From 01ebd982dac7c1be733466899dd756c67cbe3d3c Mon Sep 17 00:00:00 2001 From: 3alpha <15694175+3alpha@users.noreply.github.com> Date: Mon, 16 Nov 2026 09:44:57 +0100 Subject: [PATCH 1/8] fix: bound client lock waits and match pubkeys case-insensitively UI and API validator operations now give up before the RPC timeout instead of running after the caller has stopped waiting. Updates and builder config reads resolve stored pubkeys regardless of hex casing, and reload tolerates a short signer delete response. --- .../apiServers/ui/calls/deleteValidators.ts | 82 ++--- .../apiServers/ui/calls/getBuilderConfigs.ts | 6 +- .../apiServers/ui/calls/importValidators.ts | 287 +++++++++--------- .../apiServers/ui/calls/updateValidators.ts | 38 ++- .../deleteSignerPubkeysNotInDb.ts | 7 +- packages/brain/src/modules/db/index.ts | 15 +- packages/brain/src/modules/db/utils.ts | 5 + .../modules/db/withValidatorOperationLock.ts | 48 ++- .../withValidatorOperationLock.unit.test.ts | 42 +++ .../unit/modules/keyManagement.unit.test.ts | 72 +++++ 10 files changed, 392 insertions(+), 210 deletions(-) create mode 100644 packages/brain/test/unit/modules/db/withValidatorOperationLock.unit.test.ts diff --git a/packages/brain/src/modules/apiServers/ui/calls/deleteValidators.ts b/packages/brain/src/modules/apiServers/ui/calls/deleteValidators.ts index 2aade46f..154a96ff 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/deleteValidators.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/deleteValidators.ts @@ -1,4 +1,4 @@ -import { withValidatorOperationLock } from "../../../db/withValidatorOperationLock.js"; +import { CLIENT_OPERATION_MAX_WAIT_MS, withValidatorOperationLock } from "../../../db/withValidatorOperationLock.js"; import { Web3SignerApi, ValidatorApi } from "../../../apiClients/index.js"; import { Web3signerDeleteRequest, Web3signerDeleteResponse } from "../../../apiClients/types.js"; import { CronJob } from "../../../cron/cron.js"; @@ -25,47 +25,51 @@ export async function deleteValidators({ brainDb: BrainDataBase; deleteRequest: Web3signerDeleteRequest; }): Promise { - return withValidatorOperationLock(brainDb, async () => { - try { - if ( - !deleteRequest || - !Array.isArray(deleteRequest.pubkeys) || - deleteRequest.pubkeys.length === 0 || - Array.from(deleteRequest.pubkeys).some( - (key) => typeof key !== "string" || !isValidBlsPubkey(prefix0xPubkey(key)) + return withValidatorOperationLock( + brainDb, + async () => { + try { + if ( + !deleteRequest || + !Array.isArray(deleteRequest.pubkeys) || + deleteRequest.pubkeys.length === 0 || + Array.from(deleteRequest.pubkeys).some( + (key) => typeof key !== "string" || !isValidBlsPubkey(prefix0xPubkey(key)) + ) ) - ) - throw new Error("Select valid validator public keys to delete"); - deleteRequest = { pubkeys: deleteRequest.pubkeys.map(prefix0xPubkey) }; - // Pause future ticks while this operation holds the shared validator lock. - reloadValidatorsCronTask.stop(); + throw new Error("Select valid validator public keys to delete"); + deleteRequest = { pubkeys: deleteRequest.pubkeys.map(prefix0xPubkey) }; + // Pause future ticks while this operation holds the shared validator lock. + reloadValidatorsCronTask.stop(); - // Delete keystores on web3signer API - const web3signerDeleteResponse = await signerApi.deleteRemoteKeys(deleteRequest); + // Delete keystores on web3signer API + const web3signerDeleteResponse = await signerApi.deleteRemoteKeys(deleteRequest); - const deletedPubkeys = deleteRequest.pubkeys.filter((_, index) => - ["deleted", "not_found", "not_active"].includes(web3signerDeleteResponse.data[index]?.status) - ); - // Persist before client cleanup so reload can finish an interrupted deletion. - brainDb.deleteValidators(deletedPubkeys); - if (deletedPubkeys.length > 0) { - const clientResponse = await validatorApi - .deleteRemoteKeys({ pubkeys: deletedPubkeys }) - .catch((err) => logger.error(`Error deleting validator pubkeys`, err)); - for (const [index, pubkey] of deletedPubkeys.entries()) { - if (!clientResponse || !["deleted", "not_found"].includes(clientResponse.data[index]?.status)) continue; - await validatorApi - .deleteFeeRecipient(pubkey) - .catch((err) => logger.error(`Error deleting validator feeRecipient`, err)); + const deletedPubkeys = deleteRequest.pubkeys.filter((_, index) => + ["deleted", "not_found", "not_active"].includes(web3signerDeleteResponse.data[index]?.status) + ); + // Persist before client cleanup so reload can finish an interrupted deletion. + brainDb.deleteValidators(deletedPubkeys); + if (deletedPubkeys.length > 0) { + const clientResponse = await validatorApi + .deleteRemoteKeys({ pubkeys: deletedPubkeys }) + .catch((err) => logger.error(`Error deleting validator pubkeys`, err)); + for (const [index, pubkey] of deletedPubkeys.entries()) { + if (!clientResponse || !["deleted", "not_found"].includes(clientResponse.data[index]?.status)) continue; + await validatorApi + .deleteFeeRecipient(pubkey) + .catch((err) => logger.error(`Error deleting validator feeRecipient`, err)); + } } - } - // IMPORTANT: start the cron - reloadValidatorsCronTask.start(); - return web3signerDeleteResponse; - } catch (e) { - reloadValidatorsCronTask.restart(); - throw e; - } - }); + // IMPORTANT: start the cron + reloadValidatorsCronTask.start(); + return web3signerDeleteResponse; + } catch (e) { + reloadValidatorsCronTask.restart(); + throw e; + } + }, + { maxWaitMs: CLIENT_OPERATION_MAX_WAIT_MS } + ); } diff --git a/packages/brain/src/modules/apiServers/ui/calls/getBuilderConfigs.ts b/packages/brain/src/modules/apiServers/ui/calls/getBuilderConfigs.ts index 1fe50854..5caf7041 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/getBuilderConfigs.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/getBuilderConfigs.ts @@ -1,5 +1,6 @@ import { isValidBlsPubkey, prefix0xPubkey } from "@stakingbrain/common"; import { BrainDataBase } from "../../../db/index.js"; +import { storedPubkeys } from "../../../db/utils.js"; import { ValidatorApi } from "../../../apiClients/validator/index.js"; import { ValidatorBuilderConfigResponse } from "./types.js"; @@ -21,9 +22,10 @@ export async function getBuilderConfigs({ ) throw new Error("pubkeys must be an array of validator BLS public keys"); const dbData = brainDb.getData(); + const storedKeys = storedPubkeys(dbData); const results: ValidatorBuilderConfigResponse[] = pubkeys.map((key) => { - const pubkey = prefix0xPubkey(key); - if (!dbData[pubkey]) throw new Error(`Validator ${pubkey} is not in the database`); + const pubkey = storedKeys.get(prefix0xPubkey(key).toLowerCase()); + if (!pubkey) throw new Error(`Validator ${prefix0xPubkey(key)} is not in the database`); const builderConfig = dbData[pubkey].builderConfig; return { pubkey, diff --git a/packages/brain/src/modules/apiServers/ui/calls/importValidators.ts b/packages/brain/src/modules/apiServers/ui/calls/importValidators.ts index 14b6dd78..fadf115e 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/importValidators.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/importValidators.ts @@ -1,4 +1,4 @@ -import { withValidatorOperationLock } from "../../../db/withValidatorOperationLock.js"; +import { CLIENT_OPERATION_MAX_WAIT_MS, withValidatorOperationLock } from "../../../db/withValidatorOperationLock.js"; import { isFeeRecipientEditable, isValidBlsPubkey, @@ -49,157 +49,166 @@ export async function importValidators({ signerUrl: string; brainDb: BrainDataBase; }): Promise { - return withValidatorOperationLock(brainDb, async () => { - reloadValidatorsCronTask.stop(); - try { - if ( - !postRequest || - !Array.isArray(postRequest.validatorsImportRequest) || - postRequest.validatorsImportRequest.length === 0 - ) - throw new Error("Select at least one validator to import"); - if (!["ui", "api"].includes(postRequest.importFrom)) throw new Error("Invalid import origin"); - const existingData = brainDb.getData(); - const existing = new Map(Object.keys(existingData).map((pubkey) => [pubkey.toLowerCase(), pubkey])); - const validators: ValidatorImportRequest[] = []; - const validatorsToPost: ValidatorImportRequest[] = []; - const results: Web3signerPostResponse = { data: [] }; - const seen = new Set(); - for (const [requestIndex, validator] of postRequest.validatorsImportRequest.entries()) { - try { - if (!validator || (typeof validator.keystore !== "string" && !Buffer.isBuffer(validator.keystore))) - throw new Error("Keystore must contain JSON text"); - const keystore = validator.keystore.toString(); - let parsed: { pubkey?: string } | null; + return withValidatorOperationLock( + brainDb, + async () => { + reloadValidatorsCronTask.stop(); + try { + if ( + !postRequest || + !Array.isArray(postRequest.validatorsImportRequest) || + postRequest.validatorsImportRequest.length === 0 + ) + throw new Error("Select at least one validator to import"); + if (!["ui", "api"].includes(postRequest.importFrom)) throw new Error("Invalid import origin"); + const existingData = brainDb.getData(); + const existing = new Map(Object.keys(existingData).map((pubkey) => [pubkey.toLowerCase(), pubkey])); + const validators: ValidatorImportRequest[] = []; + const validatorsToPost: ValidatorImportRequest[] = []; + const results: Web3signerPostResponse = { data: [] }; + const seen = new Set(); + for (const [requestIndex, validator] of postRequest.validatorsImportRequest.entries()) { try { - parsed = JSON.parse(keystore); - } catch { - throw new Error("Invalid keystore JSON"); + if (!validator || (typeof validator.keystore !== "string" && !Buffer.isBuffer(validator.keystore))) + throw new Error("Keystore must contain JSON text"); + const keystore = validator.keystore.toString(); + let parsed: { pubkey?: string } | null; + try { + parsed = JSON.parse(keystore); + } catch { + throw new Error("Invalid keystore JSON"); + } + if (typeof parsed?.pubkey !== "string" || !isValidBlsPubkey(prefix0xPubkey(parsed.pubkey))) + throw new Error("Keystore must contain a valid BLS public key"); + const pubkey = prefix0xPubkey(parsed.pubkey).toLowerCase(); + if (seen.has(pubkey)) throw new Error(`Duplicate public key in import request: ${shortenPubkey(pubkey)}`); + if (!tags.includes(validator.tag)) throw new Error("Invalid validator tag"); + if (typeof validator.password !== "string") throw new Error("Keystore password must be a string"); + const feeRecipient = + !["gnosis", "lukso"].includes(network) && !isFeeRecipientEditable(validator.tag, postRequest.importFrom) + ? getNonEditableFeeRecipient( + validator.tag as NonEditableFeeRecipientTag, + network, + validator.feeRecipient + ) + : validator.feeRecipient; + if (typeof feeRecipient !== "string" || !isValidEcdsaPubkey(feeRecipient)) + throw new Error(`Invalid fee recipient for ${shortenPubkey(pubkey)}`); + seen.add(pubkey); + logger.info(`Setting ${feeRecipient} as fee recipient for ${pubkey}`); + validators.push({ + requestIndex, + keystore, + password: validator.password, + tag: validator.tag, + feeRecipient: feeRecipient, + pubkey + }); + } catch (e) { + results.data[requestIndex] = { + status: "error", + message: e.message + }; } - if (typeof parsed?.pubkey !== "string" || !isValidBlsPubkey(prefix0xPubkey(parsed.pubkey))) - throw new Error("Keystore must contain a valid BLS public key"); - const pubkey = prefix0xPubkey(parsed.pubkey).toLowerCase(); - if (seen.has(pubkey)) throw new Error(`Duplicate public key in import request: ${shortenPubkey(pubkey)}`); - if (!tags.includes(validator.tag)) throw new Error("Invalid validator tag"); - if (typeof validator.password !== "string") throw new Error("Keystore password must be a string"); - const feeRecipient = - !["gnosis", "lukso"].includes(network) && !isFeeRecipientEditable(validator.tag, postRequest.importFrom) - ? getNonEditableFeeRecipient(validator.tag as NonEditableFeeRecipientTag, network, validator.feeRecipient) - : validator.feeRecipient; - if (typeof feeRecipient !== "string" || !isValidEcdsaPubkey(feeRecipient)) - throw new Error(`Invalid fee recipient for ${shortenPubkey(pubkey)}`); - seen.add(pubkey); - logger.info(`Setting ${feeRecipient} as fee recipient for ${pubkey}`); - validators.push({ - requestIndex, - keystore, - password: validator.password, - tag: validator.tag, - feeRecipient: feeRecipient, - pubkey - }); - } catch (e) { - results.data[requestIndex] = { - status: "error", - message: e.message - }; } - } - if (validators.length === 0) return results; - const toDbRecords = (entries: ValidatorImportRequest[]): StakingBrainDb => - Object.fromEntries( - entries.map(({ pubkey, tag, feeRecipient }) => [ - pubkey, - { - tag, - feeRecipient, - automaticImport: postRequest.importFrom !== "ui", - importPending: true + if (validators.length === 0) return results; + const toDbRecords = (entries: ValidatorImportRequest[]): StakingBrainDb => + Object.fromEntries( + entries.map(({ pubkey, tag, feeRecipient }) => [ + pubkey, + { + tag, + feeRecipient, + automaticImport: postRequest.importFrom !== "ui", + importPending: true + } + ]) + ); + // Check all database constraints, including size, before changing signer storage. + brainDb.validateNewValidators(toDbRecords(validators)); + // Journal new keys before the signer can change state. Keep uncertain outcomes across restarts. + brainDb.addValidators({ validators: toDbRecords(validators) }); + + // Import keystores and passwords on web3signer API + const web3signerPostResponse = await signerApi.importRemoteKeys({ + keystores: validators.map((validator) => validator.keystore), + passwords: validators.map((validator) => validator.password), + slashing_protection: postRequest.slashing_protection ? postRequest.slashing_protection.toString() : undefined + }); + + logger.debug(`Imported keystores into web3signer API: ${JSON.stringify(web3signerPostResponse.data)}`); + + // Signer API import keystore may fail for some keystores, but not all + // @see https://github.com/ConsenSys/web3signer/issues/713 + // Remove explicit new-key failures, but retain intent when a result is missing. + const rejectedNewKeys: string[] = []; + const confirmedKeys: string[] = []; + for (const [index, validator] of validators.entries()) { + const { pubkey, requestIndex } = validator; + const response = web3signerPostResponse.data[index] ?? { + status: "error", + message: "Import outcome unknown: signer did not return a result. Import intent retained for recovery." + }; + results.data[requestIndex] = response; + const postStatus = response.status; + if (postStatus === "error") { + // A missing result is uncertain, unlike an explicit per-key rejection. + if (web3signerPostResponse.data[index]?.status === "error") { + if (!existing.has(pubkey)) rejectedNewKeys.push(pubkey); + response.message = + (response.message ?? "") + + ". Check that the keystore file format is valid and the password is correct."; } - ]) - ); - // Check all database constraints, including size, before changing signer storage. - brainDb.validateNewValidators(toDbRecords(validators)); - // Journal new keys before the signer can change state. Keep uncertain outcomes across restarts. - brainDb.addValidators({ validators: toDbRecords(validators) }); - - // Import keystores and passwords on web3signer API - const web3signerPostResponse = await signerApi.importRemoteKeys({ - keystores: validators.map((validator) => validator.keystore), - passwords: validators.map((validator) => validator.password), - slashing_protection: postRequest.slashing_protection ? postRequest.slashing_protection.toString() : undefined - }); - - logger.debug(`Imported keystores into web3signer API: ${JSON.stringify(web3signerPostResponse.data)}`); - - // Signer API import keystore may fail for some keystores, but not all - // @see https://github.com/ConsenSys/web3signer/issues/713 - // Remove explicit new-key failures, but retain intent when a result is missing. - const rejectedNewKeys: string[] = []; - const confirmedKeys: string[] = []; - for (const [index, validator] of validators.entries()) { - const { pubkey, requestIndex } = validator; - const response = web3signerPostResponse.data[index] ?? { - status: "error", - message: "Import outcome unknown: signer did not return a result. Import intent retained for recovery." - }; - results.data[requestIndex] = response; - const postStatus = response.status; - if (postStatus === "error") { - // A missing result is uncertain, unlike an explicit per-key rejection. - if (web3signerPostResponse.data[index]?.status === "error") { - if (!existing.has(pubkey)) rejectedNewKeys.push(pubkey); - response.message = - (response.message ?? "") + ". Check that the keystore file format is valid and the password is correct."; + logger.error(`Error importing keystore for pubkey ${shortenPubkey(pubkey)}: ${response.message}`); + } else if (postStatus === "duplicate" || postStatus === "imported") { + if (postStatus === "duplicate") logger.warn(`Duplicate keystore for pubkey ${shortenPubkey(pubkey)}`); + const storedKey = existing.get(pubkey); + confirmedKeys.push(storedKey ?? pubkey); + if (!storedKey || existingData[storedKey].importPending) + validatorsToPost.push({ + ...validator, + pubkey: storedKey ?? pubkey, + feeRecipient: storedKey ? existingData[storedKey].feeRecipient : validator.feeRecipient + }); } - logger.error(`Error importing keystore for pubkey ${shortenPubkey(pubkey)}: ${response.message}`); - } else if (postStatus === "duplicate" || postStatus === "imported") { - if (postStatus === "duplicate") logger.warn(`Duplicate keystore for pubkey ${shortenPubkey(pubkey)}`); - const storedKey = existing.get(pubkey); - confirmedKeys.push(storedKey ?? pubkey); - if (!storedKey || existingData[storedKey].importPending) - validatorsToPost.push({ - ...validator, - pubkey: storedKey ?? pubkey, - feeRecipient: storedKey ? existingData[storedKey].feeRecipient : validator.feeRecipient - }); } - } - - brainDb.deleteValidators(rejectedNewKeys); - if (confirmedKeys.length > 0) - brainDb.updateValidators(confirmedKeys.map((pubkey) => ({ pubkey, importPending: undefined }))); - - if (validatorsToPost.length === 0) return results; - // Import pubkeys on validator API - await validatorApi - .postRemoteKeys({ - remote_keys: validatorsToPost.map((validator) => ({ - pubkey: validator.pubkey, - url: signerUrl - })) - }) - .catch((err) => logger.error(`Error setting validator pubkeys`, err)); + brainDb.deleteValidators(rejectedNewKeys); + if (confirmedKeys.length > 0) + brainDb.updateValidators(confirmedKeys.map((pubkey) => ({ pubkey, importPending: undefined }))); - logger.debug(`Added pubkeys to validator API`); + if (validatorsToPost.length === 0) return results; - // Import feeRecipient on Validator API - for (const validator of validatorsToPost) { + // Import pubkeys on validator API await validatorApi - .setFeeRecipient(validator.feeRecipient, validator.pubkey) - .catch((err) => logger.error(`Error setting validator feeRecipient for pubkey ${validator.pubkey} :`, err)); - } + .postRemoteKeys({ + remote_keys: validatorsToPost.map((validator) => ({ + pubkey: validator.pubkey, + url: signerUrl + })) + }) + .catch((err) => logger.error(`Error setting validator pubkeys`, err)); + + logger.debug(`Added pubkeys to validator API`); + + // Import feeRecipient on Validator API + for (const validator of validatorsToPost) { + await validatorApi + .setFeeRecipient(validator.feeRecipient, validator.pubkey) + .catch((err) => logger.error(`Error setting validator feeRecipient for pubkey ${validator.pubkey} :`, err)); + } - logger.debug(`Added fee recipients to validator API`); + logger.debug(`Added fee recipients to validator API`); - logger.debug(`Written on db: ${validatorsToPost.map((v) => v.pubkey).join(", ")}`); - return results; - } finally { - reloadValidatorsCronTask.start(); - } - }); + logger.debug(`Written on db: ${validatorsToPost.map((v) => v.pubkey).join(", ")}`); + return results; + } finally { + reloadValidatorsCronTask.start(); + } + }, + { maxWaitMs: CLIENT_OPERATION_MAX_WAIT_MS } + ); } function getNonEditableFeeRecipient( diff --git a/packages/brain/src/modules/apiServers/ui/calls/updateValidators.ts b/packages/brain/src/modules/apiServers/ui/calls/updateValidators.ts index 73e92ece..46f64cbf 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/updateValidators.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/updateValidators.ts @@ -1,8 +1,9 @@ import { prefix0xPubkey, isFeeRecipientEditable, ActionRequestOrigin } from "@stakingbrain/common"; -import { withValidatorOperationLock } from "../../../db/withValidatorOperationLock.js"; +import { CLIENT_OPERATION_MAX_WAIT_MS, withValidatorOperationLock } from "../../../db/withValidatorOperationLock.js"; import logger from "../../../logger/index.js"; import { CustomValidatorUpdateRequest } from "./types.js"; import { StakingBrainDb } from "../../../db/types.js"; +import { storedPubkeys } from "../../../db/utils.js"; import { CronJob } from "../../../cron/cron.js"; import { BrainDataBase } from "../../../db/index.js"; import { ValidatorApi } from "../../../apiClients/validator/index.js"; @@ -26,20 +27,24 @@ export async function updateValidators({ customValidatorUpdateRequest: requests, requestFrom }: UpdateValidatorsOptions): Promise { - return withValidatorOperationLock(brainDb, async () => { - reloadValidatorsCronTask.stop(); - try { - const currentValidators = brainDb.getData(); - const updates = prepareUpdates(requests, currentValidators, requestFrom); + return withValidatorOperationLock( + brainDb, + async () => { + reloadValidatorsCronTask.stop(); + try { + const currentValidators = brainDb.getData(); + const updates = prepareUpdates(requests, currentValidators, requestFrom); - // The database validates the entire batch before persisting any settings. - brainDb.updateValidators(updates); - const importedKeys = await importMissingBuilderKeys(validatorApi, signerUrl, updates); - await applyUpdates(validatorApi, updates, currentValidators, importedKeys); - } finally { - reloadValidatorsCronTask.start(); - } - }); + // The database validates the entire batch before persisting any settings. + brainDb.updateValidators(updates); + const importedKeys = await importMissingBuilderKeys(validatorApi, signerUrl, updates); + await applyUpdates(validatorApi, updates, currentValidators, importedKeys); + } finally { + reloadValidatorsCronTask.start(); + } + }, + { maxWaitMs: CLIENT_OPERATION_MAX_WAIT_MS } + ); } function prepareUpdates( @@ -51,11 +56,12 @@ function prepareUpdates( const updates: CustomValidatorUpdateRequest[] = []; const seen = new Set(); + const storedKeys = storedPubkeys(currentValidators); for (const request of requests) { if (!request || typeof request.pubkey !== "string") throw new Error("Validator public key is required"); - const pubkey = prefix0xPubkey(request.pubkey); + const pubkey = storedKeys.get(prefix0xPubkey(request.pubkey).toLowerCase()); + if (!pubkey) throw new Error(`Validator ${prefix0xPubkey(request.pubkey)} is not in the database`); const current = currentValidators[pubkey]; - if (!current) throw new Error(`Validator ${pubkey} is not in the database`); if (seen.has(pubkey)) throw new Error(`Duplicate validator ${pubkey}`); if (Object.keys(request).some((field) => !["pubkey", "feeRecipient", "builderConfig"].includes(field))) throw new Error("Unsupported validator setting"); diff --git a/packages/brain/src/modules/cron/reloadValidators/deleteSignerPubkeysNotInDb.ts b/packages/brain/src/modules/cron/reloadValidators/deleteSignerPubkeysNotInDb.ts index 3659d964..7e3c5739 100644 --- a/packages/brain/src/modules/cron/reloadValidators/deleteSignerPubkeysNotInDb.ts +++ b/packages/brain/src/modules/cron/reloadValidators/deleteSignerPubkeysNotInDb.ts @@ -25,12 +25,13 @@ export async function deleteSignerPubkeysNotInDb({ }); for (const [index, pubkeyToRemove] of signerPubkeysToRemove.entries()) { - const signerDeleteStatus = signerDeleteResponse.data[index].status; - if (signerDeleteStatus === "deleted" || signerDeleteStatus === "not_found") + // A missing result is uncertain; keep the key listed so later steps treat it as still present. + const result = signerDeleteResponse.data[index]; + if (result?.status === "deleted" || result?.status === "not_found") signerPubkeys.splice(signerPubkeys.indexOf(pubkeyToRemove), 1); else logger.error( - `${logPrefix}Error deleting pubkey ${pubkeyToRemove} from signer API: ${signerDeleteResponse.data[index].message}` + `${logPrefix}Error deleting pubkey ${pubkeyToRemove} from signer API: ${result?.message ?? "signer did not return a result"}` ); } } diff --git a/packages/brain/src/modules/db/index.ts b/packages/brain/src/modules/db/index.ts index fbf9fddb..ec96a9b6 100644 --- a/packages/brain/src/modules/db/index.ts +++ b/packages/brain/src/modules/db/index.ts @@ -5,7 +5,7 @@ import fs from "fs"; import logger from "../logger/index.js"; import { Web3SignerApi, ValidatorApi } from "../apiClients/index.js"; import { params } from "../../params.js"; -import { isValidTag } from "./utils.js"; +import { isValidTag, storedPubkeys } from "./utils.js"; import { shortenPubkey, isValidBlsPubkey, isValidEcdsaPubkey, validateBuilderConfig } from "@stakingbrain/common"; import { BrainDbError } from "./error.js"; @@ -120,12 +120,14 @@ export class BrainDataBase extends LowSync { this.validateDb(); this.validateUpdateValidators(validators); const updatedData = { ...this.data }; + const storedKeys = storedPubkeys(this.data); for (const { pubkey, ...changes } of validators) { - if (!this.data[pubkey]) { + const storedKey = storedKeys.get(pubkey.toLowerCase()); + if (!storedKey) { logger.warn(`Pubkey ${pubkey} not found in the database`); continue; } - updatedData[pubkey] = { ...this.data[pubkey], ...changes }; + updatedData[storedKey] = { ...this.data[storedKey], ...changes }; } this.ensureDbMaxSize(updatedData, true); this.data = updatedData; @@ -145,7 +147,7 @@ export class BrainDataBase extends LowSync { try { this.validateDb(); if (!this.data) return; - const storedKeys = new Map(Object.keys(this.data).map((key) => [key.toLowerCase(), key])); + const storedKeys = storedPubkeys(this.data); let changed = false; for (const pubkey of pubkeys) { const storedKey = storedKeys.get(pubkey.toLowerCase()); @@ -296,8 +298,9 @@ export class BrainDataBase extends LowSync { // Validate Ethereum address if (!isValidBlsPubkey(pubkey)) errors.push(`\n pubkey ${pubkeySubstr}: bls is invalid`); - if (seen.has(pubkey)) errors.push(`\n pubkey ${pubkeySubstr}: duplicate update`); - seen.add(pubkey); + const key = pubkey.toLowerCase(); + if (seen.has(key)) errors.push(`\n pubkey ${pubkeySubstr}: duplicate update`); + seen.add(key); // FeeRecipient if ("feeRecipient" in validator) { diff --git a/packages/brain/src/modules/db/utils.ts b/packages/brain/src/modules/db/utils.ts index 3f3910b8..d0f482ff 100644 --- a/packages/brain/src/modules/db/utils.ts +++ b/packages/brain/src/modules/db/utils.ts @@ -3,3 +3,8 @@ import { Tag, tags } from "@stakingbrain/common"; export function isValidTag(tag: Tag): boolean { return tags.includes(tag); } + +/** Map lowercase pubkeys to their stored spelling; clients do not agree on hex casing. */ +export function storedPubkeys(data: object): Map { + return new Map(Object.keys(data).map((pubkey) => [pubkey.toLowerCase(), pubkey])); +} diff --git a/packages/brain/src/modules/db/withValidatorOperationLock.ts b/packages/brain/src/modules/db/withValidatorOperationLock.ts index b181fdf0..1b195156 100644 --- a/packages/brain/src/modules/db/withValidatorOperationLock.ts +++ b/packages/brain/src/modules/db/withValidatorOperationLock.ts @@ -2,19 +2,57 @@ import type { BrainDataBase } from "./index.js"; const pendingOperations = new WeakMap>(); -/** Keep database snapshots and their client writes ordered, including across awaits. */ -export async function withValidatorOperationLock(brainDb: BrainDataBase, operation: () => Promise): Promise { +export class ValidatorOperationBusyError extends Error { + constructor() { + super("Brain is busy with another validator operation. Nothing was changed; try again shortly."); + this.name = "ValidatorOperationBusyError"; + } +} + +/** + * Keep database snapshots and their client writes ordered, including across awaits. + * + * With `maxWaitMs`, an operation that cannot start in time is abandoned without running, so a + * caller that has already given up (for example a timed-out UI request) never changes state later. + * The abandoned slot still waits for its predecessor, so later operations keep their order. + */ +export async function withValidatorOperationLock( + brainDb: BrainDataBase, + operation: () => Promise, + { maxWaitMs }: { maxWaitMs?: number } = {} +): Promise { const previous = pendingOperations.get(brainDb) ?? Promise.resolve(); let release!: () => void; const current = new Promise((resolve) => { release = resolve; }); pendingOperations.set(brainDb, current); - await previous; + const cleanup = (): void => { + release(); + if (pendingOperations.get(brainDb) === current) pendingOperations.delete(brainDb); + }; + + if (maxWaitMs !== undefined) { + let timer: NodeJS.Timeout | undefined; + const acquired = await Promise.race([ + previous.then(() => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), maxWaitMs); + }) + ]); + clearTimeout(timer); + if (!acquired) { + void previous.then(cleanup); + throw new ValidatorOperationBusyError(); + } + } else await previous; + try { return await operation(); } finally { - release(); - if (pendingOperations.get(brainDb) === current) pendingOperations.delete(brainDb); + cleanup(); } } + +/** Leaves most of the UI's RPC_TIMEOUT_MS for the operation itself once it starts. */ +export const CLIENT_OPERATION_MAX_WAIT_MS = 30 * 1000; diff --git a/packages/brain/test/unit/modules/db/withValidatorOperationLock.unit.test.ts b/packages/brain/test/unit/modules/db/withValidatorOperationLock.unit.test.ts new file mode 100644 index 00000000..ed1d32d1 --- /dev/null +++ b/packages/brain/test/unit/modules/db/withValidatorOperationLock.unit.test.ts @@ -0,0 +1,42 @@ +import { expect } from "chai"; +import sinon from "sinon"; +import type { BrainDataBase } from "../../../../src/modules/db/index.js"; +import { + ValidatorOperationBusyError, + withValidatorOperationLock +} from "../../../../src/modules/db/withValidatorOperationLock.js"; + +describe("Validator operation lock", () => { + afterEach(() => sinon.restore()); + + it("keeps later operations behind the running one after a waiter is abandoned", async () => { + const clock = sinon.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const brainDb = {} as BrainDataBase; + const events: string[] = []; + let finishFirst!: () => void; + + const first = withValidatorOperationLock(brainDb, async () => { + events.push("first:start"); + await new Promise((resolve) => (finishFirst = resolve)); + events.push("first:end"); + }); + const abandoned = withValidatorOperationLock(brainDb, async () => events.push("abandoned"), { + maxWaitMs: 1000 + }).catch((error: Error) => error); + const third = withValidatorOperationLock(brainDb, async () => events.push("third")); + + await clock.tickAsync(1000); + expect(await abandoned).to.be.instanceOf(ValidatorOperationBusyError); + expect(events).to.deep.equal(["first:start"]); + + clock.restore(); + finishFirst(); + await Promise.all([first, third]); + expect(events).to.deep.equal(["first:start", "first:end", "third"]); + }); + + it("runs a bounded operation that acquires the lock in time", async () => { + const brainDb = {} as BrainDataBase; + expect(await withValidatorOperationLock(brainDb, async () => "done", { maxWaitMs: 1000 })).to.equal("done"); + }); +}); diff --git a/packages/brain/test/unit/modules/keyManagement.unit.test.ts b/packages/brain/test/unit/modules/keyManagement.unit.test.ts index 356c41f7..6f0d57b5 100644 --- a/packages/brain/test/unit/modules/keyManagement.unit.test.ts +++ b/packages/brain/test/unit/modules/keyManagement.unit.test.ts @@ -13,6 +13,11 @@ import { updateValidators } from "../../../src/modules/apiServers/ui/calls/updat import { reloadValidators } from "../../../src/modules/cron/reloadValidators/index.js"; import type { CustomImportRequest } from "../../../src/modules/apiServers/ui/calls/types.js"; import logger from "../../../src/modules/logger/index.js"; +import { + CLIENT_OPERATION_MAX_WAIT_MS, + ValidatorOperationBusyError, + withValidatorOperationLock +} from "../../../src/modules/db/withValidatorOperationLock.js"; const keys = ["11", "22", "33"].map((byte) => "0x" + byte.repeat(48)); const recipient = "0x" + "ab".repeat(20); @@ -278,4 +283,71 @@ describe("Key management partial failures", () => { expect(validatorApi.postRemoteKeys.called).to.equal(false); expect(Object.keys(brainDb.getData())).to.deep.equal([pubkey]); }); + + it("updates the stored record when the request uses different hex casing", async () => { + const pubkey = "0x" + "ab".repeat(48); + const other = "0x" + "cd".repeat(20); + brainDb.addValidators({ validators: { [pubkey]: { ...record } } }); + await updateValidators({ + brainDb, + validatorApi, + signerUrl: "http://signer:9000", + reloadValidatorsCronTask, + customValidatorUpdateRequest: [{ pubkey: "0x" + "AB".repeat(48), feeRecipient: other }] + }); + expect(brainDb.getData()).to.deep.equal({ [pubkey]: { ...record, feeRecipient: other } }); + expect(validatorApi.setFeeRecipient.calledOnceWith(other, pubkey)).to.equal(true); + }); + + it("rejects the same validator twice in one update even with different hex casing", async () => { + const pubkey = "0x" + "ab".repeat(48); + brainDb.addValidators({ validators: { [pubkey]: { ...record } } }); + const result = await updateValidators({ + brainDb, + validatorApi, + signerUrl: "http://signer:9000", + reloadValidatorsCronTask, + customValidatorUpdateRequest: [ + { pubkey, feeRecipient: recipient }, + { pubkey: "0x" + "AB".repeat(48), feeRecipient: recipient } + ] + }).catch((error: Error) => error); + expect(result).to.be.instanceOf(Error).and.have.property("message").that.includes("Duplicate validator"); + expect(validatorApi.setFeeRecipient.called).to.equal(false); + }); + + it("keeps a signer key listed when its deletion result is missing", async () => { + brainDb.addValidators({ validators: { [keys[0]]: { ...record } } }); + signerApi.upcheck.resolves(); + signerApi.listRemoteKeys.resolves({ + data: [keys[0], keys[1]].map((validating_pubkey) => ({ validating_pubkey, derivation_path: "", readonly: false })) + }); + signerApi.deleteRemoteKeys.resolves({ data: [] }); + validatorApi.getRemoteKeys.resolves({ data: [{ pubkey: keys[0], readonly: false, url: "http://signer" }] }); + await reloadValidators(signerApi, "http://signer", validatorApi, brainDb); + expect(signerApi.deleteRemoteKeys.calledOnceWith({ pubkeys: [keys[1]] })).to.equal(true); + // Reload continues past the short response instead of aborting. + expect(validatorApi.setFeeRecipient.calledOnceWith(recipient, keys[0])).to.equal(true); + }); + + it("abandons a client operation that cannot start before its wait budget", async () => { + const clock = sinon.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + let finishReload!: () => void; + const reload = withValidatorOperationLock(brainDb, () => new Promise((resolve) => (finishReload = resolve))); + const update = updateValidators({ + brainDb, + validatorApi, + signerUrl: "http://signer:9000", + reloadValidatorsCronTask, + customValidatorUpdateRequest: [{ pubkey: keys[0], feeRecipient: recipient }] + }).catch((error: Error) => error); + await clock.tickAsync(CLIENT_OPERATION_MAX_WAIT_MS); + expect(await update).to.be.instanceOf(ValidatorOperationBusyError); + clock.restore(); + finishReload(); + await reload; + // The abandoned call never touched clients or the cron, even after the lock was released. + expect(reloadValidatorsCronTask.stop.called).to.equal(false); + expect(validatorApi.setFeeRecipient.called).to.equal(false); + }); }); From 79094c9e5c14989fdb8db8628a75f6e4773eca24 Mon Sep 17 00:00:00 2001 From: 3alpha <15694175+3alpha@users.noreply.github.com> Date: Mon, 16 Nov 2026 09:53:49 +0100 Subject: [PATCH 2/8] feat: improve UX for large validator batches Import runs in batches of 20 with live progress, stop and retry; each batch carries only its own slashing history. The Brain pushes a validatorsChanged event after every database write so open lists refresh themselves, and the list keeps refreshing while rows are marked detailsPending, without blanking the table. Fee checks are cached like beacon data so large lists converge. Signer imports get a 10 minute timeout, fee recipients are applied with bounded concurrency on import and reload, and the import form pages its keystore cards and points at files that still need input. --- .../src/modules/apiClients/signer/index.ts | 8 +- .../apiServers/ui/calls/getValidators.ts | 44 +++- .../apiServers/ui/calls/importValidators.ts | 9 +- .../src/modules/apiServers/ui/calls/types.ts | 2 + .../modules/apiServers/ui/startUiServer.ts | 15 +- .../postValidatorsFeeRecipientsFromDb.ts | 12 +- packages/brain/src/modules/db/index.ts | 15 ++ .../src/modules/utils/forEachConcurrently.ts | 19 ++ .../unit/modules/db/onChange.unit.test.ts | 46 ++++ .../unit/modules/validatorList.unit.test.ts | 45 ++++ packages/common/src/rpc.ts | 2 + packages/ui/src/ImportScreen.tsx | 125 ++++++++-- .../src/components/Dialogs/ImportDialog.tsx | 181 ++++++++++++--- .../components/Dialogs/ValidatorResult.tsx | 2 +- .../src/components/FileCards/FileCardList.tsx | 214 +++++++++--------- .../ValidatorList/KeystoresDataGrid.tsx | 19 +- .../ValidatorList/ValidatorList.tsx | 120 +++++++++- packages/ui/src/socket/rpcClient.ts | 8 + packages/ui/src/utils/importBatches.ts | 153 +++++++++++++ packages/ui/src/utils/importForm.ts | 28 ++- packages/ui/src/utils/validatorRefresh.ts | 30 +++ packages/ui/test/importBatches.test.ts | 137 +++++++++++ packages/ui/test/validatorRefresh.test.ts | 40 ++++ 23 files changed, 1068 insertions(+), 206 deletions(-) create mode 100644 packages/brain/src/modules/utils/forEachConcurrently.ts create mode 100644 packages/brain/test/unit/modules/db/onChange.unit.test.ts create mode 100644 packages/ui/src/utils/importBatches.ts create mode 100644 packages/ui/src/utils/validatorRefresh.ts create mode 100644 packages/ui/test/importBatches.test.ts create mode 100644 packages/ui/test/validatorRefresh.test.ts diff --git a/packages/brain/src/modules/apiClients/signer/index.ts b/packages/brain/src/modules/apiClients/signer/index.ts index 8ed5d405..16ab82aa 100644 --- a/packages/brain/src/modules/apiClients/signer/index.ts +++ b/packages/brain/src/modules/apiClients/signer/index.ts @@ -15,6 +15,9 @@ import path from "node:path"; import { prefix0xPubkey } from "../prefix0xPubkey.js"; import { SignerApiError } from "./error.js"; +/** Longer than the UI's RPC timeout: an import already sent should be recorded even if the UI gave up. */ +export const SIGNER_IMPORT_TIMEOUT_MS = 10 * 60 * 1000; + /** * Key Manager API standard * https://ethereum.github.io/keymanager-APIs/ @@ -121,7 +124,10 @@ export class Web3SignerApi extends StandardApi { method: "POST", endpoint: this.localKeymanagerEndpoint, body: JSON.stringify(postRequest), - headers: this.originHeader + headers: this.originHeader, + // Keystore decryption is slow. Dropping the connection does not stop the signer, it only + // discards the result, so wait long enough to record the outcome of large imports. + timeout: SIGNER_IMPORT_TIMEOUT_MS }); } catch (e) { throw new SignerApiError(`Error importing (POST) keystores to remote signer. ${e.message}`); diff --git a/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts b/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts index 840fff89..73cda398 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts @@ -9,9 +9,11 @@ type ListProgress = { beaconNext: number; feeNext: number; beacon: Map; + // Keyed by the expected fee recipient so an edit is re-checked immediately. + fee: Map; }; const listProgress = new WeakMap(); -const BEACON_CACHE_MS = 60_000; +const BEACON_CACHE_MS = 60_000; // Also bounds how stale a cached fee recipient check can be. /** Read a stable snapshot; independent queues rotate unfinished work across bounded refreshes. */ export async function getValidators({ @@ -26,12 +28,15 @@ export async function getValidators({ beaconchainApi: BeaconchainApi; }): Promise { const entries = Object.entries(brainDb.getData()); - const progress = listProgress.get(brainDb) ?? { beaconNext: 0, feeNext: 0, beacon: new Map() }; + const progress = listProgress.get(brainDb) ?? { beaconNext: 0, feeNext: 0, beacon: new Map(), fee: new Map() }; listProgress.set(brainDb, progress); const currentKeys = new Set(entries.map(([pubkey]) => pubkey)); for (const [pubkey, cached] of progress.beacon) { if (!currentKeys.has(pubkey) || cached.expires <= Date.now()) progress.beacon.delete(pubkey); } + for (const [pubkey, cached] of progress.fee) { + if (!currentKeys.has(pubkey) || cached.expires <= Date.now()) progress.fee.delete(pubkey); + } if (entries.length === 0) return []; const results: CustomValidatorGetResponse[] = entries.map(([pubkey, details]) => ({ pubkey, @@ -48,9 +53,14 @@ export async function getValidators({ })); const controller = new AbortController(); const deadline = setTimeout(() => controller.abort(), 10000); + // Reads cut short by the deadline stay pending so the UI knows to refresh; definite failures do not. + const beaconPending = new Set(results.map((_, index) => index)); + const feePending = new Set(); + let membershipPending = false; async function readQueue( cursor: "beaconNext" | "feeNext", + pending: Set, read: (result: CustomValidatorGetResponse) => Promise ) { const start = progress[cursor] % results.length; @@ -62,7 +72,9 @@ export async function getValidators({ progress[cursor] = (index + 1) % results.length; try { await read(results[index]); + pending.delete(index); } catch (error) { + if (!controller.signal.aborted) pending.delete(index); logger.error(error); } } @@ -76,14 +88,28 @@ export async function getValidators({ try { const remoteKeys = await validatorApi.getRemoteKeys(controller.signal); const keys = new Set(remoteKeys.data.map(({ pubkey }) => pubkey.toLowerCase())); - for (const result of results) result.validatorImported = keys.has(result.pubkey.toLowerCase()); - await readQueue("feeNext", async (result) => { + results.forEach((result, index) => { + result.validatorImported = keys.has(result.pubkey.toLowerCase()); + if (result.validatorImported) feePending.add(index); + }); + await readQueue("feeNext", feePending, async (result) => { if (!result.validatorImported) return; + const feeRecipient = result.feeRecipient.toLowerCase(); + const cached = progress.fee.get(result.pubkey); + if (cached?.feeRecipient === feeRecipient) { + result.validatorFeeRecipientCorrect = cached.correct; + return; + } const response = await validatorApi.getFeeRecipient(result.pubkey, controller.signal); - result.validatorFeeRecipientCorrect = - response.data.ethaddress.toLowerCase() === result.feeRecipient.toLowerCase(); + result.validatorFeeRecipientCorrect = response.data.ethaddress.toLowerCase() === feeRecipient; + progress.fee.set(result.pubkey, { + feeRecipient, + correct: result.validatorFeeRecipientCorrect, + expires: Date.now() + BEACON_CACHE_MS + }); }); } catch (error) { + if (controller.signal.aborted) membershipPending = true; logger.error(error); } })(), @@ -93,10 +119,11 @@ export async function getValidators({ const keys = new Set(signerKeys.data.map(({ validating_pubkey }) => validating_pubkey.toLowerCase())); for (const result of results) result.signerImported = keys.has(result.pubkey.toLowerCase()); } catch (error) { + if (controller.signal.aborted) membershipPending = true; logger.error(error); } })(), - readQueue("beaconNext", async (result) => { + readQueue("beaconNext", beaconPending, async (result) => { const cached = progress.beacon.get(result.pubkey); if (cached) { Object.assign(result, cached.details); @@ -123,6 +150,9 @@ export async function getValidators({ progress.beacon.set(result.pubkey, { details, expires: Date.now() + BEACON_CACHE_MS }); }) ]); + results.forEach((result, index) => { + result.detailsPending = membershipPending || beaconPending.has(index) || feePending.has(index); + }); return results; } finally { clearTimeout(deadline); diff --git a/packages/brain/src/modules/apiServers/ui/calls/importValidators.ts b/packages/brain/src/modules/apiServers/ui/calls/importValidators.ts index fadf115e..b4e0d10f 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/importValidators.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/importValidators.ts @@ -15,6 +15,7 @@ import { LIDO_FEE_RECIPIENT_HOODI } from "@stakingbrain/common"; import { CustomImportRequest } from "./types.js"; +import { forEachConcurrently, VALIDATOR_CLIENT_CONCURRENCY } from "../../../utils/forEachConcurrently.js"; import { CronJob } from "../../../cron/cron.js"; import { BrainDataBase } from "../../../db/index.js"; import { Web3signerPostResponse } from "../../../apiClients/types.js"; @@ -193,11 +194,11 @@ export async function importValidators({ logger.debug(`Added pubkeys to validator API`); // Import feeRecipient on Validator API - for (const validator of validatorsToPost) { - await validatorApi + await forEachConcurrently(validatorsToPost, VALIDATOR_CLIENT_CONCURRENCY, (validator) => + validatorApi .setFeeRecipient(validator.feeRecipient, validator.pubkey) - .catch((err) => logger.error(`Error setting validator feeRecipient for pubkey ${validator.pubkey} :`, err)); - } + .catch((err) => logger.error(`Error setting validator feeRecipient for pubkey ${validator.pubkey} :`, err)) + ); logger.debug(`Added fee recipients to validator API`); diff --git a/packages/brain/src/modules/apiServers/ui/calls/types.ts b/packages/brain/src/modules/apiServers/ui/calls/types.ts index 183c29cf..5d65d885 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/types.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/types.ts @@ -75,4 +75,6 @@ export interface CustomValidatorGetResponse { validatorFeeRecipientCorrect?: boolean; builderConfig?: BuilderConfig | null; builderConfigStatus?: BuilderConfigStatus; + /** True when the read deadline cut this row's checks short; a later refresh continues them. */ + detailsPending?: boolean; } diff --git a/packages/brain/src/modules/apiServers/ui/startUiServer.ts b/packages/brain/src/modules/apiServers/ui/startUiServer.ts index b17d2dfa..6f4da279 100644 --- a/packages/brain/src/modules/apiServers/ui/startUiServer.ts +++ b/packages/brain/src/modules/apiServers/ui/startUiServer.ts @@ -1,4 +1,6 @@ -import { Network } from "@stakingbrain/common"; +import { Network, VALIDATORS_CHANGED_EVENT } from "@stakingbrain/common"; + +const VALIDATORS_CHANGED_DEBOUNCE_MS = 500; import cors from "cors"; import express from "express"; import path from "path"; @@ -57,6 +59,17 @@ export function startUiServer({ // Socket io const io = new Server(server, uiSocketOptions(uiOrigins)); + // One import or reload writes several times; coalesce them into a single refresh hint. + let changeTimer: NodeJS.Timeout | undefined; + brainDb.onChange(() => { + if (changeTimer) return; + changeTimer = setTimeout(() => { + changeTimer = undefined; + io.emit(VALIDATORS_CHANGED_EVENT); + }, VALIDATORS_CHANGED_DEBOUNCE_MS); + changeTimer.unref(); + }); + io.on("connection", (socket) => { logger.debug("A user connected"); diff --git a/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts b/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts index 6fc9c7f5..d5e6162a 100644 --- a/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts +++ b/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts @@ -1,10 +1,12 @@ import { ValidatorApi } from "../../apiClients/index.js"; import { StakingBrainDb } from "../../db/types.js"; import logger from "../../logger/index.js"; +import { forEachConcurrently, VALIDATOR_CLIENT_CONCURRENCY } from "../../utils/forEachConcurrently.js"; import { logPrefix } from "./logPrefix.js"; /** - * Post in the validator API fee recipients that are in the DB in the validator API + * Post in the validator API fee recipients that are in the DB in the validator API. + * Bounded concurrency keeps the reload (and the validator lock it holds) short for large key sets. */ export async function postValidatorsFeeRecipientsFromDb({ validatorApi, @@ -13,13 +15,13 @@ export async function postValidatorsFeeRecipientsFromDb({ validatorApi: ValidatorApi; dbData: StakingBrainDb; }): Promise { - for (const [pubkey, { feeRecipient }] of Object.entries(dbData)) { - await validatorApi + await forEachConcurrently(Object.entries(dbData), VALIDATOR_CLIENT_CONCURRENCY, ([pubkey, { feeRecipient }]) => + validatorApi .setFeeRecipient(feeRecipient, pubkey) .catch((e) => logger.error( `${logPrefix}Error posting fee recipient ${feeRecipient} for pubkey ${pubkey} to validator API: ${e}` ) - ); - } + ) + ); } diff --git a/packages/brain/src/modules/db/index.ts b/packages/brain/src/modules/db/index.ts index ec96a9b6..7bb2fcef 100644 --- a/packages/brain/src/modules/db/index.ts +++ b/packages/brain/src/modules/db/index.ts @@ -33,11 +33,26 @@ export class BrainDataBase extends LowSync { this.dbName = dbName; } + private changeListeners = new Set<() => void>(); + public override write(): void { super.write(); // The JSON adapter replaces the file atomically; restore the private mode on // the replacement too, since records may contain builder authorization data. this.setOwnerWriteRead(); + for (const listener of this.changeListeners) { + try { + listener(); + } catch (e) { + logger.error("Database change listener failed", e); + } + } + } + + /** Notify after each persisted change, e.g. to push live updates to open UIs. Returns an unsubscribe. */ + public onChange(listener: () => void): () => void { + this.changeListeners.add(listener); + return () => this.changeListeners.delete(listener); } /** diff --git a/packages/brain/src/modules/utils/forEachConcurrently.ts b/packages/brain/src/modules/utils/forEachConcurrently.ts new file mode 100644 index 00000000..7a9eee83 --- /dev/null +++ b/packages/brain/src/modules/utils/forEachConcurrently.ts @@ -0,0 +1,19 @@ +/** Run `task` for every item with at most `limit` in flight; per-item errors are the task's to handle. */ +export async function forEachConcurrently( + items: readonly T[], + limit: number, + task: (item: T, index: number) => Promise +): Promise { + let next = 0; + await Promise.all( + Array.from({ length: Math.min(limit, items.length) }, async () => { + while (next < items.length) { + const index = next++; + await task(items[index], index); + } + }) + ); +} + +/** Keeps per-validator client calls fast for large batches without flooding the validator client. */ +export const VALIDATOR_CLIENT_CONCURRENCY = 8; diff --git a/packages/brain/test/unit/modules/db/onChange.unit.test.ts b/packages/brain/test/unit/modules/db/onChange.unit.test.ts new file mode 100644 index 00000000..573a4c12 --- /dev/null +++ b/packages/brain/test/unit/modules/db/onChange.unit.test.ts @@ -0,0 +1,46 @@ +import { expect } from "chai"; +import sinon from "sinon"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { BrainDataBase } from "../../../../src/modules/db/index.js"; +import logger from "../../../../src/modules/logger/index.js"; + +describe("Database change notifications", () => { + let directory: string; + let brainDb: BrainDataBase; + const pubkey = "0x" + "11".repeat(48); + const record = { tag: "solo" as const, feeRecipient: "0x" + "ab".repeat(20), automaticImport: false }; + + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), "stakingbrain-change-")); + brainDb = new BrainDataBase(path.join(directory, "db.json")); + sinon.stub(logger, "warn"); + sinon.stub(logger, "error"); + }); + afterEach(() => { + sinon.restore(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it("notifies after persisted changes only, until unsubscribed", () => { + const listener = sinon.spy(); + const unsubscribe = brainDb.onChange(listener); + brainDb.addValidators({ validators: { [pubkey]: { ...record } } }); + expect(listener.callCount).to.equal(1); + // A delete of an unknown key writes nothing, so it must not trigger UI refreshes. + brainDb.deleteValidators(["0x" + "22".repeat(48)]); + expect(listener.callCount).to.equal(1); + unsubscribe(); + brainDb.deleteValidators([pubkey]); + expect(listener.callCount).to.equal(1); + }); + + it("keeps writing when a listener throws", () => { + brainDb.onChange(() => { + throw new Error("socket gone"); + }); + brainDb.addValidators({ validators: { [pubkey]: { ...record } } }); + expect(Object.keys(brainDb.getData())).to.deep.equal([pubkey]); + }); +}); diff --git a/packages/brain/test/unit/modules/validatorList.unit.test.ts b/packages/brain/test/unit/modules/validatorList.unit.test.ts index 0f6ed551..bb4de33a 100644 --- a/packages/brain/test/unit/modules/validatorList.unit.test.ts +++ b/packages/brain/test/unit/modules/validatorList.unit.test.ts @@ -165,4 +165,49 @@ describe("Validator list bulk reads", () => { ).to.equal(true); expect(results.map(({ index }) => index)).to.deep.equal(keys.map((_, index) => String(index))); }); + it("marks rows pending only when the deadline cut their reads short", async () => { + const clock = sinon.useFakeTimers(); + beaconchainApi.getStateValidator.callsFake(({ pubkey, signal }) => + // Validator 0 is not deposited yet: a definite failure must not keep the UI polling. + pubkey === keys[0] + ? Promise.reject(new Error("not deposited")) + : new Promise((resolve, reject) => { + const abort = () => { + clearTimeout(timer); + reject(new Error("Aborted")); + }; + const timer = setTimeout(() => { + signal!.removeEventListener("abort", abort); + resolve(beaconResponse(pubkey, "0x01" + "00".repeat(31))); + }, 4000); + signal!.addEventListener("abort", abort, { once: true }); + }) + ); + const first = read(); + await clock.tickAsync(10000); + const firstResults = await first; + expect(firstResults[0].detailsPending).to.equal(false); + expect(firstResults.some(({ detailsPending }) => detailsPending)).to.equal(true); + expect( + firstResults.filter(({ detailsPending }) => !detailsPending).every(({ index }, i) => i === 0 || index !== "") + ).to.be.true; + const second = read(); + await clock.tickAsync(10000); + expect((await second).every(({ detailsPending }) => !detailsPending)).to.equal(true); + expect(clock.countTimers()).to.equal(0); + }); + + it("caches fee checks per expected recipient and re-checks after an edit", async () => { + beaconchainApi.getStateValidator.rejects(new Error("not deposited")); + await read(); + expect(validatorApi.getFeeRecipient.callCount).to.equal(keys.length); + await read(); + expect(validatorApi.getFeeRecipient.callCount).to.equal(keys.length); + const other = "0x" + "cd".repeat(20); + brainDb.updateValidators([{ pubkey: keys[0], feeRecipient: other }]); + const results = await read(); + expect(validatorApi.getFeeRecipient.callCount).to.equal(keys.length + 1); + expect(results[0].validatorFeeRecipientCorrect).to.equal(false); + expect(results.every(({ detailsPending }) => detailsPending === false)).to.equal(true); + }); }); diff --git a/packages/common/src/rpc.ts b/packages/common/src/rpc.ts index 34dc908c..2d36714b 100644 --- a/packages/common/src/rpc.ts +++ b/packages/common/src/rpc.ts @@ -1,3 +1,5 @@ // The database allows 6 MiB. Leave room for RPC framing and bulk settings requests. export const RPC_MAX_PAYLOAD_BYTES = 8 * 1024 * 1024; export const RPC_TIMEOUT_MS = 120 * 1000; +/** Server-to-UI push: validator records changed; clients should refresh their list. Carries no data. */ +export const VALIDATORS_CHANGED_EVENT = "validatorsChanged"; diff --git a/packages/ui/src/ImportScreen.tsx b/packages/ui/src/ImportScreen.tsx index caa4bc56..ff73bcf7 100644 --- a/packages/ui/src/ImportScreen.tsx +++ b/packages/ui/src/ImportScreen.tsx @@ -18,7 +18,7 @@ import { Stack } from "@mui/material"; import { Link } from "react-router-dom"; -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import BackupIcon from "@mui/icons-material/Backup"; import { ImportStatus, KeystoreInfo, TagSelectOption } from "./types"; import FileCardList from "./components/FileCards/FileCardList"; @@ -28,10 +28,13 @@ import CloseIcon from "@mui/icons-material/Close"; import { rpcClient } from "./socket"; import ArrowCircleLeftOutlinedIcon from "@mui/icons-material/ArrowCircleLeftOutlined"; import { extractPubkey } from "./utils/dataUtils"; -import type { Web3signerPostResponse } from "@stakingbrain/brain"; import { createImportRequest } from "./utils/createImportRequest"; -import { appendImportFiles, isImportFormValid, ImportFormState } from "./utils/importForm"; +import { appendImportFiles, incompleteImportIndexes, isImportFormValid, ImportFormState } from "./utils/importForm"; import PageLayout, { PageHeader } from "./components/Layout/PageLayout"; +import { ImportProgress, ImportResult, notAttempted, planImportBatches, runImportBatches } from "./utils/importBatches"; + +// Each card holds up to three inputs; rendering hundreds at once makes typing laggy. +const FILE_CARDS_PAGE = 25; export default function ImportScreen({ network, @@ -40,7 +43,11 @@ export default function ImportScreen({ network: Network; isMevBoostSet: boolean; }): JSX.Element { - const [keystoresPostResponse, setKeystoresPostResponse] = useState(); + const [importResults, setImportResults] = useState(); + const [importProgress, setImportProgress] = useState(); + const [importStopped, setImportStopped] = useState<"user" | "error">(); + const [stopRequested, setStopRequested] = useState(false); + const stopRef = useRef(false); const [keystoresPostError, setKeystoresPostError] = useState(); const [openDialog, setOpenDialog] = useState(false); const [importForm, setImportForm] = useState({ @@ -63,6 +70,8 @@ export default function ImportScreen({ const [slashingFile, setSlashingFile] = useState(); const [showMevWarning, setShowMevWarning] = useState(false); const [isButtonDisabled, setIsButtonDisabled] = useState(true); + const [visibleFiles, setVisibleFiles] = useState(FILE_CARDS_PAGE); + const incomplete = incompleteImportIndexes(importForm); // If the user has selected the 'lido' tag and MEV-Boost is not set, show the warning useEffect(() => { @@ -137,11 +146,18 @@ export default function ImportScreen({ setIsButtonDisabled(disable); // Set the state based on the conditions }, [passwords, tags, feeRecipients, acceptedFiles, slashingFile, slashingProtectionIncluded, showMevWarning]); - async function importKeystores() { + /** Import in batches with live progress. `only` retries a subset while keeping earlier results. */ + async function importKeystores(only?: number[]) { try { setImportStatus(ImportStatus.Importing); setKeystoresPostError(undefined); - setKeystoresPostResponse(undefined); + setImportStopped(undefined); + stopRef.current = false; + setStopRequested(false); + if (!only) { + setImportResults(undefined); + setImportProgress(undefined); + } handleClickOpenDialog(); const importRequest = await createImportRequest({ @@ -151,11 +167,24 @@ export default function ImportScreen({ feeRecipients, slashingFile: slashingProtectionIncluded ? slashingFile : undefined }); + const batches = planImportBatches( + importRequest, + acceptedFiles.map(({ pubkey }) => pubkey), + { only } + ); + const { results, stopped } = await runImportBatches({ + batches, + results: (only && importResults) || acceptedFiles.map(notAttempted), + importBatch: (request) => rpcClient.call("importValidators", request), + onProgress: (progress, results) => { + setImportProgress(progress); + setImportResults(results); + }, + shouldStop: () => stopRef.current + }); - const response = await rpcClient.call("importValidators", importRequest); - - setKeystoresPostResponse(response); - setKeystoresPostError(undefined); + setImportResults(results); + setImportStopped(stopped); setImportStatus(ImportStatus.Imported); } catch (e) { console.error(e); @@ -328,6 +357,23 @@ export default function ImportScreen({ )} + {acceptedFiles.length > 1 && ( + + {acceptedFiles.length} keystores selected + + + )} + {FileCardList( acceptedFiles, setAcceptedFiles, @@ -342,7 +388,35 @@ export default function ImportScreen({ useSameFeerecipient, getFeeRecipientFieldHelperText, isFeeRecipientFieldWrong, - tagSelectOptions + tagSelectOptions, + visibleFiles + )} + + {acceptedFiles.length > visibleFiles && ( + + + + Showing {visibleFiles} of {acceptedFiles.length} keystores. + {!(useSamePassword && useSameTag && useSameFeerecipient) && + " Hidden keystores still need their own values; turn on shared settings to fill them all at once."} + + + + + + + )} @@ -397,6 +471,21 @@ export default function ImportScreen({ ) : null} + {incomplete.length > 0 && acceptedFiles.length > 1 && ( + = visibleFiles ? ( + + ) : undefined + } + > + {incomplete.length} of {acceptedFiles.length} keystores still need a password, protocol or valid fee recipient + (first: #{incomplete[0] + 1}, {acceptedFiles[incomplete[0]]?.file.name}). + + )} {showMevWarning && ( } disabled={isButtonDisabled || importStatus === ImportStatus.Importing} - onClick={importKeystores} + onClick={() => importKeystores()} sx={{ borderRadius: 2 }} > Import validators @@ -448,10 +537,18 @@ export default function ImportScreen({ { + stopRef.current = true; + setStopRequested(true); + }} + onRetry={(indexes) => importKeystores(indexes)} /> ); diff --git a/packages/ui/src/components/Dialogs/ImportDialog.tsx b/packages/ui/src/components/Dialogs/ImportDialog.tsx index dcc29bcb..06c553cd 100644 --- a/packages/ui/src/components/Dialogs/ImportDialog.tsx +++ b/packages/ui/src/components/Dialogs/ImportDialog.tsx @@ -1,12 +1,19 @@ import type { JSX } from "react"; +import { useMemo, useState } from "react"; import { Alert, Dialog, DialogContent, Box, Button, + Chip, DialogActions, Divider, + FormControlLabel, + LinearProgress, + Stack, + Switch, + Typography, useMediaQuery, useTheme } from "@mui/material"; @@ -18,30 +25,70 @@ import { SlideTransition } from "./Transitions"; import UploadFileIcon from "@mui/icons-material/UploadFile"; import DialogHeader from "./DialogHeader"; import ValidatorResult from "./ValidatorResult"; -import type { Web3signerPostResponse } from "@stakingbrain/brain"; +import { + ImportProgress, + ImportResult, + NOT_ATTEMPTED, + retryableIndexes, + summarizeImport +} from "../../utils/importBatches"; + +// Rendering hundreds of result cards at once makes the dialog sluggish; reveal more on demand. +const RESULTS_PAGE = 100; + +const summaryLabels: [status: string, label: string, color: "success" | "default" | "error"][] = [ + ["imported", "imported", "success"], + ["duplicate", "already present", "default"], + ["error", "failed", "error"], + [NOT_ATTEMPTED, "not attempted", "default"] +]; export default function ImportDialog({ open, setOpen, - keystoresPostResponse, - keystoresPostError, + results, + progress, + error, + stopped, + stopRequested, importStatus, - acceptedFiles + acceptedFiles, + onStop, + onRetry }: { open: boolean; setOpen: (open: boolean) => void; - keystoresPostResponse: Web3signerPostResponse | undefined; - keystoresPostError: string | undefined; + results: ImportResult[] | undefined; + progress: ImportProgress | undefined; + error: string | undefined; + stopped: "user" | "error" | undefined; + stopRequested: boolean; importStatus: string; acceptedFiles: KeystoreInfo[]; + onStop: () => void; + onRetry: (indexes: number[]) => void; }): JSX.Element { const navigate = useNavigate(); const isMobile = useMediaQuery(useTheme().breakpoints.down("sm")); + const importing = importStatus === ImportStatus.Importing; + const [onlyProblems, setOnlyProblems] = useState(true); + const [visible, setVisible] = useState(RESULTS_PAGE); const handleClose = () => { setOpen(false); navigate({ pathname: "/", search: window.location.search }); }; + const summary = useMemo(() => summarizeImport(results ?? []), [results]); + const retry = useMemo(() => retryableIndexes(results ?? []), [results]); + const problems = (summary.error ?? 0) + (summary[NOT_ATTEMPTED] ?? 0); + // While importing, untouched rows would only be noise; afterwards show problems first when there are any. + const filtering = (results?.length ?? 0) > 10 && onlyProblems && (problems > 0 || importing); + const shown = (results ?? []) + .map((result, index) => ({ result, index })) + .filter(({ result }) => + filtering ? (importing ? result.status === "error" : !["imported", "duplicate"].includes(result.status)) : true + ); + return ( { - if (!reason) { - handleClose(); - } + if (!reason && !importing) handleClose(); }} aria-labelledby="alert-dialog-title" aria-describedby="alert-dialog-description" TransitionComponent={SlideTransition} > } count={acceptedFiles.length} onClose={handleClose} - busy={importStatus === ImportStatus.Importing} + busy={importing} /> - {keystoresPostError ? ( - {keystoresPostError} - ) : keystoresPostResponse ? ( -
- {keystoresPostResponse.data.map((result, index) => ( - - ))} -
- ) : ( + {error ? ( + {error} + ) : !results || !progress ? ( + ) : ( + + + + + {progress.done} of {progress.total} validators processed + + {progress.batches > 1 && ( + + Batch {Math.min(progress.batch + (importing ? 1 : 0), progress.batches)} of {progress.batches} + + )} + + + {importing && ( + + Decrypting keystores takes time. Keep this window open; you can stop after the current batch. + + )} + + + + {summaryLabels + .filter(([status]) => summary[status]) + .map(([status, label, color]) => ( + + ))} + + + {stopped === "error" && ( + + The import stopped because a batch failed. Validators not yet imported can be retried; keys that did + reach the signer will show as already present. + + )} + {stopped === "user" && ( + Import stopped. You can retry the validators that were not imported. + )} + + {results.length > 10 && (problems > 0 || importing) && ( + setOnlyProblems(checked)} />} + label="Show only problems" + /> + )} + +
+ {shown.slice(0, visible).map(({ result, index }) => ( + + ))} + {shown.length > visible && ( + + )} +
+
)}
- + ) : ( + retry.length > 0 && + !error && ( + + ) + )} + diff --git a/packages/ui/src/components/Dialogs/ValidatorResult.tsx b/packages/ui/src/components/Dialogs/ValidatorResult.tsx index c4515541..b81962d5 100644 --- a/packages/ui/src/components/Dialogs/ValidatorResult.tsx +++ b/packages/ui/src/components/Dialogs/ValidatorResult.tsx @@ -21,7 +21,7 @@ export default function ValidatorResult({ {shortenPubkey(pubkey)} - + {message && ( diff --git a/packages/ui/src/components/FileCards/FileCardList.tsx b/packages/ui/src/components/FileCards/FileCardList.tsx index 2c1af428..ba7c3cf6 100644 --- a/packages/ui/src/components/FileCards/FileCardList.tsx +++ b/packages/ui/src/components/FileCards/FileCardList.tsx @@ -18,7 +18,9 @@ export default function FileCardList( useSameFeeRecipient: boolean, getFeeRecipientFieldHelperText: (index: number) => string, isFeeRecipientFieldWrong: (index: number) => boolean, - tagSelectOptions: TagSelectOption[] + tagSelectOptions: TagSelectOption[], + // Only the first `limit` cards render; indexes stay aligned because the slice starts at 0. + limit = Infinity ): JSX.Element[] { const removeFileFromList = ( fileInfo: KeystoreInfo, @@ -38,115 +40,119 @@ export default function FileCardList( setTags(tags.filter((f, index) => index !== indexToRemove)); }; - return Array.from(fileInfos).map((fileInfo, index) => ( - - - - - {fileInfo.file.name} - - ( + + + + + {fileInfo.file.name} + + + + + aria-label={`Remove ${fileInfo.file.name}`} + onClick={() => + removeFileFromList( + fileInfo, + fileInfos, + setAcceptedFiles, + passwords, + setPasswords, + feeRecipients, + setFeeRecipients, + tags, + setTags + ) + } + > + + - - removeFileFromList( - fileInfo, - fileInfos, - setAcceptedFiles, - passwords, - setPasswords, - feeRecipients, - setFeeRecipients, - tags, - setTags - ) - } - > - - - - {(!useSameTag || !useSameFeeRecipient || !useSamePassword) && ( - - {!useSamePassword && ( - { - const newPasswords = [...passwords]; - newPasswords[index] = event.target.value; - setPasswords(newPasswords); - }} - helperText="Password used to create this keystore." - /> - )} - {!useSameTag && ( - <> + {(!useSameTag || !useSameFeeRecipient || !useSamePassword) && ( + + {!useSamePassword && ( { - const newTags = [...tags]; - newTags[index] = event.target.value as Tag; - setTags(newTags); + const newPasswords = [...passwords]; + newPasswords[index] = event.target.value; + setPasswords(newPasswords); + }} + helperText="Password used to create this keystore." + /> + )} + {!useSameTag && ( + <> + { + const newTags = [...tags]; + newTags[index] = event.target.value as Tag; + setTags(newTags); - if (!useSameFeeRecipient && !isFeeRecipientEditable(event.target.value as Tag)) { - const newFeeRecipients = [...feeRecipients]; - newFeeRecipients[index] = ""; - setFeeRecipients(newFeeRecipients); - } + if (!useSameFeeRecipient && !isFeeRecipientEditable(event.target.value as Tag)) { + const newFeeRecipients = [...feeRecipients]; + newFeeRecipients[index] = ""; + setFeeRecipients(newFeeRecipients); + } + }} + > + {tagSelectOptions.map((option) => ( + + {option.label} + + ))} + + + )} + {!useSameFeeRecipient && ( + { + const newFeeRecipients = [...feeRecipients]; + newFeeRecipients[index] = event.target.value; + setFeeRecipients(newFeeRecipients); }} - > - {tagSelectOptions.map((option) => ( - - {option.label} - - ))} - - - )} - {!useSameFeeRecipient && ( - { - const newFeeRecipients = [...feeRecipients]; - newFeeRecipients[index] = event.target.value; - setFeeRecipients(newFeeRecipients); - }} - error={isFeeRecipientFieldWrong(index)} - helperText={getFeeRecipientFieldHelperText(index)} - disabled={!!tags[index] && !isFeeRecipientEditable(tags[index])} - /> - )} - - )} - - )); + error={isFeeRecipientFieldWrong(index)} + helperText={getFeeRecipientFieldHelperText(index)} + disabled={!!tags[index] && !isFeeRecipientEditable(tags[index])} + /> + )} + + )} + + )); } diff --git a/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx b/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx index 69916950..ae409f58 100644 --- a/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx +++ b/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx @@ -61,10 +61,6 @@ export default function KeystoresDataGrid({ const [paginationModel, setPaginationModel] = useState({ page: 0, pageSize: 10 }); const blockExplorerUrl = blockExplorerUrls.get(network); - useEffect(() => { - setSummaryUrlBuildingStatus(BeaconchaUrlBuildingStatus.NotStarted); - }, [selectedRows, network, rows]); - const [validatorsSubscriptionMap, setValidatorsSubscriptionMap] = useState(null); const [oracleCallError, setOracleCallError] = useState(); @@ -154,11 +150,18 @@ export default function KeystoresDataGrid({ setOracleCallError("Error fetching subscription status. Oracle might be down: " + e.message); } }; + // The list refreshes live and in rounds; only query the external oracle when what it is asked + // about changes, and not on every round while details are still loading. + const oracleQueryKey = rows.map((row) => `${row.pubkey}:${row.tag}:${row.index}`).join(","); + const detailsLoading = rows.some((row) => row.detailsPending); + // Live refreshes replace `rows` often; only reset the dashboard hint when what it depends on changes. useEffect(() => { - if (rows.length > 0) { - fetchValidatorsData(rows); - } - }, [rows]); + setSummaryUrlBuildingStatus(BeaconchaUrlBuildingStatus.NotStarted); + }, [selectedRows, network, oracleQueryKey]); + useEffect(() => { + if (rows.length === 0 || (detailsLoading && validatorsSubscriptionMap !== null)) return; + fetchValidatorsData(rows); + }, [oracleQueryKey, detailsLoading, network]); const columns: GridColDef[] = [ { diff --git a/packages/ui/src/components/ValidatorList/ValidatorList.tsx b/packages/ui/src/components/ValidatorList/ValidatorList.tsx index 1baa83f8..d880b7b2 100644 --- a/packages/ui/src/components/ValidatorList/ValidatorList.tsx +++ b/packages/ui/src/components/ValidatorList/ValidatorList.tsx @@ -3,10 +3,10 @@ import KeystoresDataGrid from "./KeystoresDataGrid"; import KeystoresDeleteDialog from "../Dialogs/KeystoresDeleteDialog"; import EditFeesDialog from "../Dialogs/EditFeesDialog"; import BuilderConfigDialog from "../Dialogs/BuilderConfigDialog"; -import { Alert, Box, CircularProgress } from "@mui/material"; +import { Alert, Box, Button, CircularProgress, LinearProgress, Typography } from "@mui/material"; import type { GridRowId } from "@mui/x-data-grid"; -import { StakerConfig as StakerConfigType } from "@stakingbrain/common"; -import { useEffect, useState } from "react"; +import { StakerConfig as StakerConfigType, VALIDATORS_CHANGED_EVENT } from "@stakingbrain/common"; +import { useEffect, useRef, useState } from "react"; import { BeaconchaUrlBuildingStatus } from "../../types"; import { rpcClient } from "../../socket"; import StakerConfig from "../StakerConfig/StakerConfig"; @@ -14,6 +14,13 @@ import KeystoresExitDialog from "../Dialogs/KeystoresExitDialog"; import { getSmoothAddressByNetwork } from "../../utils/addresses"; import type { CustomValidatorGetResponse } from "@stakingbrain/brain"; import PageLayout from "../Layout/PageLayout"; +import { + countPending, + MAX_STALLED_REFRESHES, + nextStallCount, + PENDING_REFRESH_MS, + remapSelection +} from "../../utils/validatorRefresh"; export default function ValidatorList({ stakerConfig, @@ -34,27 +41,81 @@ export default function ValidatorList({ const { network, isMevBoostSet } = stakerConfig; const smoothAddress = getSmoothAddressByNetwork(network); + const anyDialogOpen = deleteOpen || editFeesOpen || exitOpen || builderConfigOpen; + const dialogOpenRef = useRef(anyDialogOpen); + dialogOpenRef.current = anyDialogOpen; + const rowsRef = useRef(undefined); + const inFlight = useRef(false); + const queued = useRef(false); + const stalls = useRef(0); + const lastPending = useRef(undefined); + const [stalled, setStalled] = useState(false); + // Re-render table after delete/update validators useEffect(() => { - if (!deleteOpen && !editFeesOpen && !exitOpen && !builderConfigOpen) { - getValidators(); + if (!anyDialogOpen) { + // Only the first load blanks the page; later refreshes keep the table visible. + getValidators(rowsRef.current !== undefined); setSelectedRows([]); } - }, [deleteOpen, editFeesOpen, exitOpen, builderConfigOpen]); + }, [anyDialogOpen]); + + // Live updates: the Brain pushes a hint after every validator change, from any tab or the reload job. + // Dialogs act on row positions, so refreshes wait until they close (closing always refreshes). + useEffect(() => { + const refresh = () => { + if (!dialogOpenRef.current) void getValidators(true); + }; + const offChanged = rpcClient.on(VALIDATORS_CHANGED_EVENT, refresh); + // Catch up on hints missed while disconnected. + const offConnect = rpcClient.on("connect", refresh); + return () => { + offChanged(); + offConnect(); + }; + }, []); - async function getValidators() { + // Large lists load in bounded rounds; keep refreshing while rows are pending and progress is made. + useEffect(() => { + if (anyDialogOpen || stalled || !validatorsGet || countPending(validatorsGet) === 0) return; + const timer = setTimeout(() => getValidators(true), PENDING_REFRESH_MS); + return () => clearTimeout(timer); + }, [validatorsGet, anyDialogOpen, stalled]); + + async function getValidators(background = false) { + // Coalesce overlapping triggers into one follow-up refresh. + if (inFlight.current) { + queued.current = true; + return; + } + inFlight.current = true; try { - setLoading(true); - setValidatorsGet(await rpcClient.call("getValidators", undefined)); + if (!background) setLoading(true); + const rows = await rpcClient.call("getValidators", undefined); + if (background) setSelectedRows((selected) => remapSelection(rowsRef.current, rows, selected)); + rowsRef.current = rows; + const pending = countPending(rows); + stalls.current = nextStallCount(stalls.current, lastPending.current, pending); + lastPending.current = pending; + setStalled(stalls.current >= MAX_STALLED_REFRESHES); + setValidatorsGet(rows); setValidatorsGetError(undefined); - setLoading(false); } catch (e) { console.error(e); - setValidatorsGetError(e.message); - setLoading(false); + // A failed background refresh keeps the last good table instead of replacing it with an error. + if (!background || !rowsRef.current) setValidatorsGetError(e.message); + } finally { + if (!background) setLoading(false); + inFlight.current = false; + if (queued.current) { + queued.current = false; + if (!dialogOpenRef.current) void getValidators(true); + } } } + const pendingCount = validatorsGet ? countPending(validatorsGet) : 0; + const SmoothBanner = () => { return ( @@ -102,6 +163,41 @@ export default function ValidatorList({ ) : validatorsGet ? ( <> + {pendingCount > 0 && + (stalled ? ( + { + stalls.current = 0; + lastPending.current = undefined; + setStalled(false); + void getValidators(true); + }} + > + Retry + + } + > + Details for {pendingCount} of {validatorsGet.length} validators are still loading slowly. Your + consensus or validator client may be busy or syncing. + + ) : ( + + + Loading details for {pendingCount} of {validatorsGet.length} validators… + + + + ))} void): () => void { + this.socket.on(event, handler); + return () => { + this.socket.off(event, handler); + }; + } + async call(method: T, params: RoutesArguments[T]): Promise { return new Promise((resolve, reject) => { const rpcRequest = { diff --git a/packages/ui/src/utils/importBatches.ts b/packages/ui/src/utils/importBatches.ts new file mode 100644 index 00000000..82211366 --- /dev/null +++ b/packages/ui/src/utils/importBatches.ts @@ -0,0 +1,153 @@ +import type { CustomImportRequest, Web3signerPostResponse } from "@stakingbrain/brain"; +import { prefix0xPubkey, RPC_MAX_PAYLOAD_BYTES } from "@stakingbrain/common"; + +/** + * Keystore decryption in the signer is slow, so one RPC call per batch keeps every call well inside + * the RPC timeout, lets the UI report progress and lets the user stop between batches. + */ +export const IMPORT_BATCH_SIZE = 20; +// Well below the transport limit: JSON escaping roughly doubles keystore text and slashing data rides along. +const IMPORT_BATCH_MAX_BYTES = RPC_MAX_PAYLOAD_BYTES / 4; + +export interface ImportResult { + status: string; + message?: string; +} + +export interface ImportBatch { + /** Positions in the original file list, so results stay aligned with the files the user picked. */ + indexes: number[]; + request: CustomImportRequest; +} + +export interface ImportProgress { + done: number; + total: number; + batch: number; + batches: number; +} + +export const NOT_ATTEMPTED = "not_attempted"; + +export function notAttempted(): ImportResult { + return { status: NOT_ATTEMPTED, message: "Not imported yet: the import stopped before reaching this validator." }; +} + +/** Keep only the slashing history of the batch's keys; the signer only needs those. */ +export function filterInterchange(interchange: string, pubkeys: ReadonlySet): string { + let parsed: unknown; + try { + parsed = JSON.parse(interchange); + } catch { + return interchange; // Let the signer report the malformed file once, as before. + } + if (!parsed || typeof parsed !== "object" || !Array.isArray((parsed as { data?: unknown }).data)) return interchange; + const record = parsed as { data: { pubkey?: unknown }[] }; + return JSON.stringify({ + ...record, + data: record.data.filter( + (entry) => typeof entry?.pubkey === "string" && pubkeys.has(prefix0xPubkey(entry.pubkey).toLowerCase()) + ) + }); +} + +/** Split an import into ordered batches by count and size. `only` limits a retry to some files. */ +export function planImportBatches( + request: CustomImportRequest, + pubkeys: readonly string[], + { + only, + maxKeystores = IMPORT_BATCH_SIZE, + maxBytes = IMPORT_BATCH_MAX_BYTES + }: { only?: readonly number[]; maxKeystores?: number; maxBytes?: number } = {} +): ImportBatch[] { + const selected = only ?? request.validatorsImportRequest.map((_, index) => index); + const groups: number[][] = []; + let group: number[] = []; + let bytes = 0; + for (const index of selected) { + const { keystore, password } = request.validatorsImportRequest[index]; + const size = keystore.length + password.length; + if (group.length > 0 && (group.length >= maxKeystores || bytes + size > maxBytes)) { + groups.push(group); + group = []; + bytes = 0; + } + group.push(index); + bytes += size; + } + if (group.length > 0) groups.push(group); + + return groups.map((indexes) => ({ + indexes, + request: { + importFrom: request.importFrom, + slashing_protection: + request.slashing_protection === undefined + ? undefined + : filterInterchange( + request.slashing_protection, + new Set(indexes.map((index) => prefix0xPubkey(pubkeys[index]).toLowerCase())) + ), + validatorsImportRequest: indexes.map((index) => request.validatorsImportRequest[index]) + } + })); +} + +/** + * Import batches in order. A failed batch stops the run: its error may be a timeout whose outcome is + * unknown, and the remaining files are left "not attempted" so the user can retry them safely + * (keys that did land are reported as duplicates). + */ +export async function runImportBatches({ + batches, + results: initial, + importBatch, + onProgress, + shouldStop +}: { + batches: readonly ImportBatch[]; + results: readonly ImportResult[]; + importBatch: (request: CustomImportRequest) => Promise; + onProgress: (progress: ImportProgress, results: ImportResult[]) => void; + shouldStop: () => boolean; +}): Promise<{ results: ImportResult[]; stopped?: "user" | "error" }> { + const results = [...initial]; + for (const { indexes } of batches) for (const index of indexes) results[index] = notAttempted(); + const total = batches.reduce((sum, { indexes }) => sum + indexes.length, 0); + let done = 0; + onProgress({ done, total, batch: 0, batches: batches.length }, [...results]); + + for (const [position, batch] of batches.entries()) { + if (shouldStop()) return { results, stopped: "user" }; + let failed = false; + try { + const response = await importBatch(batch.request); + batch.indexes.forEach((index, offset) => { + results[index] = response.data[offset] ?? { + status: "error", + message: "Import outcome unknown: Brain returned no result for this key. Retry to check." + }; + }); + } catch (error) { + failed = true; + const message = error instanceof Error ? error.message : String(error); + for (const index of batch.indexes) results[index] = { status: "error", message }; + } + done += batch.indexes.length; + onProgress({ done, total, batch: position + 1, batches: batches.length }, [...results]); + if (failed) return { results, stopped: "error" }; + } + return { results }; +} + +/** Files worth retrying: anything not confirmed imported or already present. */ +export function retryableIndexes(results: readonly ImportResult[]): number[] { + return results.flatMap((result, index) => (["imported", "duplicate"].includes(result?.status) ? [] : [index])); +} + +export function summarizeImport(results: readonly ImportResult[]): Record { + const summary: Record = {}; + for (const { status } of results) summary[status] = (summary[status] ?? 0) + 1; + return summary; +} diff --git a/packages/ui/src/utils/importForm.ts b/packages/ui/src/utils/importForm.ts index 6390db6a..d262c4b5 100644 --- a/packages/ui/src/utils/importForm.ts +++ b/packages/ui/src/utils/importForm.ts @@ -32,18 +32,22 @@ export function appendImportFiles( }; } -export function isImportFormValid({ acceptedFiles, passwords, tags, feeRecipients }: ImportFormState): boolean { +function isImportEntryValid({ passwords, tags, feeRecipients }: ImportFormState, index: number): boolean { + const tag = tags[index]; return ( - acceptedFiles.length > 0 && - acceptedFiles.every((_, index) => { - const tag = tags[index]; - return ( - typeof passwords[index] === "string" && - passwords[index].length > 0 && - !!tag && - (!isFeeRecipientEditable(tag) || - (isValidEcdsaPubkey(feeRecipients[index]) && feeRecipients[index] !== BURN_ADDRESS)) - ); - }) + typeof passwords[index] === "string" && + passwords[index].length > 0 && + !!tag && + (!isFeeRecipientEditable(tag) || + (isValidEcdsaPubkey(feeRecipients[index]) && feeRecipients[index] !== BURN_ADDRESS)) ); } + +/** Positions of files still missing a password, protocol or valid fee recipient. */ +export function incompleteImportIndexes(state: ImportFormState): number[] { + return state.acceptedFiles.flatMap((_, index) => (isImportEntryValid(state, index) ? [] : [index])); +} + +export function isImportFormValid(state: ImportFormState): boolean { + return state.acceptedFiles.length > 0 && incompleteImportIndexes(state).length === 0; +} diff --git a/packages/ui/src/utils/validatorRefresh.ts b/packages/ui/src/utils/validatorRefresh.ts new file mode 100644 index 00000000..b813a4c2 --- /dev/null +++ b/packages/ui/src/utils/validatorRefresh.ts @@ -0,0 +1,30 @@ +import type { GridRowId } from "@mui/x-data-grid"; + +/** Delay between refreshes while the Brain reports rows whose checks were cut short. */ +export const PENDING_REFRESH_MS = 3000; +/** Stop auto-refreshing after this many refreshes in a row without fewer pending rows. */ +export const MAX_STALLED_REFRESHES = 3; + +/** Grid ids are row positions; keep the same validators selected when a refresh reorders or removes rows. */ +export function remapSelection( + previous: readonly { pubkey: string }[] | undefined, + next: readonly { pubkey: string }[], + selected: readonly GridRowId[] +): GridRowId[] { + if (!previous) return []; + const positions = new Map(next.map(({ pubkey }, index) => [pubkey.toLowerCase(), index])); + return selected.flatMap((id) => { + const position = positions.get(previous[id as number]?.pubkey.toLowerCase() ?? ""); + return position === undefined ? [] : [position]; + }); +} + +export function countPending(rows: readonly { detailsPending?: boolean }[]): number { + return rows.filter(({ detailsPending }) => detailsPending).length; +} + +/** Track consecutive refreshes that did not reduce the number of pending rows. */ +export function nextStallCount(stalls: number, previousPending: number | undefined, pending: number): number { + if (pending === 0) return 0; + return previousPending !== undefined && pending >= previousPending ? stalls + 1 : 0; +} diff --git a/packages/ui/test/importBatches.test.ts b/packages/ui/test/importBatches.test.ts new file mode 100644 index 00000000..04960db0 --- /dev/null +++ b/packages/ui/test/importBatches.test.ts @@ -0,0 +1,137 @@ +import { expect } from "chai"; +import type { CustomImportRequest, Web3signerPostResponse } from "@stakingbrain/brain"; +import { + filterInterchange, + ImportProgress, + notAttempted, + NOT_ATTEMPTED, + planImportBatches, + retryableIndexes, + runImportBatches, + summarizeImport +} from "../src/utils/importBatches.js"; + +const pubkeys = Array.from({ length: 45 }, (_, index) => "0x" + (index + 1).toString(16).padStart(96, "0")); +const interchange = JSON.stringify({ + metadata: { interchange_format_version: "5", genesis_validators_root: "0x" + "00".repeat(32) }, + data: pubkeys.map((pubkey) => ({ pubkey, signed_blocks: [], signed_attestations: [] })) +}); +const request = (slashing?: string): CustomImportRequest => ({ + importFrom: "ui", + slashing_protection: slashing, + validatorsImportRequest: pubkeys.map((pubkey) => ({ + keystore: JSON.stringify({ pubkey: pubkey.slice(2) }), + password: "password", + tag: "solo", + feeRecipient: "0x" + "ab".repeat(20) + })) +}); +const imported = (request: CustomImportRequest): Web3signerPostResponse => ({ + data: request.validatorsImportRequest.map(() => ({ status: "imported", message: "" })) +}); + +describe("Batched validator import", () => { + it("splits by count, keeps file order and sends each batch only its own slashing history", () => { + const batches = planImportBatches(request(interchange), pubkeys); + expect(batches.map(({ indexes }) => indexes.length)).to.deep.equal([20, 20, 5]); + expect(batches.flatMap(({ indexes }) => indexes)).to.deep.equal(pubkeys.map((_, index) => index)); + const second = JSON.parse(batches[1].request.slashing_protection!); + expect(second.metadata.interchange_format_version).to.equal("5"); + expect(second.data.map(({ pubkey }: { pubkey: string }) => pubkey)).to.deep.equal(pubkeys.slice(20, 40)); + }); + + it("splits by size so a batch stays under the transport limit", () => { + const batches = planImportBatches(request(), pubkeys, { maxBytes: 1000 }); + expect(batches.every(({ indexes }) => indexes.length < 20)).to.equal(true); + expect(batches.flatMap(({ indexes }) => indexes)).to.have.length(45); + }); + + it("leaves a malformed slashing file untouched for the signer to report", () => { + expect(filterInterchange("{not json", new Set(pubkeys))).to.equal("{not json"); + expect(planImportBatches(request(), pubkeys)[0].request.slashing_protection).to.equal(undefined); + }); + + it("matches slashing entries regardless of 0x prefix and hex casing", () => { + const data = JSON.stringify({ metadata: {}, data: [{ pubkey: pubkeys[0].slice(2).toUpperCase() }] }); + expect(JSON.parse(filterInterchange(data, new Set([pubkeys[0]]))).data).to.have.length(1); + }); + + it("reports progress after each batch and keeps results aligned with files", async () => { + const progress: ImportProgress[] = []; + const { results, stopped } = await runImportBatches({ + batches: planImportBatches(request(), pubkeys), + results: pubkeys.map(notAttempted), + importBatch: async (batch) => imported(batch), + onProgress: (update) => progress.push(update), + shouldStop: () => false + }); + expect(stopped).to.equal(undefined); + expect(progress.map(({ done }) => done)).to.deep.equal([0, 20, 40, 45]); + expect(results.every(({ status }) => status === "imported")).to.equal(true); + }); + + it("stops after a failed batch and leaves the rest retryable", async () => { + let calls = 0; + const { results, stopped } = await runImportBatches({ + batches: planImportBatches(request(), pubkeys), + results: pubkeys.map(notAttempted), + importBatch: async (batch) => { + if (++calls === 2) throw new Error("Connection lost or request timed out."); + return imported(batch); + }, + onProgress: () => undefined, + shouldStop: () => false + }); + expect(stopped).to.equal("error"); + expect(calls).to.equal(2); + expect(summarizeImport(results)).to.deep.equal({ imported: 20, error: 20, [NOT_ATTEMPTED]: 5 }); + expect(results[25].message).to.include("timed out"); + expect(retryableIndexes(results)).to.deep.equal(pubkeys.map((_, index) => index).slice(20)); + }); + + it("honours a stop request between batches", async () => { + let stop = false; + const { results, stopped } = await runImportBatches({ + batches: planImportBatches(request(), pubkeys), + results: pubkeys.map(notAttempted), + importBatch: async (batch) => { + stop = true; + return imported(batch); + }, + onProgress: () => undefined, + shouldStop: () => stop + }); + expect(stopped).to.equal("user"); + expect(summarizeImport(results)).to.deep.equal({ imported: 20, [NOT_ATTEMPTED]: 25 }); + }); + + it("retries only the requested files and keeps earlier results", async () => { + const earlier = pubkeys.map((_, index) => + index < 40 ? { status: "imported", message: "" } : { status: "error", message: "boom" } + ); + const sent: number[] = []; + const { results } = await runImportBatches({ + batches: planImportBatches(request(), pubkeys, { only: retryableIndexes(earlier) }), + results: earlier, + importBatch: async (batch) => { + sent.push(batch.validatorsImportRequest.length); + return { data: batch.validatorsImportRequest.map(() => ({ status: "duplicate", message: "" })) }; + }, + onProgress: () => undefined, + shouldStop: () => false + }); + expect(sent).to.deep.equal([5]); + expect(summarizeImport(results)).to.deep.equal({ imported: 40, duplicate: 5 }); + }); + + it("marks keys without a returned result as errors instead of success", async () => { + const { results } = await runImportBatches({ + batches: planImportBatches(request(), pubkeys.slice(0, 2), { only: [0, 1] }), + results: [notAttempted(), notAttempted()], + importBatch: async () => ({ data: [{ status: "imported", message: "" }] }), + onProgress: () => undefined, + shouldStop: () => false + }); + expect(results.map(({ status }) => status)).to.deep.equal(["imported", "error"]); + }); +}); diff --git a/packages/ui/test/validatorRefresh.test.ts b/packages/ui/test/validatorRefresh.test.ts new file mode 100644 index 00000000..fe78472b --- /dev/null +++ b/packages/ui/test/validatorRefresh.test.ts @@ -0,0 +1,40 @@ +import { expect } from "chai"; +import { countPending, nextStallCount, remapSelection } from "../src/utils/validatorRefresh.js"; +import { incompleteImportIndexes } from "../src/utils/importForm.js"; + +const row = (pubkey: string, detailsPending?: boolean) => ({ pubkey, detailsPending }); + +describe("Live validator list refresh", () => { + it("keeps the same validators selected when rows move or disappear", () => { + const previous = [row("0xaa"), row("0xbb"), row("0xcc")]; + const next = [row("0xCC"), row("0xaa")]; + expect(remapSelection(previous, next, [0, 1, 2])).to.deep.equal([1, 0]); + }); + + it("clears the selection when there was no earlier list", () => { + expect(remapSelection(undefined, [row("0xaa")], [0])).to.deep.equal([]); + }); + + it("counts pending rows and stops after refreshes that make no progress", () => { + expect(countPending([row("0xaa", true), row("0xbb", false), row("0xcc")])).to.equal(1); + expect(nextStallCount(0, undefined, 10)).to.equal(0); + expect(nextStallCount(0, 10, 6)).to.equal(0); + expect(nextStallCount(0, 6, 6)).to.equal(1); + expect(nextStallCount(2, 6, 7)).to.equal(3); + expect(nextStallCount(2, 6, 0)).to.equal(0); + }); +}); + +describe("Large import forms", () => { + it("points at the files that still need input", () => { + const acceptedFiles = ["a", "b", "c"].map((name) => ({ file: new File(["{}"], name), pubkey: "0x" + name })); + expect( + incompleteImportIndexes({ + acceptedFiles, + passwords: ["pw", "", "pw"], + tags: ["solo", "solo", ""], + feeRecipients: ["0x" + "ab".repeat(20), "0x" + "ab".repeat(20), ""] + }) + ).to.deep.equal([1, 2]); + }); +}); From 829529c9d7c83197f051c5516b36aa9886476d76 Mon Sep 17 00:00:00 2001 From: 3alpha <15694175+3alpha@users.noreply.github.com> Date: Mon, 16 Nov 2026 09:59:49 +0100 Subject: [PATCH 3/8] fix: update Node to 24.22.3 for November 2026 security release Covers the HTTP/2, TLS, HTTPS, zlib and permission model fixes published on 2026-11-11 (https://nodejs.org/en/blog/vulnerability/november-2026-security-releases). --- .github/workflows/main.yml | 2 +- .nvmrc | 2 +- Dockerfile | 2 +- Dockerfile.brain.dev | 2 +- package.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b3dbfb99..025c8a32 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,7 +37,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v7 with: - node-version: 24.22.1 + node-version: 24.22.3 - name: Enable Corepack run: corepack enable diff --git a/.nvmrc b/.nvmrc index ce62013c..229b3ce8 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -24.22.1 +24.22.3 diff --git a/Dockerfile b/Dockerfile index 07fb3425..34543349 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG DOCKER_IMAGE=node:24.22.1-alpine3.24 +ARG DOCKER_IMAGE=node:24.22.3-alpine3.24 # Build FROM ${DOCKER_IMAGE} AS build-stage diff --git a/Dockerfile.brain.dev b/Dockerfile.brain.dev index 9dca83a3..35c4e936 100644 --- a/Dockerfile.brain.dev +++ b/Dockerfile.brain.dev @@ -1,4 +1,4 @@ -ARG DOCKER_IMAGE=node:24.22.1-alpine3.24 +ARG DOCKER_IMAGE=node:24.22.3-alpine3.24 FROM ${DOCKER_IMAGE} AS build-stage RUN apk add --no-cache --upgrade ca-certificates ca-certificates-bundle && update-ca-certificates diff --git a/package.json b/package.json index 49f6ea2d..a1660b17 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "type": "module", "private": true, "engines": { - "node": ">=24.22.1" + "node": ">=24.22.3" }, "workspaces": [ "packages/*" From b12d0ab9d9f6b728600594733cc1b5b5fc638fef Mon Sep 17 00:00:00 2001 From: 3alpha <15694175+3alpha@users.noreply.github.com> Date: Mon, 16 Nov 2026 09:59:49 +0100 Subject: [PATCH 4/8] fix: do not re-query the Smooth oracle after a failed call A failed call left the subscription map unset, so every live refresh round while validator details were loading queried the oracle again. --- packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx b/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx index ae409f58..9573d033 100644 --- a/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx +++ b/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx @@ -147,6 +147,8 @@ export default function KeystoresDataGrid({ setValidatorsSubscriptionMap(newValidatorSubscriptionStatus); setOracleCallError(undefined); } catch (e) { + // Record the attempt (all UNKNOWN) so live refresh rounds do not re-query an oracle that is down. + setValidatorsSubscriptionMap(newValidatorSubscriptionStatus); setOracleCallError("Error fetching subscription status. Oracle might be down: " + e.message); } }; From fd8e99f3d73d4a11acaab22c39924623a077a9cb Mon Sep 17 00:00:00 2001 From: 3alpha <15694175+3alpha@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:05:33 +0100 Subject: [PATCH 5/8] feat: disable builder configuration for now Gate the Gloas builder feature behind featureFlags.builderConfig (off): the UI hides builder settings, builder RPCs and update fields are rejected, reload no longer replays stored overrides, and BUILDER_IDS is ignored without consuming its one-time import. Stored builder settings are kept so the feature can be re-enabled without migration. --- docs/gloas-builder-config.md | 2 + packages/brain/src/index.ts | 30 ++++--- .../apiServers/ui/calls/getValidators.ts | 13 +-- .../src/modules/apiServers/ui/calls/index.ts | 11 ++- .../apiServers/ui/calls/updateValidators.ts | 13 ++- .../modules/cron/reloadValidators/index.ts | 3 +- .../builderConfig/builderConfig.unit.test.ts | 80 ++++++++++++++++++- packages/common/src/featureFlags.ts | 16 ++++ packages/common/src/index.ts | 1 + .../ValidatorList/KeystoresDataGrid.tsx | 53 ++++++------ .../ValidatorList/ValidatorList.tsx | 4 +- 11 files changed, 176 insertions(+), 50 deletions(-) create mode 100644 packages/common/src/featureFlags.ts diff --git a/docs/gloas-builder-config.md b/docs/gloas-builder-config.md index 608185f0..337cade6 100644 --- a/docs/gloas-builder-config.md +++ b/docs/gloas-builder-config.md @@ -1,5 +1,7 @@ # Gloas / ePBS validator configuration +> **Disabled for now.** This feature is switched off with `featureFlags.builderConfig` in `packages/common/src/featureFlags.ts`. While it is off, the UI hides builder settings, the builder RPCs and builder fields in updates are rejected, reload does not apply stored settings, and `BUILDER_IDS` is ignored (its one-time import still runs once the feature is enabled). Settings already saved stay in the database unchanged, and overrides already applied to a validator client by 0.2.0-rc.0 are not removed. + Brain manages configuration. The validator client signs slot-specific proposer preferences and builder authorizations and supplies the resolved configuration in proposal requests. The beacon node solicits and selects bids. Web3Signer diff --git a/packages/brain/src/index.ts b/packages/brain/src/index.ts index bc66f94d..5a3d0ce3 100644 --- a/packages/brain/src/index.ts +++ b/packages/brain/src/index.ts @@ -9,6 +9,7 @@ import { params } from "./params.js"; import { getCrons } from "./modules/cron/index.js"; import { brainConfig } from "./modules/config/index.js"; import { importBuilderIds } from "./modules/config/importBuilderIds.js"; +import { featureFlags } from "@stakingbrain/common"; logger.info(`Starting brain...`); @@ -34,18 +35,23 @@ await brainDb.initialize(signerApi, validatorApi); logger.debug(`Loaded ${Object.keys(brainDb.getData()).length} validators`); // Import builder choices once before cron/API writes can begin. -try { - const imported = importBuilderIds({ - builderIds: process.env.BUILDER_IDS, - network: config.chain.network, - brainDb, - markerPath: path.join(path.dirname(brainDbPath), "builder-config-imported") - }); - if (imported) - logger.info(`Builder settings imported: ${imported.imported} validators; ${imported.skipped} preserved`); -} catch (error) { - // A failed optional handoff must not interrupt ordinary key/fee reconciliation. - logger.error("Unable to import BUILDER_IDS; import will retry at next startup", error); +if (featureFlags.builderConfig) { + try { + const imported = importBuilderIds({ + builderIds: process.env.BUILDER_IDS, + network: config.chain.network, + brainDb, + markerPath: path.join(path.dirname(brainDbPath), "builder-config-imported") + }); + if (imported) + logger.info(`Builder settings imported: ${imported.imported} validators; ${imported.skipped} preserved`); + } catch (error) { + // A failed optional handoff must not interrupt ordinary key/fee reconciliation. + logger.error("Unable to import BUILDER_IDS; import will retry at next startup", error); + } +} else if (process.env.BUILDER_IDS) { + // Leave the one-time marker unwritten so the import still runs once the feature is enabled. + logger.warn("BUILDER_IDS is set but builder configuration is disabled in this version; ignoring it"); } // CRON diff --git a/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts b/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts index 73cda398..c0c4f648 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts @@ -1,4 +1,4 @@ -import { isValidWithdrawableBlsAddress, isValidNonWithdrawableBlsAddress } from "@stakingbrain/common"; +import { isValidWithdrawableBlsAddress, isValidNonWithdrawableBlsAddress, featureFlags } from "@stakingbrain/common"; import { CustomValidatorGetResponse } from "./types.js"; import { BrainDataBase } from "../../../db/index.js"; import { ValidatorApi, Web3SignerApi, BeaconchainApi } from "../../../apiClients/index.js"; @@ -43,10 +43,13 @@ export async function getValidators({ index: details.index?.toString() ?? "", tag: details.tag, feeRecipient: details.feeRecipient, - builderConfig: details.builderConfig, - builderConfigStatus: - validatorApi.getBuilderConfigStatus(pubkey) ?? - (details.builderConfig === undefined ? undefined : { status: "pending" }), + // Stored settings are kept while the feature is off, but not surfaced. + ...(featureFlags.builderConfig && { + builderConfig: details.builderConfig, + builderConfigStatus: + validatorApi.getBuilderConfigStatus(pubkey) ?? + (details.builderConfig === undefined ? undefined : { status: "pending" }) + }), withdrawalCredentials: { address: "", format: "error" }, ...progress.beacon.get(pubkey)?.details // Membership and fee checks stay undefined until a successful read establishes their value. diff --git a/packages/brain/src/modules/apiServers/ui/calls/index.ts b/packages/brain/src/modules/apiServers/ui/calls/index.ts index fe59f2a1..f7a72997 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/index.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/index.ts @@ -15,6 +15,7 @@ import { NumberOfDaysToQuery } from "../../../validatorsDataIngest/types.js"; import { BeaconchainApi, PostgresClient, ValidatorApi, Web3SignerApi } from "../../../apiClients/index.js"; import { BrainConfig } from "../../../config/types.js"; import { getBuilderRelayPresets } from "../../../config/builderRelayPresets.js"; +import { featureFlags, FeatureDisabledError } from "@stakingbrain/common"; export const createRpcMethods = ({ postgresClient, @@ -37,8 +38,14 @@ export const createRpcMethods = ({ const { minGenesisTime, secondsPerSlot, network, isMevBoostSet, executionClient, consensusClient } = brainConfig.chain; return { - getBuilderRelayPresets: async () => getBuilderRelayPresets(network), - getBuilderConfigs: async (pubkeys: string[]) => await getBuilderConfigs({ pubkeys, brainDb, validatorApi }), + getBuilderRelayPresets: async () => { + if (!featureFlags.builderConfig) throw new FeatureDisabledError("Builder configuration"); + return getBuilderRelayPresets(network); + }, + getBuilderConfigs: async (pubkeys: string[]) => { + if (!featureFlags.builderConfig) throw new FeatureDisabledError("Builder configuration"); + return await getBuilderConfigs({ pubkeys, brainDb, validatorApi }); + }, deleteValidators: async (deleteRequest: Web3signerDeleteRequest) => await deleteValidators({ brainDb, reloadValidatorsCronTask, validatorApi, signerApi, deleteRequest }), exitValidators: async ({ pubkeys }: { pubkeys: string[] }) => diff --git a/packages/brain/src/modules/apiServers/ui/calls/updateValidators.ts b/packages/brain/src/modules/apiServers/ui/calls/updateValidators.ts index 46f64cbf..93a9917f 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/updateValidators.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/updateValidators.ts @@ -1,4 +1,10 @@ -import { prefix0xPubkey, isFeeRecipientEditable, ActionRequestOrigin } from "@stakingbrain/common"; +import { + prefix0xPubkey, + isFeeRecipientEditable, + ActionRequestOrigin, + featureFlags, + FeatureDisabledError +} from "@stakingbrain/common"; import { CLIENT_OPERATION_MAX_WAIT_MS, withValidatorOperationLock } from "../../../db/withValidatorOperationLock.js"; import logger from "../../../logger/index.js"; import { CustomValidatorUpdateRequest } from "./types.js"; @@ -70,7 +76,10 @@ function prepareUpdates( const update: CustomValidatorUpdateRequest = { pubkey }; if (feeRecipient !== undefined && isFeeRecipientEditable(current.tag, requestFrom)) update.feeRecipient = feeRecipient; - if (builderConfig !== undefined) update.builderConfig = builderConfig; + if (builderConfig !== undefined) { + if (!featureFlags.builderConfig) throw new FeatureDisabledError("Builder configuration"); + update.builderConfig = builderConfig; + } if (update.feeRecipient !== undefined || update.builderConfig !== undefined) { updates.push(update); seen.add(pubkey); diff --git a/packages/brain/src/modules/cron/reloadValidators/index.ts b/packages/brain/src/modules/cron/reloadValidators/index.ts index b456b5af..f474338b 100644 --- a/packages/brain/src/modules/cron/reloadValidators/index.ts +++ b/packages/brain/src/modules/cron/reloadValidators/index.ts @@ -1,3 +1,4 @@ +import { featureFlags } from "@stakingbrain/common"; import { withValidatorOperationLock } from "../../db/withValidatorOperationLock.js"; import { Web3SignerApi, ValidatorApi } from "../../apiClients/index.js"; import { BrainDataBase } from "../../db/index.js"; @@ -81,7 +82,7 @@ export async function reloadValidators( }); // Independently retry managed configuration after key imports and fee destinations. - await postValidatorsBuilderConfigsFromDb({ validatorApi, dbData: activeData }); + if (featureFlags.builderConfig) await postValidatorsBuilderConfigsFromDb({ validatorApi, dbData: activeData }); logger.debug(`${logPrefix}Finished reloading data`); } catch (e) { logger.error(`${logPrefix}Error reloading data`, e); diff --git a/packages/brain/test/unit/modules/builderConfig/builderConfig.unit.test.ts b/packages/brain/test/unit/modules/builderConfig/builderConfig.unit.test.ts index 371636dd..d1a27008 100644 --- a/packages/brain/test/unit/modules/builderConfig/builderConfig.unit.test.ts +++ b/packages/brain/test/unit/modules/builderConfig/builderConfig.unit.test.ts @@ -3,7 +3,15 @@ import sinon from "sinon"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { BuilderConfig, Network, MEV_SP_ADDRESS_MAINNET, ConsensusClient, ExecutionClient } from "@stakingbrain/common"; +import { + BuilderConfig, + Network, + MEV_SP_ADDRESS_MAINNET, + ConsensusClient, + ExecutionClient, + featureFlags, + FeatureDisabledError +} from "@stakingbrain/common"; import { ValidatorApi, Web3SignerApi, BeaconchainApi } from "../../../../src/modules/apiClients/index.js"; import { AllowedMethods } from "../../../../src/modules/apiClients/types.js"; import { HttpApiError } from "../../../../src/modules/apiClients/error.js"; @@ -13,6 +21,7 @@ import { reloadValidators } from "../../../../src/modules/cron/reloadValidators/ import { updateValidators } from "../../../../src/modules/apiServers/ui/calls/updateValidators.js"; import { getBuilderConfigs } from "../../../../src/modules/apiServers/ui/calls/getBuilderConfigs.js"; import { getValidators } from "../../../../src/modules/apiServers/ui/calls/getValidators.js"; +import { createRpcMethods } from "../../../../src/modules/apiServers/ui/calls/index.js"; import { setActiveValidatorsLoadedInBrain } from "../../../../src/modules/cron/trackValidatorsPerformance/setActiveValidatorsLoadedInBrain.js"; import { ValidatorStatus } from "../../../../src/modules/apiClients/types.js"; import logger from "../../../../src/modules/logger/index.js"; @@ -86,8 +95,11 @@ describe("Builder configuration management", () => { api.remoteKeys = [...keys]; cron = sinon.createStubInstance(CronJob); sinon.stub(logger, "error"); + // The feature is switched off in this version; this suite covers it for when it returns. + featureFlags.builderConfig = true; }); afterEach(() => { + featureFlags.builderConfig = false; sinon.restore(); fs.rmSync(directory, { recursive: true, force: true }); }); @@ -562,3 +574,69 @@ describe("Builder configuration management", () => { }); } }); + +describe("Builder configuration while disabled", () => { + let directory: string; + let db: BrainDataBase; + let api: RecordingValidator; + const config: BuilderConfig = { builders: [{ url: "https://hoodi.aestus.live" }] }; + beforeEach(() => { + expect(featureFlags.builderConfig).to.equal(false); + directory = fs.mkdtempSync(path.join(os.tmpdir(), "stakingbrain-builders-off-")); + db = new BrainDataBase(path.join(directory, "db.json")); + // A record saved while the feature was on must survive untouched. + db.addValidators({ + validators: { [keys[0]]: { tag: "solo", feeRecipient: recipient, automaticImport: false, builderConfig: config } } + }); + api = new RecordingValidator(); + api.remoteKeys = [keys[0]]; + sinon.stub(logger, "error"); + }); + afterEach(() => { + sinon.restore(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it("rejects builder updates without touching stored settings or the client", async () => { + const error = await updateValidators({ + reloadValidatorsCronTask: sinon.createStubInstance(CronJob), + brainDb: db, + validatorApi: api, + signerUrl: "http://signer", + customValidatorUpdateRequest: [{ pubkey: keys[0], builderConfig: null }] + }).catch((e: Error) => e); + expect(error).to.be.instanceOf(FeatureDisabledError); + expect(db.getData()[keys[0]].builderConfig).to.deep.equal(config); + expect(api.calls.filter(({ endpoint }) => endpoint.includes("builder_config"))).to.deep.equal([]); + }); + + it("does not replay stored settings on reload and hides them from the list", async () => { + const signer = sinon.createStubInstance(Web3SignerApi); + signer.upcheck.resolves(); + signer.listRemoteKeys.resolves({ data: [{ validating_pubkey: keys[0], derivation_path: "", readonly: false }] }); + await reloadValidators(signer, "http://signer", api, db); + expect(api.calls.filter(({ endpoint }) => endpoint.includes("builder_config"))).to.deep.equal([]); + expect(db.getData()[keys[0]].builderConfig).to.deep.equal(config); + + const beacon = sinon.createStubInstance(BeaconchainApi); + beacon.getStateValidator.rejects(new Error("not deposited")); + const [row] = await getValidators({ brainDb: db, validatorApi: api, signerApi: signer, beaconchainApi: beacon }); + expect(row).to.not.have.property("builderConfig"); + expect(row).to.not.have.property("builderConfigStatus"); + }); + + it("refuses the builder RPCs", async () => { + const methods = createRpcMethods({ + postgresClient: {} as never, + beaconchainApi: {} as never, + validatorApi: api, + signerApi: {} as never, + brainDb: db, + reloadValidatorsCronTask: {} as never, + brainConfig: { apis: {}, chain: { network: Network.Hoodi } } as never + }); + for (const call of [() => methods.getBuilderRelayPresets(), () => methods.getBuilderConfigs([keys[0]])]) + expect(await call().catch((e: Error) => e)).to.be.instanceOf(FeatureDisabledError); + expect(api.calls).to.deep.equal([]); + }); +}); diff --git a/packages/common/src/featureFlags.ts b/packages/common/src/featureFlags.ts new file mode 100644 index 00000000..cfe34132 --- /dev/null +++ b/packages/common/src/featureFlags.ts @@ -0,0 +1,16 @@ +/** + * Features shipped in code but switched off for now. Turning one back on needs no data migration: + * the Brain keeps persisted settings (e.g. each validator's `builderConfig`) while a feature is off. + * The object is mutable only so tests can exercise disabled features; application code must not write it. + */ +export const featureFlags = { + /** Gloas/ePBS per-validator builder configuration: UI, RPCs, reload re-apply and BUILDER_IDS import. */ + builderConfig: false +}; + +export class FeatureDisabledError extends Error { + constructor(feature: string) { + super(`${feature} is disabled in this version of the Staking Brain`); + this.name = "FeatureDisabledError"; + } +} diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts index 94830437..55200214 100644 --- a/packages/common/src/index.ts +++ b/packages/common/src/index.ts @@ -4,3 +4,4 @@ export * from "./params.js"; export * from "./builderConfig.js"; export * from "./builderRelayPresets.js"; export * from "./rpc.js"; +export * from "./featureFlags.js"; diff --git a/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx b/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx index 9573d033..4f1d3aec 100644 --- a/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx +++ b/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx @@ -6,7 +6,8 @@ import { MAINNET_ORACLE_URL, TESTNET_ORACLE_URL, nonEditableFeeRecipientTags, - NonEditableFeeRecipientTag + NonEditableFeeRecipientTag, + featureFlags } from "@stakingbrain/common"; import SmoothStatus from "./SmoothStatus"; import { GridColDef } from "@mui/x-data-grid"; @@ -269,24 +270,24 @@ export default function KeystoresDataGrid({ id: index })); + const builderConfigColumn: GridColDef = { + field: "builderConfigStatus", + headerName: "Builder config", + description: "Last application attempt; open Builder config to inspect client values and refresh status", + width: 180, + sortable: false, + renderCell: ({ row }) => ( + + + {capitalize(row.builderConfigStatus?.status ?? (row.builderConfig === undefined ? "Unmanaged" : "Pending"))} + + + ) + }; + if (userMode === "advanced") columns.push( - { - field: "builderConfigStatus", - headerName: "Builder config", - description: "Last application attempt; open Builder config to inspect client values and refresh status", - width: 180, - sortable: false, - renderCell: ({ row }) => ( - - - {capitalize( - row.builderConfigStatus?.status ?? (row.builderConfig === undefined ? "Unmanaged" : "Pending") - )} - - - ) - }, + ...(featureFlags.builderConfig ? [builderConfigColumn] : []), { field: "explorerUrl", headerName: "URL", @@ -488,14 +489,16 @@ export default function KeystoresDataGrid({ - + {featureFlags.builderConfig && ( + + )} setDeleteOpen(true)}> diff --git a/packages/ui/src/components/ValidatorList/ValidatorList.tsx b/packages/ui/src/components/ValidatorList/ValidatorList.tsx index d880b7b2..2f5e654d 100644 --- a/packages/ui/src/components/ValidatorList/ValidatorList.tsx +++ b/packages/ui/src/components/ValidatorList/ValidatorList.tsx @@ -5,7 +5,7 @@ import EditFeesDialog from "../Dialogs/EditFeesDialog"; import BuilderConfigDialog from "../Dialogs/BuilderConfigDialog"; import { Alert, Box, Button, CircularProgress, LinearProgress, Typography } from "@mui/material"; import type { GridRowId } from "@mui/x-data-grid"; -import { StakerConfig as StakerConfigType, VALIDATORS_CHANGED_EVENT } from "@stakingbrain/common"; +import { featureFlags, StakerConfig as StakerConfigType, VALIDATORS_CHANGED_EVENT } from "@stakingbrain/common"; import { useEffect, useRef, useState } from "react"; import { BeaconchaUrlBuildingStatus } from "../../types"; import { rpcClient } from "../../socket"; @@ -247,7 +247,7 @@ export default function ValidatorList({ /> )} - {builderConfigOpen && ( + {featureFlags.builderConfig && builderConfigOpen && ( validatorsGet[row as number])} From 904921faa162ed88750962e5299b6f02ac1d175e Mon Sep 17 00:00:00 2001 From: 3alpha <15694175+3alpha@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:12:04 +0100 Subject: [PATCH 6/8] test: add end-to-end tests for large validator batches Run the real UI server, RPC handler, database, lock, reload and API clients against fake web3signer, validator client and beacon node HTTP servers, and drive them over Socket.IO with the UI's own RpcClient and batch import code. Covers batched import of 120 keys, a failed batch with retry, per-key signer rejections, live validatorsChanged pushes, progressive list loading past the 10s deadline, and builder configuration staying disabled. yarn test now runs them in CI. --- packages/brain/package.json | 4 +- .../modules/apiServers/ui/startUiServer.ts | 9 +- packages/brain/test/e2e/fakeClients.ts | 177 +++++++++++ .../brain/test/e2e/largeBatches.e2e.test.ts | 275 ++++++++++++++++++ packages/brain/tsconfig.test.json | 3 +- packages/ui/src/utils/validatorRefresh.ts | 3 +- yarn.lock | 1 + 7 files changed, 466 insertions(+), 6 deletions(-) create mode 100644 packages/brain/test/e2e/fakeClients.ts create mode 100644 packages/brain/test/e2e/largeBatches.e2e.test.ts diff --git a/packages/brain/package.json b/packages/brain/package.json index a135c294..20c19d45 100644 --- a/packages/brain/package.json +++ b/packages/brain/package.json @@ -6,8 +6,9 @@ "main": "dist/index.js", "types": "dist/types.d.ts", "scripts": { - "test": "yarn run test:unit", + "test": "yarn run test:unit && yarn run test:e2e", "test:unit": "mocha --exit --recursive 'test/**/*.unit.test.ts'", + "test:e2e": "mocha --exit --recursive 'test/**/*.e2e.test.ts'", "dev": "yarn run link:ui && tsx --watch src/index.ts", "link:ui": "ln -s ../ui/build/ ./uiBuild", "build": "yarn run clean:dist && yarn run clean:ln && tsc -p tsconfig.json", @@ -33,6 +34,7 @@ "mocha": "^12.0.3", "rimraf": "^6.1.3", "sinon": "^22.1.0", + "socket.io-client": "^4.8.4", "typescript": "~6.0.3" } } diff --git a/packages/brain/src/modules/apiServers/ui/startUiServer.ts b/packages/brain/src/modules/apiServers/ui/startUiServer.ts index 6f4da279..186f5015 100644 --- a/packages/brain/src/modules/apiServers/ui/startUiServer.ts +++ b/packages/brain/src/modules/apiServers/ui/startUiServer.ts @@ -26,7 +26,8 @@ export function startUiServer({ uiBuildPath, brainConfig, reloadValidatorsCronTask, - allowedOriginsFromEnv + allowedOriginsFromEnv, + port = params.uiPort }: { brainDb: BrainDataBase; beaconchainApi: BeaconchainApi; @@ -37,6 +38,8 @@ export function startUiServer({ brainConfig: BrainConfig; reloadValidatorsCronTask: CronJob; allowedOriginsFromEnv: string | string[] | null; + /** Defaults to the production UI port; tests pass 0 for an ephemeral port. */ + port?: number; }): http.Server { const { network } = brainConfig.chain; // create index.html modified with network @@ -100,8 +103,8 @@ export function startUiServer({ res.sendFile(path.join(uiBuildPath, "index.html")); }); - server.listen(params.uiPort, () => { - logger.info(`Server listening on *:${params.uiPort}`); + server.listen(port, () => { + logger.info(`Server listening on *:${port}`); }); return server; diff --git a/packages/brain/test/e2e/fakeClients.ts b/packages/brain/test/e2e/fakeClients.ts new file mode 100644 index 00000000..588cabdf --- /dev/null +++ b/packages/brain/test/e2e/fakeClients.ts @@ -0,0 +1,177 @@ +import express from "express"; +import http from "node:http"; +import type { AddressInfo } from "node:net"; + +/** Minimal in-memory stand-ins for web3signer, a validator client and a beacon node, served over real HTTP. */ + +type Started = { url: string; close: () => Promise }; + +async function serve(app: express.Express): Promise { + const server = http.createServer(app); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + return { + url: `http://127.0.0.1:${port}`, + close: () => + new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }) + }; +} + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +const prefixed = (pubkey: string) => (pubkey.startsWith("0x") ? pubkey : `0x${pubkey}`).toLowerCase(); + +export class FakeSigner { + keys = new Set(); + importRequests: { pubkeys: string[]; slashingPubkeys?: string[] }[] = []; + /** Simulated keystore decryption time. */ + importDelayPerKeyMs = 0; + /** 1-based import call number that answers HTTP 500, as a crashed or overloaded signer would. */ + failImportCall?: number; + + async start(): Promise { + const app = express(); + app.use(express.json({ limit: "64mb" })); + app.get("/upcheck", (_req, res) => { + res.type("text/plain").send("OK"); + }); + app.get("/eth/v1/keystores", (_req, res) => { + res.json({ + data: [...this.keys].map((validating_pubkey) => ({ validating_pubkey, derivation_path: "", readonly: false })) + }); + }); + app.post("/eth/v1/keystores", async (req, res) => { + const { keystores, passwords, slashing_protection } = req.body as { + keystores: string[]; + passwords: string[]; + slashing_protection?: string; + }; + const pubkeys = keystores.map((keystore) => prefixed(JSON.parse(keystore).pubkey)); + this.importRequests.push({ + pubkeys, + slashingPubkeys: slashing_protection + ? JSON.parse(slashing_protection).data.map(({ pubkey }: { pubkey: string }) => prefixed(pubkey)) + : undefined + }); + await delay(this.importDelayPerKeyMs * keystores.length); + if (this.importRequests.length === this.failImportCall) { + res.status(500).json({ message: "Internal signer error" }); + return; + } + res.json({ + data: pubkeys.map((pubkey, index) => { + if (passwords[index] !== "password") return { status: "error", message: "Invalid password" }; + if (this.keys.has(pubkey)) return { status: "duplicate", message: "" }; + this.keys.add(pubkey); + return { status: "imported", message: "" }; + }) + }); + }); + app.delete("/eth/v1/keystores", (req, res) => { + const pubkeys = (req.body.pubkeys as string[]).map(prefixed); + res.json({ + data: pubkeys.map((pubkey) => ({ status: this.keys.delete(pubkey) ? "deleted" : "not_found", message: "" })), + slashing_protection: JSON.stringify({ metadata: {}, data: [] }) + }); + }); + return serve(app); + } +} + +export class FakeValidatorClient { + remoteKeys = new Map(); + feeRecipients = new Map(); + /** Every request path, to assert what the Brain did (and did not) call. */ + paths: string[] = []; + + async start(): Promise { + const app = express(); + app.use(express.json({ limit: "64mb" })); + app.use((req, _res, next) => { + this.paths.push(`${req.method} ${req.path}`); + next(); + }); + app.get("/eth/v1/remotekeys", (_req, res) => { + res.json({ data: [...this.remoteKeys].map(([pubkey, url]) => ({ pubkey, url, readonly: false })) }); + }); + app.post("/eth/v1/remotekeys", (req, res) => { + const remoteKeys = req.body.remote_keys as { pubkey: string; url: string }[]; + res.json({ + data: remoteKeys.map(({ pubkey, url }) => { + const key = prefixed(pubkey); + const status = this.remoteKeys.has(key) ? "DUPLICATE" : "IMPORTED"; + this.remoteKeys.set(key, url); + return { status, message: "" }; + }) + }); + }); + app.delete("/eth/v1/remotekeys", (req, res) => { + res.json({ + data: (req.body.pubkeys as string[]).map((pubkey) => ({ + status: this.remoteKeys.delete(prefixed(pubkey)) ? "DELETED" : "NOT_FOUND", + message: "" + })) + }); + }); + app.get("/eth/v1/validator/:pubkey/feerecipient", (req, res) => { + const ethaddress = this.feeRecipients.get(prefixed(req.params.pubkey)); + if (!ethaddress) res.status(404).json({ message: "Validator not found" }); + else res.json({ data: { pubkey: req.params.pubkey, ethaddress } }); + }); + app.post("/eth/v1/validator/:pubkey/feerecipient", (req, res) => { + this.feeRecipients.set(prefixed(req.params.pubkey), req.body.ethaddress); + res.status(202).end(); + }); + app.delete("/eth/v1/validator/:pubkey/feerecipient", (req, res) => { + this.feeRecipients.delete(prefixed(req.params.pubkey)); + res.status(204).end(); + }); + return serve(app); + } +} + +export class FakeBeaconNode { + indexes = new Map(); + /** Simulated per-validator lookup latency of a busy or syncing beacon node. */ + lookupDelayMs = 0; + lookups = 0; + + async start(): Promise { + const app = express(); + app.get("/eth/v1/beacon/states/head/validators/:pubkey", async (req, res) => { + this.lookups++; + // Stop early when the Brain aborts the request at its deadline. + let closed = false; + req.on("close", () => (closed = true)); + await delay(this.lookupDelayMs); + if (closed) return; + const pubkey = prefixed(req.params.pubkey); + const index = this.indexes.get(pubkey); + if (index === undefined) { + res.status(404).json({ message: "Validator not found" }); + return; + } + res.json({ + execution_optimistic: false, + data: { + index: String(index), + balance: "32000000000", + status: "active_ongoing", + validator: { + pubkey, + withdrawal_credentials: "0x01" + "00".repeat(11) + "ab".repeat(20), + effective_balance: "32000000000", + slashed: false, + activation_eligibility_epoch: "0", + activation_epoch: "0", + exit_epoch: "18446744073709551615", + withdrawable_epoch: "18446744073709551615" + } + } + }); + }); + return serve(app); + } +} diff --git a/packages/brain/test/e2e/largeBatches.e2e.test.ts b/packages/brain/test/e2e/largeBatches.e2e.test.ts new file mode 100644 index 00000000..296025c3 --- /dev/null +++ b/packages/brain/test/e2e/largeBatches.e2e.test.ts @@ -0,0 +1,275 @@ +import { expect } from "chai"; +import sinon from "sinon"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { once } from "node:events"; +import type http from "node:http"; +import type { AddressInfo } from "node:net"; +import { io, Socket } from "socket.io-client"; +import { Network, VALIDATORS_CHANGED_EVENT } from "@stakingbrain/common"; +import { BeaconchainApi, ValidatorApi, Web3SignerApi } from "../../src/modules/apiClients/index.js"; +import { BrainDataBase } from "../../src/modules/db/index.js"; +import { CronJob } from "../../src/modules/cron/cron.js"; +import { reloadValidators } from "../../src/modules/cron/reloadValidators/index.js"; +import { startUiServer } from "../../src/modules/apiServers/ui/startUiServer.js"; +import type { BrainConfig } from "../../src/modules/config/types.js"; +import logger from "../../src/modules/logger/index.js"; +// The UI's own client code, so the test drives the Brain exactly as the browser does. +import { RpcClient } from "../../../ui/src/socket/rpcClient.js"; +import { + notAttempted, + planImportBatches, + retryableIndexes, + runImportBatches, + summarizeImport +} from "../../../ui/src/utils/importBatches.js"; +import { countPending, MAX_STALLED_REFRESHES, nextStallCount } from "../../../ui/src/utils/validatorRefresh.js"; +import type { CustomImportRequest } from "../../src/modules/apiServers/ui/calls/types.js"; +import { FakeBeaconNode, FakeSigner, FakeValidatorClient } from "./fakeClients.js"; + +const feeRecipient = "0x" + "ab".repeat(20); +const pubkeyAt = (index: number) => "0x" + (index + 1).toString(16).padStart(96, "0"); + +function importRequest(pubkeys: string[], password = "password"): CustomImportRequest { + return { + importFrom: "ui", + // One interchange covering every key, as users export it from their previous setup. + slashing_protection: JSON.stringify({ + metadata: { interchange_format_version: "5", genesis_validators_root: "0x" + "00".repeat(32) }, + data: pubkeys.map((pubkey) => ({ pubkey, signed_blocks: [], signed_attestations: [] })) + }), + validatorsImportRequest: pubkeys.map((pubkey) => ({ + keystore: JSON.stringify({ version: 4, pubkey: pubkey.slice(2), crypto: { kdf: { function: "scrypt" } } }), + password, + tag: "solo", + feeRecipient + })) + }; +} + +async function waitFor(condition: () => boolean, timeoutMs = 5000): Promise { + const deadline = Date.now() + timeoutMs; + while (!condition()) { + if (Date.now() > deadline) throw new Error("Timed out waiting for condition"); + await new Promise((resolve) => setTimeout(resolve, 20)); + } +} + +describe("E2E: large validator batches through the UI socket", function () { + this.timeout(60_000); + + let directory: string; + let signer: FakeSigner; + let validator: FakeValidatorClient; + let beacon: FakeBeaconNode; + let closeFakes: (() => Promise)[]; + let signerApi: Web3SignerApi; + let validatorApi: ValidatorApi; + let signerUrl: string; + let brainDb: BrainDataBase; + let cron: CronJob; + let server: http.Server; + let sockets: Socket[]; + let client: RpcClient; + + const connect = async (): Promise<{ socket: Socket; client: RpcClient }> => { + const { port } = server.address() as AddressInfo; + const socket = io(`http://127.0.0.1:${port}`, { transports: ["websocket"], reconnection: false }); + sockets.push(socket); + const rpc = new RpcClient(socket); + if (!socket.connected) await new Promise((resolve) => socket.once("connect", () => resolve())); + return { socket, client: rpc }; + }; + + const importAll = ( + request: CustomImportRequest, + pubkeys: string[], + only?: number[], + earlier?: { status: string }[] + ) => + runImportBatches({ + batches: planImportBatches(request, pubkeys, { only }), + results: earlier ?? pubkeys.map(notAttempted), + importBatch: (batch) => client.call("importValidators", batch), + onProgress: () => undefined, + shouldStop: () => false + }); + + beforeEach(async () => { + for (const level of ["debug", "info", "warn", "error"] as const) sinon.stub(logger, level); + sinon.stub(console, "log"); + directory = fs.mkdtempSync(path.join(os.tmpdir(), "stakingbrain-e2e-")); + signer = new FakeSigner(); + validator = new FakeValidatorClient(); + beacon = new FakeBeaconNode(); + const [signerServer, validatorServer, beaconServer] = await Promise.all([ + signer.start(), + validator.start(), + beacon.start() + ]); + closeFakes = [signerServer.close, validatorServer.close, beaconServer.close]; + signerUrl = signerServer.url; + signerApi = new Web3SignerApi({ baseUrl: signerServer.url, authToken: "token" }, Network.Hoodi); + validatorApi = new ValidatorApi({ baseUrl: validatorServer.url, authToken: "token" }, Network.Hoodi); + const beaconchainApi = new BeaconchainApi({ baseUrl: beaconServer.url }, Network.Hoodi); + + brainDb = new BrainDataBase(path.join(directory, "brain-db.json")); + await brainDb.initialize(signerApi, validatorApi); + cron = new CronJob(60_000, () => reloadValidators(signerApi, signerUrl, validatorApi, brainDb)); + cron.start(); + + const uiBuildPath = path.join(directory, "ui"); + fs.mkdirSync(uiBuildPath); + fs.writeFileSync(path.join(uiBuildPath, "index.html"), ""); + server = startUiServer({ + brainDb, + beaconchainApi, + validatorApi, + signerApi, + postgresClient: {} as never, + uiBuildPath, + brainConfig: { + chain: { network: Network.Hoodi, minGenesisTime: 0, secondsPerSlot: 12, isMevBoostSet: false }, + apis: { signerUrl, validatorUrl: validatorServer.url, beaconchainUrl: beaconServer.url } + } as unknown as BrainConfig, + reloadValidatorsCronTask: cron, + allowedOriginsFromEnv: null, + port: 0 + }); + if (!server.listening) await once(server, "listening"); + sockets = []; + ({ client } = await connect()); + }); + + afterEach(async () => { + for (const socket of sockets) socket.disconnect(); + cron.stop(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(resolve)); + await Promise.all(closeFakes.map((close) => close())); + sinon.restore(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it("imports 120 keystores in batches and configures every client", async () => { + const pubkeys = Array.from({ length: 120 }, (_, index) => pubkeyAt(index)); + signer.importDelayPerKeyMs = 5; + const { results, stopped } = await importAll(importRequest(pubkeys), pubkeys); + + expect(stopped).to.equal(undefined); + expect(summarizeImport(results)).to.deep.equal({ imported: 120 }); + // Six bounded calls, each carrying only its own keys' slashing history. + expect(signer.importRequests.map(({ pubkeys }) => pubkeys.length)).to.deep.equal([20, 20, 20, 20, 20, 20]); + for (const { pubkeys: batch, slashingPubkeys } of signer.importRequests) + expect(slashingPubkeys).to.deep.equal(batch); + expect(signer.keys.size).to.equal(120); + expect(validator.remoteKeys.size).to.equal(120); + expect([...validator.remoteKeys.values()].every((url) => url === signerUrl)).to.equal(true); + expect(validator.feeRecipients.size).to.equal(120); + expect([...validator.feeRecipients.values()].every((address) => address === feeRecipient)).to.equal(true); + const records = Object.values(brainDb.getData()); + expect(records).to.have.length(120); + expect(records.some(({ importPending }) => importPending)).to.equal(false); + + // The reload job agrees with the result and removes nothing. + await reloadValidators(signerApi, signerUrl, validatorApi, brainDb); + expect(Object.keys(brainDb.getData())).to.have.length(120); + expect(validator.remoteKeys.size).to.equal(120); + }); + + it("stops at a failed batch, keeps the journal, and a retry finishes the import", async () => { + const pubkeys = Array.from({ length: 100 }, (_, index) => pubkeyAt(index)); + signer.failImportCall = 3; + const first = await importAll(importRequest(pubkeys), pubkeys); + + expect(first.stopped).to.equal("error"); + expect(summarizeImport(first.results)).to.deep.equal({ imported: 40, error: 20, not_attempted: 40 }); + expect(first.results[45].message).to.include("500"); + // The failed batch stays journaled as pending, and reload must not delete those records. + await reloadValidators(signerApi, signerUrl, validatorApi, brainDb); + const pending = Object.values(brainDb.getData()).filter(({ importPending }) => importPending); + expect(pending).to.have.length(20); + + const retry = retryableIndexes(first.results); + expect(retry).to.have.length(60); + const second = await importAll(importRequest(pubkeys), pubkeys, retry, first.results); + expect(second.stopped).to.equal(undefined); + expect(summarizeImport(second.results)).to.deep.equal({ imported: 100 }); + expect(signer.keys.size).to.equal(100); + expect(validator.remoteKeys.size).to.equal(100); + expect(validator.feeRecipients.size).to.equal(100); + expect(Object.values(brainDb.getData()).some(({ importPending }) => importPending)).to.equal(false); + }); + + it("reports per-key signer rejections without failing the rest of the batch", async () => { + const pubkeys = Array.from({ length: 30 }, (_, index) => pubkeyAt(index)); + const request = importRequest(pubkeys); + request.validatorsImportRequest[25].password = "wrong"; + const { results, stopped } = await importAll(request, pubkeys); + expect(stopped).to.equal(undefined); + expect(summarizeImport(results)).to.deep.equal({ imported: 29, error: 1 }); + expect(results[25].message).to.include("password"); + expect(brainDb.getData()[pubkeys[25]]).to.equal(undefined); + expect(retryableIndexes(results)).to.deep.equal([25]); + }); + + it("pushes validatorsChanged to other open UIs on import and delete", async () => { + const watcher = await connect(); + let events = 0; + watcher.client.on(VALIDATORS_CHANGED_EVENT, () => events++); + const pubkeys = Array.from({ length: 45 }, (_, index) => pubkeyAt(index)); + await importAll(importRequest(pubkeys), pubkeys); + await waitFor(() => events > 0); + // Many database writes are coalesced into few hints rather than one per write. + expect(events).to.be.lessThan(6); + expect(await watcher.client.call("getValidators", undefined)).to.have.length(45); + + const before = events; + await client.call("deleteValidators", { pubkeys: pubkeys.slice(0, 5) }); + await waitFor(() => events > before); + expect(await watcher.client.call("getValidators", undefined)).to.have.length(40); + expect(validator.remoteKeys.size).to.equal(40); + }); + + it("loads a slow list progressively until every row is complete", async () => { + const pubkeys = Array.from({ length: 70 }, (_, index) => pubkeyAt(index)); + await importAll(importRequest(pubkeys), pubkeys); + pubkeys.forEach((pubkey, index) => beacon.indexes.set(pubkey, 1000 + index)); + // 8 parallel lookups of 1.5s cannot cover 70 validators inside the 10s list deadline. + beacon.lookupDelayMs = 1500; + + // Refresh the way the validator list does until nothing is pending or progress stalls. + const pendingByRound: number[] = []; + let stalls = 0; + let rows = await client.call("getValidators", undefined); + pendingByRound.push(countPending(rows)); + while (countPending(rows) > 0 && stalls < MAX_STALLED_REFRESHES) { + rows = await client.call("getValidators", undefined); + stalls = nextStallCount(stalls, pendingByRound.at(-1), countPending(rows)); + pendingByRound.push(countPending(rows)); + } + + expect(pendingByRound[0]).to.be.greaterThan(0).and.lessThan(70); + expect(pendingByRound.at(-1)).to.equal(0); + expect(pendingByRound.length).to.be.at.most(3); + expect(rows.map(({ index }) => index)).to.deep.equal(pubkeys.map((_, index) => String(1000 + index))); + expect(rows.every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect === true)).to.equal(true); + expect(rows.every(({ signerImported, validatorImported }) => signerImported && validatorImported)).to.equal(true); + }); + + it("keeps builder configuration switched off end to end", async () => { + const pubkeys = [pubkeyAt(0)]; + await importAll(importRequest(pubkeys), pubkeys); + const error = await client.call("getBuilderRelayPresets", undefined).catch((e: Error) => e); + expect(error).to.be.instanceOf(Error).and.have.property("message").that.includes("disabled"); + const update = await client + .call("updateValidators", [{ pubkey: pubkeys[0], builderConfig: null }]) + .catch((e: Error) => e); + expect(update).to.be.instanceOf(Error).and.have.property("message").that.includes("disabled"); + await reloadValidators(signerApi, signerUrl, validatorApi, brainDb); + expect(validator.paths.filter((p) => p.includes("builder_config"))).to.deep.equal([]); + const [row] = await client.call("getValidators", undefined); + expect(row).to.not.have.property("builderConfig"); + }); +}); diff --git a/packages/brain/tsconfig.test.json b/packages/brain/tsconfig.test.json index d992a247..fee679a2 100644 --- a/packages/brain/tsconfig.test.json +++ b/packages/brain/tsconfig.test.json @@ -1,7 +1,8 @@ { "extends": "../../tsconfig.json", "compilerOptions": { - "rootDir": "." // Root directory of input files + // The e2e tests import the UI's client modules to drive the Brain exactly as the browser does. + "rootDir": ".." }, "include": ["test"], "exclude": ["node_modules", "dist"] diff --git a/packages/ui/src/utils/validatorRefresh.ts b/packages/ui/src/utils/validatorRefresh.ts index b813a4c2..67f15686 100644 --- a/packages/ui/src/utils/validatorRefresh.ts +++ b/packages/ui/src/utils/validatorRefresh.ts @@ -1,4 +1,5 @@ -import type { GridRowId } from "@mui/x-data-grid"; +/** Grid row ids (positions here). Kept free of UI-library types so the Brain e2e tests can import this module. */ +type GridRowId = string | number; /** Delay between refreshes while the Brain reports rows whose checks were cut short. */ export const PENDING_REFRESH_MS = 3000; diff --git a/yarn.lock b/yarn.lock index afe3f120..bbbb3493 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1052,6 +1052,7 @@ __metadata: rimraf: "npm:^6.1.3" sinon: "npm:^22.1.0" socket.io: "npm:^4.8.4" + socket.io-client: "npm:^4.8.4" typescript: "npm:~6.0.3" languageName: unknown linkType: soft From fea83ea3ffefe1693e3ff273c9081a36f68fae39 Mon Sep 17 00:00:00 2001 From: 3alpha <15694175+3alpha@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:29:42 +0100 Subject: [PATCH 7/8] fix: address review findings on large-batch changes - Capture the previous rows before scheduling the selection update, so a live refresh cannot move the selection onto a different validator. - Size import batches by exact encoded bytes, including each key's slashing history; report a key too large for any request instead of stopping. - Hold change notifications while a validator operation runs and send one when it finishes, so other UIs refresh after clients are configured. Reload also announces validator-client membership repairs. --- .../modules/cron/reloadValidators/index.ts | 17 +-- packages/brain/src/modules/db/index.ts | 30 +++++ .../modules/db/withValidatorOperationLock.ts | 3 + packages/brain/test/e2e/fakeClients.ts | 5 +- .../brain/test/e2e/largeBatches.e2e.test.ts | 21 +++ .../unit/modules/db/onChange.unit.test.ts | 29 +++++ .../withValidatorOperationLock.unit.test.ts | 6 +- .../ValidatorList/ValidatorList.tsx | 5 +- packages/ui/src/utils/importBatches.ts | 123 +++++++++++++----- packages/ui/src/utils/validatorRefresh.ts | 11 ++ packages/ui/test/importBatches.test.ts | 61 +++++++++ packages/ui/test/validatorRefresh.test.ts | 13 +- 12 files changed, 278 insertions(+), 46 deletions(-) diff --git a/packages/brain/src/modules/cron/reloadValidators/index.ts b/packages/brain/src/modules/cron/reloadValidators/index.ts index f474338b..9f00086b 100644 --- a/packages/brain/src/modules/cron/reloadValidators/index.ts +++ b/packages/brain/src/modules/cron/reloadValidators/index.ts @@ -61,19 +61,14 @@ export async function reloadValidators( const dbKeys = new Set(dbPubkeys.map((pubkey) => pubkey.toLowerCase())); // 4. POST to validator API pubkeys that are in DB and not in validator API - await postValidatorPubkeysFromDb({ - validatorApi, - signerUrl, - brainDbPubkeysToAdd: Object.keys(activeData).filter((pubkey) => !validatorKeys.has(pubkey.toLowerCase())), - validatorPubkeys - }); + const brainDbPubkeysToAdd = Object.keys(activeData).filter((pubkey) => !validatorKeys.has(pubkey.toLowerCase())); + await postValidatorPubkeysFromDb({ validatorApi, signerUrl, brainDbPubkeysToAdd, validatorPubkeys }); // 5. DELETE to validator API pubkeys that are in validator API and not in DB - await deleteValidatorPubkeysNotInDB({ - validatorApi, - validatorPubkeys, - validatorPubkeysToRemove: validatorPubkeys.filter((pubkey) => !dbKeys.has(pubkey.toLowerCase())) - }); + const validatorPubkeysToRemove = validatorPubkeys.filter((pubkey) => !dbKeys.has(pubkey.toLowerCase())); + await deleteValidatorPubkeysNotInDB({ validatorApi, validatorPubkeys, validatorPubkeysToRemove }); + // Client membership is shown in the UI; repairs here change it without touching the database. + if (brainDbPubkeysToAdd.length > 0 || validatorPubkeysToRemove.length > 0) brainDb.notifyChanged(); // 6. POST to validator API fee recipients that are in DB in validator API await postValidatorsFeeRecipientsFromDb({ diff --git a/packages/brain/src/modules/db/index.ts b/packages/brain/src/modules/db/index.ts index 7bb2fcef..c3ed5120 100644 --- a/packages/brain/src/modules/db/index.ts +++ b/packages/brain/src/modules/db/index.ts @@ -34,12 +34,23 @@ export class BrainDataBase extends LowSync { } private changeListeners = new Set<() => void>(); + private notificationHolds = 0; + private changedWhileHeld = false; public override write(): void { super.write(); // The JSON adapter replaces the file atomically; restore the private mode on // the replacement too, since records may contain builder authorization data. this.setOwnerWriteRead(); + this.notifyChanged(); + } + + /** Announce a validator change, including client-side changes that involve no database write. */ + public notifyChanged(): void { + if (this.notificationHolds > 0) { + this.changedWhileHeld = true; + return; + } for (const listener of this.changeListeners) { try { listener(); @@ -49,6 +60,25 @@ export class BrainDataBase extends LowSync { } } + /** + * Hold change notifications until the returned release runs, then send at most one. An operation + * writes the database before configuring clients, so notifying mid-operation would let UIs read + * client state that is about to change and never hear about the final result. + */ + public holdChangeNotifications(): () => void { + this.notificationHolds++; + let released = false; + return () => { + if (released) return; + released = true; + this.notificationHolds--; + if (this.notificationHolds === 0 && this.changedWhileHeld) { + this.changedWhileHeld = false; + this.notifyChanged(); + } + }; + } + /** Notify after each persisted change, e.g. to push live updates to open UIs. Returns an unsubscribe. */ public onChange(listener: () => void): () => void { this.changeListeners.add(listener); diff --git a/packages/brain/src/modules/db/withValidatorOperationLock.ts b/packages/brain/src/modules/db/withValidatorOperationLock.ts index 1b195156..79fda0ad 100644 --- a/packages/brain/src/modules/db/withValidatorOperationLock.ts +++ b/packages/brain/src/modules/db/withValidatorOperationLock.ts @@ -47,9 +47,12 @@ export async function withValidatorOperationLock( } } else await previous; + // Announce the operation's changes once it has finished configuring clients, not per write. + const releaseNotifications = brainDb.holdChangeNotifications(); try { return await operation(); } finally { + releaseNotifications(); cleanup(); } } diff --git a/packages/brain/test/e2e/fakeClients.ts b/packages/brain/test/e2e/fakeClients.ts index 588cabdf..6a49ae6d 100644 --- a/packages/brain/test/e2e/fakeClients.ts +++ b/packages/brain/test/e2e/fakeClients.ts @@ -85,12 +85,15 @@ export class FakeValidatorClient { feeRecipients = new Map(); /** Every request path, to assert what the Brain did (and did not) call. */ paths: string[] = []; + /** Simulated latency of a busy validator client on every write. */ + writeDelayMs = 0; async start(): Promise { const app = express(); app.use(express.json({ limit: "64mb" })); - app.use((req, _res, next) => { + app.use(async (req, _res, next) => { this.paths.push(`${req.method} ${req.path}`); + if (req.method !== "GET") await delay(this.writeDelayMs); next(); }); app.get("/eth/v1/remotekeys", (_req, res) => { diff --git a/packages/brain/test/e2e/largeBatches.e2e.test.ts b/packages/brain/test/e2e/largeBatches.e2e.test.ts index 296025c3..6797d682 100644 --- a/packages/brain/test/e2e/largeBatches.e2e.test.ts +++ b/packages/brain/test/e2e/largeBatches.e2e.test.ts @@ -232,6 +232,27 @@ describe("E2E: large validator batches through the UI socket", function () { expect(validator.remoteKeys.size).to.equal(40); }); + it("pushes a final update after slow validator-client writes so other UIs end up current", async () => { + validator.writeDelayMs = 1500; + const watcher = await connect(); + // Behave like the validator list: refresh on every push and keep the latest rows. + let latest: Awaited>> = []; + let refreshes = Promise.resolve(); + watcher.client.on(VALIDATORS_CHANGED_EVENT, () => { + refreshes = refreshes.then(async () => { + latest = await watcher.client.call("getValidators", undefined); + }); + }); + const pubkeys = Array.from({ length: 10 }, (_, index) => pubkeyAt(index)); + await importAll(importRequest(pubkeys), pubkeys); + // Let the last push and its refresh land. + await new Promise((resolve) => setTimeout(resolve, 1000)); + await refreshes; + expect(latest).to.have.length(10); + expect(latest.every(({ validatorImported }) => validatorImported === true)).to.equal(true); + expect(latest.every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect === true)).to.equal(true); + }); + it("loads a slow list progressively until every row is complete", async () => { const pubkeys = Array.from({ length: 70 }, (_, index) => pubkeyAt(index)); await importAll(importRequest(pubkeys), pubkeys); diff --git a/packages/brain/test/unit/modules/db/onChange.unit.test.ts b/packages/brain/test/unit/modules/db/onChange.unit.test.ts index 573a4c12..fb763419 100644 --- a/packages/brain/test/unit/modules/db/onChange.unit.test.ts +++ b/packages/brain/test/unit/modules/db/onChange.unit.test.ts @@ -5,6 +5,7 @@ import os from "node:os"; import path from "node:path"; import { BrainDataBase } from "../../../../src/modules/db/index.js"; import logger from "../../../../src/modules/logger/index.js"; +import { withValidatorOperationLock } from "../../../../src/modules/db/withValidatorOperationLock.js"; describe("Database change notifications", () => { let directory: string; @@ -36,6 +37,34 @@ describe("Database change notifications", () => { expect(listener.callCount).to.equal(1); }); + it("sends one notification after a held operation, and none if nothing changed", () => { + const listener = sinon.spy(); + brainDb.onChange(listener); + const release = brainDb.holdChangeNotifications(); + const nested = brainDb.holdChangeNotifications(); + brainDb.addValidators({ validators: { [pubkey]: { ...record } } }); + brainDb.updateValidators([{ pubkey, feeRecipient: "0x" + "cd".repeat(20) }]); + nested(); + expect(listener.callCount).to.equal(0); + release(); + release(); + expect(listener.callCount).to.equal(1); + brainDb.holdChangeNotifications()(); + expect(listener.callCount).to.equal(1); + }); + + it("defers notifications for the whole validator operation", async () => { + const listener = sinon.spy(); + brainDb.onChange(listener); + await withValidatorOperationLock(brainDb, async () => { + brainDb.addValidators({ validators: { [pubkey]: { ...record } } }); + await new Promise((resolve) => setTimeout(resolve, 5)); + // Clients are configured after the write; nobody should have been told yet. + expect(listener.callCount).to.equal(0); + }); + expect(listener.callCount).to.equal(1); + }); + it("keeps writing when a listener throws", () => { brainDb.onChange(() => { throw new Error("socket gone"); diff --git a/packages/brain/test/unit/modules/db/withValidatorOperationLock.unit.test.ts b/packages/brain/test/unit/modules/db/withValidatorOperationLock.unit.test.ts index ed1d32d1..1aa60804 100644 --- a/packages/brain/test/unit/modules/db/withValidatorOperationLock.unit.test.ts +++ b/packages/brain/test/unit/modules/db/withValidatorOperationLock.unit.test.ts @@ -6,12 +6,14 @@ import { withValidatorOperationLock } from "../../../../src/modules/db/withValidatorOperationLock.js"; +const fakeDb = () => ({ holdChangeNotifications: () => () => undefined }) as unknown as BrainDataBase; + describe("Validator operation lock", () => { afterEach(() => sinon.restore()); it("keeps later operations behind the running one after a waiter is abandoned", async () => { const clock = sinon.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); - const brainDb = {} as BrainDataBase; + const brainDb = fakeDb(); const events: string[] = []; let finishFirst!: () => void; @@ -36,7 +38,7 @@ describe("Validator operation lock", () => { }); it("runs a bounded operation that acquires the lock in time", async () => { - const brainDb = {} as BrainDataBase; + const brainDb = fakeDb(); expect(await withValidatorOperationLock(brainDb, async () => "done", { maxWaitMs: 1000 })).to.equal("done"); }); }); diff --git a/packages/ui/src/components/ValidatorList/ValidatorList.tsx b/packages/ui/src/components/ValidatorList/ValidatorList.tsx index 2f5e654d..7390de8c 100644 --- a/packages/ui/src/components/ValidatorList/ValidatorList.tsx +++ b/packages/ui/src/components/ValidatorList/ValidatorList.tsx @@ -19,7 +19,7 @@ import { MAX_STALLED_REFRESHES, nextStallCount, PENDING_REFRESH_MS, - remapSelection + selectionAfterRefresh } from "../../utils/validatorRefresh"; export default function ValidatorList({ @@ -92,7 +92,8 @@ export default function ValidatorList({ try { if (!background) setLoading(true); const rows = await rpcClient.call("getValidators", undefined); - if (background) setSelectedRows((selected) => remapSelection(rowsRef.current, rows, selected)); + // Capture the rows the current selection refers to before replacing them (see selectionAfterRefresh). + if (background) setSelectedRows(selectionAfterRefresh(rowsRef.current, rows)); rowsRef.current = rows; const pending = countPending(rows); stalls.current = nextStallCount(stalls.current, lastPending.current, pending); diff --git a/packages/ui/src/utils/importBatches.ts b/packages/ui/src/utils/importBatches.ts index 82211366..dc3e5985 100644 --- a/packages/ui/src/utils/importBatches.ts +++ b/packages/ui/src/utils/importBatches.ts @@ -6,8 +6,10 @@ import { prefix0xPubkey, RPC_MAX_PAYLOAD_BYTES } from "@stakingbrain/common"; * the RPC timeout, lets the UI report progress and lets the user stop between batches. */ export const IMPORT_BATCH_SIZE = 20; -// Well below the transport limit: JSON escaping roughly doubles keystore text and slashing data rides along. -const IMPORT_BATCH_MAX_BYTES = RPC_MAX_PAYLOAD_BYTES / 4; +// Sizes below are exact encoded bytes; keep a margin under the transport limit for RPC framing. +const IMPORT_BATCH_MAX_BYTES = Math.floor(RPC_MAX_PAYLOAD_BYTES * 0.9); +const encoder = new TextEncoder(); +const byteLength = (text: string) => encoder.encode(text).byteLength; export interface ImportResult { status: string; @@ -18,6 +20,8 @@ export interface ImportBatch { /** Positions in the original file list, so results stay aligned with the files the user picked. */ indexes: number[]; request: CustomImportRequest; + /** Set when a single key cannot fit one request; it is reported as an error without being sent. */ + tooLarge?: { bytes: number; maxBytes: number }; } export interface ImportProgress { @@ -33,24 +37,62 @@ export function notAttempted(): ImportResult { return { status: NOT_ATTEMPTED, message: "Not imported yet: the import stopped before reaching this validator." }; } -/** Keep only the slashing history of the batch's keys; the signer only needs those. */ -export function filterInterchange(interchange: string, pubkeys: ReadonlySet): string { - let parsed: unknown; +type Interchange = { data: { pubkey?: unknown }[] }; + +function parseInterchange(interchange: string): Interchange | undefined { try { - parsed = JSON.parse(interchange); + const parsed: unknown = JSON.parse(interchange); + if (parsed && typeof parsed === "object" && Array.isArray((parsed as { data?: unknown }).data)) + return parsed as Interchange; } catch { - return interchange; // Let the signer report the malformed file once, as before. + // Fall through: let the signer report the malformed file, as before. } - if (!parsed || typeof parsed !== "object" || !Array.isArray((parsed as { data?: unknown }).data)) return interchange; - const record = parsed as { data: { pubkey?: unknown }[] }; + return undefined; +} + +const entryPubkey = (entry: { pubkey?: unknown }) => + typeof entry?.pubkey === "string" ? prefix0xPubkey(entry.pubkey).toLowerCase() : undefined; + +/** Keep only the slashing history of the batch's keys; the signer only needs those. */ +export function filterInterchange(interchange: string, pubkeys: ReadonlySet): string { + const record = parseInterchange(interchange); + if (!record) return interchange; return JSON.stringify({ ...record, - data: record.data.filter( - (entry) => typeof entry?.pubkey === "string" && pubkeys.has(prefix0xPubkey(entry.pubkey).toLowerCase()) - ) + data: record.data.filter((entry) => pubkeys.has(entryPubkey(entry) ?? "")) }); } +/** + * Bytes each part adds to the serialized RPC request. `slashing_protection` travels as a JSON string + * inside JSON, so its entries are measured after a second round of escaping. + */ +function measureImport(request: CustomImportRequest, pubkeys: readonly string[]) { + const record = request.slashing_protection === undefined ? undefined : parseInterchange(request.slashing_protection); + const slashingBytes = new Map(); + for (const entry of record?.data ?? []) { + const key = entryPubkey(entry); + if (key) + slashingBytes.set(key, (slashingBytes.get(key) ?? 0) + byteLength(JSON.stringify(JSON.stringify(entry))) - 1); + } + const emptyRequest: CustomImportRequest = { + importFrom: request.importFrom, + slashing_protection: + request.slashing_protection === undefined + ? undefined + : record + ? JSON.stringify({ ...record, data: [] }) + : request.slashing_protection, + validatorsImportRequest: [] + }; + const base = byteLength(JSON.stringify({ jsonrpc: "2.0", method: "importValidators", params: emptyRequest, id: 0 })); + const perKey = (index: number) => + byteLength(JSON.stringify(request.validatorsImportRequest[index])) + + 1 + + (pubkeys[index] ? (slashingBytes.get(prefix0xPubkey(pubkeys[index]).toLowerCase()) ?? 0) : 0); + return { base, perKey }; +} + /** Split an import into ordered batches by count and size. `only` limits a retry to some files. */ export function planImportBatches( request: CustomImportRequest, @@ -62,36 +104,49 @@ export function planImportBatches( }: { only?: readonly number[]; maxKeystores?: number; maxBytes?: number } = {} ): ImportBatch[] { const selected = only ?? request.validatorsImportRequest.map((_, index) => index); + const { base, perKey } = measureImport(request, pubkeys); const groups: number[][] = []; + const oversized = new Map(); let group: number[] = []; - let bytes = 0; + let bytes = base; for (const index of selected) { - const { keystore, password } = request.validatorsImportRequest[index]; - const size = keystore.length + password.length; + const size = perKey(index); + if (base + size > maxBytes) { + oversized.set(index, base + size); + continue; + } if (group.length > 0 && (group.length >= maxKeystores || bytes + size > maxBytes)) { groups.push(group); group = []; - bytes = 0; + bytes = base; } group.push(index); bytes += size; } if (group.length > 0) groups.push(group); - return groups.map((indexes) => ({ - indexes, - request: { - importFrom: request.importFrom, - slashing_protection: - request.slashing_protection === undefined - ? undefined - : filterInterchange( - request.slashing_protection, - new Set(indexes.map((index) => prefix0xPubkey(pubkeys[index]).toLowerCase())) - ), - validatorsImportRequest: indexes.map((index) => request.validatorsImportRequest[index]) - } + const tooLarge: ImportBatch[] = [...oversized].map(([index, size]) => ({ + indexes: [index], + request: { ...request, validatorsImportRequest: [] }, + tooLarge: { bytes: size, maxBytes } })); + return [ + ...tooLarge, + ...groups.map((indexes) => ({ + indexes, + request: { + importFrom: request.importFrom, + slashing_protection: + request.slashing_protection === undefined + ? undefined + : filterInterchange( + request.slashing_protection, + new Set(indexes.map((index) => prefix0xPubkey(pubkeys[index]).toLowerCase())) + ), + validatorsImportRequest: indexes.map((index) => request.validatorsImportRequest[index]) + } + })) + ]; } /** @@ -120,6 +175,16 @@ export async function runImportBatches({ for (const [position, batch] of batches.entries()) { if (shouldStop()) return { results, stopped: "user" }; + if (batch.tooLarge) { + const mb = (bytes: number) => (bytes / 1024 / 1024).toFixed(1); + results[batch.indexes[0]] = { + status: "error", + message: `This keystore and its slashing protection history are too large to import in one request (${mb(batch.tooLarge.bytes)} MB, limit ${mb(batch.tooLarge.maxBytes)} MB). Import it with a pruned slashing protection file.` + }; + done += 1; + onProgress({ done, total, batch: position + 1, batches: batches.length }, [...results]); + continue; + } let failed = false; try { const response = await importBatch(batch.request); diff --git a/packages/ui/src/utils/validatorRefresh.ts b/packages/ui/src/utils/validatorRefresh.ts index 67f15686..eebca961 100644 --- a/packages/ui/src/utils/validatorRefresh.ts +++ b/packages/ui/src/utils/validatorRefresh.ts @@ -20,6 +20,17 @@ export function remapSelection( }); } +/** + * Build the selection updater for a refresh. `previous` must be captured when the refresh lands: + * React may run the updater later, after the caller has already replaced its own reference to the rows. + */ +export function selectionAfterRefresh( + previous: readonly { pubkey: string }[] | undefined, + next: readonly { pubkey: string }[] +): (selected: GridRowId[]) => GridRowId[] { + return (selected) => remapSelection(previous, next, selected); +} + export function countPending(rows: readonly { detailsPending?: boolean }[]): number { return rows.filter(({ detailsPending }) => detailsPending).length; } diff --git a/packages/ui/test/importBatches.test.ts b/packages/ui/test/importBatches.test.ts index 04960db0..a945cec1 100644 --- a/packages/ui/test/importBatches.test.ts +++ b/packages/ui/test/importBatches.test.ts @@ -1,4 +1,5 @@ import { expect } from "chai"; +import { RPC_MAX_PAYLOAD_BYTES } from "@stakingbrain/common"; import type { CustomImportRequest, Web3signerPostResponse } from "@stakingbrain/brain"; import { filterInterchange, @@ -46,6 +47,66 @@ describe("Batched validator import", () => { expect(batches.flatMap(({ indexes }) => indexes)).to.have.length(45); }); + it("counts each key's slashing history and JSON encoding toward the batch size", () => { + // 4,000 attestations per key: twenty of these do not fit one RPC payload. + const attestations = Array.from({ length: 4000 }, (_, epoch) => ({ + source_epoch: String(epoch), + target_epoch: String(epoch + 1), + signing_root: "0x" + "ab".repeat(32) + })); + const heavy = JSON.stringify({ + metadata: { interchange_format_version: "5", genesis_validators_root: "0x" + "00".repeat(32) }, + data: pubkeys.map((pubkey) => ({ pubkey, signed_blocks: [], signed_attestations: attestations })) + }); + const batches = planImportBatches(request(heavy), pubkeys); + expect(batches.length).to.be.greaterThan(Math.ceil(pubkeys.length / 20)); + expect(batches.flatMap(({ indexes }) => indexes)).to.deep.equal(pubkeys.map((_, index) => index)); + for (const { request: batch } of batches) { + // The same framing the RPC client measures before sending. + const rpcBytes = new TextEncoder().encode( + JSON.stringify({ jsonrpc: "2.0", method: "importValidators", params: batch, id: Date.now() }) + ).byteLength; + expect(rpcBytes + 64).to.be.at.most(RPC_MAX_PAYLOAD_BYTES); + } + }); + + it("reports a key too large for any request and imports the rest", async () => { + const huge = Array.from({ length: 60000 }, (_, epoch) => ({ + source_epoch: String(epoch), + target_epoch: String(epoch + 1), + signing_root: "0x" + "cd".repeat(32) + })); + const keys = pubkeys.slice(0, 3); + const slashing = JSON.stringify({ + metadata: { interchange_format_version: "5", genesis_validators_root: "0x" + "00".repeat(32) }, + data: keys.map((pubkey, index) => ({ + pubkey, + signed_blocks: [], + signed_attestations: index === 1 ? huge : [] + })) + }); + const full = request(slashing); + const batches = planImportBatches( + { ...full, validatorsImportRequest: full.validatorsImportRequest.slice(0, 3) }, + keys + ); + const sent: number[][] = []; + const { results, stopped } = await runImportBatches({ + batches, + results: keys.map(notAttempted), + importBatch: async (batch) => { + sent.push(batch.validatorsImportRequest.map((_, i) => i)); + return imported(batch); + }, + onProgress: () => undefined, + shouldStop: () => false + }); + expect(stopped).to.equal(undefined); + expect(results.map(({ status }) => status)).to.deep.equal(["imported", "error", "imported"]); + expect(results[1].message).to.include("too large"); + expect(sent).to.have.length(1); + }); + it("leaves a malformed slashing file untouched for the signer to report", () => { expect(filterInterchange("{not json", new Set(pubkeys))).to.equal("{not json"); expect(planImportBatches(request(), pubkeys)[0].request.slashing_protection).to.equal(undefined); diff --git a/packages/ui/test/validatorRefresh.test.ts b/packages/ui/test/validatorRefresh.test.ts index fe78472b..d6303242 100644 --- a/packages/ui/test/validatorRefresh.test.ts +++ b/packages/ui/test/validatorRefresh.test.ts @@ -1,5 +1,5 @@ import { expect } from "chai"; -import { countPending, nextStallCount, remapSelection } from "../src/utils/validatorRefresh.js"; +import { countPending, nextStallCount, remapSelection, selectionAfterRefresh } from "../src/utils/validatorRefresh.js"; import { incompleteImportIndexes } from "../src/utils/importForm.js"; const row = (pubkey: string, detailsPending?: boolean) => ({ pubkey, detailsPending }); @@ -11,6 +11,17 @@ describe("Live validator list refresh", () => { expect(remapSelection(previous, next, [0, 1, 2])).to.deep.equal([1, 0]); }); + it("keeps the selected validator when React applies the update after the rows were replaced", () => { + const rowsRef: { current: { pubkey: string }[] } = { current: [row("0xaa"), row("0xbb"), row("0xcc")] }; + const next = [row("0xbb"), row("0xcc")]; + // B is selected. Like ValidatorList, schedule the update and then replace the rows before it runs. + const updater = selectionAfterRefresh(rowsRef.current, next); + rowsRef.current = next; + expect(updater([1])).to.deep.equal([0]); + // Reading the reference inside the updater (the previous code) selects C instead. + expect(remapSelection(rowsRef.current, next, [1])).to.deep.equal([1]); + }); + it("clears the selection when there was no earlier list", () => { expect(remapSelection(undefined, [row("0xaa")], [0])).to.deep.equal([]); }); From add3f8ea753006458eddff324abfdc0a393f5926 Mon Sep 17 00:00:00 2001 From: 3alpha <15694175+3alpha@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:55:32 +0100 Subject: [PATCH 8/8] refactor: read validator status with bounded client reads and a beacon store Restore the validator-client behavior from main: sequential writes in import, update, delete cleanup and reconciliation, fee-recipient correctness from getFeeRecipient() reads, and live signer/validator membership reads on every list request, reported as unavailable when they fail. - getValidators reads membership and fee recipients within a 10s budget and 8 parallel reads. Cut-off reads are pending and the cursor advances on dispatch, so follow-up requests continue past hung keys. Matching fee checks are cached for 60s only alongside a current membership read; mismatches are dropped on change notifications, reads overlapping a change are not cached, and keys absent from the client lose their cached check. - BeaconStatusStore holds beacon details only, fetched in chunked bulk requests by its own cron job (separate from reconciliation and its lock), looked up for new keys on database changes, and persisted as indexes. - Exits look up indexes with chunked bulk requests. - The UI keys grid rows by pubkey and passes selected validators to dialogs; it requests the list again only while reads were cut off (bounded, coalesced), waits for pushes for beacon details, and shows errors with Retry instead of polling. - Launchpad fee-recipient routes read stored values directly; busy lock errors return 503. --- packages/brain/src/index.ts | 16 +- .../modules/apiClients/beaconchain/index.ts | 7 +- .../brain/src/modules/apiServers/index.ts | 9 +- .../launchpad/routes/feeRecipients/route.ts | 30 +- .../launchpad/routes/keystores/route.ts | 14 +- .../apiServers/launchpad/startLaunchpadApi.ts | 16 +- .../apiServers/ui/calls/exitValidators.ts | 23 +- .../apiServers/ui/calls/getValidators.ts | 261 ++++++++-------- .../apiServers/ui/calls/importValidators.ts | 9 +- .../src/modules/apiServers/ui/calls/index.ts | 21 +- .../src/modules/apiServers/ui/calls/types.ts | 15 +- .../modules/apiServers/ui/startUiServer.ts | 6 +- .../brain/src/modules/beaconStatus/index.ts | 152 ++++++++++ packages/brain/src/modules/cron/cron.ts | 26 +- packages/brain/src/modules/cron/index.ts | 10 +- .../deleteValidatorPubkeysNotInDb.ts | 8 +- .../postValidatorPubkeysFromDb.ts | 7 +- .../postValidatorsFeeRecipientsFromDb.ts | 12 +- .../src/modules/utils/forEachConcurrently.ts | 3 - packages/brain/test/e2e/fakeClients.ts | 39 ++- .../brain/test/e2e/largeBatches.e2e.test.ts | 116 +++++-- packages/brain/test/helpers/beacon.ts | 7 + .../unit/modules/beaconStatus.unit.test.ts | 130 ++++++++ .../builderConfig/builderConfig.unit.test.ts | 18 +- .../unit/modules/exitValidators.unit.test.ts | 67 +++++ .../modules/jsonBodyEncoding.unit.test.ts | 4 +- .../unit/modules/validatorList.unit.test.ts | 284 ++++++++---------- .../components/Dialogs/DeletionWarning.tsx | 13 +- .../src/components/Dialogs/EditFeesDialog.tsx | 38 +-- .../ui/src/components/Dialogs/ExitWarning.tsx | 13 +- .../Dialogs/KeystoresDeleteDialog.tsx | 13 +- .../Dialogs/KeystoresExitDialog.tsx | 15 +- .../ValidatorList/KeystoresDataGrid.tsx | 15 +- .../ValidatorList/ValidatorList.tsx | 165 +++++----- packages/ui/src/utils/importBatches.ts | 3 +- packages/ui/src/utils/listFollowUp.ts | 15 + packages/ui/src/utils/validatorRefresh.ts | 42 --- packages/ui/test/importForm.test.ts | 21 +- packages/ui/test/listFollowUp.test.ts | 18 ++ packages/ui/test/validatorRefresh.test.ts | 51 ---- 40 files changed, 1090 insertions(+), 642 deletions(-) create mode 100644 packages/brain/src/modules/beaconStatus/index.ts create mode 100644 packages/brain/test/helpers/beacon.ts create mode 100644 packages/brain/test/unit/modules/beaconStatus.unit.test.ts create mode 100644 packages/brain/test/unit/modules/exitValidators.unit.test.ts create mode 100644 packages/ui/src/utils/listFollowUp.ts delete mode 100644 packages/ui/src/utils/validatorRefresh.ts create mode 100644 packages/ui/test/listFollowUp.test.ts delete mode 100644 packages/ui/test/validatorRefresh.test.ts diff --git a/packages/brain/src/index.ts b/packages/brain/src/index.ts index 5a3d0ce3..58d9c4c1 100644 --- a/packages/brain/src/index.ts +++ b/packages/brain/src/index.ts @@ -9,6 +9,7 @@ import { params } from "./params.js"; import { getCrons } from "./modules/cron/index.js"; import { brainConfig } from "./modules/config/index.js"; import { importBuilderIds } from "./modules/config/importBuilderIds.js"; +import { BeaconStatusStore } from "./modules/beaconStatus/index.js"; import { featureFlags } from "@stakingbrain/common"; logger.info(`Starting brain...`); @@ -54,8 +55,11 @@ if (featureFlags.builderConfig) { logger.warn("BUILDER_IDS is set but builder configuration is disabled in this version; ignoring it"); } +// Beacon chain details (index, withdrawal credentials) for listing validators. +const beaconStatus = new BeaconStatusStore(brainDb, beaconchainApi); + // CRON -const { reloadValidatorsCronTask } = getCrons({ +const { reloadValidatorsCronTask, beaconStatusCronTask } = getCrons({ sendNotification: true, postgresClient, prometheusApi, @@ -65,9 +69,13 @@ const { reloadValidatorsCronTask } = getCrons({ brainDb, chainConfig: config.chain, beaconchainApi, - dappmanagerApi + dappmanagerApi, + beaconStatus }); reloadValidatorsCronTask.start(); +beaconStatusCronTask.start(); +// Reads only: load beacon details now rather than after the first interval. +void beaconStatusCronTask.runNow(); //trackValidatorsPerformanceCronTask.start(); // Start server APIs @@ -80,13 +88,15 @@ const { uiServer, launchpadServer, brainApiServer } = getServers({ beaconchainApi, brainDb, reloadValidatorsCronTask, - allowedOriginsFromEnv: config.apis.cors + allowedOriginsFromEnv: config.apis.cors, + beaconStatus }); // Graceful shutdown function handle(signal: string): void { logger.info(`${signal} received. Shutting down...`); reloadValidatorsCronTask.stop(); + beaconStatusCronTask.stop(); //trackValidatorsPerformanceCronTask.stop(); brainDb.close(); postgresClient.close().catch((err) => logger.error(`Error closing postgres client`, err)); // postgresClient db connection is the only external resource that needs to be closed diff --git a/packages/brain/src/modules/apiClients/beaconchain/index.ts b/packages/brain/src/modules/apiClients/beaconchain/index.ts index 2ea4881f..28ba3bdf 100644 --- a/packages/brain/src/modules/apiClients/beaconchain/index.ts +++ b/packages/brain/src/modules/apiClients/beaconchain/index.ts @@ -166,16 +166,19 @@ export class BeaconchainApi extends StandardApi { */ public async postStateValidators({ stateId, - body + body, + timeout }: { stateId: BlockId; body: { ids: string[]; statuses: ValidatorStatus[] }; + timeout?: number; }): Promise { try { return await this.request({ method: "POST", endpoint: path.join(this.beaconchainEndpoint, "states", stateId, "validators"), - body: JSON.stringify(body) + body: JSON.stringify(body), + timeout }); } catch (e) { throw new BeaconchainApiError(`Error getting (POST) state validators from beaconchain. ${e.message}`); diff --git a/packages/brain/src/modules/apiServers/index.ts b/packages/brain/src/modules/apiServers/index.ts index dcdcf754..d3c40ea6 100644 --- a/packages/brain/src/modules/apiServers/index.ts +++ b/packages/brain/src/modules/apiServers/index.ts @@ -9,6 +9,7 @@ import { BeaconchainApi } from "../apiClients/beaconchain/index.js"; import { CronJob } from "../cron/cron.js"; import { startLaunchpadApi } from "./launchpad/index.js"; import http from "http"; +import { BeaconStatusStore } from "../beaconStatus/index.js"; export const getServers = ({ brainConfig, @@ -19,7 +20,8 @@ export const getServers = ({ beaconchainApi, brainDb, reloadValidatorsCronTask, - allowedOriginsFromEnv + allowedOriginsFromEnv, + beaconStatus }: { brainConfig: BrainConfig; uiBuildPath: string; @@ -30,6 +32,7 @@ export const getServers = ({ brainDb: BrainDataBase; reloadValidatorsCronTask: CronJob; allowedOriginsFromEnv: string | string[] | null; + beaconStatus: BeaconStatusStore; }): { uiServer: http.Server; launchpadServer: http.Server; @@ -45,13 +48,13 @@ export const getServers = ({ validatorApi, beaconchainApi, postgresClient, - allowedOriginsFromEnv + allowedOriginsFromEnv, + beaconStatus }), launchpadServer: startLaunchpadApi({ brainDb, signerApi, validatorApi, - beaconchainApi, reloadValidatorsCronTask, network: brainConfig.chain.network, signerUrl: brainConfig.apis.signerUrl, diff --git a/packages/brain/src/modules/apiServers/launchpad/routes/feeRecipients/route.ts b/packages/brain/src/modules/apiServers/launchpad/routes/feeRecipients/route.ts index 89e60cd0..f07a95f3 100644 --- a/packages/brain/src/modules/apiServers/launchpad/routes/feeRecipients/route.ts +++ b/packages/brain/src/modules/apiServers/launchpad/routes/feeRecipients/route.ts @@ -1,16 +1,17 @@ import express from "express"; import { isValidBlsPubkey } from "@stakingbrain/common"; import { updateValidators } from "../../../ui/calls/updateValidators.js"; -import { getValidators } from "../../../ui/calls/getValidators.js"; import { BrainPubkeysFeeRecipients } from "../../types.js"; import { validateUpdateFeeRecipientRequestBody } from "./validation.js"; -import type { CustomValidatorGetResponse } from "../../../ui/calls/types.js"; import logger from "../../../../logger/index.js"; import { BrainDataBase } from "../../../../db/index.js"; import { CronJob } from "../../../../cron/cron.js"; -import { BeaconchainApi } from "../../../../apiClients/beaconchain/index.js"; -import { Web3SignerApi } from "../../../../apiClients/signer/index.js"; import { ValidatorApi } from "../../../../apiClients/validator/index.js"; +import { ValidatorOperationBusyError } from "../../../../db/withValidatorOperationLock.js"; + +/** Stored fee recipients; client status is not needed, so reading them makes no client calls. */ +const storedFeeRecipients = (brainDb: BrainDataBase) => + Object.entries(brainDb.getData()).map(([pubkey, { feeRecipient }]) => ({ pubkey, feeRecipient })); /** * Retrieves fee recipient information for specified validators via GET request. @@ -40,17 +41,13 @@ import { ValidatorApi } from "../../../../apiClients/validator/index.js"; export const createFeeRecipientsRouter = ({ brainDb, - signerApi, signerUrl, validatorApi, - beaconchainApi, reloadValidatorsCronTask }: { brainDb: BrainDataBase; - signerApi: Web3SignerApi; signerUrl: string; validatorApi: ValidatorApi; - beaconchainApi: BeaconchainApi; reloadValidatorsCronTask: CronJob; }) => { const feeRecipientsRouter = express.Router(); @@ -58,12 +55,7 @@ export const createFeeRecipientsRouter = ({ feeRecipientsRouter.get(feeRecipientsEndpoint, async (req, res) => { try { - const validators: CustomValidatorGetResponse[] = await getValidators({ - brainDb, - signerApi, - validatorApi, - beaconchainApi - }); + const validators = storedFeeRecipients(brainDb); const pubkeysParam = req.query.pubkeys as string | undefined; if (!pubkeysParam) { @@ -143,12 +135,7 @@ export const createFeeRecipientsRouter = ({ } try { - const currentValidators: CustomValidatorGetResponse[] = await getValidators({ - brainDb, - signerApi, - validatorApi, - beaconchainApi - }); + const currentValidators = storedFeeRecipients(brainDb); const validatorsToUpdate = currentValidators.filter((validator) => requestBody.validators.some( @@ -177,7 +164,8 @@ export const createFeeRecipientsRouter = ({ res.status(200).send(); } catch (error) { logger.error("Failed to update validators:", error); - res.status(500).send({ message: "Internal server error" }); + if (error instanceof ValidatorOperationBusyError) res.status(503).send({ message: error.message }); + else res.status(500).send({ message: "Internal server error" }); } }); diff --git a/packages/brain/src/modules/apiServers/launchpad/routes/keystores/route.ts b/packages/brain/src/modules/apiServers/launchpad/routes/keystores/route.ts index 32085794..c8afcd37 100644 --- a/packages/brain/src/modules/apiServers/launchpad/routes/keystores/route.ts +++ b/packages/brain/src/modules/apiServers/launchpad/routes/keystores/route.ts @@ -1,4 +1,5 @@ import { Network, Tag } from "@stakingbrain/common"; +import { ValidatorOperationBusyError } from "../../../../db/withValidatorOperationLock.js"; import express from "express"; import { deleteValidators } from "../../../ui/calls/deleteValidators.js"; import { importValidators } from "../../../ui/calls/importValidators.js"; @@ -11,6 +12,13 @@ import { BrainDataBase } from "../../../../db/index.js"; import { Web3SignerApi } from "../../../../apiClients/signer/index.js"; import { ValidatorApi } from "../../../../apiClients/validator/index.js"; +/** Busy is temporary and safe to retry: nothing was changed. Other failures stay opaque. */ +function sendOperationError(res: express.Response, error: Error): void { + logger.error(error); + if (error instanceof ValidatorOperationBusyError) res.status(503).send({ message: error.message }); + else res.status(500).send({ message: "Internal server error" }); +} + export const createKeystoresRouter = ({ signerApi, reloadValidatorsCronTask, @@ -62,8 +70,7 @@ export const createKeystoresRouter = ({ res.status(200).send(importResponse); } catch (e) { - logger.error(e); - res.status(500).send({ message: "Internal server error" }); + sendOperationError(res, e); } }); @@ -89,8 +96,7 @@ export const createKeystoresRouter = ({ res.status(200).send(deleteResponse); } catch (e) { - logger.error(e); - res.status(500).send({ message: "Internal server error" }); + sendOperationError(res, e); } }); diff --git a/packages/brain/src/modules/apiServers/launchpad/startLaunchpadApi.ts b/packages/brain/src/modules/apiServers/launchpad/startLaunchpadApi.ts index 853ef314..c4f7b5f2 100644 --- a/packages/brain/src/modules/apiServers/launchpad/startLaunchpadApi.ts +++ b/packages/brain/src/modules/apiServers/launchpad/startLaunchpadApi.ts @@ -8,14 +8,12 @@ import { createKeystoresRouter, createFeeRecipientsRouter } from "./routes/index import { CronJob } from "../../cron/cron.js"; import { BrainDataBase } from "../../db/index.js"; import { Network, RPC_MAX_PAYLOAD_BYTES } from "@stakingbrain/common"; -import { BeaconchainApi } from "../../apiClients/beaconchain/index.js"; import { Web3SignerApi } from "../../apiClients/signer/index.js"; import { ValidatorApi } from "../../apiClients/validator/index.js"; export function startLaunchpadApi({ signerApi, validatorApi, - beaconchainApi, reloadValidatorsCronTask, brainDb, network, @@ -24,7 +22,6 @@ export function startLaunchpadApi({ }: { signerApi: Web3SignerApi; validatorApi: ValidatorApi; - beaconchainApi: BeaconchainApi; reloadValidatorsCronTask: CronJob; brainDb: BrainDataBase; network: Network; @@ -35,14 +32,21 @@ export function startLaunchpadApi({ app.use(express.json({ limit: RPC_MAX_PAYLOAD_BYTES, inflate: false })); app.use(cors({ origin: allowedOriginsFromEnv ?? allowedOrigins })); - app.use(createKeystoresRouter({ reloadValidatorsCronTask, brainDb, network, validatorApi, signerApi, signerUrl })); app.use( - createFeeRecipientsRouter({ + createKeystoresRouter({ + reloadValidatorsCronTask, brainDb, + network, + validatorApi, signerApi, + signerUrl + }) + ); + app.use( + createFeeRecipientsRouter({ + brainDb, signerUrl, validatorApi, - beaconchainApi, reloadValidatorsCronTask }) ); diff --git a/packages/brain/src/modules/apiServers/ui/calls/exitValidators.ts b/packages/brain/src/modules/apiServers/ui/calls/exitValidators.ts index 0035750b..ca2d4641 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/exitValidators.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/exitValidators.ts @@ -4,6 +4,7 @@ import { ValidatorExitExecute, ValidatorExitGet } from "../../../apiClients/types.js"; +import { BEACON_CHUNK_SIZE } from "../../../beaconStatus/index.js"; import logger from "../../../logger/index.js"; /** @@ -96,15 +97,19 @@ async function _getExitValidators({ // Get the genesis from the beaconchain API to sign the voluntary exit const genesis = await beaconchainApi.getGenesis(); - // Get the validators indexes from the validator API - const validatorPubkeysIndexes = ( - await Promise.all(pubkeys.map((pubkey) => beaconchainApi.getStateValidator({ state: "head", pubkey }))) - ).map((validator) => { - return { - pubkey: validator.data.validator.pubkey, - index: validator.data.index - }; - }); + // Look up indexes with bulk requests: one request per lookup would flood the beacon node for large batches. + const found = new Map(); + for (let start = 0; start < pubkeys.length; start += BEACON_CHUNK_SIZE) { + const { data } = await beaconchainApi.postStateValidators({ + stateId: "head", + body: { ids: pubkeys.slice(start, start + BEACON_CHUNK_SIZE), statuses: [] } + }); + for (const validator of data) found.set(validator.validator.pubkey.toLowerCase(), validator.index); + } + const missing = pubkeys.filter((pubkey) => !found.has(pubkey.toLowerCase())); + if (missing.length > 0) + throw new Error(`Validators not found on the beacon chain, so they cannot exit: ${missing.join(", ")}`); + const validatorPubkeysIndexes = pubkeys.map((pubkey) => ({ pubkey, index: found.get(pubkey.toLowerCase())! })); const validatorsExit: ValidatorExitGet[] = []; for (const validatorIndex of validatorPubkeysIndexes) { diff --git a/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts b/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts index c0c4f648..5887a8ec 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts @@ -1,163 +1,180 @@ -import { isValidWithdrawableBlsAddress, isValidNonWithdrawableBlsAddress, featureFlags } from "@stakingbrain/common"; +import { featureFlags } from "@stakingbrain/common"; import { CustomValidatorGetResponse } from "./types.js"; import { BrainDataBase } from "../../../db/index.js"; -import { ValidatorApi, Web3SignerApi, BeaconchainApi } from "../../../apiClients/index.js"; +import { ValidatorApi, Web3SignerApi } from "../../../apiClients/index.js"; +import { BeaconStatusStore } from "../../../beaconStatus/index.js"; +import { forEachConcurrently } from "../../../utils/forEachConcurrently.js"; import logger from "../../../logger/index.js"; -type BeaconDetails = Pick; -type ListProgress = { - beaconNext: number; +/** Bound on one list request; unfinished reads are reported as pending for the UI to request again. */ +export const LIST_DEADLINE_MS = 10_000; +const READ_CONCURRENCY = 8; +const FEE_CHECK_TTL_MS = 60_000; + +interface FeeCheck { + expected: string; + correct: boolean; + expires: number; +} +interface ReadState { + /** Where the next request resumes fee reads, so successive requests cover different validators. */ feeNext: number; - beacon: Map; - // Keyed by the expected fee recipient so an edit is re-checked immediately. - fee: Map; -}; -const listProgress = new WeakMap(); -const BEACON_CACHE_MS = 60_000; // Also bounds how stale a cached fee recipient check can be. + fee: Map; + /** Bumped on every validator change; a read that overlaps one is not cached. */ + generation: number; +} +const readStates = new WeakMap(); + +function readStateFor(brainDb: BrainDataBase): ReadState { + let state = readStates.get(brainDb); + if (!state) { + const created: ReadState = { feeNext: 0, fee: new Map(), generation: 0 }; + // Change notifications arrive after an operation has configured the validator client. A check read + // mid-operation saw the new expected address but the old client value, so only mismatches can be + // stale; drop them, and refuse to cache reads that straddle the change. + brainDb.onChange(() => { + for (const [pubkey, check] of created.fee) if (!check.correct) created.fee.delete(pubkey); + created.generation++; + }); + readStates.set(brainDb, created); + state = created; + } + return state; +} + +const message = (error: unknown) => (error instanceof Error ? error.message : String(error)); -/** Read a stable snapshot; independent queues rotate unfinished work across bounded refreshes. */ +/** + * List validators with their signer and validator client status, read within one bounded request. + * Reads cut off by the deadline are marked `detailsPending` (request again); failed reads are reported + * in `detailsError` rather than replaced by an earlier result. Beacon details come from the beacon + * store, which announces its updates. + */ export async function getValidators({ brainDb, validatorApi, signerApi, - beaconchainApi + beaconStatus }: { brainDb: BrainDataBase; validatorApi: ValidatorApi; signerApi: Web3SignerApi; - beaconchainApi: BeaconchainApi; + beaconStatus: BeaconStatusStore; }): Promise { const entries = Object.entries(brainDb.getData()); - const progress = listProgress.get(brainDb) ?? { beaconNext: 0, feeNext: 0, beacon: new Map(), fee: new Map() }; - listProgress.set(brainDb, progress); - const currentKeys = new Set(entries.map(([pubkey]) => pubkey)); - for (const [pubkey, cached] of progress.beacon) { - if (!currentKeys.has(pubkey) || cached.expires <= Date.now()) progress.beacon.delete(pubkey); - } - for (const [pubkey, cached] of progress.fee) { - if (!currentKeys.has(pubkey) || cached.expires <= Date.now()) progress.fee.delete(pubkey); - } if (entries.length === 0) return []; - const results: CustomValidatorGetResponse[] = entries.map(([pubkey, details]) => ({ - pubkey, - index: details.index?.toString() ?? "", - tag: details.tag, - feeRecipient: details.feeRecipient, - // Stored settings are kept while the feature is off, but not surfaced. - ...(featureFlags.builderConfig && { - builderConfig: details.builderConfig, - builderConfigStatus: - validatorApi.getBuilderConfigStatus(pubkey) ?? - (details.builderConfig === undefined ? undefined : { status: "pending" }) - }), - withdrawalCredentials: { address: "", format: "error" }, - ...progress.beacon.get(pubkey)?.details - // Membership and fee checks stay undefined until a successful read establishes their value. - })); - const controller = new AbortController(); - const deadline = setTimeout(() => controller.abort(), 10000); - // Reads cut short by the deadline stay pending so the UI knows to refresh; definite failures do not. - const beaconPending = new Set(results.map((_, index) => index)); - const feePending = new Set(); - let membershipPending = false; + const state = readStateFor(brainDb); + const now = Date.now(); + const present = new Set(entries.map(([pubkey]) => pubkey)); + for (const [pubkey, check] of state.fee) if (!present.has(pubkey) || check.expires <= now) state.fee.delete(pubkey); - async function readQueue( - cursor: "beaconNext" | "feeNext", - pending: Set, - read: (result: CustomValidatorGetResponse) => Promise - ) { - const start = progress[cursor] % results.length; - let next = 0; - await Promise.all( - Array.from({ length: Math.min(8, results.length) }, async () => { - while (next < results.length && !controller.signal.aborted) { - const index = (start + next++) % results.length; - progress[cursor] = (index + 1) % results.length; - try { - await read(results[index]); - pending.delete(index); - } catch (error) { - if (!controller.signal.aborted) pending.delete(index); - logger.error(error); - } - } + const rows: CustomValidatorGetResponse[] = entries.map(([pubkey, details]) => { + const { beaconError, ...beacon } = beaconStatus.view(pubkey); + return { + pubkey, + tag: details.tag, + feeRecipient: details.feeRecipient, + ...beacon, + index: details.index?.toString() ?? beacon.index ?? "", + detailsPending: false, + detailsError: beaconError, + // Stored settings are kept while the feature is off, but not surfaced. + ...(featureFlags.builderConfig && { + builderConfig: details.builderConfig, + builderConfigStatus: + validatorApi.getBuilderConfigStatus(pubkey) ?? + (details.builderConfig === undefined ? undefined : { status: "pending" }) }) - ); - } + }; + }); + const errors = rows.map((row) => (row.detailsError ? [row.detailsError] : [])); + const pending = new Set(); + + const controller = new AbortController(); + const deadline = setTimeout(() => controller.abort(), LIST_DEADLINE_MS); + const { signal } = controller; + /** Cut off by the deadline: pending. Anything else: a definite failure to report. */ + const failed = (indexes: Iterable, what: string, error: unknown) => { + for (const index of indexes) { + if (signal.aborted) pending.add(index); + else errors[index].push(`${what}: ${message(error)}`); + } + }; try { await Promise.all([ (async () => { try { - const remoteKeys = await validatorApi.getRemoteKeys(controller.signal); - const keys = new Set(remoteKeys.data.map(({ pubkey }) => pubkey.toLowerCase())); - results.forEach((result, index) => { - result.validatorImported = keys.has(result.pubkey.toLowerCase()); - if (result.validatorImported) feePending.add(index); - }); - await readQueue("feeNext", feePending, async (result) => { - if (!result.validatorImported) return; - const feeRecipient = result.feeRecipient.toLowerCase(); - const cached = progress.fee.get(result.pubkey); - if (cached?.feeRecipient === feeRecipient) { - result.validatorFeeRecipientCorrect = cached.correct; - return; - } - const response = await validatorApi.getFeeRecipient(result.pubkey, controller.signal); - result.validatorFeeRecipientCorrect = response.data.ethaddress.toLowerCase() === feeRecipient; - progress.fee.set(result.pubkey, { - feeRecipient, - correct: result.validatorFeeRecipientCorrect, - expires: Date.now() + BEACON_CACHE_MS - }); - }); + const keys = new Set( + (await signerApi.listRemoteKeys(signal)).data.map(({ validating_pubkey }) => + validating_pubkey.toLowerCase() + ) + ); + for (const row of rows) row.signerImported = keys.has(row.pubkey.toLowerCase()); } catch (error) { - if (controller.signal.aborted) membershipPending = true; logger.error(error); + failed(rows.keys(), "Could not read the signer's keys", error); } })(), (async () => { + let keys: Set; try { - const signerKeys = await signerApi.listRemoteKeys(controller.signal); - const keys = new Set(signerKeys.data.map(({ validating_pubkey }) => validating_pubkey.toLowerCase())); - for (const result of results) result.signerImported = keys.has(result.pubkey.toLowerCase()); + keys = new Set((await validatorApi.getRemoteKeys(signal)).data.map(({ pubkey }) => pubkey.toLowerCase())); } catch (error) { - if (controller.signal.aborted) membershipPending = true; logger.error(error); - } - })(), - readQueue("beaconNext", beaconPending, async (result) => { - const cached = progress.beacon.get(result.pubkey); - if (cached) { - Object.assign(result, cached.details); + failed(rows.keys(), "Could not read the validator client's keys", error); return; } - const { data } = await beaconchainApi.getStateValidator({ - state: "head", - pubkey: result.pubkey, - signal: controller.signal + // Fee recipients are only checked for keys the client has; cached checks need a current + // membership read, so an outage is never hidden behind an earlier success. + const toRead: number[] = []; + rows.forEach((row, index) => { + row.validatorImported = keys.has(row.pubkey.toLowerCase()); + // A check made before the key left the client says nothing about it once it returns. + if (!row.validatorImported) { + state.fee.delete(row.pubkey); + return; + } + const cached = state.fee.get(row.pubkey); + if (cached?.expected === row.feeRecipient.toLowerCase()) row.validatorFeeRecipientCorrect = cached.correct; + else toRead.push(index); }); - const address = data.validator.withdrawal_credentials; - const details: BeaconDetails = { - index: data.index, - withdrawalCredentials: { - address, - format: isValidWithdrawableBlsAddress(address) - ? "ecdsa" - : isValidNonWithdrawableBlsAddress(address) - ? "bls" - : "unknown" + // Resume where the previous request stopped, so a large list converges over a few requests. + const start = toRead.findIndex((index) => index >= state.feeNext % rows.length); + const order = start <= 0 ? toRead : [...toRead.slice(start), ...toRead.slice(0, start)]; + order.forEach((index) => pending.add(index)); + await forEachConcurrently(order, READ_CONCURRENCY, async (index) => { + if (signal.aborted) return; + // Advance on dispatch: reads that hang until the deadline must not hold the next request back. + state.feeNext = index + 1; + const row = rows[index]; + const expected = row.feeRecipient.toLowerCase(); + const generation = state.generation; + try { + const { data } = await validatorApi.getFeeRecipient(row.pubkey, signal); + row.validatorFeeRecipientCorrect = data.ethaddress.toLowerCase() === expected; + pending.delete(index); + if (state.generation === generation) + state.fee.set(row.pubkey, { + expected, + correct: row.validatorFeeRecipientCorrect, + expires: Date.now() + FEE_CHECK_TTL_MS + }); + } catch (error) { + if (!signal.aborted) pending.delete(index); + logger.error(error); + failed([index], "Could not read the fee recipient", error); } - }; - Object.assign(result, details); - progress.beacon.set(result.pubkey, { details, expires: Date.now() + BEACON_CACHE_MS }); - }) + }); + })() ]); - results.forEach((result, index) => { - result.detailsPending = membershipPending || beaconPending.has(index) || feePending.has(index); - }); - return results; } finally { clearTimeout(deadline); } + + rows.forEach((row, index) => { + row.detailsPending = pending.has(index); + row.detailsError = errors[index].length > 0 ? errors[index].join("; ") : undefined; + }); + return rows; } diff --git a/packages/brain/src/modules/apiServers/ui/calls/importValidators.ts b/packages/brain/src/modules/apiServers/ui/calls/importValidators.ts index b4e0d10f..fadf115e 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/importValidators.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/importValidators.ts @@ -15,7 +15,6 @@ import { LIDO_FEE_RECIPIENT_HOODI } from "@stakingbrain/common"; import { CustomImportRequest } from "./types.js"; -import { forEachConcurrently, VALIDATOR_CLIENT_CONCURRENCY } from "../../../utils/forEachConcurrently.js"; import { CronJob } from "../../../cron/cron.js"; import { BrainDataBase } from "../../../db/index.js"; import { Web3signerPostResponse } from "../../../apiClients/types.js"; @@ -194,11 +193,11 @@ export async function importValidators({ logger.debug(`Added pubkeys to validator API`); // Import feeRecipient on Validator API - await forEachConcurrently(validatorsToPost, VALIDATOR_CLIENT_CONCURRENCY, (validator) => - validatorApi + for (const validator of validatorsToPost) { + await validatorApi .setFeeRecipient(validator.feeRecipient, validator.pubkey) - .catch((err) => logger.error(`Error setting validator feeRecipient for pubkey ${validator.pubkey} :`, err)) - ); + .catch((err) => logger.error(`Error setting validator feeRecipient for pubkey ${validator.pubkey} :`, err)); + } logger.debug(`Added fee recipients to validator API`); diff --git a/packages/brain/src/modules/apiServers/ui/calls/index.ts b/packages/brain/src/modules/apiServers/ui/calls/index.ts index f7a72997..881c29a5 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/index.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/index.ts @@ -16,6 +16,7 @@ import { BeaconchainApi, PostgresClient, ValidatorApi, Web3SignerApi } from "../ import { BrainConfig } from "../../../config/types.js"; import { getBuilderRelayPresets } from "../../../config/builderRelayPresets.js"; import { featureFlags, FeatureDisabledError } from "@stakingbrain/common"; +import { BeaconStatusStore } from "../../../beaconStatus/index.js"; export const createRpcMethods = ({ postgresClient, @@ -24,7 +25,8 @@ export const createRpcMethods = ({ signerApi, brainDb, reloadValidatorsCronTask, - brainConfig + brainConfig, + beaconStatus }: { postgresClient: PostgresClient; beaconchainApi: BeaconchainApi; @@ -33,6 +35,7 @@ export const createRpcMethods = ({ brainDb: BrainDataBase; reloadValidatorsCronTask: CronJob; brainConfig: BrainConfig; + beaconStatus: BeaconStatusStore; }): RpcMethods => { const { beaconchainUrl, executionClientUrl, signerUrl, validatorUrl } = brainConfig.apis; const { minGenesisTime, secondsPerSlot, network, isMevBoostSet, executionClient, consensusClient } = @@ -47,7 +50,13 @@ export const createRpcMethods = ({ return await getBuilderConfigs({ pubkeys, brainDb, validatorApi }); }, deleteValidators: async (deleteRequest: Web3signerDeleteRequest) => - await deleteValidators({ brainDb, reloadValidatorsCronTask, validatorApi, signerApi, deleteRequest }), + await deleteValidators({ + brainDb, + reloadValidatorsCronTask, + validatorApi, + signerApi, + deleteRequest + }), exitValidators: async ({ pubkeys }: { pubkeys: string[] }) => await exitValidators({ pubkeys, beaconchainApi, signerApi }), getExitValidators: async ({ pubkeys }: { pubkeys: string[] }) => @@ -74,13 +83,7 @@ export const createRpcMethods = ({ signerUrl, requestFrom }), - getValidators: async () => - await getValidators({ - brainDb, - validatorApi, - signerApi, - beaconchainApi - }), + getValidators: async () => getValidators({ brainDb, validatorApi, signerApi, beaconStatus }), signerGetStatus: async () => await signerGetStatus({ signerApi }), getStakerConfig: async () => await getStakerConfig({ diff --git a/packages/brain/src/modules/apiServers/ui/calls/types.ts b/packages/brain/src/modules/apiServers/ui/calls/types.ts index 5d65d885..27693453 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/types.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/types.ts @@ -1,3 +1,4 @@ +import type { BeaconStatusView } from "../../../beaconStatus/index.js"; import { BuilderConfig, BuilderConfigStatus, BuilderRelayPresets, StakerConfig, Tag } from "@stakingbrain/common"; import type { Web3signerDeleteRequest, @@ -60,21 +61,19 @@ export interface ValidatorBuilderConfigResponse { message?: string; } -export type WithdrawalCredentialsFormat = "ecdsa" | "bls" | "unknown" | "error"; -export interface CustomValidatorGetResponse { +export type { WithdrawalCredentialsFormat } from "../../../beaconStatus/index.js"; +export interface CustomValidatorGetResponse extends Omit { pubkey: string; index: string; tag: Tag; feeRecipient: string; - withdrawalCredentials: { - format: WithdrawalCredentialsFormat; - address: string; - }; signerImported?: boolean; validatorImported?: boolean; validatorFeeRecipientCorrect?: boolean; + /** A client read was cut off by the list deadline; request the list again to continue. */ + detailsPending: boolean; + /** Why a check failed, when it failed rather than being cut off. */ + detailsError?: string; builderConfig?: BuilderConfig | null; builderConfigStatus?: BuilderConfigStatus; - /** True when the read deadline cut this row's checks short; a later refresh continues them. */ - detailsPending?: boolean; } diff --git a/packages/brain/src/modules/apiServers/ui/startUiServer.ts b/packages/brain/src/modules/apiServers/ui/startUiServer.ts index 186f5015..05c8f477 100644 --- a/packages/brain/src/modules/apiServers/ui/startUiServer.ts +++ b/packages/brain/src/modules/apiServers/ui/startUiServer.ts @@ -16,6 +16,7 @@ import { BrainDataBase } from "../../db/index.js"; import { BrainConfig } from "../../config/types.js"; import { BeaconchainApi, PostgresClient, ValidatorApi, Web3SignerApi } from "../../apiClients/index.js"; import { allowedOrigins, uiSocketOptions } from "./config.js"; +import { BeaconStatusStore } from "../../beaconStatus/index.js"; export function startUiServer({ brainDb, @@ -27,6 +28,7 @@ export function startUiServer({ brainConfig, reloadValidatorsCronTask, allowedOriginsFromEnv, + beaconStatus, port = params.uiPort }: { brainDb: BrainDataBase; @@ -38,6 +40,7 @@ export function startUiServer({ brainConfig: BrainConfig; reloadValidatorsCronTask: CronJob; allowedOriginsFromEnv: string | string[] | null; + beaconStatus: BeaconStatusStore; /** Defaults to the production UI port; tests pass 0 for an ephemeral port. */ port?: number; }): http.Server { @@ -53,7 +56,8 @@ export function startUiServer({ signerApi, brainDb, reloadValidatorsCronTask, - brainConfig + brainConfig, + beaconStatus }); const app = express(); diff --git a/packages/brain/src/modules/beaconStatus/index.ts b/packages/brain/src/modules/beaconStatus/index.ts new file mode 100644 index 00000000..357b8d17 --- /dev/null +++ b/packages/brain/src/modules/beaconStatus/index.ts @@ -0,0 +1,152 @@ +import { isValidNonWithdrawableBlsAddress, isValidWithdrawableBlsAddress } from "@stakingbrain/common"; +import type { BeaconchainApi } from "../apiClients/index.js"; +import type { BrainDataBase } from "../db/index.js"; +import { storedPubkeys } from "../db/utils.js"; +import logger from "../logger/index.js"; + +/** Beacon lookups per request; large key sets need only a few requests per refresh. */ +export const BEACON_CHUNK_SIZE = 500; +/** A slow or syncing beacon node must not hold up a refresh for long. */ +export const BEACON_TIMEOUT_MS = 30_000; + +export type WithdrawalCredentialsFormat = "ecdsa" | "bls" | "unknown" | "error"; + +export interface BeaconStatusView { + index?: string; + withdrawalCredentials: { format: WithdrawalCredentialsFormat; address: string }; + /** Not looked up yet; the Brain pushes validatorsChanged when the lookup completes. */ + beaconPending: boolean; + /** Why the lookup failed, while no earlier result is available. */ + beaconError?: string; +} + +interface Entry { + /** null: unknown to the beacon chain (e.g. deposit not processed yet). */ + beacon?: { index: string; withdrawalCredentials: string } | null; + error?: string; +} + +const key = (pubkey: string) => pubkey.toLowerCase(); + +/** + * Beacon chain details (index, withdrawal credentials) for every validator, fetched with bulk requests + * by its own periodic job, so neither listing validators nor reconciling clients waits on the beacon + * node. Validators added to the database are looked up as soon as the change is announced, and every + * change here is announced in turn so open UIs refresh. Indexes are persisted for the Brain API. + */ +export class BeaconStatusStore { + private entries = new Map(); + private queue?: Set | "all"; + private run?: Promise; + + constructor( + private readonly brainDb: BrainDataBase, + private readonly beaconchainApi: BeaconchainApi + ) { + brainDb.onChange(() => { + // Only keys never attempted: failed lookups wait for the periodic job, so a failure that is + // itself announced cannot trigger another lookup. + const unseen = Object.keys(this.brainDb.getData()).filter((pubkey) => !this.entries.has(key(pubkey))); + if (unseen.length > 0) + this.refresh(unseen).catch((e) => logger.error("Error looking up new validators on the beacon chain", e)); + }); + } + + /** + * Look up the given validators, or all of them. Calls while a lookup runs merge into one follow-up; + * the periodic job skips ticks while it is still running, so slow lookups never pile up. + */ + refresh(pubkeys?: readonly string[]): Promise { + if (pubkeys === undefined || this.queue === "all") this.queue = "all"; + else this.queue = new Set([...(this.queue ?? []), ...pubkeys.map(key)]); + this.run ??= this.drain().finally(() => (this.run = undefined)); + return this.run; + } + + /** Resolves once no lookup is running, e.g. before closing the database. */ + async whenIdle(): Promise { + while (this.run) await this.run.catch(() => undefined); + } + + view(pubkey: string): BeaconStatusView { + const entry = this.entries.get(key(pubkey)); + const address = entry?.beacon?.withdrawalCredentials ?? ""; + return { + index: entry?.beacon?.index, + withdrawalCredentials: { + address, + format: entry?.beacon + ? isValidWithdrawableBlsAddress(address) + ? "ecdsa" + : isValidNonWithdrawableBlsAddress(address) + ? "bls" + : "unknown" + : "error" + }, + beaconPending: entry === undefined, + beaconError: entry?.beacon === undefined ? entry?.error : undefined + }; + } + + private async drain(): Promise { + while (this.queue) { + const queued = this.queue; + this.queue = undefined; + const stored = storedPubkeys(this.brainDb.getData()); + // Forget validators that left the database, so a later re-import starts fresh. + for (const pubkey of this.entries.keys()) if (!stored.has(pubkey)) this.entries.delete(pubkey); + const targets = + queued === "all" ? [...stored.values()] : [...queued].flatMap((pubkey) => stored.get(pubkey) ?? []); + for (let start = 0; start < targets.length; start += BEACON_CHUNK_SIZE) + await this.lookUp(targets.slice(start, start + BEACON_CHUNK_SIZE)); + } + } + + private async lookUp(pubkeys: string[]): Promise { + let changed = false; + try { + const { data } = await this.beaconchainApi.postStateValidators({ + stateId: "head", + body: { ids: pubkeys, statuses: [] }, + timeout: BEACON_TIMEOUT_MS + }); + const found = new Map(data.map((validator) => [key(validator.validator.pubkey), validator])); + const stored = this.brainDb.getData(); + const indexUpdates: { pubkey: string; index: number }[] = []; + for (const pubkey of pubkeys) { + const validator = found.get(key(pubkey)); + const beacon = validator + ? { index: validator.index, withdrawalCredentials: validator.validator.withdrawal_credentials } + : null; + const current = this.entries.get(key(pubkey)); + const same = + current?.error === undefined && + current !== undefined && + (current.beacon === beacon || + (!!current.beacon && + !!beacon && + current.beacon.index === beacon.index && + current.beacon.withdrawalCredentials === beacon.withdrawalCredentials)); + if (!same) { + this.entries.set(key(pubkey), { beacon }); + changed = true; + } + // Persist indexes: the Brain API serves them and other tools (e.g. Lido CSM) rely on them. + if (validator && stored[pubkey] && stored[pubkey].index !== Number(validator.index)) + indexUpdates.push({ pubkey, index: Number(validator.index) }); + } + if (indexUpdates.length > 0) this.brainDb.updateValidators(indexUpdates); + } catch (e) { + logger.error("Unable to look up validators on the beacon chain", e); + const error = `Could not reach the beacon node: ${e.message}`; + // Keep earlier results; rows never looked up report the failure instead of waiting forever. + for (const pubkey of pubkeys) { + const current = this.entries.get(key(pubkey)); + if (current?.error === error) continue; + this.entries.set(key(pubkey), { beacon: current?.beacon, error }); + changed = true; + } + } + if (changed) this.brainDb.notifyChanged(); + } +} diff --git a/packages/brain/src/modules/cron/cron.ts b/packages/brain/src/modules/cron/cron.ts index 2b6af3df..cf35c6a5 100644 --- a/packages/brain/src/modules/cron/cron.ts +++ b/packages/brain/src/modules/cron/cron.ts @@ -17,21 +17,23 @@ export class CronJob { public start(): void { if (this.task === null) { logger.info(`Starting cron job with interval ${this.interval}ms`); - this.task = setInterval(async () => { - // A slow reload must not queue another copy on every interval. - if (this.running) return; - this.running = true; - try { - await this.jobFunction(); - } catch (error) { - logger.error("Error running cron job", error); - } finally { - this.running = false; - } - }, this.interval); + this.task = setInterval(() => void this.runNow(), this.interval); } else logger.warn("Task is already running. Use restart to restart the job."); } + /** Run the job now unless a run is already in progress; a slow run must not queue more copies. */ + public async runNow(): Promise { + if (this.running) return; + this.running = true; + try { + await this.jobFunction(); + } catch (error) { + logger.error("Error running cron job", error); + } finally { + this.running = false; + } + } + public stop(): void { if (this.task !== null) { logger.info("Stopping cron job."); diff --git a/packages/brain/src/modules/cron/index.ts b/packages/brain/src/modules/cron/index.ts index f55c5a80..f20e78d9 100644 --- a/packages/brain/src/modules/cron/index.ts +++ b/packages/brain/src/modules/cron/index.ts @@ -9,6 +9,7 @@ import { PrometheusApi } from "../apiClients/prometheus/index.js"; import { PostgresClient } from "../apiClients/postgres/index.js"; import { BeaconchainApi } from "../apiClients/beaconchain/index.js"; import { DappmanagerApi } from "../apiClients/dappmanager/index.js"; +import { BeaconStatusStore } from "../beaconStatus/index.js"; export const getCrons = ({ sendNotification, @@ -20,7 +21,8 @@ export const getCrons = ({ brainDb, chainConfig, beaconchainApi, - dappmanagerApi + dappmanagerApi, + beaconStatus }: { sendNotification: boolean; postgresClient: PostgresClient; @@ -32,6 +34,7 @@ export const getCrons = ({ chainConfig: ChainConfig; beaconchainApi: BeaconchainApi; dappmanagerApi: DappmanagerApi; + beaconStatus: BeaconStatusStore; }) => { const { executionClient, consensusClient, secondsPerSlot, slotsPerEpoch } = chainConfig; return { @@ -50,6 +53,9 @@ export const getCrons = ({ }), reloadValidatorsCronTask: new CronJob(60 * 1000, () => reloadValidators(signerApi, signerUrl, validatorApi, brainDb) - ) + ), + // Beacon details for listing, on their own schedule: a slow beacon node must not delay + // reconciliation, and a run still in progress makes later ticks skip rather than queue. + beaconStatusCronTask: new CronJob(60 * 1000, () => beaconStatus.refresh()) }; }; diff --git a/packages/brain/src/modules/cron/reloadValidators/deleteValidatorPubkeysNotInDb.ts b/packages/brain/src/modules/cron/reloadValidators/deleteValidatorPubkeysNotInDb.ts index fbf6b2cd..590c5b37 100644 --- a/packages/brain/src/modules/cron/reloadValidators/deleteValidatorPubkeysNotInDb.ts +++ b/packages/brain/src/modules/cron/reloadValidators/deleteValidatorPubkeysNotInDb.ts @@ -22,13 +22,13 @@ export async function deleteValidatorPubkeysNotInDB({ }); for (const [index, pubkeyToRemove] of validatorPubkeysToRemove.entries()) { - const deleteValidatorKeyStatus = deleteValidatorKeysResponse.data[index].status; - - if (deleteValidatorKeyStatus === "deleted" || deleteValidatorKeyStatus === "not_found") + // A missing result is uncertain; keep the key listed so the next reload tries again. + const result = deleteValidatorKeysResponse.data[index]; + if (result?.status === "deleted" || result?.status === "not_found") validatorPubkeys.splice(validatorPubkeys.indexOf(pubkeyToRemove), 1); else logger.error( - `${logPrefix}Error deleting pubkey ${pubkeyToRemove} from validator API: ${deleteValidatorKeyStatus} ${deleteValidatorKeysResponse.data[index].message}` + `${logPrefix}Error deleting pubkey ${pubkeyToRemove} from validator API: ${result?.status ?? "no result"} ${result?.message ?? ""}` ); } } diff --git a/packages/brain/src/modules/cron/reloadValidators/postValidatorPubkeysFromDb.ts b/packages/brain/src/modules/cron/reloadValidators/postValidatorPubkeysFromDb.ts index 9a5c4f19..0e619b2e 100644 --- a/packages/brain/src/modules/cron/reloadValidators/postValidatorPubkeysFromDb.ts +++ b/packages/brain/src/modules/cron/reloadValidators/postValidatorPubkeysFromDb.ts @@ -26,11 +26,12 @@ export async function postValidatorPubkeysFromDb({ }); for (const [index, pubkeyToAdd] of brainDbPubkeysToAdd.entries()) { - const postKeyStatus = postKeysResponse.data[index].status; - if (postKeyStatus === "imported" || postKeyStatus === "duplicate") validatorPubkeys.push(pubkeyToAdd); + // A missing result is uncertain; leave the key out so the next reload tries again. + const result = postKeysResponse.data[index]; + if (result?.status === "imported" || result?.status === "duplicate") validatorPubkeys.push(pubkeyToAdd); else logger.error( - `${logPrefix}Error adding pubkey ${pubkeyToAdd} to validator API: ${postKeyStatus} ${postKeysResponse.data[index].message}` + `${logPrefix}Error adding pubkey ${pubkeyToAdd} to validator API: ${result?.status ?? "no result"} ${result?.message ?? ""}` ); } } diff --git a/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts b/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts index d5e6162a..6fc9c7f5 100644 --- a/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts +++ b/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts @@ -1,12 +1,10 @@ import { ValidatorApi } from "../../apiClients/index.js"; import { StakingBrainDb } from "../../db/types.js"; import logger from "../../logger/index.js"; -import { forEachConcurrently, VALIDATOR_CLIENT_CONCURRENCY } from "../../utils/forEachConcurrently.js"; import { logPrefix } from "./logPrefix.js"; /** - * Post in the validator API fee recipients that are in the DB in the validator API. - * Bounded concurrency keeps the reload (and the validator lock it holds) short for large key sets. + * Post in the validator API fee recipients that are in the DB in the validator API */ export async function postValidatorsFeeRecipientsFromDb({ validatorApi, @@ -15,13 +13,13 @@ export async function postValidatorsFeeRecipientsFromDb({ validatorApi: ValidatorApi; dbData: StakingBrainDb; }): Promise { - await forEachConcurrently(Object.entries(dbData), VALIDATOR_CLIENT_CONCURRENCY, ([pubkey, { feeRecipient }]) => - validatorApi + for (const [pubkey, { feeRecipient }] of Object.entries(dbData)) { + await validatorApi .setFeeRecipient(feeRecipient, pubkey) .catch((e) => logger.error( `${logPrefix}Error posting fee recipient ${feeRecipient} for pubkey ${pubkey} to validator API: ${e}` ) - ) - ); + ); + } } diff --git a/packages/brain/src/modules/utils/forEachConcurrently.ts b/packages/brain/src/modules/utils/forEachConcurrently.ts index 7a9eee83..09777326 100644 --- a/packages/brain/src/modules/utils/forEachConcurrently.ts +++ b/packages/brain/src/modules/utils/forEachConcurrently.ts @@ -14,6 +14,3 @@ export async function forEachConcurrently( }) ); } - -/** Keeps per-validator client calls fast for large batches without flooding the validator client. */ -export const VALIDATOR_CLIENT_CONCURRENCY = 8; diff --git a/packages/brain/test/e2e/fakeClients.ts b/packages/brain/test/e2e/fakeClients.ts index 6a49ae6d..278bd475 100644 --- a/packages/brain/test/e2e/fakeClients.ts +++ b/packages/brain/test/e2e/fakeClients.ts @@ -87,12 +87,18 @@ export class FakeValidatorClient { paths: string[] = []; /** Simulated latency of a busy validator client on every write. */ writeDelayMs = 0; + /** Answer every request with 503, as a restarting client would. */ + unavailable = false; async start(): Promise { const app = express(); app.use(express.json({ limit: "64mb" })); - app.use(async (req, _res, next) => { + app.use(async (req, res, next) => { this.paths.push(`${req.method} ${req.path}`); + if (this.unavailable) { + res.status(503).json({ message: "Validator client is restarting" }); + return; + } if (req.method !== "GET") await delay(this.writeDelayMs); next(); }); @@ -140,9 +146,40 @@ export class FakeBeaconNode { /** Simulated per-validator lookup latency of a busy or syncing beacon node. */ lookupDelayMs = 0; lookups = 0; + /** Simulated latency of one bulk lookup, whatever its size. */ + bulkDelayMs = 0; + bulkLookups: number[] = []; + unavailable = false; async start(): Promise { const app = express(); + app.use(express.json({ limit: "64mb" })); + app.post("/eth/v1/beacon/states/head/validators", async (req, res) => { + const ids = (req.body.ids as string[]).map(prefixed); + this.bulkLookups.push(ids.length); + await delay(this.bulkDelayMs); + if (this.unavailable) { + res.status(503).json({ message: "Beacon node is syncing" }); + return; + } + res.json({ + execution_optimistic: false, + finalized: false, + data: ids.flatMap((pubkey) => { + const index = this.indexes.get(pubkey); + return index === undefined + ? [] + : [ + { + index: String(index), + balance: "32000000000", + status: "active_ongoing", + validator: { pubkey, withdrawal_credentials: "0x01" + "00".repeat(11) + "ab".repeat(20) } + } + ]; + }) + }); + }); app.get("/eth/v1/beacon/states/head/validators/:pubkey", async (req, res) => { this.lookups++; // Stop early when the Brain aborts the request at its deadline. diff --git a/packages/brain/test/e2e/largeBatches.e2e.test.ts b/packages/brain/test/e2e/largeBatches.e2e.test.ts index 6797d682..d6b86165 100644 --- a/packages/brain/test/e2e/largeBatches.e2e.test.ts +++ b/packages/brain/test/e2e/largeBatches.e2e.test.ts @@ -24,8 +24,8 @@ import { runImportBatches, summarizeImport } from "../../../ui/src/utils/importBatches.js"; -import { countPending, MAX_STALLED_REFRESHES, nextStallCount } from "../../../ui/src/utils/validatorRefresh.js"; import type { CustomImportRequest } from "../../src/modules/apiServers/ui/calls/types.js"; +import { BeaconStatusStore } from "../../src/modules/beaconStatus/index.js"; import { FakeBeaconNode, FakeSigner, FakeValidatorClient } from "./fakeClients.js"; const feeRecipient = "0x" + "ab".repeat(20); @@ -69,6 +69,7 @@ describe("E2E: large validator batches through the UI socket", function () { let signerUrl: string; let brainDb: BrainDataBase; let cron: CronJob; + let beaconStatus: BeaconStatusStore; let server: http.Server; let sockets: Socket[]; let client: RpcClient; @@ -116,6 +117,8 @@ describe("E2E: large validator batches through the UI socket", function () { brainDb = new BrainDataBase(path.join(directory, "brain-db.json")); await brainDb.initialize(signerApi, validatorApi); + // As in production: reconciliation and beacon details are separate jobs. + beaconStatus = new BeaconStatusStore(brainDb, beaconchainApi); cron = new CronJob(60_000, () => reloadValidators(signerApi, signerUrl, validatorApi, brainDb)); cron.start(); @@ -135,6 +138,7 @@ describe("E2E: large validator batches through the UI socket", function () { } as unknown as BrainConfig, reloadValidatorsCronTask: cron, allowedOriginsFromEnv: null, + beaconStatus, port: 0 }); if (!server.listening) await once(server, "listening"); @@ -145,6 +149,7 @@ describe("E2E: large validator batches through the UI socket", function () { afterEach(async () => { for (const socket of sockets) socket.disconnect(); cron.stop(); + await beaconStatus.whenIdle(); server.closeAllConnections(); await new Promise((resolve) => server.close(resolve)); await Promise.all(closeFakes.map((close) => close())); @@ -243,40 +248,105 @@ describe("E2E: large validator batches through the UI socket", function () { latest = await watcher.client.call("getValidators", undefined); }); }); - const pubkeys = Array.from({ length: 10 }, (_, index) => pubkeyAt(index)); + // Writes are sequential, so a few keys are enough to outlast the notification debounce. + const pubkeys = Array.from({ length: 4 }, (_, index) => pubkeyAt(index)); await importAll(importRequest(pubkeys), pubkeys); // Let the last push and its refresh land. await new Promise((resolve) => setTimeout(resolve, 1000)); await refreshes; - expect(latest).to.have.length(10); + expect(latest).to.have.length(4); expect(latest.every(({ validatorImported }) => validatorImported === true)).to.equal(true); expect(latest.every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect === true)).to.equal(true); }); - it("loads a slow list progressively until every row is complete", async () => { - const pubkeys = Array.from({ length: 70 }, (_, index) => pubkeyAt(index)); - await importAll(importRequest(pubkeys), pubkeys); + it("lists 600 validators at once and fills in beacon data with a push, without polling", async () => { + const pubkeys = Array.from({ length: 600 }, (_, index) => pubkeyAt(index)); pubkeys.forEach((pubkey, index) => beacon.indexes.set(pubkey, 1000 + index)); - // 8 parallel lookups of 1.5s cannot cover 70 validators inside the 10s list deadline. - beacon.lookupDelayMs = 1500; + // A slow beacon node must not slow the import or the list down. + beacon.bulkDelayMs = 1500; + const watcher = await connect(); + // Behave like the validator list: refresh on every push and keep the latest rows. + let latest: Awaited>> = []; + let refreshes = Promise.resolve(); + watcher.client.on(VALIDATORS_CHANGED_EVENT, () => { + refreshes = refreshes.then(async () => { + latest = await watcher.client.call("getValidators", undefined); + }); + }); - // Refresh the way the validator list does until nothing is pending or progress stalls. - const pendingByRound: number[] = []; - let stalls = 0; - let rows = await client.call("getValidators", undefined); - pendingByRound.push(countPending(rows)); - while (countPending(rows) > 0 && stalls < MAX_STALLED_REFRESHES) { - rows = await client.call("getValidators", undefined); - stalls = nextStallCount(stalls, pendingByRound.at(-1), countPending(rows)); - pendingByRound.push(countPending(rows)); - } + await importAll(importRequest(pubkeys), pubkeys); + const started = Date.now(); + let rows = await watcher.client.call("getValidators", undefined); + expect(Date.now() - started).to.be.lessThan(5000); + expect(rows).to.have.length(600); + // Signer and client membership and fee recipients are read live, within one bounded request. + expect(rows.every(({ signerImported, validatorImported }) => signerImported && validatorImported)).to.equal(true); + expect(rows.every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect === true)).to.equal(true); - expect(pendingByRound[0]).to.be.greaterThan(0).and.lessThan(70); - expect(pendingByRound.at(-1)).to.equal(0); - expect(pendingByRound.length).to.be.at.most(3); + await waitFor( + () => + latest.length === 600 && latest.every(({ detailsPending, beaconPending }) => !detailsPending && !beaconPending), + 20_000 + ); + await refreshes; + rows = latest; + expect(rows.some(({ detailsPending }) => detailsPending)).to.equal(false); expect(rows.map(({ index }) => index)).to.deep.equal(pubkeys.map((_, index) => String(1000 + index))); - expect(rows.every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect === true)).to.equal(true); - expect(rows.every(({ signerImported, validatorImported }) => signerImported && validatorImported)).to.equal(true); + expect(rows.every(({ withdrawalCredentials }) => withdrawalCredentials.format === "ecdsa")).to.equal(true); + // Lookups for the 30 import batches merge, and each covers many keys at once. + expect(beacon.bulkLookups.length).to.be.lessThan(10); + expect(beacon.lookups).to.equal(0); + // Indexes are stored, so the Brain API and the explorer link have them. + expect(Object.values(brainDb.getData()).every(({ index }) => index !== undefined)).to.equal(true); + }); + + it("reports unreachable clients as errors instead of loading forever, then recovers", async () => { + const pubkeys = Array.from({ length: 5 }, (_, index) => pubkeyAt(index)); + pubkeys.forEach((pubkey, index) => beacon.indexes.set(pubkey, index)); + beacon.unavailable = true; + await importAll(importRequest(pubkeys), pubkeys); + await beaconStatus.refresh(); + let rows = await client.call("getValidators", undefined); + expect(rows.every(({ detailsPending, beaconPending }) => !detailsPending && !beaconPending)).to.equal(true); + expect(rows[0].detailsError).to.include("beacon node"); + + // A validator client outage after a successful check is reported, not hidden by the earlier result. + validator.unavailable = true; + rows = await client.call("getValidators", undefined); + expect( + rows.every( + ({ validatorImported, validatorFeeRecipientCorrect }) => + validatorImported === undefined && validatorFeeRecipientCorrect === undefined + ) + ).to.equal(true); + expect(rows[0].detailsError).to.include("validator client"); + + validator.unavailable = false; + beacon.unavailable = false; + await beaconStatus.refresh(); + rows = await client.call("getValidators", undefined); + expect( + rows.every( + ({ detailsError, detailsPending, beaconPending }) => !detailsError && !detailsPending && !beaconPending + ) + ).to.equal(true); + }); + + it("repairs a validator client that lost its keys and shows the repair", async () => { + const pubkeys = Array.from({ length: 20 }, (_, index) => pubkeyAt(index)); + await importAll(importRequest(pubkeys), pubkeys); + // The validator client restarted without its remote keys or fee recipients. + validator.remoteKeys.clear(); + validator.feeRecipients.clear(); + await reloadValidators(signerApi, signerUrl, validatorApi, brainDb); + expect(validator.remoteKeys.size).to.equal(20); + expect(validator.feeRecipients.size).to.equal(20); + const rows = await client.call("getValidators", undefined); + expect( + rows.every( + ({ validatorImported, validatorFeeRecipientCorrect }) => validatorImported && validatorFeeRecipientCorrect + ) + ).to.equal(true); }); it("keeps builder configuration switched off end to end", async () => { diff --git a/packages/brain/test/helpers/beacon.ts b/packages/brain/test/helpers/beacon.ts new file mode 100644 index 00000000..24554fc9 --- /dev/null +++ b/packages/brain/test/helpers/beacon.ts @@ -0,0 +1,7 @@ +import type { BeaconchainApi } from "../../src/modules/apiClients/index.js"; + +/** A beacon node that knows none of the validators, for tests that do not exercise beacon data. */ +export const emptyBeacon = (): BeaconchainApi => + ({ + postStateValidators: async () => ({ execution_optimistic: false, finalized: true, data: [] }) + }) as unknown as BeaconchainApi; diff --git a/packages/brain/test/unit/modules/beaconStatus.unit.test.ts b/packages/brain/test/unit/modules/beaconStatus.unit.test.ts new file mode 100644 index 00000000..4835007b --- /dev/null +++ b/packages/brain/test/unit/modules/beaconStatus.unit.test.ts @@ -0,0 +1,130 @@ +import { expect } from "chai"; +import sinon from "sinon"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { BrainDataBase } from "../../../src/modules/db/index.js"; +import type { BeaconchainApi } from "../../../src/modules/apiClients/index.js"; +import { BEACON_CHUNK_SIZE, BeaconStatusStore } from "../../../src/modules/beaconStatus/index.js"; +import logger from "../../../src/modules/logger/index.js"; + +const pubkeyAt = (index: number) => "0x" + (index + 1).toString(16).padStart(96, "0"); +const credentials = "0x01" + "00".repeat(11) + "cd".repeat(20); +const response = (pubkeys: string[]) => ({ + execution_optimistic: false, + finalized: true, + data: pubkeys.map((pubkey) => ({ + index: String(parseInt(pubkey.slice(-6), 16) + 1000), + status: "active_ongoing", + validator: { pubkey, withdrawal_credentials: credentials } + })) +}); + +describe("Beacon details store", () => { + let directory: string; + let brainDb: BrainDataBase; + let beacon: { postStateValidators: sinon.SinonStub }; + let store: BeaconStatusStore; + let notifications: sinon.SinonSpy; + const keys = Array.from({ length: 3 }, (_, index) => pubkeyAt(index)); + const addKeys = (pubkeys: string[]) => + brainDb.addValidators({ + validators: Object.fromEntries( + pubkeys.map((key) => [key, { tag: "solo", feeRecipient: "0x" + "ab".repeat(20), automaticImport: false }]) + ) + }); + const settle = () => new Promise((resolve) => setImmediate(resolve)); + + beforeEach(() => { + sinon.stub(logger, "error"); + sinon.stub(logger, "warn"); + directory = fs.mkdtempSync(path.join(os.tmpdir(), "stakingbrain-beacon-")); + brainDb = new BrainDataBase(path.join(directory, "db.json")); + addKeys(keys); + beacon = { postStateValidators: sinon.stub().callsFake(async ({ body }) => response(body.ids)) }; + store = new BeaconStatusStore(brainDb, beacon as unknown as BeaconchainApi); + notifications = sinon.spy(); + brainDb.onChange(notifications); + }); + afterEach(() => { + sinon.restore(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it("reports validators as pending until looked up, then their index and credentials", async () => { + expect(keys.map((key) => store.view(key).beaconPending)).to.deep.equal([true, true, true]); + await store.refresh(); + const view = store.view(keys[0]); + expect(view).to.deep.include({ index: "1001", beaconPending: false, beaconError: undefined }); + expect(view.withdrawalCredentials).to.deep.equal({ address: credentials, format: "ecdsa" }); + expect(notifications.called).to.equal(true); + }); + + it("persists indexes so the Brain API can serve them", async () => { + await store.refresh(); + expect(Object.values(brainDb.getData()).map(({ index }) => index)).to.deep.equal([1001, 1002, 1003]); + }); + + it("looks up large key sets in chunks", async () => { + addKeys(Array.from({ length: 1200 }, (_, index) => pubkeyAt(index + 10))); + await settle(); + beacon.postStateValidators.resetHistory(); + await store.refresh(); + expect(beacon.postStateValidators.getCalls().map(({ args }) => args[0].body.ids.length)).to.deep.equal([ + BEACON_CHUNK_SIZE, + BEACON_CHUNK_SIZE, + 1203 - 2 * BEACON_CHUNK_SIZE + ]); + }); + + it("treats validators unknown to the beacon chain as complete", async () => { + beacon.postStateValidators.callsFake(async ({ body }) => response(body.ids.slice(1))); + await store.refresh(); + expect(store.view(keys[0])).to.deep.include({ index: undefined, beaconPending: false, beaconError: undefined }); + expect(store.view(keys[0]).withdrawalCredentials.format).to.equal("error"); + }); + + it("reports a failed lookup, keeps earlier results, and recovers", async () => { + beacon.postStateValidators.rejects(new Error("socket hang up")); + await store.refresh(); + expect(store.view(keys[0])).to.deep.include({ beaconPending: false }); + expect(store.view(keys[0]).beaconError).to.include("socket hang up"); + + beacon.postStateValidators.callsFake(async ({ body }) => response(body.ids)); + await store.refresh(); + expect(store.view(keys[0])).to.deep.include({ index: "1001", beaconError: undefined }); + + beacon.postStateValidators.rejects(new Error("socket hang up")); + await store.refresh(); + expect(store.view(keys[0])).to.deep.include({ index: "1001", beaconError: undefined }); + }); + + it("looks up validators added to the database, without looping on failures", async () => { + await store.refresh(); + beacon.postStateValidators.resetHistory(); + beacon.postStateValidators.rejects(new Error("syncing")); + addKeys([pubkeyAt(50)]); + await settle(); + await settle(); + // One lookup for the new key; its announced failure must not trigger another. + expect(beacon.postStateValidators.getCalls().map(({ args }) => args[0].body.ids)).to.deep.equal([[pubkeyAt(50)]]); + expect(store.view(pubkeyAt(50)).beaconError).to.include("syncing"); + }); + + it("merges overlapping refreshes into one follow-up", async () => { + let release!: () => void; + beacon.postStateValidators.onFirstCall().callsFake( + ({ body }) => + new Promise((resolve) => { + release = () => resolve(response(body.ids)); + }) + ); + const runs = [store.refresh([keys[0]]), store.refresh([keys[1]]), store.refresh([keys[2]])]; + release(); + await Promise.all(runs); + expect(beacon.postStateValidators.getCalls().map(({ args }) => args[0].body.ids)).to.deep.equal([ + [keys[0]], + [keys[1], keys[2]] + ]); + }); +}); diff --git a/packages/brain/test/unit/modules/builderConfig/builderConfig.unit.test.ts b/packages/brain/test/unit/modules/builderConfig/builderConfig.unit.test.ts index d1a27008..98bb20a1 100644 --- a/packages/brain/test/unit/modules/builderConfig/builderConfig.unit.test.ts +++ b/packages/brain/test/unit/modules/builderConfig/builderConfig.unit.test.ts @@ -25,6 +25,8 @@ import { createRpcMethods } from "../../../../src/modules/apiServers/ui/calls/in import { setActiveValidatorsLoadedInBrain } from "../../../../src/modules/cron/trackValidatorsPerformance/setActiveValidatorsLoadedInBrain.js"; import { ValidatorStatus } from "../../../../src/modules/apiClients/types.js"; import logger from "../../../../src/modules/logger/index.js"; +import { BeaconStatusStore } from "../../../../src/modules/beaconStatus/index.js"; +import { emptyBeacon } from "../../../helpers/beacon.js"; function deferred(): { promise: Promise; resolve: () => void } { let resolve!: () => void; @@ -557,13 +559,11 @@ describe("Builder configuration management", () => { signer.listRemoteKeys.resolves({ data: keys.map((validating_pubkey) => ({ validating_pubkey, derivation_path: "", readonly: false })) }); - const beacon = sinon.createStubInstance(BeaconchainApi); - beacon.getStateValidator.rejects(new Error("not deposited")); const result = await getValidators({ brainDb: db, validatorApi: validator, signerApi: signer, - beaconchainApi: beacon + beaconStatus: new BeaconStatusStore(db, emptyBeacon()) }); expect(result.map(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect)).to.deep.equal([ true, @@ -618,9 +618,12 @@ describe("Builder configuration while disabled", () => { expect(api.calls.filter(({ endpoint }) => endpoint.includes("builder_config"))).to.deep.equal([]); expect(db.getData()[keys[0]].builderConfig).to.deep.equal(config); - const beacon = sinon.createStubInstance(BeaconchainApi); - beacon.getStateValidator.rejects(new Error("not deposited")); - const [row] = await getValidators({ brainDb: db, validatorApi: api, signerApi: signer, beaconchainApi: beacon }); + const [row] = await getValidators({ + brainDb: db, + validatorApi: api, + signerApi: signer, + beaconStatus: new BeaconStatusStore(db, emptyBeacon()) + }); expect(row).to.not.have.property("builderConfig"); expect(row).to.not.have.property("builderConfigStatus"); }); @@ -633,7 +636,8 @@ describe("Builder configuration while disabled", () => { signerApi: {} as never, brainDb: db, reloadValidatorsCronTask: {} as never, - brainConfig: { apis: {}, chain: { network: Network.Hoodi } } as never + brainConfig: { apis: {}, chain: { network: Network.Hoodi } } as never, + beaconStatus: new BeaconStatusStore(db, emptyBeacon()) }); for (const call of [() => methods.getBuilderRelayPresets(), () => methods.getBuilderConfigs([keys[0]])]) expect(await call().catch((e: Error) => e)).to.be.instanceOf(FeatureDisabledError); diff --git a/packages/brain/test/unit/modules/exitValidators.unit.test.ts b/packages/brain/test/unit/modules/exitValidators.unit.test.ts new file mode 100644 index 00000000..0c13df28 --- /dev/null +++ b/packages/brain/test/unit/modules/exitValidators.unit.test.ts @@ -0,0 +1,67 @@ +import { expect } from "chai"; +import sinon from "sinon"; +import { BeaconchainApi, Web3SignerApi } from "../../../src/modules/apiClients/index.js"; +import { getExitValidators } from "../../../src/modules/apiServers/ui/calls/exitValidators.js"; +import { BEACON_CHUNK_SIZE } from "../../../src/modules/beaconStatus/index.js"; +import logger from "../../../src/modules/logger/index.js"; + +const pubkeyAt = (index: number) => "0x" + (index + 1).toString(16).padStart(96, "0"); + +describe("Validator exits for large selections", () => { + let beacon: sinon.SinonStubbedInstance; + let signer: sinon.SinonStubbedInstance; + beforeEach(() => { + sinon.stub(logger, "debug"); + beacon = sinon.createStubInstance(BeaconchainApi); + signer = sinon.createStubInstance(Web3SignerApi); + beacon.getEpochHeader.resolves(10); + beacon.getForkFromState.resolves({ + data: { previous_version: "0x00", current_version: "0x01", epoch: "1" } + } as never); + beacon.getGenesis.resolves({ + data: { genesis_time: "0", genesis_validators_root: "0x" + "00".repeat(32), genesis_fork_version: "0x00" } + } as never); + signer.signVoluntaryExit.callsFake(async ({ pubkey }) => ({ signature: `sig-${pubkey}` })); + }); + afterEach(() => sinon.restore()); + + it("looks up indexes in bulk chunks and keeps the selection order", async () => { + const pubkeys = Array.from({ length: 1200 }, (_, index) => pubkeyAt(index)); + beacon.postStateValidators.callsFake( + async ({ body }) => + ({ + execution_optimistic: false, + finalized: true, + // The beacon API does not promise any order. + data: [...body.ids].reverse().map((pubkey) => ({ + index: String(pubkeys.indexOf(pubkey)), + balance: "0", + status: "active_ongoing", + validator: { pubkey: pubkey.toUpperCase().replace("0X", "0x") } + })) + }) as never + ); + const exits = await getExitValidators({ beaconchainApi: beacon, signerApi: signer, pubkeys }); + expect(beacon.postStateValidators.getCalls().map(({ args }) => args[0].body.ids.length)).to.deep.equal([ + BEACON_CHUNK_SIZE, + BEACON_CHUNK_SIZE, + 200 + ]); + expect(beacon.getStateValidator.called).to.equal(false); + // Signing follows the selection order, whatever order the beacon node answered in. + expect(signer.signVoluntaryExit.getCalls().map(({ args }) => args[0].pubkey)).to.deep.equal(pubkeys); + expect(exits.map(({ message }) => message.validator_index)).to.deep.equal(pubkeys.map((_, index) => String(index))); + }); + + it("refuses to exit when a validator is unknown to the beacon chain", async () => { + const pubkeys = [pubkeyAt(0), pubkeyAt(1)]; + beacon.postStateValidators.resolves({ + execution_optimistic: false, + finalized: true, + data: [{ index: "0", validator: { pubkey: pubkeys[0] } }] + } as never); + const error = await getExitValidators({ beaconchainApi: beacon, signerApi: signer, pubkeys }).catch((e) => e); + expect(error).to.be.instanceOf(Error).and.have.property("message").that.includes(pubkeys[1]); + expect(signer.signVoluntaryExit.called).to.equal(false); + }); +}); diff --git a/packages/brain/test/unit/modules/jsonBodyEncoding.unit.test.ts b/packages/brain/test/unit/modules/jsonBodyEncoding.unit.test.ts index 906ff49e..b4a8e00c 100644 --- a/packages/brain/test/unit/modules/jsonBodyEncoding.unit.test.ts +++ b/packages/brain/test/unit/modules/jsonBodyEncoding.unit.test.ts @@ -15,6 +15,7 @@ import { BeaconchainApi, PostgresClient, ValidatorApi, Web3SignerApi } from "../ import { BrainDataBase } from "../../../src/modules/db/index.js"; import { CronJob } from "../../../src/modules/cron/cron.js"; import { params } from "../../../src/params.js"; +import { BeaconStatusStore } from "../../../src/modules/beaconStatus/index.js"; function postJson(server: http.Server, body: Buffer, encoding?: string): Promise { const address = server.address() as AddressInfo; @@ -57,6 +58,7 @@ describe("JSON request content encoding", () => { const reloadValidatorsCronTask = sinon.createStubInstance(CronJob); const signerUrl = "http://127.0.0.1:1"; const allowedOriginsFromEnv = null; + const beaconStatus = new BeaconStatusStore(brainDb, beaconchainApi); servers.set("Brain", startBrainApi({ brainDb, allowedOriginsFromEnv })); servers.set( "Launchpad", @@ -64,7 +66,6 @@ describe("JSON request content encoding", () => { brainDb, validatorApi, signerApi, - beaconchainApi, reloadValidatorsCronTask, signerUrl, network: Network.Hoodi, @@ -82,6 +83,7 @@ describe("JSON request content encoding", () => { postgresClient: sinon.createStubInstance(PostgresClient), uiBuildPath: directory, allowedOriginsFromEnv, + beaconStatus, brainConfig: { chain: { network: Network.Hoodi, diff --git a/packages/brain/test/unit/modules/validatorList.unit.test.ts b/packages/brain/test/unit/modules/validatorList.unit.test.ts index bb4de33a..af51020a 100644 --- a/packages/brain/test/unit/modules/validatorList.unit.test.ts +++ b/packages/brain/test/unit/modules/validatorList.unit.test.ts @@ -4,40 +4,28 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { BrainDataBase } from "../../../src/modules/db/index.js"; -import { ValidatorApi, Web3SignerApi, BeaconchainApi } from "../../../src/modules/apiClients/index.js"; -import { getValidators } from "../../../src/modules/apiServers/ui/calls/getValidators.js"; +import { ValidatorApi, Web3SignerApi } from "../../../src/modules/apiClients/index.js"; +import { getValidators, LIST_DEADLINE_MS } from "../../../src/modules/apiServers/ui/calls/getValidators.js"; +import { BeaconStatusStore } from "../../../src/modules/beaconStatus/index.js"; import logger from "../../../src/modules/logger/index.js"; -import type { BeaconchainValidatorFromStateGetResponse } from "../../../src/modules/apiClients/beaconchain/types.js"; -import { ValidatorStatus } from "../../../src/modules/apiClients/beaconchain/types.js"; +import { emptyBeacon } from "../../helpers/beacon.js"; const keys = Array.from({ length: 30 }, (_, index) => "0x" + (index + 1).toString(16).padStart(96, "0")); const feeRecipient = "0x" + "ab".repeat(20); -const beaconResponse = (pubkey: string, address: string): BeaconchainValidatorFromStateGetResponse => ({ - execution_optimistic: false, - data: { - index: String(keys.indexOf(pubkey)), - balance: "32000000000", - status: ValidatorStatus.ACTIVE_ONGOING, - validator: { - pubkey, - withdrawal_credentials: address, - effective_balance: "32000000000", - slashed: false, - activation_eligibility_epoch: "0", - activation_epoch: "0", - exit_epoch: "18446744073709551615", - withdrawable_epoch: "18446744073709551615" - } - } -}); +const hang = ({ signal }: { signal?: AbortSignal }) => + new Promise((_resolve, reject) => { + signal!.addEventListener("abort", () => reject(new Error("Aborted")), { once: true }); + }); -describe("Validator list bulk reads", () => { +describe("Validator list reads", () => { let directory: string; let brainDb: BrainDataBase; let validatorApi: sinon.SinonStubbedInstance; let signerApi: sinon.SinonStubbedInstance; - let beaconchainApi: sinon.SinonStubbedInstance; + let beaconStatus: BeaconStatusStore; + beforeEach(() => { + sinon.stub(logger, "error"); directory = fs.mkdtempSync(path.join(os.tmpdir(), "stakingbrain-list-")); brainDb = new BrainDataBase(path.join(directory, "db.json")); brainDb.addValidators({ @@ -45,169 +33,159 @@ describe("Validator list bulk reads", () => { }); validatorApi = sinon.createStubInstance(ValidatorApi); signerApi = sinon.createStubInstance(Web3SignerApi); - beaconchainApi = sinon.createStubInstance(BeaconchainApi); - validatorApi.getRemoteKeys.resolves({ - data: keys.map((pubkey) => ({ pubkey, readonly: false, url: "http://signer" })) - }); + beaconStatus = new BeaconStatusStore(brainDb, emptyBeacon()); + validatorApi.getRemoteKeys.resolves({ data: keys.map((pubkey) => ({ pubkey, readonly: false, url: "" })) }); signerApi.listRemoteKeys.resolves({ data: keys.map((validating_pubkey) => ({ validating_pubkey, derivation_path: "", readonly: false })) }); - validatorApi.getFeeRecipient.resolves({ data: { ethaddress: feeRecipient } }); - sinon.stub(logger, "error"); + validatorApi.getFeeRecipient.resolves({ data: { ethaddress: feeRecipient.toUpperCase().replace("0X", "0x") } }); }); afterEach(() => { sinon.restore(); fs.rmSync(directory, { recursive: true, force: true }); }); - const read = () => getValidators({ brainDb, validatorApi, signerApi, beaconchainApi }); + const list = () => getValidators({ brainDb, validatorApi, signerApi, beaconStatus }); - it("keeps successful fee checks when a different validator fails", async () => { - validatorApi.getFeeRecipient.withArgs(keys[0]).rejects(new Error("key missing")); - beaconchainApi.getStateValidator.rejects(new Error("not deposited")); - const results = await read(); - expect(results.map(({ pubkey }) => pubkey)).to.deep.equal(keys); - expect(results[0].validatorFeeRecipientCorrect).to.equal(undefined); - expect(results.slice(1).every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect)).to.equal(true); + it("reads membership and fee recipients from the clients", async () => { + validatorApi.getFeeRecipient.withArgs(keys[1]).resolves({ data: { ethaddress: "0x" + "00".repeat(20) } }); + signerApi.listRemoteKeys.resolves({ data: [] }); + const rows = await list(); + expect(rows.every(({ signerImported }) => signerImported === false)).to.equal(true); + expect(rows.every(({ validatorImported }) => validatorImported)).to.equal(true); + expect(rows.map(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect).slice(0, 3)).to.deep.equal([ + true, + false, + true + ]); + expect(rows.every(({ detailsPending, detailsError }) => !detailsPending && !detailsError)).to.equal(true); }); - it("bounds concurrent beacon reads and stops the whole list at its deadline", async () => { + it("bounds reads by concurrency and deadline and reports cut-off rows as pending", async () => { const clock = sinon.useFakeTimers(); - beaconchainApi.getStateValidator.callsFake( - async ({ signal }) => - new Promise((_resolve, reject) => { - if (signal?.aborted) reject(new Error("Aborted")); - signal!.addEventListener("abort", () => reject(new Error("Aborted")), { once: true }); - }) - ); - const pending = read(); - await clock.tickAsync(9999); - expect(beaconchainApi.getStateValidator.callCount).to.equal(8); + validatorApi.getFeeRecipient.callsFake(((_pubkey: string, signal?: AbortSignal) => hang({ signal })) as never); + const rows = list(); + await clock.tickAsync(LIST_DEADLINE_MS - 1); + expect(validatorApi.getFeeRecipient.callCount).to.equal(8); await clock.tickAsync(1); - const results = await pending; - expect(results.map(({ pubkey }) => pubkey)).to.deep.equal(keys); - expect(results.every(({ withdrawalCredentials }) => withdrawalCredentials.format === "error")).to.equal(true); - expect(beaconchainApi.getStateValidator.callCount).to.equal(8); + const result = await rows; + expect(result.every(({ detailsPending, detailsError }) => detailsPending && !detailsError)).to.equal(true); expect(clock.countTimers()).to.equal(0); }); - it("finishes later beacon reads on refresh and retains earlier results without delaying fee checks", async () => { + it("converges over follow-up requests: only unread rows are pending", async () => { const clock = sinon.useFakeTimers(); - const address = "0x01" + "00".repeat(11) + "ab".repeat(20); - beaconchainApi.getStateValidator.callsFake( - ({ pubkey, signal }) => + validatorApi.getFeeRecipient.callsFake( + ((_pubkey: string, signal?: AbortSignal) => new Promise((resolve, reject) => { - const abort = () => { + const timer = setTimeout(() => resolve({ data: { ethaddress: feeRecipient } }), 4000); + signal?.addEventListener("abort", () => { clearTimeout(timer); reject(new Error("Aborted")); - }; - const timer = setTimeout(() => { - signal!.removeEventListener("abort", abort); - resolve(beaconResponse(pubkey, address)); - }, 4000); - signal!.addEventListener("abort", abort, { once: true }); - }) - ); - const first = read(); - await clock.tickAsync(10000); - const firstResults = await first; - expect(firstResults.filter(({ index }) => index !== "").length).to.equal(16); - expect(firstResults.every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect)).to.equal(true); - const second = read(); - await clock.tickAsync(10000); - const secondResults = await second; - expect(secondResults.map(({ index }) => index)).to.deep.equal(keys.map((_, index) => String(index))); - expect(secondResults.every(({ withdrawalCredentials }) => withdrawalCredentials.address === address)).to.equal( - true + }); + })) as never ); - // Expiration forces fresh reads so changed withdrawal credentials are not cached forever. - await clock.tickAsync(60000); - beaconchainApi.getStateValidator.resetHistory(); - const third = read(); - await clock.tickAsync(10000); - await third; - expect(beaconchainApi.getStateValidator.callCount).to.be.greaterThan(0); - expect(clock.countTimers()).to.equal(0); + const pendingByRound: number[] = []; + for (let round = 0; round < 3; round++) { + const rows = list(); + await clock.tickAsync(LIST_DEADLINE_MS); + const result = await rows; + const pending = result.filter(({ detailsPending }) => detailsPending); + // Pending means exactly "not read yet". + expect(pending.every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect === undefined)).to.equal( + true + ); + pendingByRound.push(pending.length); + } + expect(pendingByRound).to.deep.equal([14, 0, 0]); }); - it("rotates stalled fee reads and keeps unverified results unknown", async () => { + it("moves past reads that time out, so responsive validators are not starved", async () => { const clock = sinon.useFakeTimers(); - beaconchainApi.getStateValidator.rejects(new Error("not deposited")); - validatorApi.getFeeRecipient.callsFake( - async (_pubkey, signal) => - new Promise((_resolve, reject) => { - signal!.addEventListener("abort", () => reject(new Error("Aborted")), { once: true }); - }) - ); - const first = read(); - await clock.tickAsync(10000); + const stuck = new Set(keys.slice(0, 8)); + validatorApi.getFeeRecipient.callsFake(((pubkey: string, signal?: AbortSignal) => + stuck.has(pubkey) ? hang({ signal }) : Promise.resolve({ data: { ethaddress: feeRecipient } })) as never); + for (let round = 0; round < 2; round++) { + const rows = list(); + await clock.tickAsync(LIST_DEADLINE_MS); + await rows; + } + const rows = list(); + await clock.tickAsync(LIST_DEADLINE_MS); + const result = await rows; expect( - (await first).every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect === undefined) + result + .slice(8) + .every(({ detailsPending, validatorFeeRecipientCorrect }) => !detailsPending && validatorFeeRecipientCorrect) ).to.equal(true); - expect(validatorApi.getFeeRecipient.args.map(([pubkey]) => pubkey)).to.deep.equal(keys.slice(0, 8)); - validatorApi.getFeeRecipient.resetHistory(); - const second = read(); - await clock.tickAsync(10000); - await second; - expect(validatorApi.getFeeRecipient.args.map(([pubkey]) => pubkey)).to.deep.equal(keys.slice(8, 16)); - expect(clock.countTimers()).to.equal(0); + expect(result.slice(0, 8).every(({ detailsPending }) => detailsPending)).to.equal(true); }); - it("keeps membership unknown on list failures while independently reading beacon data", async () => { - validatorApi.getRemoteKeys.rejects(new Error("unavailable")); - signerApi.listRemoteKeys.rejects(new Error("unavailable")); - beaconchainApi.getStateValidator.callsFake(async ({ pubkey }) => beaconResponse(pubkey, "0x01" + "00".repeat(31))); - const results = await read(); - expect( - results.every( - ({ signerImported, validatorImported, validatorFeeRecipientCorrect }) => - signerImported === undefined && validatorImported === undefined && validatorFeeRecipientCorrect === undefined - ) - ).to.equal(true); - expect(results.map(({ index }) => index)).to.deep.equal(keys.map((_, index) => String(index))); + it("forgets a cached fee check when the key leaves the validator client", async () => { + await list(); + validatorApi.getRemoteKeys.resolves({ + data: keys.slice(1).map((pubkey) => ({ pubkey, readonly: false, url: "" })) + }); + await list(); + // The key returns, but its fee recipient was not restored. + validatorApi.getRemoteKeys.resolves({ data: keys.map((pubkey) => ({ pubkey, readonly: false, url: "" })) }); + validatorApi.getFeeRecipient.withArgs(keys[0]).resolves({ data: { ethaddress: "0x" + "00".repeat(20) } }); + brainDb.notifyChanged(); + expect((await list())[0].validatorFeeRecipientCorrect).to.equal(false); }); - it("marks rows pending only when the deadline cut their reads short", async () => { - const clock = sinon.useFakeTimers(); - beaconchainApi.getStateValidator.callsFake(({ pubkey, signal }) => - // Validator 0 is not deposited yet: a definite failure must not keep the UI polling. - pubkey === keys[0] - ? Promise.reject(new Error("not deposited")) - : new Promise((resolve, reject) => { - const abort = () => { - clearTimeout(timer); - reject(new Error("Aborted")); - }; - const timer = setTimeout(() => { - signal!.removeEventListener("abort", abort); - resolve(beaconResponse(pubkey, "0x01" + "00".repeat(31))); - }, 4000); - signal!.addEventListener("abort", abort, { once: true }); - }) + + it("reports an unreachable client instead of an earlier success", async () => { + await list(); + validatorApi.getRemoteKeys.rejects(new Error("ECONNREFUSED")); + const rows = await list(); + expect(rows.every(({ validatorImported }) => validatorImported === undefined)).to.equal(true); + expect(rows.every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect === undefined)).to.equal(true); + expect(rows.every(({ detailsPending }) => !detailsPending)).to.equal(true); + expect(rows[0].detailsError).to.include("validator client").and.include("ECONNREFUSED"); + }); + + it("reports a failed fee read for that row only", async () => { + validatorApi.getFeeRecipient.withArgs(keys[0]).rejects(new Error("HTTP 404")); + const rows = await list(); + expect(rows[0].detailsPending).to.equal(false); + expect(rows[0].validatorFeeRecipientCorrect).to.equal(undefined); + expect(rows[0].detailsError).to.include("HTTP 404"); + expect(rows.slice(1).every(({ detailsError }) => !detailsError)).to.equal(true); + }); + + it("re-reads a fee recipient checked mid-update once the update announces itself", async () => { + const other = "0x" + "cd".repeat(20); + brainDb.updateValidators([{ pubkey: keys[0], feeRecipient: other }]); + let finishRead!: () => void; + validatorApi.getFeeRecipient.withArgs(keys[0]).callsFake( + () => + new Promise((resolve) => { + finishRead = () => resolve({ data: { ethaddress: feeRecipient } }); + }) as never ); - const first = read(); - await clock.tickAsync(10000); - const firstResults = await first; - expect(firstResults[0].detailsPending).to.equal(false); - expect(firstResults.some(({ detailsPending }) => detailsPending)).to.equal(true); - expect( - firstResults.filter(({ detailsPending }) => !detailsPending).every(({ index }, i) => i === 0 || index !== "") - ).to.be.true; - const second = read(); - await clock.tickAsync(10000); - expect((await second).every(({ detailsPending }) => !detailsPending)).to.equal(true); - expect(clock.countTimers()).to.equal(0); + const during = list(); + await new Promise((resolve) => setImmediate(resolve)); + validatorApi.getFeeRecipient.withArgs(keys[0]).resolves({ data: { ethaddress: other } }); + brainDb.notifyChanged(); + finishRead(); + expect((await during)[0].validatorFeeRecipientCorrect).to.equal(false); + expect((await list())[0].validatorFeeRecipientCorrect).to.equal(true); }); - it("caches fee checks per expected recipient and re-checks after an edit", async () => { - beaconchainApi.getStateValidator.rejects(new Error("not deposited")); - await read(); + it("caches matching fee checks between requests and re-checks after an edit", async () => { + await list(); expect(validatorApi.getFeeRecipient.callCount).to.equal(keys.length); - await read(); + await list(); expect(validatorApi.getFeeRecipient.callCount).to.equal(keys.length); - const other = "0x" + "cd".repeat(20); - brainDb.updateValidators([{ pubkey: keys[0], feeRecipient: other }]); - const results = await read(); + brainDb.updateValidators([{ pubkey: keys[0], feeRecipient: "0x" + "cd".repeat(20) }]); + const rows = await list(); expect(validatorApi.getFeeRecipient.callCount).to.equal(keys.length + 1); - expect(results[0].validatorFeeRecipientCorrect).to.equal(false); - expect(results.every(({ detailsPending }) => detailsPending === false)).to.equal(true); + expect(rows[0].validatorFeeRecipientCorrect).to.equal(false); + }); + + it("takes beacon details from the beacon store, pending until it has looked them up", async () => { + const rows = await list(); + expect(rows.every(({ beaconPending, detailsPending }) => beaconPending && !detailsPending)).to.equal(true); + await beaconStatus.refresh(); + expect((await list()).every(({ beaconPending }) => !beaconPending)).to.equal(true); }); }); diff --git a/packages/ui/src/components/Dialogs/DeletionWarning.tsx b/packages/ui/src/components/Dialogs/DeletionWarning.tsx index d01cf23a..3017a2d2 100644 --- a/packages/ui/src/components/Dialogs/DeletionWarning.tsx +++ b/packages/ui/src/components/Dialogs/DeletionWarning.tsx @@ -1,21 +1,14 @@ import type { JSX } from "react"; -import type { GridRowId } from "@mui/x-data-grid"; import { shortenPubkey } from "@stakingbrain/common"; import type { CustomValidatorGetResponse } from "@stakingbrain/brain"; -export default function DeletionWarning({ - selectedRows, - rows -}: { - selectedRows: GridRowId[]; - rows: CustomValidatorGetResponse[]; -}): JSX.Element { +export default function DeletionWarning({ validators }: { validators: CustomValidatorGetResponse[] }): JSX.Element { return ( <> Are you sure you want to delete these keystores?
    - {selectedRows.map((row, i) => ( -
  • {shortenPubkey(rows[parseInt(row.toString())].pubkey)}
  • + {validators.map(({ pubkey }) => ( +
  • {shortenPubkey(pubkey)}
  • ))}
After deletion, these keystores won't be used for signing anymore and your slashing protection data will be diff --git a/packages/ui/src/components/Dialogs/EditFeesDialog.tsx b/packages/ui/src/components/Dialogs/EditFeesDialog.tsx index 84adbaba..d9c30150 100644 --- a/packages/ui/src/components/Dialogs/EditFeesDialog.tsx +++ b/packages/ui/src/components/Dialogs/EditFeesDialog.tsx @@ -22,7 +22,6 @@ import { import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; import EditOutlinedIcon from "@mui/icons-material/EditOutlined"; import DialogHeader from "./DialogHeader"; -import type { GridRowId } from "@mui/x-data-grid"; import { BURN_ADDRESS, isValidEcdsaPubkey, areAllFeeRecipientsEditable, Network } from "@stakingbrain/common"; import React from "react"; import type { @@ -47,15 +46,14 @@ import { useTheme, useMediaQuery } from "@mui/material"; export default function FeeRecipientDialog({ open, setOpen, - rows, - selectedRows, + validators, network, isMevBoostSet }: { open: boolean; setOpen: (open: boolean) => void; - rows: CustomValidatorGetResponse[]; - selectedRows: GridRowId[]; + /** The selected validators, resolved by pubkey. */ + validators: CustomValidatorGetResponse[]; network: Network; isMevBoostSet: boolean; }): JSX.Element { @@ -75,7 +73,7 @@ export default function FeeRecipientDialog({ useEffect(() => { isAnyWithdrawalCredentialsDiff("ecdsa") && getNonEcdsaValidatorsData(); getSmoothValidatorsSelected(); - }, [selectedRows]); + }, [validators]); const mevSpAddress = getSmoothAddressByNetwork(network); const smoothUrl = getSmoothUrlByNetwork(network); @@ -141,9 +139,7 @@ export default function FeeRecipientDialog({ async function updateValidators() { const validatorsData: CustomValidatorUpdateRequest[] = []; - selectedRows.forEach((rowId) => { - const row = rows[+rowId]; - + validators.forEach((row) => { if (row) { validatorsData.push({ pubkey: row.pubkey, @@ -170,18 +166,18 @@ export default function FeeRecipientDialog({ }; function areAllSelectedFeeRecipientsEditable() { - const selectedTags = selectedRows.map((rowId) => rows[+rowId].tag).flat(); + const selectedTags = validators.map((row) => row.tag).flat(); return areAllFeeRecipientsEditable(selectedTags); } function areAllOldFrsSameAsGiven(givenFr: string) { - const oldFeeRecipients = selectedRows.map((rowId) => rows[+rowId].feeRecipient).flat(); + const oldFeeRecipients = validators.map((row) => row.feeRecipient).flat(); return oldFeeRecipients.every((fr) => fr === givenFr); } function isRemovingMevSpFr() { - const oldFeeRecipients = selectedRows.map((rowId) => rows[+rowId].feeRecipient).flat(); + const oldFeeRecipients = validators.map((row) => row.feeRecipient).flat(); return ( mevSpAddress !== null && @@ -198,8 +194,7 @@ export default function FeeRecipientDialog({ // Given a possible withdrawal credentials format, checks if any of the selected validators has it const isAnyWithdrawalCredentialsEqual = (givenFormat: WithdrawalCredentialsFormat): boolean => { let isAnyAsGivenFormat = false; - for (const row of selectedRows) { - const rowData = rows[+row]; + for (const rowData of validators) { const withdrawalFormat = rowData.withdrawalCredentials.format; if (withdrawalFormat === givenFormat) { isAnyAsGivenFormat = true; @@ -212,8 +207,7 @@ export default function FeeRecipientDialog({ // Given a possible withdrawal credentials format, checks if any of the selected validators has a different format const isAnyWithdrawalCredentialsDiff = (givenFormat: WithdrawalCredentialsFormat): boolean => { let isAnyAsGivenFormat = false; - for (const row of selectedRows) { - const rowData = rows[+row]; + for (const rowData of validators) { const withdrawalFormat = rowData.withdrawalCredentials.format; if (withdrawalFormat !== givenFormat) { isAnyAsGivenFormat = true; @@ -225,9 +219,8 @@ export default function FeeRecipientDialog({ // Update the smoothValidatorsPubkeys state with the selected validators that have Smooth's fee recipient const getSmoothValidatorsSelected = (): void => { - const smoothValidatorsPubkeys: string[] = selectedRows - .map((row) => { - const rowData = rows[+row]; + const smoothValidatorsPubkeys: string[] = validators + .map((rowData) => { return rowData.feeRecipient === mevSpAddress ? rowData.pubkey : null; }) .filter((pubkey): pubkey is string => pubkey !== null); @@ -237,9 +230,8 @@ export default function FeeRecipientDialog({ // Update the nonEcdsaValidatorsData state with the selected validators that have a withdrawal address format different from ecdsa const getNonEcdsaValidatorsData = (): void => { - const filteredValidators = selectedRows - .map((row) => { - const rowData = rows[+row]; + const filteredValidators = validators + .map((rowData) => { return rowData.withdrawalCredentials.format !== "ecdsa" ? { pubkey: rowData.pubkey, @@ -620,7 +612,7 @@ export default function FeeRecipientDialog({ title="Fee recipient" subtitle="Choose where validator proposal fees are sent." icon={} - count={selectedRows.length} + count={validators.length} onClose={handleClose} busy={loading} /> diff --git a/packages/ui/src/components/Dialogs/ExitWarning.tsx b/packages/ui/src/components/Dialogs/ExitWarning.tsx index 8e1e9abc..d7e96645 100644 --- a/packages/ui/src/components/Dialogs/ExitWarning.tsx +++ b/packages/ui/src/components/Dialogs/ExitWarning.tsx @@ -1,22 +1,15 @@ import type { JSX } from "react"; import { Alert } from "@mui/material"; -import type { GridRowId } from "@mui/x-data-grid"; import { shortenPubkey } from "@stakingbrain/common"; import type { CustomValidatorGetResponse } from "@stakingbrain/brain"; -export default function ExitWarning({ - selectedRows, - rows -}: { - selectedRows: GridRowId[]; - rows: CustomValidatorGetResponse[]; -}): JSX.Element { +export default function ExitWarning({ validators }: { validators: CustomValidatorGetResponse[] }): JSX.Element { return ( <> Are you sure you want to exit these validators?
    - {selectedRows.map((row, i) => ( -
  • {shortenPubkey(rows[parseInt(row.toString())].pubkey)}
  • + {validators.map(({ pubkey }) => ( +
  • {shortenPubkey(pubkey)}
  • ))}
diff --git a/packages/ui/src/components/Dialogs/KeystoresDeleteDialog.tsx b/packages/ui/src/components/Dialogs/KeystoresDeleteDialog.tsx index e23415d4..81b0345d 100644 --- a/packages/ui/src/components/Dialogs/KeystoresDeleteDialog.tsx +++ b/packages/ui/src/components/Dialogs/KeystoresDeleteDialog.tsx @@ -14,14 +14,13 @@ import type { CustomValidatorGetResponse, Web3signerDeleteResponse } from "@stak import { useTheme, useMediaQuery } from "@mui/material"; export default function KeystoresDeleteDialog({ - rows, - selectedRows, + validators, setSelectedRows, open, setOpen }: { - rows: CustomValidatorGetResponse[]; - selectedRows: GridRowId[]; + /** The selected validators, resolved by pubkey. */ + validators: CustomValidatorGetResponse[]; setSelectedRows: (selectedRows: GridRowId[]) => void; open: boolean; setOpen: (open: boolean) => void; @@ -37,7 +36,7 @@ export default function KeystoresDeleteDialog({ try { setKeystoresDelete(undefined); setLoading(true); - const pubkeysToDelete = selectedRows.map((row) => rows[parseInt(row.toString())]?.pubkey); + const pubkeysToDelete = validators.map(({ pubkey }) => pubkey); setDeletedPubkeys(pubkeysToDelete); setKeystoresDelete( await rpcClient.call("deleteValidators", { @@ -76,7 +75,7 @@ export default function KeystoresDeleteDialog({ title={keystoresDelete ? "Deletion results" : "Delete keystores"} subtitle="Remove validator keys from this signer." icon={} - count={keystoresDelete ? deletedPubkeys.length : selectedRows.length} + count={keystoresDelete ? deletedPubkeys.length : validators.length} onClose={handleClose} busy={loading} /> @@ -121,7 +120,7 @@ export default function KeystoresDeleteDialog({ ) : ( - + )} diff --git a/packages/ui/src/components/Dialogs/KeystoresExitDialog.tsx b/packages/ui/src/components/Dialogs/KeystoresExitDialog.tsx index b88c4cac..7f69c28e 100644 --- a/packages/ui/src/components/Dialogs/KeystoresExitDialog.tsx +++ b/packages/ui/src/components/Dialogs/KeystoresExitDialog.tsx @@ -24,14 +24,13 @@ import type { CustomValidatorGetResponse, ValidatorExitExecute } from "@stakingb import { useTheme, useMediaQuery } from "@mui/material"; export default function KeystoresExitDialog({ - rows, - selectedRows, + validators, setSelectedRows, open, setOpen }: { - rows: CustomValidatorGetResponse[]; - selectedRows: GridRowId[]; + /** The selected validators, resolved by pubkey. */ + validators: CustomValidatorGetResponse[]; setSelectedRows: (selectedRows: GridRowId[]) => void; open: boolean; setOpen: (open: boolean) => void; @@ -49,7 +48,7 @@ export default function KeystoresExitDialog({ try { setLoading(true); const exitKeysores = await rpcClient.call("getExitValidators", { - pubkeys: selectedRows.map((row) => rows[parseInt(row.toString())].pubkey) + pubkeys: validators.map(({ pubkey }) => pubkey) }); exitKeysores.forEach((validator) => { @@ -75,7 +74,7 @@ export default function KeystoresExitDialog({ setLoading(true); setValidatorsExitResponse( await rpcClient.call("exitValidators", { - pubkeys: selectedRows.map((row) => rows[parseInt(row.toString())].pubkey) + pubkeys: validators.map(({ pubkey }) => pubkey) }) ); setLoading(false); @@ -109,7 +108,7 @@ export default function KeystoresExitDialog({ title={validatorsExitResponse ? "Exit results" : "Exit validators"} subtitle="Review and confirm your validator exits." icon={} - count={validatorsExitResponse?.length ?? selectedRows.length} + count={validatorsExitResponse?.length ?? validators.length} onClose={handleClose} busy={loading} /> @@ -135,7 +134,7 @@ export default function KeystoresExitDialog({ ) : ( <> - +
diff --git a/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx b/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx index 4f1d3aec..ea99857b 100644 --- a/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx +++ b/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx @@ -157,6 +157,8 @@ export default function KeystoresDataGrid({ // about changes, and not on every round while details are still loading. const oracleQueryKey = rows.map((row) => `${row.pubkey}:${row.tag}:${row.index}`).join(","); const detailsLoading = rows.some((row) => row.detailsPending); + const selectedIds = new Set(selectedRows); + const selectedValidators = rows.filter((row) => selectedIds.has(row.pubkey)); // Live refreshes replace `rows` often; only reset the dashboard hint when what it depends on changes. useEffect(() => { setSummaryUrlBuildingStatus(BeaconchaUrlBuildingStatus.NotStarted); @@ -255,7 +257,7 @@ export default function KeystoresDataGrid({ } ]; - const customRows = rows.map((row, index) => ({ + const customRows = rows.map((row) => ({ pubkey: row.pubkey, explorerUrl: blockExplorerUrl ? blockExplorerUrl + "/validator/" + row.pubkey : "", feeRecipient: row.feeRecipient, @@ -267,7 +269,8 @@ export default function KeystoresDataGrid({ feeRecipientImported: row.validatorFeeRecipientCorrect, builderConfig: row.builderConfig, builderConfigStatus: row.builderConfigStatus, - id: index + // Pubkeys, not positions: live refreshes reorder and remove rows under a selection. + id: row.pubkey })); const builderConfigColumn: GridColDef = { @@ -418,7 +421,7 @@ export default function KeystoresDataGrid({ } const summaryUrl = buildValidatorSummaryURL({ - validatorIndexes: selectedRows.map((row) => rows[row as number]?.index ?? ""), + validatorIndexes: selectedValidators.map((row) => row.index), network }); @@ -432,12 +435,12 @@ export default function KeystoresDataGrid({ } function areAllSelectedRowsExitable() { - return selectedRows.every((row) => rows[row as number].tag !== "rocketpool"); + return selectedValidators.every((row) => row.tag !== "rocketpool"); } function areAllSelectedRowsUneditable() { - return selectedRows.every((row) => { - const tag = rows[row as number].tag; + return selectedValidators.every((row) => { + const tag = row.tag; return nonEditableFeeRecipientTags.includes(tag as NonEditableFeeRecipientTag); }); } diff --git a/packages/ui/src/components/ValidatorList/ValidatorList.tsx b/packages/ui/src/components/ValidatorList/ValidatorList.tsx index 7390de8c..f9a86a84 100644 --- a/packages/ui/src/components/ValidatorList/ValidatorList.tsx +++ b/packages/ui/src/components/ValidatorList/ValidatorList.tsx @@ -6,7 +6,7 @@ import BuilderConfigDialog from "../Dialogs/BuilderConfigDialog"; import { Alert, Box, Button, CircularProgress, LinearProgress, Typography } from "@mui/material"; import type { GridRowId } from "@mui/x-data-grid"; import { featureFlags, StakerConfig as StakerConfigType, VALIDATORS_CHANGED_EVENT } from "@stakingbrain/common"; -import { useEffect, useRef, useState } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; import { BeaconchaUrlBuildingStatus } from "../../types"; import { rpcClient } from "../../socket"; import StakerConfig from "../StakerConfig/StakerConfig"; @@ -14,13 +14,7 @@ import KeystoresExitDialog from "../Dialogs/KeystoresExitDialog"; import { getSmoothAddressByNetwork } from "../../utils/addresses"; import type { CustomValidatorGetResponse } from "@stakingbrain/brain"; import PageLayout from "../Layout/PageLayout"; -import { - countPending, - MAX_STALLED_REFRESHES, - nextStallCount, - PENDING_REFRESH_MS, - selectionAfterRefresh -} from "../../utils/validatorRefresh"; +import { countUnfinished, FOLLOW_UP_MS, MAX_STALLED_FOLLOW_UPS, nextStallCount } from "../../utils/listFollowUp"; export default function ValidatorList({ stakerConfig, @@ -44,24 +38,26 @@ export default function ValidatorList({ const anyDialogOpen = deleteOpen || editFeesOpen || exitOpen || builderConfigOpen; const dialogOpenRef = useRef(anyDialogOpen); dialogOpenRef.current = anyDialogOpen; - const rowsRef = useRef(undefined); + const loaded = useRef(false); const inFlight = useRef(false); const queued = useRef(false); + const [refreshError, setRefreshError] = useState(); const stalls = useRef(0); - const lastPending = useRef(undefined); + const lastUnfinished = useRef(undefined); const [stalled, setStalled] = useState(false); - // Re-render table after delete/update validators + // Refresh when a dialog closes (its action may have changed validators) and on first load. useEffect(() => { if (!anyDialogOpen) { // Only the first load blanks the page; later refreshes keep the table visible. - getValidators(rowsRef.current !== undefined); + void getValidators(loaded.current); setSelectedRows([]); } }, [anyDialogOpen]); - // Live updates: the Brain pushes a hint after every validator change, from any tab or the reload job. - // Dialogs act on row positions, so refreshes wait until they close (closing always refreshes). + // Live updates: the Brain pushes a hint whenever a validator or its client status changes, including + // when pending checks complete, so the list never needs to poll. Refreshes wait while a dialog is + // open so its content does not shift; closing a dialog always refreshes. useEffect(() => { const refresh = () => { if (!dialogOpenRef.current) void getValidators(true); @@ -75,12 +71,22 @@ export default function ValidatorList({ }; }, []); - // Large lists load in bounded rounds; keep refreshing while rows are pending and progress is made. + // Client reads cut off by the Brain's deadline only continue when the list is requested again, so keep + // requesting (one request at a time) while that makes progress. A failed request does not poll: it + // shows an error with Retry. Beacon details arrive by push and need no follow-up. useEffect(() => { - if (anyDialogOpen || stalled || !validatorsGet || countPending(validatorsGet) === 0) return; - const timer = setTimeout(() => getValidators(true), PENDING_REFRESH_MS); + if (anyDialogOpen || stalled || refreshError || !validatorsGet || countUnfinished(validatorsGet) === 0) return; + const timer = setTimeout(() => void getValidators(true), FOLLOW_UP_MS); return () => clearTimeout(timer); - }, [validatorsGet, anyDialogOpen, stalled]); + }, [validatorsGet, anyDialogOpen, stalled, refreshError]); + + function retry() { + stalls.current = 0; + lastUnfinished.current = undefined; + setStalled(false); + setRefreshError(undefined); + void getValidators(true); + } async function getValidators(background = false) { // Coalesce overlapping triggers into one follow-up refresh. @@ -92,19 +98,22 @@ export default function ValidatorList({ try { if (!background) setLoading(true); const rows = await rpcClient.call("getValidators", undefined); - // Capture the rows the current selection refers to before replacing them (see selectionAfterRefresh). - if (background) setSelectedRows(selectionAfterRefresh(rowsRef.current, rows)); - rowsRef.current = rows; - const pending = countPending(rows); - stalls.current = nextStallCount(stalls.current, lastPending.current, pending); - lastPending.current = pending; - setStalled(stalls.current >= MAX_STALLED_REFRESHES); + loaded.current = true; + // Rows are identified by pubkey, so a refresh only has to drop validators that are gone. + const present = new Set(rows.map(({ pubkey }) => pubkey)); + setSelectedRows((selected) => selected.filter((id) => present.has(String(id)))); + const unfinished = countUnfinished(rows); + stalls.current = nextStallCount(stalls.current, lastUnfinished.current, unfinished); + lastUnfinished.current = unfinished; + setStalled(stalls.current >= MAX_STALLED_FOLLOW_UPS); setValidatorsGet(rows); setValidatorsGetError(undefined); + setRefreshError(undefined); } catch (e) { console.error(e); - // A failed background refresh keeps the last good table instead of replacing it with an error. - if (!background || !rowsRef.current) setValidatorsGetError(e.message); + // A failed background refresh keeps the last good table and offers a retry. + if (!background || !loaded.current) setValidatorsGetError(e.message); + else setRefreshError(e.message); } finally { if (!background) setLoading(false); inFlight.current = false; @@ -115,7 +124,13 @@ export default function ValidatorList({ } } - const pendingCount = validatorsGet ? countPending(validatorsGet) : 0; + const selectedValidators = useMemo(() => { + const selected = new Set(selectedRows.map(String)); + return (validatorsGet ?? []).filter(({ pubkey }) => selected.has(pubkey)); + }, [validatorsGet, selectedRows]); + const pendingCount = + validatorsGet?.filter(({ detailsPending, beaconPending }) => detailsPending || beaconPending).length ?? 0; + const detailsErrors = [...new Set((validatorsGet ?? []).flatMap(({ detailsError }) => detailsError ?? []))]; const SmoothBanner = () => { return ( @@ -164,41 +179,50 @@ export default function ValidatorList({ ) : validatorsGet ? ( <> - {pendingCount > 0 && - (stalled ? ( - { - stalls.current = 0; - lastPending.current = undefined; - setStalled(false); - void getValidators(true); - }} - > - Retry - - } - > - Details for {pendingCount} of {validatorsGet.length} validators are still loading slowly. Your - consensus or validator client may be busy or syncing. - - ) : ( - - - Loading details for {pendingCount} of {validatorsGet.length} validators… - - - - ))} + {refreshError && ( + + Retry + + } + > + The validator list could not be refreshed: {refreshError} + + )} + {detailsErrors.length > 0 && ( + + Some validator details are unavailable: {detailsErrors.join("; ")} + + )} + {stalled && ( + + Retry + + } + > + Details for {validatorsGet && countUnfinished(validatorsGet)} validators are loading slowly. Your + validator client or signer may be busy. + + )} + {pendingCount > 0 && ( + + + Checking {pendingCount} of {validatorsGet.length} validators with your clients… + + + + )} validatorsGet[row as number])} - setOpen={setBuilderConfigOpen} - /> + )} {exitOpen && ( ({ indexes: [index], - request: { ...request, validatorsImportRequest: [] }, + // Never sent: the run reports it as an error instead. + request: { importFrom: request.importFrom, validatorsImportRequest: [] }, tooLarge: { bytes: size, maxBytes } })); return [ diff --git a/packages/ui/src/utils/listFollowUp.ts b/packages/ui/src/utils/listFollowUp.ts new file mode 100644 index 00000000..f6e71938 --- /dev/null +++ b/packages/ui/src/utils/listFollowUp.ts @@ -0,0 +1,15 @@ +/** Delay before requesting the list again while client reads were cut off by the Brain's deadline. */ +export const FOLLOW_UP_MS = 3000; +/** Stop requesting after this many follow-ups in a row that did not reduce the pending rows. */ +export const MAX_STALLED_FOLLOW_UPS = 3; + +/** Rows whose client reads were cut off: requesting the list again continues them. */ +export function countUnfinished(rows: readonly { detailsPending: boolean }[]): number { + return rows.filter(({ detailsPending }) => detailsPending).length; +} + +/** Consecutive follow-ups that made no progress; resets whenever pending rows decrease or clear. */ +export function nextStallCount(stalls: number, previousPending: number | undefined, pending: number): number { + if (pending === 0) return 0; + return previousPending !== undefined && pending >= previousPending ? stalls + 1 : 0; +} diff --git a/packages/ui/src/utils/validatorRefresh.ts b/packages/ui/src/utils/validatorRefresh.ts deleted file mode 100644 index eebca961..00000000 --- a/packages/ui/src/utils/validatorRefresh.ts +++ /dev/null @@ -1,42 +0,0 @@ -/** Grid row ids (positions here). Kept free of UI-library types so the Brain e2e tests can import this module. */ -type GridRowId = string | number; - -/** Delay between refreshes while the Brain reports rows whose checks were cut short. */ -export const PENDING_REFRESH_MS = 3000; -/** Stop auto-refreshing after this many refreshes in a row without fewer pending rows. */ -export const MAX_STALLED_REFRESHES = 3; - -/** Grid ids are row positions; keep the same validators selected when a refresh reorders or removes rows. */ -export function remapSelection( - previous: readonly { pubkey: string }[] | undefined, - next: readonly { pubkey: string }[], - selected: readonly GridRowId[] -): GridRowId[] { - if (!previous) return []; - const positions = new Map(next.map(({ pubkey }, index) => [pubkey.toLowerCase(), index])); - return selected.flatMap((id) => { - const position = positions.get(previous[id as number]?.pubkey.toLowerCase() ?? ""); - return position === undefined ? [] : [position]; - }); -} - -/** - * Build the selection updater for a refresh. `previous` must be captured when the refresh lands: - * React may run the updater later, after the caller has already replaced its own reference to the rows. - */ -export function selectionAfterRefresh( - previous: readonly { pubkey: string }[] | undefined, - next: readonly { pubkey: string }[] -): (selected: GridRowId[]) => GridRowId[] { - return (selected) => remapSelection(previous, next, selected); -} - -export function countPending(rows: readonly { detailsPending?: boolean }[]): number { - return rows.filter(({ detailsPending }) => detailsPending).length; -} - -/** Track consecutive refreshes that did not reduce the number of pending rows. */ -export function nextStallCount(stalls: number, previousPending: number | undefined, pending: number): number { - if (pending === 0) return 0; - return previousPending !== undefined && pending >= previousPending ? stalls + 1 : 0; -} diff --git a/packages/ui/test/importForm.test.ts b/packages/ui/test/importForm.test.ts index b502a21f..5b1be350 100644 --- a/packages/ui/test/importForm.test.ts +++ b/packages/ui/test/importForm.test.ts @@ -1,5 +1,10 @@ import { expect } from "chai"; -import { appendImportFiles, isImportFormValid, ImportFormState } from "../src/utils/importForm.js"; +import { + appendImportFiles, + incompleteImportIndexes, + isImportFormValid, + ImportFormState +} from "../src/utils/importForm.js"; import { createImportRequest } from "../src/utils/createImportRequest.js"; import { extractPubkey } from "../src/utils/dataUtils.js"; @@ -67,3 +72,17 @@ describe("Bulk import form", () => { } }); }); + +describe("Large import forms", () => { + it("points at the files that still need input", () => { + const acceptedFiles = ["a", "b", "c"].map((name) => ({ file: new File(["{}"], name), pubkey: "0x" + name })); + expect( + incompleteImportIndexes({ + acceptedFiles, + passwords: ["pw", "", "pw"], + tags: ["solo", "solo", ""], + feeRecipients: ["0x" + "ab".repeat(20), "0x" + "ab".repeat(20), ""] + }) + ).to.deep.equal([1, 2]); + }); +}); diff --git a/packages/ui/test/listFollowUp.test.ts b/packages/ui/test/listFollowUp.test.ts new file mode 100644 index 00000000..40bd57c8 --- /dev/null +++ b/packages/ui/test/listFollowUp.test.ts @@ -0,0 +1,18 @@ +import { expect } from "chai"; +import { countUnfinished, nextStallCount } from "../src/utils/listFollowUp.js"; + +describe("Validator list follow-up requests", () => { + it("counts only rows whose client reads were cut off", () => { + expect(countUnfinished([{ detailsPending: true }, { detailsPending: false }, { detailsPending: true }])).to.equal( + 2 + ); + }); + + it("stops after follow-ups that make no progress and resets on progress", () => { + expect(nextStallCount(0, undefined, 10)).to.equal(0); + expect(nextStallCount(0, 10, 6)).to.equal(0); + expect(nextStallCount(0, 6, 6)).to.equal(1); + expect(nextStallCount(2, 6, 7)).to.equal(3); + expect(nextStallCount(2, 6, 0)).to.equal(0); + }); +}); diff --git a/packages/ui/test/validatorRefresh.test.ts b/packages/ui/test/validatorRefresh.test.ts deleted file mode 100644 index d6303242..00000000 --- a/packages/ui/test/validatorRefresh.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { expect } from "chai"; -import { countPending, nextStallCount, remapSelection, selectionAfterRefresh } from "../src/utils/validatorRefresh.js"; -import { incompleteImportIndexes } from "../src/utils/importForm.js"; - -const row = (pubkey: string, detailsPending?: boolean) => ({ pubkey, detailsPending }); - -describe("Live validator list refresh", () => { - it("keeps the same validators selected when rows move or disappear", () => { - const previous = [row("0xaa"), row("0xbb"), row("0xcc")]; - const next = [row("0xCC"), row("0xaa")]; - expect(remapSelection(previous, next, [0, 1, 2])).to.deep.equal([1, 0]); - }); - - it("keeps the selected validator when React applies the update after the rows were replaced", () => { - const rowsRef: { current: { pubkey: string }[] } = { current: [row("0xaa"), row("0xbb"), row("0xcc")] }; - const next = [row("0xbb"), row("0xcc")]; - // B is selected. Like ValidatorList, schedule the update and then replace the rows before it runs. - const updater = selectionAfterRefresh(rowsRef.current, next); - rowsRef.current = next; - expect(updater([1])).to.deep.equal([0]); - // Reading the reference inside the updater (the previous code) selects C instead. - expect(remapSelection(rowsRef.current, next, [1])).to.deep.equal([1]); - }); - - it("clears the selection when there was no earlier list", () => { - expect(remapSelection(undefined, [row("0xaa")], [0])).to.deep.equal([]); - }); - - it("counts pending rows and stops after refreshes that make no progress", () => { - expect(countPending([row("0xaa", true), row("0xbb", false), row("0xcc")])).to.equal(1); - expect(nextStallCount(0, undefined, 10)).to.equal(0); - expect(nextStallCount(0, 10, 6)).to.equal(0); - expect(nextStallCount(0, 6, 6)).to.equal(1); - expect(nextStallCount(2, 6, 7)).to.equal(3); - expect(nextStallCount(2, 6, 0)).to.equal(0); - }); -}); - -describe("Large import forms", () => { - it("points at the files that still need input", () => { - const acceptedFiles = ["a", "b", "c"].map((name) => ({ file: new File(["{}"], name), pubkey: "0x" + name })); - expect( - incompleteImportIndexes({ - acceptedFiles, - passwords: ["pw", "", "pw"], - tags: ["solo", "solo", ""], - feeRecipients: ["0x" + "ab".repeat(20), "0x" + "ab".repeat(20), ""] - }) - ).to.deep.equal([1, 2]); - }); -});