Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 7 additions & 32 deletions packages/brain/src/modules/apiServers/ui/calls/getValidators.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,43 +5,13 @@ import { ValidatorApi, Web3SignerApi } from "../../../apiClients/index.js";
import { BeaconStatusStore } from "../../../beaconStatus/index.js";
import { forEachConcurrently } from "../../../utils/forEachConcurrently.js";
import logger from "../../../logger/index.js";
import { listReadStateFor } from "../../../validatorList/readState.js";

/** Bound on one list request; unfinished reads are reported as pending for the UI to request again. */
export const LIST_DEADLINE_MS = 10_000;
const READ_CONCURRENCY = 8;
const FEE_CHECK_TTL_MS = 60_000;

interface FeeCheck {
expected: string;
correct: boolean;
expires: number;
}
interface ReadState {
/** Where the next request resumes fee reads, so successive requests cover different validators. */
feeNext: number;
fee: Map<string, FeeCheck>;
/** Bumped on every validator change; a read that overlaps one is not cached. */
generation: number;
}
const readStates = new WeakMap<BrainDataBase, ReadState>();

function readStateFor(brainDb: BrainDataBase): ReadState {
let state = readStates.get(brainDb);
if (!state) {
const created: ReadState = { feeNext: 0, fee: new Map(), generation: 0 };
// Change notifications arrive after an operation has configured the validator client. A check read
// mid-operation saw the new expected address but the old client value, so only mismatches can be
// stale; drop them, and refuse to cache reads that straddle the change.
brainDb.onChange(() => {
for (const [pubkey, check] of created.fee) if (!check.correct) created.fee.delete(pubkey);
created.generation++;
});
readStates.set(brainDb, created);
state = created;
}
return state;
}

const message = (error: unknown) => (error instanceof Error ? error.message : String(error));

/**
Expand All @@ -63,7 +33,7 @@ export async function getValidators({
}): Promise<CustomValidatorGetResponse[]> {
const entries = Object.entries(brainDb.getData());
if (entries.length === 0) return [];
const state = readStateFor(brainDb);
const state = listReadStateFor(brainDb);
const now = Date.now();
const present = new Set(entries.map(([pubkey]) => pubkey));
for (const [pubkey, check] of state.fee) if (!present.has(pubkey) || check.expires <= now) state.fee.delete(pubkey);
Expand Down Expand Up @@ -176,5 +146,10 @@ export async function getValidators({
row.detailsPending = pending.has(index);
row.detailsError = errors[index].length > 0 ? errors[index].join("; ") : undefined;
});
// Problems reconciliation may fix: ask it to announce itself when it next completes.
state.recheck = rows.some(
({ detailsPending, detailsError, validatorFeeRecipientCorrect }) =>
detailsPending || detailsError !== undefined || validatorFeeRecipientCorrect === false
);
return rows;
}
30 changes: 26 additions & 4 deletions packages/brain/src/modules/cron/reloadValidators/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { deleteDbPubkeysNotInSigner } from "./deleteDbPubkeysNotInSigner.js";
import { deleteSignerPubkeysNotInDb } from "./deleteSignerPubkeysNotInDb.js";
import { deleteValidatorPubkeysNotInDB } from "./deleteValidatorPubkeysNotInDb.js";
import { logPrefix } from "./logPrefix.js";
import { takeListRecheck } from "../../validatorList/readState.js";
import { postValidatorPubkeysFromDb } from "./postValidatorPubkeysFromDb.js";
import { postValidatorsFeeRecipientsFromDb } from "./postValidatorsFeeRecipientsFromDb.js";
import { postValidatorsBuilderConfigsFromDb } from "./postValidatorsBuilderConfigsFromDb.js";
Expand All @@ -21,6 +22,15 @@ import { postValidatorsBuilderConfigsFromDb } from "./postValidatorsBuilderConfi
* - POST to validator API fee recipients that are in DB and not in validator API
*
*/
/** Steps that tidy up rather than configure: log a failure and carry on with the pass. */
async function continueOnFailure(step: string, run: () => Promise<void>): Promise<void> {
try {
await run();
} catch (e) {
logger.error(`${logPrefix}${step} failed; continuing so desired validators are still configured`, e);
}
}

export async function reloadValidators(
signerApi: Web3SignerApi,
signerUrl: string,
Expand All @@ -38,8 +48,11 @@ export async function reloadValidators(
const dbPubkeys = Object.keys(brainDb.getData());
const signerPubkeys = (await signerApi.listRemoteKeys()).data.map((keystore) => keystore.validating_pubkey);

// 2. DELETE from signer API pubkeys that are not in DB
await deleteSignerPubkeysNotInDb({ signerApi, signerPubkeys, dbPubkeys });
// 2. DELETE from signer API pubkeys that are not in DB. Keys whose deletion fails stay listed, so
// the database keeps them and the next pass retries.
await continueOnFailure("Deleting signer keys not in the database", () =>
deleteSignerPubkeysNotInDb({ signerApi, signerPubkeys, dbPubkeys })
);

// 3. DELETE from DB pubkeys that are not in signer API
await deleteDbPubkeysNotInSigner({ brainDb, dbPubkeys, signerPubkeys });
Expand All @@ -62,11 +75,17 @@ export async function reloadValidators(

// 4. POST to validator API pubkeys that are in DB and not in validator API
const brainDbPubkeysToAdd = Object.keys(activeData).filter((pubkey) => !validatorKeys.has(pubkey.toLowerCase()));
await postValidatorPubkeysFromDb({ validatorApi, signerUrl, brainDbPubkeysToAdd, validatorPubkeys });
await continueOnFailure("Adding keys to the validator client", () =>
postValidatorPubkeysFromDb({ validatorApi, signerUrl, brainDbPubkeysToAdd, validatorPubkeys })
);

// 5. DELETE to validator API pubkeys that are in validator API and not in DB
const validatorPubkeysToRemove = validatorPubkeys.filter((pubkey) => !dbKeys.has(pubkey.toLowerCase()));
await deleteValidatorPubkeysNotInDB({ validatorApi, validatorPubkeys, validatorPubkeysToRemove });
// Cleanup must not stop desired validators from being configured; failed removals stay listed and
// are retried next pass.
await continueOnFailure("Removing obsolete keys from the validator client", () =>
deleteValidatorPubkeysNotInDB({ validatorApi, validatorPubkeys, validatorPubkeysToRemove })
);
// Client membership is shown in the UI; repairs here change it without touching the database.
if (brainDbPubkeysToAdd.length > 0 || validatorPubkeysToRemove.length > 0) brainDb.notifyChanged();

Expand All @@ -78,6 +97,9 @@ export async function reloadValidators(

// Independently retry managed configuration after key imports and fee destinations.
if (featureFlags.builderConfig) await postValidatorsBuilderConfigsFromDb({ validatorApi, dbData: activeData });
// Repairs such as restored fee recipients change no database record. If an open UI last saw a
// problem, announce the pass so it lists again (and checks the fee recipients by GET).
if (takeListRecheck(brainDb)) brainDb.notifyChanged();
logger.debug(`${logPrefix}Finished reloading data`);
} catch (e) {
logger.error(`${logPrefix}Error reloading data`, e);
Expand Down
51 changes: 51 additions & 0 deletions packages/brain/src/modules/validatorList/readState.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
import type { BrainDataBase } from "../db/index.js";

export interface FeeCheck {
expected: string;
correct: boolean;
expires: number;
}

/** State shared by successive validator list requests for one database. */
export interface ListReadState {
/** Where the next request resumes fee reads, so successive requests cover different validators. */
feeNext: number;
fee: Map<string, FeeCheck>;
/** Bumped on every validator change; a read that overlaps one is not cached. */
generation: number;
/**
* The last list response showed something reconciliation may fix (a fee mismatch, a failed or
* unfinished read). Reconciliation then announces itself when it completes, so idle UIs re-check.
*/
recheck: boolean;
}

const states = new WeakMap<BrainDataBase, ListReadState>();

export function listReadStateFor(brainDb: BrainDataBase): ListReadState {
let state = states.get(brainDb);
if (!state) {
const created: ListReadState = { feeNext: 0, fee: new Map(), generation: 0, recheck: false };
// Change notifications arrive after an operation or reconciliation has configured the validator
// client. A mismatch read before then may be stale, so drop it, and refuse to cache reads that
// straddle the change. Matches stay: a changed expected address misses the cache anyway.
brainDb.onChange(() => {
for (const [pubkey, check] of created.fee) if (!check.correct) created.fee.delete(pubkey);
created.generation++;
});
states.set(brainDb, created);
state = created;
}
return state;
}

/**
* Called when reconciliation completes: whether the last list response needs re-checking. Clears the
* flag; the next list response sets it again if the problem persists.
*/
export function takeListRecheck(brainDb: BrainDataBase): boolean {
const state = states.get(brainDb);
if (!state?.recheck) return false;
state.recheck = false;
return true;
}
28 changes: 28 additions & 0 deletions packages/brain/test/e2e/largeBatches.e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,34 @@ describe("E2E: large validator batches through the UI socket", function () {
).to.equal(true);
});

it("tells an idle UI when reload repairs a fee recipient it showed as wrong", async () => {
const pubkeys = Array.from({ length: 3 }, (_, index) => pubkeyAt(index));
await importAll(importRequest(pubkeys), pubkeys);
// Let the import's own announcements (including its beacon lookup) land before counting.
await beaconStatus.whenIdle();
await new Promise((resolve) => setTimeout(resolve, 700));
const watcher = await connect();
let events = 0;
watcher.client.on(VALIDATORS_CHANGED_EVENT, () => events++);

// The validator client's fee recipient changes behind the Brain's back; the UI shows it, then idles.
validator.feeRecipients.set(pubkeys[1], "0x" + "00".repeat(20));
const shown = await watcher.client.call("getValidators", undefined);
expect(shown[1].validatorFeeRecipientCorrect).to.equal(false);

// Reload restores it without changing any database record, and still tells the UI.
await reloadValidators(signerApi, signerUrl, validatorApi, brainDb);
await waitFor(() => events > 0);
const repaired = await watcher.client.call("getValidators", undefined);
expect(repaired.every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect)).to.equal(true);

// Nothing wrong any more: reconciling announces nothing, so idle UIs are not made to list again.
const before = events;
await reloadValidators(signerApi, signerUrl, validatorApi, brainDb);
await new Promise((resolve) => setTimeout(resolve, 700));
expect(events).to.equal(before);
});

it("keeps builder configuration switched off end to end", async () => {
const pubkeys = [pubkeyAt(0)];
await importAll(importRequest(pubkeys), pubkeys);
Expand Down
50 changes: 50 additions & 0 deletions packages/brain/test/unit/modules/keyManagement.unit.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,56 @@ describe("Key management partial failures", () => {
expect(validatorApi.setFeeRecipient.calledOnceWith(recipient, keys[0])).to.equal(true);
});

describe("reload keeps configuring desired validators when cleanup fails", () => {
const obsolete = "0x" + "ee".repeat(48);
beforeEach(() => {
brainDb.addValidators({ validators: Object.fromEntries(keys.map((key) => [key, { ...record }])) });
signerApi.upcheck.resolves();
signerApi.listRemoteKeys.resolves({
data: keys.map((validating_pubkey) => ({ validating_pubkey, derivation_path: "", readonly: false }))
});
// The validator client lost the desired keys but still holds one the database no longer has.
validatorApi.getRemoteKeys.resolves({ data: [{ pubkey: obsolete, readonly: false, url: "http://signer" }] });
});

it("restores fee recipients when removing obsolete validator keys fails, and retries the removal", async () => {
validatorApi.deleteRemoteKeys.rejects(new Error("HTTP 500"));
for (let pass = 0; pass < 2; pass++) await reloadValidators(signerApi, "http://signer", validatorApi, brainDb);
expect(validatorApi.postRemoteKeys.callCount).to.equal(2);
expect(validatorApi.deleteRemoteKeys.callCount).to.equal(2);
expect(validatorApi.deleteRemoteKeys.firstCall.args[0].pubkeys).to.deep.equal([obsolete]);
// Every pass configures every desired validator despite the failing cleanup.
expect(validatorApi.setFeeRecipient.callCount).to.equal(2 * keys.length);
});

it("restores fee recipients when adding keys to the validator client fails", async () => {
validatorApi.postRemoteKeys.rejects(new Error("HTTP 500"));
await reloadValidators(signerApi, "http://signer", validatorApi, brainDb);
expect(validatorApi.setFeeRecipient.callCount).to.equal(keys.length);
});

it("keeps database records and restores fee recipients when signer cleanup fails", async () => {
signerApi.listRemoteKeys.resolves({
data: [...keys, obsolete].map((validating_pubkey) => ({
validating_pubkey,
derivation_path: "",
readonly: false
}))
});
signerApi.deleteRemoteKeys.rejects(new Error("HTTP 500"));
await reloadValidators(signerApi, "http://signer", validatorApi, brainDb);
expect(Object.keys(brainDb.getData())).to.deep.equal(keys);
expect(validatorApi.setFeeRecipient.callCount).to.equal(keys.length);
});

it("still stops before any change when the signer cannot be listed", async () => {
signerApi.listRemoteKeys.rejects(new Error("ECONNREFUSED"));
await reloadValidators(signerApi, "http://signer", validatorApi, brainDb);
expect(validatorApi.getRemoteKeys.called || validatorApi.setFeeRecipient.called).to.equal(false);
expect(Object.keys(brainDb.getData())).to.deep.equal(keys);
});
});

it("abandons a client operation that cannot start before its wait budget", async () => {
const clock = sinon.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
let finishReload!: () => void;
Expand Down
Loading
Loading