diff --git a/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts b/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts index 5887a8ec..5b1d398a 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts @@ -5,43 +5,13 @@ import { ValidatorApi, Web3SignerApi } from "../../../apiClients/index.js"; import { BeaconStatusStore } from "../../../beaconStatus/index.js"; import { forEachConcurrently } from "../../../utils/forEachConcurrently.js"; import logger from "../../../logger/index.js"; +import { listReadStateFor } from "../../../validatorList/readState.js"; /** Bound on one list request; unfinished reads are reported as pending for the UI to request again. */ export const LIST_DEADLINE_MS = 10_000; const READ_CONCURRENCY = 8; const FEE_CHECK_TTL_MS = 60_000; -interface FeeCheck { - expected: string; - correct: boolean; - expires: number; -} -interface ReadState { - /** Where the next request resumes fee reads, so successive requests cover different validators. */ - feeNext: number; - fee: Map; - /** Bumped on every validator change; a read that overlaps one is not cached. */ - generation: number; -} -const readStates = new WeakMap(); - -function readStateFor(brainDb: BrainDataBase): ReadState { - let state = readStates.get(brainDb); - if (!state) { - const created: ReadState = { feeNext: 0, fee: new Map(), generation: 0 }; - // Change notifications arrive after an operation has configured the validator client. A check read - // mid-operation saw the new expected address but the old client value, so only mismatches can be - // stale; drop them, and refuse to cache reads that straddle the change. - brainDb.onChange(() => { - for (const [pubkey, check] of created.fee) if (!check.correct) created.fee.delete(pubkey); - created.generation++; - }); - readStates.set(brainDb, created); - state = created; - } - return state; -} - const message = (error: unknown) => (error instanceof Error ? error.message : String(error)); /** @@ -63,7 +33,7 @@ export async function getValidators({ }): Promise { const entries = Object.entries(brainDb.getData()); if (entries.length === 0) return []; - const state = readStateFor(brainDb); + const state = listReadStateFor(brainDb); const now = Date.now(); const present = new Set(entries.map(([pubkey]) => pubkey)); for (const [pubkey, check] of state.fee) if (!present.has(pubkey) || check.expires <= now) state.fee.delete(pubkey); @@ -176,5 +146,10 @@ export async function getValidators({ row.detailsPending = pending.has(index); row.detailsError = errors[index].length > 0 ? errors[index].join("; ") : undefined; }); + // Problems reconciliation may fix: ask it to announce itself when it next completes. + state.recheck = rows.some( + ({ detailsPending, detailsError, validatorFeeRecipientCorrect }) => + detailsPending || detailsError !== undefined || validatorFeeRecipientCorrect === false + ); return rows; } diff --git a/packages/brain/src/modules/cron/reloadValidators/index.ts b/packages/brain/src/modules/cron/reloadValidators/index.ts index 9f00086b..f2af6a26 100644 --- a/packages/brain/src/modules/cron/reloadValidators/index.ts +++ b/packages/brain/src/modules/cron/reloadValidators/index.ts @@ -7,6 +7,7 @@ import { deleteDbPubkeysNotInSigner } from "./deleteDbPubkeysNotInSigner.js"; import { deleteSignerPubkeysNotInDb } from "./deleteSignerPubkeysNotInDb.js"; import { deleteValidatorPubkeysNotInDB } from "./deleteValidatorPubkeysNotInDb.js"; import { logPrefix } from "./logPrefix.js"; +import { takeListRecheck } from "../../validatorList/readState.js"; import { postValidatorPubkeysFromDb } from "./postValidatorPubkeysFromDb.js"; import { postValidatorsFeeRecipientsFromDb } from "./postValidatorsFeeRecipientsFromDb.js"; import { postValidatorsBuilderConfigsFromDb } from "./postValidatorsBuilderConfigsFromDb.js"; @@ -21,6 +22,15 @@ import { postValidatorsBuilderConfigsFromDb } from "./postValidatorsBuilderConfi * - POST to validator API fee recipients that are in DB and not in validator API * */ +/** Steps that tidy up rather than configure: log a failure and carry on with the pass. */ +async function continueOnFailure(step: string, run: () => Promise): Promise { + try { + await run(); + } catch (e) { + logger.error(`${logPrefix}${step} failed; continuing so desired validators are still configured`, e); + } +} + export async function reloadValidators( signerApi: Web3SignerApi, signerUrl: string, @@ -38,8 +48,11 @@ export async function reloadValidators( const dbPubkeys = Object.keys(brainDb.getData()); const signerPubkeys = (await signerApi.listRemoteKeys()).data.map((keystore) => keystore.validating_pubkey); - // 2. DELETE from signer API pubkeys that are not in DB - await deleteSignerPubkeysNotInDb({ signerApi, signerPubkeys, dbPubkeys }); + // 2. DELETE from signer API pubkeys that are not in DB. Keys whose deletion fails stay listed, so + // the database keeps them and the next pass retries. + await continueOnFailure("Deleting signer keys not in the database", () => + deleteSignerPubkeysNotInDb({ signerApi, signerPubkeys, dbPubkeys }) + ); // 3. DELETE from DB pubkeys that are not in signer API await deleteDbPubkeysNotInSigner({ brainDb, dbPubkeys, signerPubkeys }); @@ -62,11 +75,17 @@ export async function reloadValidators( // 4. POST to validator API pubkeys that are in DB and not in validator API const brainDbPubkeysToAdd = Object.keys(activeData).filter((pubkey) => !validatorKeys.has(pubkey.toLowerCase())); - await postValidatorPubkeysFromDb({ validatorApi, signerUrl, brainDbPubkeysToAdd, validatorPubkeys }); + await continueOnFailure("Adding keys to the validator client", () => + postValidatorPubkeysFromDb({ validatorApi, signerUrl, brainDbPubkeysToAdd, validatorPubkeys }) + ); // 5. DELETE to validator API pubkeys that are in validator API and not in DB const validatorPubkeysToRemove = validatorPubkeys.filter((pubkey) => !dbKeys.has(pubkey.toLowerCase())); - await deleteValidatorPubkeysNotInDB({ validatorApi, validatorPubkeys, validatorPubkeysToRemove }); + // Cleanup must not stop desired validators from being configured; failed removals stay listed and + // are retried next pass. + await continueOnFailure("Removing obsolete keys from the validator client", () => + deleteValidatorPubkeysNotInDB({ validatorApi, validatorPubkeys, validatorPubkeysToRemove }) + ); // Client membership is shown in the UI; repairs here change it without touching the database. if (brainDbPubkeysToAdd.length > 0 || validatorPubkeysToRemove.length > 0) brainDb.notifyChanged(); @@ -78,6 +97,9 @@ export async function reloadValidators( // Independently retry managed configuration after key imports and fee destinations. if (featureFlags.builderConfig) await postValidatorsBuilderConfigsFromDb({ validatorApi, dbData: activeData }); + // Repairs such as restored fee recipients change no database record. If an open UI last saw a + // problem, announce the pass so it lists again (and checks the fee recipients by GET). + if (takeListRecheck(brainDb)) brainDb.notifyChanged(); logger.debug(`${logPrefix}Finished reloading data`); } catch (e) { logger.error(`${logPrefix}Error reloading data`, e); diff --git a/packages/brain/src/modules/validatorList/readState.ts b/packages/brain/src/modules/validatorList/readState.ts new file mode 100644 index 00000000..e2dbb4e7 --- /dev/null +++ b/packages/brain/src/modules/validatorList/readState.ts @@ -0,0 +1,51 @@ +import type { BrainDataBase } from "../db/index.js"; + +export interface FeeCheck { + expected: string; + correct: boolean; + expires: number; +} + +/** State shared by successive validator list requests for one database. */ +export interface ListReadState { + /** Where the next request resumes fee reads, so successive requests cover different validators. */ + feeNext: number; + fee: Map; + /** Bumped on every validator change; a read that overlaps one is not cached. */ + generation: number; + /** + * The last list response showed something reconciliation may fix (a fee mismatch, a failed or + * unfinished read). Reconciliation then announces itself when it completes, so idle UIs re-check. + */ + recheck: boolean; +} + +const states = new WeakMap(); + +export function listReadStateFor(brainDb: BrainDataBase): ListReadState { + let state = states.get(brainDb); + if (!state) { + const created: ListReadState = { feeNext: 0, fee: new Map(), generation: 0, recheck: false }; + // Change notifications arrive after an operation or reconciliation has configured the validator + // client. A mismatch read before then may be stale, so drop it, and refuse to cache reads that + // straddle the change. Matches stay: a changed expected address misses the cache anyway. + brainDb.onChange(() => { + for (const [pubkey, check] of created.fee) if (!check.correct) created.fee.delete(pubkey); + created.generation++; + }); + states.set(brainDb, created); + state = created; + } + return state; +} + +/** + * Called when reconciliation completes: whether the last list response needs re-checking. Clears the + * flag; the next list response sets it again if the problem persists. + */ +export function takeListRecheck(brainDb: BrainDataBase): boolean { + const state = states.get(brainDb); + if (!state?.recheck) return false; + state.recheck = false; + return true; +} diff --git a/packages/brain/test/e2e/largeBatches.e2e.test.ts b/packages/brain/test/e2e/largeBatches.e2e.test.ts index d6b86165..25ed0b11 100644 --- a/packages/brain/test/e2e/largeBatches.e2e.test.ts +++ b/packages/brain/test/e2e/largeBatches.e2e.test.ts @@ -349,6 +349,34 @@ describe("E2E: large validator batches through the UI socket", function () { ).to.equal(true); }); + it("tells an idle UI when reload repairs a fee recipient it showed as wrong", async () => { + const pubkeys = Array.from({ length: 3 }, (_, index) => pubkeyAt(index)); + await importAll(importRequest(pubkeys), pubkeys); + // Let the import's own announcements (including its beacon lookup) land before counting. + await beaconStatus.whenIdle(); + await new Promise((resolve) => setTimeout(resolve, 700)); + const watcher = await connect(); + let events = 0; + watcher.client.on(VALIDATORS_CHANGED_EVENT, () => events++); + + // The validator client's fee recipient changes behind the Brain's back; the UI shows it, then idles. + validator.feeRecipients.set(pubkeys[1], "0x" + "00".repeat(20)); + const shown = await watcher.client.call("getValidators", undefined); + expect(shown[1].validatorFeeRecipientCorrect).to.equal(false); + + // Reload restores it without changing any database record, and still tells the UI. + await reloadValidators(signerApi, signerUrl, validatorApi, brainDb); + await waitFor(() => events > 0); + const repaired = await watcher.client.call("getValidators", undefined); + expect(repaired.every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect)).to.equal(true); + + // Nothing wrong any more: reconciling announces nothing, so idle UIs are not made to list again. + const before = events; + await reloadValidators(signerApi, signerUrl, validatorApi, brainDb); + await new Promise((resolve) => setTimeout(resolve, 700)); + expect(events).to.equal(before); + }); + it("keeps builder configuration switched off end to end", async () => { const pubkeys = [pubkeyAt(0)]; await importAll(importRequest(pubkeys), pubkeys); diff --git a/packages/brain/test/unit/modules/keyManagement.unit.test.ts b/packages/brain/test/unit/modules/keyManagement.unit.test.ts index 6f0d57b5..1d80a000 100644 --- a/packages/brain/test/unit/modules/keyManagement.unit.test.ts +++ b/packages/brain/test/unit/modules/keyManagement.unit.test.ts @@ -330,6 +330,56 @@ describe("Key management partial failures", () => { expect(validatorApi.setFeeRecipient.calledOnceWith(recipient, keys[0])).to.equal(true); }); + describe("reload keeps configuring desired validators when cleanup fails", () => { + const obsolete = "0x" + "ee".repeat(48); + beforeEach(() => { + brainDb.addValidators({ validators: Object.fromEntries(keys.map((key) => [key, { ...record }])) }); + signerApi.upcheck.resolves(); + signerApi.listRemoteKeys.resolves({ + data: keys.map((validating_pubkey) => ({ validating_pubkey, derivation_path: "", readonly: false })) + }); + // The validator client lost the desired keys but still holds one the database no longer has. + validatorApi.getRemoteKeys.resolves({ data: [{ pubkey: obsolete, readonly: false, url: "http://signer" }] }); + }); + + it("restores fee recipients when removing obsolete validator keys fails, and retries the removal", async () => { + validatorApi.deleteRemoteKeys.rejects(new Error("HTTP 500")); + for (let pass = 0; pass < 2; pass++) await reloadValidators(signerApi, "http://signer", validatorApi, brainDb); + expect(validatorApi.postRemoteKeys.callCount).to.equal(2); + expect(validatorApi.deleteRemoteKeys.callCount).to.equal(2); + expect(validatorApi.deleteRemoteKeys.firstCall.args[0].pubkeys).to.deep.equal([obsolete]); + // Every pass configures every desired validator despite the failing cleanup. + expect(validatorApi.setFeeRecipient.callCount).to.equal(2 * keys.length); + }); + + it("restores fee recipients when adding keys to the validator client fails", async () => { + validatorApi.postRemoteKeys.rejects(new Error("HTTP 500")); + await reloadValidators(signerApi, "http://signer", validatorApi, brainDb); + expect(validatorApi.setFeeRecipient.callCount).to.equal(keys.length); + }); + + it("keeps database records and restores fee recipients when signer cleanup fails", async () => { + signerApi.listRemoteKeys.resolves({ + data: [...keys, obsolete].map((validating_pubkey) => ({ + validating_pubkey, + derivation_path: "", + readonly: false + })) + }); + signerApi.deleteRemoteKeys.rejects(new Error("HTTP 500")); + await reloadValidators(signerApi, "http://signer", validatorApi, brainDb); + expect(Object.keys(brainDb.getData())).to.deep.equal(keys); + expect(validatorApi.setFeeRecipient.callCount).to.equal(keys.length); + }); + + it("still stops before any change when the signer cannot be listed", async () => { + signerApi.listRemoteKeys.rejects(new Error("ECONNREFUSED")); + await reloadValidators(signerApi, "http://signer", validatorApi, brainDb); + expect(validatorApi.getRemoteKeys.called || validatorApi.setFeeRecipient.called).to.equal(false); + expect(Object.keys(brainDb.getData())).to.deep.equal(keys); + }); + }); + it("abandons a client operation that cannot start before its wait budget", async () => { const clock = sinon.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); let finishReload!: () => void; diff --git a/packages/ui/src/components/Dialogs/BatchProgressView.tsx b/packages/ui/src/components/Dialogs/BatchProgressView.tsx new file mode 100644 index 00000000..e9368b32 --- /dev/null +++ b/packages/ui/src/components/Dialogs/BatchProgressView.tsx @@ -0,0 +1,149 @@ +import type { JSX } from "react"; +import { useMemo, useState } from "react"; +import { + Alert, + Box, + Button, + Chip, + CircularProgress, + FormControlLabel, + LinearProgress, + Stack, + Switch, + Typography +} from "@mui/material"; +import ValidatorResult from "./ValidatorResult"; +import { BatchProgress, BatchResult, NOT_ATTEMPTED, QUEUED, summarize } from "../../utils/batchRunner"; + +// Rendering hundreds of result cards at once makes the dialog sluggish; reveal more on demand. +const RESULTS_PAGE = 100; + +export type SummaryLabel = [status: string, label: string, color: "success" | "default" | "error" | "info"]; + +/** Progress and per-validator results of a batched operation (import, delete). */ +export default function BatchProgressView({ + results, + progress, + running, + pubkeys, + verb, + activeStatus, + doneStatuses, + labels, + runningHint, + stopped, + stoppedMessages +}: { + results: BatchResult[]; + progress: BatchProgress; + running: boolean; + /** Pubkey for each result, by position. */ + pubkeys: readonly string[]; + /** e.g. "Importing", for "Importing batch 3 of 15…". */ + verb: string; + activeStatus: string; + /** Outcomes that are not problems. */ + doneStatuses: readonly string[]; + /** Summary chips for the operation's own outcomes; failure, in-flight, queued and not attempted are added. */ + labels: SummaryLabel[]; + runningHint?: string; + stopped?: "user" | "error"; + stoppedMessages: { user: string; error: string }; +}): JSX.Element { + const [onlyProblems, setOnlyProblems] = useState(true); + const [visible, setVisible] = useState(RESULTS_PAGE); + const summary = useMemo(() => summarize(results), [results]); + const allLabels: SummaryLabel[] = [ + ...labels, + ["error", "failed", "error"], + [activeStatus, `${verb.toLowerCase()} now`, "info"], + [QUEUED, "queued", "default"], + [NOT_ATTEMPTED, "not attempted", "default"] + ]; + const problems = (summary.error ?? 0) + (summary[NOT_ATTEMPTED] ?? 0); + // While running, show the batch in flight and failures (queued rows are noise); afterwards, problems. + const filtering = results.length > 10 && onlyProblems && (problems > 0 || running); + const shown = results + .map((result, index) => ({ result, index })) + .filter(({ result }) => + filtering + ? running + ? [activeStatus, "error"].includes(result.status) + : !doneStatuses.includes(result.status) + : true + ); + + return ( + + + + + {progress.done} of {progress.total} validators processed + + {running && progress.current > 0 ? ( + + + + {progress.batches > 1 + ? `${verb} batch ${progress.batch + 1} of ${progress.batches}…` + : `${verb} ${progress.current} validators…`} + + + ) : ( + progress.batches > 1 && ( + + {progress.batch} of {progress.batches} batches done + + ) + )} + + {/* The buffer shows the batch in flight, so the bar moves while a long batch runs. */} + + {running && runningHint && ( + + {runningHint} + + )} + + + + {allLabels + .filter(([status]) => summary[status]) + .map(([status, label, color]) => ( + + ))} + + + {stopped === "error" && {stoppedMessages.error}} + {stopped === "user" && {stoppedMessages.user}} + + {results.length > 10 && (problems > 0 || running) && ( + setOnlyProblems(checked)} />} + label="Show only problems" + /> + )} + +
+ {shown.slice(0, visible).map(({ result, index }) => ( + + ))} + {shown.length > visible && ( + + )} +
+
+ ); +} diff --git a/packages/ui/src/components/Dialogs/DeletionWarning.tsx b/packages/ui/src/components/Dialogs/DeletionWarning.tsx index 3017a2d2..a3b0ae34 100644 --- a/packages/ui/src/components/Dialogs/DeletionWarning.tsx +++ b/packages/ui/src/components/Dialogs/DeletionWarning.tsx @@ -2,14 +2,18 @@ import type { JSX } from "react"; import { shortenPubkey } from "@stakingbrain/common"; import type { CustomValidatorGetResponse } from "@stakingbrain/brain"; +// Long selections would push the warning and the button out of view. +const PREVIEW = 10; + export default function DeletionWarning({ validators }: { validators: CustomValidatorGetResponse[] }): JSX.Element { return ( <> Are you sure you want to delete these keystores?
    - {validators.map(({ pubkey }) => ( + {validators.slice(0, PREVIEW).map(({ pubkey }) => (
  • {shortenPubkey(pubkey)}
  • ))} + {validators.length > PREVIEW &&
  • …and {validators.length - PREVIEW} more
  • }
After deletion, these keystores won't be used for signing anymore and your slashing protection data will be downloaded.
diff --git a/packages/ui/src/components/Dialogs/ImportDialog.tsx b/packages/ui/src/components/Dialogs/ImportDialog.tsx index 06c553cd..18e77e38 100644 --- a/packages/ui/src/components/Dialogs/ImportDialog.tsx +++ b/packages/ui/src/components/Dialogs/ImportDialog.tsx @@ -1,19 +1,13 @@ import type { JSX } from "react"; -import { useMemo, useState } from "react"; +import { useMemo } from "react"; import { Alert, Dialog, DialogContent, Box, Button, - Chip, DialogActions, Divider, - FormControlLabel, - LinearProgress, - Stack, - Switch, - Typography, useMediaQuery, useTheme } from "@mui/material"; @@ -24,25 +18,15 @@ import WaitBox from "../WaitBox/WaitBox"; import { SlideTransition } from "./Transitions"; import UploadFileIcon from "@mui/icons-material/UploadFile"; import DialogHeader from "./DialogHeader"; -import ValidatorResult from "./ValidatorResult"; +import BatchProgressView from "./BatchProgressView"; import { + IMPORTED_STATUSES, ImportProgress, ImportResult, - NOT_ATTEMPTED, - retryableIndexes, - summarizeImport + IMPORTING, + retryableIndexes } from "../../utils/importBatches"; -// Rendering hundreds of result cards at once makes the dialog sluggish; reveal more on demand. -const RESULTS_PAGE = 100; - -const summaryLabels: [status: string, label: string, color: "success" | "default" | "error"][] = [ - ["imported", "imported", "success"], - ["duplicate", "already present", "default"], - ["error", "failed", "error"], - [NOT_ATTEMPTED, "not attempted", "default"] -]; - export default function ImportDialog({ open, setOpen, @@ -71,23 +55,11 @@ export default function ImportDialog({ const navigate = useNavigate(); const isMobile = useMediaQuery(useTheme().breakpoints.down("sm")); const importing = importStatus === ImportStatus.Importing; - const [onlyProblems, setOnlyProblems] = useState(true); - const [visible, setVisible] = useState(RESULTS_PAGE); const handleClose = () => { setOpen(false); navigate({ pathname: "/", search: window.location.search }); }; - - const summary = useMemo(() => summarizeImport(results ?? []), [results]); const retry = useMemo(() => retryableIndexes(results ?? []), [results]); - const problems = (summary.error ?? 0) + (summary[NOT_ATTEMPTED] ?? 0); - // While importing, untouched rows would only be noise; afterwards show problems first when there are any. - const filtering = (results?.length ?? 0) > 10 && onlyProblems && (problems > 0 || importing); - const shown = (results ?? []) - .map((result, index) => ({ result, index })) - .filter(({ result }) => - filtering ? (importing ? result.status === "error" : !["imported", "duplicate"].includes(result.status)) : true - ); return ( ) : ( - - - - - {progress.done} of {progress.total} validators processed - - {progress.batches > 1 && ( - - Batch {Math.min(progress.batch + (importing ? 1 : 0), progress.batches)} of {progress.batches} - - )} - - - {importing && ( - - Decrypting keystores takes time. Keep this window open; you can stop after the current batch. - - )} - - - - {summaryLabels - .filter(([status]) => summary[status]) - .map(([status, label, color]) => ( - - ))} - - - {stopped === "error" && ( - - The import stopped because a batch failed. Validators not yet imported can be retried; keys that did - reach the signer will show as already present. - - )} - {stopped === "user" && ( - Import stopped. You can retry the validators that were not imported. - )} - - {results.length > 10 && (problems > 0 || importing) && ( - setOnlyProblems(checked)} />} - label="Show only problems" - /> - )} - -
- {shown.slice(0, visible).map(({ result, index }) => ( - - ))} - {shown.length > visible && ( - - )} -
-
+ pubkey)} + verb="Importing" + activeStatus={IMPORTING} + doneStatuses={IMPORTED_STATUSES} + labels={[ + ["imported", "imported", "success"], + ["duplicate", "already present", "default"] + ]} + runningHint="Decrypting keystores takes time. Keep this window open; you can stop after the current batch." + stopped={stopped} + stoppedMessages={{ + error: + "The import stopped because a batch failed. Validators not yet imported can be retried; keys that did reach the signer will show as already present.", + user: "Import stopped. You can retry the validators that were not imported." + }} + /> )} diff --git a/packages/ui/src/components/Dialogs/KeystoresDeleteDialog.tsx b/packages/ui/src/components/Dialogs/KeystoresDeleteDialog.tsx index 81b0345d..112fc1c9 100644 --- a/packages/ui/src/components/Dialogs/KeystoresDeleteDialog.tsx +++ b/packages/ui/src/components/Dialogs/KeystoresDeleteDialog.tsx @@ -1,17 +1,35 @@ import type { JSX } from "react"; -import { useState } from "react"; -import { Dialog, DialogContent, Box, Button, DialogContentText, DialogActions, Alert, Divider } from "@mui/material"; +import { useMemo, useRef, useState } from "react"; +import { + Dialog, + DialogContent, + Box, + Button, + DialogContentText, + DialogActions, + Alert, + Divider, + Stack +} from "@mui/material"; import type { GridRowId } from "@mui/x-data-grid"; import { importDialogBoxStyle } from "../../Styles/dialogStyles"; -import WaitBox from "../WaitBox/WaitBox"; import DeletionWarning from "./DeletionWarning"; import { rpcClient } from "../../socket"; import { SlideTransition } from "./Transitions"; import DeleteOutlineIcon from "@mui/icons-material/DeleteOutline"; import DialogHeader from "./DialogHeader"; -import ValidatorResult from "./ValidatorResult"; -import type { CustomValidatorGetResponse, Web3signerDeleteResponse } from "@stakingbrain/brain"; +import BatchProgressView from "./BatchProgressView"; +import type { CustomValidatorGetResponse } from "@stakingbrain/brain"; import { useTheme, useMediaQuery } from "@mui/material"; +import { BatchProgress, BatchResult, notAttempted } from "../../utils/batchRunner"; +import { + DELETED_STATUSES, + DELETING, + mergeInterchanges, + planDeleteBatches, + retryableDeletes, + runDeleteBatches +} from "../../utils/deleteBatches"; export default function KeystoresDeleteDialog({ validators, @@ -25,36 +43,45 @@ export default function KeystoresDeleteDialog({ open: boolean; setOpen: (open: boolean) => void; }): JSX.Element { - const [keystoresDelete, setKeystoresDelete] = useState(); - const [keystoresDeleteError, setKeystoresDeleteError] = useState(); - const [loading, setLoading] = useState(false); - const [deletedPubkeys, setDeletedPubkeys] = useState([]); + // Fixed when deletion starts, so later list refreshes cannot change what is being deleted. + const [targets, setTargets] = useState(); + const [results, setResults] = useState(); + const [progress, setProgress] = useState(); + const [stopped, setStopped] = useState<"user" | "error">(); + const [running, setRunning] = useState(false); + const [stopRequested, setStopRequested] = useState(false); + const stopRef = useRef(false); + // Every batch exports the slashing history of its keys; keep all of it, across retries too. + const [exports, setExports] = useState([]); const theme = useTheme(); const isMobile = useMediaQuery(theme.breakpoints.down("sm")); - async function deleteSelectedKeystores() { - try { - setKeystoresDelete(undefined); - setLoading(true); - const pubkeysToDelete = validators.map(({ pubkey }) => pubkey); - setDeletedPubkeys(pubkeysToDelete); - setKeystoresDelete( - await rpcClient.call("deleteValidators", { - pubkeys: pubkeysToDelete - }) - ); - setLoading(false); - setKeystoresDeleteError(undefined); - setSelectedRows([]); - } catch (e) { - console.error(e); - setLoading(false); - setKeystoresDeleteError(e.message); - } + async function deleteKeystores(pubkeys: string[], only?: number[]) { + setTargets(pubkeys); + setRunning(true); + setStopped(undefined); + stopRef.current = false; + setStopRequested(false); + const outcome = await runDeleteBatches({ + batches: planDeleteBatches(pubkeys, only), + results: (only && results) || pubkeys.map(notAttempted), + deleteBatch: (request) => rpcClient.call("deleteValidators", request), + onProgress: (update, latest) => { + setProgress(update); + setResults(latest); + }, + shouldStop: () => stopRef.current + }); + setResults(outcome.results); + setStopped(outcome.stopped); + setExports((previous) => [...previous, ...outcome.slashingProtection]); + setRunning(false); + setSelectedRows([]); } - const handleClose = () => { - setOpen(false); - }; + + const retry = useMemo(() => retryableDeletes(results ?? []), [results]); + const downloads = useMemo(() => mergeInterchanges(exports), [exports]); + const handleClose = () => setOpen(false); return ( { - if (!reason) { - handleClose(); - } + if (!reason && !running) handleClose(); }} aria-labelledby="alert-dialog-title" aria-describedby="alert-dialog-description" TransitionComponent={SlideTransition} > } - count={keystoresDelete ? deletedPubkeys.length : validators.length} + count={targets?.length ?? validators.length} onClose={handleClose} - busy={loading} + busy={running} /> - {keystoresDeleteError ? ( - {keystoresDeleteError} - ) : keystoresDelete?.data ? ( -
- {keystoresDelete.data.map((result, index) => { - const pubkey = deletedPubkeys[index]; - return ( - - ); - })} - {keystoresDelete.slashing_protection ? ( + {targets && results && progress ? ( + + + {!running && downloads.length > 0 && (
- - It is strongly recommended to stop the validator and watch at least 3 missed attestations in the - explorer before uploading the keys to another machine. + + Keep the slashing protection data for when you import these keystores elsewhere. It is strongly + recommended to stop the validator and watch at least 3 missed attestations in the explorer before + uploading the keys to another machine. - - + + {downloads.map((interchange, index) => ( + + ))} +
- ) : null} -
- ) : ( -
- {loading ? ( - - ) : ( - - - )} -
+ + ) : ( + + + )}
- + ) : ( + targets && + retry.length > 0 && ( + + ) + )} + - {!keystoresDelete && !loading ? ( + {!targets && ( - ) : null} + )}
); diff --git a/packages/ui/src/components/Dialogs/ValidatorResult.tsx b/packages/ui/src/components/Dialogs/ValidatorResult.tsx index b81962d5..cb6e153b 100644 --- a/packages/ui/src/components/Dialogs/ValidatorResult.tsx +++ b/packages/ui/src/components/Dialogs/ValidatorResult.tsx @@ -1,6 +1,9 @@ -import { Chip, Paper, Stack, Typography } from "@mui/material"; +import { Chip, CircularProgress, Paper, Stack, Typography } from "@mui/material"; import { shortenPubkey } from "@stakingbrain/common"; +/** Statuses of validators in the batch being sent right now. */ +const IN_FLIGHT = ["importing", "deleting"]; + export default function ValidatorResult({ pubkey, status, @@ -14,14 +17,22 @@ export default function ValidatorResult({ ? "success" : ["error", "Not exited"].includes(status) ? "error" - : "default"; + : IN_FLIGHT.includes(status) + ? "info" + : "default"; return ( {shortenPubkey(pubkey)} - + : undefined} + /> {message && ( diff --git a/packages/ui/src/utils/batchRunner.ts b/packages/ui/src/utils/batchRunner.ts new file mode 100644 index 00000000..4f332bf0 --- /dev/null +++ b/packages/ui/src/utils/batchRunner.ts @@ -0,0 +1,109 @@ +/** Per-validator outcome of a batched operation (import, delete), aligned with the user's selection. */ +export interface BatchResult { + status: string; + message?: string; +} + +export interface BatchProgress { + done: number; + total: number; + /** Batches finished. */ + batch: number; + batches: number; + /** Validators in the batch being sent right now; 0 between batches. */ + current: number; +} + +export interface Batch { + /** Positions in the selection, so results stay aligned with what the user picked. */ + indexes: number[]; + request: Request; + /** Never sent; reported as an error with this message (e.g. too large for one request). */ + rejected?: string; +} + +export const NOT_ATTEMPTED = "not_attempted"; +/** Waiting for earlier batches while an operation runs. */ +export const QUEUED = "queued"; + +export function notAttempted(): BatchResult { + return { status: NOT_ATTEMPTED, message: "Not processed: the operation stopped before reaching this validator." }; +} + +/** + * Send batches in order, one request at a time. While running, validators are `active` (current batch) + * or "queued" (later batches). A failed batch stops the run: its error may be a timeout whose outcome is + * unknown, so the validators it never reached become "not attempted" and can be retried safely. + */ +export async function runBatches({ + batches, + results: initial, + send, + toResults, + active, + onProgress, + shouldStop +}: { + batches: readonly Batch[]; + results: readonly BatchResult[]; + send: (request: Request) => Promise; + /** One result per index of the batch, in order. */ + toResults: (response: Response, batch: Batch) => BatchResult[]; + active: BatchResult; + onProgress: (progress: BatchProgress, results: BatchResult[]) => void; + shouldStop: () => boolean; +}): Promise<{ results: BatchResult[]; stopped?: "user" | "error" }> { + const results = [...initial]; + const queued: BatchResult = { status: QUEUED, message: "Waiting for earlier batches to finish." }; + for (const { indexes } of batches) for (const index of indexes) results[index] = queued; + const total = batches.reduce((sum, { indexes }) => sum + indexes.length, 0); + let done = 0; + let finished = 0; + const report = (current: number) => + onProgress({ done, total, batch: finished, batches: batches.length, current }, [...results]); + // Once the run ends early, nothing is waiting any more: say so instead of "queued". + const end = (stopped: "user" | "error") => { + for (const [index, result] of results.entries()) if (result?.status === QUEUED) results[index] = notAttempted(); + report(0); + return { results, stopped }; + }; + report(0); + + for (const [position, batch] of batches.entries()) { + if (shouldStop()) return end("user"); + if (batch.rejected) { + for (const index of batch.indexes) results[index] = { status: "error", message: batch.rejected }; + done += batch.indexes.length; + finished = position + 1; + report(0); + continue; + } + for (const index of batch.indexes) results[index] = active; + report(batch.indexes.length); + let failed = false; + try { + const batchResults = toResults(await send(batch.request), batch); + batch.indexes.forEach((index, offset) => (results[index] = batchResults[offset])); + } catch (error) { + failed = true; + const message = error instanceof Error ? error.message : String(error); + for (const index of batch.indexes) results[index] = { status: "error", message }; + } + done += batch.indexes.length; + finished = position + 1; + if (failed) return end("error"); + report(0); + } + return { results }; +} + +/** Positions worth retrying: anything without one of the given successful statuses. */ +export function retryableIndexes(results: readonly BatchResult[], succeeded: readonly string[]): number[] { + return results.flatMap((result, index) => (succeeded.includes(result?.status) ? [] : [index])); +} + +export function summarize(results: readonly BatchResult[]): Record { + const summary: Record = {}; + for (const { status } of results) summary[status] = (summary[status] ?? 0) + 1; + return summary; +} diff --git a/packages/ui/src/utils/deleteBatches.ts b/packages/ui/src/utils/deleteBatches.ts new file mode 100644 index 00000000..6d2f6d82 --- /dev/null +++ b/packages/ui/src/utils/deleteBatches.ts @@ -0,0 +1,90 @@ +import type { Web3signerDeleteResponse } from "@stakingbrain/brain"; +import { Batch, BatchProgress, BatchResult, retryableIndexes, runBatches } from "./batchRunner"; + +/** + * Deleting is quicker than importing, but each key still costs signer, validator client and fee + * recipient calls; batches keep each call short and let the UI show progress and stop between them. + */ +export const DELETE_BATCH_SIZE = 25; +/** In the batch being deleted right now. */ +export const DELETING = "deleting"; +/** Statuses that need no retry: the signer no longer holds the key. */ +export const DELETED_STATUSES = ["deleted", "not_active", "not_found"] as const; + +export type DeleteBatch = Batch<{ pubkeys: string[] }>; + +export function planDeleteBatches(pubkeys: readonly string[], only?: readonly number[]): DeleteBatch[] { + const selected = only ?? pubkeys.map((_, index) => index); + const batches: DeleteBatch[] = []; + for (let start = 0; start < selected.length; start += DELETE_BATCH_SIZE) { + const indexes = selected.slice(start, start + DELETE_BATCH_SIZE); + batches.push({ indexes, request: { pubkeys: indexes.map((index) => pubkeys[index]) } }); + } + return batches; +} + +/** + * Combine the slashing protection exports of several batches into one interchange file. They come + * from the same signer, so they share metadata. An export that cannot be parsed is kept as its own + * file rather than dropped: losing slashing history is not acceptable. + */ +export function mergeInterchanges(exports: readonly string[]): string[] { + const parsed: { metadata?: unknown; data: unknown[] }[] = []; + const unparsable: string[] = []; + for (const text of exports) { + try { + const value = JSON.parse(text); + if (value && typeof value === "object" && Array.isArray(value.data)) parsed.push(value); + else unparsable.push(text); + } catch { + unparsable.push(text); + } + } + const merged = + parsed.length > 0 + ? [JSON.stringify({ metadata: parsed[0].metadata, data: parsed.flatMap(({ data }) => data) })] + : []; + return [...merged, ...unparsable]; +} + +/** Delete in batches with live progress; slashing protection exports are collected from every batch. */ +export async function runDeleteBatches({ + batches, + results, + deleteBatch, + onProgress, + shouldStop +}: { + batches: readonly DeleteBatch[]; + results: readonly BatchResult[]; + deleteBatch: (request: { pubkeys: string[] }) => Promise; + onProgress: (progress: BatchProgress, results: BatchResult[]) => void; + shouldStop: () => boolean; +}): Promise<{ results: BatchResult[]; stopped?: "user" | "error"; slashingProtection: string[] }> { + const exports: string[] = []; + const outcome = await runBatches({ + batches, + results, + send: async (request: { pubkeys: string[] }) => { + const response = await deleteBatch(request); + if (response.slashing_protection) exports.push(response.slashing_protection); + return response; + }, + toResults: (response, batch) => + batch.indexes.map( + (_, offset) => + response.data[offset] ?? { + status: "error", + message: "Deletion outcome unknown: Brain returned no result for this key. Retry to check." + } + ), + active: { status: DELETING, message: "Deleting…" }, + onProgress, + shouldStop + }); + return { ...outcome, slashingProtection: exports }; +} + +export function retryableDeletes(results: readonly BatchResult[]): number[] { + return retryableIndexes(results, DELETED_STATUSES); +} diff --git a/packages/ui/src/utils/importBatches.ts b/packages/ui/src/utils/importBatches.ts index a05c287b..4b2f4663 100644 --- a/packages/ui/src/utils/importBatches.ts +++ b/packages/ui/src/utils/importBatches.ts @@ -1,5 +1,16 @@ import type { CustomImportRequest, Web3signerPostResponse } from "@stakingbrain/brain"; import { prefix0xPubkey, RPC_MAX_PAYLOAD_BYTES } from "@stakingbrain/common"; +import { + Batch, + BatchProgress, + BatchResult, + NOT_ATTEMPTED, + notAttempted, + QUEUED, + retryableIndexes as retryable, + runBatches, + summarize +} from "./batchRunner"; /** * Keystore decryption in the signer is slow, so one RPC call per batch keeps every call well inside @@ -11,31 +22,15 @@ const IMPORT_BATCH_MAX_BYTES = Math.floor(RPC_MAX_PAYLOAD_BYTES * 0.9); const encoder = new TextEncoder(); const byteLength = (text: string) => encoder.encode(text).byteLength; -export interface ImportResult { - status: string; - message?: string; -} - -export interface ImportBatch { - /** Positions in the original file list, so results stay aligned with the files the user picked. */ - indexes: number[]; - request: CustomImportRequest; - /** Set when a single key cannot fit one request; it is reported as an error without being sent. */ - tooLarge?: { bytes: number; maxBytes: number }; -} - -export interface ImportProgress { - done: number; - total: number; - batch: number; - batches: number; -} - -export const NOT_ATTEMPTED = "not_attempted"; +export type ImportResult = BatchResult; +export type ImportProgress = BatchProgress; +export type ImportBatch = Batch; +export { NOT_ATTEMPTED, QUEUED, notAttempted }; -export function notAttempted(): ImportResult { - return { status: NOT_ATTEMPTED, message: "Not imported yet: the import stopped before reaching this validator." }; -} +/** In the batch being imported right now. */ +export const IMPORTING = "importing"; +/** Statuses that need no retry. */ +export const IMPORTED_STATUSES = ["imported", "duplicate"] as const; type Interchange = { data: { pubkey?: unknown }[] }; @@ -125,11 +120,12 @@ export function planImportBatches( } if (group.length > 0) groups.push(group); + const mb = (bytes: number) => (bytes / 1024 / 1024).toFixed(1); const tooLarge: ImportBatch[] = [...oversized].map(([index, size]) => ({ indexes: [index], // Never sent: the run reports it as an error instead. request: { importFrom: request.importFrom, validatorsImportRequest: [] }, - tooLarge: { bytes: size, maxBytes } + rejected: `This keystore and its slashing protection history are too large to import in one request (${mb(size)} MB, limit ${mb(maxBytes)} MB). Import it with a pruned slashing protection file.` })); return [ ...tooLarge, @@ -150,14 +146,10 @@ export function planImportBatches( ]; } -/** - * Import batches in order. A failed batch stops the run: its error may be a timeout whose outcome is - * unknown, and the remaining files are left "not attempted" so the user can retry them safely - * (keys that did land are reported as duplicates). - */ -export async function runImportBatches({ +/** Import batches in order with live progress; see runBatches for stop and failure semantics. */ +export function runImportBatches({ batches, - results: initial, + results, importBatch, onProgress, shouldStop @@ -168,52 +160,27 @@ export async function runImportBatches({ onProgress: (progress: ImportProgress, results: ImportResult[]) => void; shouldStop: () => boolean; }): Promise<{ results: ImportResult[]; stopped?: "user" | "error" }> { - const results = [...initial]; - for (const { indexes } of batches) for (const index of indexes) results[index] = notAttempted(); - const total = batches.reduce((sum, { indexes }) => sum + indexes.length, 0); - let done = 0; - onProgress({ done, total, batch: 0, batches: batches.length }, [...results]); - - for (const [position, batch] of batches.entries()) { - if (shouldStop()) return { results, stopped: "user" }; - if (batch.tooLarge) { - const mb = (bytes: number) => (bytes / 1024 / 1024).toFixed(1); - results[batch.indexes[0]] = { - status: "error", - message: `This keystore and its slashing protection history are too large to import in one request (${mb(batch.tooLarge.bytes)} MB, limit ${mb(batch.tooLarge.maxBytes)} MB). Import it with a pruned slashing protection file.` - }; - done += 1; - onProgress({ done, total, batch: position + 1, batches: batches.length }, [...results]); - continue; - } - let failed = false; - try { - const response = await importBatch(batch.request); - batch.indexes.forEach((index, offset) => { - results[index] = response.data[offset] ?? { - status: "error", - message: "Import outcome unknown: Brain returned no result for this key. Retry to check." - }; - }); - } catch (error) { - failed = true; - const message = error instanceof Error ? error.message : String(error); - for (const index of batch.indexes) results[index] = { status: "error", message }; - } - done += batch.indexes.length; - onProgress({ done, total, batch: position + 1, batches: batches.length }, [...results]); - if (failed) return { results, stopped: "error" }; - } - return { results }; + return runBatches({ + batches, + results, + send: importBatch, + toResults: (response, batch) => + batch.indexes.map( + (_, offset) => + response.data[offset] ?? { + status: "error", + message: "Import outcome unknown: Brain returned no result for this key. Retry to check." + } + ), + active: { status: IMPORTING, message: "Importing…" }, + onProgress, + shouldStop + }); } /** Files worth retrying: anything not confirmed imported or already present. */ export function retryableIndexes(results: readonly ImportResult[]): number[] { - return results.flatMap((result, index) => (["imported", "duplicate"].includes(result?.status) ? [] : [index])); + return retryable(results, IMPORTED_STATUSES); } -export function summarizeImport(results: readonly ImportResult[]): Record { - const summary: Record = {}; - for (const { status } of results) summary[status] = (summary[status] ?? 0) + 1; - return summary; -} +export const summarizeImport = summarize; diff --git a/packages/ui/test/deleteBatches.test.ts b/packages/ui/test/deleteBatches.test.ts new file mode 100644 index 00000000..79191b1a --- /dev/null +++ b/packages/ui/test/deleteBatches.test.ts @@ -0,0 +1,94 @@ +import { expect } from "chai"; +import type { Web3signerDeleteResponse } from "@stakingbrain/brain"; +import { notAttempted, summarize } from "../src/utils/batchRunner.js"; +import { + DELETE_BATCH_SIZE, + mergeInterchanges, + planDeleteBatches, + retryableDeletes, + runDeleteBatches +} from "../src/utils/deleteBatches.js"; + +const pubkeys = Array.from({ length: 60 }, (_, index) => "0x" + (index + 1).toString(16).padStart(96, "0")); +const metadata = { interchange_format_version: "5", genesis_validators_root: "0x" + "00".repeat(32) }; +const signerDelete = (request: { pubkeys: string[] }): Web3signerDeleteResponse => ({ + data: request.pubkeys.map((pubkey) => ({ status: pubkey === pubkeys[5] ? "not_found" : "deleted", message: "" })), + slashing_protection: JSON.stringify({ metadata, data: request.pubkeys.map((pubkey) => ({ pubkey })) }) +}); + +describe("Batched validator deletion", () => { + it("plans batches in selection order, or only the given positions for a retry", () => { + expect(planDeleteBatches(pubkeys).map(({ indexes }) => indexes.length)).to.deep.equal([ + DELETE_BATCH_SIZE, + DELETE_BATCH_SIZE, + 60 - 2 * DELETE_BATCH_SIZE + ]); + expect(planDeleteBatches(pubkeys, [3, 40]).map(({ request }) => request.pubkeys)).to.deep.equal([ + [pubkeys[3], pubkeys[40]] + ]); + }); + + it("deletes in batches with progress, keeps results aligned and merges slashing history", async () => { + const inFlight: number[] = []; + const { results, stopped, slashingProtection } = await runDeleteBatches({ + batches: planDeleteBatches(pubkeys), + results: pubkeys.map(notAttempted), + deleteBatch: async (request) => signerDelete(request), + onProgress: (progress, latest) => { + if (progress.current) inFlight.push(latest.filter(({ status }) => status === "deleting").length); + }, + shouldStop: () => false + }); + expect(stopped).to.equal(undefined); + expect(inFlight).to.deep.equal([25, 25, 10]); + expect(summarize(results)).to.deep.equal({ deleted: 59, not_found: 1 }); + expect(results[5].status).to.equal("not_found"); + expect(retryableDeletes(results)).to.deep.equal([]); + const [merged, ...rest] = mergeInterchanges(slashingProtection); + expect(rest).to.deep.equal([]); + expect(JSON.parse(merged)).to.deep.equal({ metadata, data: pubkeys.map((pubkey) => ({ pubkey })) }); + }); + + it("stops at a failed batch, keeps the history already exported and offers the rest for retry", async () => { + let calls = 0; + const { results, stopped, slashingProtection } = await runDeleteBatches({ + batches: planDeleteBatches(pubkeys), + results: pubkeys.map(notAttempted), + deleteBatch: async (request) => { + if (++calls === 2) throw new Error("Connection lost or request timed out."); + return signerDelete(request); + }, + onProgress: () => undefined, + shouldStop: () => false + }); + expect(stopped).to.equal("error"); + expect(summarize(results)).to.deep.equal({ deleted: 24, not_found: 1, error: 25, not_attempted: 10 }); + expect(retryableDeletes(results)).to.deep.equal(pubkeys.map((_, index) => index).slice(25)); + expect(JSON.parse(mergeInterchanges(slashingProtection)[0]).data).to.have.length(25); + }); + + it("honours a stop request between batches", async () => { + let stop = false; + const { results, stopped } = await runDeleteBatches({ + batches: planDeleteBatches(pubkeys), + results: pubkeys.map(notAttempted), + deleteBatch: async (request) => { + stop = true; + return signerDelete(request); + }, + onProgress: () => undefined, + shouldStop: () => stop + }); + expect(stopped).to.equal("user"); + expect(summarize(results)).to.deep.equal({ deleted: 24, not_found: 1, not_attempted: 35 }); + }); + + it("never drops slashing history it cannot merge", () => { + const valid = JSON.stringify({ metadata, data: [{ pubkey: pubkeys[0] }] }); + expect(mergeInterchanges([valid, "{broken", valid])).to.deep.equal([ + JSON.stringify({ metadata, data: [{ pubkey: pubkeys[0] }, { pubkey: pubkeys[0] }] }), + "{broken" + ]); + expect(mergeInterchanges([])).to.deep.equal([]); + }); +}); diff --git a/packages/ui/test/importBatches.test.ts b/packages/ui/test/importBatches.test.ts index a945cec1..c072b55d 100644 --- a/packages/ui/test/importBatches.test.ts +++ b/packages/ui/test/importBatches.test.ts @@ -127,10 +127,30 @@ describe("Batched validator import", () => { shouldStop: () => false }); expect(stopped).to.equal(undefined); - expect(progress.map(({ done }) => done)).to.deep.equal([0, 20, 40, 45]); + // One report as each batch starts (so the UI can show it in flight) and one as it finishes. + expect(progress.map(({ done }) => done)).to.deep.equal([0, 0, 20, 20, 40, 40, 45]); + expect(progress.map(({ current }) => current)).to.deep.equal([0, 20, 0, 20, 0, 5, 0]); expect(results.every(({ status }) => status === "imported")).to.equal(true); }); + it("shows the batch in flight as importing and later ones as queued, never as not attempted", async () => { + const snapshots: { progress: ImportProgress; summary: Record }[] = []; + await runImportBatches({ + batches: planImportBatches(request(), pubkeys), + results: pubkeys.map(notAttempted), + importBatch: async (batch) => imported(batch), + onProgress: (progress, results) => snapshots.push({ progress, summary: summarizeImport(results) }), + shouldStop: () => false + }); + // While the second batch is in flight: 20 done, 20 importing, 5 queued. + const second = snapshots.find(({ progress }) => progress.batch === 1 && progress.current > 0)!; + expect(second.progress).to.deep.equal({ done: 20, total: 45, batch: 1, batches: 3, current: 20 }); + expect(second.summary).to.deep.equal({ imported: 20, importing: 20, queued: 5 }); + expect(snapshots.some(({ summary }) => summary.not_attempted)).to.equal(false); + expect(snapshots.at(-1)!.summary).to.deep.equal({ imported: 45 }); + expect(snapshots.at(-1)!.progress.current).to.equal(0); + }); + it("stops after a failed batch and leaves the rest retryable", async () => { let calls = 0; const { results, stopped } = await runImportBatches({