diff --git a/src/commands/create.ts b/src/commands/create.ts index f0b6ae75e..e71e4f7cc 100644 --- a/src/commands/create.ts +++ b/src/commands/create.ts @@ -55,6 +55,7 @@ import { isNodeVersionSupported, isPythonVersionSupported, printJsonToStdout, + retryTransient, setLocalConfig, setLocalEnv, } from '../lib/utils.js'; @@ -214,7 +215,7 @@ export class CreateCommand extends ApifyCommand { // Start fetching manifest immediately to prevent // annoying delays that sometimes happen on CLI startup. - const manifestPromise = fetchManifest().catch((err) => { + const manifestPromise = retryTransient(async () => fetchManifest()).catch((err) => { return new Error(`Could not fetch template list from server. Cause: ${err?.message}`); }); diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 0bdfea073..e1a72a39f 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -703,12 +703,39 @@ export const isNodeVersionSupported = (installedNodeVersion: string) => { return gte(installedNodeVersion, minimumSupportedNodeVersion); }; +const TRANSIENT_ATTEMPTS = 3; +const TRANSIENT_RETRY_DELAY_MS = 1000; +const TRANSIENT_HTTP_STATUSES = new Set([408, 429, 500, 502, 503, 504]); + +/** + * Retries a network call that failed with a thrown error, or whose result `shouldRetry` marks as transient. + * The last attempt's result or error is passed through unchanged. + */ +export async function retryTransient(fn: () => Promise, shouldRetry: (result: T) => boolean = () => false) { + for (let attempt = 1; ; attempt++) { + const isLastAttempt = attempt === TRANSIENT_ATTEMPTS; + + try { + const result = await fn(); + if (isLastAttempt || !shouldRetry(result)) return result; + } catch (err) { + if (isLastAttempt) throw err; + } + + await new Promise((resolve) => setTimeout(resolve, TRANSIENT_RETRY_DELAY_MS * attempt)); + } +} + export const downloadZip = async (url: string) => { - const response = await axios.get(url, { - responseType: 'arraybuffer', - validateStatus: () => true, - headers: { 'User-Agent': `Apify CLI/${useCLIMetadata().version} (https://github.com/apify/apify-cli)` }, - }); + const response = await retryTransient( + async () => + axios.get(url, { + responseType: 'arraybuffer', + validateStatus: () => true, + headers: { 'User-Agent': `Apify CLI/${useCLIMetadata().version} (https://github.com/apify/apify-cli)` }, + }), + (res) => TRANSIENT_HTTP_STATUSES.has(res.status), + ); if (response.status < 200 || response.status >= 300) { const body = Buffer.from(response.data).toString('utf8').trim().slice(0, 500); diff --git a/test/__setup__/build-utils.ts b/test/__setup__/build-utils.ts index eea798373..a2f217531 100644 --- a/test/__setup__/build-utils.ts +++ b/test/__setup__/build-utils.ts @@ -18,3 +18,25 @@ export const waitForBuildToFinishWithTimeout = async (client: ApifyClient, build const result = await Promise.race([buildPromise, timeoutPromise]); if (!result) throw new Error(`Timed out after ${timeoutSecs} seconds`); }; + +/** + * Waits until the platform reports a `latest` build for the Actor, then waits for that build to finish. + * The builds list and tagged builds are updated asynchronously after `apify push` returns. + */ +export const waitForLatestBuildToFinish = async (client: ApifyClient, actorId: string, timeoutSecs = 60) => { + const deadline = Date.now() + timeoutSecs * 1000; + + while (Date.now() < deadline) { + const actor = await client.actor(actorId).get(); + const buildId = actor?.taggedBuilds?.latest?.buildId; + + if (buildId) { + await waitForBuildToFinishWithTimeout(client, buildId, Math.ceil((deadline - Date.now()) / 1000)); + return; + } + + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + + throw new Error(`No latest build appeared for Actor ${actorId} within ${timeoutSecs} seconds`); +}; diff --git a/test/api/commands/call.test.ts b/test/api/commands/call.test.ts index 20f6f4dbc..91e263b54 100644 --- a/test/api/commands/call.test.ts +++ b/test/api/commands/call.test.ts @@ -2,7 +2,7 @@ import { readFileSync, writeFileSync } from 'node:fs'; import { testRunCommand } from '../../../src/lib/command-framework/apify-command.js'; import { getLocalKeyValueStorePath } from '../../../src/lib/utils.js'; -import { waitForBuildToFinishWithTimeout } from '../../__setup__/build-utils.js'; +import { waitForLatestBuildToFinish } from '../../__setup__/build-utils.js'; import { testUserClient } from '../../__setup__/config.js'; import { TEST_TIMEOUT } from '../../__setup__/consts.js'; import { safeLogin, useAuthSetup } from '../../__setup__/hooks/useAuthSetup.js'; @@ -75,9 +75,7 @@ describe('[api] apify call', () => { actorId = `${username}/${ACTOR_NAME}`; // Build must finish before doing `apify call`, otherwise we would get nonexisting build with "LATEST" tag error. - const builds = await testUserClient.actor(actorId).builds().list(); - const lastBuild = builds.items.pop(); - await waitForBuildToFinishWithTimeout(testUserClient, lastBuild!.id); + await waitForLatestBuildToFinish(testUserClient, actorId); apifyId = await testUserClient .actor(actorId) diff --git a/test/api/commands/task/run.test.ts b/test/api/commands/task/run.test.ts index 9fb0c773b..5ae797dd2 100644 --- a/test/api/commands/task/run.test.ts +++ b/test/api/commands/task/run.test.ts @@ -1,7 +1,7 @@ import { writeFileSync } from 'node:fs'; import { testRunCommand } from '../../../../src/lib/command-framework/apify-command.js'; -import { waitForBuildToFinishWithTimeout } from '../../../__setup__/build-utils.js'; +import { waitForLatestBuildToFinish } from '../../../__setup__/build-utils.js'; import { testUserClient } from '../../../__setup__/config.js'; import { TEST_TIMEOUT } from '../../../__setup__/consts.js'; import { safeLogin, useAuthSetup } from '../../../__setup__/hooks/useAuthSetup.js'; @@ -66,9 +66,7 @@ describe('[api] apify task run', () => { actorId = `${username}/${actName}`; // Build must finish before doing `apify call`, otherwise we would get nonexisting build with "LATEST" tag error. - const builds = await testUserClient.actor(actorId).builds().list(); - const lastBuild = builds.items.pop(); - await waitForBuildToFinishWithTimeout(testUserClient, lastBuild!.id); + await waitForLatestBuildToFinish(testUserClient, actorId); // Make a task for this actor const task = await testUserClient.tasks().create({ diff --git a/test/local/lib/utils.test.ts b/test/local/lib/utils.test.ts index 719efa9d2..2b2a69fa5 100644 --- a/test/local/lib/utils.test.ts +++ b/test/local/lib/utils.test.ts @@ -95,6 +95,17 @@ describe('Utils', () => { await expect(downloadAndUnzip({ url: 'https://example.com/a.zip', pathTo: '.' })).rejects.toThrow(/HTTP 403/); }); + it('should retry transient failures', async () => { + const get = vitest + .spyOn(axios, 'get') + .mockRejectedValueOnce(new Error('socket hang up')) + .mockResolvedValueOnce({ status: 503, data: Buffer.from('unavailable') }) + .mockResolvedValue({ status: 404, data: Buffer.from('gone') }); + + await expect(downloadAndUnzip({ url: 'https://example.com/a.zip', pathTo: '.' })).rejects.toThrow(/HTTP 404/); + expect(get).toHaveBeenCalledTimes(3); + }); + it('should throw an actionable error when the response body is not a zip', async () => { vitest.spyOn(axios, 'get').mockResolvedValue({ status: 200, data: Buffer.from('block page') });