diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a54b639c1..3675cbf4e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,9 +87,9 @@ jobs: # workflow cannot exfiltrate it. The `if` also keeps the job off everything except a direct push to # the `main` branch: tag pushes and `workflow_call` (release) carry a non-main ref, so they are # skipped — which both avoids a branch-policy failure and matches where the secret is allowed, and - # it is scoped to the canonical repo so forks don't run it. It deploys a UNIQUELY-named worker - # (VOID_E2E_RUN_ID) to a throwaway account and tears it down. A preflight step fails the job when a - # credential is empty (unset / deleted / not-yet-rotated token), so a missing secret can never turn + # it is scoped to the canonical repo so forks don't run it. It updates a reusable test Worker + # and retains its resources; job concurrency serializes that shared target. A preflight step fails + # the job when a credential is empty (unset / deleted / not-yet-rotated token), so a missing secret can never turn # the live-deploy gate into a silent green no-op (the `test` job's uncredentialed run still skips). live-e2e: if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'voidzero-dev/void' }} @@ -97,10 +97,13 @@ jobs: permissions: contents: read environment: cloudflare-e2e + concurrency: + group: cloudflare-live-e2e + cancel-in-progress: false env: VOID_E2E_CF_TOKEN: ${{ secrets.VOID_E2E_CF_TOKEN }} VOID_E2E_CF_ACCOUNT_ID: ${{ secrets.VOID_E2E_CF_ACCOUNT_ID }} - VOID_E2E_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} + VOID_E2E_NAMESPACE: ci steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/docs/guide/platform/installation/prerequisites.md b/docs/guide/platform/installation/prerequisites.md index 06d59f2f7..300b57e79 100644 --- a/docs/guide/platform/installation/prerequisites.md +++ b/docs/guide/platform/installation/prerequisites.md @@ -13,16 +13,17 @@ To run a Void platform, you need: Void creates the Workers, storage, queues, routing, and database tables through the CLI. -Workers for Platforms requires a paid plan. Your database and Cloudflare usage are billed separately. +Workers for Platforms requires its own paid plan. Analytics Engine is available on Workers Free, but you must enable it for the account. Review [Analytics Engine pricing and allowances](https://developers.cloudflare.com/analytics/analytics-engine/pricing/). Your database and Cloudflare usage are billed separately. ## Prepare Your Cloudflare Account and Domain In the [Cloudflare dashboard](https://dash.cloudflare.com/), select the account where you want the platform to live: 1. Open **Workers & Pages** once to let Cloudflare provision Workers for the account. -2. Open **Workers for Platforms** and enable its plan. Review [Workers for Platforms pricing](https://developers.cloudflare.com/cloudflare-for-platforms/workers-for-platforms/platform/pricing/) before confirming. +2. Open **Workers for Platforms** and enable its plan. Review [Workers for Platforms pricing](https://developers.cloudflare.com/cloudflare-for-platforms/workers-for-platforms/reference/pricing/) before confirming. 3. Open **R2 Object Storage** and complete its activation. Void creates the bucket later. -4. For a domain installation, choose a domain you own, such as `example.app`. If you need one, register it with your preferred registrar. Use a spare domain's root: apps will be served at `my-app.example.app`. For workers.dev testing, skip this step and the DNS setup below. +4. Open **Analytics Engine** and select **Enable** if prompted. You do not need to create a dataset manually. If the page has no **Enable** action and installation reports code 10089, follow the [steps below](#if-cloudflare-rejects-analytics-engine). +5. For a domain installation, choose a domain you own, such as `example.app`. If you need one, register it with your preferred registrar. Use a spare domain's root: apps will be served at `my-app.example.app`. For workers.dev testing, skip this step and the DNS setup below. If that domain is already in this Cloudflare account, use its existing zone. Otherwise, [add the domain to Cloudflare](https://developers.cloudflare.com/dns/zone-setups/full-setup/setup/) and follow the nameserver instructions until the zone is active. A zone is Cloudflare's DNS configuration for a domain; creating one does not buy the domain. @@ -30,7 +31,9 @@ You can also let Void create the zone during installation. Setting it up now let ### If Cloudflare rejects Analytics Engine -The platform stores request metrics and logs in Analytics Engine. If installation stops with "You need to enable Analytics Engine" (code 10089), add an Analytics Engine binding to a Worker that is not part of the platform under **Settings** > **Bindings**, and deploy it once from the dashboard. A new Hello World Worker works. Do not edit the platform's own Workers. Then run the printed `void platform install --resume` command. +The platform stores request metrics and logs in Analytics Engine. If installation stops with "You need to enable Analytics Engine" (code 10089), open the Analytics Engine dashboard link in the error and select **Enable** for that account. Activation is available on Workers Free; this error does not require a Workers plan upgrade or a replacement API token. Then run the printed `void platform install --resume` command. + +If your account's dashboard has no **Enable** action, create a disposable Hello World Worker, add an Analytics Engine binding under **Settings** > **Bindings**, and deploy it once from the dashboard. Then resume installation. Use a separate Worker for this step; leave the platform's own Workers unchanged. ## Install the CLI and Preview @@ -82,6 +85,8 @@ Use any PostgreSQL provider that accepts connections from your computer and Clou | [PlanetScale Postgres](https://planetscale.com/docs/postgres/connecting) | Choose **Postgres**, not Vitess/MySQL. Open **Connect**, create role credentials, and use the direct primary connection on port `5432`, not PgBouncer on `6432`. | | [Neon](https://neon.com/docs/get-started-with-neon/connect-neon) | Open **Connect**, select the database and owner role, and turn **connection pooling off**. | +When sizing your database, include the connection pools for every platform installation and application sharing that server. Adjust each Hyperdrive's [origin connection limit](https://developers.cloudflare.com/hyperdrive/configuration/tune-connection-pool/) to fit the server's capacity, leaving room for migrations, administration, provider-reserved connections, and temporary bursts. + Once installation claims the database, continue using that same database for resume and maintenance commands. If installation stops because it cannot use the database, for example because the database is not empty, set `VOID_PLATFORM_DATABASE_URL` to another database's URL before you resume. Uninstall never deletes external PostgreSQL. ### Use a provider CLI diff --git a/packages/void/src/cli/platform-cloudflare.ts b/packages/void/src/cli/platform-cloudflare.ts index 43a1e8cd7..27713b12c 100644 --- a/packages/void/src/cli/platform-cloudflare.ts +++ b/packages/void/src/cli/platform-cloudflare.ts @@ -1,4 +1,5 @@ import { createHash, randomUUID } from 'node:crypto'; +import { formatUrl } from './output.ts'; import { resolvePinnedApiBaseUrl } from './provision.ts'; type CloudflareEnvelope = Readonly<{ @@ -25,6 +26,16 @@ export class PlatformCloudflareApiError extends Error { } } +export class PlatformCloudflarePrerequisiteError extends PlatformCloudflareApiError { + override readonly cause: PlatformCloudflareApiError; + + constructor(message: string, cause: PlatformCloudflareApiError) { + super(message, cause.status, cause.requestPath); + this.name = 'PlatformCloudflarePrerequisiteError'; + this.cause = cause; + } +} + export class PlatformWorkerCustomDomainsChangedError extends Error { constructor(message: string, options?: ErrorOptions) { super(message, options); @@ -326,11 +337,17 @@ export class PlatformCloudflareClient { } if (!response.ok || envelope.success !== true || envelope.result === undefined) { const details = cloudflareErrorDetails(envelope, response.status); - throw new PlatformCloudflareApiError( - `Cloudflare API error: ${details} (HTTP ${response.status}; ${init?.method ?? 'GET'} ${path.split('?')[0]}).`, + const requestPath = path.split('?')[0]; + const error = new PlatformCloudflareApiError( + `Cloudflare API error: ${details} (HTTP ${response.status}; ${init?.method ?? 'GET'} ${requestPath}).`, response.status, - path.split('?')[0], + requestPath, ); + const prerequisite = cloudflarePrerequisiteHelp(envelope, response.status, requestPath); + if (prerequisite) { + throw new PlatformCloudflarePrerequisiteError(prerequisite, error); + } + throw error; } return { result: envelope.result, resultInfo: envelope.result_info }; } @@ -1427,6 +1444,37 @@ function deletionPath( } } +function cloudflarePrerequisiteHelp( + envelope: CloudflareEnvelope, + status: number, + path: string, +): string | undefined { + if (status !== 403) { + return; + } + const [, scope, accountId, service, resource] = path.split('/'); + if (scope !== 'accounts' || !accountId) { + return; + } + if (service === 'r2' && envelope.errors?.some((error) => error.code === 10042)) { + return ( + 'Cloudflare R2 is not enabled for this account (code 10042). ' + + `Complete R2 activation at ${formatUrl(`https://dash.cloudflare.com/${accountId}/r2/overview`)}, then retry the same command.` + ); + } + if ( + service === 'workers' && + resource === 'dispatch' && + envelope.errors?.some((error) => error.code === 10121) + ) { + return ( + 'Cloudflare Workers for Platforms requires a paid subscription for this account (code 10121). ' + + `Enable it at ${formatUrl(`https://dash.cloudflare.com/${accountId}/workers-for-platforms`)}. ` + + 'Enterprise customers can contact their account team. Then retry the same command.' + ); + } +} + function cloudflareErrorDetails(envelope: CloudflareEnvelope, status: number): string { return ( envelope.errors diff --git a/packages/void/src/cli/platform-installer-cutover.ts b/packages/void/src/cli/platform-installer-cutover.ts index c77bb2905..792174fbb 100644 --- a/packages/void/src/cli/platform-installer-cutover.ts +++ b/packages/void/src/cli/platform-installer-cutover.ts @@ -86,6 +86,13 @@ export function recordWorkerSecretUpdateState( ); } +export function workerCutoverUpdatesSecrets( + manifest: PlatformInstallationManifest, + generation: number, +): boolean { + return manifest.steps.some((step) => step.key === `worker-secret-updates:${generation}:pending`); +} + export function recordWorkerCutoverStage( manifest: PlatformInstallationManifest, generation: number, diff --git a/packages/void/src/cli/platform-installer-errors.ts b/packages/void/src/cli/platform-installer-errors.ts index 5efdb6d41..0c0f28028 100644 --- a/packages/void/src/cli/platform-installer-errors.ts +++ b/packages/void/src/cli/platform-installer-errors.ts @@ -4,11 +4,10 @@ import { stabilizeCloudflareVersionTriggers, type CloudflareCommandResult, } from '@void/deploy-cloudflare'; +import { formatUrl } from './output.ts'; // Cloudflare refuses a Worker with an Analytics Engine binding (code 10089) -// until the account has used Analytics Engine once. No API turns it on, and -// the dashboard page may offer only "Create dataset"; deploying a binding from -// the dashboard once enables it (cloudflare/workers-sdk#9312). +// until Analytics Engine is enabled for the account, including on Workers Free. const ANALYTICS_ENGINE_NOT_ENABLED = 10089; // Without a terminal, the Cloudflare CLI stops an upload when it must ask @@ -67,6 +66,7 @@ function uploadConflictMessage(result: CloudflareCommandResult, operation: strin export function assertCleanPlatformWrangler( result: CloudflareCommandResult, operation: string, + accountId?: string, ): void { if (result.status !== 0 || result.signal || result.error) { const conflict = uploadConflictMessage(result, operation); @@ -80,12 +80,22 @@ export function assertCleanPlatformWrangler( const analyticsEngineDisabled = result.errorCode === ANALYTICS_ENGINE_NOT_ENABLED || result.errors?.some((error) => error.code === ANALYTICS_ENGINE_NOT_ENABLED) || - /\b10089\b/.test(diagnostic ?? ''); + (result.errorCode === undefined && + !result.errors?.some((error) => error.code !== undefined) && + (diagnostic?.includes('[code: 10089]') || diagnostic?.includes('[10089]'))); + if (analyticsEngineDisabled) { + const dashboard = accountId + ? `https://dash.cloudflare.com/${accountId}/workers/analytics-engine` + : 'https://dash.cloudflare.com/'; + throw new Error( + `platform: Cloudflare could not ${operation}: Analytics Engine is not enabled for this account (code 10089). ` + + `Open ${formatUrl(dashboard)}${accountId ? '' : ', select your account, and open Analytics Engine'}, then select Enable. ` + + 'Analytics Engine is available on Workers Free. After enabling it, use the printed recovery command if one is shown; otherwise retry this command. ' + + `If Enable is unavailable, see ${formatUrl('https://void.cloud/guide/platform/installation/prerequisites#if-cloudflare-rejects-analytics-engine')}.`, + ); + } throw new Error( - `platform: Wrangler failed to ${operation} (${diagnostic || `exit ${result.status}`}).` + - (analyticsEngineDisabled - ? ' If the Analytics Engine page offers no way to enable it, follow https://void.cloud/guide/platform/installation/prerequisites#if-cloudflare-rejects-analytics-engine, then retry.' - : ''), + `platform: Cloudflare failed to ${operation} (${diagnostic || `exit ${result.status}`}).`, ); } } diff --git a/packages/void/src/cli/platform-installer-health.ts b/packages/void/src/cli/platform-installer-health.ts index a92480e70..4bd42d4ac 100644 --- a/packages/void/src/cli/platform-installer-health.ts +++ b/packages/void/src/cli/platform-installer-health.ts @@ -1,3 +1,4 @@ +import { setTimeout as delay } from 'node:timers/promises'; import { normalizeApplicationDomain, PLATFORM_HEALTH_HOSTNAME_LABEL } from '@void/platform'; import { cfAccessHeaders } from '../shared/cf-access.ts'; import { type PlatformHealthCheckContext } from './platform-installer-types.ts'; @@ -72,16 +73,28 @@ export async function verifyPlatformHealth( dispatchOrigin?: string, context?: PlatformHealthCheckContext, ): Promise { - const request = (url: string) => - fetcher(url, { - headers: { - 'X-Void-Platform-Probe': healthProbeToken, - ...cfAccessHeaders(process.env, url), - }, - // A login redirect must not forward the installation's probe credential. - redirect: 'manual', - signal: AbortSignal.timeout(30_000), - }); + const request = async (url: string): Promise => { + for (let attempt = 0; ; attempt++) { + try { + return await fetcher(url, { + headers: { + 'X-Void-Platform-Probe': healthProbeToken, + ...cfAccessHeaders(process.env, url), + }, + // A login redirect must not forward the installation's probe credential. + redirect: 'manual', + signal: AbortSignal.timeout(30_000), + }); + } catch (error) { + if (attempt === 2) { + throw new Error(`platform: Health check could not reach ${url} after 3 attempts.`, { + cause: error, + }); + } + await delay(250 * (attempt + 1)); + } + } + }; // Project Zero Trust leaves only this application-domain hostname outside Access. const dispatchUrl = dispatchOrigin ?? `https://${PLATFORM_HEALTH_HOSTNAME_LABEL}.${applicationBaseDomain}`; diff --git a/packages/void/src/cli/platform-installer-inspection.ts b/packages/void/src/cli/platform-installer-inspection.ts index 39326162e..c5dffba80 100644 --- a/packages/void/src/cli/platform-installer-inspection.ts +++ b/packages/void/src/cli/platform-installer-inspection.ts @@ -30,7 +30,7 @@ export async function readActivePlatformWorkerVersion( true, ); if (result.status !== 0) { - assertCleanPlatformWrangler(result, `read active version for ${workerName}`); + assertCleanPlatformWrangler(result, `read active version for ${workerName}`, accountId); } const value = result.data; if ( @@ -136,7 +136,7 @@ export async function readPlatformWorkerDeployments( true, ); if (result.status !== 0) { - assertCleanPlatformWrangler(result, `read deployment lineage for ${workerName}`); + assertCleanPlatformWrangler(result, `read deployment lineage for ${workerName}`, accountId); } const value = result.data; if (!Array.isArray(value)) { @@ -342,7 +342,7 @@ export async function readPlatformWorkerVersionSnapshot( process.cwd(), true, ); - assertCleanPlatformWrangler(result, `read rollback metadata for ${workerName}`); + assertCleanPlatformWrangler(result, `read rollback metadata for ${workerName}`, accountId); const value = result.data; if (!value || typeof value !== 'object') { throw new Error(`platform: Wrangler returned malformed version metadata for ${workerName}.`); diff --git a/packages/void/src/cli/platform-installer-outside-changes.ts b/packages/void/src/cli/platform-installer-outside-changes.ts index 2f5720e26..42f759e0d 100644 --- a/packages/void/src/cli/platform-installer-outside-changes.ts +++ b/packages/void/src/cli/platform-installer-outside-changes.ts @@ -7,7 +7,10 @@ import { PlatformCloudflareApiError, type PlatformCloudflareClient, } from './platform-cloudflare.ts'; -import { interruptedWorkerCutover } from './platform-installer-cutover.ts'; +import { + interruptedWorkerCutover, + workerCutoverUpdatesSecrets, +} from './platform-installer-cutover.ts'; import { platformWorkerVersionHasTriggerRecord, readActivePlatformWorkerDeployment, @@ -116,9 +119,9 @@ function needsReviewUpload(change: PlatformWorkerOutsideChange): boolean { * The earlier version of each Worker that the installer makes active again * before it deploys. A run that stopped during a Worker cutover left Void's * version active, and the next run first restores the earlier one. Like the - * installer, keep Void's version while the platform changes to configurable - * sign-in and already requires sign-in. Every Worker that this check reads - * exists, so the installation had Workers before. + * installer, retain rotated credentials or a configurable sign-in transition + * that already requires sign-in. Every Worker that this check reads exists, + * so the installation had Workers before. */ async function restoredPlatformWorkerVersions( installation: PlatformOutsideChangeInstallation, @@ -127,6 +130,7 @@ async function restoredPlatformWorkerVersions( const cutover = interruptedWorkerCutover(installation.manifest); if ( !cutover || + workerCutoverUpdatesSecrets(installation.manifest, cutover.generation) || (installation.manifest.deployment.authentication.configurable !== true && runtime.manifest.authenticationProtocolVersion === 1 && (await installation.readAuthenticationState?.())?.sessionsRequired === true) diff --git a/packages/void/src/cli/platform-installer-workers.ts b/packages/void/src/cli/platform-installer-workers.ts index bdfe30ee8..60cab9551 100644 --- a/packages/void/src/cli/platform-installer-workers.ts +++ b/packages/void/src/cli/platform-installer-workers.ts @@ -39,6 +39,7 @@ import { recordWorkerTrafficState, withoutCompletedPlatformWorkerMigrations, resolvePlatformPreviousTriggerConfig, + workerCutoverUpdatesSecrets, } from './platform-installer-cutover.ts'; import { assertPlatformTriggerStabilization, @@ -126,8 +127,8 @@ export function platformWorkerTrafficCallbacks(input: { }; } -// Restore staged Workers before starting a new generation, then replan any -// newly created Worker that recovery had to retain. +// Restore staged Workers, or retain their rotated credentials, before starting +// a new guarded generation. Replan any newly created Worker recovery retained. export async function recoverInterruptedPlatformWorkerCutover( input: Readonly<{ manifest: PlatformInstallationManifest; @@ -165,6 +166,10 @@ export async function recoverInterruptedPlatformWorkerCutover( } = input; let { manifest, workerActions } = input; if (interruptedCutover) { + const retainUpdatedCredentials = workerCutoverUpdatesSecrets( + manifest, + interruptedCutover.generation, + ); const recoveryOrder: ReadonlyArray = [ 'dispatch', 'tail', @@ -189,7 +194,32 @@ export async function recoverInterruptedPlatformWorkerCutover( { ...previousConfigs[key], name: snapshot.workerName }, preservedRoutes, ); - if (dependencies.recoverWorkerCutover) { + if (retainUpdatedCredentials) { + // Recovery may have restored some Workers before Cloudflare refused an + // old secret version. Accept only this cutover's old or staged version; + // the next guarded generation reapplies the authorized credentials. + const directory = mkdtempSync(join(tmpdir(), 'void-platform-resume-')); + try { + const configPath = join(directory, 'wrangler.json'); + writeFileSync(configPath, JSON.stringify(recoveryConfig), { mode: 0o600 }); + const currentVersionId = await readActivePlatformWorkerVersion( + configPath, + plan.account.id, + snapshot.workerName, + runPinnedPlatformWrangler, + ); + if ( + currentVersionId !== snapshot.stagedVersionId && + (snapshot.previousVersionId === null || currentVersionId !== snapshot.previousVersionId) + ) { + throw new Error( + `Worker ${snapshot.workerName} changed concurrently after the credential update; refusing to overwrite it.`, + ); + } + } finally { + rmSync(directory, { force: true, recursive: true }); + } + } else if (dependencies.recoverWorkerCutover) { await dependencies.recoverWorkerCutover({ key, preserveCustomDomains: interruptedCutover.unsafeTraffic.has(key), @@ -232,13 +262,15 @@ export async function recoverInterruptedPlatformWorkerCutover( manifest = recordWorkerCutoverResolution( manifest, interruptedCutover.generation, - 'restored', + retainUpdatedCredentials ? 'committed' : 'restored', now(), ); manifest = recordWorkerTrafficState( manifest, interruptedCutover.generation, - previousPublicEndpoints && interruptedCutover.unsafeTraffic.size === 0, + !retainUpdatedCredentials && + previousPublicEndpoints && + interruptedCutover.unsafeTraffic.size === 0, now(), ); onManifestChange(manifest); @@ -745,6 +777,7 @@ export async function deployPlatformWorker( assertCleanPlatformWrangler( uploaded, `${options.workerExisted ? 'upload' : 'bootstrap'} ${workerName}`, + accountId, ); } catch (error) { uploadError = error; @@ -792,7 +825,7 @@ export async function deployPlatformWorker( directory, true, ); - assertCleanPlatformWrangler(activated, `activate ${workerName}`); + assertCleanPlatformWrangler(activated, `activate ${workerName}`, accountId); } const deploymentsAfterActivation = await readPlatformWorkerDeployments( configPath, @@ -1230,7 +1263,11 @@ export async function restorePlatformWorkerDeployment(input: { input.directory, true, ); - assertCleanPlatformWrangler(restored, `restore deployment ${input.restoration.id}`); + assertCleanPlatformWrangler( + restored, + `restore deployment ${input.restoration.id}`, + input.accountId, + ); const after = await readPlatformWorkerDeployments( input.configPath, input.accountId, diff --git a/packages/void/src/cli/platform-installer.ts b/packages/void/src/cli/platform-installer.ts index aa161e3ea..09ef67d54 100644 --- a/packages/void/src/cli/platform-installer.ts +++ b/packages/void/src/cli/platform-installer.ts @@ -29,6 +29,7 @@ import { enableInstallationServices } from './platform-install-activation.ts'; import { prepareInstallationAuthentication } from './platform-install-preparation.ts'; import { provisionInstallationServices } from './platform-install-provisioning.ts'; import { uploadInstallationWorkers } from './platform-install-workers.ts'; +import { workerCutoverUpdatesSecrets } from './platform-installer-cutover.ts'; import { assertVoidCreatedHyperdriveTrustsDatabase, claimPlatformDatabase, @@ -101,6 +102,7 @@ async function recoverFailedPlatformInstallation( workerTrafficGuarded: boolean; workerDeploymentCommitted: boolean; workerRollbacks: Array; + workerSecretUpdateIntent: PlatformWorkerSecretUpdateIntent | null; authenticationEnforcementActive: boolean; authenticationProtocolTransition: boolean; workerCutoverGeneration: number; @@ -145,11 +147,13 @@ async function recoverFailedPlatformInstallation( ); } } - // With authentication enforced mid-transition, traffic stays closed and a - // retry resumes toward a fully compatible Worker set instead of rolling back. + // Rotated credentials cannot safely be rolled back to immutable versions + // containing the old secrets. Keep traffic closed and roll forward on retry. + // Authentication transitions likewise retain the compatible Worker set. if ( !workerDeploymentCommitted && workerRollbacks.length > 0 && + !state.workerSecretUpdateIntent && !(authenticationEnforcementActive && authenticationProtocolTransition) ) { const rollbackFailures: Array = []; @@ -357,9 +361,13 @@ export async function installPlatform( const workerSecretUpdateIntent: PlatformWorkerSecretUpdateIntent | null = hasExplicitWorkerSecretUpdates ? { - baselineGeneration: latestPlatformWorkerCutoverGeneration(manifest), - generation: null, - id: randomUUID(), + // Keep the recovery authority until the next generation is durably + // staged, including a crash after resolving the previous cutover. + baselineGeneration: + resumedWorkerSecretUpdateIntent?.baselineGeneration ?? + latestPlatformWorkerCutoverGeneration(manifest), + generation: resumedWorkerSecretUpdateIntent?.generation ?? null, + id: resumedWorkerSecretUpdateIntent?.id ?? randomUUID(), schema: 1, updates: { ...resumedWorkerSecretUpdateIntent?.updates, @@ -574,6 +582,9 @@ export async function installPlatform( !preserveDisabled && targetSandboxDrainProtocol >= 1 && sandboxDrainState === 'pending'; const requiresTrafficGuard = !previousPublicEndpoints || + (hadPriorWorkers && !!state.workerSecretUpdateIntent) || + (!!interruptedCutover && + workerCutoverUpdatesSecrets(state.manifest, interruptedCutover.generation)) || recoveringUnsafeTraffic || createsWorker || recoveringCreatedWorker || @@ -703,6 +714,13 @@ export async function installPlatform( })); state.workerCutoverGeneration = nextWorkerCutoverGeneration(state.manifest); if (state.workerSecretUpdateIntent) { + // Keep this authorized intent resumable on either side of the next + // manifest checkpoint before binding it to the new generation. + state.workerSecretUpdateIntent = { ...state.workerSecretUpdateIntent, generation: null }; + writePendingInstallationSecrets( + state.manifest, + secretCheckpoint(state.secrets, state.workerSecretUpdateIntent), + ); state.manifest = recordWorkerSecretUpdateState( state.manifest, state.workerCutoverGeneration, diff --git a/packages/void/src/cli/platform-plan.ts b/packages/void/src/cli/platform-plan.ts index 104e12755..7e11f7319 100644 --- a/packages/void/src/cli/platform-plan.ts +++ b/packages/void/src/cli/platform-plan.ts @@ -19,7 +19,10 @@ import { type PlatformInstallationManifest, } from '@void/platform'; import { pc } from './output.ts'; -import { PlatformCloudflareApiError } from './platform-cloudflare.ts'; +import { + PlatformCloudflareApiError, + PlatformCloudflarePrerequisiteError, +} from './platform-cloudflare.ts'; import type { CloudflareAccount, CloudflareDnsRecord, @@ -346,6 +349,14 @@ export async function createPlatformInstallPlan( } throw error; }); + const prerequisites: Array = []; + const collectPrerequisite = (error: unknown): [] => { + if (!(error instanceof PlatformCloudflarePrerequisiteError)) { + throw error; + } + prerequisites.push(error); + return []; + }; const [ aiGateways, routingKv, @@ -363,17 +374,26 @@ export async function createPlatformInstallPlan( client.listKvNamespaces(account.id, names.routingKv), client.listKvNamespaces(account.id, names.isrCacheKv), client.listKvNamespaces(account.id, names.legacyAssetsKv), - client.listR2Buckets(account.id), + client.listR2Buckets(account.id).catch(collectPrerequisite), client.listQueues(account.id, names.usageQueue), client.listQueues(account.id, names.cronQueue), (options?.deploymentExecution ?? existing?.resources.some((resource) => resource.key === 'deployment-queue')) ? client.listQueues(account.id, names.deploymentQueue) : Promise.resolve([]), - client.listDispatchNamespaces(account.id), + client.listDispatchNamespaces(account.id).catch(collectPrerequisite), client.listHyperdriveConfigs(account.id), client.listWorkerNames(account.id), ]); + if (prerequisites.length === 1) { + throw prerequisites[0]; + } + if (prerequisites.length > 1) { + throw new AggregateError( + prerequisites, + `platform: Cloudflare account prerequisites are missing:\n\n${prerequisites.map((error) => error.message).join('\n\n')}`, + ); + } const dnsRecords = zone ? await listZoneDnsRecords(client, zone, wildcardHostname) : []; const remoteOwnership = existing ? await readRemoteOwnership(client, account.id, routingKv, existing) diff --git a/packages/void/test/e2e/README.md b/packages/void/test/e2e/README.md index f481384b4..8d4bb497a 100644 --- a/packages/void/test/e2e/README.md +++ b/packages/void/test/e2e/README.md @@ -1,89 +1,69 @@ # `packages/void/test/e2e` -Gated live-account end-to-end tests. These run REAL Cloudflare operations against a REAL account, so -they **skip unless credentials are present**. Locally that means exporting the env vars below; in CI -they run on **pushes to `main`** (see [In CI](#in-ci)). +Gated live-account end-to-end tests. These run real Cloudflare operations against the selected +account and skip when credentials are absent. They never delete remote resources. ## `cloudflare-deploy-live.test.ts` -Deploys `playground/spa` (D1 + KV + cron) to the target account via `void deploy --platform cloudflare`, -verifies the live worker answers HTTP 200, then deletes the worker and every resource it created. +Deploys an isolated copy of `playground/spa` (D1 + KV + cron) through +`void deploy --platform cloudflare`. It checks HTTP responses, D1 and KV reads/writes, the registered +cron, and a second deployment that preserves resource IDs and test data. -The test starts with an intentionally incomplete root config and runs only -`void deploy --platform cloudflare`. The deploy automatically creates D1 + KV, persists their real -ids, builds, applies the checked migration, and uploads the worker. This exercises the complete -one-command path in the same non-interactive environment used by CI. +The first run creates missing D1 and KV resources. Later runs reuse the same Worker and the actual +IDs of its named test database and namespace. The generated config, lockfile, and build output stay +in the ignored fixture directory printed by the test. The tracked playground is not modified. ### Prerequisites -The suite is skipped unless BOTH env vars are set: +The suite skips unless both credential variables are set. Credentialed runs also require an explicit, +stable namespace reserved for this suite in the selected test account. -| Env var | Purpose | -| ------------------------ | -------------------------------------------------------------------------- | -| `VOID_E2E_CF_TOKEN` | Cloudflare API token that can create + delete Workers Scripts, D1, and KV. | -| `VOID_E2E_CF_ACCOUNT_ID` | Cloudflare account id to deploy into. | +| Environment variable | Purpose | +| ------------------------ | ----------------------------------------------------------------------------------------------- | +| `VOID_E2E_CF_TOKEN` | Cloudflare API token with Workers Scripts: Edit, D1: Edit, and Workers KV Storage: Edit. | +| `VOID_E2E_CF_ACCOUNT_ID` | Test account ID. | +| `VOID_E2E_NAMESPACE` | Short lowercase account-local namespace, such as a team or operator name. Reuse it across runs. | -The token needs, on that account, **Workers Scripts: Edit**, **D1: Edit**, and **Workers KV Storage: -Edit** (Edit implies read, covering the ownership/teardown list calls). Cron triggers ride along with -the worker deploy — no extra permission. +The token's Edit permissions include the reads needed to discover existing resources. Cron triggers +are published with the Worker. This fixture does not require paid-only services. -The account must also have a **workers.dev subdomain** already registered (with its wildcard TLS cert -provisioned). Without one, `void deploy` uploads the worker but cannot publish a public URL, and -the HTTP-200 assertion has nothing to hit. Opening the Workers & Pages dashboard once registers a -subdomain automatically; a freshly-registered subdomain takes a few minutes for its cert to go live, -during which the first run may see a TLS handshake failure — rerun once the cert is ready. - -If Zero Trust **Block traffic to all domains in this account** is enabled, add an Access application -with a **Bypass + Everyone** policy for -`void-e2e-cf-deploy-*..workers.dev`. The suite uses a new Worker name for every run, -so a rule for one hostname is insufficient. Without this Access application, Cloudflare returns error -1050 and Void cannot verify the deployment. Keep the account-wide block enabled. - -With no env vars the suite SKIPS cleanly (0 run, N skipped). +The account needs an existing workers.dev subdomain with its TLS certificate ready. If Cloudflare +Access blocks the test hostname, configure a public test hostname or an explicit **Bypass + Everyone** +policy for `void-e2e-cf-deploy-..workers.dev`. **Allow + Everyone** still +requires authentication. Keep unrelated account policies unchanged. ### Running it +Build the CLI, then supply credentials through the environment: + ```sh -# Ensure the CLI is built first (dist/cli/cli.mjs): vp run build:core -VOID_E2E_CF_TOKEN=... VOID_E2E_CF_ACCOUNT_ID=... vp test run packages/void/test/e2e +VOID_E2E_CF_TOKEN=... VOID_E2E_CF_ACCOUNT_ID=... VOID_E2E_NAMESPACE=my-team \ + vp test run packages/void/test/e2e ``` +Use a short namespace so the Worker and its derived names fit Cloudflare's naming limits. Never +select production resource names. The suite reserves: + +- Worker: `void-e2e-cf-deploy-` +- D1: `void-e2e-cf-deploy--db` +- KV: `void-e2e-cf-deploy--kv` + +Runs with the same account and namespace must be serialized, including across local checkouts. +Different operators should use different stable namespaces. Random values are used only for test +records and KV keys; resources and test data are retained after successful and failed runs. + +Any eventual cleanup is a separate, manually reviewed operation against exact resource IDs. Do not +add teardown deletion to this suite. See [the live-testing guidance](../../../../TESTING.md#native-cloudflare-live-e2e-tests). + ### In CI -`VOID_E2E_CF_TOKEN` and `VOID_E2E_CF_ACCOUNT_ID` are **environment secrets** on the `cloudflare-e2e` -GitHub Environment (not repo secrets), and a dedicated `live-e2e` job in `.github/workflows/ci.yml` -consumes them — the main `test` job carries no creds at all. Two layers keep the account token off -untrusted code: - -- The `live-e2e` job runs only on a direct push to `main` (`github.event_name == 'push' && github.ref - == 'refs/heads/main'`), so `pull_request` runs, tag pushes, and `workflow_call` (release) skip it. -- The `cloudflare-e2e` environment's deployment branch policy allows `main` **only**, enforced by - GitHub server-side on the run's ref. A `pull_request` run's ref is `refs/pull//merge`, so it can - never reach the secret **even if a PR edits this workflow** — the expression-level gate alone would - be bypassable that way; the environment ref-check is the real boundary. - -So the credentialed live deploy runs on every merge to `main`; everywhere else the creds are empty and -the suite SKIPS (the `test` job still imports the file and skips it). The `live-e2e` job itself is -fail-closed — a preflight step errors if either credential is empty (a missing, deleted, or -not-yet-rotated secret), so a misconfigured gate is loud instead of a silent green skip. Each -`live-e2e` run also gets `VOID_E2E_RUN_ID = -` for the unique per-run resource names -below. - -### Resource naming & cleanup - -Each run derives a UNIQUE resource name — `void-e2e-cf-deploy-` — so overlapping runs never -collide on remote resources. The suffix is `VOID_E2E_RUN_ID` (CI: `-`) or, locally, a -timestamp + random. Derived names mirror the provisioner: `-db` (D1), `-kv` (KV). The -shared `void-e2e-cf-deploy-` prefix keeps every resource greppable, so an orphan from a hard-killed run -is easy to find and clean up. A pre-run ownership gate refuses to start if the run's names -already exist (failing closed on any lookup it cannot confirm), so the test never adopts or deletes a -resource it does not own. Teardown runs even when the deploy or an assertion fails. - -> **Local concurrency:** unique names isolate _remote_ resources, but a run also snapshots and -> overwrites the shared `playground/spa/wrangler.jsonc` and builds in place. Do **not** run two live -> e2e invocations against the same checkout at once — they would race that file. CI is unaffected (each -> runner is an isolated checkout, and only the single `main`-push `live-e2e` job deploys). The deploy -> test also asserts the returned host's worker label equals this run's `WORKER_NAME`, so a clobbered -> overlay fails loudly instead of validating another run's worker. +The dedicated `live-e2e` job runs only on direct pushes to `main` in the canonical repository. Its +`cloudflare-e2e` GitHub Environment supplies `VOID_E2E_CF_TOKEN` and `VOID_E2E_CF_ACCOUNT_ID`; a +preflight fails if either credential is empty. The job uses the stable `ci` namespace and a concurrency +group to serialize updates to that account's test resources. + +The GitHub Environment must allow deployments from `main` only. This server-side branch policy +keeps its credentials away from pull-request code even if a PR changes the workflow. Other test jobs +receive no Cloudflare credentials and skip the live suite. diff --git a/packages/void/test/e2e/cloudflare-deploy-live.test.ts b/packages/void/test/e2e/cloudflare-deploy-live.test.ts index 9624db848..4b753a635 100644 --- a/packages/void/test/e2e/cloudflare-deploy-live.test.ts +++ b/packages/void/test/e2e/cloudflare-deploy-live.test.ts @@ -1,16 +1,15 @@ -import { existsSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { randomUUID } from 'node:crypto'; +import { readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; -import type { CloudflareOperation } from '@void/deploy-cloudflare'; import spawn from 'cross-spawn'; import { afterAll, beforeAll, describe, expect, it } from 'vite-plus/test'; -import { - encodeCloudflareOperation, - decodeCloudflareOperationResult, -} from '../../src/cli/cloudflare-process.ts'; +import { createIsolatedPlayground } from '../../../../test/integration-helpers.ts'; +import { renderVoidConfig } from '../../src/config-write.ts'; +import { readConfig } from '../../src/config.ts'; // Gated live-account e2e for the `void deploy --platform cloudflare` path. Runs only when // VOID_E2E_CF_TOKEN and VOID_E2E_CF_ACCOUNT_ID are set, and skips each test otherwise. See -// README.md in this directory for prerequisites, CI wiring, and resource naming/cleanup. +// README.md in this directory for prerequisites, CI wiring, and resource naming and retention. // Read env inside hooks and tests, never at module load: Vitest collects this file before the // runner injects the job env, so a top-level read sees empty credentials. @@ -30,64 +29,47 @@ const NO_CREDS_NOTE = const ROOT = join(import.meta.dirname, '..', '..', '..', '..'); const PACKAGE_ROOT = join(ROOT, 'packages', 'void'); const CLI_ENTRY = join(PACKAGE_ROOT, 'dist', 'cli', 'cli.mjs'); -const CF_BIN = join(PACKAGE_ROOT, 'node_modules', 'cf', 'bin', 'cf'); -const OPERATION_ENTRY = join(PACKAGE_ROOT, 'dist', 'cli', 'cloudflare-operation-process.mjs'); -// `spawn.sync` blocks the event loop, so Vitest cannot time out a hung call and teardown would -// never run. Bound every spawn below its hook timeout. +// `spawn.sync` blocks the event loop, so Vitest cannot time out a hung call. +// Bound every spawn below its hook timeout. const DEPLOY_TIMEOUT_MS = 480_000; -const CF_TIMEOUT_MS = 120_000; const REQUEST_TIMEOUT_MS = 20_000; -// Total cf time per hook, since several blocking calls run back to back. Setup: 90s + -// DEPLOY_TIMEOUT_MS < 600s; teardown: 200s + the ~64s worker probe < 300s. -const SETUP_CF_BUDGET_MS = 90_000; -const TEARDOWN_CF_BUDGET_MS = 200_000; - -// The SPA playground uses D1, KV and a cron. -const APP = join(ROOT, 'playground', 'spa'); -const APP_WRANGLER = join(APP, 'wrangler.jsonc'); - -// A unique name per run keeps overlapping runs apart; the shared prefix keeps orphans greppable. -// D1/KV names mirror the provisioner's `suffixedResourceName`. -function uniqueRunSuffix(): string { - // `||`, not `??`: vite.config.ts forwards an unset VOID_E2E_RUN_ID as ''. - const raw = - process.env.VOID_E2E_RUN_ID || - `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`; - // Cloudflare names are lowercase [a-z0-9-], cannot start or end with '-', and max out at 63 chars. - const cleaned = raw - .toLowerCase() - .replace(/[^a-z0-9]+/g, '-') - .slice(0, 28) - .replace(/^-+|-+$/g, ''); - return cleaned || 'local'; -} -let WORKER_NAME = ''; -let D1_NAME = ''; -let KV_TITLE = ''; -function computeResourceNames(): void { - WORKER_NAME = `void-e2e-cf-deploy-${uniqueRunSuffix()}`; - D1_NAME = `${WORKER_NAME}-db`; - KV_TITLE = `${WORKER_NAME}-kv`; -} - -const COMPAT_DATE = '2026-02-24'; - -let wranglerSnapshotTaken = false; -let wranglerOriginal: string | undefined; +let app = ''; +let workerName = ''; +let d1Name = ''; +let kvTitle = ''; let deployOutput = ''; let deployStatus: number | null = null; -// Teardown deletes only after the ownership gate proved the names were free before the deploy. -let ownershipVerified = false; +let firstResourceIds: ReturnType; +let createdEmail = ''; +let kvKey = ''; +let kvValue = ''; + +// Randomness belongs in test records, never in retained remote resource names. +function uniqueRecordSuffix(): string { + return `${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`; +} -/** The credentialed env every real Cloudflare call (deploy + teardown) shares. */ +function resourceIds() { + const config = JSON.parse(readFileSync(join(app, '.void-wrangler.jsonc'), 'utf8')) as { + d1_databases?: Array<{ binding: string; database_id: string }>; + kv_namespaces?: Array<{ binding: string; id: string }>; + }; + return { + d1: config.d1_databases?.find((binding) => binding.binding === 'DB')?.database_id, + kv: config.kv_namespaces?.find((binding) => binding.binding === 'KV')?.id, + }; +} + +/** The credentialed env every real Cloudflare call shares. */ function cloudEnv(): NodeJS.ProcessEnv { const env: NodeJS.ProcessEnv = { ...process.env, CLOUDFLARE_API_TOKEN: cfToken(), CLOUDFLARE_ACCOUNT_ID: cfAccountId(), }; - // The direct fetches use the public api.cloudflare.com, so cf must too. Otherwise teardown could - // miss a worker deployed to another API base and leak it. + // The CLI receives only its standard token variable, which it strips from app builds. + delete env.VOID_E2E_CF_TOKEN; + // The direct fetches and the CLI must use the same public Cloudflare API. delete env.CLOUDFLARE_API_BASE_URL; delete env.CF_API_BASE_URL; delete env.WRANGLER_API_ENVIRONMENT; @@ -95,87 +77,6 @@ function cloudEnv(): NodeJS.ProcessEnv { return env; } -// Deadline for the current hook's total cf time (see SETUP_/TEARDOWN_CF_BUDGET_MS). -let cfDeadline = Number.POSITIVE_INFINITY; - -/** Best-effort pinned cf invocation for setup and teardown. */ -function cf(args: Array | CloudflareOperation): { - status: number; - stdout: string; - output: string; -} { - // Once the hook budget is spent, fail the call instead of spawning; every caller fails closed. - const remaining = cfDeadline - Date.now(); - if (remaining <= 0) { - return { - status: 1, - stdout: '', - output: `cf ${Array.isArray(args) ? args.join(' ') : args.type} skipped — hook Cloudflare time budget exhausted`, - }; - } - const res = spawn.sync( - process.execPath, - Array.isArray(args) ? [CF_BIN, ...args] : [OPERATION_ENTRY, ...encodeCloudflareOperation(args)], - { - cwd: APP, - encoding: 'utf8', - maxBuffer: 10 * 1024 * 1024, - env: { ...cloudEnv(), CF_DELEGATION: '1' }, - timeout: Math.min(CF_TIMEOUT_MS, remaining), - }, - ); - const decoded = Array.isArray(args) - ? res - : decodeCloudflareOperationResult({ - status: res.status, - stdout: res.stdout ?? '', - stderr: res.stderr ?? '', - signal: res.signal, - error: res.error?.message, - }); - const stdout = - 'data' in decoded && decoded.data !== undefined - ? JSON.stringify(decoded.data) - : (decoded.stdout ?? ''); - const stderr = decoded.stderr ?? ''; - // Parse JSON from `stdout` only: cf can print notices to stderr on success. - return { - status: decoded.status ?? 1, - stdout, - output: `${stdout}${stderr}${res.error ? `\n${res.error}` : ''}`, - }; -} - -/** - * Ask the CF API whether the worker exists: `cf` exits nonzero for a missing worker too, so its - * exit code cannot tell. Returns 'unknown' after three tries without a 200 or 404. - */ -async function workerExistence(): Promise<'exists' | 'absent' | 'unknown'> { - for (let attempt = 0; attempt < 3; attempt++) { - try { - const res = await fetch( - `https://api.cloudflare.com/client/v4/accounts/${cfAccountId()}/workers/scripts/${WORKER_NAME}`, - { - headers: { Authorization: `Bearer ${cfToken()}` }, - signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), - }, - ); - if (res.status === 200) { - return 'exists'; - } - if (res.status === 404) { - return 'absent'; - } - } catch { - // Retry network errors. - } - if (attempt < 2) { - await new Promise((resolve) => setTimeout(resolve, 2000)); - } - } - return 'unknown'; -} - /** Extract the deployed workers.dev URL the CLI prints on a successful deploy. */ function extractWorkerUrl(output: string): string | undefined { // The host has two labels before workers.dev: `.`. @@ -252,7 +153,7 @@ async function fetchWithRetry( async function workersDevEnabled(): Promise { try { const res = await fetch( - `https://api.cloudflare.com/client/v4/accounts/${cfAccountId()}/workers/scripts/${WORKER_NAME}/subdomain`, + `https://api.cloudflare.com/client/v4/accounts/${cfAccountId()}/workers/scripts/${workerName}/subdomain`, { headers: { Authorization: `Bearer ${cfToken()}` }, signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), @@ -268,7 +169,7 @@ async function workersDevEnabled(): Promise { /** Authenticated GET against the Cloudflare API for THIS deployed worker (Bearer `cfToken()`). */ async function cfWorkerApi(path: string, signal?: AbortSignal): Promise { return fetch( - `https://api.cloudflare.com/client/v4/accounts/${cfAccountId()}/workers/scripts/${WORKER_NAME}${path}`, + `https://api.cloudflare.com/client/v4/accounts/${cfAccountId()}/workers/scripts/${workerName}${path}`, { headers: { Authorization: `Bearer ${cfToken()}` }, signal: signal @@ -279,83 +180,55 @@ async function cfWorkerApi(path: string, signal?: AbortSignal): Promise { - beforeAll(async () => { + beforeAll(() => { if (!e2eEnabled()) { return; } - cfDeadline = Date.now() + SETUP_CF_BUDGET_MS; - computeResourceNames(); - - // Ownership gate: provisioning adopts an existing D1/KV with the same name and teardown deletes - // by name, so refuse to run on any collision. An ambiguous lookup counts as a collision. - const collisions: Array = []; - - const worker = await workerExistence(); - if (worker === 'exists') { - collisions.push(`worker ${WORKER_NAME}`); - } else if (worker === 'unknown') { - collisions.push(`worker ${WORKER_NAME} (could not confirm it is free — lookup errored)`); - } - - // `d1 info` cannot tell a missing database from an error, so list them all. - const d1List = cf({ type: 'd1.list' }); - if (d1List.status !== 0) { - collisions.push(`d1 database ${D1_NAME} (could not list databases to rule out a collision)`); - } else { - try { - const databases = JSON.parse(d1List.stdout) as Array<{ name: string }>; - if (databases.some((d) => d.name === D1_NAME)) { - collisions.push(`d1 database ${D1_NAME}`); - } - } catch { - collisions.push( - `d1 database ${D1_NAME} (non-JSON database list — cannot rule out a collision)`, - ); - } - } - - const kvList = cf({ type: 'kv.namespace.list' }); - if (kvList.status !== 0) { - collisions.push( - `kv namespace ${KV_TITLE} (could not list namespaces to rule out a collision)`, - ); - } else { - try { - const namespaces = JSON.parse(kvList.stdout) as Array<{ title: string }>; - if (namespaces.some((n) => n.title === KV_TITLE)) { - collisions.push(`kv namespace ${KV_TITLE}`); - } - } catch { - collisions.push( - `kv namespace ${KV_TITLE} (non-JSON namespace list — cannot rule out a collision)`, - ); - } - } - if (collisions.length > 0) { + const namespace = process.env.VOID_E2E_NAMESPACE?.trim(); + if (!namespace) { throw new Error( - 'live e2e refuses to run — these throwaway-named resources already exist in the target ' + - 'account, so provisioning would adopt/migrate them and teardown would delete them ' + - `(risking data this test does not own):\n ${collisions.join('\n ')}\n` + - 'Remove them (or change WORKER_NAME) and rerun.', + 'Set VOID_E2E_NAMESPACE to a stable namespace reserved for this test account.', ); } - ownershipVerified = true; - - // Snapshot before writing so an early failure never deletes a local file. - wranglerOriginal = existsSync(APP_WRANGLER) ? readFileSync(APP_WRANGLER, 'utf-8') : undefined; - wranglerSnapshotTaken = true; + workerName = `void-e2e-cf-deploy-${namespace}`; + d1Name = `${workerName}-db`; + kvTitle = `${workerName}-kv`; + app = createIsolatedPlayground(ROOT, 'playground/spa').root; + // Start from fixture source, not an operator's environment or old provider identity. + for (const name of readdirSync(app)) { + if ( + name === '.env' || + name.startsWith('.env.') || + name === '.dev.vars' || + name.startsWith('.dev.vars.') || + [ + '.void-wrangler.jsonc', + 'void.lock.json', + 'wrangler.jsonc', + 'wrangler.json', + 'wrangler.toml', + ].includes(name) + ) { + rmSync(join(app, name), { force: true }); + } + } - // No D1/KV ids, so the deploy must provision them. `account_id` pins every step to the test - // account. The deploy migrates this file into void.config.ts. - const overlay = { - name: WORKER_NAME, - compatibility_date: COMPAT_DATE, - account_id: cfAccountId(), - }; - writeFileSync(APP_WRANGLER, `${JSON.stringify(overlay, null, 2)}\n`); + // The explicit account-local namespace is reserved for this suite. The production provisioner + // discovers and pins its actual D1/KV IDs, including on retry after a partial deployment. + const config = readConfig(app); + writeFileSync( + join(app, 'void.config.ts'), + renderVoidConfig({ + ...config, + cloudflare: { + name: workerName, + account_id: cfAccountId(), + }, + }), + ); const res = spawn.sync('node', [CLI_ENTRY, 'deploy', '--platform', 'cloudflare'], { - cwd: APP, + cwd: app, encoding: 'utf8', maxBuffer: 20 * 1024 * 1024, env: cloudEnv(), @@ -365,82 +238,25 @@ describe('void deploy --platform cloudflare (live account)', () => { deployOutput = `${res.stdout ?? ''}${res.stderr ?? ''}${res.error ? `\n${res.error}` : ''}`; }, 600_000); - afterAll(async () => { - // Independent best-effort deletes, even after a failed deploy. A failed delete or a lookup that - // cannot confirm a resource is gone is collected and thrown after the config restore. - const failures: Array = []; - const attemptDelete = (label: string, args: Array): void => { - const res = cf(args); - if (res.status !== 0) { - failures.push(`${label} (exit ${res.status})\n${res.output.trim()}`); - } - }; - if (e2eEnabled() && ownershipVerified) { - cfDeadline = Date.now() + TEARDOWN_CF_BUDGET_MS; - // Skip only a worker the API confirms absent; 'unknown' still tries the delete. - const worker = await workerExistence(); - if (worker !== 'absent') { - attemptDelete(`worker ${WORKER_NAME}`, ['workers', 'delete', WORKER_NAME, '--force']); - } - const d1List = cf({ type: 'd1.list' }); - if (d1List.status !== 0) { - failures.push(`d1 list (exit ${d1List.status})\n${d1List.output.trim()}`); - } else { - try { - const databases = JSON.parse(d1List.stdout) as Array<{ name: string; uuid: string }>; - const database = databases.find((d) => d.name === D1_NAME); - if (database && typeof database.uuid === 'string' && database.uuid) { - attemptDelete(`d1 ${D1_NAME}`, ['d1', 'delete', database.uuid, '--force']); - } else if (database) { - failures.push(`d1 ${D1_NAME} has no ID — delete the database manually`); - } - } catch { - failures.push( - `d1 list returned non-JSON output — delete the ${D1_NAME} database manually`, - ); - } - } - // KV delete needs the namespace id; match the exact title. - const list = cf({ type: 'kv.namespace.list' }); - if (list.status !== 0) { - failures.push(`kv namespace list (exit ${list.status})\n${list.output.trim()}`); - } else { - try { - const namespaces = JSON.parse(list.stdout) as Array<{ id: string; title: string }>; - const ns = namespaces.find((n) => n.title === KV_TITLE); - if (ns && typeof ns.id === 'string' && ns.id) { - attemptDelete(`kv ${KV_TITLE}`, ['kv', 'namespaces', 'delete', ns.id, '--force']); - } else if (ns) { - failures.push(`kv ${KV_TITLE} has no ID — delete the namespace manually`); - } - } catch { - failures.push( - `kv namespace list returned non-JSON output — delete the ${KV_TITLE} namespace manually`, - ); - } - } - } - // Restore the root config before surfacing teardown failures. - if (wranglerSnapshotTaken) { - if (wranglerOriginal === undefined) { - rmSync(APP_WRANGLER, { force: true }); - } else { - writeFileSync(APP_WRANGLER, wranglerOriginal); - } - } - if (failures.length > 0) { - throw new Error(`live e2e teardown failed — manual cleanup needed:\n${failures.join('\n')}`); + afterAll(() => { + if (app) { + // Keep the lockfile and remote resources for the next run and any failure investigation. + console.info( + `Live E2E targets retained when created: ${workerName}, ${d1Name}, ${kvTitle}. Fixture: ${app}`, + ); } - }, 300_000); + }); it('deploys the worker to the live account', (ctx) => { ctx.skip(!e2eEnabled(), NO_CREDS_NOTE); expect(deployStatus, deployOutput).toBe(0); const url = extractWorkerUrl(deployOutput); expect(url, deployOutput).toBeDefined(); - // A concurrent local run sharing the playground could have deployed another worker. const workerLabel = new URL(url!).hostname.split('.')[0]; - expect(workerLabel, `deployed host for ${url}`).toBe(WORKER_NAME); + expect(workerLabel, `deployed host for ${url}`).toBe(workerName); + firstResourceIds = resourceIds(); + expect(firstResourceIds.d1).toBeTruthy(); + expect(firstResourceIds.kv).toBeTruthy(); }); it('serves a live 200 from the deployed worker', async (ctx) => { @@ -451,7 +267,7 @@ describe('void deploy --platform cloudflare (live account)', () => { const res = await fetchWithRetry(url, {}, 40, 3000, ctx.signal); const diagnostic = res.status === 200 ? '' : ` — GET ${url} → ${res.status}; ${await workersDevEnabled()}`; - expect(res.status, `worker ${WORKER_NAME}${diagnostic}`).toBe(200); + expect(res.status, `worker ${workerName}${diagnostic}`).toBe(200); }, 200_000); it('persists data through the live D1 binding', async (ctx) => { @@ -459,20 +275,15 @@ describe('void deploy --platform cloudflare (live account)', () => { const base = extractWorkerUrl(deployOutput); expect(base, deployOutput).toBeDefined(); const root = base!.replace(/\/$/, ''); - // A 500 means the remote D1 migration or binding failed. The live-200 test already waited for - // the route, so shorter retries only cover cold start. - const email = `e2e-${uniqueRunSuffix()}@example.com`; - const created = await fetchWithRetry( - `${root}/api/users`, - { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ name: 'E2E User', email }), - }, - 15, - 2000, - ctx.signal, - ); + // Submit once: a failed response does not establish that the insert was rejected. + const email = `e2e-${uniqueRecordSuffix()}@example.com`; + createdEmail = email; + const created = await fetch(`${root}/api/users`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ name: 'E2E User', email }), + signal: AbortSignal.any([AbortSignal.timeout(REQUEST_TIMEOUT_MS), ctx.signal]), + }); const createdBody = await created.text(); expect(created.status, createdBody).toBe(200); expect(JSON.parse(createdBody)).toEqual({ created: true }); @@ -492,8 +303,10 @@ describe('void deploy --platform cloudflare (live account)', () => { const base = extractWorkerUrl(deployOutput); expect(base, deployOutput).toBeDefined(); const root = base!.replace(/\/$/, ''); - const key = `probe-${uniqueRunSuffix()}`; + const key = `probe-${uniqueRecordSuffix()}`; const value = `kv-${Date.now().toString(36)}`; + kvKey = key; + kvValue = value; const put = await fetchWithRetry( `${root}/api/kv/${key}`, { method: 'PUT', body: value }, @@ -544,7 +357,7 @@ describe('void deploy --platform cloudflare (live account)', () => { ctx.skip(!e2eEnabled(), NO_CREDS_NOTE); // Runs last: redeploys with the config the first deploy wrote, reusing the worker, D1 and KV. const res = spawn.sync('node', [CLI_ENTRY, 'deploy', '--platform', 'cloudflare'], { - cwd: APP, + cwd: app, encoding: 'utf8', maxBuffer: 20 * 1024 * 1024, env: cloudEnv(), @@ -555,12 +368,18 @@ describe('void deploy --platform cloudflare (live account)', () => { // Read the URL from this deploy's own output, not the first deploy's. const base = extractWorkerUrl(output); expect(base, output).toBeDefined(); - expect(new URL(base!).hostname.split('.')[0], output).toBe(WORKER_NAME); + expect(new URL(base!).hostname.split('.')[0], output).toBe(workerName); const root = base!.replace(/\/$/, ''); - // /api/users reads D1, so a dropped binding would return 500. + expect(resourceIds()).toEqual(firstResourceIds); + // Verify the records survive the update, not only that the bindings answer. const served = await fetchWithRetry(`${root}/api/hello`, {}, 15, 2000, ctx.signal); expect(served.status).toBe(200); const dbRoute = await fetchWithRetry(`${root}/api/users`, {}, 15, 2000, ctx.signal); - expect(dbRoute.status, await dbRoute.text()).toBe(200); + const users = (await dbRoute.json()) as Array<{ email: string }>; + expect(dbRoute.status).toBe(200); + expect(users.some((user) => user.email === createdEmail)).toBe(true); + const kv = await fetchWithRetry(`${root}/api/kv/${kvKey}`, {}, 40, 2000, ctx.signal); + expect(kv.status).toBe(200); + expect(await kv.text()).toBe(kvValue); }, 600_000); }); diff --git a/packages/void/test/unit/platform-cloudflare.test.ts b/packages/void/test/unit/platform-cloudflare.test.ts index e4dc9397a..b35515604 100644 --- a/packages/void/test/unit/platform-cloudflare.test.ts +++ b/packages/void/test/unit/platform-cloudflare.test.ts @@ -56,6 +56,77 @@ describe('platform Cloudflare client', () => { expect((error as Error).message).not.toContain('query-value'); expect((error as Error).message).not.toContain('secret-token'); }); + + it.each([ + { + code: 10042, + path: '/accounts/account/r2/buckets', + message: 'Please enable R2 through the Cloudflare Dashboard.', + guidance: 'Cloudflare R2 is not enabled for this account (code 10042).', + dashboard: 'https://dash.cloudflare.com/account/r2/overview', + }, + { + code: 10121, + path: '/accounts/account/workers/dispatch/namespaces', + message: + 'You do not have access to dispatch namespaces. Purchase at https://dash.cloudflare.com?to=/:account/workers-for-platforms', + guidance: + 'Cloudflare Workers for Platforms requires a paid subscription for this account (code 10121).', + dashboard: 'https://dash.cloudflare.com/account/workers-for-platforms', + }, + ])('explains prerequisite code $code while preserving provider diagnostics', async (test) => { + const client = new PlatformCloudflareClient('secret-token', { + fetcher: async () => + Response.json( + { success: false, errors: [{ code: test.code, message: test.message }] }, + { status: 403 }, + ), + }); + const error = await client.request(`${test.path}?secret=query-value`).catch((error) => error); + if (!(error instanceof PlatformCloudflareApiError)) { + throw new Error('Expected a Cloudflare API error', { cause: error }); + } + expect(error).toMatchObject({ status: 403, requestPath: test.path }); + expect(error.message.startsWith(test.guidance)).toBe(true); + expect(error.message).toContain(test.dashboard); + expect(error.message).toContain('retry the same command'); + expect(error.cause).toBeInstanceOf(PlatformCloudflareApiError); + expect(error.cause).toMatchObject({ + status: 403, + requestPath: test.path, + message: `Cloudflare API error: ${test.message} (code ${test.code}) (HTTP 403; GET ${test.path}).`, + }); + expect(error.message).not.toContain(test.message); + expect(error.message).not.toContain('HTTP 403'); + expect(error.message).not.toContain(test.path); + expect(error.message).not.toContain('--resume'); + expect(error.message).not.toContain('query-value'); + expect(error.message).not.toContain('secret-token'); + if (test.code === 10121) { + expect(error.message).toContain('Enterprise customers'); + } + }); + + it.each([ + { status: 401, code: 10000, path: '/accounts/account/r2/buckets' }, + { status: 403, code: 10000, path: '/accounts/account/r2/buckets' }, + { status: 403, code: 10042, path: '/accounts/account/ai-gateway/gateways' }, + { status: 500, code: 7000, path: '/accounts/account/ai-gateway/gateways' }, + ])('keeps unrelated provider errors unchanged: $status/$code', async ({ status, code, path }) => { + const client = new PlatformCloudflareClient('token', { + fetcher: async () => + Response.json( + { success: false, errors: [{ code, message: 'Please enable R2' }] }, + { status }, + ), + }); + await expect(client.request(path)).rejects.toMatchObject({ + status, + requestPath: path, + message: `Cloudflare API error: Please enable R2 (code ${code}) (HTTP ${status}; GET ${path}).`, + }); + }); + it.each([401, 403, 429, 500, 503])( 'distinguishes HTTP %s from missing workers.dev onboarding', async (status) => { diff --git a/packages/void/test/unit/platform-install-cutover.test.ts b/packages/void/test/unit/platform-install-cutover.test.ts index 264f935b4..234f0a755 100644 --- a/packages/void/test/unit/platform-install-cutover.test.ts +++ b/packages/void/test/unit/platform-install-cutover.test.ts @@ -5,17 +5,28 @@ import type { AddressInfo } from 'node:net'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { CloudflareOperation } from '@void/deploy-cloudflare'; -import { PLATFORM_DISABLED_KEY, type PlatformInstallationManifest } from '@void/platform'; +import { + createPlatformDeploymentManifest, + PLATFORM_DISABLED_KEY, + PLATFORM_REMOTE_INSTALLATION_MANIFEST_KEY, + type PlatformInstallationManifest, +} from '@void/platform'; import { afterEach, beforeEach, expect, it, vi } from 'vite-plus/test'; import { hyperdriveOriginFromConnectionString } from '../../src/cli/platform-cloudflare.ts'; import { verifyPlatformWorkerCutoverVersions } from '../../src/cli/platform-installer-cutover.ts'; -import { type PlatformInstallerDependencies } from '../../src/cli/platform-installer-types.ts'; +import { + type PlatformInstallerDependencies, + type PlatformInstallationSecrets, +} from '../../src/cli/platform-installer-types.ts'; import { createPlatformInstallationSecrets, installPlatform, } from '../../src/cli/platform-installer.ts'; import { createPlatformInstallPlan } from '../../src/cli/platform-plan.ts'; -import { listInstallationManifests } from '../../src/cli/platform-registry.ts'; +import { + listInstallationManifests, + readInstallationRecoverySecrets, +} from '../../src/cli/platform-registry.ts'; import type { RunWrangler } from '../../src/cli/provision.ts'; // Runs the real Worker deploy, cutover checkpoint, and pre-traffic verification @@ -305,6 +316,9 @@ async function createFakeCloudflare() { worker.versions.set(replacement.id, replacement); activate(name, replacement.id); }, + restorePreviousVersion: (name: string) => { + activate(name, workers.get(name)!.deployments[1]!.versions[0]!.version_id); + }, }; } @@ -356,29 +370,36 @@ async function runInstall( options: { sharedEmail?: boolean; bootstrapEmail?: PlatformInstallerDependencies['bootstrapEmail']; + secrets?: PlatformInstallationSecrets; + verifyHealth?: PlatformInstallerDependencies['verifyHealth']; } = {}, ) { const plan = await planInstallation(cloud, existing, options.sharedEmail); - return installPlatform(plan, installationSecrets(options.sharedEmail), existing, { - acquireDatabaseLifecycleLock: async () => - Object.assign(async () => {}, { - prepareSandboxDrain: async () => ({ - admissionProtocol: 3 as const, - generation: 'generation-1', - state: 'complete' as const, + return installPlatform( + plan, + options.secrets ?? installationSecrets(options.sharedEmail), + existing, + { + acquireDatabaseLifecycleLock: async () => + Object.assign(async () => {}, { + prepareSandboxDrain: async () => ({ + admissionProtocol: 3 as const, + generation: 'generation-1', + state: 'complete' as const, + }), }), + claimDatabase: async () => {}, + client: cloud.client, + prepareDatabase: async () => ({ + appliedMigrations: 0, + previousSchemaDigest: '', + schemaDigest: '', }), - claimDatabase: async () => {}, - client: cloud.client, - prepareDatabase: async () => ({ - appliedMigrations: 0, - previousSchemaDigest: '', - schemaDigest: '', - }), - revalidateBeforeWorkerDeploy: async () => {}, - verifyHealth: async () => {}, - ...(options.bootstrapEmail ? { bootstrapEmail: options.bootstrapEmail } : {}), - }); + revalidateBeforeWorkerDeploy: async () => {}, + verifyHealth: options.verifyHealth ?? (async () => {}), + ...(options.bootstrapEmail ? { bootstrapEmail: options.bootstrapEmail } : {}), + }, + ); } function currentManifest(): PlatformInstallationManifest { @@ -391,6 +412,270 @@ function workerStepKeys(manifest: PlatformInstallationManifest) { const WORKER_KEYS = ['api', 'proxy', 'emailGateway', 'tail', 'dispatch']; +function rotateSecrets(manifest: PlatformInstallationManifest) { + return createPlatformInstallationSecrets({ + ...installationSecrets(), + checkpoint: readInstallationRecoverySecrets(manifest), + cloudflareApiToken: 'rotated-runtime-token', + existingInstallation: true, + updateExistingWorkers: true, + }); +} + +function resumeSecrets(manifest: PlatformInstallationManifest) { + return createPlatformInstallationSecrets({ + ...installationSecrets(), + checkpoint: readInstallationRecoverySecrets(manifest), + cloudflareApiToken: '', + githubClientId: '', + githubClientSecret: '', + r2AccessKeyId: '', + r2SecretAccessKey: '', + existingInstallation: true, + updateExistingWorkers: true, + }); +} + +it('rolls forward a partially restored credential update without restoring old secret versions', async () => { + const cloud = await createFakeCloudflare(); + const installed = await runInstall(cloud, null); + const existing = { + ...installed.manifest, + deployment: createPlatformDeploymentManifest({ + ...installed.manifest.deployment, + authentication: { ...installed.manifest.deployment.authentication, configurable: true }, + }), + }; + const start = cloud.commands.length; + await expect( + runInstall(cloud, existing, { + secrets: rotateSecrets(existing), + verifyHealth: async () => { + throw new Error('network unavailable'); + }, + }), + ).rejects.toThrow('network unavailable'); + const failed = currentManifest(); + const routing = failed.resources.find((resource) => resource.key === 'routing-kv')!; + expect(cloud.kv.has(`${routing.id}/${PLATFORM_DISABLED_KEY}`)).toBe(true); + expect( + cloud.commands + .slice(start) + .filter( + ({ args }) => + args.type === 'versions.deploy' && args.message?.startsWith('Void platform restoration'), + ), + ).toEqual([]); + + // Older installers restored these Workers before Cloudflare rejected the + // API/proxy rollback because CF_API_TOKEN and R2 credentials had changed. + for (const name of [ + failed.deployment.resources.emailGatewayWorker, + failed.deployment.resources.tailWorker, + failed.deployment.resources.dispatchWorker, + ]) { + cloud.restorePreviousVersion(name); + } + const resumeStart = cloud.commands.length; + let healthChecks = 0; + const result = await runInstall(cloud, failed, { + secrets: rotateSecrets(failed), + verifyHealth: async () => { + if (healthChecks++ === 0) { + expect(cloud.kv.has(`${routing.id}/${PLATFORM_DISABLED_KEY}`)).toBe(true); + } + }, + }); + expect(result.manifest.status).toBe('installed'); + expect(cloud.kv.has(`${routing.id}/${PLATFORM_DISABLED_KEY}`)).toBe(false); + expect( + cloud.commands + .slice(resumeStart) + .filter( + ({ args }) => + args.type === 'versions.deploy' && args.message?.startsWith('Void platform restoration'), + ), + ).toEqual([]); + expect(readInstallationRecoverySecrets(result.manifest)?.workerSecretUpdatesJson).toBeUndefined(); +}); + +it('refuses a concurrent version replacement while recovering a credential update', async () => { + const cloud = await createFakeCloudflare(); + const installed = await runInstall(cloud, null); + const existing = { + ...installed.manifest, + deployment: createPlatformDeploymentManifest({ + ...installed.manifest.deployment, + authentication: { ...installed.manifest.deployment.authentication, configurable: true }, + }), + }; + await expect( + runInstall(cloud, existing, { + secrets: rotateSecrets(existing), + verifyHealth: async () => { + throw new Error('network unavailable'); + }, + }), + ).rejects.toThrow('network unavailable'); + const failed = currentManifest(); + cloud.replaceActiveVersion(failed.deployment.resources.apiWorker); + const start = cloud.commands.length; + await expect(runInstall(cloud, failed, { secrets: rotateSecrets(failed) })).rejects.toThrow( + 'changed concurrently after the credential update', + ); + expect( + cloud.commands + .slice(start) + .filter(({ args }) => + ['versions.deploy', 'versions.upload', 'worker.deploy'].includes(args.type), + ), + ).toEqual([]); +}); + +it('resumes from encrypted credentials after the prior generation commits but before staging', async () => { + const cloud = await createFakeCloudflare(); + const installed = await runInstall(cloud, null); + const existing = { + ...installed.manifest, + deployment: createPlatformDeploymentManifest({ + ...installed.manifest.deployment, + authentication: { ...installed.manifest.deployment.authentication, configurable: true }, + }), + }; + await expect( + runInstall(cloud, existing, { + secrets: rotateSecrets(existing), + verifyHealth: async () => { + throw new Error('network unavailable'); + }, + }), + ).rejects.toThrow('network unavailable'); + const failed = currentManifest(); + const put = cloud.client.putKvValue; + let stopped = false; + cloud.client.putKvValue = async (...args) => { + await put(...args); + if ( + !stopped && + args[2] === PLATFORM_REMOTE_INSTALLATION_MANIFEST_KEY && + workerStepKeys(JSON.parse(args[3])).includes('worker-cutover:3:committed') + ) { + stopped = true; + throw new Error('interrupted after recovery checkpoint'); + } + }; + await expect(runInstall(cloud, failed, { secrets: rotateSecrets(failed) })).rejects.toThrow( + 'interrupted after recovery checkpoint', + ); + const checkpoint = currentManifest(); + expect(stagedKeys(checkpoint, 4)).toEqual([]); + const routing = checkpoint.resources.find((resource) => resource.key === 'routing-kv')!; + expect(cloud.kv.has(`${routing.id}/${PLATFORM_DISABLED_KEY}`)).toBe(true); + const recovered = resumeSecrets(checkpoint); + expect(recovered.pendingWorkerSecretUpdateIntent?.updates.CF_API_TOKEN).toBe( + 'rotated-runtime-token', + ); + const resumed = await runInstall(cloud, checkpoint, { secrets: recovered }); + expect(resumed.manifest.status).toBe('installed'); + expect(stagedKeys(resumed.manifest, 5)).toEqual([...WORKER_KEYS].sort()); +}); + +it('resumes a credential update that stopped before activating any Worker', async () => { + const cloud = await createFakeCloudflare(); + const installed = await runInstall(cloud, null); + cloud.failUpload(installed.manifest.deployment.resources.apiWorker); + await expect( + runInstall(cloud, installed.manifest, { secrets: rotateSecrets(installed.manifest) }), + ).rejects.toThrow('upload failed'); + const failed = currentManifest(); + expect(stagedKeys(failed, 3)).toEqual([]); + const resumed = await runInstall(cloud, failed, { secrets: resumeSecrets(failed) }); + expect(resumed.manifest.status).toBe('installed'); + expect(stagedKeys(resumed.manifest, 5)).toEqual([...WORKER_KEYS].sort()); +}); + +it('resumes when the next pending generation is saved before its encrypted checkpoint', async () => { + const cloud = await createFakeCloudflare(); + const installed = await runInstall(cloud, null); + const existing = { + ...installed.manifest, + deployment: createPlatformDeploymentManifest({ + ...installed.manifest.deployment, + authentication: { ...installed.manifest.deployment.authentication, configurable: true }, + }), + }; + await expect( + runInstall(cloud, existing, { + secrets: rotateSecrets(existing), + verifyHealth: async () => { + throw new Error('network unavailable'); + }, + }), + ).rejects.toThrow('network unavailable'); + const failed = currentManifest(); + const intentId = resumeSecrets(failed).pendingWorkerSecretUpdateIntent!.id; + const put = cloud.client.putKvValue; + let stopped = false; + cloud.client.putKvValue = async (...args) => { + await put(...args); + if ( + !stopped && + args[2] === PLATFORM_REMOTE_INSTALLATION_MANIFEST_KEY && + workerStepKeys(JSON.parse(args[3])).includes('worker-secret-updates:4:pending') + ) { + stopped = true; + throw new Error('interrupted after next pending marker'); + } + }; + await expect(runInstall(cloud, failed, { secrets: resumeSecrets(failed) })).rejects.toThrow( + 'interrupted after next pending marker', + ); + const checkpoint = currentManifest(); + expect(workerStepKeys(checkpoint)).toContain('worker-secret-updates:4:pending'); + expect(stagedKeys(checkpoint, 4)).toEqual([]); + const recovered = resumeSecrets(checkpoint); + expect(recovered.pendingWorkerSecretUpdateIntent).toMatchObject({ + id: intentId, + updates: { CF_API_TOKEN: 'rotated-runtime-token' }, + }); + const routing = checkpoint.resources.find((resource) => resource.key === 'routing-kv')!; + expect(cloud.kv.has(`${routing.id}/${PLATFORM_DISABLED_KEY}`)).toBe(true); + const resumed = await runInstall(cloud, checkpoint, { secrets: recovered }); + expect(resumed.manifest.status).toBe('installed'); + expect(cloud.kv.has(`${routing.id}/${PLATFORM_DISABLED_KEY}`)).toBe(false); +}); + +it('still restores a failed cutover without credential updates', async () => { + const cloud = await createFakeCloudflare(); + const installed = await runInstall(cloud, null); + const existing = { + ...installed.manifest, + usageProtocolVersion: 1 as const, + deployment: createPlatformDeploymentManifest({ + ...installed.manifest.deployment, + authentication: { ...installed.manifest.deployment.authentication, configurable: true }, + }), + }; + const start = cloud.commands.length; + await expect( + runInstall(cloud, existing, { + secrets: resumeSecrets(existing), + verifyHealth: async () => { + throw new Error('health unavailable'); + }, + }), + ).rejects.toThrow('health unavailable'); + expect(workerStepKeys(currentManifest())).toContain('worker-cutover:3:restored'); + expect( + cloud.commands + .slice(start) + .some( + ({ args }) => + args.type === 'versions.deploy' && args.message?.startsWith('Void platform restoration'), + ), + ).toBe(true); +}); + function stagedKeys(manifest: PlatformInstallationManifest, generation: number) { const prefix = `worker-cutover:${generation}:staged:`; return workerStepKeys(manifest) diff --git a/packages/void/test/unit/platform-installer-errors.test.ts b/packages/void/test/unit/platform-installer-errors.test.ts index 440d6b6fd..8fcadaeb1 100644 --- a/packages/void/test/unit/platform-installer-errors.test.ts +++ b/packages/void/test/unit/platform-installer-errors.test.ts @@ -168,21 +168,37 @@ it('reports structured provider errors when stderr is empty using existing crede 'upload test-worker', ), ).toThrow( - 'platform: Wrangler failed to upload test-worker (Invalid Authorization: Bearer [redacted]).', + 'platform: Cloudflare failed to upload test-worker (Invalid Authorization: Bearer [redacted]).', ); }); -it('points to the Analytics Engine setup when Cloudflare refuses its binding', () => { - const raw = - 'You need to enable Analytics Engine. Head to the Cloudflare Dashboard to enable: https://dash.cloudflare.com/account/workers/analytics-engine [code: 10089]'; - const hint = - ' If the Analytics Engine page offers no way to enable it, follow https://void.cloud/guide/platform/installation/prerequisites#if-cloudflare-rejects-analytics-engine, then retry.'; - expect(() => +it('replaces the Analytics Engine APIError token advice with account activation and recovery steps', () => { + const accountId = '0123456789abcdef0123456789abcdef'; + const dashboard = `https://dash.cloudflare.com/${accountId}/workers/analytics-engine`; + // The provider appended token advice to this account-activation error on Workers Free. + const message = `You need to enable Analytics Engine. Head to the Cloudflare Dashboard to enable: ${dashboard} [code: 10089]`; + const upload = () => assertCleanPlatformWrangler( - { status: 1, stdout: '', stderr: raw, errorCode: 10089 }, + { + status: 1, + stdout: '', + stderr: `\n┌ APIError\n│ [code: 10089] ${message}\n│ 403 Forbidden\n└\n\n→ Check your API token permissions\n`, + errorCode: 10089, + errors: [{ code: 10089, message }], + }, 'bootstrap void-team-tail', - ), - ).toThrow(`platform: Wrangler failed to bootstrap void-team-tail (${raw}).${hint}`); + accountId, + ); + expect(upload).toThrow( + 'platform: Cloudflare could not bootstrap void-team-tail: Analytics Engine is not enabled for this account (code 10089). ' + + `Open ${dashboard}, then select Enable. ` + + 'Analytics Engine is available on Workers Free. After enabling it, use the printed recovery command if one is shown; otherwise retry this command. ' + + 'If Enable is unavailable, see https://void.cloud/guide/platform/installation/prerequisites#if-cloudflare-rejects-analytics-engine.', + ); + expect(upload).not.toThrow(/API token|403 Forbidden|Wrangler/); +}); + +it('explains how to select the account when only a structured Analytics Engine error is available', () => { expect(() => assertCleanPlatformWrangler( { @@ -193,21 +209,35 @@ it('points to the Analytics Engine setup when Cloudflare refuses its binding', ( }, 'bootstrap void-team-tail', ), - ).toThrow(hint); - expect(() => - assertCleanPlatformWrangler( - { status: 1, stdout: '', stderr: 'You need to enable Analytics Engine. [10089]' }, - 'bootstrap void-team-tail', - ), - ).toThrow(hint); + ).toThrow( + 'Open https://dash.cloudflare.com/, select your account, and open Analytics Engine, then select Enable.', + ); +}); + +it.each(['[code: 10089]', '[10089]'])( + 'recognizes a legacy Analytics Engine diagnostic with %s when no structured code is available', + (code) => { + expect(() => + assertCleanPlatformWrangler( + { status: 1, stdout: '', stderr: `You need to enable Analytics Engine. ${code}` }, + 'bootstrap void-team-tail', + ), + ).toThrow('Analytics Engine is not enabled for this account (code 10089).'); + }, +); + +it.each([ + { errorCode: 10000, errors: undefined }, + { errorCode: undefined, errors: [{ code: 12345, message: 'Unknown provider failure' }] }, +])('preserves other structured provider errors and their token advice: %j', (codes) => { + const diagnostic = + 'The request failed. Check your API token permissions. Previous attempt: [code: 10089]'; expect(() => assertCleanPlatformWrangler( - { status: 1, stdout: '', stderr: 'Authentication error [code: 10000]', errorCode: 10000 }, - 'bootstrap void-team-tail', + { status: 1, stdout: '', stderr: diagnostic, ...codes }, + 'upload test-worker', ), - ).toThrow( - /^platform: Wrangler failed to bootstrap void-team-tail \(Authentication error \[code: 10000\]\)\.$/, - ); + ).toThrow(`platform: Cloudflare failed to upload test-worker (${diagnostic}).`); }); it('retains the exit status when stdout contains only binding values', () => { @@ -216,5 +246,5 @@ it('retains the exit status when stdout contains only binding values', () => { { status: 1, stdout: 'env.PRIVATE_VALUE: "do-not-echo"', stderr: '' }, 'upload test-worker', ), - ).toThrow('platform: Wrangler failed to upload test-worker (exit 1).'); + ).toThrow('platform: Cloudflare failed to upload test-worker (exit 1).'); }); diff --git a/packages/void/test/unit/platform-installer-health.test.ts b/packages/void/test/unit/platform-installer-health.test.ts new file mode 100644 index 000000000..cbdf46747 --- /dev/null +++ b/packages/void/test/unit/platform-installer-health.test.ts @@ -0,0 +1,76 @@ +import { beforeEach, expect, it, vi } from 'vite-plus/test'; +import { verifyPlatformHealth } from '../../src/cli/platform-installer-health.ts'; + +const wait = vi.hoisted(() => vi.fn(async (_milliseconds: number) => {})); +vi.mock('node:timers/promises', async (importOriginal) => ({ + ...(await importOriginal()), + setTimeout: wait, +})); + +beforeEach(() => wait.mockClear()); + +const api = 'https://api.example.com'; +const proxy = 'https://proxy.example.com'; +const check = (fetcher: typeof fetch) => + verifyPlatformHealth( + api, + 'installation', + proxy, + 'apps.example.com', + '1.2.1', + 2, + 'private-probe', + fetcher, + ); +const healthy = (url: string) => + Response.json( + url.endsWith('/.well-known/void-platform') + ? { installationId: 'installation', platformVersion: '1.2.1', usageProtocolVersion: 2 } + : { ok: true }, + ); + +it('retries a transient transport failure without repeating successful health probes', async () => { + const calls: Array = []; + const fetcher = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { + const url = String(input); + calls.push(url); + expect(new Headers(init?.headers).get('X-Void-Platform-Probe')).toBe('private-probe'); + expect(init?.redirect).toBe('manual'); + if (url === `${proxy}/health` && calls.filter((value) => value === url).length === 1) { + throw new TypeError('fetch failed'); + } + return healthy(url); + }) as typeof fetch; + await expect(check(fetcher)).resolves.toBeUndefined(); + expect(calls.filter((url) => url === `${proxy}/health`)).toHaveLength(2); + expect(calls.filter((url) => url === `${api}/health`)).toHaveLength(1); + expect(wait.mock.calls).toEqual([[250]]); +}); + +it('identifies the unreachable endpoint after bounded transport retries', async () => { + const failure = new TypeError('fetch failed'); + const fetcher = vi.fn(async (input: string | URL | Request) => { + const url = String(input); + if (url === `${proxy}/health`) { + throw failure; + } + return healthy(url); + }) as typeof fetch; + const error = await check(fetcher).catch((value: unknown) => value); + expect(error).toMatchObject({ + message: `platform: Health check could not reach ${proxy}/health after 3 attempts.`, + cause: failure, + }); + expect(String(error)).not.toContain('private-probe'); + expect(wait.mock.calls).toEqual([[250], [500]]); +}); + +it('does not retry HTTP failures or follow an authentication redirect', async () => { + const fetcher = vi.fn(async (input: string | URL | Request) => + String(input) === `${api}/health` + ? new Response(null, { status: 302, headers: { Location: 'https://login.example.com' } }) + : healthy(String(input)), + ) as typeof fetch; + await expect(check(fetcher)).rejects.toThrow(`Health check failed at ${api}/health (HTTP 302)`); + expect(wait).not.toHaveBeenCalled(); +}); diff --git a/packages/void/test/unit/platform-installer-outside-changes.test.ts b/packages/void/test/unit/platform-installer-outside-changes.test.ts index 73f7401cd..7f5e4c665 100644 --- a/packages/void/test/unit/platform-installer-outside-changes.test.ts +++ b/packages/void/test/unit/platform-installer-outside-changes.test.ts @@ -37,6 +37,7 @@ import { import { recordWorkerCutoverResolution, recordWorkerCutoverStage, + recordWorkerSecretUpdateState, } from '../../src/cli/platform-installer-cutover.ts'; import { findPlatformWorkerOutsideChanges, @@ -559,6 +560,21 @@ describe('platform Workers changed outside Void', () => { expect(earlierReads()).toHaveLength(1); }); + it('does not inspect old secret versions when resuming a credential update', async () => { + await expect( + find({ + manifest: recordWorkerSecretUpdateState( + stopped(), + 1, + 'pending', + 'rotation', + '2026-10-23T00:00:00.000Z', + ), + }), + ).resolves.toEqual([]); + expect(earlierReads()).toEqual([]); + }); + it('decides like the installer whether the run restores the earlier version', async () => { const configurable = (manifest: PlatformInstallationManifest) => ({ ...manifest, diff --git a/packages/void/test/unit/platform-plan.test.ts b/packages/void/test/unit/platform-plan.test.ts index 70bc9fdf6..55d5a2686 100644 --- a/packages/void/test/unit/platform-plan.test.ts +++ b/packages/void/test/unit/platform-plan.test.ts @@ -650,6 +650,80 @@ describe('platform installation planning', () => { ), ).rejects.toThrow('KV access denied'); }); + + it('reports both unpaid-account prerequisites from the existing read-only inventory', async () => { + const requests: Array<{ path: string; method: string }> = []; + const provider = new PlatformCloudflareClient('token', { + fetcher: async (input, init) => { + const path = new URL(String(input)).pathname; + requests.push({ path, method: init?.method ?? 'GET' }); + const code = path.endsWith('/r2/buckets') ? 10042 : 10121; + return Response.json( + { success: false, errors: [{ code, message: 'Account prerequisite missing' }] }, + { status: 403 }, + ); + }, + }); + const error = await createPlatformInstallPlan( + client({ + listR2Buckets: provider.listR2Buckets.bind(provider), + listDispatchNamespaces: provider.listDispatchNamespaces.bind(provider), + }), + input, + null, + ).catch((error) => error); + expect(error).toBeInstanceOf(AggregateError); + expect(error.message).toContain('Cloudflare R2 is not enabled'); + expect(error.message).toContain('Workers for Platforms requires a paid subscription'); + expect(error.message).toContain('retry the same command'); + expect(error.message).not.toContain('--resume'); + expect(error.errors).toEqual( + expect.arrayContaining([ + expect.objectContaining({ status: 403, requestPath: '/accounts/account/r2/buckets' }), + expect.objectContaining({ + status: 403, + requestPath: '/accounts/account/workers/dispatch/namespaces', + }), + ]), + ); + expect(error.errors).toHaveLength(2); + expect(requests).toEqual([ + { path: '/client/v4/accounts/account/r2/buckets', method: 'GET' }, + { path: '/client/v4/accounts/account/workers/dispatch/namespaces', method: 'GET' }, + ]); + }); + + it.each([401, 403, 500])( + 'does not replace an HTTP %s inventory failure with a known prerequisite', + async (status) => { + const unknown = new PlatformCloudflareApiError( + 'Unrelated provider failure', + status, + '/accounts/account/workers/dispatch/namespaces', + ); + const provider = new PlatformCloudflareClient('token', { + fetcher: async () => + Response.json( + { success: false, errors: [{ code: 10042, message: 'Please enable R2' }] }, + { status: 403 }, + ), + }); + await expect( + createPlatformInstallPlan( + client({ + listR2Buckets: provider.listR2Buckets.bind(provider), + listDispatchNamespaces: async () => { + await Promise.resolve(); + throw unknown; + }, + }), + input, + null, + ), + ).rejects.toBe(unknown); + }, + ); + it('names the DNS permission when the token cannot read DNS records', async () => { const deniedDns = async () => { throw new PlatformCloudflareApiError('Authentication error', 403, '/zones/zone/dns_records'); diff --git a/platform/README.md b/platform/README.md index 0c58ed6af..9bc8674c5 100644 --- a/platform/README.md +++ b/platform/README.md @@ -110,6 +110,11 @@ Forks build their own runtime and use `void platform install` or - Static applications are served from platform assets. Worker applications also mirror assets in KV/R2 so dispatch can recover from the Cloudflare missing-asset response; see the [fallback design](./meta/design-docs/0066-wfp-asset-r2-fallback.md). +- Managed ISR bodies use the existing assets R2 bucket under `isr-cache/` so a + deleted KV value cannot repopulate the edge cache during propagation. Reads + never fall back to legacy ISR KV. Purges and deployment cleanup still scan + legacy KV to remove old edge entries; project cleanup removes the R2 prefix. + This storage change needs no new binding and does not change native app ISR. - Keep routing changes, cron synchronization, and resource cleanup ordered around the successful deployment cutover. A pre-cutover failure must preserve the previous application's active configuration. diff --git a/platform/packages/api/package.json b/platform/packages/api/package.json index 238b58937..b3ee2e1fa 100644 --- a/platform/packages/api/package.json +++ b/platform/packages/api/package.json @@ -8,6 +8,7 @@ "./api": "./src/api.ts", "./reserved-slugs": "./src/reserved-slugs.ts", "./routing": "./src/routing.ts", + "./isr-store": "./src/isr-store.ts", "./usage-auth": "./src/usage-auth.ts", "./proxy-token": "./src/proxy-token.ts", "./admin-jsx/jsx-runtime": "./src/admin/react-jsx/jsx-runtime.ts", diff --git a/platform/packages/api/src/assets.ts b/platform/packages/api/src/assets.ts index 038826ed4..5e2797879 100644 --- a/platform/packages/api/src/assets.ts +++ b/platform/packages/api/src/assets.ts @@ -2,6 +2,7 @@ import { extname } from 'node:path'; import { createHasher } from 'blake3-jit'; import mime from 'mime'; import { retryWithBackoff } from './cf-api.ts'; +import { isrObjectPrefix } from './isr-store.ts'; import type { AssetConfig } from './types.ts'; class AssetUploadError extends Error { @@ -763,6 +764,7 @@ export async function deleteWorkerBundle( export async function deleteProjectAssets(r2: R2Bucket, projectId: string): Promise { await paginatedR2Delete(r2, `assets/${projectId}/`); await paginatedR2Delete(r2, `deployment-inputs/${projectId}/`); + await paginatedR2Delete(r2, isrObjectPrefix(projectId)); } /** diff --git a/platform/packages/api/src/deploy/cache.ts b/platform/packages/api/src/deploy/cache.ts index 99f6984fe..680c19de9 100644 --- a/platform/packages/api/src/deploy/cache.ts +++ b/platform/packages/api/src/deploy/cache.ts @@ -1,9 +1,10 @@ import { isrCachePrefix } from '@void/isr'; import { R2_CONCURRENCY, uploadPool } from '../concurrency.ts'; import { legacyIsrCachePrefix } from '../isr-legacy-keys.ts'; +import type { IsrStore } from '../isr-store.ts'; export async function clearIsrCache( - kv: KVNamespace, + cache: Pick, projectId: string, deploymentId: string, ): Promise { @@ -19,17 +20,20 @@ export async function clearIsrCache( // `revalidate-core.ts`. The two prefixes are disjoint (`…/isr/v3/…` vs // `…/isr/v2/…`), so nothing is listed or deleted twice. Transition-only: // drop the v2 sweep once no v2 entries remain in production. - await deleteIsrKeysByPrefix(kv, isrCachePrefix(projectId, deploymentId)); - await deleteIsrKeysByPrefix(kv, legacyIsrCachePrefix(projectId, deploymentId)); + await deleteIsrKeysByPrefix(cache, isrCachePrefix(projectId, deploymentId)); + await deleteIsrKeysByPrefix(cache, legacyIsrCachePrefix(projectId, deploymentId)); } -async function deleteIsrKeysByPrefix(kv: KVNamespace, prefix: string): Promise { +async function deleteIsrKeysByPrefix( + cache: Pick, + prefix: string, +): Promise { let cursor: string | undefined; do { - const listed = await kv.list({ prefix, cursor }); + const listed = await cache.list({ prefix, cursor }); if (listed.keys.length > 0) { - await uploadPool(listed.keys, R2_CONCURRENCY, (key) => kv.delete(key.name)); + await uploadPool(listed.keys, R2_CONCURRENCY, (key) => cache.delete(key.name)); } cursor = listed.list_complete ? undefined : listed.cursor; } while (cursor); diff --git a/platform/packages/api/src/deploy/compensate.ts b/platform/packages/api/src/deploy/compensate.ts index c86760f4f..6b712f8da 100644 --- a/platform/packages/api/src/deploy/compensate.ts +++ b/platform/packages/api/src/deploy/compensate.ts @@ -16,6 +16,7 @@ import { } from '../db.ts'; import { sendDeployFailedEmail } from '../email.ts'; import type { ApiEnv } from '../env.ts'; +import { createIsrStore } from '../isr-store.ts'; import { type DeployLogger } from '../observability/logger.ts'; import { ProjectAccessDeniedError } from '../project-access.ts'; import { reconcileManagedDurableObjects } from '../provisioning/durable-objects.ts'; @@ -478,7 +479,11 @@ export async function recoverFailedDeployment(input: { }), ); await cleanupSandboxController(); - await clearIsrCache(context.env.ISR_CACHE, projectId, failedDeployment.id).catch((e) => + await clearIsrCache( + createIsrStore(context.env.STATIC_ASSETS_R2, context.env.ISR_CACHE), + projectId, + failedDeployment.id, + ).catch((e) => logger.error('catch', 'isr_cache_clear_failed', e, { deploymentId: failedDeployment.id, }), diff --git a/platform/packages/api/src/deploy/publish-worker.ts b/platform/packages/api/src/deploy/publish-worker.ts index f8ad01900..5242899ea 100644 --- a/platform/packages/api/src/deploy/publish-worker.ts +++ b/platform/packages/api/src/deploy/publish-worker.ts @@ -24,6 +24,7 @@ import { type Db, } from '../db.ts'; import type { ApiEnv } from '../env.ts'; +import { createIsrStore } from '../isr-store.ts'; import { getProjectQueueName } from '../naming.ts'; import { type DeployLogger } from '../observability/logger.ts'; import { resolvePlanLimits } from '../plans.ts'; @@ -268,7 +269,11 @@ export async function publishWorkerDeployment(input: { const hasCrons = manifest.schedules && manifest.schedules.length > 0; - await clearIsrCache(context.env.ISR_CACHE, projectId, deployment.id); + await clearIsrCache( + createIsrStore(context.env.STATIC_ASSETS_R2, context.env.ISR_CACHE), + projectId, + deployment.id, + ); await assertDeploymentCanProceed(context.get('db'), deployment.id, projectId); input.currentStep = 'prerender'; @@ -331,7 +336,7 @@ export async function publishWorkerDeployment(input: { const headers = serializePrerenderHeaders(response); const body = await response.arrayBuffer(); const packed = packIsrValue(headers, body); - await context.env.ISR_CACHE.put(key, packed, { + await createIsrStore(context.env.STATIC_ASSETS_R2).put(key, packed, { // `c` (createdAt) + `t` (ttl) are REQUIRED by dispatch's // `readIsrCache` — it returns null (permanent cold miss) // without them. `satisfies IsrCacheMetadata` makes any field @@ -603,7 +608,11 @@ export async function publishWorkerDeployment(input: { }); } } - await clearIsrCache(context.env.ISR_CACHE, projectId, old.id).catch((e) => + await clearIsrCache( + createIsrStore(context.env.STATIC_ASSETS_R2, context.env.ISR_CACHE), + projectId, + old.id, + ).catch((e) => input.logger.error('retention_cleanup', 'evicted_isr_cache_clear_failed', e, { evictedDeploymentId: old.id, }), diff --git a/platform/packages/api/src/isr-store.ts b/platform/packages/api/src/isr-store.ts new file mode 100644 index 000000000..ed8895aa8 --- /dev/null +++ b/platform/packages/api/src/isr-store.ts @@ -0,0 +1,82 @@ +import type { IsrCacheMetadata } from '@void/isr'; + +const PREFIX = 'isr-cache/'; + +export function isrObjectPrefix(projectId: string): string { + return `${PREFIX}${projectId}/`; +} + +export interface IsrStore { + getWithMetadata( + key: string, + type?: 'arrayBuffer', + ): Promise<{ + value: ArrayBuffer | null; + metadata: Metadata | null; + }>; + put(key: string, value: ArrayBuffer, options: { metadata: IsrCacheMetadata }): Promise; + has(key: string): Promise; + delete(key: string): Promise; + list(options: { prefix: string; cursor?: string; limit?: number }): Promise<{ + keys: Array<{ name: string }>; + list_complete: boolean; + cursor?: string; + }>; +} + +/** + * R2 is authoritative: an eventually consistent KV read after a purge can + * otherwise repopulate the edge with deleted HTML for its entire original TTL. + * Legacy KV participates only in purge/cleanup, never in serving or writing. + */ +export function createIsrStore(bucket: R2Bucket, legacy?: KVNamespace): IsrStore { + return { + async getWithMetadata(key) { + const object = await bucket.get(PREFIX + key); + if (!object) { + return { value: null, metadata: null }; + } + return { + value: await object.arrayBuffer(), + metadata: JSON.parse(object.customMetadata?.isr ?? 'null'), + }; + }, + async put(key, value, { metadata }) { + await bucket.put(PREFIX + key, value, { + customMetadata: { isr: JSON.stringify(metadata) }, + }); + }, + async delete(key) { + await bucket.delete(PREFIX + key); + try { + await legacy?.delete(key); + } catch (error) { + // Legacy bodies are never served; their cleanup must not block the edge purge. + console.warn('[api] Legacy ISR KV cleanup failed.', error); + } + }, + async has(key) { + return (await bucket.head(PREFIX + key)) !== null; + }, + async list({ prefix, cursor, limit }) { + if (legacy && cursor?.startsWith('kv:')) { + const page = await legacy.list({ prefix, cursor: cursor.slice(3) || undefined, limit }); + return { + keys: page.keys, + list_complete: page.list_complete, + cursor: page.list_complete ? undefined : `kv:${page.cursor}`, + }; + } + const page = await bucket.list({ + prefix: PREFIX + prefix, + cursor: cursor?.slice(3), + limit, + }); + return { + keys: page.objects.map(({ key }) => ({ name: key.slice(PREFIX.length) })), + list_complete: !page.truncated && !legacy, + cursor: page.truncated ? `r2:${page.cursor}` : legacy ? 'kv:' : undefined, + }; + }, + }; +} diff --git a/platform/packages/api/src/revalidate-core.ts b/platform/packages/api/src/revalidate-core.ts index 372d974fd..6d0c6575a 100644 --- a/platform/packages/api/src/revalidate-core.ts +++ b/platform/packages/api/src/revalidate-core.ts @@ -18,6 +18,7 @@ import { legacyIsrCachePrefix, legacyIsrCacheVariantPrefix, } from './isr-legacy-keys.ts'; +import { createIsrStore } from './isr-store.ts'; import { purgeZoneCache } from './provisioning/cloudflare-api.ts'; type RevalidateProject = Readonly<{ @@ -44,7 +45,7 @@ type RevalidateRequestBody = Readonly<{ * entries have aged out. * * Two-layer purge: - * 1. KV (R2): + * 1. R2 body storage plus legacy KV cleanup: * `{all}` — per deployment id in the union, scans TWO disjoint prefixes: * - `/isr/v3//` (host-scoped): the first tail * segment IS the host, guaranteed by the version. @@ -66,7 +67,7 @@ type RevalidateRequestBody = Readonly<{ * * The `{paths}` branch issues exactly one `purgeZoneCache` call per invocation * (it batches internally at 100 URLs per CF API request); the `{all}` branch - * flushes per KV list page to bound peak memory (one page ~1000 keys). + * flushes per storage list page to bound peak memory (one page ~1000 keys). */ export async function runRevalidate( context: Context, @@ -76,6 +77,7 @@ export async function runRevalidate( db: Db = context.get('db'), ): Promise { const application = readPlatformApplication(context.env); + const cache = createIsrStore(context.env.STATIC_ASSETS_R2, context.env.ISR_CACHE); const customDomains = await getDomainsByProjectId(db, projectId); // Include `pending` domains, not only `active`. Activation writes the @@ -109,7 +111,7 @@ export async function runRevalidate( // Edge purge fans out to `|keys| × 2` URL strings. For high-cardinality // projects (tens of thousands of paths, dozens of variants each, multiple // hostnames/ids) that array can pin hundreds of MB and OOM the Worker. - // Flush per KV list page instead so peak memory is bounded by a single + // Flush per storage list page instead so peak memory is bounded by a single // page (~1000 keys). CF tolerates duplicate purge URLs across calls, so // cross-page dedup loss is fine. const pathnameSet = new Set(); @@ -127,7 +129,7 @@ export async function runRevalidate( let cursor: string | undefined; do { - const listed = await context.env.ISR_CACHE.list({ prefix, cursor }); + const listed = await cache.list({ prefix, cursor }); if (listed.keys.length > 0) { const pageDirect = new Map(); const pageVariants: Array = []; @@ -136,7 +138,7 @@ export async function runRevalidate( const slashIndex = tail.indexOf('/'); if (slashIndex === -1) { // Malformed v3 key (no `/` segment). Can't derive an edge - // URL without guessing a host, so skip the edge purge — the KV + // URL without guessing a host, so skip the edge purge — the storage // delete below still wipes it (an `all` purge clears the whole // deployment namespace). continue; @@ -158,9 +160,7 @@ export async function runRevalidate( }); } } - await uploadPool(listed.keys, R2_CONCURRENCY, (key) => - context.env.ISR_CACHE.delete(key.name), - ); + await uploadPool(listed.keys, R2_CONCURRENCY, (key) => cache.delete(key.name)); await purgeEdgeCache(context, [...pageDirect.values(), ...pageVariants]); } cursor = listed.list_complete ? undefined : listed.cursor; @@ -176,7 +176,7 @@ export async function runRevalidate( let legacyCursor: string | undefined; do { - const listed = await context.env.ISR_CACHE.list({ + const listed = await cache.list({ prefix: legacyPrefix, cursor: legacyCursor, }); @@ -202,9 +202,7 @@ export async function runRevalidate( } } } - await uploadPool(listed.keys, R2_CONCURRENCY, (key) => - context.env.ISR_CACHE.delete(key.name), - ); + await uploadPool(listed.keys, R2_CONCURRENCY, (key) => cache.delete(key.name)); await purgeEdgeCache(context, [...pageDirect.values(), ...pageVariants]); } legacyCursor = listed.list_complete ? undefined : listed.cursor; @@ -212,7 +210,7 @@ export async function runRevalidate( } // `purged` = unique pathnames purged (mirrors what the caller asked for, - // not the raw count of KV keys deleted across all deployment namespaces). + // not the raw count of stored keys deleted across all deployment namespaces). return context.json({ purged: pathnameSet.size }); } @@ -231,13 +229,13 @@ export async function runRevalidate( for (const [hostname, ids] of perHostname) { for (const deploymentId of ids) { for (const path of body.paths) { - await context.env.ISR_CACHE.delete(isrCacheKey(projectId, deploymentId, hostname, path)); + await cache.delete(isrCacheKey(projectId, deploymentId, hostname, path)); edgeTargets.push({ host: hostname, pathname: path, deploymentId, variantToken: null }); const variantPrefix = isrCacheVariantPrefix(projectId, deploymentId, hostname, path); let cursor: string | undefined; do { - const listed = await context.env.ISR_CACHE.list({ prefix: variantPrefix, cursor }); + const listed = await cache.list({ prefix: variantPrefix, cursor }); if (listed.keys.length > 0) { for (const key of listed.keys) { // `variantPrefix` ends with the `#rw=` separator, so the @@ -245,9 +243,7 @@ export async function runRevalidate( const variantToken = key.name.slice(variantPrefix.length); edgeTargets.push({ host: hostname, pathname: path, deploymentId, variantToken }); } - await uploadPool(listed.keys, R2_CONCURRENCY, (key) => - context.env.ISR_CACHE.delete(key.name), - ); + await uploadPool(listed.keys, R2_CONCURRENCY, (key) => cache.delete(key.name)); } cursor = listed.list_complete ? undefined : listed.cursor; } while (cursor); @@ -279,12 +275,12 @@ export async function runRevalidate( for (const deploymentId of union) { const hostsForId = hostsResolvingTo(perHostname, deploymentId); for (const path of body.paths) { - await context.env.ISR_CACHE.delete(legacyIsrCacheKey(projectId, deploymentId, path)); + await cache.delete(legacyIsrCacheKey(projectId, deploymentId, path)); const legacyVariantPrefix = legacyIsrCacheVariantPrefix(projectId, deploymentId, path); let cursor: string | undefined; do { - const listed = await context.env.ISR_CACHE.list({ prefix: legacyVariantPrefix, cursor }); + const listed = await cache.list({ prefix: legacyVariantPrefix, cursor }); if (listed.keys.length > 0) { for (const key of listed.keys) { // `legacyVariantPrefix` ends with the `#rw=` separator, so the @@ -294,9 +290,7 @@ export async function runRevalidate( edgeTargets.push({ host, pathname: path, deploymentId, variantToken }); } } - await uploadPool(listed.keys, R2_CONCURRENCY, (key) => - context.env.ISR_CACHE.delete(key.name), - ); + await uploadPool(listed.keys, R2_CONCURRENCY, (key) => cache.delete(key.name)); } cursor = listed.list_complete ? undefined : listed.cursor; } while (cursor); @@ -304,7 +298,7 @@ export async function runRevalidate( } await purgeEdgeCache(context, edgeTargets); - // `purged` = unique path entries the caller asked for. The per-host KV + // `purged` = unique path entries the caller asked for. The per-host storage // deletes above are an implementation detail of the drift-handling hedge. return context.json({ purged: body.paths.length }); } @@ -323,14 +317,14 @@ function hostsResolvingTo( } type PurgeEdgeTarget = Readonly<{ - /** Hostname the KV key / edge entry is scoped to. */ + /** Hostname the stored key / edge entry is scoped to. */ host: string; /** Public pathname (denormalized, e.g. `/about`). */ pathname: string; /** Deployment id namespace the key lives under (edge `__void_isr` value). */ deploymentId: string; /** - * Pre-hashed variant token from the KV key, or null for a direct-hit key. + * Pre-hashed variant token from the stored key, or null for a direct-hit key. * The original URL is not recoverable post-hash; we only need the token to * rebuild the matching `__void_orig=` edge cache URL via * `isrEdgeCacheUrlForToken`. diff --git a/platform/packages/api/src/routes/github.ts b/platform/packages/api/src/routes/github.ts index 93c03aef7..41c4e400c 100644 --- a/platform/packages/api/src/routes/github.ts +++ b/platform/packages/api/src/routes/github.ts @@ -1267,7 +1267,8 @@ function isValidWorkflowPath(path: string): boolean { // route checks the required live project capability. // --------------------------------------------------------------------------- export const githubProjectRoutes = new Hono() - .use('*', capabilityMiddleware('managedBuilds')) + // Other routers share /projects, so this gate must only cover GitHub connections. + .use('/:id/github', capabilityMiddleware('managedBuilds')) // Connect a project to a repo within one of the caller's installations. .post('/:id/github', async (context) => { diff --git a/platform/packages/api/src/routes/usage.ts b/platform/packages/api/src/routes/usage.ts index 7d4a48ed3..c179ad54f 100644 --- a/platform/packages/api/src/routes/usage.ts +++ b/platform/packages/api/src/routes/usage.ts @@ -595,14 +595,16 @@ async function queryProjectAiUsage( // anyone can write a log row with any metadata. The proxy tags its rows with // a value signed under JWT_SECRET; filtering on that value means rows the // platform did not write are never counted. + // The proxy owns the whole metadata object and writes only this projectId. const usageTag = await deriveAiUsageTag(jwtSecret, projectId); const query = `query { viewer { accounts(filter: { accountTag: "${accountId}" }) { - aiGatewayRequests( + aiGatewayRequests: aiGatewayRequestsAdaptiveGroups( + limit: 1 filter: { - gatewayId: "${gatewayId}" - metadata_projectId: "${usageTag}" + gateway: "${gatewayId}" + metadataRaw: ${JSON.stringify(JSON.stringify({ projectId: usageTag }))} datetime_geq: "${since.toISOString()}" ${until ? `datetime_lt: "${until.toISOString()}"` : ''} } diff --git a/platform/packages/api/test/integration/api-delete-accounting.test.ts b/platform/packages/api/test/integration/api-delete-accounting.test.ts index b07dc109e..d3811ac77 100644 --- a/platform/packages/api/test/integration/api-delete-accounting.test.ts +++ b/platform/packages/api/test/integration/api-delete-accounting.test.ts @@ -213,7 +213,9 @@ it.each(['fresh', 'initialized'] as const)( expect(queries).toHaveLength(2); const usageTag = await deriveAiUsageTag(env.JWT_SECRET, project.id); for (const query of queries) { - expect(query).toContain(`metadata_projectId: "${usageTag}"`); + expect(query).toContain( + `metadataRaw: ${JSON.stringify(JSON.stringify({ projectId: usageTag }))}`, + ); expect(query).toContain(`datetime_geq: "${ownershipStartedAt}"`); expect(query).toContain(`datetime_lt: "${window!.ownershipEndedAt}"`); } diff --git a/platform/packages/api/test/integration/api-delete.test.ts b/platform/packages/api/test/integration/api-delete.test.ts index 6ebc6ceb7..a797f677a 100644 --- a/platform/packages/api/test/integration/api-delete.test.ts +++ b/platform/packages/api/test/integration/api-delete.test.ts @@ -314,6 +314,10 @@ describe('DELETE /projects/:id', () => { const user = await createTestUser(env.db); const project = await createTestProject(env.db, user.id, 'full-app'); mockCfApi(); + const isrKey = `isr-cache/${project.id}/isr/v3/dpl_test1/full-app.void.app/page`; + const otherIsrKey = 'isr-cache/other-project/isr/v3/other/other.void.app/page'; + await env.STATIC_ASSETS_R2.put(isrKey, 'cached page'); + await env.STATIC_ASSETS_R2.put(otherIsrKey, 'unrelated cache'); await addResource(project.id, 'd1', 'd1-uuid-123', 'DB'); await addResource(project.id, 'kv', 'kv-id-456', 'KV'); @@ -358,6 +362,8 @@ describe('DELETE /projects/:id', () => { const routing = await env.ROUTING.get('full-app'); expect(routing).toBeNull(); + expect(await env.STATIC_ASSETS_R2.get(isrKey)).toBeNull(); + expect(await env.STATIC_ASSETS_R2.get(otherIsrKey)).not.toBeNull(); }); it('terminalizes in-flight builds and tears down their container DOs', async () => { diff --git a/platform/packages/api/test/integration/api-deploy.test.ts b/platform/packages/api/test/integration/api-deploy.test.ts index 67f3dcaac..72685dd0f 100644 --- a/platform/packages/api/test/integration/api-deploy.test.ts +++ b/platform/packages/api/test/integration/api-deploy.test.ts @@ -24,6 +24,7 @@ import { type Db, } from '../../src/db.ts'; import app from '../../src/index.ts'; +import { createIsrStore } from '../../src/isr-store.ts'; import { writePlanCatalog } from '../../src/plans.ts'; import * as projectAccess from '../../src/project-access.ts'; import * as projectIngress from '../../src/project-ingress.ts'; @@ -5537,8 +5538,10 @@ describe('Deploy with prerendering', () => { .where(eq(deployments.project_id, project.id)); expect(deployment).toBeDefined(); - // Verify ISR cache entries were written to KV (packed binary format) - const { value: homeValue, metadata: homeMeta } = await env.ISR_CACHE.getWithMetadata<{ + // Verify ISR cache entries were written to authoritative R2 (packed binary format) + const { value: homeValue, metadata: homeMeta } = await createIsrStore( + env.STATIC_ASSETS_R2, + ).getWithMetadata<{ s: number; c: number; t: number; @@ -5552,7 +5555,7 @@ describe('Deploy with prerendering', () => { expect(homeMeta?.s).toBe(200); expect(homeMeta?.t).toBe(3600); - const { value: aboutValue } = await env.ISR_CACHE.getWithMetadata( + const { value: aboutValue } = await createIsrStore(env.STATIC_ASSETS_R2).getWithMetadata( `${project.id}/isr/v3/${deployment!.id}/${project.slug}.void.app/about`, 'arrayBuffer', ); @@ -5592,7 +5595,7 @@ describe('Deploy with prerendering', () => { .select({ id: deployments.id }) .from(deployments) .where(eq(deployments.project_id, project.id)); - const { value } = await env.ISR_CACHE.getWithMetadata( + const { value } = await createIsrStore(env.STATIC_ASSETS_R2).getWithMetadata( `${project.id}/isr/v3/${deployment!.id}/${project.slug}.void.app/data`, 'arrayBuffer', ); @@ -5642,9 +5645,11 @@ describe('Deploy with prerendering', () => { // deployments' v2 entries remain intact. expect(await env.ISR_CACHE.get(`${project.id}/isr/v2/previous/about`)).toBe('previous-v2'); expect( - await env.ISR_CACHE.get( - `${project.id}/isr/v3/${deployment!.id}/${project.slug}.void.app/about`, - ), + ( + await createIsrStore(env.STATIC_ASSETS_R2).getWithMetadata( + `${project.id}/isr/v3/${deployment!.id}/${project.slug}.void.app/about`, + ) + ).value, ).not.toBeNull(); }); @@ -5688,14 +5693,14 @@ describe('Deploy with prerendering', () => { expect(deployment).toBeDefined(); // /about should have ISR entry - const { value: aboutValue } = await env.ISR_CACHE.getWithMetadata( + const { value: aboutValue } = await createIsrStore(env.STATIC_ASSETS_R2).getWithMetadata( `${project.id}/isr/v3/${deployment!.id}/${project.slug}.void.app/about`, 'arrayBuffer', ); expect(aboutValue).not.toBeNull(); // /private should NOT have ISR entry - const { value: privateValue } = await env.ISR_CACHE.getWithMetadata( + const { value: privateValue } = await createIsrStore(env.STATIC_ASSETS_R2).getWithMetadata( `${project.id}/isr/v3/${deployment!.id}/${project.slug}.void.app/private`, 'arrayBuffer', ); @@ -5768,24 +5773,26 @@ describe('Deploy with prerendering', () => { const base = `${project.id}/isr/v3/${deployment!.id}/${project.slug}.void.app`; // The normal 200 render is warmed. - const { value: warmValue, metadata: warmMetadata } = await env.ISR_CACHE.getWithMetadata<{ + const { value: warmValue, metadata: warmMetadata } = await createIsrStore( + env.STATIC_ASSETS_R2, + ).getWithMetadata<{ p: number; }>(`${base}/warm`, 'arrayBuffer'); expect(warmValue).not.toBeNull(); expect(warmMetadata?.p).toBe(SHARED_CACHE_POLICY_VERSION); // Neither opt-out render is warmed. - const { value: headerOptOut } = await env.ISR_CACHE.getWithMetadata( + const { value: headerOptOut } = await createIsrStore(env.STATIC_ASSETS_R2).getWithMetadata( `${base}/optout-header`, 'arrayBuffer', ); expect(headerOptOut).toBeNull(); - const { value: ccOptOut } = await env.ISR_CACHE.getWithMetadata( + const { value: ccOptOut } = await createIsrStore(env.STATIC_ASSETS_R2).getWithMetadata( `${base}/optout-cc`, 'arrayBuffer', ); expect(ccOptOut).toBeNull(); - const { value: cookieOptOut } = await env.ISR_CACHE.getWithMetadata( + const { value: cookieOptOut } = await createIsrStore(env.STATIC_ASSETS_R2).getWithMetadata( `${base}/optout-cookie`, 'arrayBuffer', ); @@ -5841,12 +5848,12 @@ describe('Deploy with prerendering', () => { expect(deployment).toBeDefined(); const base = `${project.id}/isr/v3/${deployment!.id}/${project.slug}.void.app`; - const { value: otherValue } = await env.ISR_CACHE.getWithMetadata( + const { value: otherValue } = await createIsrStore(env.STATIC_ASSETS_R2).getWithMetadata( `${base}/blog/other`, 'arrayBuffer', ); expect(otherValue).not.toBeNull(); - const { value: privateValue } = await env.ISR_CACHE.getWithMetadata( + const { value: privateValue } = await createIsrStore(env.STATIC_ASSETS_R2).getWithMetadata( `${base}/blog/private`, 'arrayBuffer', ); @@ -5899,14 +5906,14 @@ describe('Deploy with prerendering', () => { expect(deployment).toBeDefined(); // /blog/hello should have an ISR entry. - const { value: helloValue } = await env.ISR_CACHE.getWithMetadata( + const { value: helloValue } = await createIsrStore(env.STATIC_ASSETS_R2).getWithMetadata( `${project.id}/isr/v3/${deployment!.id}/${project.slug}.void.app/blog/hello`, 'arrayBuffer', ); expect(helloValue).not.toBeNull(); // /blog/a/b must NOT have an ISR entry (single-segment anchor). - const { value: nestedValue } = await env.ISR_CACHE.getWithMetadata( + const { value: nestedValue } = await createIsrStore(env.STATIC_ASSETS_R2).getWithMetadata( `${project.id}/isr/v3/${deployment!.id}/${project.slug}.void.app/blog/a/b`, 'arrayBuffer', ); @@ -5964,7 +5971,9 @@ describe('Deploy with prerendering', () => { // `c`) exists as bytes but is a permanent cold miss — dispatch's // `readIsrCache` returns null without `c`/`t`. Assert the readable shape, // not just presence. - const { value: introValue, metadata: introMeta } = await env.ISR_CACHE.getWithMetadata<{ + const { value: introValue, metadata: introMeta } = await createIsrStore( + env.STATIC_ASSETS_R2, + ).getWithMetadata<{ s: number; c: number; t: number; @@ -5975,11 +5984,12 @@ describe('Deploy with prerendering', () => { expect(introMeta!.c).toBeGreaterThan(0); // The NESTED path /docs/guide/setup must ALSO have a readable ISR entry. - const { value: nestedCatchAllValue, metadata: nestedMeta } = - await env.ISR_CACHE.getWithMetadata<{ s: number; c: number; t: number }>( - `${project.id}/isr/v3/${deployment!.id}/${project.slug}.void.app/docs/guide/setup`, - 'arrayBuffer', - ); + const { value: nestedCatchAllValue, metadata: nestedMeta } = await createIsrStore( + env.STATIC_ASSETS_R2, + ).getWithMetadata<{ s: number; c: number; t: number }>( + `${project.id}/isr/v3/${deployment!.id}/${project.slug}.void.app/docs/guide/setup`, + 'arrayBuffer', + ); expect(nestedCatchAllValue).not.toBeNull(); expect(nestedMeta).toMatchObject({ s: 200, t: 3600 }); expect(typeof nestedMeta?.c).toBe('number'); @@ -6097,7 +6107,9 @@ describe('Deploy with prerendering', () => { .where(eq(deployments.project_id, project.id)); expect(deployment).toBeDefined(); - const { value: pValue, metadata: pMeta } = await env.ISR_CACHE.getWithMetadata<{ + const { value: pValue, metadata: pMeta } = await createIsrStore( + env.STATIC_ASSETS_R2, + ).getWithMetadata<{ s: number; c: number; t: number; @@ -6156,13 +6168,15 @@ describe('Deploy with prerendering', () => { .where(eq(deployments.project_id, project.id)); expect(deployment).toBeDefined(); - const { value: specificValue } = await env.ISR_CACHE.getWithMetadata( + const { value: specificValue } = await createIsrStore(env.STATIC_ASSETS_R2).getWithMetadata( `${project.id}/isr/v3/${deployment!.id}/${project.slug}.void.app/specific`, 'arrayBuffer', ); expect(specificValue).toBeNull(); - const { value: otherValue, metadata: otherMeta } = await env.ISR_CACHE.getWithMetadata<{ + const { value: otherValue, metadata: otherMeta } = await createIsrStore( + env.STATIC_ASSETS_R2, + ).getWithMetadata<{ s: number; c: number; t: number; diff --git a/platform/packages/api/test/integration/api-project-access-routes.test.ts b/platform/packages/api/test/integration/api-project-access-routes.test.ts index 37bc5ea5b..2a852debf 100644 --- a/platform/packages/api/test/integration/api-project-access-routes.test.ts +++ b/platform/packages/api/test/integration/api-project-access-routes.test.ts @@ -1,6 +1,7 @@ import { eq } from 'drizzle-orm'; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vite-plus/test'; import { verifyPrerenderToken } from '../../src/auth/deploy-token.ts'; +import { getPlatformCapabilities } from '../../src/capabilities.ts'; import app from '../../src/index.ts'; import * as projectAccess from '../../src/project-access.ts'; import { deriveAiUsageTag } from '../../src/proxy-token.ts'; @@ -568,6 +569,73 @@ describe('project route membership capabilities', () => { }); }); +describe('managed build platform capability', () => { + function withoutManagedBuilds(): TestEnv { + return { + ...env, + PLATFORM_CAPABILITIES: JSON.stringify({ + ...getPlatformCapabilities(env), + managedBuilds: false, + }), + }; + } + + it('serves project usage when managed builds are disabled', async () => { + const owner = await createTestUser(env.db); + const project = await createTestProject(env.db, owner.id); + vi.stubGlobal( + 'fetch', + vi.fn(async (input: string | URL | Request) => { + const url = + typeof input === 'string' ? input : input instanceof URL ? input.href : input.url; + if (url.includes('/analytics_engine/sql')) { + return Response.json({ data: [{ requests: 12, bandwidth_bytes: 34 }] }); + } + if (url.includes('/graphql')) { + return Response.json({ + data: { + viewer: { + accounts: [{ aiGatewayRequests: [{ sum: { cost: 0.000022 } }] }], + }, + }, + }); + } + throw new Error(`unexpected fetch: ${url}`); + }), + ); + + const response = await app.fetch( + authed(`/projects/${project.id}/usage`, owner.jwt), + withoutManagedBuilds(), + ); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + projectId: project.id, + requests: 12, + bandwidthBytes: 34, + aiNeurons: 2, + }); + }); + + it.each(['GET', 'POST', 'PATCH', 'DELETE'])( + 'keeps %s project GitHub connections unavailable when managed builds are disabled', + async (method) => { + const owner = await createTestUser(env.db); + const project = await createTestProject(env.db, owner.id); + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + + const response = await app.fetch( + authed(`/projects/${project.id}/github`, owner.jwt, method, {}), + withoutManagedBuilds(), + ); + expect(response.status).toBe(501); + expect(await response.json()).toMatchObject({ capability: 'managedBuilds' }); + expect(fetchMock).not.toHaveBeenCalled(); + }, + ); +}); + describe('ownership-period account usage', () => { it('reports unavailable usage rather than zero when analytics fails', async () => { const owner = await createTestUser(env.db, 'usage-owner'); @@ -669,7 +737,9 @@ describe('ownership-period account usage', () => { ); expect(graphqlQueries).toHaveLength(1); expect(graphqlQueries[0]).toContain( - `metadata_projectId: "${await deriveAiUsageTag(JWT_SECRET, project.id)}"`, + `metadataRaw: ${JSON.stringify( + JSON.stringify({ projectId: await deriveAiUsageTag(JWT_SECRET, project.id) }), + )}`, ); expect(graphqlQueries[0]).toContain(`datetime_geq: "${ownedFrom.toISOString()}"`); expect(graphqlQueries[0]).toContain(`datetime_lt: "${transferredAt.toISOString()}"`); diff --git a/platform/packages/api/test/integration/api-revalidate.test.ts b/platform/packages/api/test/integration/api-revalidate.test.ts index 6b80cb28e..5c7f36d4d 100644 --- a/platform/packages/api/test/integration/api-revalidate.test.ts +++ b/platform/packages/api/test/integration/api-revalidate.test.ts @@ -12,6 +12,7 @@ import { } from 'vite-plus/test'; import { hmacSignState } from '../../src/auth/hmac.ts'; import app from '../../src/index.ts'; +import { createIsrStore } from '../../src/isr-store.ts'; import { domains, projectMembers } from '../../src/schema.ts'; import { setupEnv, @@ -64,6 +65,96 @@ afterEach(() => { }); describe('POST /projects/:id/revalidate', () => { + it.each([false, true])('purges R2 pages and legacy KV edge variants (all=%s)', async (all) => { + const user = await createTestUser(env.db); + const project = await createTestProject(env.db, user.id); + const deployment = await createTestDeployment(env.db, project.id, 'live'); + const host = `${project.slug}.void.app`; + const direct = isrKey(project.id, deployment.id, host, 'docs'); + const variant = `${direct}#rw=${hashOriginalUrl(`https://${host}/old-docs`)}`; + const legacy = `${legacyIsrKey(project.id, deployment.id, 'docs')}#rw=${hashOriginalUrl(`https://${host}/legacy-docs`)}`; + const cache = createIsrStore(env.STATIC_ASSETS_R2); + for (const key of [direct, variant]) { + await cache.put(key, new ArrayBuffer(4), { + metadata: { s: 200, c: Date.now(), t: 600, p: 3 }, + }); + } + await env.ISR_CACHE.put(legacy, 'legacy body'); + const fetchMock = mockCfApi(); + const response = await app.fetch( + new Request(`http://localhost/projects/${project.id}/revalidate`, { + method: 'POST', + headers: { Authorization: `Bearer ${user.jwt}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(all ? { all: true } : { paths: ['/docs'] }), + }), + env, + ); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ purged: 1 }); + expect((await cache.getWithMetadata(direct)).value).toBeNull(); + expect((await cache.getWithMetadata(variant)).value).toBeNull(); + expect(await env.ISR_CACHE.get(legacy)).toBeNull(); + const urls = fetchMock.mock.calls + .filter(([url]) => String(url).includes('/purge_cache')) + .flatMap(([, options]) => JSON.parse(options!.body as string).files as Array); + expect(urls).toContain(`https://${host}/docs?__void_isr=${deployment.id}`); + expect(urls).toContain( + `https://${host}/docs?__void_isr=${deployment.id}&__void_orig=${hashOriginalUrl(`https://${host}/old-docs`)}`, + ); + expect(urls).toContain( + `https://${host}/docs?__void_isr=${deployment.id}&__void_orig=${hashOriginalUrl(`https://${host}/legacy-docs`)}`, + ); + }); + + it.each([false, true])( + 'continues edge purging when legacy KV cleanup fails (all=%s)', + async (all) => { + const user = await createTestUser(env.db); + const project = await createTestProject(env.db, user.id); + const deployment = await createTestDeployment(env.db, project.id, 'live'); + const host = `${project.slug}.void.app`; + const direct = isrKey(project.id, deployment.id, host, 'docs'); + const token = hashOriginalUrl(`https://${host}/old-docs`); + const legacy = `${legacyIsrKey(project.id, deployment.id, 'docs')}#rw=${token}`; + const cache = createIsrStore(env.STATIC_ASSETS_R2); + await cache.put(direct, new ArrayBuffer(4), { + metadata: { s: 200, c: Date.now(), t: 600, p: 3 }, + }); + await env.ISR_CACHE.put(legacy, 'stale legacy HTML'); + const failure = new Error('Legacy KV unavailable'); + const remove = vi.spyOn(env.ISR_CACHE, 'delete').mockRejectedValue(failure); + const warning = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const fetchMock = mockCfApi(); + + const response = await app.fetch( + new Request(`http://localhost/projects/${project.id}/revalidate`, { + method: 'POST', + headers: { Authorization: `Bearer ${user.jwt}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(all ? { all: true } : { paths: ['/docs'] }), + }), + env, + ); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ purged: 1 }); + expect((await cache.getWithMetadata(direct)).value).toBeNull(); + expect(await env.ISR_CACHE.get(legacy)).toBe('stale legacy HTML'); + expect(remove).toHaveBeenCalledWith(legacy); + expect(warning).toHaveBeenCalledWith('[api] Legacy ISR KV cleanup failed.', failure); + const urls = fetchMock.mock.calls + .filter(([url]) => String(url).includes('/purge_cache')) + .flatMap(([, options]) => JSON.parse(options!.body as string).files as Array); + expect(urls).toEqual( + expect.arrayContaining([ + `https://${host}/docs?__void_isr=${deployment.id}`, + `https://${host}/docs?__void=json&__void_isr=${deployment.id}`, + `https://${host}/docs?__void_isr=${deployment.id}&__void_orig=${token}`, + `https://${host}/docs?__void=json&__void_isr=${deployment.id}&__void_orig=${token}`, + ]), + ); + }, + ); + it('returns 401 without auth', async () => { const res = await app.fetch( new Request('http://localhost/projects/proj_fake/revalidate', { diff --git a/platform/packages/api/test/integration/deploy-direct-r2.test.ts b/platform/packages/api/test/integration/deploy-direct-r2.test.ts index 2d8c230e2..fc415b80d 100644 --- a/platform/packages/api/test/integration/deploy-direct-r2.test.ts +++ b/platform/packages/api/test/integration/deploy-direct-r2.test.ts @@ -442,10 +442,14 @@ describe('POST /projects/:id/deploy (JSON mode — direct-to-R2)', () => { const head = vi.spyOn(env.STATIC_ASSETS_R2, 'head'); const originalList = env.STATIC_ASSETS_R2.list.bind(env.STATIC_ASSETS_R2); let releaseList: (() => void) | undefined; + const assetPrefix = `assets/${project.id}/`; const list = vi.spyOn(env.STATIC_ASSETS_R2, 'list').mockImplementation(async (options) => { - await new Promise((resolve) => { - releaseList = resolve; - }); + // Only asset GC is deferred; ISR cleanup uses the same bucket. + if (options?.prefix === assetPrefix) { + await new Promise((resolve) => { + releaseList = resolve; + }); + } return originalList(options); }); const pending: Array> = []; @@ -479,7 +483,11 @@ describe('POST /projects/:id/deploy (JSON mode — direct-to-R2)', () => { expect(events.find((event) => isEvent(event, 'done'))).toBeDefined(); expect(head).not.toHaveBeenCalled(); expect(pending).toHaveLength(1); - await vi.waitFor(() => expect(list).toHaveBeenCalledOnce()); + await vi.waitFor(() => + expect(list.mock.calls.filter(([options]) => options?.prefix === assetPrefix)).toHaveLength( + 1, + ), + ); } finally { releaseList?.(); await Promise.allSettled(pending); diff --git a/platform/packages/api/test/integration/helpers.ts b/platform/packages/api/test/integration/helpers.ts index 76ea4f970..f3f164e92 100644 --- a/platform/packages/api/test/integration/helpers.ts +++ b/platform/packages/api/test/integration/helpers.ts @@ -262,6 +262,7 @@ export class MemoryKVNamespace { type MemoryR2Entry = { body: ArrayBuffer; httpMetadata?: unknown; + customMetadata?: Record; etag: string; uploaded: Date; }; @@ -322,6 +323,7 @@ class MemoryR2Bucket { value: ArrayBuffer | ArrayBufferView | ReadableStream | Blob | string, options?: { httpMetadata?: unknown; + customMetadata?: Record; onlyIf?: { etagMatches?: string; etagDoesNotMatch?: string }; }, ): Promise<{ key: string; size: number; etag: string; uploaded: Date } | null> { @@ -351,6 +353,7 @@ class MemoryR2Bucket { const entry: MemoryR2Entry = { body, httpMetadata: options?.httpMetadata, + customMetadata: options?.customMetadata, etag: memoryR2Etag(), uploaded: new Date(), }; @@ -360,6 +363,7 @@ class MemoryR2Bucket { async get(key: string): Promise<{ key: string; + customMetadata?: Record; size: number; etag: string; uploaded: Date; @@ -374,6 +378,7 @@ class MemoryR2Bucket { } return { key, + customMetadata: entry.customMetadata, size: entry.body.byteLength, etag: entry.etag, uploaded: entry.uploaded, diff --git a/platform/packages/api/test/unit/clear-isr-cache.test.ts b/platform/packages/api/test/unit/clear-isr-cache.test.ts index 1f4f3fac4..8f28bd0aa 100644 --- a/platform/packages/api/test/unit/clear-isr-cache.test.ts +++ b/platform/packages/api/test/unit/clear-isr-cache.test.ts @@ -1,6 +1,7 @@ import { isrCachePrefix } from '@void/isr'; import { describe, expect, it } from 'vite-plus/test'; import { legacyIsrCachePrefix } from '../../src/isr-legacy-keys.ts'; +import { createIsrStore } from '../../src/isr-store.ts'; import { clearIsrCache } from '../../src/routes/deploy.ts'; // Minimal KV mock: `list({ prefix })` returns seeded keys whose name starts with @@ -27,6 +28,25 @@ function makeKv(keys: Array) { } describe('clearIsrCache', () => { + it('clears R2 and legacy KV for an evicted deployment while retaining its neighbor', async () => { + const current = `${isrCachePrefix('p', 'old')}host/page`; + const retained = `${isrCachePrefix('p', 'retained')}host/page`; + const { kv, store: legacy } = makeKv([current, retained]); + const objects = new Set([`isr-cache/${current}`, `isr-cache/${retained}`]); + const bucket = { + list: async ({ prefix }: { prefix: string }) => ({ + objects: [...objects].filter((key) => key.startsWith(prefix)).map((key) => ({ key })), + truncated: false, + }), + delete: async (key: string) => { + objects.delete(key); + }, + } as unknown as R2Bucket; + await clearIsrCache(createIsrStore(bucket, kv), 'p', 'old'); + expect([...objects]).toEqual([`isr-cache/${retained}`]); + expect([...legacy]).toEqual([retained]); + }); + it('sweeps BOTH the v3 host-scoped and the legacy v2 hostless namespace for the deployment', async () => { const proj = 'proj1'; const dep = 'dep1'; diff --git a/platform/packages/api/test/unit/isr-store.test.ts b/platform/packages/api/test/unit/isr-store.test.ts new file mode 100644 index 000000000..a340eb087 --- /dev/null +++ b/platform/packages/api/test/unit/isr-store.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it, vi } from 'vite-plus/test'; +import { createIsrStore } from '../../src/isr-store.ts'; + +describe('managed ISR storage', () => { + it('reads the authoritative R2 body and metadata without consulting stale KV', async () => { + const value = new TextEncoder().encode('fresh HTML').buffer; + const metadata = { s: 200, c: 1_000, t: 600, p: 3 } as const; + const get = vi.fn(async () => ({ + arrayBuffer: async () => value, + customMetadata: { isr: JSON.stringify(metadata) }, + })); + const legacyGet = vi.fn(); + const store = createIsrStore( + { get } as unknown as R2Bucket, + { getWithMetadata: legacyGet } as unknown as KVNamespace, + ); + expect(await store.getWithMetadata('project/isr/v3/deploy/host/path')).toEqual({ + value, + metadata, + }); + expect(get).toHaveBeenCalledWith('isr-cache/project/isr/v3/deploy/host/path'); + get.mockResolvedValueOnce(null!); + expect(await store.getWithMetadata('project/isr/v3/deploy/host/path')).toEqual({ + value: null, + metadata: null, + }); + expect(legacyGet).not.toHaveBeenCalled(); + }); + + it('writes only R2 and removes both R2 and the legacy KV entry', async () => { + const put = vi.fn(async () => ({})); + const remove = vi.fn(async () => {}); + const legacyPut = vi.fn(); + const legacyDelete = vi.fn(async () => {}); + const store = createIsrStore( + { put, delete: remove } as unknown as R2Bucket, + { put: legacyPut, delete: legacyDelete } as unknown as KVNamespace, + ); + const body = new ArrayBuffer(4); + const metadata = { s: 200, c: 1_000, t: 600, p: 3 } as const; + await store.put('key', body, { metadata }); + expect(put).toHaveBeenCalledWith('isr-cache/key', body, { + customMetadata: { isr: JSON.stringify(metadata) }, + }); + expect(legacyPut).not.toHaveBeenCalled(); + await store.delete('key'); + expect(remove).toHaveBeenCalledWith('isr-cache/key'); + expect(legacyDelete).toHaveBeenCalledWith('key'); + }); + + it('propagates authoritative R2 deletion failures before attempting legacy cleanup', async () => { + const failure = new Error('R2 unavailable'); + const remove = vi.fn().mockRejectedValue(failure); + const legacyDelete = vi.fn(); + const store = createIsrStore( + { delete: remove } as unknown as R2Bucket, + { delete: legacyDelete } as unknown as KVNamespace, + ); + + await expect(store.delete('key')).rejects.toBe(failure); + expect(legacyDelete).not.toHaveBeenCalled(); + }); + + it('visits empty nonfinal R2 pages before all legacy KV pages', async () => { + const list = vi + .fn() + .mockResolvedValueOnce({ objects: [], truncated: true, cursor: 'next-r2' }) + .mockResolvedValueOnce({ + objects: [{ key: 'isr-cache/p/isr/v3/d/h/new' }], + truncated: false, + }); + const legacyList = vi + .fn() + .mockResolvedValueOnce({ + keys: [{ name: 'p/isr/v3/d/h/old' }], + list_complete: false, + cursor: 'next-kv', + }) + .mockResolvedValueOnce({ keys: [{ name: 'p/isr/v3/d/h/last' }], list_complete: true }); + const store = createIsrStore( + { list } as unknown as R2Bucket, + { list: legacyList } as unknown as KVNamespace, + ); + const keys: Array = []; + let cursor: string | undefined; + do { + const page = await store.list({ prefix: 'p/isr/', cursor, limit: 2 }); + keys.push(...page.keys.map(({ name }) => name)); + cursor = page.list_complete ? undefined : page.cursor; + } while (cursor); + expect(keys).toEqual(['p/isr/v3/d/h/new', 'p/isr/v3/d/h/old', 'p/isr/v3/d/h/last']); + expect(list.mock.calls).toEqual([ + [{ prefix: 'isr-cache/p/isr/', cursor: undefined, limit: 2 }], + [{ prefix: 'isr-cache/p/isr/', cursor: 'next-r2', limit: 2 }], + ]); + expect(legacyList.mock.calls).toEqual([ + [{ prefix: 'p/isr/', cursor: undefined, limit: 2 }], + [{ prefix: 'p/isr/', cursor: 'next-kv', limit: 2 }], + ]); + }); + + it('finishes the R2-only list without adding a legacy scan', async () => { + const list = vi.fn(async () => ({ objects: [], truncated: false })); + expect(await createIsrStore({ list } as unknown as R2Bucket).list({ prefix: 'p/' })).toEqual({ + keys: [], + list_complete: true, + cursor: undefined, + }); + }); +}); diff --git a/platform/packages/api/test/unit/usage-ai-query.test.ts b/platform/packages/api/test/unit/usage-ai-query.test.ts index c69287d04..6276754dd 100644 --- a/platform/packages/api/test/unit/usage-ai-query.test.ts +++ b/platform/packages/api/test/unit/usage-ai-query.test.ts @@ -43,10 +43,17 @@ describe('queryProjectAiUsage strict accounting', () => { const [url, init] = vi.mocked(fetch).mock.calls[0]; expect(url).toBe('https://api.cloudflare.com/client/v4/graphql'); const request = JSON.parse(String(init?.body)); + expect(request.query).toContain('aiGatewayRequests: aiGatewayRequestsAdaptiveGroups('); + expect(request.query).toContain('limit: 1'); + expect(request.query).toContain('gateway: "void"'); expect(request.query).toContain( - `metadata_projectId: "${await deriveAiUsageTag(jwtSecret, 'project-1')}"`, + `metadataRaw: ${JSON.stringify( + JSON.stringify({ projectId: await deriveAiUsageTag(jwtSecret, 'project-1') }), + )}`, ); - expect(request.query).not.toContain('metadata_projectId: "project-1"'); + expect(request.query).not.toContain('metadata_projectId'); + expect(request.query).not.toContain('gatewayId:'); + expect(request.query).not.toContain('"project-1"'); expect(request.query).toContain(`datetime_geq: "${since.toISOString()}"`); expect(request.query).toContain(`datetime_lt: "${until.toISOString()}"`); }); diff --git a/platform/packages/dispatch/README.md b/platform/packages/dispatch/README.md index 0158f73a7..dc9f1cc99 100644 --- a/platform/packages/dispatch/README.md +++ b/platform/packages/dispatch/README.md @@ -11,7 +11,7 @@ Edge router for deployed Void apps. It routes requests to user workers, serves s 5. Serve hashed assets from edge cache when possible. 6. For static/SPA projects, serve current direct-upload assets from `STATIC_ASSETS_R2`; preserve `STATIC_ASSETS` KV reads for assets carried forward from legacy deploys. Storage reads get one delayed hedge within the existing 10-second request budget, then use `assetConfig` path resolution. 7. Apply `fallbackRules` (post-asset) only when static asset resolution misses — instead of returning 404, match the pathname against fallback rules (status=200 only) and rewrite. Intended for SPA shells (`/* → /index.html`) and default-locale catch-alls. On static deployments, a matched fallback serves the resolved asset directly without dispatching to a user worker, so `X-Void-Original-URL` is never set, `c.originalUrl()` is unavailable, and the response is not ISR-cached (see `packages/dispatch/test/unit/fallback-rules.test.ts`). -8. For `/api/*`, forward to the user worker in the dispatch namespace with plan-specific CPU/subrequest limits. +8. For `/api/*`, forward to the user worker in the dispatch namespace with plan-specific CPU/subrequest limits. Foreground worker responses, including SSR and ISR cache misses, have no default wall-time deadline while the client remains connected. 9. For SSR with ISR enabled, use `caches.default` plus `ISR_CACHE` KV for stale-while-revalidate behavior. 10. For WfP deploys, inspect worker and cache responses for the Cloudflare asset-dedupe-poisoning stub (`Assets have not yet been deployed to this script.`). On detection, purge any poisoned cache entry and re-serve from the platform-owned mirror (`STATIC_ASSETS`/`STATIC_ASSETS_R2`). If the mirror is unavailable or incomplete, return an attributed `503` instead of passing through and caching the known-bad `200` stub. See the [asset fallback design](../../meta/design-docs/0066-wfp-asset-r2-fallback.md). 11. Apply header rules and record usage in Analytics Engine. If `userId` is present on the routing entry, enqueue usage events as well. Platform-owned static storage failures carry a `failureKind`: they remain metered and queryable but are excluded from customer application-error emails. @@ -89,7 +89,9 @@ applying a valid event. ### Explaining a dispatch-generated 503 -A user worker cut at the dispatch budget (`DISPATCH_TIMEOUT_MS`, default 10s) never returns, so it emits no log of its own and its tail event records only `canceled` — the same outcome a real client hang-up produces. Dispatch is the only party that knows it was the one that hung up. +`DISPATCH_TIMEOUT_MS` optionally imposes a foreground worker response deadline. Without it, requests follow Cloudflare's client-connected lifetime; platform storage and background ISR revalidation retain their 10-second budgets. An explicitly configured deadline can cancel a worker after an external side effect has completed, so a timeout does not establish that a mutation failed. + +A user worker cut at that configured budget never returns, so it emits no log of its own and its tail event records only `canceled` — the same outcome a real client hang-up produces. Dispatch is the only party that knows it was the one that hung up. On those three worker-timeout paths, `recordPlatformLog` (`metering.ts`) writes one row into the `void_logs` dataset through the `LOGS` binding, so `void project logs` explains the 503 instead of showing nothing. The row uses the same seven-blob/two-double shape `packages/tail` writes, and is keyed by splitting `RoutingEntry.workerName` (`user--dpl_`) into `index1` and `blob7` exactly as the tail worker does — the read path in `GET /projects/:id/logs` selects on `index1`, so a differently-keyed row would be unreadable. Do not key it off the metering `slug`, which falls back to the hostname on a custom domain. diff --git a/platform/packages/dispatch/src/index.ts b/platform/packages/dispatch/src/index.ts index 77e54b43d..6bb54ae20 100644 --- a/platform/packages/dispatch/src/index.ts +++ b/platform/packages/dispatch/src/index.ts @@ -705,9 +705,11 @@ const dispatcher = { return createMaintenanceResponse(request); } const startTime = Date.now(); - // Budget for the bounded hot paths (static-asset reads, primary dispatch). - // See `resolveTimeoutMs` for the operator kill switch. + // Keep platform storage/background work bounded. Foreground user workers + // follow the client's lifetime unless the operator configures a deadline: + // returning early can cancel a mutation after its external side effect. const timeoutMs = resolveTimeoutMs(env.DISPATCH_TIMEOUT_MS); + const workerTimeoutMs = resolveTimeoutMs(env.DISPATCH_TIMEOUT_MS, 0); // Block Void-internal routes from external HTTP access. // Only /__void/* is reserved by the platform — other __ prefixes @@ -885,7 +887,7 @@ const dispatcher = { method: request.method, url: request.url, status: 503, - timeoutMs, + timeoutMs: workerTimeoutMs, }); return meter(timeoutResponse(503, 'dispatch')); }; @@ -1335,11 +1337,15 @@ const dispatcher = { requestType = 'isr'; const worker = getWorker(env, routing); const isrWorkerStartedAt = Date.now(); + const isrWorkerDeadline = + env.DISPATCH_TIMEOUT_MS === undefined + ? undefined + : { startedAt: isrWorkerStartedAt, timeoutMs: workerTimeoutMs }; let workerResponse: Response; try { workerResponse = await withTimeout( worker.fetch(workerRequest), - remainingTimeoutBudget(isrWorkerStartedAt, timeoutMs, 'dispatch'), + remainingTimeoutBudget(isrWorkerStartedAt, workerTimeoutMs, 'dispatch'), 'dispatch', ); } catch (error) { @@ -1364,7 +1370,7 @@ const dispatcher = { pathname, response: workerResponse, userAgent: request.headers.get('user-agent'), - deadline: { startedAt: isrWorkerStartedAt, timeoutMs }, + deadline: isrWorkerDeadline, }); workerResponse = assetFallbackResult.response; workerResponseFailureKind = assetFallbackResult.failureKind; @@ -1383,7 +1389,7 @@ const dispatcher = { originalRequestUrl: request.url, originalUrlHeader, originalUrlVariant: isrOriginalUrlVariant, - deadline: { startedAt: isrWorkerStartedAt, timeoutMs }, + deadline: isrWorkerDeadline, }); } catch (error) { if (error instanceof DispatchTimeoutError) { @@ -1403,7 +1409,7 @@ const dispatcher = { pathname: fallbackResult.pathname, response: workerResponse, userAgent: request.headers.get('user-agent'), - deadline: { startedAt: isrWorkerStartedAt, timeoutMs }, + deadline: isrWorkerDeadline, }); workerResponse = assetFallbackResult.response; workerResponseFailureKind = assetFallbackResult.failureKind; @@ -1730,9 +1736,13 @@ const dispatcher = { const dispatchReqForFallback = hasBodyMethod && routing.fallbackRules?.length ? dispatchReq.clone() : dispatchReq; const workerDispatchStartedAt = Date.now(); + const workerDeadline = + env.DISPATCH_TIMEOUT_MS === undefined + ? undefined + : { startedAt: workerDispatchStartedAt, timeoutMs: workerTimeoutMs }; let response = await withTimeout( worker.fetch(dispatchReq), - remainingTimeoutBudget(workerDispatchStartedAt, timeoutMs, 'dispatch'), + remainingTimeoutBudget(workerDispatchStartedAt, workerTimeoutMs, 'dispatch'), 'dispatch', ); let isEventStream = isEventStreamResponse(response); @@ -1748,7 +1758,7 @@ const dispatcher = { pathname, response, userAgent: request.headers.get('user-agent'), - deadline: { startedAt: workerDispatchStartedAt, timeoutMs }, + deadline: workerDeadline, }); response = assetFallbackResult.response; responseFailureKind = assetFallbackResult.failureKind; @@ -1776,7 +1786,7 @@ const dispatcher = { originalRequestUrl: request.url, originalUrlHeader, originalUrlVariant, - deadline: { startedAt: workerDispatchStartedAt, timeoutMs }, + deadline: workerDeadline, }); if (fallbackResult.applied) { writebackFallbackApplied = true; @@ -1792,7 +1802,7 @@ const dispatcher = { pathname, response, userAgent: request.headers.get('user-agent'), - deadline: { startedAt: workerDispatchStartedAt, timeoutMs }, + deadline: workerDeadline, }); response = assetFallbackResult.response; responseFailureKind = assetFallbackResult.failureKind; diff --git a/platform/packages/dispatch/src/isr.ts b/platform/packages/dispatch/src/isr.ts index facc78360..ce6552b53 100644 --- a/platform/packages/dispatch/src/isr.ts +++ b/platform/packages/dispatch/src/isr.ts @@ -15,6 +15,7 @@ import { type IsrCacheMetadata, } from '@void/isr/core'; import { MAX_ISR_VARIANTS_PER_PATH } from '@void/isr/runtime'; +import { createIsrStore } from '@voidcloud/api/isr-store'; import type { Env } from './env.ts'; import { matchSegmentPattern } from './pattern.ts'; @@ -42,8 +43,8 @@ export { isrCacheKey } from '@void/isr'; export type { IsrCacheEntry, IsrCacheMetadata } from '@void/isr/core'; /** - * Read an ISR cache entry from KV. - * Returns null on cache miss, malformed metadata, or a KV read failure so the + * Read an ISR cache entry from the strongly consistent shared store. + * Returns null on cache miss, malformed metadata, or a storage read failure so the * caller renders the page live. */ export async function readIsrCache( @@ -56,10 +57,7 @@ export async function readIsrCache( ): Promise { const key = isrCacheKey(projectId, deploymentId, host, pathname, originalUrl); try { - const { value, metadata } = await env.ISR_CACHE.getWithMetadata( - key, - 'arrayBuffer', - ); + const { value, metadata } = await createIsrStore(env.STATIC_ASSETS_R2).getWithMetadata(key); return decodeIsrCacheEntry(value, metadata); } catch { return null; @@ -67,7 +65,7 @@ export async function readIsrCache( } /** - * Delete an ISR cache entry from KV. + * Delete an ISR cache entry from the shared store. * Used when a worker permanently opts out via x-revalidate: 0. */ export async function deleteIsrCache( @@ -79,11 +77,11 @@ export async function deleteIsrCache( originalUrl?: string | null, ): Promise { const key = isrCacheKey(projectId, deploymentId, host, pathname, originalUrl); - await env.ISR_CACHE.delete(key); + await createIsrStore(env.STATIC_ASSETS_R2, env.ISR_CACHE).delete(key); } /** - * Write an ISR cache entry to KV. Variant writes are gated by + * Write an ISR cache entry to R2. Variant writes are gated by * `MAX_ISR_VARIANTS_PER_PATH` — past the cap, only updates to existing * variant slots go through. Direct-hit entries (no `originalUrl`) are * never gated. @@ -117,7 +115,7 @@ export async function writeIsrCache( const body = await responseClone.arrayBuffer(); const packed = packIsrValue(headers, body); - await env.ISR_CACHE.put(key, packed, { + await createIsrStore(env.STATIC_ASSETS_R2).put(key, packed, { metadata: { s: status, c: Date.now(), @@ -132,10 +130,9 @@ export async function writeIsrCache( * per-pathname cap. Always allows updates to existing variants (so ISR * revalidation keeps working after a path saturates). * - * The KV list call is alphabetically ordered; we list up to `1000` entries - * (the KV per-call max) which comfortably covers any realistic cap. If a - * pathname has more variants than the listing returns, we conservatively - * refuse new writes — that case shouldn't occur with the cap enforced. + * R2 can return short or empty nonfinal pages. Count until the cap or end; + * once full, a point lookup permits updating an existing variant without + * scanning beyond the cap. */ async function isVariantWriteAllowed( env: Env, @@ -146,12 +143,21 @@ async function isVariantWriteAllowed( key: string, ): Promise { const prefix = isrCacheVariantPrefix(projectId, deploymentId, host, pathname); - const listed = await env.ISR_CACHE.list({ prefix, limit: 1000 }); - if (listed.keys.length < MAX_ISR_VARIANTS_PER_PATH) { - return true; - } - // At or over cap: only allow updates to existing keys. - return listed.keys.some((cacheKey) => cacheKey.name === key); + const cache = createIsrStore(env.STATIC_ASSETS_R2); + let count = 0; + let cursor: string | undefined; + do { + const listed = await cache.list({ prefix, cursor, limit: MAX_ISR_VARIANTS_PER_PATH }); + if (listed.keys.some((entry) => entry.name === key)) { + return true; + } + count += listed.keys.length; + if (count >= MAX_ISR_VARIANTS_PER_PATH) { + return cache.has(key); + } + cursor = listed.list_complete ? undefined : listed.cursor; + } while (cursor); + return true; } /** diff --git a/platform/packages/dispatch/test/unit/access-cache.test.ts b/platform/packages/dispatch/test/unit/access-cache.test.ts index 491eaced1..68c50cd61 100644 --- a/platform/packages/dispatch/test/unit/access-cache.test.ts +++ b/platform/packages/dispatch/test/unit/access-cache.test.ts @@ -51,14 +51,25 @@ function createHarness(mode: 'hashed' | 'isr') { vi.stubGlobal('caches', { default: edgeCache }); const isrEntries = new Map(); const isrCache = { - getWithMetadata: vi.fn( - async (key: string) => isrEntries.get(key) ?? { value: null, metadata: null }, - ), - put: vi.fn(async (key: string, value: ArrayBuffer, options?: { metadata?: unknown }) => { - isrEntries.set(key, { value, metadata: options?.metadata }); + get: vi.fn(async (key: string) => { + const entry = isrEntries.get(key.slice('isr-cache/'.length)); + return entry + ? { + arrayBuffer: async () => entry.value, + customMetadata: { isr: JSON.stringify(entry.metadata) }, + } + : null; }), + put: vi.fn( + async (key: string, value: ArrayBuffer, options: { customMetadata: { isr: string } }) => { + isrEntries.set(key.slice('isr-cache/'.length), { + value, + metadata: JSON.parse(options.customMetadata.isr), + }); + }, + ), delete: vi.fn(async () => {}), - list: vi.fn(async () => ({ keys: [], list_complete: true, cursor: '' })), + list: vi.fn(async () => ({ objects: [], truncated: false })), }; const origin = vi.fn(async (request: Request) => { const assertion = request.headers.get('Cf-Access-Jwt-Assertion'); @@ -80,7 +91,8 @@ function createHarness(mode: 'hashed' | 'isr') { const env = { ROUTING: { get: async () => JSON.stringify(routing) }, DISPATCH: { get: () => ({ fetch: origin }) }, - ISR_CACHE: isrCache, + STATIC_ASSETS_R2: isrCache, + ISR_CACHE: { delete: async () => {} }, ANALYTICS: { writeDataPoint() {} }, } as unknown as Env; const url = `https://profile.void.app${mode === 'hashed' ? '/assets/profile-a1.json' : '/profile'}`; @@ -150,7 +162,7 @@ describe.each(['hashed', 'isr'] as const)('Access assertion isolation for %s res expect(origin).toHaveBeenCalledOnce(); expect(edgeCache.match).toHaveBeenCalledTimes(2); expect(edgeCache.put).toHaveBeenCalledOnce(); - expect(isrCache.getWithMetadata).toHaveBeenCalledTimes(mode === 'isr' ? 1 : 0); + expect(isrCache.get).toHaveBeenCalledTimes(mode === 'isr' ? 1 : 0); expect(isrCache.put).toHaveBeenCalledTimes(mode === 'isr' ? 1 : 0); }); @@ -181,7 +193,7 @@ describe.each(['hashed', 'isr'] as const)('Access assertion isolation for %s res }); }); -it('replaces ISR KV responses admitted before Access assertions bypassed caching', async () => { +it('replaces ISR storage responses admitted before Access assertions bypassed caching', async () => { const { isrEntries, isrCache, origin, request } = createHarness('isr'); const key = isrCacheKey('project-profile', 'deployment', 'profile.void.app', '/profile'); isrEntries.set(key, { @@ -194,7 +206,7 @@ it('replaces ISR KV responses admitted before Access assertions bypassed caching expect(await (await request()).text()).toBe('profile: anonymous'); expect(origin).toHaveBeenCalledOnce(); - expect(isrCache.getWithMetadata).toHaveBeenCalledOnce(); + expect(isrCache.get).toHaveBeenCalledOnce(); expect(isrCache.put).toHaveBeenCalledOnce(); expect(isrEntries.get(key)!.metadata).toMatchObject({ p: SHARED_CACHE_POLICY_VERSION }); }); diff --git a/platform/packages/dispatch/test/unit/index.test.ts b/platform/packages/dispatch/test/unit/index.test.ts index aa162a4be..077acf34a 100644 --- a/platform/packages/dispatch/test/unit/index.test.ts +++ b/platform/packages/dispatch/test/unit/index.test.ts @@ -163,39 +163,41 @@ function createEnv(options: { DISPATCH: { get: dispatchGet, }, - ISR_CACHE: { - getWithMetadata: async (key: string) => { - const entry = kvStore.get(key); + STATIC_ASSETS_R2: { + get: async (key: string) => { + const entry = kvStore.get(key.slice('isr-cache/'.length)); if (!entry) { - return { value: null, metadata: null }; + return null; } - return { value: entry.value, metadata: entry.metadata }; + return { + arrayBuffer: async () => entry.value, + customMetadata: { isr: JSON.stringify(entry.metadata) }, + }; }, - put: async (key: string, value: ArrayBuffer, opts?: { metadata?: unknown }) => { - kvStore.set(key, { + put: async (key: string, value: ArrayBuffer, opts: { customMetadata: { isr: string } }) => { + kvStore.set(key.slice('isr-cache/'.length), { value, - metadata: opts?.metadata as { - s: number; - c: number; - t: number; - p: typeof SHARED_CACHE_POLICY_VERSION; - }, + metadata: JSON.parse(opts.customMetadata.isr), }); }, delete: async (key: string) => { - kvStore.delete(key); + kvStore.delete(key.slice('isr-cache/'.length)); }, list: async (opts: { prefix: string; limit?: number }) => { - const all = [...kvStore.keys()].filter((k) => k.startsWith(opts.prefix)).sort(); + const all = [...kvStore.keys()] + .map((key) => `isr-cache/${key}`) + .filter((key) => key.startsWith(opts.prefix)) + .sort(); const limit = opts.limit ?? 1000; const slice = all.slice(0, limit); return { - keys: slice.map((name) => ({ name })), - list_complete: slice.length === all.length, + objects: slice.map((key) => ({ key })), + truncated: slice.length !== all.length, cursor: '', }; }, }, + ISR_CACHE: { delete: async () => {} }, ANALYTICS: { writeDataPoint: (data: unknown) => analyticsWrites.push(data), }, @@ -470,6 +472,26 @@ describe('custom domain routing', () => { }); describe('ISR (stale-while-revalidate)', () => { + it('does not resurrect a purged 600-second page from stale legacy KV', async () => { + const { env, workerFetch } = createEnv({ + routing: { workerName: 'user-app', projectId: 'proj_1', ssr: true, revalidate: 600 }, + workerResponse: new Response('fresh after purge'), + }); + const staleRead = vi.fn(async () => + packMockEntry({ + body: 'deleted but still visible in KV', + createdAt: Date.now(), + ttl: 600, + }), + ); + Object.assign(env.ISR_CACHE, { getWithMetadata: staleRead }); + const response = await app.fetch(new Request('https://my-app.void.app/page'), env, createCtx()); + expect(await response.text()).toBe('fresh after purge'); + expect(response.headers.get('x-isr-cache')).toBe('MISS'); + expect(workerFetch).toHaveBeenCalledOnce(); + expect(staleRead).not.toHaveBeenCalled(); + }); + it('keeps framework data requests separate from a cached document at the same URL', async () => { let renders = 0; const { env, workerFetch } = createEnv({ @@ -874,7 +896,7 @@ describe('ISR (stale-while-revalidate)', () => { headers: { 'Content-Type': 'text/html' }, }), }); - vi.spyOn(env.ISR_CACHE, 'getWithMetadata').mockRejectedValueOnce(new Error('KV unavailable')); + vi.spyOn(env.STATIC_ASSETS_R2, 'get').mockRejectedValueOnce(new Error('R2 unavailable')); const response = await app.fetch( new Request('https://my-app.void.app/dashboard'), @@ -6567,6 +6589,135 @@ describe('static asset content-type normalization', () => { }); }); +describe('foreground worker duration', () => { + beforeEach(() => vi.useFakeTimers()); + afterEach(() => vi.useRealTimers()); + + it.each([ + ['API mutation', '/api/email', 'POST', undefined], + ['SSR page', '/slow', 'GET', undefined], + ['ISR cache miss', '/slow', 'GET', 60], + ] as const)('waits for a slow %s to finish by default', async (_, path, method, revalidate) => { + const completed = vi.fn(); + const { env, logWrites, analyticsWrites } = createEnv({ + routing: { + workerName: 'user-app-dpl_1', + projectId: 'proj_1', + type: 'void', + ssr: true, + revalidate, + }, + workerFetchImpl: async () => { + await new Promise((resolve) => setTimeout(resolve, 11_000)); + completed(); + return new Response('completed', { headers: { 'Content-Type': 'text/html' } }); + }, + }); + const responsePromise = app.fetch( + new Request(`https://my-app.void.app${path}`, { method }), + env, + createCtx(), + ); + await vi.advanceTimersByTimeAsync(11_000); + + const response = await responsePromise; + expect(response.status).toBe(200); + expect(await response.text()).toBe('completed'); + expect(completed).toHaveBeenCalledOnce(); + expect(logWrites).toHaveLength(0); + expect(analyticsWrites).toHaveLength(1); + }); + + it.each([undefined, 60])( + 'waits for a slow worker fallback with ISR TTL %s', + async (revalidate) => { + let calls = 0; + const { env, workerFetch, logWrites } = createEnv({ + routing: { + workerName: 'user-app-dpl_1', + projectId: 'proj_1', + type: 'void', + ssr: true, + revalidate, + fallbackRules: [{ source: '/missing', destination: '/slow', status: 200 }], + }, + workerFetchImpl: async () => { + if (++calls === 1) { + return new Response('missing', { status: 404, headers: { 'X-Void-No-Match': '1' } }); + } + await new Promise((resolve) => setTimeout(resolve, 11_000)); + return new Response('fallback', { headers: { 'Content-Type': 'text/html' } }); + }, + }); + const responsePromise = app.fetch( + new Request('https://my-app.void.app/missing'), + env, + createCtx(), + ); + await vi.advanceTimersByTimeAsync(11_000); + + const response = await responsePromise; + expect(response.status).toBe(200); + expect(await response.text()).toBe('fallback'); + expect(workerFetch).toHaveBeenCalledTimes(2); + expect(logWrites).toHaveLength(0); + }, + ); + + it('keeps a separate bounded storage budget after a slow worker returns an asset stub', async () => { + const error = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const stubBody = 'Assets have not yet been deployed to this script.\n'; + const { env, logWrites } = createEnv({ + routing: { + workerName: 'user-app-dpl_1', + projectId: 'proj_1', + type: 'void', + ssr: true, + assetManifestId: 'deploy-1', + assetConfig: { run_worker_first: true }, + }, + assetManifest: { '/bundle.js': '!bundle' }, + workerFetchImpl: async () => { + await new Promise((resolve) => setTimeout(resolve, 11_000)); + return new Response(stubBody, { + headers: { + 'Content-Type': 'text/plain;charset=UTF-8', + 'Content-Length': String(stubBody.length), + }, + }); + }, + }); + const get = vi.fn(() => new Promise(() => {})); + Object.assign(env, { STATIC_ASSETS_R2: { get } }); + let settled = false; + const responsePromise = app + .fetch( + new Request('https://my-app.void.app/bundle.js', { headers: { Cookie: 'a=1' } }), + env, + createCtx(), + ) + .then((response) => { + settled = true; + return response; + }); + await vi.advanceTimersByTimeAsync(11_000); + expect(get).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(9_999); + expect(settled).toBe(false); + await vi.advanceTimersByTimeAsync(1); + + const response = await responsePromise; + expect(response.status).toBe(503); + expect(response.headers.get('X-Void-Timeout')).toBe('static-asset'); + expect(get).toHaveBeenCalledTimes(2); + expect(logWrites).toHaveLength(0); + } finally { + error.mockRestore(); + } + }); +}); + describe('worker timeout explains itself in the project log stream', () => { // A worker cut at the dispatch budget returns nothing, so it logs nothing and // `void project logs` was empty for the one failure users most need diff --git a/platform/packages/dispatch/test/unit/isr.test.ts b/platform/packages/dispatch/test/unit/isr.test.ts index fad4334d5..f8c7dc5fd 100644 --- a/platform/packages/dispatch/test/unit/isr.test.ts +++ b/platform/packages/dispatch/test/unit/isr.test.ts @@ -705,36 +705,42 @@ describe('writeIsrCache — variant cap', () => { } { const kvStore = seed; const env = { - ISR_CACHE: { - getWithMetadata: async (key: string) => { - const value = kvStore.get(key); + STATIC_ASSETS_R2: { + head: async (key: string) => (kvStore.has(key.slice('isr-cache/'.length)) ? { key } : null), + get: async (key: string) => { + const value = kvStore.get(key.slice('isr-cache/'.length)); return value ? { - value, - metadata: { - s: 200, - c: Date.now(), - t: 60, - p: SHARED_CACHE_POLICY_VERSION, + arrayBuffer: async () => value, + customMetadata: { + isr: JSON.stringify({ + s: 200, + c: Date.now(), + t: 60, + p: SHARED_CACHE_POLICY_VERSION, + }), }, } - : { value: null, metadata: null }; + : null; }, put: async (key: string, value: ArrayBuffer) => { - kvStore.set(key, value); + kvStore.set(key.slice('isr-cache/'.length), value); }, delete: async (key: string) => { - kvStore.delete(key); + kvStore.delete(key.slice('isr-cache/'.length)); }, list: async (opts: { prefix: string; limit?: number; cursor?: string }) => { - const all = [...kvStore.keys()].filter((k) => k.startsWith(opts.prefix)).sort(); + const all = [...kvStore.keys()] + .map((key) => `isr-cache/${key}`) + .filter((k) => k.startsWith(opts.prefix)) + .sort(); const limit = opts.limit ?? 1000; const start = opts.cursor ? parseInt(opts.cursor, 10) : 0; const slice = all.slice(start, start + limit); const list_complete = start + limit >= all.length; return { - keys: slice.map((name) => ({ name })), - list_complete, + objects: slice.map((key) => ({ key })), + truncated: !list_complete, cursor: list_complete ? '' : String(start + limit), }; }, @@ -793,6 +799,22 @@ describe('writeIsrCache — variant cap', () => { ); }); + it('does not treat an empty nonfinal R2 page as free variant capacity', async () => { + const { env, kvStore } = createKvMock(); + seedVariants(kvStore, MAX_ISR_VARIANTS_PER_PATH); + const list = vi.spyOn(env.STATIC_ASSETS_R2, 'list').mockResolvedValueOnce({ + objects: [], + truncated: true, + cursor: '0', + delimitedPrefixes: [], + }); + const url = 'https://h/search?q=OVERFLOW'; + await writeIsrCache(env, 'p', 'd', 'h', '/search', new Response('fresh'), 600, url); + expect(list).toHaveBeenCalledTimes(2); + expect(kvStore.has(isrCacheKey('p', 'd', 'h', '/search', url))).toBe(false); + expect(kvStore.size).toBe(MAX_ISR_VARIANTS_PER_PATH); + }); + it('still updates an existing variant past the cap (revalidation must keep working)', async () => { const { env, kvStore } = createKvMock(); seedVariants(kvStore, MAX_ISR_VARIANTS_PER_PATH); diff --git a/platform/packages/dispatch/test/unit/wfp-asset-fallback.test.ts b/platform/packages/dispatch/test/unit/wfp-asset-fallback.test.ts index 1c204bed7..4a851b8f4 100644 --- a/platform/packages/dispatch/test/unit/wfp-asset-fallback.test.ts +++ b/platform/packages/dispatch/test/unit/wfp-asset-fallback.test.ts @@ -155,12 +155,18 @@ function createEnv(h: Harness): { return { value: asset.content, metadata: asset.ct ? { ct: asset.ct } : null }; }, }, - STATIC_ASSETS_R2: { get: async () => null }, - ISR_CACHE: { - getWithMetadata: async () => h.isrCacheEntry ?? { value: null, metadata: null }, + STATIC_ASSETS_R2: { + get: async (key: string) => + key.startsWith('isr-cache/') && h.isrCacheEntry + ? { + arrayBuffer: async () => h.isrCacheEntry!.value, + customMetadata: { isr: JSON.stringify(h.isrCacheEntry.metadata) }, + } + : null, put: async () => {}, delete: isrDelete, }, + ISR_CACHE: { delete: async () => {} }, ANALYTICS: { writeDataPoint: analyticsWrite }, USAGE_QUEUE: { send: usageSend }, DISPATCH_TIMEOUT_MS: h.dispatchTimeoutMs, diff --git a/vite.config.ts b/vite.config.ts index df369179f..6f2325ead 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -35,15 +35,15 @@ const sdkPgliteUnitTests = [ ]; // Vitest forwards a FILTERED env to test workers — in CI only VOID_E2E_CF_TOKEN survives, while -// VOID_E2E_CF_ACCOUNT_ID and VOID_E2E_RUN_ID are dropped. The gated live Cloudflare deploy e2e +// VOID_E2E_CF_ACCOUNT_ID and VOID_E2E_NAMESPACE are dropped. The gated live Cloudflare deploy e2e // (packages/void/test/e2e/cloudflare-deploy-live.test.ts) needs all three, so forward them into the // core-markdown workers explicitly from this main config process (which has the runner's step env). -// Absent/unset → empty strings → the suite skips itself as designed. +// Absent credentials become empty strings so ordinary test runs skip the live suite. const e2eWorkerEnv = { VOID_TEST_BROWSER: process.env.VOID_TEST_BROWSER ?? 'chromium', VOID_E2E_CF_TOKEN: process.env.VOID_E2E_CF_TOKEN ?? '', VOID_E2E_CF_ACCOUNT_ID: process.env.VOID_E2E_CF_ACCOUNT_ID ?? '', - VOID_E2E_RUN_ID: process.env.VOID_E2E_RUN_ID ?? '', + VOID_E2E_NAMESPACE: process.env.VOID_E2E_NAMESPACE ?? '', }; // Windows CI runners are 2-3x slower for builds and server startup. @@ -90,7 +90,7 @@ export default defineConfig({ // skips even when the CI job injects them. `untrackedEnv` forwards them without folding them // into the task's cache fingerprint (credentials must not bust the cache). cache: { - untrackedEnv: ['VOID_E2E_CF_TOKEN', 'VOID_E2E_CF_ACCOUNT_ID', 'VOID_E2E_RUN_ID'], + untrackedEnv: ['VOID_E2E_CF_TOKEN', 'VOID_E2E_CF_ACCOUNT_ID', 'VOID_E2E_NAMESPACE'], }, }, },