From 2b799be989b8d72d77bb0b414ad55c33e9f1b75e Mon Sep 17 00:00:00 2001 From: Lance Willett Date: Wed, 2 Sep 2026 17:57:02 -0700 Subject: [PATCH] Build/Test Tools: Retry and verify the Gutenberg download on the 7.0 branch. [62873] merged [62859] and [62862] to this branch, but carried only the workflow half: it touched a single file, .github/workflows/phpunit-tests.yml. The downloader rewrite from [62859] never landed here. So tools/gutenberg/download.js still streams the archive from the GitHub Container Registry as fetch body, gunzip, then tar stdin in one pipeline, with no temporary file, no size or SHA-256 check, no timeout, and no retries. An early stream close raises Node's ERR_STREAM_PREMATURE_CLOSE and the build fails with "Download/extraction failed: Premature close" before a test runs. tools/gutenberg/utils.js has the same gap in its token and manifest requests. The PHPUnit workflow on this branch is insulated, because [62873] wired it to consume the shared prepare-gutenberg run artifact and it never calls the downloader. Anything else that checks out this branch and runs `npm run build:dev` still hits the raw downloader, once per job. Merge the downloader half of [62859]: stage the blob to a temporary file, verify the manifest size and SHA-256, extract only after verification, and retry interrupted transfers up to three times. Metadata requests get the same bounded retries and a timeout. This also makes the branch honour GUTENBERG_EXPECTED_SHA, which reusable-phpunit-tests-v3.yml already passes it. tools/gutenberg/copy.js also differs between the branches, but that drift is from unrelated work and is left alone. Fixes #66028. --- tools/gutenberg/download.js | 362 ++++++++++++++++++++++++++++-------- tools/gutenberg/utils.js | 153 +++++++++++---- 2 files changed, 406 insertions(+), 109 deletions(-) diff --git a/tools/gutenberg/download.js b/tools/gutenberg/download.js index fd76c6c7a7836..97df476161412 100644 --- a/tools/gutenberg/download.js +++ b/tools/gutenberg/download.js @@ -1,11 +1,14 @@ #!/usr/bin/env node +/* global AbortSignal */ /** * Download Gutenberg Repository Script. * * This script downloads a pre-built Gutenberg tar.gz artifact from the GitHub - * Container Registry and extracts it into the ./gutenberg directory. Any - * existing gutenberg directory is removed before extraction. + * Container Registry and extracts it into the ./gutenberg directory. The + * archive is downloaded and verified (SHA-256 and manifest size) before + * extraction; the existing gutenberg directory is then removed and replaced + * with the extracted contents. * * The artifact is identified by the "gutenberg.sha" value in the root * package.json, which is used as the OCI tag for the gutenberg-wp-develop-build @@ -20,10 +23,13 @@ */ const { spawn } = require( 'child_process' ); +const crypto = require( 'crypto' ); const fs = require( 'fs' ); +const os = require( 'os' ); +const path = require( 'path' ); const { Readable } = require( 'stream' ); const { pipeline } = require( 'stream/promises' ); -const zlib = require( 'zlib' ); +const { Transform } = require( 'stream' ); const { gutenbergDir, readGutenbergConfig, @@ -31,6 +37,257 @@ const { fetchManifest, } = require( './utils' ); +const MAX_DOWNLOAD_ATTEMPTS = 3; +const RETRY_DELAY_MS = 2000; +const DOWNLOAD_TIMEOUT_MS = 120000; + +/** + * Convert bytes into a readable string for download diagnostics. + * + * @param {number} bytes Number of bytes. + * @return {string} Formatted byte count. + */ +function formatBytes( bytes ) { + return `${ ( bytes / 1024 / 1024 ).toFixed( 2 ) } MiB (${ bytes } bytes)`; +} + +/** + * Wait before retrying a failed download. + * + * @param {number} milliseconds Time to wait in milliseconds. + * @return {Promise} Resolves after the requested delay. + */ +function delay( milliseconds ) { + return new Promise( ( resolve ) => setTimeout( resolve, milliseconds ) ); +} + +/** + * Create an error that retains the HTTP status code for retry decisions. + * + * @param {string} message Error message. + * @param {number} status HTTP status code. + * @return {Error & { status: number }} Error with status code. + */ +function createHttpError( message, status ) { + const error = /** @type {Error & { status: number }} */ ( new Error( message ) ); + error.status = status; + return error; +} + +/** + * Determine whether a failed download might succeed on a later attempt. + * + * @param {Error & { status?: number }} error Download error. + * @return {boolean} Whether the error is retryable. + */ +function isRetryableDownloadError( error ) { + return ! error.status || error.status === 408 || error.status === 429 || error.status >= 500; +} + +/** + * Extract a SHA-256 hash from an OCI layer digest. + * + * @param {string} digest OCI layer digest. + * @return {string} Expected SHA-256 hash. + * @throws {Error} If the digest is not a SHA-256 digest. + */ +function getExpectedSha256( digest ) { + const match = /^sha256:([a-f0-9]{64})$/i.exec( digest ); + if ( ! match ) { + throw new Error( `Unsupported OCI layer digest: ${ digest }` ); + } + + return match[ 1 ].toLowerCase(); +} + +/** + * Download a blob to disk and verify its SHA-256 digest and byte count. + * + * @param {string} url Blob URL. + * @param {string} token Bearer token for GHCR. + * @param {string} digest OCI layer digest. + * @param {number|undefined} expectedSize Expected layer size from the manifest. + * @param {string} destination Path where the compressed blob is written. + * @return {Promise} Resolves after the download is verified. + */ +async function downloadAndVerifyBlob( url, token, digest, expectedSize, destination ) { + const expectedSha256 = getExpectedSha256( digest ); + const response = await fetch( url, { + headers: { + Authorization: `Bearer ${ token }`, + }, + signal: AbortSignal.timeout( DOWNLOAD_TIMEOUT_MS ), + } ); + + console.log( + ` Response: ${ response.status } ${ response.statusText } from ${ new URL( response.url ).hostname }` + ); + + if ( ! response.ok ) { + throw createHttpError( + `Failed to download blob: ${ response.status } ${ response.statusText }`, + response.status + ); + } + + if ( ! response.body ) { + throw new Error( 'Blob response has no body' ); + } + + const contentLength = Number( response.headers.get( 'content-length' ) ); + if ( Number.isFinite( contentLength ) && contentLength > 0 ) { + console.log( ` Content-Length: ${ formatBytes( contentLength ) }` ); + } + if ( expectedSize ) { + console.log( ` Manifest size: ${ formatBytes( expectedSize ) }` ); + } + + let downloadedBytes = 0; + const hash = crypto.createHash( 'sha256' ); + const meter = new Transform( { + transform( chunk, _encoding, callback ) { + downloadedBytes += chunk.length; + hash.update( chunk ); + callback( null, chunk ); + }, + } ); + + try { + await pipeline( + Readable.fromWeb( + /** @type {import('stream/web').ReadableStream} */ ( response.body ) + ), + meter, + fs.createWriteStream( destination ) + ); + } catch ( error ) { + throw new Error( + `Download interrupted after ${ formatBytes( downloadedBytes ) }: ${ /** @type {Error} */ ( error ).message }` + ); + } + + if ( expectedSize && downloadedBytes !== expectedSize ) { + throw new Error( + `Downloaded ${ formatBytes( downloadedBytes ) }, but manifest size was ${ formatBytes( expectedSize ) }` + ); + } + + const actualSha256 = hash.digest( 'hex' ); + if ( actualSha256 !== expectedSha256 ) { + throw new Error( + `SHA-256 mismatch: expected ${ expectedSha256 } but received ${ actualSha256 }` + ); + } + + console.log( `āœ… Downloaded ${ formatBytes( downloadedBytes ) } and verified SHA-256` ); +} + +/** + * Download a blob with bounded retries and remove incomplete files between attempts. + * + * The GHCR bearer token can age out during a long retry window. If a 401 is + * encountered, a fresh token is fetched once (via `fetchGhcrToken`) and the + * download is retried with it. + * + * @param {string} url Blob URL. + * @param {string} token Bearer token for GHCR. + * @param {string} digest OCI layer digest. + * @param {number|undefined} expectedSize Expected layer size from the manifest. + * @param {string} ghcrRepo The "owner/repo/package" path on ghcr.io, used to refresh an expired token. + * @return {Promise} Path to the verified compressed blob. + */ +async function downloadBlobWithRetries( url, token, digest, expectedSize, ghcrRepo ) { + const destination = path.join( + os.tmpdir(), + `wordpress-gutenberg-${ process.pid }.tar.gz` + ); + + let currentToken = token; + let hasRefetchedToken = false; + + for ( let attempt = 1; attempt <= MAX_DOWNLOAD_ATTEMPTS; attempt++ ) { + console.log( `\nšŸ“„ Download attempt ${ attempt }/${ MAX_DOWNLOAD_ATTEMPTS }...` ); + fs.rmSync( destination, { force: true } ); + + try { + await downloadAndVerifyBlob( + url, + currentToken, + digest, + expectedSize, + destination + ); + return destination; + } catch ( error ) { + const downloadError = /** @type {Error & { status?: number }} */ ( error ); + fs.rmSync( destination, { force: true } ); + console.error( `āŒ Download attempt ${ attempt } failed: ${ downloadError.message }` ); + + if ( + downloadError.status === 401 && + ! hasRefetchedToken && + attempt < MAX_DOWNLOAD_ATTEMPTS + ) { + hasRefetchedToken = true; + console.log( ' Bearer token may have expired mid-retry; fetching a fresh token...' ); + currentToken = await fetchGhcrToken( ghcrRepo ); + + console.log( ` Retrying in ${ RETRY_DELAY_MS / 1000 } seconds...` ); + await delay( RETRY_DELAY_MS ); + continue; + } + + if ( + attempt === MAX_DOWNLOAD_ATTEMPTS || + ! isRetryableDownloadError( downloadError ) + ) { + throw downloadError; + } + + console.log( ` Retrying in ${ RETRY_DELAY_MS / 1000 } seconds...` ); + await delay( RETRY_DELAY_MS ); + } + } + + throw new Error( 'Download failed without an error' ); +} + +/** + * Extract a verified archive directly into the Gutenberg directory, removing + * any existing directory first. + * + * @param {string} archivePath Path to the verified compressed blob. + * @param {string} expectedSha Expected immutable Gutenberg source SHA. + * @return {Promise} Resolves after extraction completes. + */ +async function extractVerifiedArchive( archivePath, expectedSha ) { + fs.rmSync( gutenbergDir, { recursive: true, force: true } ); + fs.mkdirSync( gutenbergDir, { recursive: true } ); + + const tar = spawn( 'tar', [ '-xzf', archivePath, '-C', gutenbergDir ], { + stdio: [ 'ignore', 'inherit', 'inherit' ], + } ); + + await new Promise( ( resolve, reject ) => { + tar.on( 'close', ( code ) => { + if ( code !== 0 ) { + reject( new Error( `tar exited with code ${ code }` ) ); + return; + } + resolve( undefined ); + } ); + tar.on( 'error', reject ); + } ); + + const extractedHashPath = path.join( gutenbergDir, '.gutenberg-hash' ); + const extractedSha = fs.readFileSync( extractedHashPath, 'utf8' ).trim(); + if ( extractedSha !== expectedSha ) { + throw new Error( + `Extracted Gutenberg SHA mismatch: expected ${ expectedSha } but found ${ extractedSha }` + ); + } +} + /** * Resolve the manifest to use for downloading. * @@ -43,7 +300,7 @@ const { * * @param {{ ref: string, ghcrRepo: string, isMutable: boolean }} config * @param {string} token - * @return {Promise<{ manifest: Record, resolvedRef: string }>} + * @return {Promise<{ manifest: Record, resolvedRef: string, expectedSha: string }>} */ async function resolveDownloadManifest( config, token ) { const { ref, ghcrRepo, isMutable } = config; @@ -51,27 +308,26 @@ async function resolveDownloadManifest( config, token ) { const initialManifest = await fetchManifest( ref, ghcrRepo, token ); if ( ! isMutable ) { - return { manifest: initialManifest, resolvedRef: ref }; + return { manifest: initialManifest, resolvedRef: ref, expectedSha: ref }; } const revision = initialManifest?.annotations?.[ 'org.opencontainers.image.revision' ]; - if ( ! revision ) { - console.log( - `ā„¹ļø No image.revision annotation on "${ ref }"; using mutable tag for download.` + if ( ! revision || ! /^[a-f0-9]{40}$/i.test( revision ) ) { + throw new Error( + `Manifest for mutable ref "${ ref }" has no valid org.opencontainers.image.revision SHA` ); - return { manifest: initialManifest, resolvedRef: ref }; } try { const immutableManifest = await fetchManifest( revision, ghcrRepo, token ); - return { manifest: immutableManifest, resolvedRef: revision }; + return { manifest: immutableManifest, resolvedRef: revision, expectedSha: revision }; } catch ( error ) { if ( /** @type {{ status?: number }} */ ( error ).status === 404 ) { console.log( `ā„¹ļø Immutable SHA tag ${ revision } unavailable; falling back to mutable tag "${ ref }".` ); - return { manifest: initialManifest, resolvedRef: ref }; + return { manifest: initialManifest, resolvedRef: ref, expectedSha: revision }; } throw error; } @@ -116,9 +372,9 @@ async function main() { // Step 2: Resolve the manifest to use for download. console.log( `\nšŸ“‹ Fetching manifest for ${ config.ref }...` ); - let manifest, resolvedRef; + let manifest, resolvedRef, expectedSha; try { - ( { manifest, resolvedRef } = await resolveDownloadManifest( + ( { manifest, resolvedRef, expectedSha } = await resolveDownloadManifest( config, token ) ); @@ -130,78 +386,36 @@ async function main() { process.exit( 1 ); } - const digest = manifest?.layers?.[ 0 ]?.digest; + const layer = manifest?.layers?.[ 0 ]; + const digest = layer?.digest; if ( ! digest ) { console.error( 'āŒ No layer digest found in manifest' ); process.exit( 1 ); } console.log( `āœ… Blob digest: ${ digest }` ); - // Remove existing gutenberg directory so the extraction is clean. - if ( fs.existsSync( gutenbergDir ) ) { - console.log( '\nšŸ—‘ļø Removing existing gutenberg directory...' ); - fs.rmSync( gutenbergDir, { recursive: true, force: true } ); - } - - fs.mkdirSync( gutenbergDir, { recursive: true } ); - - /* - * Step 3: Stream the blob directly through gunzip into tar, writing - * into ./gutenberg with no temporary file on disk. - */ - console.log( `\nšŸ“„ Downloading and extracting artifact...` ); + // Step 3: Download and verify the compressed blob before extraction. + let archivePath; try { - const response = await fetch( `https://ghcr.io/v2/${ config.ghcrRepo }/blobs/${ digest }`, { - headers: { - Authorization: `Bearer ${ token }`, - }, - } ); - if ( ! response.ok ) { - throw new Error( `Failed to download blob: ${ response.status } ${ response.statusText }` ); - } - if ( ! response.body ) { - throw new Error( 'Blob response has no body' ); - } - - /* - * Spawn tar to read from stdin and extract into gutenbergDir. - * `tar` is available on macOS, Linux, and Windows 10+. - */ - const tar = spawn( 'tar', [ '-x', '-C', gutenbergDir ], { - stdio: [ 'pipe', 'inherit', 'inherit' ], - } ); - - /** @type {Promise} */ - const tarDone = new Promise( ( resolve, reject ) => { - tar.on( 'close', ( code ) => { - if ( code !== 0 ) { - reject( new Error( `tar exited with code ${ code }` ) ); - } else { - resolve(); - } - } ); - tar.on( 'error', reject ); - } ); - - /* - * Pipe: fetch body → gunzip → tar stdin. - * Decompressing in Node keeps the pipeline error handling - * consistent and means tar only sees plain tar data on stdin. - */ - await pipeline( - Readable.fromWeb( - /** @type {import('stream/web').ReadableStream} */ ( response.body ) - ), - zlib.createGunzip(), - tar.stdin, + archivePath = await downloadBlobWithRetries( + `https://ghcr.io/v2/${ config.ghcrRepo }/blobs/${ digest }`, + token, + digest, + layer.size, + config.ghcrRepo ); - await tarDone; - - console.log( 'āœ… Download and extraction complete' ); + console.log( '\nšŸ“¦ Extracting verified artifact...' ); + await extractVerifiedArchive( archivePath, expectedSha ); + console.log( 'āœ… Extraction complete' ); } catch ( error ) { console.error( 'āŒ Download/extraction failed:', /** @type {Error} */ ( error ).message ); - process.exit( 1 ); + process.exitCode = 1; + return; + } finally { + if ( archivePath ) { + fs.rmSync( archivePath, { force: true } ); + } } console.log( '\nāœ… Gutenberg download complete!' ); diff --git a/tools/gutenberg/utils.js b/tools/gutenberg/utils.js index 3ba95199578b4..b43e760735efe 100644 --- a/tools/gutenberg/utils.js +++ b/tools/gutenberg/utils.js @@ -1,4 +1,5 @@ #!/usr/bin/env node +/* global AbortSignal */ /** * Gutenberg build utilities. @@ -26,6 +27,73 @@ const SHA_PATTERN = /^[a-f0-9]{40}$/i; const MANIFEST_ACCEPT = 'application/vnd.oci.image.manifest.v1+json'; +// Retry/timeout settings for the token and manifest requests. These run +// before the blob download and are now a single point of failure for the +// whole build, so they share the blob download's attempt count and backoff +// (see MAX_DOWNLOAD_ATTEMPTS and RETRY_DELAY_MS in download.js). +const MAX_METADATA_ATTEMPTS = 3; +const RETRY_DELAY_MS = 2000; +const METADATA_TIMEOUT_MS = 30000; + +/** + * Wait before retrying a failed metadata request. + * + * @param {number} milliseconds Time to wait in milliseconds. + * @return {Promise} Resolves after the requested delay. + */ +function delay( milliseconds ) { + return new Promise( ( resolve ) => setTimeout( resolve, milliseconds ) ); +} + +/** + * Create an error that retains the HTTP status code for retry decisions. + * + * @param {string} message Error message. + * @param {number} status HTTP status code. + * @return {Error & { status: number }} Error with status code. + */ +function createHttpError( message, status ) { + const error = /** @type {Error & { status: number }} */ ( new Error( message ) ); + error.status = status; + return error; +} + +/** + * Determine whether a failed metadata request might succeed on a later attempt. + * + * @param {Error & { status?: number }} error Request error. + * @return {boolean} Whether the error is retryable. + */ +function isRetryableMetadataError( error ) { + return ! error.status || error.status === 408 || error.status === 429 || error.status >= 500; +} + +/** + * Run a metadata request with bounded retries, matching the blob download's + * retry semantics. Non-retryable errors (e.g. a 404) are thrown immediately. + * + * @param {string} description Human-readable label for retry log messages. + * @param {() => Promise} request Function that performs one request attempt. + * @return {Promise} Resolves with the request's result. + */ +async function withMetadataRetries( description, request ) { + for ( let attempt = 1; attempt <= MAX_METADATA_ATTEMPTS; attempt++ ) { + try { + return await request(); + } catch ( error ) { + const requestError = /** @type {Error & { status?: number }} */ ( error ); + if ( attempt === MAX_METADATA_ATTEMPTS || ! isRetryableMetadataError( requestError ) ) { + throw requestError; + } + console.error( `āŒ ${ description } attempt ${ attempt } failed: ${ requestError.message }` ); + console.log( ` Retrying in ${ RETRY_DELAY_MS / 1000 } seconds...` ); + await delay( RETRY_DELAY_MS ); + } + } + + throw new Error( `${ description } failed without an error` ); +} + /** * Read Gutenberg configuration from package.json. * @@ -64,19 +132,23 @@ function readGutenbergConfig() { * @return {Promise} The bearer token. */ async function fetchGhcrToken( ghcrRepo ) { - const response = await fetch( - `https://ghcr.io/token?scope=repository:${ ghcrRepo }:pull&service=ghcr.io` - ); - if ( ! response.ok ) { - throw new Error( - `Failed to fetch GHCR token: ${ response.status } ${ response.statusText }` + return withMetadataRetries( 'Fetch GHCR token', async () => { + const response = await fetch( + `https://ghcr.io/token?scope=repository:${ ghcrRepo }:pull&service=ghcr.io`, + { signal: AbortSignal.timeout( METADATA_TIMEOUT_MS ) } ); - } - const data = await response.json(); - if ( ! data.token ) { - throw new Error( 'No token in GHCR response' ); - } - return data.token; + if ( ! response.ok ) { + throw createHttpError( + `Failed to fetch GHCR token: ${ response.status } ${ response.statusText }`, + response.status + ); + } + const data = await response.json(); + if ( ! data.token ) { + throw new Error( 'No token in GHCR response' ); + } + return data.token; + } ); } /** @@ -88,25 +160,25 @@ async function fetchGhcrToken( ghcrRepo ) { * @return {Promise>} Parsed manifest JSON. */ async function fetchManifest( ref, ghcrRepo, token ) { - const response = await fetch( - `https://ghcr.io/v2/${ ghcrRepo }/manifests/${ ref }`, - { - headers: { - Authorization: `Bearer ${ token }`, - Accept: MANIFEST_ACCEPT, - }, - } - ); - if ( ! response.ok ) { - const error = /** @type {Error & { status?: number }} */ ( - new Error( - `Failed to fetch manifest for "${ ref }": ${ response.status } ${ response.statusText }` - ) + return withMetadataRetries( `Fetch manifest for "${ ref }"`, async () => { + const response = await fetch( + `https://ghcr.io/v2/${ ghcrRepo }/manifests/${ ref }`, + { + headers: { + Authorization: `Bearer ${ token }`, + Accept: MANIFEST_ACCEPT, + }, + signal: AbortSignal.timeout( METADATA_TIMEOUT_MS ), + } ); - error.status = response.status; - throw error; - } - return response.json(); + if ( ! response.ok ) { + throw createHttpError( + `Failed to fetch manifest for "${ ref }": ${ response.status } ${ response.statusText }`, + response.status + ); + } + return response.json(); + } ); } /** @@ -121,6 +193,17 @@ async function fetchManifest( ref, ghcrRepo, token ) { * @return {Promise} The expected SHA. */ async function resolveExpectedSha( { ref, ghcrRepo, isMutable } ) { + const workflowSha = process.env.GUTENBERG_EXPECTED_SHA; + if ( workflowSha ) { + if ( ! SHA_PATTERN.test( workflowSha ) ) { + throw new Error( + `GUTENBERG_EXPECTED_SHA must be a 40-character Git SHA, received "${ workflowSha }"` + ); + } + + return workflowSha; + } + if ( ! isMutable ) { return ref; } @@ -157,11 +240,11 @@ function downloadGutenberg() { /** * Verify that the installed Gutenberg version matches the expected SHA. * - * For SHA refs, the expected SHA is the configured value. For mutable refs, - * the expected SHA is whatever the mutable tag currently points to in GHCR - * (read from the manifest's image.revision annotation). The installed - * `.gutenberg-hash` is compared against the expected SHA; on mismatch, a - * fresh download is triggered. + * A calling workflow may supply GUTENBERG_EXPECTED_SHA after resolving a build + * once. This avoids re-resolving a mutable tag in every matrix job. Otherwise, + * SHA refs use the configured value and mutable refs resolve their current + * image.revision annotation. The installed `.gutenberg-hash` is compared + * against the expected SHA; on mismatch, a fresh download is triggered. */ async function verifyGutenbergVersion() { console.log( '\nšŸ” Verifying Gutenberg version...' );