From b8e6b93638bb1cf4dd6fdcd8218c05a7257d2600 Mon Sep 17 00:00:00 2001 From: pieh Date: Wed, 1 Jul 2026 10:45:28 +0200 Subject: [PATCH 1/2] feat: initial new ef uploads --- src/utils/deploy/deploy-site.ts | 15 +++- src/utils/deploy/hash-edge-functions.ts | 73 +++++++++++++++++++ src/utils/deploy/upload-files.ts | 18 +++++ .../utils/deploy/hash-edge-functions.test.ts | 57 +++++++++++++++ tests/unit/utils/deploy/upload-files.test.ts | 35 +++++++++ 5 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 src/utils/deploy/hash-edge-functions.ts create mode 100644 tests/unit/utils/deploy/hash-edge-functions.test.ts diff --git a/src/utils/deploy/deploy-site.ts b/src/utils/deploy/deploy-site.ts index 18efd1969b1..ecdbf124b4b 100644 --- a/src/utils/deploy/deploy-site.ts +++ b/src/utils/deploy/deploy-site.ts @@ -15,6 +15,7 @@ import { DEFAULT_SYNC_LIMIT, } from './constants.js' import { hashConfig } from './hash-config.js' +import hashEdgeFunctions from './hash-edge-functions.js' import hashFiles from './hash-files.js' import hashFns from './hash-fns.js' import { @@ -104,6 +105,7 @@ export const deploySite = async ( { files: staticFiles, filesShaMap: staticShaMap }, { fnConfig, fnShaMap, functionSchedules, functions, functionsWithNativeModules }, configFile, + { edgeFunctions, edgeFnShaMap }, ] = await Promise.all([ hashFiles({ assetType, @@ -125,6 +127,7 @@ export const deploySite = async ( rootDir: siteRoot, }), hashConfig({ config }), + hashEdgeFunctions(edgeFunctionsDistPath, { hashAlgorithm, statusCb }), ]) const files = { ...staticFiles, [configFile.normalizedPath]: configFile.hash } @@ -181,6 +184,7 @@ For more information, visit https://ntl.fyi/cli-native-modules.`) body: { files, functions, + edge_functions: edgeFunctions, function_schedules: functionSchedules, functions_config: fnConfig, async: Object.keys(files).length > syncFileLimit, @@ -195,19 +199,24 @@ For more information, visit https://ntl.fyi/cli-native-modules.`) if (deployParams.body.async) deploy = await waitForDiff(api, deploy.id, siteId, deployTimeout) - const { required: requiredFiles, required_functions: requiredFns } = deploy + const { + required: requiredFiles, + required_functions: requiredFns, + required_edge_functions: requiredEdgeFns, + } = deploy statusCb({ type: 'create-deploy', msg: `CDN requesting ${requiredFiles.length} files${ Array.isArray(requiredFns) ? ` and ${requiredFns.length} functions` : '' - }`, + }${Array.isArray(requiredEdgeFns) ? ` and ${requiredEdgeFns.length} edge functions` : ''}`, phase: 'stop', }) const filesUploadList = getUploadList(requiredFiles, filesShaMap) const functionsUploadList = getUploadList(requiredFns, fnShaMap) - const uploadList = [...filesUploadList, ...functionsUploadList] + const edgeFunctionsUploadList = getUploadList(requiredEdgeFns, edgeFnShaMap) + const uploadList = [...filesUploadList, ...functionsUploadList, ...edgeFunctionsUploadList] await uploadFiles(api, deployId, uploadList, { concurrentUpload, statusCb, maxRetry }) diff --git a/src/utils/deploy/hash-edge-functions.ts b/src/utils/deploy/hash-edge-functions.ts new file mode 100644 index 00000000000..84ed2537bd4 --- /dev/null +++ b/src/utils/deploy/hash-edge-functions.ts @@ -0,0 +1,73 @@ +import { createHash } from 'node:crypto' +import { createReadStream } from 'node:fs' +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' +import { pipeline } from 'node:stream/promises' + +import { $TSFixMe } from '../../commands/types.js' + +import type { StatusCallback } from './status-cb.js' + +interface ManifestBundle { + asset: string + format: string +} + +const hashBundle = async (filepath: string, hashAlgorithm: string): Promise => { + const hasher = createHash(hashAlgorithm) + await pipeline([createReadStream(filepath), hasher]) + return hasher.digest('hex') +} + +// Reads the edge-bundler manifest from the dist directory and, for every bundle, computes its +// `code_sha` (sha256 of the bundle bytes — the deploy identity, recomputed rather than trusting the +// bundler's asset filename) so we can both declare it on deploy create and stream it on upload. We +// declare every format; bitballoon decides which ones actually ride this path and returns them in +// `required_edge_functions`. +const hashEdgeFunctions = async ( + edgeFunctionsDistPath: string | undefined, + { hashAlgorithm = 'sha256', statusCb }: { hashAlgorithm?: string; statusCb: StatusCallback }, +): Promise<{ + // edge_functions: { format => code_sha } sent on deploy create + edgeFunctions: Record + // code_sha => [fileObj] consumed by the upload arm + edgeFnShaMap: Record +}> => { + const edgeFunctions: Record = {} + const edgeFnShaMap: Record = {} + + if (!edgeFunctionsDistPath) { + return { edgeFunctions, edgeFnShaMap } + } + + let manifest: { bundles?: ManifestBundle[] } + try { + manifest = JSON.parse(await readFile(join(edgeFunctionsDistPath, 'manifest.json'), 'utf8')) as { + bundles?: ManifestBundle[] + } + } catch { + // No manifest (or an unreadable one) means there are no edge functions to declare. + return { edgeFunctions, edgeFnShaMap } + } + + const bundles = Array.isArray(manifest.bundles) ? manifest.bundles : [] + for (const bundle of bundles) { + const filepath = join(edgeFunctionsDistPath, bundle.asset) + const codeSha = await hashBundle(filepath, hashAlgorithm) + + edgeFunctions[bundle.format] = codeSha + + const fileObj = { assetType: 'edge-function', filepath, normalizedPath: codeSha, hash: codeSha } + if (Array.isArray(edgeFnShaMap[codeSha])) { + edgeFnShaMap[codeSha].push(fileObj) + } else { + edgeFnShaMap[codeSha] = [fileObj] + } + + statusCb({ type: 'hashing', msg: `Hashing edge function bundle ${bundle.asset}`, phase: 'progress' }) + } + + return { edgeFunctions, edgeFnShaMap } +} + +export default hashEdgeFunctions diff --git a/src/utils/deploy/upload-files.ts b/src/utils/deploy/upload-files.ts index 7244c96ceb0..3282c5c61d0 100644 --- a/src/utils/deploy/upload-files.ts +++ b/src/utils/deploy/upload-files.ts @@ -60,6 +60,24 @@ const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, maxRet }, maxRetry) break } + case 'edge-function': { + // @ts-expect-error TS(7006) FIXME: Parameter 'retryCount' implicitly has an 'any' type. + response = await retryUpload((retryCount) => { + const params = { + body: readStreamCtor, + deployId, + codeSha: normalizedPath, + } + + if (retryCount > 0) { + // @ts-expect-error TS(2339) FIXME: Property 'xNfRetryCount' does not exist on type '{... Remove this comment to see the full error message + params.xNfRetryCount = retryCount + } + + return api.uploadDeployEdgeFunction(params) + }, maxRetry) + break + } default: { const error = new Error('File Object missing assetType property') // @ts-expect-error TS(2339) FIXME: Property 'fileObj' does not exist on type 'Error'. diff --git a/tests/unit/utils/deploy/hash-edge-functions.test.ts b/tests/unit/utils/deploy/hash-edge-functions.test.ts new file mode 100644 index 00000000000..9b6f01dee17 --- /dev/null +++ b/tests/unit/utils/deploy/hash-edge-functions.test.ts @@ -0,0 +1,57 @@ +import { createHash } from 'node:crypto' +import { mkdir, writeFile } from 'node:fs/promises' +import { join } from 'node:path' + +import { expect, test } from 'vitest' + +import hashEdgeFunctions from '../../../../src/utils/deploy/hash-edge-functions.js' +import { temporaryDirectory } from '../../../../src/utils/temporary-file.js' + +const sha256 = (contents: string) => createHash('sha256').update(contents).digest('hex') + +const writeManifest = async (dir: string, bundles: { asset: string; format: string; contents: string }[]) => { + await mkdir(dir, { recursive: true }) + await Promise.all(bundles.map(({ asset, contents }) => writeFile(join(dir, asset), contents))) + await writeFile( + join(dir, 'manifest.json'), + JSON.stringify({ bundles: bundles.map(({ asset, format }) => ({ asset, format })) }), + ) +} + +test('declares every bundle format, keyed by the recomputed code_sha', async () => { + const dir = temporaryDirectory() + await writeManifest(dir, [ + { asset: 'aaa.tar.gz', format: 'tar', contents: 'tar-bundle-bytes' }, + { asset: 'bbb.eszip', format: 'eszip2', contents: 'eszip-bundle-bytes' }, + ]) + + const { edgeFunctions, edgeFnShaMap } = await hashEdgeFunctions(dir, { statusCb() {} }) + + const tarSha = sha256('tar-bundle-bytes') + const eszipSha = sha256('eszip-bundle-bytes') + // We declare all formats; bitballoon filters and only asks for the ones that ride this path. + expect(edgeFunctions).toEqual({ tar: tarSha, eszip2: eszipSha }) + expect(Object.keys(edgeFnShaMap).sort()).toEqual([tarSha, eszipSha].sort()) + expect(edgeFnShaMap[tarSha][0]).toMatchObject({ + assetType: 'edge-function', + filepath: join(dir, 'aaa.tar.gz'), + normalizedPath: tarSha, + }) +}) + +test('returns empty maps when there is no dist path', async () => { + const { edgeFunctions, edgeFnShaMap } = await hashEdgeFunctions(undefined, { statusCb() {} }) + + expect(edgeFunctions).toEqual({}) + expect(edgeFnShaMap).toEqual({}) +}) + +test('returns empty maps when the manifest is missing', async () => { + const dir = temporaryDirectory() + await mkdir(dir, { recursive: true }) + + const { edgeFunctions, edgeFnShaMap } = await hashEdgeFunctions(dir, { statusCb() {} }) + + expect(edgeFunctions).toEqual({}) + expect(edgeFnShaMap).toEqual({}) +}) diff --git a/tests/unit/utils/deploy/upload-files.test.ts b/tests/unit/utils/deploy/upload-files.test.ts index d3c1c089b83..7b2a660804e 100644 --- a/tests/unit/utils/deploy/upload-files.test.ts +++ b/tests/unit/utils/deploy/upload-files.test.ts @@ -51,6 +51,41 @@ test('Adds a retry count to function upload requests', async () => { expect(uploadDeployFunction).toHaveBeenNthCalledWith(3, expect.objectContaining({ xNfRetryCount: 2 })) }) +test('Adds a retry count to edge function upload requests', async () => { + const uploadDeployEdgeFunction = vi.fn() + const mockError = new Error('Uh-oh') + + // @ts-expect-error TS(2339) FIXME: Property 'status' does not exist on type 'Error'. + mockError.status = 500 + + uploadDeployEdgeFunction.mockRejectedValueOnce(mockError) + uploadDeployEdgeFunction.mockResolvedValueOnce(undefined) + + const mockApi = { + uploadDeployEdgeFunction, + } + const deployId = crypto.randomUUID() + const files = [ + { + assetType: 'edge-function', + filepath: '/some/path/abc123.tar.gz', + normalizedPath: 'abc123', + }, + ] + const options = { + concurrentUpload: 1, + maxRetry: 3, + statusCb: vi.fn(), + } + + await uploadFiles(mockApi, deployId, files, options) + + expect(uploadDeployEdgeFunction).toHaveBeenCalledTimes(2) + expect(uploadDeployEdgeFunction).toHaveBeenNthCalledWith(1, expect.objectContaining({ codeSha: 'abc123' })) + expect(uploadDeployEdgeFunction).toHaveBeenNthCalledWith(1, expect.not.objectContaining({ xNfRetryCount: 1 })) + expect(uploadDeployEdgeFunction).toHaveBeenNthCalledWith(2, expect.objectContaining({ xNfRetryCount: 1 })) +}) + test('Does not retry on 400 response from function upload requests', async () => { const uploadDeployFunction = vi.fn() const mockError = new Error('Uh-oh') From 3732c4dc6422ccd6683125b2b9848921102701c4 Mon Sep 17 00:00:00 2001 From: pieh Date: Wed, 1 Jul 2026 10:47:57 +0200 Subject: [PATCH 2/2] refactor: type retryUpload --- src/utils/deploy/upload-files.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/utils/deploy/upload-files.ts b/src/utils/deploy/upload-files.ts index 3282c5c61d0..bb8a29b1f4b 100644 --- a/src/utils/deploy/upload-files.ts +++ b/src/utils/deploy/upload-files.ts @@ -40,7 +40,6 @@ const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, maxRet break } case 'function': { - // @ts-expect-error TS(7006) FIXME: Parameter 'retryCount' implicitly has an 'any' typ... Remove this comment to see the full error message response = await retryUpload((retryCount) => { const params = { body: readStreamCtor, @@ -61,7 +60,6 @@ const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, maxRet break } case 'edge-function': { - // @ts-expect-error TS(7006) FIXME: Parameter 'retryCount' implicitly has an 'any' type. response = await retryUpload((retryCount) => { const params = { body: readStreamCtor, @@ -98,8 +96,7 @@ const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, maxRet return results } -// @ts-expect-error TS(7006) FIXME: Parameter 'uploadFn' implicitly has an 'any' type. -const retryUpload = (uploadFn, maxRetry) => +const retryUpload = (uploadFn: (retryCount: number) => Promise, maxRetry: number) => new Promise((resolve, reject) => { // @ts-expect-error TS(7034) FIXME: Variable 'lastError' implicitly has type 'any' in ... Remove this comment to see the full error message let lastError