[do-not-merge] feat: new edge function upload

src/utils/deploy/deploy-site.ts

@@ -15,6 +15,7 @@ import {
   DEFAULT_SYNC_LIMIT,
 } from './constants.js'
 import { hashConfig } from './hash-config.js'
+import hashEdgeFunctions from './hash-edge-functions.js'
 import hashFiles from './hash-files.js'
 import hashFns from './hash-fns.js'
 import {
@@ -104,6 +105,7 @@ export const deploySite = async (
     { files: staticFiles, filesShaMap: staticShaMap },
     { fnConfig, fnShaMap, functionSchedules, functions, functionsWithNativeModules },
     configFile,
+    { edgeFunctions, edgeFnShaMap },
   ] = await Promise.all([
     hashFiles({
       assetType,
@@ -125,6 +127,7 @@ export const deploySite = async (
       rootDir: siteRoot,
     }),
     hashConfig({ config }),
+    hashEdgeFunctions(edgeFunctionsDistPath, { hashAlgorithm, statusCb }),
   ])
 
   const files = { ...staticFiles, [configFile.normalizedPath]: configFile.hash }
@@ -181,6 +184,7 @@ For more information, visit https://ntl.fyi/cli-native-modules.`)
     body: {
       files,
       functions,
+      edge_functions: edgeFunctions,
       function_schedules: functionSchedules,
       functions_config: fnConfig,
       async: Object.keys(files).length > syncFileLimit,
@@ -195,19 +199,24 @@ For more information, visit https://ntl.fyi/cli-native-modules.`)
 
   if (deployParams.body.async) deploy = await waitForDiff(api, deploy.id, siteId, deployTimeout)
 
-  const { required: requiredFiles, required_functions: requiredFns } = deploy
+  const {
+    required: requiredFiles,
+    required_functions: requiredFns,
+    required_edge_functions: requiredEdgeFns,
+  } = deploy
 
   statusCb({
     type: 'create-deploy',
     msg: `CDN requesting ${requiredFiles.length} files${
       Array.isArray(requiredFns) ? ` and ${requiredFns.length} functions` : ''
-    }`,
+    }${Array.isArray(requiredEdgeFns) ? ` and ${requiredEdgeFns.length} edge functions` : ''}`,
     phase: 'stop',
   })
 
   const filesUploadList = getUploadList(requiredFiles, filesShaMap)
   const functionsUploadList = getUploadList(requiredFns, fnShaMap)
-  const uploadList = [...filesUploadList, ...functionsUploadList]
+  const edgeFunctionsUploadList = getUploadList(requiredEdgeFns, edgeFnShaMap)
+  const uploadList = [...filesUploadList, ...functionsUploadList, ...edgeFunctionsUploadList]
 
   await uploadFiles(api, deployId, uploadList, { concurrentUpload, statusCb, maxRetry })
 

src/utils/deploy/hash-edge-functions.ts

@@ -0,0 +1,73 @@
+import { createHash } from 'node:crypto'
+import { createReadStream } from 'node:fs'
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { pipeline } from 'node:stream/promises'
+
+import { $TSFixMe } from '../../commands/types.js'
+
+import type { StatusCallback } from './status-cb.js'
+
+interface ManifestBundle {
+  asset: string
+  format: string
+}
+
+const hashBundle = async (filepath: string, hashAlgorithm: string): Promise<string> => {
+  const hasher = createHash(hashAlgorithm)
+  await pipeline([createReadStream(filepath), hasher])
+  return hasher.digest('hex')
+}
+
+// Reads the edge-bundler manifest from the dist directory and, for every bundle, computes its
+// `code_sha` (sha256 of the bundle bytes — the deploy identity, recomputed rather than trusting the
+// bundler's asset filename) so we can both declare it on deploy create and stream it on upload. We
+// declare every format; bitballoon decides which ones actually ride this path and returns them in
+// `required_edge_functions`.
+const hashEdgeFunctions = async (
+  edgeFunctionsDistPath: string | undefined,
+  { hashAlgorithm = 'sha256', statusCb }: { hashAlgorithm?: string; statusCb: StatusCallback },
+): Promise<{
+  // edge_functions: { format => code_sha } sent on deploy create
+  edgeFunctions: Record<string, string>
+  // code_sha => [fileObj] consumed by the upload arm
+  edgeFnShaMap: Record<string, $TSFixMe[]>
+}> => {
+  const edgeFunctions: Record<string, string> = {}
+  const edgeFnShaMap: Record<string, $TSFixMe[]> = {}
+
+  if (!edgeFunctionsDistPath) {
+    return { edgeFunctions, edgeFnShaMap }
+  }
+
+  let manifest: { bundles?: ManifestBundle[] }
+  try {
+    manifest = JSON.parse(await readFile(join(edgeFunctionsDistPath, 'manifest.json'), 'utf8')) as {
+      bundles?: ManifestBundle[]
+    }
+  } catch {
+    // No manifest (or an unreadable one) means there are no edge functions to declare.
+    return { edgeFunctions, edgeFnShaMap }
+  }
+
+  const bundles = Array.isArray(manifest.bundles) ? manifest.bundles : []
+  for (const bundle of bundles) {
+    const filepath = join(edgeFunctionsDistPath, bundle.asset)
+    const codeSha = await hashBundle(filepath, hashAlgorithm)
+
+    edgeFunctions[bundle.format] = codeSha
+
+    const fileObj = { assetType: 'edge-function', filepath, normalizedPath: codeSha, hash: codeSha }
+    if (Array.isArray(edgeFnShaMap[codeSha])) {
+      edgeFnShaMap[codeSha].push(fileObj)
+    } else {
+      edgeFnShaMap[codeSha] = [fileObj]
+    }
+
+    statusCb({ type: 'hashing', msg: `Hashing edge function bundle ${bundle.asset}`, phase: 'progress' })
+  }
+
+  return { edgeFunctions, edgeFnShaMap }
+}
+
+export default hashEdgeFunctions

src/utils/deploy/upload-files.ts

@@ -40,7 +40,6 @@ const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, maxRet
         break
       }
       case 'function': {
-        // @ts-expect-error TS(7006) FIXME: Parameter 'retryCount' implicitly has an 'any' typ... Remove this comment to see the full error message
         response = await retryUpload((retryCount) => {
           const params = {
             body: readStreamCtor,
@@ -60,6 +59,23 @@ const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, maxRet
         }, maxRetry)
         break
       }
+      case 'edge-function': {
+        response = await retryUpload((retryCount) => {
+          const params = {
+            body: readStreamCtor,
+            deployId,
+            codeSha: normalizedPath,
+          }
+
+          if (retryCount > 0) {
+            // @ts-expect-error TS(2339) FIXME: Property 'xNfRetryCount' does not exist on type '{... Remove this comment to see the full error message
+            params.xNfRetryCount = retryCount
+          }
+
+          return api.uploadDeployEdgeFunction(params)
+        }, maxRetry)
+        break
+      }
       default: {
         const error = new Error('File Object missing assetType property')
         // @ts-expect-error TS(2339) FIXME: Property 'fileObj' does not exist on type 'Error'.
@@ -80,8 +96,7 @@ const uploadFiles = async (api, deployId, uploadList, { concurrentUpload, maxRet
   return results
 }
 
-// @ts-expect-error TS(7006) FIXME: Parameter 'uploadFn' implicitly has an 'any' type.
-const retryUpload = (uploadFn, maxRetry) =>
+const retryUpload = (uploadFn: (retryCount: number) => Promise<unknown>, maxRetry: number) =>
   new Promise((resolve, reject) => {
     // @ts-expect-error TS(7034) FIXME: Variable 'lastError' implicitly has type 'any' in ... Remove this comment to see the full error message
     let lastError

tests/unit/utils/deploy/hash-edge-functions.test.ts

@@ -0,0 +1,57 @@
+import { createHash } from 'node:crypto'
+import { mkdir, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+
+import { expect, test } from 'vitest'
+
+import hashEdgeFunctions from '../../../../src/utils/deploy/hash-edge-functions.js'
+import { temporaryDirectory } from '../../../../src/utils/temporary-file.js'
+
+const sha256 = (contents: string) => createHash('sha256').update(contents).digest('hex')
+
+const writeManifest = async (dir: string, bundles: { asset: string; format: string; contents: string }[]) => {
+  await mkdir(dir, { recursive: true })
+  await Promise.all(bundles.map(({ asset, contents }) => writeFile(join(dir, asset), contents)))
+  await writeFile(
+    join(dir, 'manifest.json'),
+    JSON.stringify({ bundles: bundles.map(({ asset, format }) => ({ asset, format })) }),
+  )
+}
+
+test('declares every bundle format, keyed by the recomputed code_sha', async () => {
+  const dir = temporaryDirectory()
+  await writeManifest(dir, [
+    { asset: 'aaa.tar.gz', format: 'tar', contents: 'tar-bundle-bytes' },
+    { asset: 'bbb.eszip', format: 'eszip2', contents: 'eszip-bundle-bytes' },
+  ])
+
+  const { edgeFunctions, edgeFnShaMap } = await hashEdgeFunctions(dir, { statusCb() {} })
+
+  const tarSha = sha256('tar-bundle-bytes')
+  const eszipSha = sha256('eszip-bundle-bytes')
+  // We declare all formats; bitballoon filters and only asks for the ones that ride this path.
+  expect(edgeFunctions).toEqual({ tar: tarSha, eszip2: eszipSha })
+  expect(Object.keys(edgeFnShaMap).sort()).toEqual([tarSha, eszipSha].sort())
+  expect(edgeFnShaMap[tarSha][0]).toMatchObject({
+    assetType: 'edge-function',
+    filepath: join(dir, 'aaa.tar.gz'),
+    normalizedPath: tarSha,
+  })
+})
+
+test('returns empty maps when there is no dist path', async () => {
+  const { edgeFunctions, edgeFnShaMap } = await hashEdgeFunctions(undefined, { statusCb() {} })
+
+  expect(edgeFunctions).toEqual({})
+  expect(edgeFnShaMap).toEqual({})
+})
+
+test('returns empty maps when the manifest is missing', async () => {
+  const dir = temporaryDirectory()
+  await mkdir(dir, { recursive: true })
+
+  const { edgeFunctions, edgeFnShaMap } = await hashEdgeFunctions(dir, { statusCb() {} })
+
+  expect(edgeFunctions).toEqual({})
+  expect(edgeFnShaMap).toEqual({})
+})

tests/unit/utils/deploy/upload-files.test.ts

@@ -51,6 +51,41 @@ test('Adds a retry count to function upload requests', async () => {
   expect(uploadDeployFunction).toHaveBeenNthCalledWith(3, expect.objectContaining({ xNfRetryCount: 2 }))
 })
 
+test('Adds a retry count to edge function upload requests', async () => {
+  const uploadDeployEdgeFunction = vi.fn()
+  const mockError = new Error('Uh-oh')
+
+  // @ts-expect-error TS(2339) FIXME: Property 'status' does not exist on type 'Error'.
+  mockError.status = 500
+
+  uploadDeployEdgeFunction.mockRejectedValueOnce(mockError)
+  uploadDeployEdgeFunction.mockResolvedValueOnce(undefined)
+
+  const mockApi = {
+    uploadDeployEdgeFunction,
+  }
+  const deployId = crypto.randomUUID()
+  const files = [
+    {
+      assetType: 'edge-function',
+      filepath: '/some/path/abc123.tar.gz',
+      normalizedPath: 'abc123',
+    },
+  ]
+  const options = {
+    concurrentUpload: 1,
+    maxRetry: 3,
+    statusCb: vi.fn(),
+  }
+
+  await uploadFiles(mockApi, deployId, files, options)
+
+  expect(uploadDeployEdgeFunction).toHaveBeenCalledTimes(2)
+  expect(uploadDeployEdgeFunction).toHaveBeenNthCalledWith(1, expect.objectContaining({ codeSha: 'abc123' }))
+  expect(uploadDeployEdgeFunction).toHaveBeenNthCalledWith(1, expect.not.objectContaining({ xNfRetryCount: 1 }))
+  expect(uploadDeployEdgeFunction).toHaveBeenNthCalledWith(2, expect.objectContaining({ xNfRetryCount: 1 }))
+})
+
 test('Does not retry on 400 response from function upload requests', async () => {
   const uploadDeployFunction = vi.fn()
   const mockError = new Error('Uh-oh')