Skip to content

Commit 3e2f397

Browse files
eoinshaeoinsha
committed
chore: try providing assets as ZIP
1 parent 454d319 commit 3e2f397

4 files changed

Lines changed: 172 additions & 17 deletions

File tree

‎resources/export.sh‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ export HOME=/tmp/uv-python-lambda-home
3636
ulimit -n "$UV_PYTHON_LAMBDA_NOFILE_LIMIT"
3737

3838
print_help() {
39-
echo "Usage: $NAME --output <output_dir> [--package <package_name>] [--exclude <glob>] [--before-hooks <base64_json>] [--after-hooks <base64_json>]" >&2
39+
echo "Usage: $NAME --output <output_dir> [--output-zip <output_zip>] [--package <package_name>] [--exclude <glob>] [--before-hooks <base64_json>] [--after-hooks <base64_json>]" >&2
4040
exit 1
4141
}
4242

@@ -47,17 +47,18 @@ then
4747
exit 1
4848
fi
4949

50-
opts=$(getopt --name "$NAME" --options hp:o:e:vd --longoptions help,package:,output:,exclude:,before-hooks:,after-hooks:,verbose,debug -- "$@") || print_help
50+
opts=$(getopt --name "$NAME" --options hp:o:e:vd --longoptions help,package:,output:,output-zip:,exclude:,before-hooks:,after-hooks:,verbose,debug -- "$@") || print_help
5151
eval set -- "$opts"
5252

53-
declare package="" output="" before_hooks="" after_hooks="" verbose=0 debug=0
53+
declare package="" output="" output_zip="" before_hooks="" after_hooks="" verbose=0 debug=0
5454
declare -a excludes=()
5555
while (($#))
5656
do
5757
case $1 in
5858
-h|--help) print_help;;
5959
-p|--package) package=$2; shift;;
6060
-o|--output) output=$2; shift;;
61+
--output-zip) output_zip=$2; shift;;
6162
-e|--exclude) excludes+=("$2"); shift;;
6263
--before-hooks) before_hooks=$2; shift;;
6364
--after-hooks) after_hooks=$2; shift;;
@@ -82,7 +83,14 @@ while [[ ! -f "$LOCK_FILE" ]]; do
8283
done
8384

8485
output_dir=$(realpath -m "$output")
86+
output_zip_path=""
87+
if [[ -n "$output_zip" ]]; then
88+
output_zip_path=$(realpath -m "$output_zip")
89+
fi
8590
mkdir -p "$(dirname "$output_dir")"
91+
if [[ -n "$output_zip_path" ]]; then
92+
mkdir -p "$(dirname "$output_zip_path")"
93+
fi
8694

8795
# Lock the output directory itself so two requests never write to the same asset
8896
# path concurrently. A single Lambda asset should only be produced once at a
@@ -289,5 +297,31 @@ run_hooks "$before_hooks"
289297
run_export_from_directory "$working_root"
290298
run_hooks "$after_hooks"
291299

300+
if [[ -n "$output_zip_path" ]]; then
301+
python - "$output_dir" "$output_zip_path" <<'PY'
302+
from pathlib import Path
303+
import sys
304+
import zipfile
305+
306+
root = Path(sys.argv[1])
307+
destination = Path(sys.argv[2])
308+
temporary = destination.with_suffix(destination.suffix + ".tmp")
309+
310+
temporary.unlink(missing_ok=True)
311+
destination.unlink(missing_ok=True)
312+
313+
with zipfile.ZipFile(
314+
temporary,
315+
mode="w",
316+
compression=zipfile.ZIP_DEFLATED,
317+
) as archive:
318+
for path in sorted(root.rglob("*")):
319+
if path.is_file():
320+
archive.write(path, path.relative_to(root).as_posix())
321+
322+
temporary.replace(destination)
323+
PY
324+
fi
325+
292326
# Historical cleanup: older iterations wrote a file lock into the asset dir
293327
rm -f "$output_dir/.lock"

‎src/bundling.ts‎

Lines changed: 68 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,10 @@ import type { BundlingOptions, ICommandHooks } from './types';
1919

2020
export const HASHABLE_DEPENDENCIES_EXCLUDE = [
2121
'*.pyc',
22+
'cdk.out/**',
23+
'**/cdk.out/**',
2224
'cdk/**',
25+
'**/cdk/**',
2326
'.git/**',
2427
'.venv/**',
2528
];
@@ -28,8 +31,10 @@ export const DEFAULT_ASSET_EXCLUDES = [
2831
'.venv/',
2932
'node_modules/',
3033
'cdk.out/',
34+
'**/cdk.out/**',
35+
'cdk/',
36+
'**/cdk/**',
3137
'.git/',
32-
'cdk',
3338
];
3439

3540
export const DEFAULT_UV_VERSION = '0.5.27';
@@ -87,29 +92,48 @@ export class Bundling {
8792

8893
public static bundle(options: BundlingProps): AssetCode {
8994
const {
90-
hashableAssetExclude = HASHABLE_DEPENDENCIES_EXCLUDE,
95+
hashableAssetExclude,
9196
assetHashType = AssetHashType.SOURCE,
9297
assetHash,
9398
...bundlingOptions
9499
} = options;
100+
const mergedHashableAssetExclude = dedupePatterns([
101+
...HASHABLE_DEPENDENCIES_EXCLUDE,
102+
...(hashableAssetExclude ?? []),
103+
]);
95104

96105
const bundling = new Bundling(bundlingOptions);
97106
const cdkOutDir = getCdkOutDir();
98107
const hostFunctionOutputDir = bundling.getHostFunctionOutputDir(cdkOutDir);
108+
const hostFunctionWorkspaceDir =
109+
bundling.getHostFunctionWorkspaceDir(cdkOutDir);
110+
const hostFunctionArchivePath =
111+
bundling.getHostFunctionArchivePath(cdkOutDir);
112+
113+
if (bundling.skip) {
114+
mkdirSync(hostFunctionOutputDir, { recursive: true });
115+
return Code.fromCustomCommand(
116+
hostFunctionOutputDir,
117+
bundling.createBundlingCommand(),
118+
{
119+
assetHash,
120+
assetHashType,
121+
exclude: mergedHashableAssetExclude,
122+
},
123+
);
124+
}
99125

100-
mkdirSync(hostFunctionOutputDir, { recursive: true });
126+
mkdirSync(hostFunctionWorkspaceDir, { recursive: true });
101127

102-
if (!bundling.skip) {
103-
bundling.ensureBuilderReady(cdkOutDir);
104-
}
128+
bundling.ensureBuilderReady(cdkOutDir);
105129

106130
return Code.fromCustomCommand(
107-
hostFunctionOutputDir,
131+
hostFunctionArchivePath,
108132
bundling.createBundlingCommand(),
109133
{
110134
assetHash,
111135
assetHashType,
112-
exclude: hashableAssetExclude,
136+
exclude: mergedHashableAssetExclude,
113137
},
114138
);
115139
}
@@ -147,7 +171,10 @@ export class Bundling {
147171
this.securityOpt = props.securityOpt;
148172
this.network = props.network;
149173
this.bundlingFileAccess = props.bundlingFileAccess;
150-
this.assetExcludes = props.assetExcludes ?? DEFAULT_ASSET_EXCLUDES;
174+
this.assetExcludes = dedupePatterns([
175+
...DEFAULT_ASSET_EXCLUDES,
176+
...(props.assetExcludes ?? []),
177+
]);
151178
this.commandHooks = props.commandHooks;
152179
this.outputPathSuffix = props.outputPathSuffix;
153180
this.skip = !!props.skip;
@@ -249,6 +276,7 @@ export class Bundling {
249276
}
250277

251278
const containerOutputDir = this.getContainerFunctionOutputDir();
279+
const containerArchivePath = this.getContainerFunctionArchivePath();
252280
const command = ['docker', 'exec'];
253281
const builderUser = getDockerUserArg();
254282

@@ -265,6 +293,8 @@ export class Bundling {
265293
`${BUILDER_TOOL_DIR}/export.sh`,
266294
'--output',
267295
containerOutputDir,
296+
'--output-zip',
297+
containerArchivePath,
268298
);
269299

270300
if (this.props.workspacePackage) {
@@ -315,16 +345,39 @@ export class Bundling {
315345

316346
private getContainerFunctionOutputDir() {
317347
return toPosixPath(
318-
path.join('/uvbuild', this.functionOutDir, this.outputPathSuffix ?? ''),
348+
path.join(
349+
'/uvbuild',
350+
this.functionOutDir,
351+
'bundle',
352+
this.outputPathSuffix ?? '',
353+
),
319354
);
320355
}
321356

322357
private getHostFunctionOutputDir(cdkOutDir: string) {
358+
return path.join(
359+
this.getHostFunctionWorkspaceDir(cdkOutDir),
360+
'bundle',
361+
this.outputPathSuffix ?? '',
362+
);
363+
}
364+
365+
private getContainerFunctionArchivePath() {
366+
return toPosixPath(path.join('/uvbuild', this.functionOutDir, 'asset.zip'));
367+
}
368+
369+
private getHostFunctionWorkspaceDir(cdkOutDir: string) {
323370
return path.join(
324371
cdkOutDir,
325372
this.containerBuilderKey,
326373
this.functionOutDir,
327-
this.outputPathSuffix ?? '',
374+
);
375+
}
376+
377+
private getHostFunctionArchivePath(cdkOutDir: string) {
378+
return path.join(
379+
this.getHostFunctionWorkspaceDir(cdkOutDir),
380+
'asset.zip',
328381
);
329382
}
330383

@@ -365,6 +418,10 @@ function toPosixPath(value: string) {
365418
return value.split(path.sep).join(path.posix.sep);
366419
}
367420

421+
function dedupePatterns(patterns: string[]) {
422+
return [...new Set(patterns)];
423+
}
424+
368425
function getBuilderEnvironment(
369426
environment?: Record<string, string>,
370427
): Record<string, string> {

‎test/bundling.test.ts‎

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,13 +120,15 @@ describe('Bundling', () => {
120120
expect(afterIndex).toBeGreaterThan(-1);
121121
expect(decodeCommands(command[beforeIndex + 1])).toEqual(['echo before']);
122122
expect(decodeCommands(command[afterIndex + 1])).toEqual(['echo after']);
123+
expect(command).toContain('--output-zip');
124+
expect(command).toContain('/uvbuild/app/asset.zip');
123125
expect(commandHooks.beforeBundling).toHaveBeenCalledWith(
124126
'/src',
125-
'/uvbuild/app',
127+
'/uvbuild/app/bundle',
126128
);
127129
expect(commandHooks.afterBundling).toHaveBeenCalledWith(
128130
'/src',
129-
'/uvbuild/app',
131+
'/uvbuild/app/bundle',
130132
);
131133
});
132134

@@ -172,6 +174,44 @@ describe('Bundling', () => {
172174
expect(command).toContain('UV_PYTHON_LAMBDA_NOFILE_LIMIT=1048576');
173175
});
174176

177+
test('includes nested cdk output excludes by default', () => {
178+
const bundling = new Bundling({
179+
rootDir: '/tmp/project-default-excludes',
180+
runtime: Runtime.PYTHON_3_12,
181+
architecture: Architecture.X86_64,
182+
workspacePackage: 'app',
183+
});
184+
185+
const command = Reflect.get(bundling, 'createBundlingCommand').call(
186+
bundling,
187+
) as string[];
188+
189+
expect(command).toContain('--exclude');
190+
expect(command).toContain('cdk.out/');
191+
expect(command).toContain('**/cdk.out/**');
192+
expect(command).toContain('cdk/');
193+
expect(command).toContain('**/cdk/**');
194+
});
195+
196+
test('merges custom asset excludes with the library defaults', () => {
197+
const bundling = new Bundling({
198+
rootDir: '/tmp/project-custom-excludes',
199+
runtime: Runtime.PYTHON_3_12,
200+
architecture: Architecture.X86_64,
201+
workspacePackage: 'app',
202+
assetExcludes: ['dist/', '**/dist/**'],
203+
});
204+
205+
const command = Reflect.get(bundling, 'createBundlingCommand').call(
206+
bundling,
207+
) as string[];
208+
209+
expect(command).toContain('cdk.out/');
210+
expect(command).toContain('**/cdk.out/**');
211+
expect(command).toContain('dist/');
212+
expect(command).toContain('**/dist/**');
213+
});
214+
175215
test('runs export commands as the host user when uid and gid are available', () => {
176216
const bundling = new Bundling({
177217
rootDir: '/tmp/project-user',

‎test/function.test.ts‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { exec } from 'node:child_process';
1+
import { exec, execFile } from 'node:child_process';
22
import * as fs from 'node:fs/promises';
33
import * as os from 'node:os';
44
import * as path from 'node:path';
@@ -14,6 +14,7 @@ import {
1414
} from '../src/build-container';
1515

1616
const execAsync = promisify(exec);
17+
const execFileAsync = promisify(execFile);
1718
const resourcesPath = path.resolve(__dirname, 'resources');
1819
const TEST_TIMEOUT = Number(process.env.TEST_TIMEOUT ?? '999999');
1920

@@ -318,6 +319,13 @@ test('Reject non-python runtimes', async () => {
318319
async function getFunctionAssetContents(functionResource: any, app: App) {
319320
const assetRelPath = functionResource.Metadata['uv-python-lambda:asset-path'];
320321
const assetPath = path.join(app.outdir, assetRelPath);
322+
323+
if (assetPath.endsWith('.zip')) {
324+
const files = await listZipEntries(assetPath);
325+
const rootEntries = [...new Set(files.map((file) => file.split('/')[0]))];
326+
return { rootEntries, files };
327+
}
328+
321329
const rootEntries = await fs.readdir(assetPath);
322330
const files: string[] = [];
323331

@@ -342,3 +350,19 @@ async function getFunctionAssetContents(functionResource: any, app: App) {
342350

343351
return { rootEntries, files };
344352
}
353+
354+
async function listZipEntries(assetPath: string): Promise<string[]> {
355+
const command = [
356+
'-c',
357+
'import json, sys, zipfile; archive = zipfile.ZipFile(sys.argv[1]); print(json.dumps([info.filename for info in archive.infolist() if not info.is_dir()]))',
358+
assetPath,
359+
];
360+
361+
try {
362+
const { stdout } = await execFileAsync('python3', command);
363+
return JSON.parse(stdout) as string[];
364+
} catch {
365+
const { stdout } = await execFileAsync('python', command);
366+
return JSON.parse(stdout) as string[];
367+
}
368+
}

0 commit comments

Comments
 (0)