diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a636ad..3f17040 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,3 +53,41 @@ jobs: # the flag onto the repo's own test script, so the strip-types flag stays single-sourced. - name: Run tests (with coverage) run: node --run test -- --experimental-test-coverage + + # The `ci` job installs the newest versions. This one runs the same checks on the oldest + # supported Node major (the newest release of it) with every peer at the lowest version its + # range allows, so a declared minimum that stopped working is caught here and not in a host. + min-versions: + runs-on: ubuntu-latest + + permissions: + contents: read + + env: + MONGOMS_DISABLE_POSTINSTALL: '1' + + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: '24' + cache: 'npm' + + - name: Cache mongod binary (mongodb-memory-server) + uses: actions/cache@v4 + with: + path: ~/.cache/mongodb-binaries + key: mongod-${{ runner.os }}-mms11 + + - name: npm clean install + run: npm ci + + # Read the minimums from package.json, so this job follows the peer ranges. + - name: Install the minimum peer versions + run: | + npm install --no-save $(node -p "Object.entries(require('./package.json').peerDependencies).map(([name, range]) => name + '@' + range.replace(/^[^0-9]*/, '')).join(' ')") + + - run: node --run types:check + - run: node --run build + - run: node --run test diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index b9569d8..ad2edb7 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -26,7 +26,7 @@ jobs: run: npm ci # Builds dist, packs the tarball, installs it into a throwaway consumer and verifies - # the PUBLISHED surface (17 core exports, the exports map, the optional subpaths' + # the PUBLISHED surface (the expected core exports, the exports map, the optional subpaths' # loud-fail without their AWS SDKs, the always-safe subpaths, the resize-scaffold bin) # — dist-only breakage the TS-source test suite can't see. - name: Packaging smoke test diff --git a/AGENTS.md b/AGENTS.md index a21d2af..1d61071 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,15 +26,19 @@ task. Preview identity is `resizer:pipeline:sizeKey:format:filterSig`: dispatch locks and stored previews are separate per Resizer and per pipeline, so different renderings of the same media keep separate previews (rename a pipeline, e.g. `watermark-v2`, to regenerate its images). Preview rows without `resizer`/`pipeline` belong to `default`. Legacy task rows without -a key remain valid. Storage drivers that can prove original -visibility implement `canServeOriginalPublicly`; the engine never fabricates a public URL for -a private original. +a key remain valid. The original is never served: `resolve()` returns stored previews only. For a +pipeline without `variantSteps`, a raster original no larger than a requested `WxH` box gets a +preview at its own size (no upscaling, no cropping, metadata removed; only the +`limits.resultDimension` cap can scale it down); a pipeline with `variantSteps` always gets the +full box its steps were written for. To hand the private original to its owner, call +`storage.signedUrl(original.storageRef, ttlSeconds)` yourself; the module never calls it. ## Integrate (in order) 1. Install. Every peer is OPTIONAL: a framework app already has `@adaptivestone/framework` and - `mongoose`; the AWS SDKs are needed only for the driver subpaths that use them (a missing one - fails loudly at your own import line at bootstrap): + `mongoose`; the AWS SDKs are needed only for the drivers that use them. A missing one fails at + your own import line when you import a driver subpath, or, for a driver selected in the config + file, at first use or `verify()` with `RESIZE_PEER_MISSING` (naming the packages and the file): ```bash npm i @adaptivestone/framework-module-resize @@ -51,8 +55,10 @@ a private original. ``` `--eager` emits `src/resizer.ts` + `src/config/resize.ts` (local storage, no queue). - Default (lazy) also emits `src/models/ResizeTask.ts` and `src/commands/ResizeWorker.ts` - (`import '../resizer.ts'` plus a re-export of the module's command). + Default (lazy) also emits `src/models/ResizeTask.ts` (extends the default export of + `@adaptivestone/framework-module-resize/framework/ResizeTaskModel.js`, so `npm run gen` types + `getModel('ResizeTask')`) and `src/commands/ResizeWorker.ts` (`import '../resizer.ts'` plus a + re-export of the module's command). Appends a pointer to this guide into the host's `AGENTS.md` (`--agents claude|print|skip` to redirect or suppress it). @@ -75,13 +81,15 @@ a private original. Without the framework: `new Resizer({ storage, db, tasks? })` from the main entry, with `@adaptivestone/framework-module-resize/drivers/mongo.js` → `mongoDatabase(connection, { mediaModel, timing? })` (media, locks and `.tasks` with the package's `ResizeTask` / - `ResizeLock` models; also `MongoDatabase`, `MongoTaskQueue`, `createResizeModels`), + `ResizeLock` models; also `MongoDatabase`, `MongoTaskQueue`, `createResizeModels`; call it once + and pass that `.tasks` to every Resizer — each call makes its own queue), `…/drivers/fs.js` → `LocalFsStorage`, `…/drivers/s3.js` → `S3Storage`, `…/drivers/sqs.js` → `SqsTaskQueue`. Any part may be a function (sync or async) called once on first use. A custom driver extends the exported abstract class (`ResizeStorage`, `ResizeDatabase`, `TaskQueue`) or is any object of the same shape — no `app` parameter; a driver closes over its own client. The core owns the queue logic (worker loop, retries, dead-letters, events); a `TaskQueue` only - implements atomic `add` / `claim` / `renew` / `complete` / `fail`. `claim` may wait for a task + implements atomic `add` / `claim` / `renew` / `complete` / `fail` (optional `release` gives a + task back at worker shutdown; see the README contract). `claim` may wait for a task (long poll), but must return once its `signal` aborts and never claim ahead of the call. 4. Import `src/resizer.ts` wherever you need the Resizer (a static import is fine). To fail at boot @@ -100,10 +108,15 @@ a private original. - `queue`: `{ driver: 'database' }` or `{ driver: 'sqs', queueUrl, queues?, deadLetterQueueUrl?, waitTimeSeconds?, region?, endpoint? }`, plus any timing key (`leaseMs`, `lockTtlMs`, `maxAttempts`, …; the rest default). Missing or `false` = eager only; - - `worker`: the worker command's settings. + - `worker`: the worker process's `enabled` switch and Sharp tuning. Variant parallelism is the top-level `concurrency`. A second Resizer can read its own file: - `new FrameworkResizer({ name: 'listings', configName: 'resizeListings' })`. + `new FrameworkResizer({ name: 'listings', configName: 'resizeListings' })`. Resizers on the same + backend share one task queue (every `queue: { driver: 'database' }`, or identical SQS + settings), and a queue has one timing: such files must set the same timing keys (and SQS + `waitTimeSeconds`; unset counts as the default 10). The worker process + reads `worker` from `resize.ts` whatever its Resizers read, unless started with + `npm run cli ResizeWorker -- --config=` (required when the host has no `resize.ts`). Put environment-only changes in `resize..ts` (for example, S3 in `resize.production.ts`); the framework merges that file field by field before this module reads and validates the resolved config. Do not add a second runtime merge. When an @@ -120,7 +133,9 @@ a private original. Keep `minimize: false` on the media schema (already the Framework `BaseModel` default). Direct Mongoose users must pass `{ minimize: false }` to `new Schema`. - Otherwise empty objects in opaque `storageRef` values can disappear on save/update. + Otherwise empty objects in opaque `storageRef` values can disappear on save/update. A + hand-written schema that declares preview rows as sub-documents must also give them + `identity: { type: String }`: the database stores one row per preview identity. 7. Prepare queue infrastructure outside the resizer runtime. The package's `ResizeTask` model and the framework's `Lock` model declare their indexes; the host's normal lifecycle or an explicit @@ -128,7 +143,10 @@ a private original. run. The module does not create, synchronize, drop, or repair indexes, and it has no `prepareQueue()` API. The partial unique active-request index on `{ fileId, pipeline, requestKey }` is required for the Mongo deduplication guarantee; verify it in the host's DB - rollout. Never add index creation to HTTP bootstrap or the first enqueue. + rollout. The claim index is `{ queue: 1, status: 1, availableAt: 1 }`; when upgrading, create + it before or with the new workers, then drop the old lease index (`{ status: 1, createdAt: 1 }`, + or `{ queue: 1, status: 1, createdAt: 1 }` from a pre-release build). Never add index creation + to HTTP bootstrap or the first enqueue. 8. Lazy / pre-warm modes: keep `queue: { driver: 'database' }` (or SQS) and set `worker.enabled: true` in the host `src/config/resize.ts` @@ -153,8 +171,15 @@ const original = await getResizer().uploadOriginal({ The format and dimensions come from `sharp().metadata()` for raster images and SVG. Input bytes are stored unchanged; SVG stays `.svg` (`image/svg+xml`) as a private original. SVG sizes are reported by Sharp, including sizes derived from `viewBox`; unreadable or unsized SVG -is rejected. The module does not sanitize SVG markup. The worker rasterizes accepted SVG -into the same configured public preview formats as other images. +is rejected. The module does not sanitize SVG markup. The worker renders an SVG once per task, +before any pipeline step, into a PNG, in a separate Node process, at the largest size the +requested previews need; `beforeSteps` receive that PNG (never SVG markup), and every format is +made from it. The render is killed after `limits.processingTimeoutSeconds` +(`RESIZE_SVG_RENDER_TIMEOUT`, dead-lettered at once) or when the task aborts; a crashed render is +`RESIZE_SVG_RENDER_FAILED` (retried). A render that cannot start is `ResizeSetupError` +`RESIZE_SVG_RENDER_UNAVAILABLE`: the host must allow child processes (Node's permission model: +`--allow-child-process`) and ship `svgRasterChild.js` next to `svgRaster.js` (watch out when +bundling). Eager `generate()` throws all three. Persist every original privately, then call the same `prewarm()` path for raster and SVG. The worker creates the requested Sharp previews for both. SVG is an input @@ -177,7 +202,6 @@ const { decision, output } = await getResizer().resolve({ media: fileDoc, pipeline: 'default', sizes: [{ width: 620 }, { fit: true }, { width: 300, height: 300 }], - ctx: { isOwner }, }); // `output` is your formatPublicUrls hook (undefined if no hook / hook throws). // formatPictureUrls skips filtered variants — map `decision` for those. @@ -256,22 +280,34 @@ Observers (worker side): `onPreviewGenerated`, `afterTaskComplete`, `onTaskFaile - Construct each Resizer ONCE, at one construction site, with `new FrameworkResizer`. Most hosts need one (`getResizer()`); for more, give each a `name` and, if it differs, its own config file via `configName` (`getResizer('listings')`). The same name twice throws. Every task records its Resizer and queue; a worker serves all Resizers in its process - for one queue (`--queue`, default `'default'`), with one consume loop per distinct task queue. + for one queue (`--queue`, default `'default'`), with one consume loop per task queue (Resizers + on the same backend share one), one task at a time, each processed with its own Resizer. - `ctx` does NOT cross the queue: worker-side steps and observers see `ctx === {}`. Only eager `generate()` passes the caller's `ctx` to steps. Persist per-media data on the media doc. - Watermarks belong in `variantSteps`, never in `beforeSteps` (baked once onto the original, a watermark scales away to unreadable on small variants). +- Register every pipeline in `src/resizer.ts`, so the API and the worker both have it + (`resizer.hasPipeline(name)` checks). An unregistered name (anything but `'default'`) never + renders: `resolve()` serves only previews already stored for it and queues nothing, `prewarm()` + reports the non-retryable issue `RESIZE_PIPELINE_UNKNOWN`, and `generate()` and the worker throw + `ResizeSetupError` with that code (a queued task retries, then dead-letters). When you add or + rename a pipeline, deploy the worker before the API requests it. - The scaffolded `resize.ts` is complete. The framework merges environment overrides and the module reads that final config without a second merge. Arrays in environment overrides replace. - Format ids are open strings. `formats` controls generated outputs, `upload.formats` controls - accepted originals, and `encode.formats[id]` is passed to Sharp as that encoder's options. + accepted originals, and `encode.formats[id]` is passed to Sharp as that encoder's options. A + per-call `formats` entry (or a variant a `beforeEnqueue` tap adds or rewrites) without an + `encode.formats` key is never generated: `resolve()` leaves it out of `decision.missing`, + `prewarm()` reports it in `unconfirmed` with the non-retryable issue + `RESIZE_FORMAT_NOT_CONFIGURED`, and `generate()` throws `ResizeSetupError` with that code. - Never resize/encode with sharp on the request path. `uploadOriginal()` has one bounded exception: `metadata()` inspection for raster images and SVG only; it never emits transformed bytes. - The scaffolded model/command shims re-use the package: do not vendor or fork them. The command - must keep its `import '../resizer.ts'`. Gate drift in CI with `npx resize-scaffold --check`. -- SVG originals stay private. The worker rasterizes SVG into the requested public preview - formats through the normal durable task lifecycle. Neither `resolve()` nor the original-fits - shortcut returns uploaded SVG markup. + must keep its `import '../resizer.ts'`. Gate drift in CI with `npx resize-scaffold --check` + (`--check --eager` for an eager host, which has no model or command to check). +- SVG originals stay private, and no original is ever served: `resolve()` returns stored previews + only, so uploaded SVG markup never reaches a reader. The worker renders SVG into the requested + raster preview formats through the normal durable task lifecycle. - Deleting storage objects when media is deleted is the HOST's job — the module only appends. - A queued raster task completes only with full identity coverage. Partial successes are persisted, then retried for the missing identities only; persistent gaps follow normal backoff/dead-letter. @@ -284,13 +320,19 @@ Observers (worker side): `onPreviewGenerated`, `afterTaskComplete`, `onTaskFaile | `RESIZE_DATABASE_REQUIRED` at construction | `new Resizer()` takes its `db` explicitly — framework hosts use `new FrameworkResizer()` from `…/framework.js`; plain Node: `mongoDatabase(connection, { mediaModel })` | | `RESIZE_CONFIG_STORAGE_MISSING` | add `storage: { driver: 'local', … }` (or `'s3'`) to the config file named in the message, or pass `storage` to `new FrameworkResizer()` | | `RESIZE_NOT_READY` | host code read `resizer.storage` / `db` / `tasks` before the drivers loaded — `await resizer.ready()` first (the Resizer's methods do this themselves) | -| `RESIZE_CONFIG_REMOVED_KEY` naming `queue` or `worker` | a core config passed to `new Resizer` holds image settings only — move timing to the task queue's `timing`, `worker.concurrency` to `concurrency` | -| `RESIZE_MONGO_MODEL_MISSING` from `verify()` or at worker start | the `ResizeTask` model (or the media model) does not resolve — scaffold `src/models/ResizeTask.ts`, check the model name | +| `RESIZE_CONFIG_REMOVED_KEY` naming `queue` or `worker` | a core config passed to `new Resizer` holds image settings only — move timing to the task queue's `timing`; `worker.concurrency` (in a framework config file too) moves to the top-level `concurrency` | +| `RESIZE_MONGO_MODEL_MISSING` from `verify()` or at worker start | a model does not resolve: the `ResizeTask` model (scaffold `src/models/ResizeTask.ts`), the lock model (the framework's `Lock`, or the `lockModel` passed to `MongoDatabase`), or the media model of a plain `MongoDatabase` (framework hosts get `RESIZE_CONFIG_MEDIA_MODEL_UNKNOWN` instead) | +| `RESIZE_MONGO_MODEL_OUTDATED`, or `resize-scaffold --check` reports drift in `src/models/ResizeTask.ts` | a shim from an older scaffold that imports `…/framework.js`: delete `src/models/ResizeTask.ts` and re-run `npx resize-scaffold` (`--force` would also overwrite `src/resizer.ts` and `src/config/resize.ts`). An ejected or hand-written model without the `resizer`, `queue`, `requestKey` or `availableAt` fields: port them (and the `{ queue, status, availableAt }` index) from `resizeTaskFields()` / `resizeTaskIndexes` (`…/drivers/mongo.js`), or delete it and re-run `npx resize-scaffold --eject` | +| `RESIZE_MONGO_MEDIA_MODEL_OUTDATED` from `verify()` or at worker start | the media schema declares preview rows as sub-documents without `identity` (a mixed or plain `previews` array passes) — spread the current `resizeMediaSchemaFragment`, or add `identity: { type: String }` to the row | +| `RESIZE_SVG_RENDER_UNAVAILABLE` | the SVG render process cannot start — allow child processes (`--allow-child-process` under Node's permission model) and keep `svgRasterChild.js` next to `svgRaster.js` in a bundle | +| `RESIZE_CONFIG_QUEUE_TIMING_CONFLICT` | two config files share one task queue but set different timing (the message names both files and the keys) — set the same values in both | +| a variant stays missing for ~10 min after its task was dead-lettered | `lockTtlMs.failed` (default 600000): reads do not queue a dead task's previews again until it expires — fix the cause in the logs first | | `RESIZE_MONGO_MODEL_REQUIRED` | `new MongoDatabase()` / `new MongoTaskQueue()` needs a model or a getter — or use `mongoDatabase(connection, { mediaModel })` | | `RESIZE_CONFIG_MEDIA_MODEL_UNKNOWN` at worker start | `mediaModelName` does not match a registered host model — fix the name | | `RESIZE_CONFIG_REMOVED_KEY` | a 0.2.x key is still in `resize.ts` / `resize..ts` — move it to the path named in the message | -| `formats [...] have no encode.formats entry` | add `encode.formats.` (`{}` for Sharp defaults); use `'jpeg'`, not the alias `'jpg'` | -| `ERR_MODULE_NOT_FOUND: @aws-sdk/...` at your driver import | optional peer not installed — see step 1 | +| `formats [...] have no encode.formats entry`, `RESIZE_FORMAT_NOT_CONFIGURED` | add `encode.formats.` (`{}` for Sharp defaults), or request only configured ids; use `'jpeg'`, not the alias `'jpg'` | +| `RESIZE_PIPELINE_UNKNOWN` | the pipeline is not registered in that process — register it in `src/resizer.ts`; deploy the worker before the API requests a new or renamed pipeline | +| `ERR_MODULE_NOT_FOUND: @aws-sdk/...` at your driver import, or `RESIZE_PEER_MISSING` for a driver selected in the config | optional peer not installed — install the packages named, see step 1 | | `a Resizer named '…' already exists` | each name is constructed once per process — import the single construction site; elsewhere `getResizer(name)` | | `RESIZE_NO_RESIZER` at worker start | `src/commands/ResizeWorker.ts` does not import `../resizer.ts` — delete it and re-run `npx resize-scaffold` | | `RESIZE_NO_RESIZER` in worker logs for a task | the worker process did not construct that Resizer — construct every Resizer in `src/resizer.ts`, which both the API and the worker load | diff --git a/CHANGELOG.md b/CHANGELOG.md index 741565a..848f488 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,4 @@ -# Unreleased - -Pending changes since 0.2.1. The release version will be chosen when these changes are ready. +# 0.3.0 **Breaking changes** @@ -16,8 +14,27 @@ Pending changes since 0.2.1. The release version will be chosen when these chang `bucketPrivate` differs from `bucketPublic`; `LocalFsStorage` keeps private files under `privateRootDir` (default `-private`). - SVG is an input format only. `uploadOriginal()` stores SVG privately and rejects public SVG - uploads; the worker rasterizes accepted SVG into the configured preview formats through the - normal task lifecycle. `resolve()` never returns uploaded SVG markup. + uploads. The worker renders an SVG once per task, before any pipeline step, into a PNG, in a + separate Node process, at the largest size the requested previews need; `beforeSteps` receive + that PNG, never SVG markup, and every format is made from it. The render is killed after + `limits.processingTimeoutSeconds` (`ResizeMediaError` `RESIZE_SVG_RENDER_TIMEOUT`, the task is + dead-lettered at once) or when the task is aborted; a render process that crashes or produces no + image is `RESIZE_SVG_RENDER_FAILED` (retried). A render that cannot start is `ResizeSetupError` + `RESIZE_SVG_RENDER_UNAVAILABLE`: the host must allow child processes (`--allow-child-process` + under Node's permission model) and ship `svgRasterChild.js` next to `svgRaster.js` (bundles). + Eager `generate()` throws all three. +- The original is never served. `resolve()` no longer serves an original that fits the requested + box: `decision.ready` holds stored previews only, every `ReadyEntry` has `preview` and + `contentType`, and `isOriginal` is gone. For a pipeline without `variantSteps`, a raster + original no larger than the requested width and height gets a normal preview at its own size + (no upscaling, no cropping, no sharpening, metadata such as EXIF and GPS removed; one larger than + `limits.resultDimension` is scaled down inside the cap) in every requested format; before, it + was served as it was or upscaled and cropped to the box. A pipeline with `variantSteps` still + gets the full requested box, the size its steps were written for. `ctx.isOwner` / + `ctx.isAdmin` no longer change what `resolve()` returns (hooks still receive `ctx`). + `canServeOriginalPublicly` is removed from `ResizeStorage` and both shipped drivers; the module + never calls `signedUrl`: to give an owner the private original, call + `storage.signedUrl(original.storageRef, ttlSeconds)` yourself. - The module no longer merges host config over its defaults and drops the `deepmerge` dependency. The framework's `resize.ts` + `resize..ts` merge is the only merge: the host `src/config/resize.ts` spreads `defaultFrameworkResizeConfig` from @@ -27,7 +44,7 @@ Pending changes since 0.2.1. The release version will be chosen when these chang - Config keys moved. The old keys now fail validation with `RESIZE_CONFIG_REMOVED_KEY` instead of being ignored: - | 0.2.x | Unreleased | + | 0.2.x | 0.3.0 | |---|---| | `webpAvifOnly: true` | `formats: ['webp', 'avif']` | | `encode.quality.` | `encode.formats..quality` | @@ -50,13 +67,13 @@ Pending changes since 0.2.1. The release version will be chosen when these chang framework adapter fills them from the app, read on first use. - Drivers are three contracts, exported as abstract classes: a custom driver `extends` one, or is any object of the same shape. The 0.2 media store, lock provider and queue transport are gone. - - `ResizeStorage` (files): `upload`, `download`, `publicUrl`, optional `signedUrl` and - `canServeOriginalPublicly`. + - `ResizeStorage` (files): `upload`, `download`, `publicUrl`, optional `signedUrl`. - `ResizeDatabase` (records): `loadMedia`, `appendPreviews`, `acquireLock`, `releaseLock`, and - optionally `tasks` (its own queue) and `verify()`. + optionally `tasks` (its own queue) and `verify()`. `appendPreviews` stores a preview only when + its `identity` is not stored yet and resolves with the previews it stored. - `TaskQueue` (queued tasks): atomic `add`, `claim`, `renew`, `complete` and `fail`, and optionally `findActive`, `servesQueue`, `getTiming` and `verify`. -- The core owns the queue logic. Its worker loop (`consumeQueue`, run by `runWorker`) handles +- The core owns the queue logic. Its worker loop (run by `runWorker`) handles leases and the heartbeat, the task timeout, retry with backoff, dead-lettering (after `maxAttempts`, or at once for a media without an original), the request de-duplication key and the `completed` / `failed` / `deadLettered` events, identically for every queue. Tasks carry their @@ -86,17 +103,35 @@ Pending changes since 0.2.1. The release version will be chosen when these chang to `deadLetterQueueUrl` (when set) and `onTaskDeadLettered` fires for SQS too. - Re-run `resize-scaffold` after deleting the old model and command shims. - The Mongo `requestKey` is now `v2` and covers the Resizer name and the queue. `ResizeTask` gains - `resizer` and `queue` fields, and its lease index becomes `{ queue, status, createdAt }`; create - the new index through your migration. Rows without the new fields read as `'default'`. -- `processTask(task)` runs the task with the Resizer named in it; an unknown name rejects with + `resizer`, `queue` and `availableAt` (when the task may next be claimed) fields. Rows without + `resizer`/`queue` read as `'default'`. Rows without `availableAt` (written before it existed, + or by an older worker during a rolling deploy) still wait for the retry time or lease end in + their `leaseExpiresAt`; a backfill is optional. +- The claim index `{ queue: 1, status: 1, availableAt: 1 }` replaces `{ status: 1, createdAt: 1 }` + (and the pre-release `{ queue: 1, status: 1, createdAt: 1 }`), and workers claim the task that + became due first. Create it through your migration before or + together with the new workers, then drop the old index. +- The worker runs each task with the Resizer named in it; an unknown name rejects with `RESIZE_NO_RESIZER` (the task retries, then dead-letters). -- Preview identity includes the Resizer and pipeline: `getPreviewIdentity(scope, sizeKey, format, - filters)` returns `resizer:pipeline:sizeKey:format:filterSig`. Generated preview rows store - `resizer` and `pipeline` (rows without them belong to `default`), so different pipelines and - Resizers keep separate previews of the same media instead of sharing whichever was generated - first. Dispatch and worker lock keys change accordingly. `expandMissingPreviews` and - `expandPreviewRequests` take a scope; `isCatalogCovered` takes an optional one. The - "use distinct filters per pipeline" workaround is no longer needed. +- Preview identity includes the Resizer and pipeline: `resizer:pipeline:sizeKey:format:filterSig`. + Generated preview rows store `resizer` and `pipeline` (rows without them belong to `default`), so + different pipelines and Resizers keep separate previews of the same media instead of sharing + whichever was generated first. Dispatch and worker lock keys change accordingly. + `isCatalogCovered` takes an optional scope. The "use distinct filters per pipeline" workaround is + no longer needed. +- One preview row per identity. Every new preview row has an `identity` string, and + `resizeMediaSchemaFragment` includes `previews[].identity`; the database stores a preview only + when its identity is not stored yet. A preview that another worker stored first is logged as a + warning with its storage ref and does not count as created. A hand-written media schema that + declares preview rows as sub-documents must add `identity: { type: String }` to them: + `MongoDatabase.verify()` (so `resizer.verify()` and worker start) throws `ResizeSetupError` + `RESIZE_MONGO_MEDIA_MODEL_OUTDATED` without it (a mixed or plain `previews` array passes). Rows + written earlier have no identity and are not migrated. +- `MongoDatabase` writes previews and the backfilled dimensions with `findOneAndUpdate`, one write + per preview plus one for the backfill, so a `findOneAndUpdate` middleware on the media model runs + for each; the document it receives has only `_id`, and is `null` when the preview was already + stored or the media is gone. A model-shaped `MongoMediaModel` needs `findById` and + `findOneAndUpdate` (no longer `findByIdAndUpdate`). - The framework is an adapter. The main entry imports no framework code; framework wiring lives in `@adaptivestone/framework-module-resize/framework.js`. `new FrameworkResizer({ pipelines, hooks })` (a `Resizer` subclass) builds every part from the config file on first use: the image @@ -105,6 +140,11 @@ Pending changes since 0.2.1. The release version will be chosen when these chang events. Options win over the config (`storage`, `db`, `tasks` — a `TaskQueue` or `false` —, `config`, `configName`, `logger`, `events`). The main entry no longer exports `ResizeWorker`, `ResizeTaskModel`, `runResizeWorker` or the `TResizeTask` type. +- A smaller public surface. The main entry no longer exports `calculateResizedDimensions`, + `getFilterSig`, `getImageContentType`, `getPreviewIdentity`, `processTask`, `getResizeConfig`, + `defaultResizeConfig` (use the default export of `…/config/resize.js`) or `requiredFormats`. + `runWorker` is the worker entry. `…/framework.js` exports `FrameworkResizer`, + `FrameworkDatabase`, `ResizeTaskModel`, `ResizeWorker`, `runResizeWorker` and the config types. - The scaffolded `src/resizer.ts` holds behaviour only (`new FrameworkResizer({ pipelines })`); storage and the queue live in `src/config/resize.ts`. `--eager` emits the same construction site with a config without `queue`. @@ -121,16 +161,55 @@ Pending changes since 0.2.1. The release version will be chosen when these chang - `queue`: `{ driver: 'database' }` or `{ driver: 'sqs', queueUrl, … }` with any timing keys (the rest default). Missing or `false` means eager only; `defaultFrameworkResizeConfig` no longer contains `queue`, so add `queue: { driver: 'database' }` for background generation. - - `getResizeConfig()` returns `{ image, mediaModelName, storage, queue, timing, worker }`. `ResizeConfig` no longer contains `mediaModelName`; `FrameworkResizeConfig` does. - `prewarm()` reports every requested variant: `{ status, ready, accepted, notRequired, unconfirmed, tasks, issues }` (`PrewarmResult`), instead of an `{ enqueued }` count, and still - never throws (an internal error is `incomplete` with a `RESIZE_ENQUEUE_INTERNAL_ERROR` issue). A + never throws (an internal error is `incomplete` with a `RESIZE_ENQUEUE_INTERNAL_ERROR` issue that + lists the variants requested so far). A held lock is not treated as a task receipt: a task queue with the optional `TaskQueue.findActive()` (the Mongo queue) proves exact canonical active-payload coverage, while one without it (SQS) reports lock races as retryable `incomplete`. Conflicting payloads with one preview identity are explicit - errors. There is no separate strict method: the pre-release `enqueueRequired()` is merged into - `prewarm()`. + errors. Issues describe only the variants that remain unconfirmed. There is no separate strict + method: the pre-release `enqueueRequired()` is merged into `prewarm()`. +- An unregistered pipeline name (anything but `'default'`, which always exists) is no longer + rendered as an empty pipeline. `resolve()` logs an error, serves only previews already stored + for that pipeline, reports nothing missing and queues nothing; `prewarm()` returns `incomplete` + with one non-retryable `RESIZE_PIPELINE_UNKNOWN` issue; `generate()` and the worker throw + `ResizeSetupError` (`RESIZE_PIPELINE_UNKNOWN`), so a queued task retries, then dead-letters. + Register a new or renamed pipeline in the worker process before the API requests it (deploy + workers first). New: `resizer.hasPipeline(name)`. +- Per-call `formats`, and the variants a `beforeEnqueue` tap adds or rewrites, must use keys of + `encode.formats`. `resolve()` leaves the others out of `decision.missing` and logs an error, + `prewarm()` reports them in `unconfirmed` with the non-retryable issue + `RESIZE_FORMAT_NOT_CONFIGURED`, and `generate()` throws `ResizeSetupError` + (`RESIZE_FORMAT_NOT_CONFIGURED`) before any hook runs. +- `RESIZE_SOURCE_TOO_LARGE` and `RESIZE_SOURCE_METADATA_MISSING` dead-letter a task on the first + failure, like `RESIZE_NO_ORIGINAL`: no retry can fix them. +- `worker.concurrency` in a framework config file fails with `RESIZE_CONFIG_REMOVED_KEY`; move it to + the top-level `concurrency`. +- The scaffolded `src/models/ResizeTask.ts` imports the new + `@adaptivestone/framework-module-resize/framework/ResizeTaskModel.js` export (the model class as + its default export), which lets the framework's `npm run gen` type `getModel('ResizeTask')`. + `resize-scaffold --check` reports a shim that still imports `…/framework.js` as drift: delete + `src/models/ResizeTask.ts` and re-run `npx resize-scaffold` (`--force` would also overwrite + `src/resizer.ts` and `src/config/resize.ts`). +- An ejected or hand-written `ResizeTask` model must declare the `resizer`, `queue`, `requestKey` + and `availableAt` fields (and the `{ queue, status, availableAt }` index); Mongoose would drop + undeclared fields, filing every task under Resizer and queue `'default'` and losing retry + times. `resize-scaffold --check` reports such a model, and `MongoTaskQueue.verify()` (so + `resizer.verify()` and worker start) throws `ResizeSetupError` `RESIZE_MONGO_MODEL_OUTDATED`. +- One task queue per backend in a framework app: every `FrameworkResizer` and `FrameworkDatabase` + on the database queue shares one task queue, and so do Resizers whose config selects SQS with + the same settings. The worker runs one loop per queue, one task at a time, each task processed + with its own Resizer. Timing belongs to the queue: config files that share one and set different + timing (for SQS also an effective `waitTimeSeconds`; unset counts as the default 10) fail on + first use, in `verify()` and at worker start + with `ResizeConfigError` `RESIZE_CONFIG_QUEUE_TIMING_CONFLICT`, naming both files and the keys. + Without the framework, each `mongoDatabase()` call makes its own queue: pass one database's + `tasks` to every Resizer. +- The default JPEG quality is 88 (was 80). +- A task's `leasedBy` is `:` instead of `resizer-`, so pods with the same pid + are told apart. **Features** @@ -144,8 +223,9 @@ Pending changes since 0.2.1. The release version will be chosen when these chang without code changes. - Lazy drivers: `storage`, `db` and `tasks` may be functions (sync or async), called once on first use. `resizer.ready()` loads them; every Resizer method and the worker await it, and `resolve()` - / `prewarm()` keep their never-throw guarantee when loading fails. Reading `resizer.storage` / - `db` / `tasks` before then throws `RESIZE_NOT_READY`. + / `prewarm()` keep their never-throw guarantee when loading fails. When one part fails to load, + the next call retries only that part. Reading `resizer.storage` / `db` / `tasks` before then + throws `RESIZE_NOT_READY`. - Named queues. A Resizer has a default `queue` (default `'default'`), and `resolve()` and `prewarm()` accept a per-call `queue`. `npm run cli ResizeWorker -- --queue=` consumes only that queue; without the flag it consumes `'default'`. `SqsTaskQueue` maps queue names to @@ -154,7 +234,7 @@ Pending changes since 0.2.1. The release version will be chosen when these chang named in it. It runs one loop per distinct task queue that serves the queue (a queue whose `servesQueue()` says no is skipped; `RESIZE_QUEUE_NOT_SERVED` when none does; if one loop fails, the others stop and the worker rejects with that error), and every Resizer's `verify()` runs - before claiming. `listResizers()` lists the registered Resizers. + before claiming. - The framework adapter reads the app lazily: config, models, logger and events are read on first use, so `src/resizer.ts` can be imported statically anywhere, even before `Server.init()`. The new `resizer.verify()` checks at boot what would otherwise only be logged per call: the config, the @@ -169,11 +249,30 @@ Pending changes since 0.2.1. The release version will be chosen when these chang create indexes; prepare them through the host's migration or lifecycle before rollout. - Queued raster tasks retry only the identities still missing after partial generation. Successful previews stay persisted, and permanent gaps use the normal backoff and dead-letter - path. Deleted media tasks remain successful no-ops. + path. Deleted media tasks remain successful no-ops; previews uploaded for media deleted during + the task are logged as a warning that the media no longer exists. - `ResizeDatabase` and `TaskQueue` have an optional `verify()` startup check. `Resizer.verify()` and the worker await them once before claiming tasks, so custom drivers can fail fast too; - `MongoDatabase.verify()` checks the media and lock models, and `FrameworkDatabase.verify()` + `MongoDatabase.verify()` checks the media model (including `previews.identity`) and the lock + model, and `FrameworkDatabase.verify()` checks that `mediaModelName` names a registered model and that the framework `Lock` model exists. +- Worker shutdown gives the stopped task back. The variants in progress finish and are saved, and + the task returns to the queue with no `onTaskFailed` / `onTaskDeadLettered` event and no + backoff, through the new optional `TaskQueue.release(task)`; a terminal media error + (`RESIZE_NO_ORIGINAL`, `RESIZE_SOURCE_METADATA_MISSING`, `RESIZE_SOURCE_TOO_LARGE`) is still + dead-lettered with its event. `MongoTaskQueue` does not count the attempt; `SqsTaskQueue` makes + the message visible again but still counts the delivery (SQS counts receives); a custom queue + without `release` is retried at once through `fail`, which also counts. +- `npm run cli ResizeWorker -- --config=` picks the config file whose `worker` section (the + `enabled` switch and Sharp tuning) the worker process uses; default `resize`. A host with only + named config files must pass it. +- A driver selected in a framework config file whose optional AWS SDK peer is not installed fails + at first use or in `verify()` with `ResizeSetupError` `RESIZE_PEER_MISSING`, naming the packages + and the config file. +- New timing key `lockTtlMs.failed` (default 600000 ms): after a task is dead-lettered, the worker + holds its variants' dispatch locks for this long, so `resolve()` and `prewarm()` do not queue + the same failing work for that media again until it expires. An invalid value is + `ResizeConfigError` `RESIZE_CONFIG_QUEUE_LOCK_TTL_INVALID`. **Fixes** @@ -189,13 +288,45 @@ Pending changes since 0.2.1. The release version will be chosen when these chang - Coverage builds the package before integration tests. Source-only test runs skip the Framework config integration test with a build instruction when the compiled config is absent. - Host adoption documentation uses a generic checklist without internal project names or paths. -- The resolved config object is validated once instead of on every `getResizeConfig()` call, so - the read path no longer re-runs full validation per `resolve()`. +- The resolved framework config is validated once instead of on every read, so the read path no + longer re-runs full validation per `resolve()`. - Optional peer ranges state what the code needs instead of `*`: `@adaptivestone/framework` `^5.1.0`, `mongoose` `^9.0.0`, and the AWS SDK clients `^3.572.0`. `SqsTaskQueue` reads attempt counts from `MessageSystemAttributeNames`, which older `@aws-sdk/client-sqs` versions silently drop: every delivery then counted as attempt 1, so a failing task was never dead-lettered. It now also warns once when SQS returns no `ApproximateReceiveCount`. +- `animated: true` works. WebP and GIF previews keep the animation, up to + `limits.animationFrames` frames, for a pipeline without `variantSteps`; every other format, and + every preview of a pipeline with variant steps (a watermark would land on one frame), gets the + first frame instead of all frames stacked into one tall image. `actualWidth` / `actualHeight` + are the size of one frame. `limits.sourcePixels` applies to one frame, and an animation with + more frames than `limits.sourcePixels` or `limits.inputPixels` allow is shortened to the frames + that fit instead of failing. An animation with an EXIF orientation is rendered from its first + frame. +- No output side of a width-only or height-only size exceeds `limits.resultDimension`: when the + side derived from the aspect ratio would, the image is cropped to the cap. +- Fractional size dimensions are rounded in the size key, the task and the stored preview; a + dimension that rounds to 0 is ignored; a `fit` result is at least 1 px per side. +- An EXIF-rotated original is no longer re-encoded before resizing when the pipeline has no + `beforeSteps` (no quality loss, faster). With `beforeSteps`, the steps still get upright pixels, + re-encoded in the same format without visible loss. +- Very long or thin SVG originals render instead of failing at librsvg's size limit. +- When recording previews fails after their upload, the error log names the uploaded files that + no stored row points to (and only those). +- A storage `publicUrl` that throws for one stored preview leaves out only that size and format; + the rest of the read and the enqueue continue. +- `S3Storage.publicUrl` with `forcePathStyle: true` and no `endpoint` returns + `https://s3..amazonaws.com//` instead of a relative path, and China + regions (`cn-…`) use the `amazonaws.com.cn` domain in both URL forms. +- `S3Storage` uploads `image/svg+xml` objects with `ContentDisposition: attachment`. +- Filters with an own `__proto__` key (as `JSON.parse` creates) keep their own preview identity. +- The dispatch locks of one read or pre-warm are acquired concurrently, not one after another. +- `package.json` points to the right repository (`adaptivestone/framework-module-resizer`). + +**Internal** + +- A new CI job, `min-versions`, runs the type check, build and tests on the oldest supported Node + major (the newest 24.x release) with every peer at the lowest version its range allows. # 0.2.1 diff --git a/README.md b/README.md index 4875bdd..08abce2 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,9 @@ driver, and turns the record on your media document into image URLs. | **Lazy**: previews only for sizes readers request | `resolve()` with a task queue | + a task queue and a worker | All three write the same `previews[]` and read URLs with `resolve()`, so you can mix them or -switch later without migrating data. `sharp` never runs on the read path. +switch later without migrating data. `sharp` never runs on the read path, and the original is +never served: for a pipeline without `variantSteps`, a raster original no larger than a `WxH` +size gets a preview at its own size (no upscaling, no cropping, metadata removed). ## Install @@ -37,9 +39,11 @@ part that uses it: | S3: `storage: { driver: 's3' }` or `S3Storage` (`…/drivers/s3.js`) | `@aws-sdk/client-s3` `@aws-sdk/s3-request-presigner` ≥ 3.572 | | SQS: `queue: { driver: 'sqs' }` or `SqsTaskQueue` (`…/drivers/sqs.js`) | `@aws-sdk/client-sqs` ≥ 3.572 | -A missing optional peer fails at your own import line (or, for a driver chosen in the config, when -the Resizer first loads it), not at the first upload. An older SQS client does not return receive -counts, so failing tasks would never be dead-lettered; `SqsTaskQueue` warns if that happens. +A missing optional peer fails at your own import line when you import a driver subpath. A driver +chosen in the config fails when the Resizer first loads it (the first call, or `verify()`) with +`ResizeSetupError` `RESIZE_PEER_MISSING`, which names the packages and the config file. An older +SQS client does not return receive counts, so failing tasks would never be dead-lettered; +`SqsTaskQueue` warns if that happens. ## Quick start (framework, eager) @@ -110,7 +114,11 @@ const picture = formatPictureUrls(decision, { id: String(file.id) }); The scaffolded command is `import '../resizer.ts'` plus a re-export of the module's command, so the worker has the same Resizers as the API. Keep that import, and run -`npx resize-scaffold --check` in CI to catch drift. +`npx resize-scaffold --check` in CI to catch drift (`--check --eager` for an eager app). The model +shim extends the default export of `…/framework/ResizeTaskModel.js`, which lets `npm run gen` type +`getModel('ResizeTask')`; `--check` reports a shim from an older scaffold (fix: delete +`src/models/ResizeTask.ts` and re-run `npx resize-scaffold`; `--force` would also overwrite +`src/resizer.ts` and `src/config/resize.ts`). ## Without the framework @@ -124,7 +132,8 @@ import defaultResizeConfig from '@adaptivestone/framework-module-resize/config/r import { LocalFsStorage } from '@adaptivestone/framework-module-resize/drivers/fs.js'; import { mongoDatabase } from '@adaptivestone/framework-module-resize/drivers/mongo.js'; -// Media, locks and the task queue, with the package's ResizeTask / ResizeLock models +// Media, locks and the task queue, with the package's ResizeTask / ResizeLock models. Call it +// once: each call makes its own task queue, so pass this db.tasks to every Resizer. const db = mongoDatabase(mongoose.connection, { mediaModel: File }); // File spreads resizeMediaSchemaFragment export const resizer = new Resizer({ @@ -138,20 +147,22 @@ export const resizer = new Resizer({ await runWorker({ signal, queue: 'default', sharp: { concurrency: 1, cache: false } }); ``` -Create the indexes through your migration process (for example `ResizeTask.createIndexes()`); -the module never creates them at runtime (`createResizeModels` sets `autoIndex: false`). +Create the indexes of both models through your migration process (for example +`await mongoose.connection.models.ResizeTask.createIndexes()`, and the same for `ResizeLock`); the +module never creates them at runtime (`createResizeModels` sets `autoIndex: false`). ## Package exports | Import | Contents | |---|---| -| `@adaptivestone/framework-module-resize` | `Resizer`, `getResizer`, `listResizers`, `runWorker`, `consumeQueue`, the contracts (`ResizeStorage`, `ResizeDatabase`, `TaskQueue`), helpers (`formatPictureUrls`, `isCatalogCovered`, `resizeMediaPaths`, `resizeMediaSchemaFragment`), errors, types | +| `@adaptivestone/framework-module-resize` | `Resizer`, `getResizer`, `resetResizerForTests`, `runWorker`, the contracts (`ResizeStorage`, `ResizeDatabase`, `TaskQueue`), helpers (`formatPictureUrls`, `getSizeKey`, `parseSizeKey`, `isCatalogCovered`, `resizeMediaPaths`, `resizeMediaSchemaFragment`), errors, types | | `…/config/resize.js` | Default config | | `…/drivers/fs.js` | `LocalFsStorage` | | `…/drivers/s3.js` | `S3Storage` | | `…/drivers/mongo.js` | `mongoDatabase`, `MongoDatabase`, `MongoTaskQueue`, `createResizeModels`, the schemas | | `…/drivers/sqs.js` | `SqsTaskQueue` | -| `…/framework.js` | Framework adapter: `FrameworkResizer`, `FrameworkDatabase`, `ResizeTaskModel`, `ResizeWorker`, `runResizeWorker`, `appLogger`, `appEvents`, `getResizeConfig`, `FrameworkResizeConfig` and its section types | +| `…/framework.js` | Framework adapter: `FrameworkResizer`, `FrameworkDatabase`, `ResizeTaskModel`, `ResizeWorker`, `runResizeWorker`, `FrameworkResizeConfig` and its section types | +| `…/framework/ResizeTaskModel.js` | `ResizeTaskModel` as the default export, for the scaffolded model shim | ## Drivers @@ -167,7 +178,8 @@ implement atomic operations. Drivers receive no `app` argument; each one uses it `FrameworkResizer` builds `FrameworkDatabase`: the app's media model, the framework's own `Lock` model, and the scaffolded `ResizeTask` model as its queue. Any part may also be a function (sync or async), called once on first use: `new Resizer({ storage: async () => …, db, tasks })`. -`await resizer.ready()` loads them; the Resizer's own methods and the worker do it for you. +`await resizer.ready()` loads them; the Resizer's own methods and the worker do it for you. When +one part fails to load, the next call retries only that part. **`LocalFsStorage`** @@ -191,13 +203,24 @@ symlinks into the public tree. The ref is `{ bucket, key, namespace? }`. Every read accepts only the two configured buckets, so a tampered `bucket` cannot reach another bucket. `publicUrl()` does no I/O, and public access is a -bucket policy, not a per-object ACL. +bucket policy, not a per-object ACL. Without `publicBaseUrl`, URLs are virtual-hosted +(`https://.s3..amazonaws.com/`), or path-style with `endpoint` or +`forcePathStyle` (`//`; without an endpoint, +`https://s3..amazonaws.com//`). China regions (`cn-…`) use +`amazonaws.com.cn` in both forms. SVG objects are uploaded with `Content-Disposition: attachment`. **`mongoDatabase(connection, { mediaModel, timing?, logger? })`** builds a `MongoDatabase` with the package's `ResizeTask` and `ResizeLock` models on that connection (registered with `autoIndex` off: create their indexes through your migration process). For other setups, `new MongoDatabase({ mediaModel | getMediaModel, lockModel | getLockModel, tasks })` and `new MongoTaskQueue({ model | -getModel, timing | getTiming, logger })` take the models directly. +getModel, timing | getTiming, logger })` take the models directly; a model-shaped media object +needs `findById` and `findOneAndUpdate`. Previews and the backfilled dimensions are written with +`findOneAndUpdate`, one write per preview plus one for the backfill, so the media model's +`findOneAndUpdate` middleware runs for each; the document it receives has only `_id`, and is +`null` when the preview was already stored or the media is gone. `verify()` fails with +`RESIZE_MONGO_MEDIA_MODEL_OUTDATED` when the media schema declares preview rows as sub-documents +without `identity` (a mixed or plain `previews` array passes), and with +`RESIZE_MONGO_MODEL_OUTDATED` when the `ResizeTask` schema lacks a field the queue writes. **`SqsTaskQueue`** @@ -209,7 +232,7 @@ getModel, timing | getTiming, logger })` take the models directly. | `timing` | optional | Queue timing (below); the lease is the message visibility timeout | | `waitTimeSeconds` | `10` | Long poll per claim (0–20) | | `region`, `endpoint`, `client` | optional | An existing `SQSClient`, or one built on first use | -| `logger` | `console` | Framework apps: `appLogger` | +| `logger` | `console` | A queue built from a framework config file gets the app logger | Credentials come from the AWS provider chain. Retries and dead-lettering are the core's, the same as for Mongo (attempts = `ApproximateReceiveCount`), and `onTaskDeadLettered` fires for SQS too. Use @@ -226,25 +249,39 @@ first holder's late release removes it, and at worst a variant is generated twic **Custom drivers** extend the exported abstract class, or are any object of the same shape: -- `ResizeStorage`: `upload`, `download`, `publicUrl` (pure, no I/O), and optionally `signedUrl` - and `canServeOriginalPublicly`. +- `ResizeStorage`: `upload`, `download`, `publicUrl` (pure, no I/O), and optionally + `signedUrl(ref, ttlSeconds)`. The module never calls `signedUrl`; a host calls it to give an + owner the private original. - The ref you return from `upload()` is opaque to the module and comes back unchanged. - `upload()` may receive a `namespace` hint (from `uploadOriginal`) or a `parentRef` (the original's ref, for previews). Never both. - `ResizeDatabase`: `loadMedia(id)`, `appendPreviews(id, previews, dims?)`, `acquireLock(key, ttlMs)` (resolves `true` when taken), `releaseLock(key)`; optionally `tasks` (its own queue) and `verify()`, awaited at startup — throw there to stop the worker before it claims anything. + - `appendPreviews` stores a preview only when its `identity` is not stored on the media yet, and + resolves with the previews it stored (nothing = all). The core logs a preview that another + worker stored first as a warning with its storage ref; it does not count as created and gets + no `onPreviewGenerated` event. - `TaskQueue` — each method one atomic operation: - `add(task)` stores `{ resizer, queue, mediaId, pipeline, previews, requestKey }`; an active task with the same `requestKey` may be returned instead. - - `claim(queue, leaseMs, signal?)` takes the oldest due task (a waiting one whose retry time has - passed, or one whose lease expired), increments `attempts` and returns a fencing `token`. It is - how a worker receives tasks: it may return `null` at once (the core polls every `idlePollMs`) - or wait for a task first (long poll, LISTEN/NOTIFY, a change stream). A waiting claim returns - once `signal` aborts, claims only when called (a prefetched task's lease would run out), and - treats a notification as a hint, since only one of the woken workers' claims wins. + - `claim(queue, leaseMs, signal?)` takes the task that became due first (a waiting one whose + retry time has passed, or one whose lease expired), increments `attempts` and returns a + fencing `token`. It is how a worker receives tasks: it may return `null` at once (the core + polls every `idlePollMs`) or wait for a task first (long poll, LISTEN/NOTIFY, a change + stream). A waiting claim returns once `signal` aborts, claims only when called (a + prefetched task's lease would run out), and treats a notification as a hint, since only one + of the woken workers' claims wins. - `renew(task, leaseMs)`, `complete(task)`, `fail(task, { retryAt } | 'dead', error)` act only while the token still holds (`false` = lease lost). + - Optionally `release(task)`: give a claimed task back unprocessed (`false` = lease lost). It + becomes claimable at once, and the delivery should not count as an attempt; a backend that + cannot take a delivery back may still count it (`MongoTaskQueue` takes the attempt back; + `SqsTaskQueue` cannot, since SQS counts receives). The worker calls it at shutdown for a task + it stopped, with no `onTaskFailed` / `onTaskDeadLettered` event and no backoff; a terminal + media error (`RESIZE_NO_ORIGINAL`, `RESIZE_SOURCE_METADATA_MISSING`, + `RESIZE_SOURCE_TOO_LARGE`) is still dead-lettered with its event. Without `release`, the core + retries the task at once through `fail`, which counts. - Optionally `findActive({ resizer, mediaId, pipeline })` (lets `prewarm()` confirm work queued by another request), `servesQueue(queue)`, `getTiming()` and `verify()`. @@ -260,25 +297,30 @@ created; a config function (the framework adapter passes one) on first use or `v | `upload.maxBytes` | `26214400` (25 MiB) | Largest accepted original | | `upload.formats` | `['jpeg', 'png', 'webp', 'avif', 'gif', 'svg']` | Accepted originals, detected from the bytes | | `maxSize` | `{ width: 2000, height: 1200 }` | Box for `fit` | -| `animated` | `false` | `true` keeps GIF/WebP frames | -| `encode.formats` | jpeg `{ quality: 80, mozjpeg: true, chromaSubsampling: '4:2:0' }`, webp `{ quality: 82, effort: 4 }`, avif `{ quality: 64, effort: 4 }` | Passed to `sharp.toFormat(id, options)`; `{}` keeps Sharp's defaults | +| `animated` | `false` | `true`: WebP and GIF previews of an animated original keep its frames, for a pipeline without `variantSteps`; other formats, pipelines with variant steps, and animations with an EXIF orientation get the first frame. `actualWidth`/`actualHeight` are one frame's size | +| `encode.formats` | jpeg `{ quality: 88, mozjpeg: true, chromaSubsampling: '4:2:0' }`, webp `{ quality: 82, effort: 4 }`, avif `{ quality: 64, effort: 4 }` | Passed to `sharp.toFormat(id, options)`; `{}` keeps Sharp's defaults | | `encode.sharpen` | `{ cover: true, fit: false }` | Mild sharpening after downscaling, or `false` | | `encode.flatten` | `{ formats: ['jpeg'], background: '#ffffff' }` | Formats whose transparency is flattened | | `limits.inputPixels` | `268402689` | Sharp decoder limit | -| `limits.sourcePixels` | `50000000` | Rejected before decoding | -| `limits.resultDimension` | `5000` | Largest output side for cropped sizes | -| `limits.animationFrames` | `64` | Frame limit for animated input | +| `limits.sourcePixels` | `50000000` | Largest frame, rejected before decoding | +| `limits.resultDimension` | `5000` | Largest output side for width/height sizes; when the side derived from the aspect ratio of a width-only or height-only size would exceed it, the preview is cropped to it | +| `limits.animationFrames` | `64` | Frame limit for animated input; an animation is also shortened to the frames that fit `sourcePixels` and `inputPixels` | | `limits.processingTimeoutSeconds` | `30` | Timeout per Sharp operation | | `concurrency` | `4` | Variants processed in parallel per task or `generate()` call | Queue timing and lock TTLs belong to the **task queue** (`timing` on `MongoTaskQueue`, `mongoDatabase` and `SqsTaskQueue`; `FrameworkResizer` reads them from the config file's `queue` -section). The core validates them on first use or in `verify()`. Their defaults are +section). In a framework app, every Resizer and `FrameworkDatabase` on the database queue shares +one task queue, and so do Resizers whose config selects SQS with the same settings; config files +that share a queue must set the same timing (and SQS `waitTimeSeconds`, where unset counts as the +default 10), or first use, `verify()` +and worker start fail with `ResizeConfigError` `RESIZE_CONFIG_QUEUE_TIMING_CONFLICT`, naming both +files and the keys. The core validates timing on first use or in `verify()`. Their defaults are `defaultQueueOptions` in `…/config/resize.js`: | Option | Default | Notes | |---|---|---| -| `lockTtlMs` | `{ dispatch: 60000, worker: 60000 }` | `worker` must be ≤ `leaseMs` | +| `lockTtlMs` | `{ dispatch: 60000, worker: 60000, failed: 600000 }` | `worker` must be ≤ `leaseMs`. `failed`: after a task is dead-lettered, `resolve()` and `prewarm()` do not queue its previews for that media again until it expires (`RESIZE_CONFIG_QUEUE_LOCK_TTL_INVALID` for a bad value) | | `leaseMs` | `60000` | Set to at least ~2× the slowest encode | | `retryBackoffMs` | `{ base: 5000, max: 300000 }` | Retry delay | | `maxAttempts` | `5` | Every lease counts, including reclaimed ones | @@ -303,7 +345,10 @@ does not merge again. Only `FrameworkResizer` reads the extra keys: region?, endpoint? }`, plus any of the timing options above (the rest default). Missing or `false`: eager only. - `worker`: `{ enabled: false, sharpConcurrency: 1, sharpCache: false }`, used by the - `ResizeWorker` command. `enabled` allows the command to run. + `ResizeWorker` command. `enabled` allows the command to run. The worker process reads `worker` + from `resize.ts`, whatever files its Resizers read; + `npm run cli ResizeWorker -- --config=` reads it from another file (needed when the app + has no `resize.ts`). A second Resizer can read its own file with `new FrameworkResizer({ name: 'listings', configName: 'resizeListings' })`. @@ -311,21 +356,28 @@ A second Resizer can read its own file with Without the framework, buckets, URLs and queue URLs are driver options. The 0.2.x keys `webpAvifOnly`, `encode.quality`, `encode.effort`, `encode.mozjpeg`, `encode.chromaSubsampling` and `encode.flattenBackground` fail with `RESIZE_CONFIG_REMOVED_KEY`, and so do `queue` and -`worker` in a core config. +`worker` in a core config and `worker.concurrency` in a framework config file (use the top-level +`concurrency`). ## Operations - **Task states (Mongo):** `pending → processing → completed`. - A failed attempt returns to `pending` with backoff. - - After `maxAttempts` the task is `dead`, and the lease never reclaims it. + - After `maxAttempts` the task is `dead`, and the lease never reclaims it. A task is `dead` at + once when no retry can help: the media has no original (`RESIZE_NO_ORIGINAL`), the source + has no dimensions or exceeds the pixel limits (`RESIZE_SOURCE_METADATA_MISSING`, + `RESIZE_SOURCE_TOO_LARGE`), or an SVG render ran past `limits.processingTimeoutSeconds` + (`RESIZE_SVG_RENDER_TIMEOUT`). - Completed rows expire after 24 h and dead rows after ~30 days (the `expireAfterSeconds` in the model). - **At-least-once delivery:** a task can run more than once. The worker skips previews that - already exist, so a repeat never duplicates them. A task completes only when every requested + already exist, and the database stores one row per preview identity, so a repeat never + duplicates them. A task completes only when every requested variant is stored. When an attempt is only partly successful, its previews are saved and the next attempt makes only the missing ones. -- **Retrying a dead task:** after fixing the cause, call `prewarm()` for that media. To replay the - row itself, reset it only when no active row has the same request. The partial unique index +- **Retrying a dead task:** after fixing the cause, call `prewarm()` for that media once the + `lockTtlMs.failed` cooldown has ended (until then it reports a retryable lock issue). To replay + the row itself, reset it only when no active row has the same request. The partial unique index allows one active copy: ```ts @@ -338,7 +390,7 @@ const active = await ResizeTask.findOne({ if (!active) { await ResizeTask.updateOne( { _id: row._id, status: 'dead' }, - { $set: { status: 'pending', attempts: 0, leaseExpiresAt: null } }, + { $set: { status: 'pending', attempts: 0, leaseExpiresAt: null, availableAt: new Date() } }, ); // a duplicate-key error (11000) means another operator re-queued it first } ``` diff --git a/RELEASE.md b/RELEASE.md index d8b934f..4afe800 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,9 +1,12 @@ # Releasing `@adaptivestone/framework-module-resize` Releases are **manual** (same as the sibling repos — `@adaptivestone/framework` and -`@adaptivestone/framework-module-email` ship without a publish workflow). CI (`.github/workflows/ci.yml`) -and the packaging smoke (`.github/workflows/packaging.yml`) gate every push/PR; publishing is a -human step run from a clean `main`. +`@adaptivestone/framework-module-email` ship without a publish workflow). CI +(`.github/workflows/ci.yml`: the latest Node with the locked dependencies, plus a `min-versions` +job on the oldest supported Node major, at its newest 24.x release, with every peer at its +declared minimum) and the packaging smoke +(`.github/workflows/packaging.yml`) gate every push/PR; publishing is a human step run from a +clean `main`. ## 1. Pre-flight gates (all must be green) @@ -35,9 +38,14 @@ Semver. Pre-1.0 (`0.x`), the minor is the breaking channel: Bump with `npm version ` (updates `package.json` + creates the git tag — see §4), or edit `version` by hand and tag manually. +> **Check `version` first.** For 0.3.0 no bump is needed: `package.json` already holds `0.3.0` +> and `CHANGELOG.md` has the `# 0.3.0` heading, so tag the release commit by hand. In general, +> compare `version` with `npm view @adaptivestone/framework-module-resize version`; if it is +> already ahead, do not run `npm version` on top of it (that would skip a version). + **Update `CHANGELOG.md` in the same commit as the bump.** Newest version first, `# X.Y.Z` -heading, with `**Breaking changes**` / `**Features**` / `**Internal**` groups — the format the -sibling `framework-module-email` uses. It is NOT in `files`, so it stays on GitHub and never +heading, with `**Breaking changes**` / `**Features**` / `**Fixes**` / `**Internal**` groups — +the format the sibling `framework-module-email` uses. It is NOT in `files`, so it stays on GitHub and never ships in the tarball (same as the sibling). Write it for a host developer deciding whether to upgrade: what breaks, what is new, and what they must change. @@ -70,10 +78,11 @@ all that ships. Tag the release commit `vX.Y.Z`. -> **Note:** the sibling repos actually tag **without** the `v` prefix (`framework` uses `5.3.1`, -> `framework-module-email` uses `2.0.0`) — an earlier version of this file claimed otherwise. This -> repo already published `v0.1.0` and `v0.2.0` with the prefix, so it keeps `v` for internal -> consistency rather than switching mid-stream. Use `v` here; don't "fix" it to match the siblings. +> **Note:** the sibling repos tag **without** the `v` prefix (`framework` uses `5.3.1`, +> `framework-module-email` uses `2.0.0`). This repo's existing tags are mixed: `v0.1.0` has the +> prefix, `0.2.0` does not, and 0.2.1 has no tag at all (`git tag` lists them). This file keeps +> the `v` prefix, npm's default; pick one convention before the next release and use it from +> then on. ```bash git tag v0.1.0 @@ -106,13 +115,19 @@ _Placeholder — the one check the automated smoke can't do: wire the published actual framework host and run it against real infra._ - [ ] In a real `@adaptivestone/framework` host: `npm i @adaptivestone/framework-module-resize`, - run `npx resize-scaffold`, fill the `storage` TODO in `src/resizer.ts` + `mediaModelName` - in `src/config/resize.ts`, `import ./resizer.ts` from `src/server.ts`. -- [ ] Confirm `npm run gen` (the framework's AST codegen) types the scaffolded - `class ResizeTask extends ResizeTaskModel {}`. -- [ ] Boot the server, upload an image, confirm a `resolve()` read enqueues and the - `ResizeWorker` command generates + persists previews (mongo transport + S3 storage). -- [ ] Confirm eager mode (`generate()`, no transport) on a host wired without a queue. + run `npx resize-scaffold`, and in `src/config/resize.ts` set `mediaModelName`, `storage` + (S3), `queue: { driver: 'database' }` and `worker.enabled: true`. `src/resizer.ts` stays + `new FrameworkResizer({ pipelines })`. Create the `ResizeTask` indexes through the host's + migration, and call `await resizer.verify()` after `server.init()`. +- [ ] `npx resize-scaffold --check` passes, and `npm run gen` (the framework's AST codegen) types + `getModel('ResizeTask')` through the scaffolded shim. +- [ ] Boot the server, upload an image, confirm a `resolve()` read queues the missing variants and + `npm run cli ResizeWorker` generates and persists the previews (database queue + S3 + storage). If the host uses SQS, repeat with `queue: { driver: 'sqs', … }`. +- [ ] Stop the worker during a task (SIGTERM): the task goes back to `pending` and the next + worker finishes it. +- [ ] Confirm eager mode (`generate()`, no `queue` in the config) on a host scaffolded with + `--eager`; `npx resize-scaffold --check --eager` passes. ## Notes diff --git a/docs/design/2026-09-30-multi-resizer.md b/docs/design/2026-09-30-multi-resizer.md index f8099b8..7aa9c66 100644 --- a/docs/design/2026-09-30-multi-resizer.md +++ b/docs/design/2026-09-30-multi-resizer.md @@ -272,11 +272,12 @@ published; the page still describes 0.2. **P5 — Later, each optional and separate.** - `worker.parallelTasks`. -- Merge `prewarm` into `enqueueRequired`. +- Merge `prewarm` into `enqueueRequired`. **Done:** `enqueueRequired` was merged into `prewarm`. - Strip archived-spec section references from comments. - Check the AVIF/WebP quality defaults on real photos. - Export a framework-free `ResizeTask` schema definition (fields + indexes). Today a host - without the framework copies them from `src/models/ResizeTask.ts`. + without the framework copies them from `src/models/ResizeTask.ts`. **Done:** + `resizeTaskFields` and `resizeTaskIndexes` (and `createResizeModels`) from `…/drivers/mongo.js`. ## 7. Non-goals diff --git a/docs/design/2026-10-05-database-queue-framework-wrapper.md b/docs/design/2026-10-05-database-queue-framework-wrapper.md index 009a84c..5e45f6f 100644 --- a/docs/design/2026-10-05-database-queue-framework-wrapper.md +++ b/docs/design/2026-10-05-database-queue-framework-wrapper.md @@ -1,6 +1,6 @@ # Database, queue and framework wrapper -Status: **proposal**, 2026-10-05. Follows +Status: **implemented** in pull requests #46 and #47 (proposed 2026-10-05). Follows [drivers and framework independence](./2026-10-02-drivers-and-framework-independence.md) (Q1–Q4, merged). Pre-release: breaking changes are fine; hosts migrate their own data. diff --git a/docs/host-adoption.md b/docs/host-adoption.md index 6746513..cacc3ea 100644 --- a/docs/host-adoption.md +++ b/docs/host-adoption.md @@ -26,3 +26,58 @@ Retain `minimize: false` on the media schema so empty objects inside opaque refs survive persistence. Framework `BaseModel` already supplies this default; direct Mongoose schemas must set it explicitly. The field fragment alone cannot configure the enclosing schema's options. + +## Other changes a host must make for this version + +`CHANGELOG.md` (`# 0.3.0`) has the details. + +- Construct the Resizer with `new FrameworkResizer({ pipelines, hooks })` in + `src/resizer.ts`, and move `storage` and `queue` into `src/config/resize.ts` + (`queue: { driver: 'database' }` for background generation). Move + `worker.concurrency` to the top-level `concurrency`. +- Create the `ResizeTask` indexes through the host's migration: the claim index + `{ queue: 1, status: 1, availableAt: 1 }` before or together with the new + workers, then drop the old lease index (`{ status: 1, createdAt: 1 }`, or + `{ queue: 1, status: 1, createdAt: 1 }` from a pre-release build). The partial + unique index on `{ fileId, pipeline, requestKey }` is required for + de-duplication. Backfilling `availableAt` is optional: rows without it still wait + for the retry time or lease end stored in `leaseExpiresAt`. +- Regenerate the model shim, which now imports `…/framework/ResizeTaskModel.js`: + delete `src/models/ResizeTask.ts` and re-run `npx resize-scaffold` (`--force` + would also overwrite `src/resizer.ts` and `src/config/resize.ts`). An ejected or + hand-written model must add the `resizer`, `queue`, `requestKey` and + `availableAt` fields and the claim index; `resize-scaffold --check` and + `resizer.verify()` report a model without the fields. +- A hand-written media schema that declares preview rows as sub-documents must add + `identity: { type: String }` to them (the fragment already has it); + `resizer.verify()` throws `RESIZE_MONGO_MEDIA_MODEL_OUTDATED` without it. Rows + written earlier have no identity and are not migrated. A custom model-shaped + `MongoMediaModel` needs `findById` and `findOneAndUpdate`. A `findOneAndUpdate` + middleware on the media model now runs once per preview write (and once for the + dimension backfill) and receives a document with only `_id`, or `null` when the + preview was already stored or the media is gone. +- The original is never served. Code that read `isOriginal`, or relied on + `ctx.isOwner` / `ctx.isAdmin` to get the original from `resolve()`, must change: + to give an owner the private original, call + `storage.signedUrl(original.storageRef, ttlSeconds)`. For a pipeline without + `variantSteps`, a small raster original now gets a normal preview at its own + size, made by the worker. +- SVG `beforeSteps` now receive the rendered PNG, not SVG markup. The render runs + in a child process: a host under Node's permission model needs + `--allow-child-process`, and a bundle must keep `svgRasterChild.js` next to + `svgRaster.js` (otherwise `RESIZE_SVG_RENDER_UNAVAILABLE`). +- Resizers on the database queue (and Resizers with identical SQS settings) share + one task queue with one timing: config files that share a queue must set the same + timing keys (and effective SQS `waitTimeSeconds`), or `verify()` and worker start + fail with + `RESIZE_CONFIG_QUEUE_TIMING_CONFLICT`. Without the framework, call + `mongoDatabase()` once and pass its `tasks` to every Resizer. +- A host whose Resizers read only named config files (no `resize.ts`) starts the + worker with `npm run cli ResizeWorker -- --config=`. +- Previews stored before preview identity included the Resizer and the pipeline + have no `resizer` or `pipeline` field and count as the `default` Resizer's + `default` pipeline. A named Resizer or pipeline generates its previews again, and a + `default` read may serve an old preview that was rendered by another pipeline; + remove such rows if that matters. +- Register every pipeline in `src/resizer.ts`, and deploy the worker before the API + requests a new or renamed pipeline: an unregistered pipeline is never rendered. diff --git a/package.json b/package.json index f83958a..afbfb65 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "./package.json": "./package.json", "./config/resize.js": "./dist/config/resize.js", "./framework.js": "./dist/framework/index.js", + "./framework/ResizeTaskModel.js": "./dist/framework/ResizeTaskModel.js", "./drivers/fs.js": "./dist/drivers/fs.js", "./drivers/s3.js": "./dist/drivers/s3.js", "./drivers/mongo.js": "./dist/drivers/mongo/index.js", @@ -23,7 +24,7 @@ }, "repository": { "type": "git", - "url": "git+https://github.com/adaptivestone/framework-module-resize.git" + "url": "git+https://github.com/adaptivestone/framework-module-resizer.git" }, "homepage": "https://framework.adaptivestone.com/docs/resize", "keywords": [ diff --git a/smokeTest.ts b/smokeTest.ts index f61a9f8..fe7a85b 100644 --- a/smokeTest.ts +++ b/smokeTest.ts @@ -38,11 +38,7 @@ const PKG = '@adaptivestone/framework-module-resize'; // (a) main entry: exactly the expected runtime exports, and no driver class leaks into it. const mod = await import(PKG); const expected = [ - 'calculateResizedDimensions', 'formatPictureUrls', - 'getFilterSig', - 'getImageContentType', - 'getPreviewIdentity', 'getSizeKey', 'isCatalogCovered', 'parseSizeKey', @@ -58,15 +54,11 @@ const expected = [ 'ResizeNoOriginalError', 'ResizeOriginalError', 'getResizer', - 'listResizers', - 'consumeQueue', - 'timingOf', 'Resizer', 'ResizeDatabase', 'ResizeStorage', 'TaskQueue', 'resetResizerForTests', - 'processTask', 'runWorker', ]; for (const name of expected) { @@ -133,6 +125,7 @@ const safe = [ ['/drivers/fs.js', 'LocalFsStorage'], ['/framework.js', 'FrameworkDatabase'], ['/framework.js', 'ResizeTaskModel'], + ['/framework/ResizeTaskModel.js', 'default'], ['/framework.js', 'ResizeWorker'], ['/framework.js', 'FrameworkResizer'], ['/framework.js', 'runResizeWorker'], @@ -142,6 +135,21 @@ for (const [sub, exp] of safe) { assert.ok(exp in m, sub + ' should export ' + exp); console.log(' ok ' + sub + ' imports (exports ' + exp + ')'); } +const taskModel = await import(PKG + '/framework/ResizeTaskModel.js'); +const framework = await import(PKG + '/framework.js'); +const expectedFramework = [ + 'FrameworkDatabase', + 'FrameworkResizer', + 'ResizeTaskModel', + 'ResizeWorker', + 'runResizeWorker', +]; +assert.deepEqual( + Object.keys(framework).sort(), + [...expectedFramework].sort(), + 'framework entry export surface drift', +); +assert.strictEqual(taskModel.default, framework.ResizeTaskModel); const { MongoTaskQueue, MongoDatabase } = await import(PKG + '/drivers/mongo.js'); for (const Driver of [MongoTaskQueue, MongoDatabase]) { assert.equal( @@ -224,8 +232,13 @@ import { FrameworkResizer, type FrameworkResizeConfig, } from '@adaptivestone/framework-module-resize/framework.js'; +import ResizeTaskModel from '@adaptivestone/framework-module-resize/framework/ResizeTaskModel.js'; import { defaultFrameworkResizeConfig } from '@adaptivestone/framework-module-resize/config/resize.js'; +// The scaffold imports the defining model file so framework codegen can detect its ancestor. +class ResizeTask extends ResizeTaskModel {} +void ResizeTask; + // The scaffolded config with its commented-out worker line enabled must stay a complete config. const hostConfig = { ...defaultFrameworkResizeConfig, diff --git a/src/config/resize.test.ts b/src/config/resize.test.ts index 42cfbc7..b339420 100644 --- a/src/config/resize.test.ts +++ b/src/config/resize.test.ts @@ -180,6 +180,24 @@ describe('getResizeConfig', () => { } }); + test('rejects worker.concurrency and names the config file and replacement', () => { + for (const value of [8, undefined]) { + install({ + ...makeResizeConfig(), + worker: { ...defaultWorkerOptions, concurrency: value }, + }); + assert.throws( + () => getResizeConfig('resizeListings'), + (error: unknown) => + error instanceof ResizeConfigError && + error.code === 'RESIZE_CONFIG_REMOVED_KEY' && + error.message.includes('`worker.concurrency`') && + error.message.includes('src/config/resizeListings.ts') && + error.message.includes('top-level `concurrency`'), + ); + } + }); + test('requires an encode.formats entry for every generated format', () => { // 'jpg' is a Sharp alias: it would encode JPEG without the 'jpeg' options or flatten. install(makeResizeConfig({ formats: ['jpg', 'webp'] })); @@ -244,6 +262,38 @@ describe('config split: core validation vs framework loading', () => { assert.strictEqual(getResizeConfig().worker, defaultWorkerOptions); }); + test('the dead-letter cooldown lockTtlMs.failed defaults to 10 minutes and is validated', () => { + assert.equal(defaultQueueOptions.lockTtlMs.failed, 600_000); + install( + makeResizeConfig({ + queue: { + driver: 'database', + lockTtlMs: { dispatch: 30_000, worker: 30_000 }, + }, + }), + ); + assert.deepEqual(getResizeConfig().timing.lockTtlMs, { + dispatch: 30_000, + worker: 30_000, + failed: 600_000, + }); + resetAppInstance(); + install( + makeResizeConfig({ + queue: { + driver: 'database', + lockTtlMs: { dispatch: 30_000, worker: 30_000, failed: -5 }, + }, + }), + ); + assert.throws( + () => getResizeConfig(), + (err: unknown) => + err instanceof ResizeConfigError && + err.code === 'RESIZE_CONFIG_QUEUE_LOCK_TTL_INVALID', + ); + }); + test('queue: false is eager only, and invalid queue timing is a config error', () => { install(makeResizeConfig({ queue: false })); assert.equal(getResizeConfig().queue, false); diff --git a/src/config/resize.ts b/src/config/resize.ts index a022bec..0bb996a 100644 --- a/src/config/resize.ts +++ b/src/config/resize.ts @@ -19,7 +19,7 @@ const defaultResizeConfig: ResizeConfig = { animated: false, encode: { formats: { - jpeg: { quality: 80, mozjpeg: true, chromaSubsampling: '4:2:0' }, + jpeg: { quality: 88, mozjpeg: true, chromaSubsampling: '4:2:0' }, webp: { quality: 82, effort: 4 }, avif: { quality: 64, effort: 4 }, }, @@ -37,14 +37,14 @@ const defaultResizeConfig: ResizeConfig = { }; /** Queue timing and lock TTL defaults (a task queue's missing timing values). */ -export const defaultQueueOptions: QueueTimingOptions = { - lockTtlMs: { dispatch: 60_000, worker: 60_000 }, +export const defaultQueueOptions = { + lockTtlMs: { dispatch: 60_000, worker: 60_000, failed: 600_000 }, leaseMs: 60_000, retryBackoffMs: { base: 5_000, max: 300_000 }, maxAttempts: 5, idlePollMs: 1_000, taskTimeoutMs: 600_000, -}; +} satisfies QueueTimingOptions; /** Framework worker command defaults. */ export const defaultWorkerOptions: FrameworkWorkerConfig = { diff --git a/src/contracts/database.ts b/src/contracts/database.ts index c5f9833..3173f37 100644 --- a/src/contracts/database.ts +++ b/src/contracts/database.ts @@ -8,12 +8,18 @@ export abstract class ResizeDatabase { /** Load a media document; `null` (deleted media) makes its task a logged no-op. */ abstract loadMedia(mediaId: string): Promise; - /** Append generated previews, and optionally backfill the original's dimensions, atomically. */ + /** + * Append generated previews, and optionally backfill the original's dimensions. A preview whose + * `identity` is already stored on the media is skipped, so two workers that rendered the same + * variant leave one row. Resolve with the previews that were stored; resolving with nothing means + * all of them were. + */ abstract appendPreviews( mediaId: string, previews: Preview[], backfillDims?: { width: number; height: number }, - ): Promise; + // biome-ignore lint/suspicious/noConfusingVoidType: a driver that resolves with nothing (Promise) stays valid + ): Promise; /** * Take the lock `key` for `ttlMs`: `true` if taken, `false` if someone holds it. Locks only diff --git a/src/contracts/storage.ts b/src/contracts/storage.ts index 2f20a74..a5013a1 100644 --- a/src/contracts/storage.ts +++ b/src/contracts/storage.ts @@ -19,15 +19,15 @@ export abstract class ResizeStorage { /** Upload a new object and return the JSON-compatible locator to persist. */ abstract upload(args: StorageUploadArgs): Promise; - /** Public URL of an object. Pure and synchronous: the read path calls it, so no I/O. */ + /** + * Public URL of an object. Pure and synchronous: the read path calls it for stored previews, + * so no I/O. It must refuse a ref it would not expose (e.g. a private original). + */ abstract publicUrl(ref: StorageRef): string; /** - * Optional, pure: whether an original may be served to anonymous readers. Without it every - * original is treated as private. + * Optional: a time-limited URL for a private object. The module never calls it; a host calls it + * itself, e.g. to hand a private original to its owner. */ - canServeOriginalPublicly?(ref: StorageRef): boolean; - - /** Optional: a time-limited URL for an owner/admin read of a private original. */ signedUrl?(ref: StorageRef, ttlSeconds: number): Promise; } diff --git a/src/contracts/taskQueue.ts b/src/contracts/taskQueue.ts index bcc2c57..cdd2072 100644 --- a/src/contracts/taskQueue.ts +++ b/src/contracts/taskQueue.ts @@ -1,8 +1,9 @@ // Task queue contract: where queued tasks wait. A database can hold them (MongoDatabase.tasks) or a // message broker can (SqsTaskQueue). Every method is ONE atomic operation; the core owns everything -// else — the worker loop, lease heartbeat, task timeout, retry with backoff, dead-lettering, request -// de-duplication keys and task events — so every backend behaves the same. Extend this class, or -// pass any object of the same shape (the core never checks `instanceof`). +// else — the worker loop, lease heartbeat, task timeout, retry with backoff, dead-lettering, giving +// tasks back at shutdown, request de-duplication keys and task events — so every backend behaves +// the same. Extend this class, or pass any object of the same shape (the core never checks +// `instanceof`). import type { EnqueueReceipt, MissingPreview, @@ -86,6 +87,14 @@ export abstract class TaskQueue { error: string, ): Promise; + /** + * Optional: give a claimed task back unprocessed (the worker is shutting down). It becomes + * claimable at once, and this delivery should not count as an attempt; a backend that cannot take + * a delivery back (SQS counts receives) may still count it. `false` means the lease was already + * lost. Without it the core retries the task at once through `fail`, so the delivery counts. + */ + release?(task: ClaimedTask): Promise; + /** * Optional: active tasks of this Resizer + media + pipeline on any queue, so prewarm can confirm * work another request already queued. Omit when tasks can't be queried (e.g. SQS). diff --git a/src/drivers/fs.test.ts b/src/drivers/fs.test.ts index eb9da8c..a228848 100644 --- a/src/drivers/fs.test.ts +++ b/src/drivers/fs.test.ts @@ -37,7 +37,6 @@ describe('LocalFsStorage', () => { await assert.rejects(() => readFile(join(publicRoot, '-private', key)), { code: 'ENOENT', }); - assert.equal(s.canServeOriginalPublicly(ref), false); assert.throws(() => s.publicUrl(ref), /private original/); } }); @@ -104,8 +103,8 @@ describe('LocalFsStorage', () => { await readFile(join(`${dir}-private`, 'originals/a.jpg')), Buffer.from('originals/a.jpg'), ); - assert.equal(s.canServeOriginalPublicly(privateRef), false); - assert.equal(s.canServeOriginalPublicly(publicRef), true); + // Visibility is enforced where a URL is made; the driver has no separate visibility check. + assert.equal('canServeOriginalPublicly' in s, false); assert.throws(() => s.publicUrl(privateRef), /private original/); assert.equal(s.publicUrl(publicRef), '/media/originals/a.jpg'); }); @@ -178,7 +177,6 @@ describe('LocalFsStorage', () => { }, ]) { await assert.rejects(() => s.download(ref)); - assert.throws(() => s.canServeOriginalPublicly(ref)); assert.throws(() => s.publicUrl(ref)); } }); diff --git a/src/drivers/fs.ts b/src/drivers/fs.ts index c81f7e6..2fd2e7a 100644 --- a/src/drivers/fs.ts +++ b/src/drivers/fs.ts @@ -185,8 +185,4 @@ export class LocalFsStorage extends ResizeStorage { } return `${this.#publicBaseUrl.replace(/\/+$/, '')}/${parsed.path}`; } - - canServeOriginalPublicly(ref: StorageRef): boolean { - return this.#ref(ref).visibility === 'public'; - } } diff --git a/src/drivers/mongo/database.mongo.integration.test.ts b/src/drivers/mongo/database.mongo.integration.test.ts new file mode 100644 index 0000000..7e78af9 --- /dev/null +++ b/src/drivers/mongo/database.mongo.integration.test.ts @@ -0,0 +1,366 @@ +// MongoDatabase media writes against a real MongoDB: one stored preview row per preview identity, +// however many workers render it, and the startup check for a media model without that field. +import assert from 'node:assert/strict'; +import { after, before, describe, test } from 'node:test'; +import { MongoMemoryServer } from 'mongodb-memory-server'; +import mongoose from 'mongoose'; +import { ResizeSetupError } from '../../errors.ts'; +import { resizeMediaSchemaFragment } from '../../mediaFragment.ts'; +import type { Preview } from '../../types.d.ts'; +import { createResizeModels, MongoDatabase } from './index.ts'; + +let server: MongoMemoryServer; +let connection: mongoose.Connection; + +before(async () => { + server = await MongoMemoryServer.create({ instance: { ip: '127.0.0.1' } }); + connection = await mongoose + .createConnection(server.getUri(), { + dbName: `resize_mongo_database_${process.pid}`, + }) + .asPromise(); +}); + +after(async () => { + await connection.close(); + await server.stop(); +}); + +const Media = () => + connection.models.IdentityMedia ?? + connection.model( + 'IdentityMedia', + new mongoose.Schema({ ...resizeMediaSchemaFragment }, { minimize: false }), + ); + +// A media model whose schema predates `previews.identity` (a hand-written or outdated fragment). +const OutdatedMedia = () => { + const { identity: _identity, ...previewFields } = + resizeMediaSchemaFragment.previews[0]; + return ( + connection.models.OutdatedMedia ?? + connection.model( + 'OutdatedMedia', + new mongoose.Schema( + { + original: resizeMediaSchemaFragment.original, + previews: [previewFields], + }, + { minimize: false }, + ), + ) + ); +}; + +// A media model whose `previews` path is declared as given (registered once per name). +const mediaModelWith = (name: string, previews: unknown) => + connection.models[name] ?? + connection.model( + name, + new mongoose.Schema( + { original: resizeMediaSchemaFragment.original, previews } as never, + { minimize: false }, + ), + ); + +function database(): MongoDatabase { + return new MongoDatabase({ + mediaModel: Media(), + lockModel: createResizeModels(connection).ResizeLock, + }); +} + +async function newMedia(previews: Preview[] = []): Promise { + const doc = await Media().create({ + original: { storageRef: { path: 'o.png' }, format: 'png' }, + previews, + }); + return String(doc._id); +} + +async function storedPreviews(mediaId: string): Promise { + const doc = (await Media().findById(mediaId).lean()) as { + previews?: Preview[]; + } | null; + return doc?.previews ?? []; +} + +function preview(identity: string | undefined, path = identity): Preview { + return { + storageRef: { path: `${path ?? 'none'}.webp` }, + ...(identity === undefined ? {} : { identity }), + sizeKey: '16x16', + format: 'webp', + contentType: 'image/webp', + } as Preview; +} + +const ID_A = 'default:default:16x16:webp:'; +const ID_B = 'default:default:32x32:webp:'; + +describe('MongoDatabase.appendPreviews: one row per preview identity', () => { + test('a second append of a stored identity leaves one row and resolves with nothing stored', async () => { + const db = database(); + const mediaId = await newMedia(); + const first = preview(ID_A, 'first'); + const second = preview(ID_A, 'second'); + + assert.deepEqual(await db.appendPreviews(mediaId, [first]), [first]); + assert.deepEqual(await db.appendPreviews(mediaId, [second]), []); + + const rows = await storedPreviews(mediaId); + assert.equal(rows.length, 1); + assert.equal(rows[0].identity, ID_A); + assert.deepEqual(rows[0].storageRef, { path: 'first.webp' }); + }); + + test('concurrent appends of one identity store exactly one row', async () => { + const db = database(); + const mediaId = await newMedia(); + const results = await Promise.all( + Array.from({ length: 8 }, (_, i) => + db.appendPreviews(mediaId, [preview(ID_A, `worker-${i}`)]), + ), + ); + + const rows = await storedPreviews(mediaId); + assert.equal(rows.length, 1); + // The one append that stored the row reports it; every other append reports nothing. + const stored = results.filter((r) => (r ?? []).length > 0); + assert.equal(stored.length, 1); + assert.deepEqual(stored[0]?.[0]?.storageRef, rows[0].storageRef); + }); + + test('different identities are all stored, and a repeat inside one call is stored once', async () => { + const db = database(); + const mediaId = await newMedia(); + const a = preview(ID_A); + const b = preview(ID_B); + + assert.deepEqual( + await db.appendPreviews(mediaId, [a, b, preview(ID_A, 'repeat')]), + [a, b], + ); + const rows = await storedPreviews(mediaId); + assert.deepEqual( + rows.map((r) => r.identity), + [ID_A, ID_B], + ); + }); + + test('a preview without an identity is always stored', async () => { + const db = database(); + const mediaId = await newMedia(); + const legacy = preview(undefined, 'legacy'); + + assert.deepEqual(await db.appendPreviews(mediaId, [legacy]), [legacy]); + assert.deepEqual(await db.appendPreviews(mediaId, [legacy]), [legacy]); + assert.equal((await storedPreviews(mediaId)).length, 2); + }); + + test('a row stored without an identity never matches a new one', async () => { + const db = database(); + const mediaId = await newMedia([preview(undefined, 'old-row')]); + const fresh = preview(ID_A, 'fresh'); + + assert.deepEqual(await db.appendPreviews(mediaId, [fresh]), [fresh]); + assert.equal((await storedPreviews(mediaId)).length, 2); + }); + + test('the dimension backfill is applied, also when every preview was skipped', async () => { + const db = database(); + const mediaId = await newMedia(); + await db.appendPreviews(mediaId, [preview(ID_A)]); + + assert.deepEqual( + await db.appendPreviews(mediaId, [preview(ID_A, 'late')], { + width: 64, + height: 48, + }), + [], + ); + const loaded = await db.loadMedia(mediaId); + assert.equal(loaded?.original?.width, 64); + assert.equal(loaded?.original?.height, 48); + assert.equal(loaded?.previews?.length, 1); + + const other = await newMedia(); + const stored = preview(ID_B); + assert.deepEqual( + await db.appendPreviews(other, [stored], { width: 10, height: 20 }), + [stored], + ); + const loadedOther = await db.loadMedia(other); + assert.equal(loadedOther?.original?.width, 10); + assert.equal(loadedOther?.original?.height, 20); + assert.equal(loadedOther?.previews?.length, 1); + }); + + test('media that no longer exists is a silent no-op', async () => { + const db = database(); + const gone = String(new mongoose.Types.ObjectId()); + assert.deepEqual( + await db.appendPreviews(gone, [preview(ID_A), preview(undefined)], { + width: 1, + height: 1, + }), + [], + ); + assert.equal(await db.loadMedia(gone), null); + }); + + test("every write runs the host's findOneAndUpdate middleware and returns only the id", async () => { + const seen: Array<{ ops: string[]; fields: string[] | null }> = []; + const schema = new mongoose.Schema( + { ...resizeMediaSchemaFragment }, + { minimize: false }, + ); + // A host that refreshes a cache or a search index after each media update. + schema.post( + 'findOneAndUpdate', + function (this: mongoose.Query, doc: unknown) { + seen.push({ + ops: Object.keys(this.getUpdate() ?? {}), + fields: doc + ? Object.keys((doc as mongoose.Document).toObject()) + : null, + }); + }, + ); + const Hooked = + connection.models.HookedMedia ?? connection.model('HookedMedia', schema); + const db = new MongoDatabase({ + mediaModel: Hooked, + lockModel: createResizeModels(connection).ResizeLock, + }); + const doc = await Hooked.create({ + original: { storageRef: { path: 'o.png' }, format: 'png' }, + previews: [], + }); + const stored = preview(ID_A); + + assert.deepEqual( + await db.appendPreviews( + String(doc._id), + [stored, preview(ID_A, 'again')], + { width: 64, height: 48 }, + ), + [stored], + ); + // The backfill, the stored preview, then the skipped one (no document matched). + assert.deepEqual(seen, [ + { ops: ['$set'], fields: ['_id'] }, + { ops: ['$push'], fields: ['_id'] }, + { ops: ['$push'], fields: null }, + ]); + const reloaded = await Hooked.findById(doc._id).lean(); + assert.equal(reloaded?.previews?.length, 1); + assert.equal(reloaded?.original?.width, 64); + }); +}); + +describe('MongoDatabase.verify(): the media model must store preview identities', () => { + test('a media model without previews.identity is a setup error', () => { + const db = new MongoDatabase({ + mediaModel: OutdatedMedia(), + lockModel: createResizeModels(connection).ResizeLock, + }); + assert.throws( + () => db.verify(), + (err: unknown) => + err instanceof ResizeSetupError && + err.code === 'RESIZE_MONGO_MEDIA_MODEL_OUTDATED' && + err.message.includes('OutdatedMedia') && + err.message.includes('resizeMediaSchemaFragment'), + ); + }); + + test('rows declared through an explicit row schema without identity are rejected too', () => { + const { identity: _identity, ...rowFields } = + resizeMediaSchemaFragment.previews[0]; + const db = new MongoDatabase({ + mediaModel: mediaModelWith('RowSchemaMedia', [ + new mongoose.Schema(rowFields), + ]), + lockModel: createResizeModels(connection).ResizeLock, + }); + assert.throws( + () => db.verify(), + (err: unknown) => + err instanceof ResizeSetupError && + err.code === 'RESIZE_MONGO_MEDIA_MODEL_OUTDATED', + ); + }); + + test('previews kept as a mixed or plain array, another type, or loose rows pass', () => { + const lockModel = createResizeModels(connection).ResizeLock; + for (const [name, previews] of [ + ['MixedArrayMedia', [mongoose.Schema.Types.Mixed]], + ['EmptyArrayMedia', []], + ['ArrayCtorMedia', Array], + ['TypeArrayMedia', { type: Array }], + ['StringArrayMedia', [String]], + [ + 'LooseRowsMedia', + [new mongoose.Schema({ sizeKey: String }, { strict: false })], + ], + ] as const) { + const db = new MongoDatabase({ + mediaModel: mediaModelWith(name, previews), + lockModel, + }); + assert.doesNotThrow(() => db.verify(), name); + } + }); + + test('a mixed or plain previews array really stores and matches identity', async () => { + const lockModel = createResizeModels(connection).ResizeLock; + for (const [name, previews] of [ + ['MixedArrayMedia', [mongoose.Schema.Types.Mixed]], + ['ArrayCtorMedia', Array], + ] as const) { + const model = mediaModelWith(name, previews); + const db = new MongoDatabase({ mediaModel: model, lockModel }); + const doc = await model.create({ + original: { storageRef: { path: 'o.png' }, format: 'png' }, + previews: [], + }); + const id = String(doc._id); + const first = preview(ID_A, 'first'); + assert.deepEqual(await db.appendPreviews(id, [first]), [first], name); + assert.deepEqual( + await db.appendPreviews(id, [preview(ID_A, 'second')]), + [], + name, + ); + const results = await Promise.all( + Array.from({ length: 6 }, (_, i) => + db.appendPreviews(id, [preview(ID_B, `worker-${i}`)]), + ), + ); + assert.equal(results.flat().length, 1, name); + const rows = ( + (await model.findById(id).lean()) as { previews: Preview[] } + ).previews; + assert.deepEqual( + rows.map((row) => row.identity), + [ID_A, ID_B], + name, + ); + } + }); + + test('the current fragment and a model-shaped object without a schema pass', () => { + const lockModel = createResizeModels(connection).ResizeLock; + assert.doesNotThrow(() => + new MongoDatabase({ mediaModel: Media(), lockModel }).verify(), + ); + const shaped = { + findById: async () => null, + findOneAndUpdate: async () => null, + }; + assert.doesNotThrow(() => + new MongoDatabase({ mediaModel: shaped, lockModel }).verify(), + ); + }); +}); diff --git a/src/drivers/mongo/database.ts b/src/drivers/mongo/database.ts index eace202..76ae491 100644 --- a/src/drivers/mongo/database.ts +++ b/src/drivers/mongo/database.ts @@ -12,11 +12,29 @@ import type { import { createResizeModels } from './models.ts'; import { MongoTaskQueue } from './taskQueue.ts'; -/** The media model methods the database calls (any Mongoose model has them). */ +/** + * The media model methods the database calls (any Mongoose model has them). Writes go through + * `findOneAndUpdate`, so the host's findOneAndUpdate middleware sees them; it resolves with the + * matched document (only its `_id` is requested) or null. When the model also exposes its + * `schema`, verify() checks it keeps the preview fields the database relies on. + */ export interface MongoMediaModel { modelName?: string; findById(id: string): PromiseLike; - findByIdAndUpdate(id: string, update: object): PromiseLike; + findOneAndUpdate( + filter: object, + update: object, + options: { projection: { _id: 1 } }, + ): PromiseLike; +} + +// Each write asks for the matched document's id only, not the whole media document. +const ID_ONLY = { projection: { _id: 1 as const } }; + +// The part of a Mongoose schema verify() reads (types only: no mongoose import). +interface SchemaLike { + path(path: string): unknown; + options: { strict?: unknown }; } /** The lock model methods the database calls (any Mongoose model has them). */ @@ -91,10 +109,38 @@ export class MongoDatabase extends ResizeDatabase { * missing lock model would otherwise fail every variant at run time and dead-letter its tasks. */ verify(): void { - this.mediaModel(); + this.verifyMediaModel(); this.lockModel(); } + /** + * The media model must resolve, and its preview rows must keep `identity`. Only rows declared as + * sub-documents with a strict shape lose it (Mongoose strict mode strips an undeclared field from + * every row, and every worker that renders a variant would then add its own row); a mixed or + * plain array, or rows with `strict: false`, keep any field. + */ + protected verifyMediaModel(): void { + const model = this.mediaModel(); + const schema = (model as { schema?: Partial }).schema; + if (typeof schema?.path !== 'function') { + return; + } + const previews = schema.path('previews') as + | { instance?: string; schema?: Partial } + | undefined; + const rows = previews?.instance === 'Array' ? previews.schema : undefined; + if ( + typeof rows?.path === 'function' && + rows.options?.strict !== false && + !rows.path('identity') + ) { + throw new ResizeSetupError( + `resize: the media model ${model.modelName ? `'${model.modelName}' ` : ''}has no previews.identity field, so Mongoose would drop it and a preview rendered twice would be stored twice — spread the current resizeMediaSchemaFragment into the media model's schema`, + { code: 'RESIZE_MONGO_MEDIA_MODEL_OUTDATED' }, + ); + } + } + async loadMedia(mediaId: string): Promise { return (await this.mediaModel().findById(mediaId)) as MediaLike | null; } @@ -103,20 +149,42 @@ export class MongoDatabase extends ResizeDatabase { mediaId: string, previews: Preview[], backfillDims?: { width: number; height: number }, - ): Promise { - // One atomic write: push the previews and, when the worker measured the original, set its - // dimensions in the same update. - const update: { - $push: { previews: { $each: Preview[] } }; - $set?: { 'original.width': number; 'original.height': number }; - } = { $push: { previews: { $each: previews } } }; + ): Promise { + const model = this.mediaModel(); + // When the worker measured the original, set its dimensions first: a reader that sees the + // previews then sees the dimensions too. Applied whether or not any preview is stored. if (backfillDims) { - update.$set = { - 'original.width': backfillDims.width, - 'original.height': backfillDims.height, - }; + await model.findOneAndUpdate( + { _id: mediaId }, + { + $set: { + 'original.width': backfillDims.width, + 'original.height': backfillDims.height, + }, + }, + ID_ONLY, + ); + } + // One conditional write per preview: the filter and the push are atomic on the document, so of + // two workers that rendered the same identity only one stores its row (the other matches + // nothing: null). A preview without an identity is always stored, and a row stored before rows + // carried one never blocks a new row. Missing media matches nothing: a silent no-op. + const stored: Preview[] = []; + for (const preview of previews) { + const filter = + preview.identity === undefined + ? { _id: mediaId } + : { _id: mediaId, 'previews.identity': { $ne: preview.identity } }; + const matched = await model.findOneAndUpdate( + filter, + { $push: { previews: preview } }, + ID_ONLY, + ); + if (matched) { + stored.push(preview); + } } - await this.mediaModel().findByIdAndUpdate(mediaId, update); + return stored; } // A lock is a document whose id is the key; a TTL index removes expired ones, and an expired diff --git a/src/drivers/mongo/schemas.ts b/src/drivers/mongo/schemas.ts index 467c36a..70de664 100644 --- a/src/drivers/mongo/schemas.ts +++ b/src/drivers/mongo/schemas.ts @@ -37,9 +37,12 @@ export function resizeTaskFields(mediaModelName: string) { default: 'pending', }, attempts: { type: Number, default: 0 }, + // When the task may next be claimed: now for a new or released task, the retry time after a + // failure, the end of the lease while a worker holds it. Claims take the earliest due task. + availableAt: { type: Date, default: Date.now }, leasedBy: { type: String }, leaseToken: { type: String }, // fencing token of the current lease - leaseExpiresAt: { type: Date }, + leaseExpiresAt: { type: Date }, // the current lease ends; null while the task waits completedAt: { type: Date }, deadAt: { type: Date }, error: { type: String }, @@ -66,9 +69,11 @@ export const resizeTaskIndexes: readonly IndexSpec[] = [ partialFilterExpression: { status: 'dead' }, }, ], - // Lease hot path: a worker consumes one queue, oldest task first. - [{ queue: 1, status: 1, createdAt: 1 }], - // Reclaiming expired leases. The partial filter, not sparseness, scopes it. + // Claim hot path: a worker consumes one queue, the earliest due task first. Tasks in backoff and + // live leases lie outside the claim's bounds, so a claim reads about one document. + [{ queue: 1, status: 1, availableAt: 1 }], + // Processing tasks by lease end, to find stuck leases (the claim does not need it). The partial + // filter, not sparseness, scopes it. [ { leaseExpiresAt: 1 }, { partialFilterExpression: { status: 'processing' } }, diff --git a/src/drivers/mongo/taskQueue.test.ts b/src/drivers/mongo/taskQueue.test.ts index 4590ddc..ad301f7 100644 --- a/src/drivers/mongo/taskQueue.test.ts +++ b/src/drivers/mongo/taskQueue.test.ts @@ -1,4 +1,5 @@ import assert from 'node:assert/strict'; +import { hostname } from 'node:os'; import { after, afterEach, @@ -31,6 +32,7 @@ import type { QueueTimingOptions, ResizeLogger, } from '../../types.d.ts'; +import { resizeTaskFields } from './schemas.ts'; import { MongoTaskQueue } from './taskQueue.ts'; // Real MongoDB atomic semantics, using the same schema and indexes as framework hosts. @@ -331,14 +333,75 @@ describe('MongoTaskQueue setup', () => { assert.equal(await tasks.renew(claimed, 300), false); assert.equal(await tasks.complete(claimed), false); assert.equal(await tasks.fail(claimed, 'dead', 'error'), false); + assert.equal(await tasks.release(claimed), false); assert.deepEqual(await tasks.findActive(claimed), []); - assert.equal(errors.length, 6); + assert.equal(errors.length, 7); + }); + + test('verify rejects a model without the fields the queue writes (an ejected 0.2 model)', () => { + // Strict mode would silently drop these from every insert: each task would then read as + // Resizer 'default' on queue 'default'. + const { + resizer: _resizer, + queue: _queue, + requestKey: _requestKey, + availableAt: _availableAt, + ...oldFields + } = resizeTaskFields('File'); + const Old = + connection.models.OldResizeTask ?? + connection.model( + 'OldResizeTask', + new mongoose.Schema(oldFields, { timestamps: true, autoIndex: false }), + ); + const queue = new MongoTaskQueue({ model: Old }); + assert.throws( + () => queue.verify(), + (error: unknown) => + error instanceof ResizeSetupError && + error.code === 'RESIZE_MONGO_MODEL_OUTDATED' && + /resizer, queue, requestKey, availableAt/.test(error.message) && + /delete src\/models\/ResizeTask\.ts and re-run resize-scaffold/.test( + error.message, + ) && + /--eject/.test(error.message) && + /resizeTaskFields\(\)/.test(error.message) && + // --force would also overwrite the host's own resizer.ts and config file. + !/--force/.test(error.message), + ); + }); + + test('verify rejects a model ejected before tasks carried availableAt', () => { + // Without the path, strict mode drops every due time: retries would skip their backoff. + const { availableAt: _availableAt, ...fields } = resizeTaskFields('File'); + const Old = + connection.models.ResizeTaskWithoutDueTime ?? + connection.model( + 'ResizeTaskWithoutDueTime', + new mongoose.Schema(fields, { timestamps: true, autoIndex: false }), + ); + assert.throws( + () => new MongoTaskQueue({ model: Old }).verify(), + (error: unknown) => + error instanceof ResizeSetupError && + error.code === 'RESIZE_MONGO_MODEL_OUTDATED' && + /has no availableAt field/.test(error.message), + ); + }); + + test('verify accepts the package model and a model-shaped object without a schema', () => { + assert.doesNotThrow(() => tasks.verify()); + const schemaless = new MongoTaskQueue({ + model: { findOneAndUpdate: async () => null }, + }); + assert.doesNotThrow(() => schemaless.verify()); }); }); describe('MongoTaskQueue.add', () => { test('maps mediaId to fileId, stores pipeline and previews, returns the taskId', async () => { const payload = request({ pipeline: 'photo' }); + const before = Date.now(); const { taskId } = await tasks.add(payload); assert.ok(taskId); const doc = await M.findById(taskId).lean(); @@ -349,6 +412,10 @@ describe('MongoTaskQueue.add', () => { assert.equal(doc.attempts, 0); assert.equal((doc.previews as unknown[]).length, 1); assert.equal(doc.requestKey, payload.requestKey); + // Due at once. + const availableAt = (doc.availableAt as Date).getTime(); + assert.ok(availableAt >= before && availableAt <= Date.now()); + assert.equal(doc.leaseExpiresAt, undefined); }); for (const { name, preview, errorPath } of [ @@ -688,22 +755,116 @@ describe('MongoTaskQueue.add', () => { }); describe('MongoTaskQueue.claim', () => { - test('claims the oldest pending task by createdAt', async () => { - // The newer row goes in first, so _id order and createdAt order disagree. - await insert({}, new Date(5000)); - const older = await insert({}, new Date(1000)); + test('claims the task that became due first; while leased it is due when the lease ends', async () => { + // The later-due row is older and goes in first: _id, createdAt and due order disagree. + await insert({ availableAt: new Date(Date.now() - 1000) }, new Date(1000)); + const first = await insert( + { availableAt: new Date(Date.now() - 5000) }, + new Date(5000), + ); const leased = await tasks.claim('default', 300); assert.ok(leased); - assert.equal(leased.taskId, String(older._id)); + assert.equal(leased.taskId, String(first._id)); assert.equal(leased.attempts, 1); assert.ok(leased.token); const row = await M.findById(leased.taskId).lean(); - assert.equal(row?.status, 'processing'); - assert.equal(row?.leasedBy, `resizer-${process.pid}`); + assert.ok(row); + assert.equal(row.status, 'processing'); + // Which pod holds the lease: every container's main process is pid 1. + assert.equal(row.leasedBy, `${hostname().slice(0, 64)}:${process.pid}`); + assert.equal( + (row.availableAt as Date).getTime(), + (row.leaseExpiresAt as Date).getTime(), + ); + }); + + test('a task in backoff is not claimed before its availableAt', async () => { + await insert({ availableAt: new Date(Date.now() + 60_000) }); + const due = await insert({ availableAt: past() }); + assert.equal((await tasks.claim('default', 300))?.taskId, String(due._id)); + assert.equal(await tasks.claim('default', 300), null); + }); + + test('a legacy pending row keeps its retry time from leaseExpiresAt', async (t) => { + // Written by the previous code: the retry time five minutes ahead, no availableAt. + const start = Date.now(); + const legacy = await insert({ + leaseExpiresAt: new Date(start + 5 * 60_000), + }); + await M.collection.updateOne( + { _id: legacy._id }, + { $unset: { availableAt: '' } }, + ); + assert.equal(await tasks.claim('default', 300), null); + t.mock.timers.enable({ apis: ['Date'], now: start + 5 * 60_000 + 1000 }); + assert.equal( + (await tasks.claim('default', 300))?.taskId, + String(legacy._id), + ); + }); + + test('a retry written by an older worker (availableAt passed, leaseExpiresAt ahead) waits', async () => { + const retried = await insert({ + availableAt: past(), + leaseExpiresAt: new Date(Date.now() + 60_000), + }); + assert.equal(await tasks.claim('default', 300), null); + await M.updateOne( + { _id: retried._id }, + { $set: { leaseExpiresAt: past() } }, + ); + assert.equal( + (await tasks.claim('default', 300))?.taskId, + String(retried._id), + ); + }); + + test('rows written before availableAt existed: pending is due at once, processing once its lease ends', async () => { + const pending = await insert({}); + const expired = await insert({ + status: 'processing', + leaseExpiresAt: past(), + }); + await insert({ + status: 'processing', + leaseExpiresAt: new Date(Date.now() + 60_000), + }); + await M.collection.updateMany({}, { $unset: { availableAt: '' } }); + const claimed = [ + (await tasks.claim('default', 300))?.taskId, + (await tasks.claim('default', 300))?.taskId, + ]; + assert.deepEqual( + claimed.sort(), + [String(pending._id), String(expired._id)].sort(), + ); + assert.equal(await tasks.claim('default', 300), null); + }); + + test('a live lease is respected even when its availableAt was not moved (an older worker)', async () => { + const leased = await insert({ + status: 'processing', + availableAt: past(), + leaseExpiresAt: new Date(Date.now() + 60_000), + }); + assert.equal(await tasks.claim('default', 300), null); + await M.updateOne( + { _id: leased._id }, + { $set: { leaseExpiresAt: past() } }, + ); + assert.equal( + (await tasks.claim('default', 300))?.taskId, + String(leased._id), + ); }); test('reclaims an expired processing lease and bumps attempts', async () => { - await insert({ status: 'processing', leaseExpiresAt: past(), attempts: 1 }); + await insert({ + status: 'processing', + leaseExpiresAt: past(), + availableAt: past(), + attempts: 1, + }); const leased = await tasks.claim('default', 300); assert.ok(leased); assert.equal(leased.attempts, 2); @@ -714,18 +875,17 @@ describe('MongoTaskQueue.claim', () => { }); test('claims exhausted pending and expired processing tasks so the core can dead-letter them', async () => { - const expired = await insert( - { - status: 'processing', - leaseExpiresAt: past(), - attempts: 3, - }, - new Date(1000), - ); - const pending = await insert( - { status: 'pending', attempts: 3 }, - new Date(2000), - ); + const expired = await insert({ + status: 'processing', + leaseExpiresAt: past(), + availableAt: past(), + attempts: 3, + }); + const pending = await insert({ + status: 'pending', + availableAt: new Date(Date.now() - 1000), + attempts: 3, + }); const a = await tasks.claim('default', 300); const b = await tasks.claim('default', 300); assert.equal(a?.taskId, String(expired._id)); @@ -743,19 +903,102 @@ describe('MongoTaskQueue.claim', () => { assert.equal((a === null) !== (b === null), true); }); - test('pending tasks without a lease, with a null lease, or with a past retry date are due', async () => { - const a = await insert({}, new Date(1000)); - const b = await insert({ leaseExpiresAt: null }, new Date(2000)); - const c = await insert({ leaseExpiresAt: past() }, new Date(3000)); - await insert({ leaseExpiresAt: new Date(Date.now() + 60_000) }); - const claimed = []; - for (let index = 0; index < 3; index++) { - claimed.push((await tasks.claim('default', 300))?.taskId); - } - assert.deepEqual( - claimed, - [a, b, c].map((row) => String(row._id)), - ); + // The claim's own filter and sort, explained against thousands of tasks that are not due: tasks + // in backoff and live leases must stay outside the index bounds, so after an outage a claim + // still reads about one document, with no in-memory sort. + async function explainClaim(): Promise<{ + docs: number; + keys: number; + plan: string; + }> { + let captured: + | { filter: object; update: object; options: { sort?: object } } + | undefined; + const recording = new Proxy(M, { + get(target, property, receiver) { + if (property === 'findOneAndUpdate') { + return async ( + filter: object, + update: object, + options: { sort?: object }, + ) => { + captured = { filter, update, options }; + return null; + }; + } + return Reflect.get(target, property, receiver); + }, + }); + await new MongoTaskQueue({ model: recording }).claim('default', 300); + assert.ok(captured && connection.db); + const explained = await connection.db.command({ + explain: { + findAndModify: M.collection.collectionName, + query: captured.filter, + sort: captured.options.sort, + update: captured.update, + }, + verbosity: 'executionStats', + }); + return { + docs: explained.executionStats.totalDocsExamined, + keys: explained.executionStats.totalKeysExamined, + plan: JSON.stringify(explained.queryPlanner.winningPlan), + }; + } + + async function seedNotDue() { + const row = (status: string, at: number, extra: object = {}) => ({ + fileId: new mongoose.Types.ObjectId(), + queue: 'default', + resizer: 'default', + pipeline: 'default', + previews: [], + status, + attempts: 1, + availableAt: new Date(at), + ...extra, + }); + const later = Date.now() + 300_000; + const leaseEnd = Date.now() + 60_000; + await M.collection.insertMany([ + ...Array.from({ length: 2000 }, () => row('pending', later)), + ...Array.from({ length: 50 }, () => + row('processing', leaseEnd, { leaseExpiresAt: new Date(leaseEnd) }), + ), + ...Array.from({ length: 500 }, () => row('completed', 0)), + ]); + } + + for (const scenario of ['a due task', 'an expired lease'] as const) { + test(`claiming ${scenario} among 2000 tasks in backoff and 50 live leases reads one document`, async () => { + await seedNotDue(); + const due = + scenario === 'a due task' + ? await insert({ availableAt: past() }) + : await insert({ + status: 'processing', + availableAt: past(), + leaseExpiresAt: past(), + }); + const { docs, keys, plan } = await explainClaim(); + assert.ok(docs <= 1, `documents examined: ${docs}`); + assert.ok(keys <= 10, `index keys examined: ${keys}`); + assert.doesNotMatch(plan, /"stage":"SORT"/, 'no in-memory sort'); + assert.doesNotMatch(plan, /COLLSCAN/); + assert.match(plan, /"availableAt":1/); + assert.equal( + (await tasks.claim('default', 300))?.taskId, + String(due._id), + ); + }); + } + + test('when nothing is due, a claim reads no document', async () => { + await seedNotDue(); + const { docs, plan } = await explainClaim(); + assert.equal(docs, 0); + assert.doesNotMatch(plan, /"stage":"SORT"/); assert.equal(await tasks.claim('default', 300), null); }); }); @@ -905,9 +1148,10 @@ describe('MongoTaskQueue.complete fencing', () => { await insert(); const first = await tasks.claim('default', 300); assert.ok(first); + // The lease ends: it is next claimable then. await M.updateOne( { _id: first.taskId }, - { $set: { leaseExpiresAt: past() } }, + { $set: { leaseExpiresAt: past(), availableAt: past() } }, ); const second = await tasks.claim('default', 300); assert.ok(second); @@ -960,7 +1204,8 @@ describe('MongoTaskQueue.fail and consumeQueue retry policy', () => { assert.ok(row); assert.equal(row.status, 'pending'); assert.equal(row.leaseToken, null); - assert.ok((row.leaseExpiresAt as Date).getTime() > before); + assert.equal(row.leaseExpiresAt, null); + assert.ok((row.availableAt as Date).getTime() > before); assert.equal(rec.failed.length, 1); assert.equal(rec.dead.length, 0); }); @@ -982,18 +1227,27 @@ describe('MongoTaskQueue.fail and consumeQueue retry policy', () => { test('RESIZE_NO_ORIGINAL with a stale token is a fenced no-op with no event', async () => { const inserted = await insert(); const rec = makeEvents(); + // Stop once the fenced fail has run (a shutdown before it would give the task back instead). + const failResults: boolean[] = []; + const realFail = tasks.fail.bind(tasks); + tasks.fail = async (...args) => { + const held = await realFail(...args); + failResults.push(held); + consumer.ctrl.abort(); + return held; + }; const consumer = startConsumer( async (task) => { await M.updateOne( { _id: task.taskId }, { $set: { leaseToken: 'another-worker' } }, ); - consumer.ctrl.abort(); throw new ResizeNoOriginalError(task.mediaId); }, { onEvent: rec.onEvent }, ); await consumer.done; + assert.deepEqual(failResults, [false]); const row = await M.findById(inserted._id).lean(); assert.equal(row?.status, 'processing'); assert.equal(row?.deadAt, undefined); @@ -1041,7 +1295,8 @@ describe('MongoTaskQueue.fail and consumeQueue retry policy', () => { assert.equal(await tasks.fail(leased, { retryAt }, 'boom'), true); const row = await M.findById(leased.taskId).lean(); assert.ok(row); - assert.equal((row.leaseExpiresAt as Date).getTime(), retryAt.getTime()); + assert.equal((row.availableAt as Date).getTime(), retryAt.getTime()); + assert.equal(row.leaseExpiresAt, null); // no lease while it waits assert.equal(row?.leaseToken, null); assert.equal(row?.error, 'boom'); assert.equal(await tasks.claim('default', 300), null); @@ -1089,6 +1344,40 @@ describe('MongoTaskQueue.fail and consumeQueue retry policy', () => { }); }); +describe('MongoTaskQueue.release', () => { + test('returns a claimed task to pending, due at once, without counting the delivery', async () => { + const inserted = await insert({ attempts: 2, error: 'earlier failure' }); + const leased = await tasks.claim('default', 60_000); + assert.ok(leased); + assert.equal(leased.attempts, 3); + assert.equal(await tasks.release(leased), true); + const row = await M.findById(inserted._id).lean(); + assert.equal(row?.status, 'pending'); + assert.equal(row?.attempts, 2); + assert.equal(row?.leaseToken, null); + assert.equal(row?.leaseExpiresAt, null); + assert.ok(row && (row.availableAt as Date).getTime() <= Date.now()); + assert.equal(row?.error, 'earlier failure'); + const again = await tasks.claim('default', 300); + assert.equal(again?.taskId, leased.taskId); + assert.equal(again?.attempts, 3); + assert.notEqual(again?.token, leased.token); + }); + + test('a stale token releases nothing', async () => { + await insert(); + const leased = await tasks.claim('default', 300); + assert.ok(leased); + assert.equal(await tasks.release({ ...leased, token: 'stale' }), false); + const row = await M.findById(leased.taskId).lean(); + assert.equal(row?.status, 'processing'); + assert.equal(row?.leaseToken, leased.token); + assert.equal(row?.attempts, 1); + assert.equal(await tasks.complete(leased), true); + assert.equal(await tasks.release(leased), false); + }); +}); + describe('MongoTaskQueue.renew fencing', () => { test('a valid token extends the lease; a stale token does not match', async () => { await insert(); @@ -1102,10 +1391,13 @@ describe('MongoTaskQueue.renew fencing', () => { assert.ok(row); const expiry = (row.leaseExpiresAt as Date).getTime(); assert.ok(expiry > firstExpiry); + // The task stays unclaimable until the renewed lease ends. + assert.equal((row.availableAt as Date).getTime(), expiry); assert.equal(await tasks.renew({ ...leased, token: 'stale' }, 2000), false); const unchanged = await M.findById(leased.taskId).lean(); assert.ok(unchanged); assert.equal((unchanged.leaseExpiresAt as Date).getTime(), expiry); + assert.equal((unchanged.availableAt as Date).getTime(), expiry); }); }); @@ -1266,6 +1558,54 @@ describe('consumeQueue with MongoTaskQueue', () => { await consumer.done; }); + for (const prior of [0, 2]) { + test(`a shutdown mid-task releases the task without counting it (${prior} earlier attempts of 3)`, { + timeout: 10_000, + }, async () => { + const inserted = await insert({ attempts: prior }); + const rec = makeEvents(); + let started = false; + const consumer = startConsumer( + // Like the worker: on abort it skips the remaining variants and rejects as incomplete. + (task, { signal }) => + new Promise((_resolve, reject) => { + started = true; + signal.addEventListener( + 'abort', + () => + reject( + new ResizeGenerateError({ + mediaId: task.mediaId, + failed: 1, + requested: 1, + code: 'RESIZE_WORKER_INCOMPLETE', + }), + ), + { once: true }, + ); + }), + { onEvent: rec.onEvent }, + ); + await waitFor(() => started); + consumer.ctrl.abort(); + await consumer.done; + const row = await M.findById(inserted._id).lean(); + assert.equal(row?.status, 'pending'); + assert.equal(row?.attempts, prior); + assert.equal(row?.leaseToken, null); + assert.equal(row?.deadAt, undefined); + assert.equal(row?.error, undefined); + assert.deepEqual( + [rec.completed.length, rec.failed.length, rec.dead.length], + [0, 0, 0], + ); + // The next worker takes it at once, as the same attempt. + const next = await tasks.claim('default', 300); + assert.equal(next?.taskId, String(inserted._id)); + assert.equal(next?.attempts, prior + 1); + }); + } + test('an idle worker stops promptly when its signal aborts', async () => { configureQueue(undefined, { idlePollMs: 10_000 }); const consumer = startConsumer(async () => {}); diff --git a/src/drivers/mongo/taskQueue.ts b/src/drivers/mongo/taskQueue.ts index e526dba..30a9203 100644 --- a/src/drivers/mongo/taskQueue.ts +++ b/src/drivers/mongo/taskQueue.ts @@ -1,6 +1,7 @@ // MongoTaskQueue: the task queue as documents of a ResizeTask model (see createResizeModels). Each // method is one atomic findOneAndUpdate; the lease token fences every write after a claim, so a -// worker that lost its lease can never complete or fail a task another worker now holds. +// worker that lost its lease can never complete, fail or release a task another worker now holds. +import { hostname } from 'node:os'; import { type ClaimedTask, type NewTask, @@ -29,6 +30,30 @@ export interface MongoTaskQueueOptions { logger?: ResizeLogger; // default: console } +// The schema paths this queue writes. Mongoose strict mode silently drops a path its schema lacks: +// a model ejected before tasks carried a Resizer, queue and request key would file every task under +// Resizer 'default' on queue 'default'; one without availableAt would lose every retry time. +const WRITTEN_PATHS = [ + 'fileId', + 'resizer', + 'queue', + 'pipeline', + 'requestKey', + 'previews', + 'status', + 'attempts', + 'availableAt', + 'leasedBy', + 'leaseToken', + 'leaseExpiresAt', + 'completedAt', + 'deadAt', + 'error', +] as const; + +// Who holds a lease: the host name tells pods apart (every container's main process is pid 1). +const LEASE_HOLDER = `${hostname().slice(0, 64)}:${process.pid}`; + // The fields of a ResizeTask document this queue reads. interface TaskDoc { _id: { toString(): string }; @@ -64,14 +89,29 @@ export class MongoTaskQueue extends TaskQueue { return this.#getTiming(); } - /** Startup check: the ResizeTask model must be registered. */ + /** + * Startup check: the ResizeTask model must be registered, and its schema (when the model exposes + * one) must have every field this queue writes. + */ verify(): void { - if (!this.#getModel()) { + const model = this.#getModel(); + if (!model) { throw new ResizeSetupError( 'resize: the ResizeTask model is not registered — scaffold src/models/ResizeTask.ts (or pass `model`)', { code: 'RESIZE_MONGO_MODEL_MISSING' }, ); } + const schema = model.schema; + if (typeof schema?.path !== 'function') { + return; + } + const missing = WRITTEN_PATHS.filter((path) => !schema.path(path)); + if (missing.length > 0) { + throw new ResizeSetupError( + `resize: the ResizeTask model has no ${missing.join(', ')} field(s), so Mongoose would drop them from every task — delete src/models/ResizeTask.ts and re-run resize-scaffold (add --eject for a full editable model); for a hand-written model, port the fields from resizeTaskFields()`, + { code: 'RESIZE_MONGO_MODEL_OUTDATED' }, + ); + } } // At runtime a missing model is logged, not thrown: reads and prewarm never throw (verify() @@ -110,6 +150,7 @@ export class MongoTaskQueue extends TaskQueue { previews: task.previews, status: 'pending', attempts: 0, + availableAt: new Date(), }, }, { @@ -155,45 +196,50 @@ export class MongoTaskQueue extends TaskQueue { } // One findOneAndUpdate that returns at once (the core polls every idlePollMs), so it takes no - // abort signal. + // abort signal. The earliest due task wins: a waiting task whose availableAt has passed, or a + // leased one whose lease ended. A lease keeps availableAt at its end, so tasks in backoff and + // live leases lie outside the { queue, status, availableAt } index bounds and a claim reads + // about one document, with no in-memory sort. Rows written before availableAt existed have none: + // they are due once their leaseExpiresAt (a retry time or a lease end) has passed. async claim(queue: string, leaseMs: number): Promise { const model = this.#model(); if (!model) { return null; } const now = new Date(); + const leaseEnd = new Date(now.getTime() + leaseMs); const doc = (await model.findOneAndUpdate( { queue: named(queue), + status: { $in: ['pending', 'processing'] }, + availableAt: { $not: { $gt: now } }, // due, or no availableAt yet + // Rows written by a worker that keeps times in leaseExpiresAt (before availableAt existed, + // or an older worker in a rolling deploy) still wait for it: a waiting row's retry time, + // a leased row's lease. New waiting rows have leaseExpiresAt null. $or: [ - { - status: 'pending', - $or: [ - { leaseExpiresAt: { $exists: false } }, - { leaseExpiresAt: null }, - { leaseExpiresAt: { $lt: now } }, - ], - }, - { status: 'processing', leaseExpiresAt: { $lt: now } }, + { status: 'pending', leaseExpiresAt: { $not: { $gt: now } } }, + { leaseExpiresAt: { $lt: now } }, ], }, { $set: { status: 'processing', - leasedBy: `resizer-${process.pid}`, + leasedBy: LEASE_HOLDER, leaseToken: randomHex(), - leaseExpiresAt: new Date(now.getTime() + leaseMs), + leaseExpiresAt: leaseEnd, + availableAt: leaseEnd, }, $inc: { attempts: 1 }, }, - { sort: { createdAt: 1 }, returnDocument: 'after' }, + { sort: { availableAt: 1 }, returnDocument: 'after' }, )) as TaskDoc | null; return doc ? toClaimedTask(doc) : null; } async renew(task: ClaimedTask, leaseMs: number): Promise { + const leaseEnd = new Date(Date.now() + leaseMs); return this.#fenced(task, { - $set: { leaseExpiresAt: new Date(Date.now() + leaseMs) }, + $set: { leaseExpiresAt: leaseEnd, availableAt: leaseEnd }, }); } @@ -213,17 +259,30 @@ export class MongoTaskQueue extends TaskQueue { next === 'dead' ? { $set: { status: 'dead', deadAt: new Date(), error } } : { - // A future leaseExpiresAt keeps a pending task unclaimable until the retry time. $set: { status: 'pending', leaseToken: null, - leaseExpiresAt: next.retryAt, + leaseExpiresAt: null, + availableAt: next.retryAt, error, }, }, ); } + // Due at once, and the claim's attempt taken back. + async release(task: ClaimedTask): Promise { + return this.#fenced(task, { + $set: { + status: 'pending', + leaseToken: null, + leaseExpiresAt: null, + availableAt: new Date(), + }, + $inc: { attempts: -1 }, + }); + } + // Any queue: a task already waiting on another queue still covers the request. async findActive(query: { resizer: string; diff --git a/src/drivers/s3.test.ts b/src/drivers/s3.test.ts index 64c6c69..95b092a 100644 --- a/src/drivers/s3.test.ts +++ b/src/drivers/s3.test.ts @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; import { S3Client } from '@aws-sdk/client-s3'; +import { ResizeSecurityError } from '../errors.ts'; import { S3Storage } from './s3.ts'; function fakeClient() { @@ -64,8 +65,13 @@ describe('S3Storage', () => { ); assert.deepEqual(await s.download(parent), Buffer.from([1, 2, 3])); assert.equal(sent[2].input.Bucket, 'priv'); - assert.equal(s.canServeOriginalPublicly(parent), false); - assert.equal(s.canServeOriginalPublicly(child), true); + // Visibility is enforced where a URL is made; the driver has no separate visibility check. + assert.equal('canServeOriginalPublicly' in s, false); + assert.throws(() => s.publicUrl(parent), /private bucket/); + assert.equal( + s.publicUrl(child), + 'https://pub.s3.us-east-1.amazonaws.com/users/u1/previews/b.jpg', + ); }); test('ungrouped public object gets a public URL without client I/O', () => { @@ -93,6 +99,118 @@ describe('S3Storage', () => { ); }); + for (const region of ['eu-west-1', undefined]) { + test(`AWS path-style URL uses ${region ?? 'us-east-1 by default'} without an endpoint`, () => { + const { client, sent } = fakeClient(); + const s = new S3Storage({ + bucketPublic: 'pub', + region, + forcePathStyle: true, + client, + }); + assert.equal( + s.publicUrl({ bucket: 'pub', key: 'previews/a.jpg' }), + `https://s3.${region ?? 'us-east-1'}.amazonaws.com/pub/previews/a.jpg`, + ); + assert.equal(sent.length, 0); + }); + } + + test('AWS China regions use the amazonaws.com.cn domain in both URL forms', () => { + const ref = { bucket: 'pub', key: 'previews/a.jpg' }; + assert.equal( + new S3Storage({ bucketPublic: 'pub', region: 'cn-north-1' }).publicUrl( + ref, + ), + 'https://pub.s3.cn-north-1.amazonaws.com.cn/previews/a.jpg', + ); + assert.equal( + new S3Storage({ + bucketPublic: 'pub', + region: 'cn-northwest-1', + forcePathStyle: true, + }).publicUrl(ref), + 'https://s3.cn-northwest-1.amazonaws.com.cn/pub/previews/a.jpg', + ); + }); + + test('explicit endpoint and CDN URL keep precedence over the AWS path-style base', () => { + const opts = { + bucketPublic: 'pub', + endpoint: 'http://localhost:9000///', + region: 'eu-west-1', + forcePathStyle: true, + }; + const ref = { bucket: 'pub', key: 'previews/a.jpg' }; + assert.equal( + new S3Storage(opts).publicUrl(ref), + 'http://localhost:9000/pub/previews/a.jpg', + ); + assert.equal( + new S3Storage({ + ...opts, + publicBaseUrl: 'https://cdn.example.com///', + }).publicUrl(ref), + 'https://cdn.example.com/previews/a.jpg', + ); + }); + + test('bucket allowlist errors keep their security code without archived citations', async () => { + const { client, sent } = fakeClient(); + const s = new S3Storage({ bucketPublic: 'pub', client }); + await assert.rejects( + () => s.download({ bucket: 'attacker', key: 'a.jpg' }), + (error) => { + assert.ok(error instanceof ResizeSecurityError); + assert.equal(error.code, 'RESIZE_S3_BUCKET_NOT_ALLOWED'); + assert.equal( + error.message, + 'resize s3: ref.bucket "attacker" is not an allowlisted bucket (bucketPublic/bucketPrivate) — refusing cross-bucket access', + ); + return true; + }, + ); + assert.equal(sent.length, 0); + }); + + test('SVG uploads set attachment disposition while preserving their bytes and content type', async () => { + const { client, sent } = fakeClient(); + const s = new S3Storage({ + bucketPublic: 'pub', + bucketPrivate: 'priv', + client, + }); + const upload = { + ...args('originals/a.svg', 'private'), + body: Buffer.from(''), + contentType: 'image/svg+xml', + }; + await s.upload(upload); + assert.equal(sent[0].constructor.name, 'PutObjectCommand'); + assert.deepEqual(sent[0].input, { + Bucket: 'priv', + Key: upload.key, + Body: upload.body, + ContentType: upload.contentType, + ContentDisposition: 'attachment', + }); + }); + + test('non-SVG uploads keep their original PutObject input without a disposition', async () => { + const { client, sent } = fakeClient(); + const s = new S3Storage({ bucketPublic: 'pub', client }); + for (const contentType of ['image/jpeg', 'image/png', 'image/webp']) { + const upload = { ...args('previews/a', 'public'), contentType }; + await s.upload(upload); + assert.deepEqual(sent.at(-1)?.input, { + Bucket: 'pub', + Key: upload.key, + Body: upload.body, + ContentType: contentType, + }); + } + }); + test('refuses private URL, unknown bucket, missing bucket and invalid refs', async () => { const { client } = fakeClient(); const s = new S3Storage({ @@ -113,7 +231,6 @@ describe('S3Storage', () => { { bucket: 'pub', key: 'users/a/originals/x.jpg', namespace: 'users/b' }, ]) { await assert.rejects(() => s.download(ref)); - assert.throws(() => s.canServeOriginalPublicly(ref)); assert.throws(() => s.publicUrl(ref)); } }); diff --git a/src/drivers/s3.ts b/src/drivers/s3.ts index 2650e45..073e51f 100644 --- a/src/drivers/s3.ts +++ b/src/drivers/s3.ts @@ -84,7 +84,7 @@ export class S3Storage extends ResizeStorage { return; } throw new ResizeSecurityError( - `resize s3: ref.bucket "${bucket}" is not an allowlisted bucket (bucketPublic/bucketPrivate) — refusing cross-bucket access (05 · §10.5)`, + `resize s3: ref.bucket "${bucket}" is not an allowlisted bucket (bucketPublic/bucketPrivate) — refusing cross-bucket access`, { code: 'RESIZE_S3_BUCKET_NOT_ALLOWED' }, ); } @@ -181,6 +181,9 @@ export class S3Storage extends ResizeStorage { Key: physicalKey, Body: body, ContentType: contentType, + ...(contentType === 'image/svg+xml' + ? { ContentDisposition: 'attachment' } + : {}), }), ); return { @@ -206,18 +209,13 @@ export class S3Storage extends ResizeStorage { return Buffer.from(bytes); } - canServeOriginalPublicly(ref: StorageRef): boolean { - const { bucket } = this.#ref(ref); - return bucket === this.#opts.bucketPublic; - } - // PURE string building — no SDK, no I/O (called on the read path). Three forms: // explicit publicUrl base → CDN; endpoint/forcePathStyle → path-style; else // virtual-hosted. publicUrl(ref: StorageRef): string { const { bucket, key } = this.#ref(ref); // A ref explicitly pointing at the configured private bucket must never be turned into a - // public CDN URL. The engine normally prevents this call; keep the driver safe when a host + // public CDN URL. The engine only asks for stored previews; keep the driver safe when a host // calls publicUrl directly too. If both buckets are the same, that bucket is intentionally // public and the check below does not reject it. if ( @@ -234,11 +232,17 @@ export class S3Storage extends ResizeStorage { if (publicBase) { return `${publicBase.replace(/\/+$/, '')}/${key}`; } + // AWS's own host name: the China regions live under amazonaws.com.cn. + const region = this.#opts.region ?? 'us-east-1'; + const awsHost = `s3.${region}.amazonaws.com${region.startsWith('cn-') ? '.cn' : ''}`; if (this.#opts.endpoint !== undefined || this.#opts.forcePathStyle) { - const base = (this.#opts.endpoint ?? '').replace(/\/+$/, ''); + const base = (this.#opts.endpoint ?? `https://${awsHost}`).replace( + /\/+$/, + '', + ); return `${base}/${bucket}/${key}`; } - return `https://${bucket}.s3.${this.#opts.region ?? 'us-east-1'}.amazonaws.com/${key}`; + return `https://${bucket}.${awsHost}/${key}`; } // Time-limited signed URL for owner/admin reads of a private original. diff --git a/src/drivers/sqs.test.ts b/src/drivers/sqs.test.ts index 984f4fe..44511de 100644 --- a/src/drivers/sqs.test.ts +++ b/src/drivers/sqs.test.ts @@ -544,6 +544,19 @@ describe('SqsTaskQueue lease operations', () => { } assert.equal(deletes.length, 0); }); + test('release makes the message visible again at once, without deleting it', async () => { + const { client, changeVis, deletes } = makeFakeSqsClient(); + const tasks = new SqsTaskQueue({ queueUrl: 'q', client }); + const task = claimedTask({ + queue: 'bulk', + token: JSON.stringify([BULK_URL, 'bulk-rh']), + }); + assert.equal(await tasks.release(task), true); + assert.deepEqual(changeVis, [ + { QueueUrl: BULK_URL, ReceiptHandle: 'bulk-rh', VisibilityTimeout: 0 }, + ]); + assert.equal(deletes.length, 0); + }); test('a dead task is sent with its error and attempt count before deletion', async () => { const { client, commands, sent, deletes } = makeFakeSqsClient(); const tasks = new SqsTaskQueue({ @@ -607,15 +620,23 @@ describe('SqsTaskQueue lease operations', () => { /mid is dead.*no deadLetterQueueUrl.*handler boom/, ); }); - test('lost receipt errors report false for renew, complete, and fail', async () => { + test('lost receipt errors report false for renew, complete, fail, and release', async () => { for (const name of [ 'ReceiptHandleIsInvalid', 'MessageNotInflight', 'InvalidParameterValue', ]) { - for (const operation of ['renew', 'complete', 'retry', 'dead']) { + for (const operation of [ + 'renew', + 'complete', + 'retry', + 'dead', + 'release', + ]) { const command = - operation === 'renew' || operation === 'retry' + operation === 'renew' || + operation === 'retry' || + operation === 'release' ? 'ChangeMessageVisibilityCommand' : 'DeleteMessageCommand'; const { client } = makeFakeSqsClient({ @@ -631,11 +652,13 @@ describe('SqsTaskQueue lease operations', () => { ? await tasks.renew(task, 60_000) : operation === 'complete' ? await tasks.complete(task) - : await tasks.fail( - task, - operation === 'dead' ? 'dead' : { retryAt: new Date() }, - 'failed', - ); + : operation === 'release' + ? await tasks.release(task) + : await tasks.fail( + task, + operation === 'dead' ? 'dead' : { retryAt: new Date() }, + 'failed', + ); assert.equal(held, false, `${operation}: ${name}`); } } @@ -898,6 +921,42 @@ describe('consumeQueue with SqsTaskQueue', () => { assert.equal(changeVis.length, 1); assert.equal(errors[0][0], 'resize worker: failed event handler failed'); }); + test('a shutdown mid-task makes the message visible at once, with no event and no delete', async () => { + const { client, changeVis, deletes, sent } = makeFakeSqsClient({ + message: message({ Attributes: { ApproximateReceiveCount: '3' } }), + }); + const tasks = new SqsTaskQueue({ + queueUrl: 'q', + client, + deadLetterQueueUrl: DEAD_URL, + timing: { ...fastTiming, maxAttempts: 3 }, + }); + const rec = makeEvents(); + const ctrl = new AbortController(); + await consumeQueue(tasks, { + signal: ctrl.signal, + queue: 'default', + handle: (_task, { signal }) => + new Promise((_resolve, reject) => { + signal.addEventListener( + 'abort', + () => reject(new Error('stopped before the remaining variants')), + { once: true }, + ); + ctrl.abort(); // SIGTERM while the task runs + }), + onEvent: rec.onEvent, + }); + assert.deepEqual(changeVis, [ + { QueueUrl: 'q', ReceiptHandle: 'rh', VisibilityTimeout: 0 }, + ]); + assert.equal(deletes.length, 0); + assert.equal(sent.length, 0); // not dead-lettered on its last delivery + assert.deepEqual( + [rec.completed.length, rec.failed.length, rec.deadLettered.length], + [0, 0, 0], + ); + }); test('an exhausted task reports deadLettered with the original error and is moved then deleted', async () => { const { client, sent, deletes, changeVis } = makeFakeSqsClient({ message: message({ Attributes: { ApproximateReceiveCount: '3' } }), @@ -942,6 +1001,22 @@ describe('SqsTaskQueue options', () => { (err: Error & { code?: string }) => err.code === 'RESIZE_CONFIG_QUEUE_LOCK_TTL_INVALID', ); + assert.throws( + () => + new SqsTaskQueue({ + queueUrl: 'q', + timing: { lockTtlMs: { dispatch: 1000, worker: 1000, failed: 0 } }, + }), + (err: Error & { code?: string }) => + err.code === 'RESIZE_CONFIG_QUEUE_LOCK_TTL_INVALID' && + /lockTtlMs\.failed/.test(err.message), + ); + // Optional: the default cooldown applies. + const tasks = new SqsTaskQueue({ + queueUrl: 'q', + timing: { lockTtlMs: { dispatch: 1000, worker: 1000 } }, + }); + assert.equal(timingOf(tasks).lockTtlMs.failed, 600_000); }); test('requires queueUrl at construction', () => { assert.throws( diff --git a/src/drivers/sqs.ts b/src/drivers/sqs.ts index 4ce5128..49852c8 100644 --- a/src/drivers/sqs.ts +++ b/src/drivers/sqs.ts @@ -3,6 +3,7 @@ // claim = ReceiveMessage (visibility timeout = lease; ApproximateReceiveCount = attempts) // renew = ChangeMessageVisibility complete = DeleteMessage // retry = ChangeMessageVisibility(delay) dead = send to deadLetterQueueUrl (if set), delete +// release = ChangeMessageVisibility(0) // The SQS client is built from region/endpoint on first use unless the host passes `client`; // credentials come from the AWS provider chain. Subpath-only entry: the optional AWS SDK peer is // resolved only when this file is imported, so a missing SDK fails at the host's import line. @@ -20,6 +21,7 @@ import { TaskQueue, } from '../contracts/taskQueue.ts'; import { ResizeSetupError } from '../errors.ts'; +import { validateFailedLockTtl } from '../queue.ts'; import { validateLockTtlMs } from '../resizeConfig.ts'; import type { MissingPreview, @@ -79,6 +81,7 @@ export class SqsTaskQueue extends TaskQueue { } if (opts.timing?.lockTtlMs !== undefined) { validateLockTtlMs(opts.timing.lockTtlMs); + validateFailedLockTtl(opts.timing.lockTtlMs.failed); } // erasableSyntaxOnly: no parameter properties — assign fields explicitly. this.#opts = opts; @@ -264,6 +267,13 @@ export class SqsTaskQueue extends TaskQueue { } } + // Visible again at once. SQS cannot lower a message's receive count, so unlike the database + // queue this delivery still counts as an attempt: a task released on its last allowed delivery + // is dead-lettered by the next claim without running. + async release(task: ClaimedTask): Promise { + return this.#setVisibility(task, 0); + } + async #setVisibility( task: ClaimedTask, visibilitySeconds: number, diff --git a/src/engine.test.ts b/src/engine.test.ts index 982ac76..a9ed012 100644 --- a/src/engine.test.ts +++ b/src/engine.test.ts @@ -55,9 +55,6 @@ function makeStorage(o: Partial = {}): ResizeStorage { download: async () => Buffer.alloc(0), upload: async () => ({ key: 'k' }), publicUrl: (ref: StorageRef) => `https://cdn/${ref.key}`, - // The default fake models a storage driver whose originals are public. Tests that need to - // exercise private-original behavior override this explicitly with `false` or omit it. - canServeOriginalPublicly: () => true, ...o, }; } @@ -214,65 +211,59 @@ describe('resolve — partitioning', () => { assert.equal(errors.length, 0); }); - test('a throwing original visibility check behaves like a private original and later missing variants enqueue', async () => { - const run = async (check: () => boolean) => { - resetResizerForTests(); - resetAppInstance(); - const { errors } = installFakeApp(); - const { tasks, calls } = makeTasks(); - const { locks } = makeLocks(true); - const r = new FrameworkResizer({ - storage: makeStorage({ canServeOriginalPublicly: check }), - tasks, - db: fakeDb({ locks }), - }); - const { decision } = await r.resolve({ - media: { - id: 'm1', - original: { - storageRef: { key: 'legacy-origin.jpg' }, - bucket: 'retired-bucket', - }, - previews: [ - { - storageRef: { key: 'public-preview.jpg' }, - contentType: 'image/jpeg', - sizeKey: '300x300', - format: 'jpeg', - }, - ], + test('a legacy original on a retired bucket never reaches the driver: its previews serve and missing ones queue', async () => { + const { errors } = installFakeApp(); + const { tasks, calls } = makeTasks(); + const { locks } = makeLocks(true); + const r = new FrameworkResizer({ + storage: makeStorage({ + publicUrl: (ref: StorageRef) => { + if (ref.key === 'legacy-origin.jpg') { + throw new Error('original bucket is no longer allowlisted'); + } + return `https://cdn/${ref.key}`; }, - sizes: [ - { width: 300, height: 300 }, - { width: 100, height: 100 }, + }), + tasks, + db: fakeDb({ locks }), + }); + const { decision } = await r.resolve({ + media: { + id: 'm1', + original: { + storageRef: { key: 'legacy-origin.jpg' }, + bucket: 'retired-bucket', + width: 50, + height: 50, + }, + previews: [ + { + storageRef: { key: 'public-preview.jpg' }, + contentType: 'image/jpeg', + sizeKey: '300x300', + format: 'jpeg', + }, ], - formats: ['jpeg'], - }); - return { calls, decision, errors }; - }; - - const throwing = await run(() => { - throw new Error('original bucket is no longer allowlisted'); + }, + sizes: [ + { width: 300, height: 300 }, + { width: 100, height: 100 }, + ], + formats: ['jpeg'], }); - const privateOriginal = await run(() => false); - - assert.deepEqual(throwing.decision, privateOriginal.decision); - assert.equal(throwing.decision.ready.length, 1); - assert.equal( - throwing.decision.ready[0]?.url, - 'https://cdn/public-preview.jpg', + assert.deepEqual( + decision.ready.map((entry) => entry.url), + ['https://cdn/public-preview.jpg'], ); assert.deepEqual( - throwing.decision.missing.map((m) => m.sizeKey), + decision.missing.map((m) => m.sizeKey), ['100x100'], ); - assert.deepEqual(throwing.calls, privateOriginal.calls); assert.deepEqual( - throwing.calls[0]?.previews.map((p) => p.sizeKey), + calls[0]?.previews.map((p) => p.sizeKey), ['100x100'], ); - assert.equal(throwing.errors.length, 1); - assert.equal(privateOriginal.errors.length, 0); + assert.equal(errors.length, 0); }); test('a filtered variant is distinct from the unfiltered same size', async () => { @@ -439,6 +430,7 @@ describe('resolve — enqueue wiring', () => { storage: makeStorage(), tasks, db: fakeDb({ locks }), + pipelines: { photo: {} }, }); const media: MediaLike = { id: 'm1', @@ -736,7 +728,6 @@ describe('resolve — SVG raster previews', () => { let signedCalls = 0; const r = new FrameworkResizer({ storage: makeStorage({ - canServeOriginalPublicly: () => true, signedUrl: async () => { signedCalls++; return 'https://signed/private.svg'; @@ -801,281 +792,426 @@ describe('resolve — SVG raster previews', () => { }); // --------------------------------------------------------------------------- -// §17 step 7 — "original already fits" fast-path +// The original is never served in place of a preview // --------------------------------------------------------------------------- -describe('resolve — original-fits fast-path', () => { - const fitsMedia = (): MediaLike => ({ +describe('resolve — the original is never served', () => { + // Smaller than every requested box: the worker makes a preview at its own size instead. + const smallOriginal = (): MediaLike => ({ id: 'm1', original: { storageRef: { key: 'orig.jpg' }, contentType: 'image/jpeg', - width: 200, - height: 150, + width: 100, + height: 100, }, }); - test('serves the original (isOriginal, no preview) when it fits both dims', async () => { + test('a small public original with a 300×300 size is missing and queued', async () => { installFakeApp(); - const r = new FrameworkResizer({ storage: makeStorage() }); - const { decision } = await r.resolve({ - media: fitsMedia(), - sizes: [{ width: 300, height: 300 }], - formats: ['jpeg'], - enqueueMissing: false, - }); - assert.equal(decision.missing.length, 0); - assert.equal(decision.ready.length, 1); - assert.deepEqual(decision.ready[0], { - sizeKey: '300x300', - format: 'jpeg', - url: 'https://cdn/orig.jpg', - isOriginal: true, - contentType: 'image/jpeg', + const { tasks, calls } = makeTasks(); + const { locks } = makeLocks(true); + const r = new FrameworkResizer({ + // A custom driver written before the removal still declares its originals public: ignored. + storage: makeStorage({ + canServeOriginalPublicly: () => true, + } as Partial), + tasks, + db: fakeDb({ locks }), }); - }); - - test('does NOT fire when only one dim fits — becomes missing', async () => { - installFakeApp(); - const r = new FrameworkResizer({ storage: makeStorage() }); - const media: MediaLike = { - id: 'm1', - original: { storageRef: { key: 'orig.jpg' }, width: 200, height: 400 }, - }; const { decision } = await r.resolve({ - media, + media: smallOriginal(), sizes: [{ width: 300, height: 300 }], - formats: ['jpeg'], - enqueueMissing: false, + formats: ['jpeg', 'webp'], }); - assert.equal(decision.ready.length, 0); - assert.equal(decision.missing.length, 1); + assert.deepEqual(decision.ready, []); + assert.deepEqual(decision.missing, [ + { + sizeKey: '300x300', + format: 'jpeg', + requestedWidth: 300, + requestedHeight: 300, + }, + { + sizeKey: '300x300', + format: 'webp', + requestedWidth: 300, + requestedHeight: 300, + }, + ]); + assert.equal(calls.length, 1); + assert.deepEqual( + calls[0].previews.map((p) => `${p.sizeKey}:${p.format}`), + ['300x300:jpeg', '300x300:webp'], + ); }); - test('does NOT fire when filters are present — becomes missing', async () => { + test('an owner or admin ctx changes nothing: the original is neither signed nor linked', async () => { installFakeApp(); - const r = new FrameworkResizer({ storage: makeStorage() }); - const media: MediaLike = { - id: 'm1', - original: { storageRef: { key: 'orig.jpg' }, width: 100, height: 100 }, - }; - const { decision } = await r.resolve({ - media, - sizes: [{ width: 300, height: 300, filters: { blur: 40 } }], - formats: ['jpeg'], - enqueueMissing: false, + const signed: StorageRef[] = []; + const linked: StorageRef[] = []; + const r = new FrameworkResizer({ + storage: makeStorage({ + publicUrl: (ref: StorageRef) => { + linked.push(ref); + return `https://cdn/${ref.key}`; + }, + signedUrl: async (ref: StorageRef) => { + signed.push(ref); + return `https://signed/${ref.key}`; + }, + }), }); - assert.equal(decision.ready.length, 0); - assert.equal(decision.missing.length, 1); - assert.deepEqual(decision.missing[0].filters, { blur: 40 }); + const read = (ctx: Record) => + r.resolve({ + media: smallOriginal(), + sizes: [{ width: 300, height: 300 }], + formats: ['jpeg'], + ctx, + enqueueMissing: false, + }); + const anonymous = await read({}); + assert.deepEqual(anonymous.decision.ready, []); + assert.equal(anonymous.decision.missing.length, 1); + for (const ctx of [{ isOwner: true }, { isAdmin: true }]) { + assert.deepEqual((await read(ctx)).decision, anonymous.decision); + } + assert.deepEqual(signed, []); + assert.deepEqual(linked, []); }); - test('does NOT fire for a width-only size — becomes missing', async () => { + test('every size shape is missing for a step-less pipeline too', async () => { installFakeApp(); - const r = new FrameworkResizer({ storage: makeStorage() }); - const media: MediaLike = { - id: 'm1', - original: { storageRef: { key: 'orig.jpg' }, width: 100, height: 100 }, - }; - const { decision } = await r.resolve({ - media, - sizes: [{ width: 300 }], - formats: ['jpeg'], - enqueueMissing: false, + const r = new FrameworkResizer({ + storage: makeStorage(), + pipelines: { plain: { beforeSteps: [], variantSteps: [] } }, }); - assert.equal(decision.ready.length, 0); - assert.equal(decision.missing.length, 1); - assert.equal(decision.missing[0].sizeKey, '300w'); + for (const pipeline of ['default', 'plain']) { + const { decision } = await r.resolve({ + media: smallOriginal(), + sizes: [ + { width: 300, height: 300 }, + { width: 300 }, + { fit: true }, + { width: 300, height: 300, filters: { blur: 40 } }, + ], + formats: ['jpeg'], + pipeline, + enqueueMissing: false, + }); + assert.deepEqual(decision.ready, []); + assert.deepEqual( + decision.missing.map((m) => m.sizeKey), + ['300x300', '300w', 'fit', '300x300'], + ); + } }); +}); - test('does NOT fire when original dims are unknown — becomes missing', async () => { - installFakeApp(); - const r = new FrameworkResizer({ storage: makeStorage() }); - const media: MediaLike = { - id: 'm1', - original: { storageRef: { key: 'orig.jpg' }, contentType: 'image/jpeg' }, - }; - const { decision } = await r.resolve({ - media, - sizes: [{ width: 300, height: 300 }], - formats: ['jpeg'], - enqueueMissing: false, - }); - assert.equal(decision.ready.length, 0); - assert.equal(decision.missing.length, 1); - }); +// --------------------------------------------------------------------------- +// §17 never-throw guarantee +// --------------------------------------------------------------------------- - test('uses signedUrl for an owner/admin when the driver supports it', async () => { - installFakeApp(); - const signedCalls: { ref: StorageRef; ttl: number }[] = []; - const storage = makeStorage({ - signedUrl: async (ref, ttl) => { - signedCalls.push({ ref, ttl }); - return `https://signed/${ref.key}?ttl=${ttl}`; +describe('resolve — never throws', () => { + // p2's ref is rejected by the driver (e.g. its bucket is no longer allowlisted). + const throwingStorage = () => + makeStorage({ + publicUrl: (ref: StorageRef) => { + if (ref.key === 'p2') { + throw new Error('bucket is not allowlisted'); + } + return `https://cdn/${ref.key}`; }, }); - const r = new FrameworkResizer({ storage }); - const { decision } = await r.resolve({ - media: fitsMedia(), - sizes: [{ width: 300, height: 300 }], - formats: ['jpeg'], - ctx: { isOwner: true }, - enqueueMissing: false, - }); - assert.equal(decision.ready.length, 1); - assert.equal(decision.ready[0].isOriginal, true); - assert.match(decision.ready[0].url, /^https:\/\/signed\/orig\.jpg/); - assert.equal(signedCalls.length, 1); - assert.equal(signedCalls[0].ref.key, 'orig.jpg'); + const storedPreview = (key: string, sizeKey: string) => ({ + storageRef: { key }, + contentType: 'image/jpeg', + sizeKey, + format: 'jpeg', + }); + const threeStored = (): MediaLike => ({ + id: 'm1', + original: { storageRef: { key: 'orig.jpg' }, width: 4000, height: 3000 }, + previews: [ + storedPreview('p1', '300x300'), + storedPreview('p2', '100x100'), + storedPreview('p3', '50x50'), + ], }); + const sizes = [ + { width: 300, height: 300 }, + { width: 100, height: 100 }, + { width: 50, height: 50 }, + { width: 600, height: 600 }, + ]; - test('falls back to publicUrl when signedUrl throws for a public original', async () => { - installFakeApp(); - const storage = makeStorage({ - signedUrl: async () => { - throw new Error('presign down'); + test('a publicUrl throw skips only that cell: later cells, the hook and the enqueue proceed', async () => { + const { errors } = installFakeApp(); + const { tasks, calls } = makeTasks(); + const { locks } = makeLocks(true); + const r = new FrameworkResizer({ + storage: throwingStorage(), + tasks, + db: fakeDb({ locks }), + hooks: { + formatPublicUrls: (decision) => + decision.ready.map((entry) => entry.url), }, }); - const r = new FrameworkResizer({ storage }); - const { decision } = await r.resolve({ - media: fitsMedia(), - sizes: [{ width: 300, height: 300 }], + const { decision, output } = await r.resolve({ + media: threeStored(), + sizes, formats: ['jpeg'], - ctx: { isAdmin: true }, - enqueueMissing: false, + enqueueMissing: true, }); - assert.equal(decision.ready.length, 1); - assert.equal(decision.ready[0].url, 'https://cdn/orig.jpg'); - assert.equal(decision.ready[0].isOriginal, true); + assert.deepEqual( + decision.ready.map((entry) => entry.url), + ['https://cdn/p1', 'https://cdn/p3'], + ); + // The rejected preview is neither ready nor missing; the absent size still queues. + assert.deepEqual( + decision.missing.map((m) => m.sizeKey), + ['600x600'], + ); + assert.deepEqual(output, ['https://cdn/p1', 'https://cdn/p3']); + assert.equal(calls.length, 1); + assert.deepEqual( + calls[0].previews.map((p) => p.sizeKey), + ['600x600'], + ); + assert.equal(errors.length, 1); }); - test('a private raster original stays missing for an anonymous reader', async () => { - installFakeApp(); - const r = new FrameworkResizer({ - storage: makeStorage({ canServeOriginalPublicly: () => false }), - }); + test('a publicUrl throw without enqueueing still returns every other cell', async () => { + const { errors } = installFakeApp(); + const r = new FrameworkResizer({ storage: throwingStorage() }); const { decision } = await r.resolve({ - media: fitsMedia(), - sizes: [{ width: 300, height: 300 }], + media: threeStored(), + sizes, formats: ['jpeg'], enqueueMissing: false, }); - assert.equal(decision.ready.length, 0); - assert.equal(decision.missing.length, 1); - assert.equal(decision.missing[0].sizeKey, '300x300'); + assert.deepEqual( + decision.ready.map((entry) => entry.url), + ['https://cdn/p1', 'https://cdn/p3'], + ); + assert.deepEqual( + decision.missing.map((m) => m.sizeKey), + ['600x600'], + ); + assert.equal(errors.length, 1); }); - test('a private original does not fall back to publicUrl when signing fails', async () => { - installFakeApp(); - let publicUrlCalls = 0; - const r = new FrameworkResizer({ - storage: makeStorage({ - canServeOriginalPublicly: () => false, - signedUrl: async () => { - throw new Error('presign down'); - }, - publicUrl: (ref: StorageRef) => { - publicUrlCalls += 1; - return `https://cdn/${ref.key}`; - }, - }), - }); - const { decision } = await r.resolve({ - media: fitsMedia(), - sizes: [{ width: 300, height: 300 }], - formats: ['jpeg'], - ctx: { isAdmin: true }, - enqueueMissing: false, - }); - assert.equal(decision.ready.length, 0); - assert.equal(decision.missing.length, 1); - assert.equal(publicUrlCalls, 0); + test('resolve(undefined) returns the logged safe empty decision', async () => { + const { errors } = installFakeApp(); + const r = new FrameworkResizer({ storage: makeStorage() }); + const { decision, output } = await r.resolve(undefined as never); + assert.deepEqual(decision, { ready: [], missing: [] }); + assert.equal(output, undefined); + assert.equal(errors.length, 1); }); - test('a custom storage without canServeOriginalPublicly has no original fast-path', async () => { - installFakeApp(); - const storage: ResizeStorage = { - download: async () => Buffer.alloc(0), - upload: async () => ({ key: 'k' }), - publicUrl: (ref: StorageRef) => `https://cdn/${ref.key}`, - }; - const r = new FrameworkResizer({ storage }); - const { decision } = await r.resolve({ - media: fitsMedia(), + test('media with no id/_id → logged safe empty decision (never-throw wrapper absorbs requireMediaId)', async () => { + const { errors } = installFakeApp(); + const r = new FrameworkResizer({ storage: makeStorage() }); + const { decision, output } = await r.resolve({ + media: { + original: { + storageRef: { key: 'orig.jpg' }, + contentType: 'image/jpeg', + }, + }, sizes: [{ width: 300, height: 300 }], formats: ['jpeg'], - enqueueMissing: false, }); - assert.equal(decision.ready.length, 0); - assert.equal(decision.missing.length, 1); + assert.deepEqual(decision, { ready: [], missing: [] }); + assert.equal(output, undefined); + assert.ok(errors.length >= 1); }); }); -// --------------------------------------------------------------------------- -// §17 never-throw guarantee -// --------------------------------------------------------------------------- - -describe('resolve — never throws', () => { - test('a mid-loop storage.publicUrl throw yields the safe value (ready-so-far, empty missing)', async () => { +describe('resolve — unknown pipeline', () => { + test('serves stored previews of that pipeline, reports nothing missing, queues nothing and logs', async () => { const { errors } = installFakeApp(); - const storage = makeStorage({ - publicUrl: (ref: StorageRef) => { - if (ref.key === 'p2') { - throw new Error('cdn boom'); - } - return `https://cdn/${ref.key}`; + const { tasks, calls } = makeTasks(); + const { locks, acquired } = makeLocks(true); + let beforeEnqueueCalls = 0; + const r = new FrameworkResizer({ + storage: makeStorage(), + tasks, + db: fakeDb({ locks }), + hooks: { + beforeEnqueue: (missing) => { + beforeEnqueueCalls += 1; + return [...missing, { sizeKey: '10x10', format: 'jpeg' }]; + }, + formatPublicUrls: (decision) => + decision.ready.map((entry) => entry.url), }, }); - const r = new FrameworkResizer({ storage }); const media: MediaLike = { id: 'm1', + // Smaller than the box: still not served, for an unknown pipeline as for any other. + original: { storageRef: { key: 'orig.jpg' }, width: 100, height: 100 }, previews: [ { - storageRef: { key: 'p1' }, - contentType: 'image/jpeg', + storageRef: { key: 'retired.webp' }, + contentType: 'image/webp', sizeKey: '300x300', - format: 'jpeg', + format: 'webp', + pipeline: 'retired', }, { - storageRef: { key: 'p2' }, + storageRef: { key: 'default.jpg' }, contentType: 'image/jpeg', - sizeKey: '100x100', + sizeKey: '300x300', format: 'jpeg', }, ], }; const { decision, output } = await r.resolve({ media, + sizes: [{ width: 300, height: 300 }, { width: 600 }], + formats: ['jpeg', 'webp'], + pipeline: 'retired', + enqueueMissing: true, + }); + assert.deepEqual( + decision.ready.map((entry) => entry.url), + ['https://cdn/retired.webp'], + ); + assert.deepEqual(decision.missing, []); + assert.deepEqual(output, ['https://cdn/retired.webp']); + assert.equal(beforeEnqueueCalls, 0); + assert.equal(calls.length, 0); + assert.equal(acquired.length, 0); + assert.equal(errors.length, 1); + assert.match(String(errors[0][0]), /'retired'/); + }); + + test('"default" is always known, registered or not', async () => { + const { errors } = installFakeApp(); + const r = new FrameworkResizer({ storage: makeStorage() }); + const { decision } = await r.resolve({ + media: { id: 'm1' }, + sizes: [{ width: 300, height: 300 }], + formats: ['jpeg'], + pipeline: 'default', + enqueueMissing: false, + }); + assert.equal(decision.missing.length, 1); + assert.equal(errors.length, 0); + }); +}); + +describe('resolve — per-call formats', () => { + test('formats without an encode.formats entry are dropped with one logged error', async () => { + const { errors } = installFakeApp(); + const { tasks, calls } = makeTasks(); + const { locks } = makeLocks(true); + const r = new FrameworkResizer({ + storage: makeStorage(), + tasks, + db: fakeDb({ locks }), + }); + const { decision } = await r.resolve({ + media: { id: 'm1', original: { storageRef: { key: 'orig.jpg' } } }, sizes: [ - { width: 300, height: 300 }, { width: 100, height: 100 }, + { width: 200, height: 200 }, ], - formats: ['jpeg'], - enqueueMissing: false, + // 'toString' is inherited, not an own key of encode.formats. + formats: ['jpg', 'webp', '../../etc', 'toString', 'jpg'], }); - assert.equal(decision.ready.length, 1); - assert.equal(decision.ready[0].url, 'https://cdn/p1'); - assert.equal(decision.missing.length, 0); - assert.equal(output, undefined); - assert.ok(errors.length >= 1); + assert.deepEqual( + decision.missing.map((m) => `${m.sizeKey}:${m.format}`), + ['100x100:webp', '200x200:webp'], + ); + assert.deepEqual( + calls[0].previews.map((p) => p.format), + ['webp', 'webp'], + ); + assert.equal(errors.length, 1); + const message = String(errors[0][0]); + for (const dropped of ['jpg', '../../etc', 'toString']) { + assert.ok(message.includes(dropped), `${dropped} named in: ${message}`); + } + assert.ok(!message.includes('webp')); }); - test('media with no id/_id → logged safe empty decision (never-throw wrapper absorbs requireMediaId)', async () => { + test('a beforeEnqueue tap cannot queue a format without an encode.formats entry', async () => { + const { errors } = installFakeApp(); + const { tasks, calls } = makeTasks(); + const { locks } = makeLocks(true); + const r = new FrameworkResizer({ + storage: makeStorage(), + tasks, + db: fakeDb({ locks }), + hooks: { + beforeEnqueue: (missing) => [ + ...missing, + { + sizeKey: '300x300', + format: 'jpg', + requestedWidth: 300, + requestedHeight: 300, + }, + ], + }, + }); + const { decision } = await r.resolve({ + media: { id: 'm1', original: { storageRef: { key: 'orig.jpg' } } }, + sizes: [{ width: 300, height: 300 }], + formats: ['webp'], + }); + assert.deepEqual( + decision.missing.map((m) => m.format), + ['webp'], + ); + assert.deepEqual( + calls[0].previews.map((p) => p.format), + ['webp'], + ); + assert.equal(errors.length, 1); + assert.match(String(errors[0][0]), /"jpg".*beforeEnqueue/); + }); + + test('a stored preview of an unconfigured format is not served', async () => { const { errors } = installFakeApp(); const r = new FrameworkResizer({ storage: makeStorage() }); - const { decision, output } = await r.resolve({ + const { decision } = await r.resolve({ media: { - original: { - storageRef: { key: 'orig.jpg' }, - contentType: 'image/jpeg', - }, + id: 'm1', + previews: [ + { + storageRef: { key: 'old.png' }, + contentType: 'image/png', + sizeKey: '300x300', + format: 'png', + }, + ], }, sizes: [{ width: 300, height: 300 }], - formats: ['jpeg'], + formats: ['png'], + enqueueMissing: false, }); assert.deepEqual(decision, { ready: [], missing: [] }); - assert.equal(output, undefined); - assert.ok(errors.length >= 1); + assert.equal(errors.length, 1); + }); + + test('the configured default formats log nothing', async () => { + const { errors } = installFakeApp(); + const r = new FrameworkResizer({ storage: makeStorage() }); + const { decision } = await r.resolve({ + media: { id: 'm1' }, + sizes: [{ width: 300, height: 300 }], + enqueueMissing: false, + }); + assert.deepEqual( + decision.missing.map((m) => m.format), + ['jpeg', 'webp', 'avif'], + ); + assert.equal(errors.length, 0); }); }); @@ -1137,7 +1273,10 @@ describe('pipelines are part of preview identity', () => { test('a default preview is not served for another pipeline', async () => { installFakeApp(); - const r = new FrameworkResizer({ storage: makeStorage() }); + const r = new FrameworkResizer({ + storage: makeStorage(), + pipelines: { watermark: {} }, + }); const media = { id: 'm1', previews: [stored] }; const clean = await r.resolve({ media, sizes, formats: ['webp'] }); const watermarked = await r.resolve({ @@ -1153,7 +1292,10 @@ describe('pipelines are part of preview identity', () => { test('a preview stored for a pipeline is served only to that pipeline', async () => { installFakeApp(); - const r = new FrameworkResizer({ storage: makeStorage() }); + const r = new FrameworkResizer({ + storage: makeStorage(), + pipelines: { watermark: {} }, + }); const media = { id: 'm1', previews: [{ ...stored, pipeline: 'watermark' }], diff --git a/src/engine.ts b/src/engine.ts index 2f20c6d..2e1253b 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -1,32 +1,29 @@ // The read-path engine (06 · §17). `resizer.resolve` delegates here: it partitions the -// requested size×format grid into ready (served from an existing preview -// or an "original already fits" raster original) vs missing (handed to enqueue), threading three -// host waterfalls (resolveSizes / beforeEnqueue / formatPublicUrls) and never throwing into -// the caller's read. All URLs come from the PURE, I/O-free storage.publicUrl; the only I/O -// is the owner/admin-gated signedUrl (itself caught + fallen back). Imports the Resizer -// TYPE only — resizer.ts imports resolveImpl as a value, so this cycle is runtime-free. +// requested size×format grid into ready (served from a stored preview) vs missing (handed to +// enqueue), threading three host waterfalls (resolveSizes / beforeEnqueue / formatPublicUrls) +// and never throwing into the caller's read. The original itself is never served: an original +// smaller than the box still gets a preview, made by the worker at the original's own size. +// Every URL comes from the PURE, I/O-free storage.publicUrl of a stored preview. Imports the +// Resizer TYPE only — resizer.ts imports resolveImpl as a value, so this cycle is runtime-free. import { canonicalizeVariants, enqueue, enqueueConfirmed } from './enqueue.ts'; import { ResizeConfigError, ResizeMediaError, ResizeSetupError, } from './errors.ts'; -import { isPositiveFinite } from './helpers/guards.ts'; import { expandPreviewRequests, - getFilterSig, getPreviewIdentity, getSizeKey, - isSvgOriginal, isUsablePreview, previewScope, requireMediaId, + toMissingPreview, } from './images.ts'; import type { Resizer } from './resizer.ts'; import type { MediaLike, MissingPreview, - Original, Preview, PreviewFormat, PrewarmResult, @@ -40,7 +37,7 @@ export interface ResolveOpts { sizes: SizeInput[]; pipeline?: string; // selects a registered pipeline; default 'default' formats?: PreviewFormat[]; // default = config.formats - ctx?: Record; // threaded to read-path hooks; ctx.isOwner/isAdmin gate signedUrl + ctx?: Record; // threaded to the read-path hooks enqueueMissing?: boolean; // default true when a task queue is set, false otherwise queue?: string; // queue for missing variants; default resizer.queue } @@ -54,11 +51,6 @@ export interface PrewarmOpts { queue?: string; // queue for missing variants; default resizer.queue } -// Owner/admin private-original reads: short-lived by design (the only read-path I/O). A -// small constant is fine — the URL is re-minted on every read, so it never needs to outlive -// one response. -const SIGNED_ORIGINAL_TTL_SECONDS = 300; // 5 minutes - /** * §17 steps 1–11. See the module header for the shape. The ENTIRE body runs inside a * try/catch (the never-throw guarantee, layer 3): on any unexpected internal error it logs @@ -69,12 +61,13 @@ export async function resolveImpl( resizer: Resizer, opts: ResolveOpts, ): Promise<{ decision: ReadDecision; output: unknown }> { - const { media } = opts; // Built incrementally so the never-throw catch can still return what was produced. const ready: ReadyEntry[] = []; const decision: ReadDecision = { ready, missing: [] }; try { + // Inside the guard, so a missing options object also returns the safe empty decision. + const { media } = opts; await resizer.ready(); // drivers given as functions load here, inside the never-throw guard const ctx = opts.ctx ?? {}; const storage = resizer.storage; // required constructor option — always present (§17.3) @@ -82,6 +75,14 @@ export async function resolveImpl( // Inside the never-throw try: a media with no id/_id logs + returns the safe empty decision // rather than enqueueing under the literal 'undefined' key (04 · papercut). const mediaId = requireMediaId(media); + // A pipeline this process does not know (e.g. registered only in another process) has + // unknown steps: serve what is already stored for it, but no task can be trusted to render it. + const knownPipeline = resizer.hasPipeline(pipeline); + if (!knownPipeline) { + resizer.logger.error( + `resize resolve: pipeline '${pipeline}' is not registered on Resizer '${resizer.name}' — serving stored previews only, nothing is queued`, + ); + } // 1. Host size magic (expand/inject/map/dedupe). Guarded per-tap inside runWaterfall. const sizes = (await resizer.runWaterfall( @@ -90,7 +91,11 @@ export async function resolveImpl( ctx, )) as SizeInput[]; - const formats = opts.formats ?? resizer.config.formats; + const { formats } = configuredFormats( + resizer, + opts.formats, + 'resize resolve', + ); // 5. previewMap keyed by identity — only complete entries (both key + contentType). Stored // previews keep the scope they were rendered in, so another pipeline or Resizer never @@ -106,35 +111,8 @@ export async function resolveImpl( } } - const original = media.original; const missing: MissingPreview[] = []; const missingSeen = new Set(); - const originalIsSvg = isSvgOriginal(original); - // Compute this lazily. A generated preview is independently public and must remain - // readable even when a legacy original now points to a retired/unavailable bucket. - // A driver that does not implement the check is deliberately conservative: a public URL - // from an arbitrary custom driver is not enough proof that an original is safe to expose. - let originalIsPublic: boolean | undefined; - const isOriginalPublic = (): boolean => { - if (originalIsPublic === undefined) { - try { - originalIsPublic = - original != null && - original.storageRef != null && - storage.canServeOriginalPublicly?.(original.storageRef) === true; - } catch (err) { - resizer.logger.error( - 'resize resolve: canServeOriginalPublicly threw — treating original as private', - err, - ); - originalIsPublic = false; - } - } - return originalIsPublic; - }; - const authorizedOriginalRead = Boolean( - (ctx.isOwner || ctx.isAdmin) && storage.signedUrl, - ); // 7. Per requested size × format. for (const size of sizes) { @@ -154,10 +132,23 @@ export async function resolveImpl( const existing = previewMap.get(identity); if (existing) { // exists → serve the generated preview. + let url: string; + try { + url = storage.publicUrl(existing.storageRef); + } catch (err) { + // A ref the driver rejects (e.g. a bucket no longer allowlisted) loses only this + // cell. It is neither ready nor missing: the preview is stored, so queueing it again + // would not help. + resizer.logger.error( + `resize resolve: publicUrl failed for stored preview ${identity} of media ${mediaId} — skipping it`, + err, + ); + continue; + } const entry: ReadyEntry = { sizeKey, format, - url: storage.publicUrl(existing.storageRef), + url, preview: existing, contentType: existing.contentType, }; @@ -167,71 +158,34 @@ export async function resolveImpl( ready.push(entry); continue; } - - // "original already fits" fast-path — ALL of (a)–(d) must hold (§17 step 7). - if ( - original && - !originalIsSvg && - (isOriginalPublic() || authorizedOriginalRead) && - getFilterSig(size.filters) === 'none' && // (a) no filters - !size.fit && - isPositiveFinite(size.width) && // (b) plain cover WxH - isPositiveFinite(size.height) && - isPositiveFinite(original.width) && // (c) original dims known - isPositiveFinite(original.height) && - original.width <= size.width && // (d) not larger than the box - original.height <= size.height - ) { - const url = await originalUrl( - resizer, - original, - ctx, - isOriginalPublic(), - ); - if (url !== undefined) { - const fits: ReadyEntry = { - sizeKey, - format, - url, - isOriginal: true, - }; - if (original.contentType) { - fits.contentType = original.contentType; - } - ready.push(fits); - continue; - } + if (!knownPipeline) { + continue; // stored previews only } - // missing → deduped by identity. + // missing → deduped by identity, whatever the original's size or the reader's ctx. if (missingSeen.has(identity)) { continue; } missingSeen.add(identity); - const mp: MissingPreview = { sizeKey, format }; - if (size.filters && Object.keys(size.filters).length > 0) { - mp.filters = size.filters; - } - if (isPositiveFinite(size.width)) { - mp.requestedWidth = size.width; - } - if (isPositiveFinite(size.height)) { - mp.requestedHeight = size.height; - } - if (size.fit) { - mp.fit = true; - } - missing.push(mp); + missing.push(toMissingPreview(size, sizeKey, format)); } } // 8. beforeEnqueue — REASSIGN the (post-hook) missing set so steps 9–10 + the host's - // formatPublicUrls all see exactly what was enqueued. - decision.missing = (await resizer.runWaterfall( - 'beforeEnqueue', - missing, - ctx, - )) as MissingPreview[]; + // formatPublicUrls all see exactly what was enqueued. An unknown pipeline skips the hook, so + // a tap cannot add variants that would be queued for it; a variant the tap added or rewrote + // to an unconfigured format is dropped. + decision.missing = knownPipeline + ? splitConfiguredVariants( + resizer, + (await resizer.runWaterfall( + 'beforeEnqueue', + missing, + ctx, + )) as MissingPreview[], + 'resize resolve', + ).configured + : []; // 9. Enqueue the missing variants. Default follows construction: a task queue means // lazy mode (enqueue), none means eager-only (do not log-on-every-read). @@ -291,8 +245,7 @@ export async function resolveImpl( * Pre-warm the catalog at UPLOAD: queue every missing variant without blocking on image work, and * report each requested variant (ready / accepted / not required / unconfirmed, with task receipts * and issues). A held dispatch lock never counts as queued: the queue's findActive() must - * confirm it. Uses the read path's `resolveSizes` / `beforeEnqueue` waterfalls; the "original - * already fits" fast-path is not consulted (that is a read-time serving decision). NEVER throws: + * confirm it. Uses the read path's `resolveSizes` / `beforeEnqueue` waterfalls. NEVER throws: * an upload must not fail because pre-warming hiccuped, so an unexpected error becomes * `status: 'incomplete'` with a RESIZE_ENQUEUE_INTERNAL_ERROR issue. */ @@ -300,9 +253,11 @@ export async function prewarmImpl( resizer: Resizer, opts: PrewarmOpts, ): Promise { + // What prewarmStrict has expanded so far: an unexpected error reports it all as unconfirmed. + const progress: PrewarmProgress = { requested: [] }; try { await resizer.ready(); // inside the never-throw guard - return await prewarmStrict(resizer, opts); + return await prewarmStrict(resizer, opts, progress); } catch (err) { logNeverThrow( resizer, @@ -311,11 +266,11 @@ export async function prewarmImpl( ); return { status: 'incomplete', - requested: [], + requested: [...progress.requested], ready: [], accepted: [], notRequired: [], - unconfirmed: [], + unconfirmed: [...progress.requested], tasks: [], issues: [ { @@ -327,17 +282,22 @@ export async function prewarmImpl( err instanceof ResizeConfigError || err instanceof ResizeSetupError ), - previews: [], + previews: [...progress.requested], }, ], }; } } +interface PrewarmProgress { + requested: MissingPreview[]; +} + /** The pre-warm itself: every missing identity is either confirmed by a task receipt or explicit. */ async function prewarmStrict( resizer: Resizer, opts: PrewarmOpts, + progress: PrewarmProgress, ): Promise { const ctx = opts.ctx ?? {}; const { media } = opts; @@ -348,9 +308,40 @@ async function prewarmStrict( opts.sizes, ctx, )) as SizeInput[]; - const formats = opts.formats ?? resizer.config.formats; const scope = { resizer: resizer.name, pipeline }; + const { formats, dropped } = configuredFormats( + resizer, + opts.formats, + 'resize prewarm', + ); + // Requested in formats that have no encoder settings: reported, never queued. + const unconfigured = expandPreviewRequests(sizes, dropped, scope); + const finish = (result: PrewarmResult): PrewarmResult => + withUnconfiguredFormats(result, unconfigured); const requestedBeforePolicy = expandPreviewRequests(sizes, formats, scope); + progress.requested = [...requestedBeforePolicy, ...unconfigured]; + if (!resizer.hasPipeline(pipeline)) { + // Its steps are unknown in this process, so no task can be trusted to render it. + const message = `pipeline '${pipeline}' is not registered on Resizer '${resizer.name}'`; + resizer.logger.error(`resize prewarm: ${message} — nothing is queued`); + return finish({ + status: 'incomplete', + requested: requestedBeforePolicy, + ready: [], + accepted: [], + notRequired: [], + unconfirmed: requestedBeforePolicy, + tasks: [], + issues: [ + { + code: 'RESIZE_PIPELINE_UNKNOWN', + message, + retryable: false, + previews: requestedBeforePolicy, + }, + ], + }); + } const empty = (): PrewarmResult => ({ status: 'not-required', reason: 'empty-request', @@ -363,7 +354,7 @@ async function prewarmStrict( issues: [], }); if (requestedBeforePolicy.length === 0) { - return empty(); + return finish(empty()); } const readyIdentities = new Set(); @@ -400,7 +391,7 @@ async function prewarmStrict( ), ), ); - const required = canonicalizeVariants( + const afterPolicy = canonicalizeVariants( (await resizer.runWaterfall( 'beforeEnqueue', missingBeforePolicy, @@ -417,6 +408,32 @@ async function prewarmStrict( ), ), ); + // A tap may add or rewrite variants: one in an unconfigured format is reported like a per-call + // one (once per identity), never queued. + const { configured: required, unconfigured: fromPolicy } = + splitConfiguredVariants(resizer, afterPolicy, 'resize prewarm'); + const unconfiguredIdentities = new Set( + unconfigured.map((preview) => + getPreviewIdentity( + scope, + preview.sizeKey, + preview.format, + preview.filters, + ), + ), + ); + for (const preview of fromPolicy) { + const identity = getPreviewIdentity( + scope, + preview.sizeKey, + preview.format, + preview.filters, + ); + if (!unconfiguredIdentities.has(identity)) { + unconfiguredIdentities.add(identity); + unconfigured.push(preview); + } + } const requiredIdentities = new Set( required.map((preview) => getPreviewIdentity( @@ -439,8 +456,9 @@ async function prewarmStrict( ), ); const requested = [...ready, ...required, ...notRequired]; + progress.requested = [...requested, ...unconfigured]; if (required.length === 0) { - return { + return finish({ status: ready.length > 0 ? 'ready' : 'not-required', ...(ready.length === 0 ? { reason: 'filtered' as const } : {}), requested, @@ -450,10 +468,10 @@ async function prewarmStrict( unconfirmed: [], tasks: [], issues: [], - }; + }); } if (media.original?.storageRef == null) { - return { + return finish({ status: 'incomplete', requested, ready, @@ -469,7 +487,7 @@ async function prewarmStrict( previews: required, }, ], - }; + }); } const attempt = await enqueueConfirmed( @@ -479,7 +497,7 @@ async function prewarmStrict( required, opts.queue ?? resizer.queue, ); - return { + return finish({ status: attempt.unconfirmed.length > 0 ? 'incomplete' : 'accepted', requested, ready, @@ -488,38 +506,116 @@ async function prewarmStrict( unconfirmed: attempt.unconfirmed, tasks: attempt.tasks, issues: attempt.issues, + }); +} + +/** + * Add the variants requested in formats without an `encode.formats` entry: requested, never + * queued, and unconfirmed with a non-retryable issue (the request itself must change). + */ +function withUnconfiguredFormats( + result: PrewarmResult, + unconfigured: MissingPreview[], +): PrewarmResult { + if (unconfigured.length === 0) { + return result; + } + const formats = [...new Set(unconfigured.map((preview) => preview.format))]; + return { + status: 'incomplete', + requested: [...result.requested, ...unconfigured], + ready: result.ready, + accepted: result.accepted, + notRequired: result.notRequired, + unconfirmed: [...result.unconfirmed, ...unconfigured], + tasks: result.tasks, + issues: [ + ...result.issues, + { + code: 'RESIZE_FORMAT_NOT_CONFIGURED', + message: `formats ${JSON.stringify(formats)} have no encode.formats entry`, + retryable: false, + previews: unconfigured, + }, + ], }; } /** - * The public URL for an original-backed ready entry. Owner/admin reads get a signed URL - * when the driver supports it — the ONLY read-path I/O. A private original has no public URL - * fallback: if signing fails, return undefined so raster callers leave the variant missing. + * The requested formats (default: config.formats) that have an own `encode.formats` entry. Per-call + * formats are open strings, so an alias such as 'jpg' or an arbitrary id would otherwise be queued + * and encoded without the configured options. The others are dropped and named in one logged error. */ -async function originalUrl( +function configuredFormats( resizer: Resizer, - original: Original, - ctx: Record, - originalIsPublic: boolean, -): Promise { - const storage = resizer.storage; - if ((ctx.isOwner || ctx.isAdmin) && storage.signedUrl) { - try { - return await storage.signedUrl( - original.storageRef, - SIGNED_ORIGINAL_TTL_SECONDS, - ); - } catch (err) { - resizer.logger.error( - 'resize resolve: signedUrl failed — private original stays unavailable', - err, - ); - if (!originalIsPublic) { - return undefined; - } + requested: readonly PreviewFormat[] | undefined, + label: string, +): { formats: PreviewFormat[]; dropped: PreviewFormat[] } { + const formats: PreviewFormat[] = []; + const dropped = new Set(); + for (const format of requested ?? resizer.config.formats) { + if (isConfiguredFormat(resizer, format)) { + formats.push(format); + } else { + dropped.add(format); + } + } + logUnconfiguredFormats(resizer, label, [...dropped], ''); + return { formats, dropped: [...dropped] }; +} + +/** + * Split what a `beforeEnqueue` tap returned by the same rule as configuredFormats: a tap may add + * or rewrite a variant to a format with no encoder settings, which a worker could never produce + * (it would fail, dead-letter, and be queued again by the next read). The unconfigured ones are + * named in one logged error. + */ +function splitConfiguredVariants( + resizer: Resizer, + variants: readonly MissingPreview[], + label: string, +): { configured: MissingPreview[]; unconfigured: MissingPreview[] } { + const configured: MissingPreview[] = []; + const unconfigured: MissingPreview[] = []; + for (const variant of variants) { + // `?.`: a tap's malformed entry is dropped like an unknown format, not thrown on. + if (isConfiguredFormat(resizer, variant?.format)) { + configured.push(variant); + } else { + unconfigured.push(variant); } } - return originalIsPublic ? storage.publicUrl(original.storageRef) : undefined; + logUnconfiguredFormats( + resizer, + label, + [...new Set(unconfigured.map((variant) => variant?.format))], + ' from a beforeEnqueue tap', + ); + return { configured, unconfigured }; +} + +/** True when `format` is an own key of `encode.formats` (format ids are open strings). */ +function isConfiguredFormat( + resizer: Resizer, + format: unknown, +): format is PreviewFormat { + return ( + typeof format === 'string' && + Object.hasOwn(resizer.config.encode.formats, format) + ); +} + +function logUnconfiguredFormats( + resizer: Resizer, + label: string, + formats: readonly unknown[], + origin: string, +): void { + if (formats.length > 0) { + resizer.logger.error( + `${label}: formats ${JSON.stringify(formats)}${origin} have no encode.formats entry — skipped (use a configured Sharp format id, e.g. 'jpeg', not 'jpg')`, + ); + } } /** Log a never-throw catch; a throwing host logger falls back to the console. */ diff --git a/src/enqueue.test.ts b/src/enqueue.test.ts index 562214b..bd81c63 100644 --- a/src/enqueue.test.ts +++ b/src/enqueue.test.ts @@ -270,6 +270,47 @@ describe('enqueue', () => { assert.ok(errors.length >= 1); }); + test('acquires the dispatch locks in parallel; held and rejected locks still drop out', async () => { + const { errors } = installFakeApp(); + const { tasks, calls } = makeTasks(); + let inFlight = 0; + let maxInFlight = 0; + const locks: FakeLocks = { + acquire: async (key) => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + await new Promise((resolve) => setImmediate(resolve)); + inFlight -= 1; + if (key.endsWith(':avif:none')) { + throw new Error('lock backend down'); + } + return !key.endsWith(':webp:none'); // webp is held by a concurrent read + }, + release: async () => {}, + }; + const r = makeResizer({ tasks, locks }); + const enqueued = await enqueue( + r, + 'm1', + 'default', + [ + variant(), + variant({ format: 'webp' }), + variant({ format: 'avif' }), + variant({ sizeKey: '100x100' }), + ], + 'default', + ); + assert.equal(maxInFlight, 4); + assert.equal(calls.length, 1); + assert.deepEqual( + calls[0].previews.map((p) => `${p.sizeKey}:${p.format}`), + ['100x100:jpeg', '300x300:jpeg'], + ); + assert.equal(enqueued, 2); + assert.equal(errors.length, 1); + }); + test('does not call the task queue when no lock survives', async () => { installFakeApp(); const { tasks, calls } = makeTasks(); diff --git a/src/enqueue.ts b/src/enqueue.ts index 1588eae..4ffd564 100644 --- a/src/enqueue.ts +++ b/src/enqueue.ts @@ -141,23 +141,23 @@ export async function enqueue( const dispatchTtlMs = timingOf(tasks).lockTtlMs.dispatch; const survivors: MissingPreview[] = []; const survivorLockKeys: string[] = []; - for (const [identity, m] of byIdentity) { - const lockKey = `resize_dispatch:${mediaId}:${identity}`; - // A rejecting acquire = this variant is NOT a survivor (log + continue); earlier survivors are - // unaffected and still reach the task queue. enqueue must never throw into the read (1.2b). - let acquired: boolean; - try { - acquired = await resizer.db.acquireLock(lockKey, dispatchTtlMs); - } catch (err) { + const attempts = await acquireDispatchLocks( + resizer, + mediaId, + byIdentity, + dispatchTtlMs, + ); + for (const attempt of attempts) { + // A rejecting acquire = this variant is NOT a survivor (log + continue); the other survivors + // are unaffected and still reach the task queue. enqueue must never throw into the read. + if (attempt.failed) { resizer.logger.error( - `resize enqueue: dispatch-lock acquire failed for ${lockKey} on media ${mediaId} — skipping this variant`, - err, + `resize enqueue: dispatch-lock acquire failed for ${attempt.lockKey} on media ${mediaId} — skipping this variant`, + attempt.error, ); - continue; - } - if (acquired) { - survivors.push(m); - survivorLockKeys.push(lockKey); + } else if (attempt.acquired) { + survivors.push(attempt.preview); + survivorLockKeys.push(attempt.lockKey); } } @@ -296,21 +296,24 @@ export async function enqueueConfirmed( const lockFailed: MissingPreview[] = []; const dispatchTtlMs = timingOf(taskQueue).lockTtlMs.dispatch; - for (const [identity, preview] of byIdentity) { - const lockKey = `resize_dispatch:${mediaId}:${identity}`; - try { - if (await resizer.db.acquireLock(lockKey, dispatchTtlMs)) { - winners.push(preview); - winnerKeys.push(lockKey); - } else { - lockContended.push(preview); - } - } catch (error) { + const attempts = await acquireDispatchLocks( + resizer, + mediaId, + byIdentity, + dispatchTtlMs, + ); + for (const attempt of attempts) { + if (attempt.failed) { resizer.logger.error( - `resize prewarm: dispatch-lock acquire failed for ${lockKey}`, - error, + `resize prewarm: dispatch-lock acquire failed for ${attempt.lockKey}`, + attempt.error, ); - lockFailed.push(preview); + lockFailed.push(attempt.preview); + } else if (attempt.acquired) { + winners.push(attempt.preview); + winnerKeys.push(attempt.lockKey); + } else { + lockContended.push(attempt.preview); } } @@ -431,62 +434,99 @@ export async function enqueueConfirmed( previews: receiptConflictPreviews, }); } - const unresolvedIdentities = new Set( - unconfirmed.map((preview) => - getPreviewIdentity( - scope, - preview.sizeKey, - preview.format, - preview.filters, - ), - ), - ); - const remainingContended = lockContended.filter((preview) => - unresolvedIdentities.has( - getPreviewIdentity( - scope, - preview.sizeKey, - preview.format, - preview.filters, - ), - ), - ); - const remainingFailed = lockFailed.filter((preview) => - unresolvedIdentities.has( - getPreviewIdentity( - scope, - preview.sizeKey, - preview.format, - preview.filters, - ), - ), - ); - if (remainingContended.length > 0) { + if (lockContended.length > 0) { issues.push({ code: 'RESIZE_ENQUEUE_LOCK_CONTENDED', message: 'dispatch lock is held but no active task coverage was confirmed', retryable: true, - previews: remainingContended, + previews: lockContended, }); } - if (remainingFailed.length > 0) { + if (lockFailed.length > 0) { issues.push({ code: 'RESIZE_ENQUEUE_LOCK_FAILED', message: 'dispatch lock could not be acquired or confirmed', retryable: true, - previews: remainingFailed, + previews: lockFailed, }); } + // An issue explains only what stayed unconfirmed: a preview that findActive later confirmed + // drops out of it, and an issue left with none is removed, so an accepted request carries + // no retryable issue. + const unresolvedIdentities = new Set( + unconfirmed.map((preview) => + getPreviewIdentity( + scope, + preview.sizeKey, + preview.format, + preview.filters, + ), + ), + ); + const remainingIssues = issues.flatMap((issue) => { + const previews = issue.previews.filter((preview) => + unresolvedIdentities.has( + getPreviewIdentity( + scope, + preview.sizeKey, + preview.format, + preview.filters, + ), + ), + ); + return previews.length > 0 ? [{ ...issue, previews }] : []; + }); + return { accepted: [...accepted.values()], unconfirmed, tasks, - issues, + issues: remainingIssues, }; } +interface LockAttempt { + preview: MissingPreview; + lockKey: string; + acquired: boolean; // false for a lock another request holds, and for a failed acquire + failed: boolean; // the acquire rejected (`error`) + error?: unknown; +} + +/** + * The dispatch lock of one preview identity of a media: while it is held, reads do not queue that + * variant. The worker holds it for a cooldown after the variant's task is dead-lettered. + */ +export function dispatchLockKey(mediaId: string, identity: string): string { + return `resize_dispatch:${mediaId}:${identity}`; +} + +/** + * Acquire one dispatch lock per identity, all at once: a read with many missing variants waits + * for the slowest acquire, not for one database round trip after another. Never rejects; the + * results keep the input order, and the caller decides how to report a held or failed lock. + */ +async function acquireDispatchLocks( + resizer: Resizer, + mediaId: string, + byIdentity: Map, + ttlMs: number, +): Promise { + return Promise.all( + [...byIdentity].map(async ([identity, preview]): Promise => { + const lockKey = dispatchLockKey(mediaId, identity); + try { + const acquired = Boolean(await resizer.db.acquireLock(lockKey, ttlMs)); + return { preview, lockKey, acquired, failed: false }; + } catch (error) { + return { preview, lockKey, acquired: false, failed: true, error }; + } + }), + ); +} + /** Best-effort release of every given lock key; a failing release is logged, not thrown. */ async function releaseAll(resizer: Resizer, lockKeys: string[]): Promise { for (const key of lockKeys) { diff --git a/src/formatPictureUrls.test.ts b/src/formatPictureUrls.test.ts index 6a04169..648f3fc 100644 --- a/src/formatPictureUrls.test.ts +++ b/src/formatPictureUrls.test.ts @@ -1,7 +1,24 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; import { formatPictureUrls } from './formatPictureUrls.ts'; -import type { ReadDecision } from './types.d.ts'; +import type { ReadDecision, ReadyEntry } from './types.d.ts'; + +// A ready entry as resolve() builds it: always backed by a stored preview. +function entry( + over: Partial & Pick, +): ReadyEntry { + const contentType = over.contentType ?? `image/${over.format}`; + return { + contentType, + preview: { + storageRef: { key: over.url }, + sizeKey: over.sizeKey, + format: over.format, + contentType, + }, + ...over, + }; +} describe('formatPictureUrls', () => { test('treats inherited size keys as data without modifying shared prototypes', () => { @@ -17,11 +34,13 @@ describe('formatPictureUrls', () => { ); try { const out = formatPictureUrls({ - ready: sizeKeys.map((sizeKey) => ({ - sizeKey, - format: 'webp', - url: `https://cdn/${sizeKey}.webp`, - })), + ready: sizeKeys.map((sizeKey) => + entry({ + sizeKey, + format: 'webp', + url: `https://cdn/${sizeKey}.webp`, + }), + ), missing: [], }); assert.deepEqual( @@ -34,7 +53,10 @@ describe('formatPictureUrls', () => { for (const sizeKey of sizeKeys) { assert.ok(Object.hasOwn(out.sizes, sizeKey)); assert.deepEqual(out.sizes[sizeKey], { - webp: { url: `https://cdn/${sizeKey}.webp` }, + webp: { + url: `https://cdn/${sizeKey}.webp`, + contentType: 'image/webp', + }, }); } assert.deepEqual(JSON.parse(JSON.stringify(out)), out); @@ -53,18 +75,24 @@ describe('formatPictureUrls', () => { test('treats special format keys from untyped callers as own data properties', () => { const formatKeys = ['__proto__', 'constructor', 'toString']; const out = formatPictureUrls({ - ready: formatKeys.map((format) => ({ - sizeKey: '320w', - format: format as ReadDecision['ready'][number]['format'], - url: `https://cdn/${format}`, - })), + ready: formatKeys.map((format) => + entry({ + sizeKey: '320w', + format: format as ReadyEntry['format'], + url: `https://cdn/${format}`, + contentType: 'image/webp', + }), + ), missing: [], }); const byFormat = out.sizes['320w']; assert.equal(Object.getPrototypeOf(byFormat), Object.prototype); for (const format of formatKeys) { assert.ok(Object.hasOwn(byFormat, format)); - assert.deepEqual(byFormat[format], { url: `https://cdn/${format}` }); + assert.deepEqual(byFormat[format], { + url: `https://cdn/${format}`, + contentType: 'image/webp', + }); } assert.deepEqual(JSON.parse(JSON.stringify(out)), out); }); @@ -72,39 +100,13 @@ describe('formatPictureUrls', () => { test('groups ready entries by sizeKey then format', () => { const decision: ReadDecision = { ready: [ - { - sizeKey: '320x320', - format: 'jpeg', - url: 'https://cdn/a.jpg', - preview: { - key: 'a.jpg', - sizeKey: '320x320', - format: 'jpeg', - contentType: 'image/jpeg', - }, - }, - { + entry({ sizeKey: '320x320', format: 'jpeg', url: 'https://cdn/a.jpg' }), + entry({ sizeKey: '320x320', format: 'webp', url: 'https://cdn/a.webp', - preview: { - key: 'a.webp', - sizeKey: '320x320', - format: 'webp', - contentType: 'image/webp', - }, - }, - { - sizeKey: 'fit', - format: 'jpeg', - url: 'https://cdn/b.jpg', - preview: { - key: 'b.jpg', - sizeKey: 'fit', - format: 'jpeg', - contentType: 'image/jpeg', - }, - }, + }), + entry({ sizeKey: 'fit', format: 'jpeg', url: 'https://cdn/b.jpg' }), ], missing: [{ sizeKey: '620w', format: 'jpeg' }], }; @@ -124,61 +126,39 @@ describe('formatPictureUrls', () => { assert.equal('620w' in out.sizes, false); }); - test('original-backed entries use contentType when known, never invent image/', () => { - const decision: ReadDecision = { + test("each cell carries its entry's contentType", () => { + const out = formatPictureUrls({ ready: [ - { + entry({ sizeKey: '300x300', - format: 'webp', - url: 'https://cdn/orig.svg', - isOriginal: true, - contentType: 'image/svg+xml', - }, + format: 'jpeg', + url: 'https://cdn/a.jpg', + contentType: 'image/jpeg', + }), ], missing: [], - }; - const out = formatPictureUrls(decision); - assert.equal(out.id, undefined); - assert.deepEqual(out.sizes['300x300'].webp, { - url: 'https://cdn/orig.svg', - contentType: 'image/svg+xml', }); - }); - - test('omits contentType when unknown rather than guessing', () => { - const decision: ReadDecision = { - ready: [ - { - sizeKey: '300x300', - format: 'webp', - url: 'https://cdn/orig.jpg', - isOriginal: true, - }, - ], - missing: [], - }; - const out = formatPictureUrls(decision); - assert.deepEqual(out.sizes['300x300'].webp, { - url: 'https://cdn/orig.jpg', + assert.equal(out.id, undefined); + assert.deepEqual(out.sizes['300x300'].jpeg, { + url: 'https://cdn/a.jpg', + contentType: 'image/jpeg', }); }); test('skips filtered variants so they cannot collide on sizeKey+format', () => { const decision: ReadDecision = { ready: [ - { + entry({ sizeKey: '300x300', format: 'jpeg', url: 'https://cdn/plain.jpg', - contentType: 'image/jpeg', - }, - { + }), + entry({ sizeKey: '300x300', format: 'jpeg', filters: { blur: 40 }, url: 'https://cdn/blur.jpg', - contentType: 'image/jpeg', - }, + }), ], missing: [], }; diff --git a/src/formatPictureUrls.ts b/src/formatPictureUrls.ts index f98984a..6188367 100644 --- a/src/formatPictureUrls.ts +++ b/src/formatPictureUrls.ts @@ -21,12 +21,10 @@ export function formatPictureUrls( byFormat = new Map(); sizes.set(entry.sizeKey, byFormat); } - const contentType = entry.contentType ?? entry.preview?.contentType; - const cell: { url: string; contentType?: string } = { url: entry.url }; - if (contentType) { - cell.contentType = contentType; - } - byFormat.set(entry.format, cell); + byFormat.set(entry.format, { + url: entry.url, + contentType: entry.contentType, + }); } const out: PictureUrls = { // fromEntries creates own data properties, including for '__proto__', while diff --git a/src/framework/ResizeTaskModel.test.ts b/src/framework/ResizeTaskModel.test.ts index 9c7fc15..92115b0 100644 --- a/src/framework/ResizeTaskModel.test.ts +++ b/src/framework/ResizeTaskModel.test.ts @@ -90,6 +90,7 @@ describe('ResizeTaskModel.modelSchema — spec/08 §12 fields', () => { test('lease / timestamp / error fields are present', () => { const s = ResizeTaskModel.modelSchema; for (const k of [ + 'availableAt', 'leasedBy', 'leaseToken', 'leaseExpiresAt', @@ -137,15 +138,26 @@ describe('ResizeTaskModel.initHooks — the five indexes (spec/08 §12)', () => }); }); - test('{ queue:1, status:1, createdAt:1 } with no options (lease hot path)', () => { - const idx = byFields(buildIndexes(), { queue: 1, status: 1, createdAt: 1 }); + test('{ queue:1, status:1, availableAt:1 } with no options (claim hot path)', () => { + const idx = byFields(buildIndexes(), { + queue: 1, + status: 1, + availableAt: 1, + }); assert.ok(idx); assert.deepEqual(idx[1], {}); - // The queue-less lease index is gone: every lease filters by queue. - assert.equal( - byFields(buildIndexes(), { status: 1, createdAt: 1 }), - undefined, - ); + // The claim no longer orders by createdAt, and every claim filters by queue. + for (const old of [ + { queue: 1, status: 1, createdAt: 1 }, + { status: 1, createdAt: 1 }, + ]) { + assert.equal(byFields(buildIndexes(), old), undefined); + } + }); + + test('availableAt defaults to the insert time', () => { + assert.equal(ResizeTaskModel.modelSchema.availableAt.type, Date); + assert.equal(ResizeTaskModel.modelSchema.availableAt.default, Date.now); }); test('{ leaseExpiresAt:1 } partial to status:processing, NOT sparse', () => { diff --git a/src/framework/ResizeTaskModel.ts b/src/framework/ResizeTaskModel.ts index 597089d..ca24faa 100644 --- a/src/framework/ResizeTaskModel.ts +++ b/src/framework/ResizeTaskModel.ts @@ -3,9 +3,10 @@ // framework's filename-keyed loader registers `getModel('ResizeTask')`. The fields and indexes come // from drivers/mongo/schemas.ts, the same source createResizeModels() uses. // -// One of the two files that import `@adaptivestone/framework` (the other is src/framework/app.ts); -// exported only from `…/framework.js`. It must stay a literal `class … extends BaseModel`: the -// loader checks `prototype instanceof BaseModel`, and `npm run gen` walks the `extends` chain. +// Exported from `…/framework.js` and directly from `…/framework/ResizeTaskModel.js`. +// It must stay a literal `class … extends BaseModel`: the loader checks +// `prototype instanceof BaseModel`, and `npm run gen` parses the superclass's defining file. +// The scaffold's direct model import lets codegen find that BaseModel ancestor and type the shim. import type { GetModelTypeFromClass, diff --git a/src/framework/ResizeWorkerCommand.test.ts b/src/framework/ResizeWorkerCommand.test.ts index abfb5f4..47251d5 100644 --- a/src/framework/ResizeWorkerCommand.test.ts +++ b/src/framework/ResizeWorkerCommand.test.ts @@ -1,8 +1,44 @@ import assert from 'node:assert/strict'; -import { describe, test } from 'node:test'; +import { afterEach, describe, test } from 'node:test'; +import { + resetAppInstance, + setAppInstance, +} from '@adaptivestone/framework/helpers/appInstance.js'; +import { makeResizeConfig } from '../testHelpers/resizeConfig.ts'; import ResizeWorker from './ResizeWorkerCommand.ts'; +afterEach(resetAppInstance); + describe('ResizeWorker CLI contract', () => { + test('exposes the config file selector as a string argument', () => { + assert.equal(ResizeWorker.commandArguments.config.type, 'string'); + assert.match(ResizeWorker.commandArguments.config.description, /worker/); + assert.match( + ResizeWorker.commandArguments.config.description, + /default 'resize'/, + ); + }); + + for (const config of [undefined, 'resizeListings']) { + test(`reads worker settings from ${config ?? 'resize'} config`, async () => { + const asked: string[] = []; + setAppInstance({ + getConfig(name: string) { + asked.push(name); + return name === (config ?? 'resize') ? makeResizeConfig() : {}; + }, + logger: { info() {}, warn() {}, error() {} }, + } as never); + const command = new ResizeWorker( + undefined, + undefined, + config === undefined ? {} : { queue: 'bulk', config }, + ); + assert.equal(await command.run(), true); + assert.deepEqual(asked, [config ?? 'resize']); + }); + } + test('provides the Mongo connection name expected by BaseCli', () => { // BaseCli lowercases the command name and passes parsedArgs.values as the second argument. const connectionName = ResizeWorker.getMongoConnectionName('resizeworker', { diff --git a/src/framework/ResizeWorkerCommand.ts b/src/framework/ResizeWorkerCommand.ts index 3c3d1c0..0c7d3ce 100644 --- a/src/framework/ResizeWorkerCommand.ts +++ b/src/framework/ResizeWorkerCommand.ts @@ -42,12 +42,21 @@ export default class ResizeWorker { description: "Queue to consume (default 'default'). Tasks on other queues are left for their own workers.", }, + config: { + type: 'string', + description: + "Config file name whose 'worker' section the process uses (default 'resize').", + }, } as const; } async run(): Promise { - const queue = (this.args as { queue?: string } | undefined)?.queue; - await runResizeWorker(queue === undefined ? {} : { queue }); + const { queue, config } = + (this.args as { queue?: string; config?: string } | undefined) ?? {}; + await runResizeWorker({ + ...(queue === undefined ? {} : { queue }), + ...(config === undefined ? {} : { configName: config }), + }); return true; } } diff --git a/src/framework/config.ts b/src/framework/config.ts index 81cf8a9..2047a52 100644 --- a/src/framework/config.ts +++ b/src/framework/config.ts @@ -112,6 +112,12 @@ export function resolveFrameworkConfig( } const timing = fillTiming(isRecord(queue) ? queue : {}); const workerOptions = worker ?? defaultWorkerOptions; + if (isRecord(workerOptions) && Object.hasOwn(workerOptions, 'concurrency')) { + throw new ResizeConfigError( + `resize config: \`worker.concurrency\` in ${file} is no longer supported — move the value to the top-level \`concurrency\``, + { code: 'RESIZE_CONFIG_REMOVED_KEY' }, + ); + } if ( !isRecord(workerOptions) || typeof workerOptions.enabled !== 'boolean' || diff --git a/src/framework/database.test.ts b/src/framework/database.test.ts index 806405a..21459ba 100644 --- a/src/framework/database.test.ts +++ b/src/framework/database.test.ts @@ -4,7 +4,8 @@ import { resetAppInstance, setAppInstance, } from '@adaptivestone/framework/helpers/appInstance.js'; -import { ResizeConfigError } from '../errors.ts'; +import mongoose from 'mongoose'; +import { ResizeConfigError, ResizeSetupError } from '../errors.ts'; import { makeResizeConfig } from '../testHelpers/resizeConfig.ts'; import type { Preview } from '../types.d.ts'; import { FrameworkDatabase } from './database.ts'; @@ -125,50 +126,112 @@ describe('FrameworkDatabase.loadMedia', () => { }); describe('FrameworkDatabase.appendPreviews', () => { - test('issues exactly ONE findByIdAndUpdate with $push {$each} and no $set without dims', async () => { - const calls: Array<[string, Record]> = []; - const model = { - findByIdAndUpdate(id: string, update: Record) { - calls.push([id, update]); - return Promise.resolve({}); + // A media model that records each findOneAndUpdate; `matched` decides whether a document is + // returned (null: nothing matched). + function recordingModel( + matched: (filter: Record) => boolean, + ) { + const calls: Array<[Record, Record]> = []; + const options: unknown[] = []; + return { + calls, + options, + model: { + findOneAndUpdate( + filter: Record, + update: Record, + opts: unknown, + ) { + calls.push([filter, update]); + options.push(opts); + return Promise.resolve(matched(filter) ? { _id: 'm1' } : null); + }, }, }; + } + + test('pushes each preview unless its identity is already stored, and resolves with the stored ones', async () => { + const { calls, options, model } = recordingModel( + (filter) => + (filter['previews.identity'] as { $ne?: string } | undefined)?.$ne !== + 'taken', + ); installApp(model); - const previews = [ - { sizeKey: '100x100', format: 'webp' }, - ] as unknown as Preview[]; - - await db.appendPreviews('m1', previews); - - assert.equal(calls.length, 1); - assert.equal(calls[0][0], 'm1'); - assert.deepEqual(calls[0][1], { - $push: { previews: { $each: previews } }, - }); - assert.equal('$set' in calls[0][1], false); + const fresh = { identity: 'fresh', sizeKey: '100x100' } as Preview; + const taken = { identity: 'taken', sizeKey: '200x200' } as Preview; + const legacy = { sizeKey: '300x300' } as Preview; + + assert.deepEqual(await db.appendPreviews('m1', [fresh, taken, legacy]), [ + fresh, + legacy, + ]); + assert.deepEqual(calls, [ + [ + { _id: 'm1', 'previews.identity': { $ne: 'fresh' } }, + { $push: { previews: fresh } }, + ], + [ + { _id: 'm1', 'previews.identity': { $ne: 'taken' } }, + { $push: { previews: taken } }, + ], + // No identity: stored unconditionally. + [{ _id: 'm1' }, { $push: { previews: legacy } }], + ]); + // Only the id comes back, not the whole media document per preview. + for (const opts of options) { + assert.deepEqual(opts, { projection: { _id: 1 } }); + } }); - test('adds $set with dotted original.width/height ONLY when backfillDims is passed', async () => { - const calls: Array> = []; - const model = { - findByIdAndUpdate(_id: string, update: Record) { - calls.push(update); - return Promise.resolve({}); - }, - }; + test('sets the dotted original.width/height ONLY when backfillDims is passed, even with nothing to push', async () => { + const { calls, options, model } = recordingModel(() => true); installApp(model); - await db.appendPreviews('m1', [] as Preview[], { - width: 800, - height: 600, - }); - - assert.equal(calls.length, 1); - assert.deepEqual(calls[0].$push, { previews: { $each: [] } }); - assert.deepEqual(calls[0].$set, { - 'original.width': 800, - 'original.height': 600, - }); + assert.deepEqual(await db.appendPreviews('m1', []), []); + assert.equal(calls.length, 0); + + assert.deepEqual( + await db.appendPreviews('m1', [] as Preview[], { + width: 800, + height: 600, + }), + [], + ); + assert.deepEqual(calls, [ + [ + { _id: 'm1' }, + { $set: { 'original.width': 800, 'original.height': 600 } }, + ], + ]); + assert.deepEqual(options, [{ projection: { _id: 1 } }]); + }); +}); + +describe('FrameworkDatabase.verify: the media schema', () => { + // A model-shaped object with a real schema whose preview rows are declared as given. + const withRows = (modelName: string, row: Record) => ({ + modelName, + schema: new mongoose.Schema({ previews: [row] }), + }); + + test('a media model without previews.identity is a setup error naming the fragment', () => { + installApp(withRows('File', { storageRef: { type: 'Mixed' } })); + assert.throws( + () => db.verify(), + (err: unknown) => + err instanceof ResizeSetupError && + err.code === 'RESIZE_MONGO_MEDIA_MODEL_OUTDATED' && + err.message.includes("'File'") && + err.message.includes('resizeMediaSchemaFragment'), + ); + }); + + test('a media model with previews.identity, or without a schema, passes', () => { + installApp(withRows('File', { identity: { type: String } })); + assert.doesNotThrow(() => db.verify()); + resetAppInstance(); + installApp({ findById: async () => null }); + assert.doesNotThrow(() => db.verify()); }); }); diff --git a/src/framework/database.ts b/src/framework/database.ts index 1a37fb5..32a1114 100644 --- a/src/framework/database.ts +++ b/src/framework/database.ts @@ -1,15 +1,30 @@ // FrameworkDatabase: MongoDatabase over the framework app's models, resolved by name on each use. // - media: `modelName`, or `mediaModelName` from the config; // - locks: the framework's own `Lock` model (no extra collection); -// - tasks: the scaffolded `ResizeTask` model, with timing from the config's `queue` section. +// - tasks: the app's one task queue over the scaffolded `ResizeTask` model (below). // The config is the file `configName` (default 'resize'), or an explicit `config`. Nothing is read // from the app until first use. +// +// One task queue per backend: every FrameworkDatabase in the process shares one MongoTaskQueue, +// and FrameworkResizers whose config selects the same SQS queue share one SqsTaskQueue, so the +// worker runs one consume loop per backend. Timing belongs to the backend: it comes from the config +// of every Resizer (or host-built FrameworkDatabase) that uses it, and those must agree. +import type { TaskQueue } from '../contracts/taskQueue.ts'; import { MongoDatabase } from '../drivers/mongo/database.ts'; import { MongoTaskQueue } from '../drivers/mongo/taskQueue.ts'; import { ResizeConfigError, ResizeSetupError } from '../errors.ts'; -import type { FrameworkResizeConfig } from '../types.d.ts'; +import { onResetResizerForTests } from '../resizer.ts'; +import type { + FrameworkQueueConfig, + FrameworkResizeConfig, + QueueTimingOptions, +} from '../types.d.ts'; import { appLogger, getApp } from './app.ts'; -import { getResizeConfig, resolveFrameworkConfig } from './config.ts'; +import { + getResizeConfig, + type ResolvedFrameworkConfig, + resolveFrameworkConfig, +} from './config.ts'; export interface FrameworkDatabaseOptions { modelName?: string; // the host media model; default: mediaModelName from the config @@ -17,7 +32,186 @@ export interface FrameworkDatabaseOptions { config?: FrameworkResizeConfig; // an explicit config instead of reading `configName` } +/** A config that may select a shared backend: a FrameworkResizer's, or a host-built database's. */ +export interface QueueUser { + source: string; // the config as messages name it, e.g. src/config/resize.ts + read: () => ResolvedFrameworkConfig; + // The shared backend this config's tasks wait on (see backendKey), or undefined for none. + backend: (config: ResolvedFrameworkConfig) => string | undefined; +} + +/** The backend key of the app's ResizeTask queue. */ +export const DATABASE_BACKEND = 'database'; + +const queueUsers = new Set(); +const backendQueues = new Map(); +let appQueue: MongoTaskQueue | undefined; +// Set while a FrameworkResizer builds its own database: the Resizer registers its use of the queue +// itself, because only it knows whether it uses the queue at all (`tasks: false`, explicit tasks). +let buildingForResizer = false; + +onResetResizerForTests(() => { + queueUsers.clear(); + backendQueues.clear(); + appQueue = undefined; +}); + +/** Register a config whose timing a shared backend must agree with. */ +export function addQueueUser(user: QueueUser): void { + queueUsers.add(user); +} + +/** The shared backend a config's `queue` section selects: its key, or undefined for none. */ +export function backendKey( + queue: FrameworkQueueConfig | false, +): string | undefined { + if (queue === false) { + return undefined; + } + if (queue.driver !== 'sqs') { + return DATABASE_BACKEND; + } + // One SQS backend per set of queue settings; the order of the named queues does not matter. + const { queueUrl, queues, deadLetterQueueUrl, region, endpoint } = queue; + const named = Object.entries(queues ?? {}).sort(([a], [b]) => + a < b ? -1 : 1, + ); + return `sqs:${JSON.stringify([queueUrl, named, deadLetterQueueUrl ?? null, region ?? null, endpoint ?? null])}`; +} + +// SqsTaskQueue's long poll when `waitTimeSeconds` is not set. Keep in step with the driver's own +// default (`this.#opts.waitTimeSeconds ?? 10` in SqsTaskQueue.claim, src/drivers/sqs.ts), which is +// not imported here: that module loads the optional AWS SDK. +const SQS_DEFAULT_WAIT_TIME_SECONDS = 10; + +// What the configs sharing a backend must agree on: the timing, and the effective SQS long poll. +function queueSettings( + config: ResolvedFrameworkConfig, +): Record { + const { queue, timing } = config; + return queue !== false && queue.driver === 'sqs' + ? { + ...timing, + waitTimeSeconds: queue.waitTimeSeconds ?? SQS_DEFAULT_WAIT_TIME_SECONDS, + } + : { ...timing }; +} + +function sameValue(a: unknown, b: unknown): boolean { + if ( + typeof a !== 'object' || + a === null || + typeof b !== 'object' || + b === null + ) { + return Object.is(a, b); + } + const keys = new Set([...Object.keys(a), ...Object.keys(b)]); + return [...keys].every((key) => + sameValue( + (a as Record)[key], + (b as Record)[key], + ), + ); +} + +/** + * The timing of the shared backend `key`: the timing of every config that uses it, which must + * agree (ResizeConfigError RESIZE_CONFIG_QUEUE_TIMING_CONFLICT otherwise). A config that cannot be + * read is skipped here: its own Resizer reports it. No config: the defaults. + */ +export function backendTiming(key: string): Partial { + let first: + | { + source: string; + settings: Record; + timing: QueueTimingOptions; + } + | undefined; + for (const user of queueUsers) { + let config: ResolvedFrameworkConfig; + try { + config = user.read(); + } catch { + continue; + } + if (user.backend(config) !== key) { + continue; + } + const settings = queueSettings(config); + if (!first) { + first = { source: user.source, settings, timing: config.timing }; + continue; + } + const reference = first.settings; + const differ = [ + ...new Set([...Object.keys(reference), ...Object.keys(settings)]), + ].filter((name) => !sameValue(reference[name], settings[name])); + if (differ.length > 0) { + throw new ResizeConfigError( + `resize config: ${first.source} and ${user.source} use the same task queue but set different ${differ.join(', ')} — one task queue has one timing, so set the same values in both`, + { code: 'RESIZE_CONFIG_QUEUE_TIMING_CONFLICT' }, + ); + } + } + return first?.timing ?? {}; +} + +/** Check the shared backend `user` selects, if any: throws on a timing conflict. */ +export function checkQueueUser(user: QueueUser): void { + const key = user.backend(user.read()); + if (key !== undefined) { + backendTiming(key); + } +} + +/** + * The task queue of the shared backend `key`, built once per process. The timing is checked on + * every call, so each Resizer that loads the queue sees a conflict. + */ +export function sharedQueue( + key: string, + build: (timing: Partial) => TaskQueue, +): TaskQueue { + const timing = backendTiming(key); + let tasks = backendQueues.get(key); + if (!tasks) { + tasks = build(timing); + backendQueues.set(key, tasks); + } + return tasks; +} + +/** The process's one MongoTaskQueue over the app's ResizeTask model. */ +function appTaskQueue(): MongoTaskQueue { + appQueue ??= new MongoTaskQueue({ + getModel: () => getApp().getModel('ResizeTask'), + getTiming: () => backendTiming(DATABASE_BACKEND), + logger: appLogger, + }); + return appQueue; +} + +/** True when `tasks` is the app's shared ResizeTask queue. */ +export function isAppTaskQueue(tasks: TaskQueue | undefined): boolean { + return tasks !== undefined && tasks === appQueue; +} + +/** A FrameworkDatabase for a FrameworkResizer, which registers its own use of the queue. */ +export function databaseForResizer( + opts: FrameworkDatabaseOptions, +): FrameworkDatabase { + buildingForResizer = true; + try { + return new FrameworkDatabase(opts); + } finally { + buildingForResizer = false; + } +} + export class FrameworkDatabase extends MongoDatabase { + readonly #queueUser: QueueUser | undefined; + constructor(opts: FrameworkDatabaseOptions = {}) { const read = () => opts.config @@ -37,23 +231,39 @@ export class FrameworkDatabase extends MongoDatabase { } return model; }, - tasks: new MongoTaskQueue({ - getModel: () => getApp().getModel('ResizeTask'), - getTiming: () => read().timing, - logger: appLogger, - }), + tasks: appTaskQueue(), }); + // A database the host builds uses the queue when its config selects the 'database' driver. + if (!buildingForResizer) { + this.#queueUser = { + source: opts.config + ? 'the config passed to new FrameworkDatabase()' + : `src/config/${opts.configName ?? 'resize'}.ts`, + read, + backend: (config) => + backendKey(config.queue) === DATABASE_BACKEND + ? DATABASE_BACKEND + : undefined, + }; + addQueueUser(this.#queueUser); + } } - /** Startup check: the media model and the framework's `Lock` model must resolve. */ + /** + * Startup check: the media model must resolve and store preview identities, the framework's + * `Lock` model must resolve, and (for a database the host built) the queue timing must agree. + */ verify(): void { - this.mediaModel(); + this.verifyMediaModel(); if (!getApp().getModel('Lock')) { throw new ResizeSetupError( "resize: the framework's Lock model is not registered — FrameworkDatabase keeps its locks there", { code: 'RESIZE_MONGO_MODEL_MISSING' }, ); } + if (this.#queueUser) { + checkQueueUser(this.#queueUser); + } } // The framework Lock TTL is in seconds; round up so a sub-second TTL never becomes a 0-second diff --git a/src/framework/index.ts b/src/framework/index.ts index 092f72d..83d9bc3 100644 --- a/src/framework/index.ts +++ b/src/framework/index.ts @@ -13,13 +13,6 @@ export type { FrameworkSqsQueueConfig, FrameworkStorageConfig, } from '../types.d.ts'; -export { - appEvents, - appLogger, - getApp, - type TMinimalResizeApp, -} from './app.ts'; -export { getResizeConfig } from './config.ts'; export { FrameworkDatabase, type FrameworkDatabaseOptions, diff --git a/src/framework/queueIndexes.mongo.integration.test.ts b/src/framework/queueIndexes.mongo.integration.test.ts index c9eae43..83d92cf 100644 --- a/src/framework/queueIndexes.mongo.integration.test.ts +++ b/src/framework/queueIndexes.mongo.integration.test.ts @@ -168,17 +168,20 @@ test('prepared indexes preserve concurrent enqueue deduplication', async () => { assert.ok(dedupe, 'fixture should create the active-request dedupe index'); assert.ok( taskIndexes.some(({ key }) => - hasExactKey(key, { queue: 1, status: 1, createdAt: 1 }), + hasExactKey(key, { queue: 1, status: 1, availableAt: 1 }), ), - 'fixture should create the queue-scoped lease index', - ); - assert.equal( - taskIndexes.some(({ key }) => - hasExactKey(key, { status: 1, createdAt: 1 }), - ), - false, - 'the queue-less lease index is gone', + 'fixture should create the queue-scoped claim index', ); + for (const old of [ + { queue: 1, status: 1, createdAt: 1 }, + { status: 1, createdAt: 1 }, + ]) { + assert.equal( + taskIndexes.some(({ key }) => hasExactKey(key, old)), + false, + `the old lease index ${JSON.stringify(old)} is gone`, + ); + } assert.equal(dedupe.unique, true); assert.deepEqual(dedupe.partialFilterExpression, { status: { $in: ['pending', 'processing'] }, diff --git a/src/framework/resizer.peers.test.ts b/src/framework/resizer.peers.test.ts new file mode 100644 index 0000000..2207398 --- /dev/null +++ b/src/framework/resizer.peers.test.ts @@ -0,0 +1,97 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { test } from 'node:test'; +import { promisify } from 'node:util'; + +const runNode = promisify(execFile); + +// Each process gets a fresh module cache, so installed SDKs elsewhere in the suite do not hide +// the missing-peer path. Resolution hooks leave node_modules and other implementers' files alone. +async function failedDriverImport( + driver: 's3' | 'sqs', + missing: string, + code = 'ERR_MODULE_NOT_FOUND', + message = `Cannot find package '${missing}' imported from driver`, +) { + const { stdout } = await runNode(process.execPath, [ + '--experimental-strip-types', + '--input-type=module', + '-e', + ` + import { registerHooks } from 'node:module'; + import { setAppInstance } from '@adaptivestone/framework/helpers/appInstance.js'; + import { FrameworkResizer } from ${JSON.stringify(new URL('./resizer.ts', import.meta.url).href)}; + import { makeResizeConfig } from ${JSON.stringify(new URL('../testHelpers/resizeConfig.ts', import.meta.url).href)}; + import { ResizeSetupError } from ${JSON.stringify(new URL('../errors.ts', import.meta.url).href)}; + const original = Object.assign(new Error(${JSON.stringify(message)}), { code: ${JSON.stringify(code)} }); + registerHooks({ + resolve(specifier, context, nextResolve) { + if (specifier === ${JSON.stringify(missing)} && context.parentURL?.endsWith('/drivers/${driver}.ts')) { + throw original; + } + return nextResolve(specifier, context); + }, + }); + const config = makeResizeConfig({ + storage: ${driver === 's3' ? "{ driver: 's3', bucketPublic: 'cdn' }" : "{ driver: 'local', rootDir: './media', publicBaseUrl: '/media' }"}, + queue: ${driver === 'sqs' ? "{ driver: 'sqs', queueUrl: 'https://sqs.example/resize' }" : 'false'}, + }); + setAppInstance({ + getConfig: () => config, + logger: { info() {}, warn() {}, error() {} }, + }); + const resizer = new FrameworkResizer({ configName: 'resizeListings' }); + try { + await resizer.ready(); + console.log(JSON.stringify({ resolved: true })); + } catch (error) { + console.log(JSON.stringify({ + setupError: error instanceof ResizeSetupError, + code: error.code, + message: error.message, + sameError: error === original, + sameCause: error.cause === original, + })); + } + `, + ]); + return JSON.parse(stdout); +} + +for (const [driver, peers] of [ + ['s3', ['@aws-sdk/client-s3', '@aws-sdk/s3-request-presigner']], + ['sqs', ['@aws-sdk/client-sqs']], +] as const) { + for (const peer of peers) { + test(`${driver}: missing ${peer} names the config and installation remedy`, async () => { + const error = await failedDriverImport(driver, peer); + assert.equal(error.setupError, true); + assert.equal(error.code, 'RESIZE_PEER_MISSING'); + assert.equal(error.sameCause, true); + assert.match(error.message, /src\/config\/resizeListings\.ts/); + assert.ok(error.message.includes(`'${driver}'`)); + assert.match(error.message, /install/); + for (const required of peers) { + assert.ok(error.message.includes(required)); + } + }); + } + + test(`${driver}: unrelated missing packages and other import failures stay unchanged`, async () => { + for (const [code, message] of [ + [ + 'ERR_MODULE_NOT_FOUND', + "Cannot find package '@aws-sdk/credential-provider-node' imported from driver", + ], + ['ERR_MODULE_NOT_FOUND', 'Cannot find module /drivers/missing.ts'], + ['ERR_MODULE_NOT_FOUND', `Cannot find module '${peers[0]}/missing'`], + ['ERR_INVALID_PACKAGE_CONFIG', `Cannot find package '${peers[0]}'`], + ]) { + const error = await failedDriverImport(driver, peers[0], code, message); + assert.equal(error.sameError, true); + assert.equal(error.setupError, false); + assert.equal(error.code, code); + assert.equal(error.message, message); + } + }); +} diff --git a/src/framework/resizer.test.ts b/src/framework/resizer.test.ts index 7e84467..83da7c7 100644 --- a/src/framework/resizer.test.ts +++ b/src/framework/resizer.test.ts @@ -14,8 +14,10 @@ import { timingOf } from '../queue.ts'; import { resetResizerForTests } from '../resizer.ts'; import { fakeDb, MemoryTaskQueue } from '../testHelpers/fakes.ts'; import { makeResizeConfig } from '../testHelpers/resizeConfig.ts'; +import { runWorker } from '../worker.ts'; import { FrameworkDatabase } from './database.ts'; import { FrameworkResizer } from './resizer.ts'; +import { runResizeWorker } from './worker.ts'; const storage: ResizeStorage = { download: async () => Buffer.alloc(0), @@ -216,10 +218,8 @@ test("a queue section without a driver is the database's queue; its timing fills await r.ready(); assert.ok(r.tasks instanceof MongoTaskQueue); assert.equal(timingOf(r.tasks).leaseMs, 1234); - assert.deepEqual(timingOf(r.tasks).lockTtlMs, { - dispatch: 60000, - worker: 1000, - }); + assert.equal(timingOf(r.tasks).lockTtlMs.dispatch, 60000); + assert.equal(timingOf(r.tasks).lockTtlMs.worker, 1000); assert.equal(timingOf(r.tasks).maxAttempts, 5); }); @@ -389,10 +389,8 @@ test('a core MongoTaskQueue needs one model, and the core validates its timing', ); const t = new MongoTaskQueue({ model: {} }); assert.equal(timingOf(t).leaseMs, 60_000); - assert.deepEqual(timingOf(t).lockTtlMs, { - dispatch: 60_000, - worker: 60_000, - }); + assert.equal(timingOf(t).lockTtlMs.dispatch, 60_000); + assert.equal(timingOf(t).lockTtlMs.worker, 60_000); }); test('nothing is read from the app until first use', async () => { @@ -473,3 +471,327 @@ test('prewarm reports a config error as a non-retryable issue', async () => { assert.equal(result.issues[0].code, 'RESIZE_ENQUEUE_INTERNAL_ERROR'); assert.equal(result.issues[0].retryable, false); }); + +// --- one task queue per backend ------------------------------------------------------------- + +const conflictBetween = + (files: string[], keys: string[], same: string[] = []) => + (err: unknown) => + err instanceof ResizeConfigError && + err.code === 'RESIZE_CONFIG_QUEUE_TIMING_CONFLICT' && + files.every((file) => err.message.includes(file)) && + keys.every((key) => err.message.includes(key)) && + same.every((key) => !err.message.includes(key)); + +test('Resizers and databases with default wiring share one database task queue', async () => { + installApp({ + resize: makeResizeConfig({ + storage: localStorage, + queue: { driver: 'database' }, + }), + resizeListings: makeResizeConfig({ + storage: localStorage, + formats: ['webp'], + queue: {}, + }), + }); + const media = new FrameworkResizer(); + const listings = new FrameworkResizer({ + name: 'listings', + configName: 'resizeListings', + }); + const direct = new FrameworkDatabase({ configName: 'resizeListings' }); + await Promise.all([media.ready(), listings.ready()]); + assert.ok(media.tasks instanceof MongoTaskQueue); + assert.notEqual(media.db, listings.db); // each Resizer keeps its own media model and config + assert.equal(listings.tasks, media.tasks); + assert.equal(media.db.tasks, media.tasks); + assert.equal(direct.tasks, media.tasks); +}); + +test('config files sharing a task queue with different timing conflict on first use, in verify() and at worker start', async () => { + resetAppInstance(); + // Construction reads nothing, so it cannot see the conflict (and must not throw). + const media = new FrameworkResizer(); + const listings = new FrameworkResizer({ + name: 'listings', + configName: 'resizeListings', + }); + installApp({ + resize: makeResizeConfig({ + storage: localStorage, + worker: { enabled: true }, + queue: { + maxAttempts: 3, + idlePollMs: 50, + lockTtlMs: { dispatch: 60000, worker: 5000 }, + }, + }), + resizeListings: makeResizeConfig({ + storage: localStorage, + formats: ['webp'], // image settings may differ + queue: { + driver: 'database', + maxAttempts: 7, + idlePollMs: 50, + lockTtlMs: { worker: 5000, dispatch: 60000 }, // same values, other key order + taskTimeoutMs: 1000, + }, + }), + }); + const conflict = conflictBetween( + ['src/config/resize.ts', 'src/config/resizeListings.ts'], + ['maxAttempts', 'taskTimeoutMs'], + ['idlePollMs', 'lockTtlMs'], + ); + await assert.rejects(() => media.verify(), conflict); + await assert.rejects(() => listings.verify(), conflict); + await assert.rejects( + () => + media.generate({ + media: { id: 'm1', original: { storageRef: { key: 'k' } } }, + sizes: [{ width: 10, height: 10 }], + }), + conflict, + ); + await assert.rejects(() => runResizeWorker(), conflict); + await assert.rejects( + () => runWorker({ signal: AbortSignal.abort() }), + conflict, + ); +}); + +test('a host-built FrameworkDatabase is checked against the Resizers that share its queue', async () => { + installApp({ + resize: makeResizeConfig({ + storage: localStorage, + queue: { maxAttempts: 3 }, + }), + resizeHost: makeResizeConfig({ queue: { maxAttempts: 4 } }), + }); + const media = new FrameworkResizer(); + const db = new FrameworkDatabase({ configName: 'resizeHost' }); + const conflict = conflictBetween( + ['src/config/resize.ts', 'src/config/resizeHost.ts'], + ['maxAttempts'], + ); + await assert.rejects(() => media.verify(), conflict); + assert.throws(() => db.verify(), conflict); + assert.ok(db.tasks); + assert.throws(() => timingOf(db.tasks as MongoTaskQueue), conflict); +}); + +test('two config files with the same timing share the queue and its timing', async () => { + const queue = { + maxAttempts: 3, + lockTtlMs: { dispatch: 60000, worker: 5000 }, + leaseMs: 5000, + }; + installApp({ + resize: makeResizeConfig({ storage: localStorage, queue }), + resizeListings: makeResizeConfig({ + storage: localStorage, + formats: ['webp'], + queue: { driver: 'database', ...queue }, + }), + }); + const media = new FrameworkResizer(); + const listings = new FrameworkResizer({ + name: 'listings', + configName: 'resizeListings', + }); + await media.verify(); + await listings.verify(); + assert.ok(media.tasks); + assert.equal(timingOf(media.tasks).maxAttempts, 3); + assert.equal(timingOf(media.tasks).leaseMs, 5000); +}); + +test('timing is compared only between configs that use the same task queue', async () => { + const withQueue = (queue: unknown) => + makeResizeConfig({ storage: localStorage, queue: queue as never }); + installApp({ + resize: withQueue({ maxAttempts: 3 }), + resizeEager: withQueue({ maxAttempts: 4 }), // tasks: false in code + resizeOwn: withQueue({ maxAttempts: 5 }), // explicit tasks in code + resizeSqs: withQueue({ + driver: 'sqs', + queueUrl: 'https://sqs.example/resize', + maxAttempts: 6, + }), + resizeOff: withQueue(false), + }); + const own = new MemoryTaskQueue(); + const media = new FrameworkResizer(); + const eager = new FrameworkResizer({ + name: 'eager', + configName: 'resizeEager', + tasks: false, + }); + const explicit = new FrameworkResizer({ + name: 'own', + configName: 'resizeOwn', + tasks: own, + }); + const sqs = new FrameworkResizer({ name: 'sqs', configName: 'resizeSqs' }); + const off = new FrameworkResizer({ name: 'off', configName: 'resizeOff' }); + for (const r of [media, eager, explicit, sqs, off]) { + await r.verify(); + } + assert.ok(media.tasks instanceof MongoTaskQueue); + assert.equal(timingOf(media.tasks).maxAttempts, 3); + assert.equal(eager.tasks, undefined); + assert.equal(off.tasks, undefined); + assert.equal(explicit.tasks, own); // an explicit queue is never shared or replaced + assert.ok(sqs.tasks instanceof SqsTaskQueue); + assert.equal(timingOf(sqs.tasks).maxAttempts, 6); +}); + +test('configs that select the same SQS queue share one SqsTaskQueue; another queue gets its own', async () => { + const sqs = (queue: Record) => + makeResizeConfig({ + storage: localStorage, + queue: { + driver: 'sqs', + queueUrl: 'https://sqs.example/resize', + region: 'eu-west-1', + deadLetterQueueUrl: 'https://sqs.example/dead', + ...queue, + } as never, + }); + installApp({ + resize: sqs({ + queues: { + bulk: 'https://sqs.example/bulk', + slow: 'https://sqs.example/slow', + }, + }), + resizeListings: sqs({ + queues: { + slow: 'https://sqs.example/slow', + bulk: 'https://sqs.example/bulk', + }, + }), + resizeOther: sqs({ queueUrl: 'https://sqs.example/other' }), + resizeRegion: sqs({ + queues: { + bulk: 'https://sqs.example/bulk', + slow: 'https://sqs.example/slow', + }, + region: 'us-east-1', + }), + }); + const media = new FrameworkResizer(); + const listings = new FrameworkResizer({ + name: 'listings', + configName: 'resizeListings', + }); + const other = new FrameworkResizer({ + name: 'other', + configName: 'resizeOther', + }); + const region = new FrameworkResizer({ + name: 'region', + configName: 'resizeRegion', + }); + await Promise.all([media, listings, other, region].map((r) => r.ready())); + assert.ok(media.tasks instanceof SqsTaskQueue); + assert.equal(listings.tasks, media.tasks); + assert.ok(other.tasks instanceof SqsTaskQueue); + assert.notEqual(other.tasks, media.tasks); + assert.notEqual(region.tasks, media.tasks); +}); + +test('configs that share an SQS queue with different timing conflict', async () => { + const sqs = (queue: Record) => + makeResizeConfig({ + storage: localStorage, + worker: { enabled: true }, + queue: { + driver: 'sqs', + queueUrl: 'https://sqs.example/resize', + ...queue, + } as never, + }); + installApp({ + resize: sqs({ maxAttempts: 3, waitTimeSeconds: 10 }), + resizeListings: sqs({ maxAttempts: 3, waitTimeSeconds: 20 }), + }); + const media = new FrameworkResizer(); + const listings = new FrameworkResizer({ + name: 'listings', + configName: 'resizeListings', + }); + const conflict = conflictBetween( + ['src/config/resize.ts', 'src/config/resizeListings.ts'], + ['waitTimeSeconds'], + ['maxAttempts'], + ); + await assert.rejects(() => listings.verify(), conflict); + await assert.rejects(() => media.verify(), conflict); + await assert.rejects(() => runResizeWorker(), conflict); +}); + +test("an omitted SQS waitTimeSeconds equals the driver's default, so it is no conflict", async () => { + const sqs = (queue: Record) => + makeResizeConfig({ + storage: localStorage, + queue: { + driver: 'sqs', + queueUrl: 'https://sqs.example/resize', + ...queue, + } as never, + }); + installApp({ + resize: sqs({}), + resizeListings: sqs({ waitTimeSeconds: 10 }), + }); + const media = new FrameworkResizer(); + const listings = new FrameworkResizer({ + name: 'listings', + configName: 'resizeListings', + }); + await media.verify(); + await listings.verify(); + assert.ok(media.tasks instanceof SqsTaskQueue); + assert.equal(listings.tasks, media.tasks); +}); + +test('resetResizerForTests forgets the shared task queues and the configs that used them', async () => { + installApp({ + resize: makeResizeConfig({ + storage: localStorage, + queue: { maxAttempts: 3 }, + }), + resizeListings: makeResizeConfig({ + storage: localStorage, + queue: { maxAttempts: 4 }, + }), + resizeSqs: makeResizeConfig({ + storage: localStorage, + queue: { driver: 'sqs', queueUrl: 'https://sqs.example/resize' }, + }), + }); + const first = new FrameworkResizer(); + const firstSqs = new FrameworkResizer({ + name: 'sqs', + configName: 'resizeSqs', + }); + await Promise.all([first.ready(), firstSqs.ready()]); + assert.ok(first.tasks); + assert.equal(timingOf(first.tasks).maxAttempts, 3); + + resetResizerForTests(); + // A forgotten Resizer's config no longer takes part, and the queues are new objects. + const next = new FrameworkResizer({ configName: 'resizeListings' }); + const nextSqs = new FrameworkResizer({ + name: 'sqs', + configName: 'resizeSqs', + }); + await next.verify(); + await nextSqs.verify(); + assert.ok(next.tasks); + assert.notEqual(next.tasks, first.tasks); + assert.equal(timingOf(next.tasks).maxAttempts, 4); + assert.notEqual(nextSqs.tasks, firstSqs.tasks); +}); diff --git a/src/framework/resizer.ts b/src/framework/resizer.ts index d475b3d..cf35969 100644 --- a/src/framework/resizer.ts +++ b/src/framework/resizer.ts @@ -7,12 +7,14 @@ // - tasks: the file's `queue` section ('database', 'sqs', or false / missing for eager only); // - logger and events: the app's. // Options win over the config file. Nothing is read from the app until first use, and the AWS -// drivers (optional peers) are imported only when the config selects them. +// drivers (optional peers) are imported only when the config selects them. The task queues this +// builds are shared per backend (see ./database.ts): Resizers whose config selects the same backend +// get the same queue object, so the worker runs one loop for it, and their timing must agree. import type { ResizeDatabase } from '../contracts/database.ts'; import type { ResizeStorage } from '../contracts/storage.ts'; import type { TaskQueue } from '../contracts/taskQueue.ts'; import { LocalFsStorage } from '../drivers/fs.ts'; -import { ResizeConfigError } from '../errors.ts'; +import { ResizeConfigError, ResizeSetupError } from '../errors.ts'; import { Resizer, type ResizerOptions } from '../resizer.ts'; import type { FrameworkResizeConfig, @@ -25,7 +27,17 @@ import { type ResolvedFrameworkConfig, resolveFrameworkConfig, } from './config.ts'; -import { FrameworkDatabase } from './database.ts'; +import { + addQueueUser, + backendKey, + backendTiming, + checkQueueUser, + DATABASE_BACKEND, + databaseForResizer, + isAppTaskQueue, + type QueueUser, + sharedQueue, +} from './database.ts'; export interface FrameworkResizerOptions extends Omit< @@ -48,6 +60,10 @@ export interface FrameworkResizerOptions * Call `await resizer.verify()` after `Server.init()` to check everything at boot. */ export class FrameworkResizer extends Resizer { + // This Resizer's config as a user of a shared task queue; unset with an explicit `tasks` or + // `tasks: false`. + readonly #queueUser: QueueUser | undefined; + constructor(opts: FrameworkResizerOptions = {}) { const { configName, @@ -68,7 +84,7 @@ export class FrameworkResizer extends Resizer { const file = `src/config/${configName ?? 'resize'}.ts`; const database = db ?? - new FrameworkDatabase( + databaseForResizer( explicitConfig ? { config: explicitConfig, configName } : { configName }, @@ -84,6 +100,62 @@ export class FrameworkResizer extends Resizer { ? {} : { tasks: tasks ?? (() => buildQueue(read(), database, file)) }), }); + // Registered once constructed (a rejected name or config registers nothing). With the + // 'database' driver, the queue is shared only when it is the app's one ResizeTask queue. + if (tasks === undefined) { + this.#queueUser = { + source: explicit + ? `the config passed to new FrameworkResizer({ name: '${this.name}' })` + : file, + read, + backend: (config) => { + const key = backendKey(config.queue); + return key === DATABASE_BACKEND && !isAppTaskQueue(database.tasks) + ? undefined + : key; + }, + }; + addQueueUser(this.#queueUser); + } + } + + /** + * The core checks, then the timing of the shared task queue again: a Resizer constructed after + * this one loaded its queue is compared here too. + */ + async verify(): Promise { + await super.verify(); + if (this.#queueUser) { + checkQueueUser(this.#queueUser); + } + } +} + +async function importDriver( + load: () => Promise, + driver: string, + peers: string[], + file: string, +): Promise { + try { + return await load(); + } catch (err) { + if ( + err instanceof Error && + 'code' in err && + (err.code === 'ERR_MODULE_NOT_FOUND' || err.code === 'MODULE_NOT_FOUND') + ) { + const missing = /^Cannot find (?:package|module) ['"]([^'"]+)['"]/.exec( + err.message, + )?.[1]; + if (missing && peers.includes(missing)) { + throw new ResizeSetupError( + `resize config: the '${driver}' driver selected in ${file} requires missing optional peer \`${missing}\` — install ${peers.join(' ')}`, + { code: 'RESIZE_PEER_MISSING', cause: err }, + ); + } + } + throw err; } } @@ -116,7 +188,12 @@ async function buildStorage( endpoint, forcePathStyle, } = storage; - const { S3Storage } = await import('../drivers/s3.ts'); + const { S3Storage } = await importDriver( + () => import('../drivers/s3.ts'), + 's3', + ['@aws-sdk/client-s3', '@aws-sdk/s3-request-presigner'], + file, + ); return new S3Storage( Object.fromEntries( Object.entries({ @@ -136,33 +213,48 @@ async function buildQueue( database: ResizeDatabase, file: string, ): Promise { - const { queue, timing } = config; + const { queue } = config; if (queue === false) { return undefined; } if (queue.driver === 'sqs') { - const { SqsTaskQueue } = await import('../drivers/sqs.ts'); - return new SqsTaskQueue({ - queueUrl: queue.queueUrl, - ...(queue.queues ? { queues: queue.queues } : {}), - ...(queue.deadLetterQueueUrl - ? { deadLetterQueueUrl: queue.deadLetterQueueUrl } - : {}), - ...(queue.waitTimeSeconds === undefined - ? {} - : { waitTimeSeconds: queue.waitTimeSeconds }), - ...(queue.region ? { region: queue.region } : {}), - ...(queue.endpoint ? { endpoint: queue.endpoint } : {}), - timing, - logger: appLogger, - }); + const { SqsTaskQueue } = await importDriver( + () => import('../drivers/sqs.ts'), + 'sqs', + ['@aws-sdk/client-sqs'], + file, + ); + // One SqsTaskQueue per SQS queue in the process, with the timing its configs agree on. + return sharedQueue( + backendKey(queue) as string, + (timing) => + new SqsTaskQueue({ + queueUrl: queue.queueUrl, + ...(queue.queues ? { queues: queue.queues } : {}), + ...(queue.deadLetterQueueUrl + ? { deadLetterQueueUrl: queue.deadLetterQueueUrl } + : {}), + ...(queue.waitTimeSeconds === undefined + ? {} + : { waitTimeSeconds: queue.waitTimeSeconds }), + ...(queue.region ? { region: queue.region } : {}), + ...(queue.endpoint ? { endpoint: queue.endpoint } : {}), + timing, + logger: appLogger, + }), + ); } - // 'database': the database's own queue (FrameworkDatabase: the ResizeTask model). + // 'database': the database's own queue (FrameworkDatabase: the app's one ResizeTask queue). if (!database.tasks) { throw new ResizeConfigError( `resize config: \`queue\` in ${file} selects the 'database' driver, but the database passed to new FrameworkResizer() has no task queue — pass \`tasks\`, or set queue: false`, { code: 'RESIZE_CONFIG_INVALID' }, ); } + // The core reads a queue's timing once; check it here so every Resizer sees a conflict on its + // own first use. + if (isAppTaskQueue(database.tasks)) { + backendTiming(DATABASE_BACKEND); + } return database.tasks; } diff --git a/src/framework/scaffold/command.test.ts b/src/framework/scaffold/command.test.ts index 9df7578..62d539a 100644 --- a/src/framework/scaffold/command.test.ts +++ b/src/framework/scaffold/command.test.ts @@ -148,6 +148,22 @@ describe('runScaffold — default run', () => { assert.match(out, new RegExp(RESIZER)); assert.match(out, new RegExp(MODEL)); }); + + test('the model shim imports the defining file so codegen can follow its ancestor', async () => { + await run([]); + assert.match( + await read(MODEL), + /^import ResizeTaskModel from '@adaptivestone\/framework-module-resize\/framework\/ResizeTaskModel\.js';$/m, + ); + }); + + test('next steps explain the queue and worker switch required to run the worker', async () => { + const { out } = await run([]); + assert.match(out, /queue: \{ driver: 'database' \}/); + assert.match(out, /'sqs'/); + assert.match(out, /worker\.enabled: true/); + assert.match(out, /for the worker to run/); + }); }); describe('runScaffold — idempotency & --force', () => { @@ -209,9 +225,15 @@ describe('runScaffold — --eject', () => { ); assert.match( model, - /\{ queue: 1, status: 1, createdAt: 1 \}/, - 'the ejected schema carries the queue-scoped lease index', + /availableAt:\s*\{\s*type:\s*Date,\s*default:\s*Date\.now\s*\}/, + 'the ejected schema records when each task is due', + ); + assert.match( + model, + /\{ queue: 1, status: 1, availableAt: 1 \}/, + 'the ejected schema carries the queue-scoped claim index', ); + assert.doesNotMatch(model, /\{ queue: 1, status: 1, createdAt: 1 \}/); // Still the full set of files. assert.equal(await exists(COMMAND), true); assert.equal(await exists(CONFIG), true); @@ -268,6 +290,11 @@ describe('runScaffold — --check', () => { const { code, out } = await run(['--check']); assert.equal(code, 1); assert.match(out, /drift/); + assert.match( + out, + /delete the file and re-run resize-scaffold for a fresh shim/, + ); + assert.doesNotMatch(out, /--force/); }); test('an ejected model passes --check (it owns its schema)', async () => { @@ -277,6 +304,71 @@ describe('runScaffold — --check', () => { assert.doesNotMatch(out, /drift/); }); + test('a shim importing the defining model subpath passes --check', async () => { + await run([]); + await writeFile( + join(root, MODEL), + "import ResizeTaskModel from '@adaptivestone/framework-module-resize/framework/ResizeTaskModel.js';\nexport default class ResizeTask extends ResizeTaskModel {}\n", + ); + const { code, out } = await run(['--check']); + assert.equal(code, 0, out); + }); + + test('the old barrel import is drift with a regeneration hint that keeps the host files', async () => { + await run([]); + const oldShim = + "import { ResizeTaskModel } from '@adaptivestone/framework-module-resize/framework.js';\nexport default class ResizeTask extends ResizeTaskModel {}\n"; + await writeFile(join(root, MODEL), oldShim); + const { code, out } = await run(['--check']); + assert.equal(code, 1); + assert.match(out, /drift\s+src\/models\/ResizeTask\.ts/); + assert.match(out, /framework\/ResizeTaskModel\.js/); + assert.match( + out, + /delete the file and re-run resize-scaffold for a fresh shim/, + ); + assert.doesNotMatch(out, /--force/); + assert.equal( + await read(MODEL), + oldShim, + '--check never rewrites the host model', + ); + }); + + for (const fields of [ + ['resizer'], + ['queue'], + ['requestKey'], + ['availableAt'], + ['resizer', 'queue', 'requestKey', 'availableAt'], + ]) { + test(`an ejected model missing ${fields.join(', ')} is drift even with its indexes intact`, async () => { + await run(['--eject']); + let model = await read(MODEL); + for (const field of fields) { + model = model.replace( + new RegExp(`^ {6}${field}: \\{[^\\n]+\\n`, 'm'), + '', + ); + } + await writeFile(join(root, MODEL), model); + const { code, out } = await run(['--check']); + assert.equal(code, 1); + assert.match(out, /drift\s+src\/models\/ResizeTask\.ts/); + assert.match( + out, + /port the resizer, queue, requestKey and availableAt fields/, + ); + assert.match(out, /delete the file and re-run resize-scaffold --eject/); + assert.doesNotMatch(out, /--force/); + assert.equal( + await read(MODEL), + model, + '--check never rewrites an ejected model', + ); + }); + } + test('an old model shim importing the removed subpath → exit 1 + drift', async () => { await run([]); await writeFile( @@ -452,6 +544,26 @@ describe('runScaffold — --agents pointer', () => { // Build/packaging smoke — cheap source assertions (no real build in the unit suite). describe('packaging smoke', () => { + test('repository metadata points at framework-module-resizer', async () => { + const pkg = JSON.parse( + await readFile(new URL('../../../package.json', import.meta.url), 'utf8'), + ); + assert.equal( + pkg.repository.url, + 'git+https://github.com/adaptivestone/framework-module-resizer.git', + ); + }); + + test('package exports the defining model file for scaffold codegen', async () => { + const pkg = JSON.parse( + await readFile(new URL('../../../package.json', import.meta.url), 'utf8'), + ); + assert.equal( + pkg.exports['./framework/ResizeTaskModel.js'], + './dist/framework/ResizeTaskModel.js', + ); + }); + test('command.ts starts with the node shebang', async () => { const src = await readFile( fileURLToPath(new URL('./command.ts', import.meta.url)), diff --git a/src/framework/scaffold/command.ts b/src/framework/scaffold/command.ts index 3abd9b8..99eef8f 100644 --- a/src/framework/scaffold/command.ts +++ b/src/framework/scaffold/command.ts @@ -1,6 +1,6 @@ #!/usr/bin/env node // `resize-scaffold` — the package bin that vendors the resize module's integration files into a -// host project (08 · §12). It runs BEFORE any host wiring exists (chicken-and-egg: the framework +// host project. It runs BEFORE any host wiring exists (chicken-and-egg: the framework // discovers models/commands by scanning host folders, so ResizeTask + ResizeWorker must be real // files in the host's src/). So this generator is STANDALONE: NO framework, NO getApp, NO other // module imports — only node builtins. Paths resolve from process.cwd() (or --out ). @@ -26,17 +26,24 @@ const CONFIG = 'src/config/resize.ts'; // Load-bearing substrings `--check` verifies (also documents what each shim MUST reference). // A construction site builds its Resizer through the framework adapter or the core class. const RESIZER_MARKERS = ['new FrameworkResizer(', 'new Resizer(']; -// The model shim extends ResizeTaskModel from the framework adapter (the old …/models/ResizeTask.js -// subpath no longer exists). +// The model shim imports the defining file so framework codegen can parse its BaseModel ancestor. const MODEL_MARKERS = [ 'extends ResizeTaskModel', - '@adaptivestone/framework-module-resize/framework.js', + '@adaptivestone/framework-module-resize/framework/ResizeTaskModel.js', ]; -// An ejected model (`--eject`) owns its schema: a full BaseModel subclass. +// An ejected model (`--eject`) owns its schema: a full BaseModel subclass with the current +// task identity fields. The patterns match schema declarations however the host formats them, +// and not requestKey's partial index filter. const EJECTED_MODEL_MARKERS = [ 'extends BaseModel', '@adaptivestone/framework/modules/BaseModel.js', ]; +const EJECTED_MODEL_FIELDS = [ + /\bresizer:\s*\{\s*type:/, + /\bqueue:\s*\{\s*type:/, + /\brequestKey:\s*\{\s*type:/, + /\bavailableAt:\s*\{\s*type:/, +]; // The worker command imports the construction site AND re-exports the module's command, so the // worker process has the Resizers its tasks name (a bare re-export starts with none). const COMMAND_MARKERS = [ @@ -160,8 +167,9 @@ async function checkFiles(root: string, eager: boolean): Promise { target: MODEL, validate: (c) => MODEL_MARKERS.every((marker) => c.includes(marker)) || - EJECTED_MODEL_MARKERS.every((marker) => c.includes(marker)), - hint: 'must extend ResizeTaskModel from @adaptivestone/framework-module-resize/framework.js (or be an ejected BaseModel) — re-run resize-scaffold for a fresh shim, or --eject for the full model', + (EJECTED_MODEL_MARKERS.every((marker) => c.includes(marker)) && + EJECTED_MODEL_FIELDS.every((field) => field.test(c))), + hint: 'must extend ResizeTaskModel from @adaptivestone/framework-module-resize/framework/ResizeTaskModel.js (or be an ejected BaseModel with resizer, queue, requestKey and availableAt fields) — delete the file and re-run resize-scaffold for a fresh shim; for an ejected model, port the resizer, queue, requestKey and availableAt fields, or delete the file and re-run resize-scaffold --eject', }, { target: COMMAND, @@ -330,6 +338,9 @@ export async function runScaffold( console.log( 'need the Resizer (a static import is fine). The ResizeWorker command imports it too.', ); + console.log( + "Set queue: { driver: 'database' } (or 'sqs') and worker.enabled: true in src/config/resize.ts for the worker to run.", + ); } return code; } diff --git a/src/framework/scaffold/templates/ResizeTask.model.full.ts.tpl b/src/framework/scaffold/templates/ResizeTask.model.full.ts.tpl index 101ffe5..afaa99c 100644 --- a/src/framework/scaffold/templates/ResizeTask.model.full.ts.tpl +++ b/src/framework/scaffold/templates/ResizeTask.model.full.ts.tpl @@ -1,4 +1,4 @@ -// src/models/ResizeTask.ts — EJECTED full model (scaffolded with `--eject`, 08 · §12). +// src/models/ResizeTask.ts — EJECTED full model (scaffolded with `--eject`). // // ⚠️ This is a VENDORED COPY of the module's ResizeTaskModel schema + indexes, for hosts that // need custom fields/indexes. Unlike the thin `extends ResizeTaskModel` shim, this copy will @@ -57,9 +57,12 @@ export default class ResizeTask extends BaseModel { }, // Capped by the queue's maxAttempts (config `queue.maxAttempts`), then dead-lettered. attempts: { type: Number, default: 0 }, + // When the task may next be claimed: now for a new or released task, the retry time after + // a failure, the end of the lease while a worker holds it. Claims take the earliest. + availableAt: { type: Date, default: Date.now }, leasedBy: { type: String }, - leaseToken: { type: String }, // fencing token (05 · §10.2) - leaseExpiresAt: { type: Date }, + leaseToken: { type: String }, // fencing token + leaseExpiresAt: { type: Date }, // the current lease ends; null while the task waits completedAt: { type: Date }, deadAt: { type: Date }, error: { type: String }, @@ -84,9 +87,10 @@ export default class ResizeTask extends BaseModel { partialFilterExpression: { status: 'dead' }, }, ); - // Lease hot path: a worker consumes one queue, oldest task first. - schema.index({ queue: 1, status: 1, createdAt: 1 }); - // Sweep/reclaim stuck leases. NOT sparse: the partial filter on status:'processing' scopes it. + // Claim hot path: a worker consumes one queue, the earliest due task first. + schema.index({ queue: 1, status: 1, availableAt: 1 }); + // Processing tasks by lease end, to find stuck leases. NOT sparse: the partial filter on + // status:'processing' scopes it. schema.index( { leaseExpiresAt: 1 }, { partialFilterExpression: { status: 'processing' } }, diff --git a/src/framework/scaffold/templates/ResizeTask.model.ts.tpl b/src/framework/scaffold/templates/ResizeTask.model.ts.tpl index f9b8672..091b716 100644 --- a/src/framework/scaffold/templates/ResizeTask.model.ts.tpl +++ b/src/framework/scaffold/templates/ResizeTask.model.ts.tpl @@ -1,8 +1,9 @@ -// src/models/ResizeTask.ts — scaffolded thin shim (08 · §12). The MODULE owns the schema + +// src/models/ResizeTask.ts — scaffolded thin shim. The MODULE owns the schema + // indexes (ResizeTaskModel); this file only NAMES the model so the framework's filename-keyed -// loader registers getModel('ResizeTask') and `npm run gen` types it. Auto-updates with the -// package — no drift. Need custom fields/indexes? re-run the scaffold with `--eject`. -import { ResizeTaskModel } from '@adaptivestone/framework-module-resize/framework.js'; +// loader registers getModel('ResizeTask'). The direct model import lets `npm run gen` parse +// the BaseModel ancestor and type getModel('ResizeTask'). Schema and indexes update with the +// package. Need custom fields/indexes? delete this file and re-run the scaffold with `--eject`. +import ResizeTaskModel from '@adaptivestone/framework-module-resize/framework/ResizeTaskModel.js'; // Point fileId at a differently-named media model with `static fileRef = 'Media'` (default 'File'). export default class ResizeTask extends ResizeTaskModel {} diff --git a/src/framework/scaffold/templates/ResizeWorker.command.ts.tpl b/src/framework/scaffold/templates/ResizeWorker.command.ts.tpl index 5fc63ae..bd7f0f3 100644 --- a/src/framework/scaffold/templates/ResizeWorker.command.ts.tpl +++ b/src/framework/scaffold/templates/ResizeWorker.command.ts.tpl @@ -1,4 +1,4 @@ -// src/commands/ResizeWorker.ts — scaffolded (08 · §12). The MODULE owns the worker command +// src/commands/ResizeWorker.ts — scaffolded. The MODULE owns the worker command // (AbstractCommand shape, isShouldInitModels=true, --queue); the framework's filename-keyed CLI // loader registers this file as `npm run cli ResizeWorker`. Importing src/resizer.ts builds the // host's Resizers in the CLI process, so the worker serves the same Resizers as the API (they read diff --git a/src/framework/worker.mongo.integration.test.ts b/src/framework/worker.mongo.integration.test.ts index 771bd68..d75a0bc 100644 --- a/src/framework/worker.mongo.integration.test.ts +++ b/src/framework/worker.mongo.integration.test.ts @@ -16,6 +16,7 @@ import mongoose from 'mongoose'; import sharp from 'sharp'; import type { ResizeDatabase } from '../contracts/database.ts'; import type { ResizeStorage } from '../contracts/storage.ts'; +import { ResizeSetupError } from '../errors.ts'; import { resizeMediaSchemaFragment } from '../mediaFragment.ts'; import { resetResizerForTests } from '../resizer.ts'; import { fakeDb } from '../testHelpers/fakes.ts'; @@ -71,26 +72,48 @@ after(async () => { await server.stop(); }); -function installApp() { +const testQueue = { + leaseMs: 5000, + idlePollMs: 20, + lockTtlMs: { dispatch: 60000, worker: 5000 }, +}; + +// A fake app over the real task and lock models. `configs` overrides config files by name; +// `models` adds models (e.g. the media model) to ResizeTask and Lock. +function installApp( + opts: { + configs?: Record; + models?: Record; + } = {}, +) { resetResizerForTests(); resetAppInstance(); + const models: Record = { + ResizeTask: taskModel, + Lock: lockModel, + ...opts.models, + }; setAppInstance({ - getConfig: () => + getConfig: (name: string) => + opts.configs?.[name] ?? makeResizeConfig({ formats: ['webp'], worker: { enabled: true }, - queue: { - leaseMs: 5000, - idlePollMs: 20, - lockTtlMs: { dispatch: 60000, worker: 5000 }, - }, + queue: testQueue, }), - getModel: (name: string) => - name === 'ResizeTask' ? taskModel : name === 'Lock' ? lockModel : false, + getModel: (name: string) => models[name] ?? false, logger: { info() {}, warn() {}, error() {} }, } as never); } +// The host media model, built from the current fragment. +const fileModel = () => + connection.models.File ?? + connection.model( + 'File', + new mongoose.Schema({ ...resizeMediaSchemaFragment }, { minimize: false }), + ); + // In-memory storage: every download returns the test PNG; uploads are recorded. function memoryStorage(): { storage: ResizeStorage; uploads: string[] } { const uploads: string[] = []; @@ -243,20 +266,10 @@ test('a bulk-queue task waits for a bulk worker', async () => { test('a FrameworkResizer wired only by its config file runs through runResizeWorker', async () => { await taskModel.deleteMany({}); const root = await mkdtemp(join(tmpdir(), 'resize-e2e-')); - const fileModel = - connection.models.File ?? - connection.model( - 'File', - new mongoose.Schema( - { ...resizeMediaSchemaFragment }, - { minimize: false }, - ), - ); - resetResizerForTests(); - resetAppInstance(); - setAppInstance({ - getConfig: () => - makeResizeConfig({ + const File = fileModel(); + installApp({ + configs: { + resize: makeResizeConfig({ formats: ['webp'], worker: { enabled: true }, storage: { @@ -264,18 +277,11 @@ test('a FrameworkResizer wired only by its config file runs through runResizeWor rootDir: join(root, 'public'), publicBaseUrl: '/media', }, - queue: { - driver: 'database', - leaseMs: 5000, - idlePollMs: 20, - lockTtlMs: { dispatch: 60000, worker: 5000 }, - }, + queue: { driver: 'database', ...testQueue }, }), - getModel: (name: string) => - ({ ResizeTask: taskModel, Lock: lockModel, File: fileModel })[name] ?? - false, - logger: { info() {}, warn() {}, error() {} }, - } as never); + }, + models: { File }, + }); try { const resizer = new FrameworkResizer(); // everything from the config file await resizer.verify(); @@ -283,7 +289,7 @@ test('a FrameworkResizer wired only by its config file runs through runResizeWor body: png, visibility: 'private', }); - const doc = await fileModel.create({ original, previews: [] }); + const doc = await File.create({ original, previews: [] }); const media = { id: String(doc._id), original, previews: [] }; const result = await resizer.prewarm({ media, sizes }); assert.equal(result.status, 'accepted'); @@ -292,7 +298,7 @@ test('a FrameworkResizer wired only by its config file runs through runResizeWor try { const until = Date.now() + 20000; while ( - ((await fileModel.findById(doc._id).lean())?.previews as unknown[]) + ((await File.findById(doc._id).lean())?.previews as unknown[]) ?.length !== 1 ) { assert.ok(Date.now() < until, 'the worker did not store the preview'); @@ -302,7 +308,7 @@ test('a FrameworkResizer wired only by its config file runs through runResizeWor process.emit('SIGTERM'); await worker; } - const stored = await fileModel.findById(doc._id).lean(); + const stored = await File.findById(doc._id).lean(); const [preview] = (stored?.previews ?? []) as Preview[]; assert.equal(preview.format, 'webp'); assert.equal( @@ -319,3 +325,159 @@ test('a FrameworkResizer wired only by its config file runs through runResizeWor await rm(root, { recursive: true, force: true }); } }); + +test('two Resizers wired by their config files share one task queue: one task at a time, each with its own Resizer', async () => { + await taskModel.deleteMany({}); + const File = fileModel(); + const queue = { driver: 'database', ...testQueue }; + installApp({ + configs: { + resize: makeResizeConfig({ formats: ['webp'], queue }), + resizeListings: makeResizeConfig({ formats: ['webp'], queue }), + }, + models: { File }, + }); + // Storage that records what it downloads and how many downloads overlap across both Resizers. + let running = 0; + let peak = 0; + const tracked = () => { + const downloads: string[] = []; + const storage: ResizeStorage = { + download: async (ref) => { + downloads.push((ref as { key: string }).key); + running += 1; + peak = Math.max(peak, running); + await new Promise((r) => setTimeout(r, 150)); + running -= 1; + return png; + }, + upload: async ({ key }) => ({ key }), + publicUrl: (ref) => `/m/${(ref as { key: string }).key}`, + }; + return { downloads, storage }; + }; + const a = tracked(); + const b = tracked(); + const media = new FrameworkResizer({ storage: a.storage }); + const listings = new FrameworkResizer({ + name: 'listings', + configName: 'resizeListings', + storage: b.storage, + }); + assert.equal(media.db === listings.db, false); + + const ids: string[] = []; + for (const [resizer, prefix] of [ + [media, 'media'], + [listings, 'listings'], + ] as const) { + for (const n of [1, 2]) { + const original = { + storageRef: { key: `${prefix}/${n}.png` }, + format: 'png', + }; + const doc = await File.create({ original, previews: [] }); + ids.push(String(doc._id)); + const result = await resizer.prewarm({ + media: { id: String(doc._id), original, previews: [] }, + sizes, + }); + assert.equal(result.status, 'accepted'); + } + } + await media.ready(); + await listings.ready(); + assert.equal(media.tasks, listings.tasks); + + const stop = new AbortController(); + const done = runWorker({ signal: stop.signal }); + try { + const until = Date.now() + 20000; + while ( + (await File.countDocuments({ + _id: { $in: ids }, + 'previews.0': { $exists: true }, + })) !== ids.length + ) { + assert.ok(Date.now() < until, 'the worker did not store every preview'); + await new Promise((r) => setTimeout(r, 20)); + } + } finally { + stop.abort(); + await done; + } + + assert.equal(peak, 1, 'one consume loop: tasks never overlap'); + assert.deepEqual(a.downloads.sort(), ['media/1.png', 'media/2.png']); + assert.deepEqual(b.downloads.sort(), ['listings/1.png', 'listings/2.png']); + const rows = await taskModel.find({}).lean(); + assert.equal(rows.length, 4); + assert.ok(rows.every((r) => r.status === 'completed')); +}); + +test("FrameworkDatabase keeps one preview row per identity in the app's media model", async () => { + const File = fileModel(); + installApp({ models: { File } }); + const db = new FrameworkDatabase(); + db.verify(); + const doc = await File.create({ + original: { storageRef: { key: 'o.png' }, format: 'png' }, + previews: [], + }); + const preview = (key: string) => + ({ + storageRef: { key }, + identity: 'default:default:16x16:webp:', + sizeKey: '16x16', + format: 'webp', + contentType: 'image/webp', + }) as Preview; + const results = await Promise.all([ + db.appendPreviews(String(doc._id), [preview('one.webp')]), + db.appendPreviews(String(doc._id), [preview('two.webp')]), + ]); + assert.equal(results.flatMap((r) => r ?? []).length, 1); + const stored = await File.findById(doc._id).lean(); + assert.equal(stored?.previews?.length, 1); + assert.equal( + (stored?.previews?.[0] as Preview | undefined)?.identity, + 'default:default:16x16:webp:', + ); +}); + +test('verify() rejects a media model registered without previews.identity', async () => { + const { identity: _identity, ...previewFields } = + resizeMediaSchemaFragment.previews[0]; + const OldFile = + connection.models.OldFile ?? + connection.model( + 'OldFile', + new mongoose.Schema( + { + original: resizeMediaSchemaFragment.original, + previews: [previewFields], + }, + { minimize: false }, + ), + ); + installApp({ + configs: { + resize: makeResizeConfig({ + mediaModelName: 'OldFile', + storage: { + driver: 'local', + rootDir: './var/media', + publicBaseUrl: '/m', + }, + }), + }, + models: { OldFile }, + }); + await assert.rejects( + () => new FrameworkResizer().verify(), + (err: unknown) => + err instanceof ResizeSetupError && + err.code === 'RESIZE_MONGO_MEDIA_MODEL_OUTDATED' && + err.message.includes("'OldFile'"), + ); +}); diff --git a/src/framework/worker.test.ts b/src/framework/worker.test.ts index 5eba5bc..e9d9846 100644 --- a/src/framework/worker.test.ts +++ b/src/framework/worker.test.ts @@ -33,6 +33,24 @@ describe('runResizeWorker', () => { await runResizeWorker(); }); + test('the disabled message names the config file the worker read', async () => { + const lines: string[] = []; + setAppInstance({ + getConfig: () => ({ + ...defaultResizeConfig, + mediaModelName: 'File', + worker: { ...defaultWorkerOptions, enabled: false }, + }), + getModel: () => undefined, + logger: { info: (m: string) => lines.push(m), warn() {}, error() {} }, + } as never); + await runResizeWorker({ configName: 'resizeListings' }); + assert.match(lines.join('\n'), /src\/config\/resizeListings\.ts/); + lines.length = 0; + await runResizeWorker(); + assert.match(lines.join('\n'), /src\/config\/resize\.ts/); + }); + test('names the command fix when the worker process built no Resizer', async () => { installApp(true); await assert.rejects( diff --git a/src/framework/worker.ts b/src/framework/worker.ts index 06a024c..6354bc5 100644 --- a/src/framework/worker.ts +++ b/src/framework/worker.ts @@ -14,7 +14,7 @@ export async function runResizeWorker( const { worker } = getResizeConfig(opts.configName); if (worker.enabled === false) { app.logger.info( - 'resize worker disabled — set config.worker.enabled=true in the host src/config/resize.ts to run it', + `resize worker disabled — set config.worker.enabled=true in the host src/config/${opts.configName ?? 'resize'}.ts to run it`, ); return; } diff --git a/src/helpers/imageFormat.ts b/src/helpers/imageFormat.ts index 46ceaa4..196dcc8 100644 --- a/src/helpers/imageFormat.ts +++ b/src/helpers/imageFormat.ts @@ -6,3 +6,13 @@ export function isAvifBuffer(body: Buffer): boolean { const brands = body.toString('ascii', 8, Math.min(body.length, 64)); return brands.includes('avif') || brands.includes('avis'); } + +// Output formats Sharp writes as an animation. Measured with Sharp 0.35: webp and gif keep every +// frame; avif, jpeg and png stack the frames into one tall image, and tiff writes them as pages, +// which no browser plays. Every other format is rendered from the first frame. +const ANIMATED_OUTPUT_FORMATS: ReadonlySet = new Set(['gif', 'webp']); + +/** True when a preview in `format` can keep the frames of an animated source. */ +export function isAnimatedFormat(format: string): boolean { + return ANIMATED_OUTPUT_FORMATS.has(format); +} diff --git a/src/helpers/svgRaster.test.ts b/src/helpers/svgRaster.test.ts new file mode 100644 index 0000000..475d7b7 --- /dev/null +++ b/src/helpers/svgRaster.test.ts @@ -0,0 +1,230 @@ +// SVG rasterization in a child process: output, hard time limit, abort, failures. +import assert from 'node:assert/strict'; +import childProcess, { type ChildProcess } from 'node:child_process'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, mock, test } from 'node:test'; +import { pathToFileURL } from 'node:url'; +import sharp from 'sharp'; +import { ResizeMediaError, ResizeSetupError } from '../errors.ts'; +import { rasterizeSvg, svgRasterChildPath } from './svgRaster.ts'; + +const realSpawn = childProcess.spawn; + +const isUnavailable = (err: unknown) => + err instanceof ResizeSetupError && + err.code === 'RESIZE_SVG_RENDER_UNAVAILABLE' && + /child process/.test(err.message) && + /svgRasterChild/.test(err.message); + +// Six turbulence-filtered rects: librsvg needs more than 10 s for this 488-byte SVG, inside one +// native call that sharp's `.timeout()` cannot interrupt. +const heavySvg = Buffer.from( + `${''.repeat(6)}`, +); + +const redSvg = Buffer.from( + '', +); + +const options = { limitInputPixels: 268402689, timeoutMs: 10_000 }; + +/** True while a process with `pid` exists (signal 0 only checks). */ +function isRunning(pid: number | undefined): boolean { + if (pid === undefined) { + return false; + } + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +function spawnSpy() { + return mock.method(childProcess, 'spawn'); +} + +afterEach(() => { + mock.restoreAll(); +}); + +describe('rasterizeSvg', () => { + test('renders a PNG at the requested density', async () => { + const png = await rasterizeSvg(redSvg, { ...options, density: 144 }); + const meta = await sharp(png).metadata(); + assert.equal(meta.format, 'png'); + assert.deepEqual([meta.width, meta.height], [60, 40]); + }); + + test('runs node itself, without a shell, with its stdout discarded', async () => { + const spawn = spawnSpy(); + await rasterizeSvg(redSvg, { ...options, density: 72 }); + assert.equal(spawn.mock.callCount(), 1); + const [command, , spawnOptions] = spawn.mock.calls[0].arguments as [ + string, + string[], + { shell?: unknown; stdio?: unknown[] } | undefined, + ]; + assert.equal(command, process.execPath); + assert.ok(!spawnOptions?.shell); + assert.equal(spawnOptions?.stdio?.[1], 'ignore'); + }); + + test('a preload that prints to stdout cannot corrupt the PNG', async (t) => { + // NODE_OPTIONS is inherited on purpose (Yarn PnP and loaders need it): the PNG travels on + // its own pipe, so whatever a preload prints cannot reach it. + const dir = await mkdtemp(join(tmpdir(), 'resize-preload-')); + const preload = join(dir, 'pre.cjs'); + await writeFile(preload, "console.log('hello from a preload');\n"); + const previous = process.env.NODE_OPTIONS; + process.env.NODE_OPTIONS = `--require "${preload}"`; + t.after(async () => { + if (previous === undefined) { + delete process.env.NODE_OPTIONS; + } else { + process.env.NODE_OPTIONS = previous; + } + await rm(dir, { recursive: true, force: true }); + }); + const png = await rasterizeSvg(redSvg, { ...options, density: 72 }); + assert.deepEqual( + [(await sharp(png).metadata()).width, png.length > 0], + [30, true], + ); + }); + + test('a spawn that throws (permission model) is RESIZE_SVG_RENDER_UNAVAILABLE', async () => { + const denied = Object.assign(new Error('Access to this API is denied'), { + code: 'ERR_ACCESS_DENIED', + }); + mock.method(childProcess, 'spawn', () => { + throw denied; + }); + await assert.rejects( + () => rasterizeSvg(redSvg, { ...options, density: 72 }), + (err: unknown) => isUnavailable(err) && (err as Error).cause === denied, + ); + }); + + test('a process that cannot start (ENOENT) is RESIZE_SVG_RENDER_UNAVAILABLE', async () => { + mock.method( + childProcess, + 'spawn', + (_command: string, args: string[], spawnOptions: object) => + realSpawn('/nonexistent/node-binary', args, spawnOptions), + ); + await assert.rejects( + () => rasterizeSvg(redSvg, { ...options, density: 72 }), + isUnavailable, + ); + }); + + test('a missing child script is RESIZE_SVG_RENDER_UNAVAILABLE', async () => { + const dir = await mkdtemp(join(tmpdir(), 'resize-nochild-')); + try { + assert.throws( + () => svgRasterChildPath(pathToFileURL(join(dir, 'svgRaster.js')).href), + isUnavailable, + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } + // Next to this module, the child script is found (a .ts sibling under type stripping). + assert.match(svgRasterChildPath(), /svgRasterChild\.ts$/); + }); + + test('a render over the time limit is killed and fails with RESIZE_SVG_RENDER_TIMEOUT', { + timeout: 8000, + }, async () => { + const spawn = spawnSpy(); + const started = Date.now(); + await assert.rejects( + () => + rasterizeSvg(heavySvg, { + ...options, + density: 72, + timeoutMs: 1000, + mediaId: 'm1', + }), + (err: unknown) => + err instanceof ResizeMediaError && + err.code === 'RESIZE_SVG_RENDER_TIMEOUT' && + err.mediaId === 'm1', + ); + const elapsed = Date.now() - started; + assert.ok(elapsed < 3000, `settled after ${elapsed} ms`); + const child = spawn.mock.calls[0].result as ChildProcess; + assert.equal(child.signalCode, 'SIGKILL'); + assert.equal(isRunning(child.pid), false); + }); + + test('an abort signal kills the render', { timeout: 8000 }, async () => { + const spawn = spawnSpy(); + const controller = new AbortController(); + setTimeout(() => controller.abort(new Error('lease lost')), 300); + await assert.rejects( + () => + rasterizeSvg(heavySvg, { + ...options, + density: 72, + signal: controller.signal, + }), + (err: unknown) => + err instanceof ResizeMediaError && + err.code === 'RESIZE_SVG_RENDER_FAILED' && + /abort/i.test(err.message), + ); + const child = spawn.mock.calls[0].result as ChildProcess; + assert.equal(child.signalCode, 'SIGKILL'); + assert.equal(isRunning(child.pid), false); + }); + + test('an already aborted signal starts no process', async () => { + const spawn = spawnSpy(); + await assert.rejects( + () => + rasterizeSvg(redSvg, { + ...options, + density: 72, + signal: AbortSignal.abort(), + }), + (err: unknown) => + err instanceof ResizeMediaError && + err.code === 'RESIZE_SVG_RENDER_FAILED', + ); + assert.equal(spawn.mock.callCount(), 0); + }); + + test('a render error fails with RESIZE_SVG_RENDER_FAILED and the end of stderr', async () => { + await assert.rejects( + () => + rasterizeSvg(Buffer.from(' + err instanceof ResizeMediaError && + err.code === 'RESIZE_SVG_RENDER_FAILED' && + /exit code 1/.test(err.message) && + /svg|xml|input|unsupported/i.test(err.message), + ); + }); + + test('the raster obeys limitInputPixels', async () => { + await assert.rejects( + () => + rasterizeSvg(redSvg, { + ...options, + density: 720, + limitInputPixels: 1000, + }), + (err: unknown) => + err instanceof ResizeMediaError && + err.code === 'RESIZE_SVG_RENDER_FAILED' && + /pixel limit/i.test(err.message), + ); + }); +}); diff --git a/src/helpers/svgRaster.ts b/src/helpers/svgRaster.ts new file mode 100644 index 0000000..5478795 --- /dev/null +++ b/src/helpers/svgRaster.ts @@ -0,0 +1,187 @@ +// SVG → PNG in a child process with a hard time limit. librsvg renders inside one native call +// that sharp's `.timeout()` cannot interrupt: a few hundred bytes of filtered shapes can hold a +// libuv thread for minutes. The child is killed (SIGKILL) at the deadline or when the caller's +// signal aborts; the parent settles only after the process has exited, so none is left behind. +import childProcess from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { extname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { ResizeMediaError, ResizeSetupError } from '../errors.ts'; + +const PNG_SIGNATURE = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, +]); + +// How much of the child's stderr an error message carries (its end: the actual failure). +const STDERR_TAIL = 2000; + +/** The host cannot start the render process at all: a setup problem, not a media one. */ +function unavailable(detail: string, cause?: unknown): ResizeSetupError { + return new ResizeSetupError( + `resize: SVG rendering is unavailable: ${detail}. SVG originals are rendered in a child process: the host must allow child processes (under Node's permission model, --allow-child-process) and ship svgRasterChild.js next to svgRaster.js`, + { code: 'RESIZE_SVG_RENDER_UNAVAILABLE', cause }, + ); +} + +/** + * The child script next to `moduleUrl` (this module by default): `src/*.ts` under type stripping + * (tests), `dist/*.js` once built (hosts). Throws RESIZE_SVG_RENDER_UNAVAILABLE when it is missing. + */ +export function svgRasterChildPath( + moduleUrl: string = import.meta.url, +): string { + const path = fileURLToPath( + new URL(`./svgRasterChild${extname(fileURLToPath(moduleUrl))}`, moduleUrl), + ); + if (!existsSync(path)) { + throw unavailable(`the child script ${path} is missing`); + } + return path; +} + +// Resolved on the first render, not at import: importing the package must not touch the disk. +let childScript: string | undefined; + +export interface SvgRasterOptions { + /** Render density in dpi; 72 renders the SVG at its own size. */ + density: number; + limitInputPixels: number; + /** Kill the render after this long. */ + timeoutMs: number; + /** Kill the render when this aborts (the task's lease was lost, the worker stops). */ + signal?: AbortSignal; + mediaId?: string; +} + +/** Render `svg` to a lossless PNG at `density`, in a child process with a hard time limit. */ +export function rasterizeSvg( + svg: Buffer, + { density, limitInputPixels, timeoutMs, signal, mediaId }: SvgRasterOptions, +): Promise { + const forMedia = mediaId === undefined ? '' : ` for media ${mediaId}`; + const failed = (detail: string, cause?: unknown) => + new ResizeMediaError(`resize: SVG rendering failed${forMedia}: ${detail}`, { + mediaId, + code: 'RESIZE_SVG_RENDER_FAILED', + cause, + }); + if (signal?.aborted) { + return Promise.reject( + failed( + 'aborted before it started (the abort signal fired)', + signal.reason, + ), + ); + } + + return new Promise((resolve, reject) => { + let child: childProcess.ChildProcess; + try { + childScript ??= svgRasterChildPath(); + // No shell: the node binary itself, with the child script and its options as arguments. + // The PNG comes back on fd 3, a pipe of its own: stdout is discarded, so nothing a host + // preload (NODE_OPTIONS --require/--import, inherited on purpose) prints can corrupt it. + child = childProcess.spawn( + process.execPath, + [childScript, JSON.stringify({ density, limitInputPixels, timeoutMs })], + { stdio: ['pipe', 'ignore', 'pipe', 'pipe'] }, + ); + } catch (err) { + // Node's permission model throws synchronously without --allow-child-process. + reject( + err instanceof ResizeSetupError + ? err + : unavailable( + `the render process could not start (${err instanceof Error ? err.message : String(err)})`, + err, + ), + ); + return; + } + const png: Buffer[] = []; + let stderr = ''; + let killedFor: 'timeout' | 'abort' | undefined; + let settled = false; + + const kill = (reason: 'timeout' | 'abort') => { + if (killedFor === undefined) { + killedFor = reason; + child.kill('SIGKILL'); + } + }; + const timer = setTimeout(() => kill('timeout'), timeoutMs); + const onAbort = () => kill('abort'); + signal?.addEventListener('abort', onAbort, { once: true }); + + const settle = (error: Error | undefined, result?: Buffer) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); + if (error) { + reject(error); + } else { + resolve(result as Buffer); + } + }; + + child.stdio[3]?.on('data', (chunk: Buffer) => png.push(chunk)); + child.stderr?.on('data', (chunk: Buffer) => { + stderr = (stderr + chunk.toString('utf8')).slice(-STDERR_TAIL); + }); + // A child that dies early closes its pipes (EPIPE); its exit status reports why. + child.stdin?.on('error', () => {}); + child.stdio[3]?.on('error', () => {}); + child.on('error', (err) => { + // The process could not start at all (ENOENT, EACCES): 'close' may never follow. + if (child.pid === undefined) { + settle( + unavailable( + `the render process could not start (${err.message})`, + err, + ), + ); + } + }); + // 'close' comes after the process has exited and its pipes are drained. + child.on('close', (code, exitSignal) => { + if (killedFor === 'timeout') { + settle( + new ResizeMediaError( + `resize: SVG rendering${forMedia} exceeded limits.processingTimeoutSeconds (${timeoutMs / 1000}s) — the render process was killed`, + { mediaId, code: 'RESIZE_SVG_RENDER_TIMEOUT' }, + ), + ); + return; + } + if (killedFor === 'abort') { + settle( + failed( + 'the abort signal fired; the render process was killed', + signal?.reason, + ), + ); + return; + } + if (code !== 0) { + const status = + code === null ? `signal ${exitSignal}` : `exit code ${code}`; + settle(failed(`${status}: ${stderr.trim() || 'no error output'}`)); + return; + } + const output = Buffer.concat(png); + if ( + output.length <= PNG_SIGNATURE.length || + !output.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE) + ) { + settle(failed('the render process produced no PNG')); + return; + } + settle(undefined, output); + }); + + child.stdin?.end(svg); + }); +} diff --git a/src/helpers/svgRasterChild.ts b/src/helpers/svgRasterChild.ts new file mode 100644 index 0000000..dec8923 --- /dev/null +++ b/src/helpers/svgRasterChild.ts @@ -0,0 +1,44 @@ +// Child process of svgRaster.ts — never import this module: it reads stdin and exits. +// Renders ONE SVG to a lossless PNG. librsvg renders inside one native call that sharp's +// `.timeout()` cannot interrupt, so the parent runs it here and kills this process at its deadline. +// argv[2]: JSON { density, limitInputPixels, timeoutMs } +// stdin: the SVG bytes +// fd 3: the PNG bytes (a pipe of its own: stdout is discarded, so whatever a host preload +// prints there cannot reach the image) +// failure: a message on stderr and a non-zero exit code +// `sharp` resolves from this file's own location (the package), whatever the host's cwd. +import { Socket } from 'node:net'; +import sharp from 'sharp'; + +const { density, limitInputPixels, timeoutMs } = JSON.parse( + process.argv[2] ?? '{}', +) as { density: number; limitInputPixels: number; timeoutMs: number }; + +// The render runs on a libuv thread, so this timer still fires during it: a safety net if the +// parent is gone and cannot kill this process. The parent's own deadline comes first. +setTimeout(() => process.exit(2), timeoutMs + 5000).unref(); + +sharp.cache(false); +sharp.concurrency(1); + +try { + const chunks: Buffer[] = []; + for await (const chunk of process.stdin) { + chunks.push(chunk as Buffer); + } + // Same safety options as any decode in the worker. A buffer input has no base location, so + // librsvg loads no external file, URL or stylesheet reference. + const png = await sharp(Buffer.concat(chunks), { + density, + failOn: 'warning', + limitInputPixels, + }) + .png({ compressionLevel: 1 }) + .toBuffer(); + // A stream over fd 3 (like process.stdout over a pipe) copes with a non-blocking pipe; exit + // by draining, not process.exit(), so the whole PNG reaches the parent. + new Socket({ fd: 3, readable: false, writable: true }).end(png); +} catch (err) { + process.stderr.write(err instanceof Error ? err.message : String(err)); + process.exitCode = 1; +} diff --git a/src/images.test.ts b/src/images.test.ts index 0a21b3c..54608ac 100644 --- a/src/images.test.ts +++ b/src/images.test.ts @@ -1,17 +1,21 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; +import { ResizeSetupError } from './errors.ts'; import { calculateResizedDimensions, + canonicalizeFilterValue, + coverDimensions, DEFAULT_SCOPE, expandMissingPreviews, getFilterSig, - getImageContentType, getPreviewIdentity, getSizeKey, isCatalogCovered, parseSizeKey, previewScope, + toMissingPreview, } from './images.ts'; +import type { SizeInput } from './types.d.ts'; describe('getSizeKey', () => { test('fit → "fit"', () => { @@ -51,6 +55,66 @@ describe('getSizeKey', () => { assert.throws(() => getSizeKey({ width: Number.NaN })); assert.throws(() => getSizeKey({ width: Number.POSITIVE_INFINITY })); }); + + test('a positive dimension that rounds to 0 does not count', () => { + // A `0w` key would reach sharp as width 0, which it rejects. + assert.throws( + () => getSizeKey({ width: 0.4 }), + (err: unknown) => + err instanceof ResizeSetupError && err.code === 'RESIZE_SIZE_INVALID', + ); + assert.throws(() => getSizeKey({ height: 0.49 })); + assert.equal(getSizeKey({ width: 0.4, height: 100 }), '100h'); + assert.equal(getSizeKey({ width: 0.5 }), '1w'); + }); +}); + +describe('toMissingPreview', () => { + const payload = (size: SizeInput, format = 'webp') => + toMissingPreview(size, getSizeKey(size), format); + + test('a fractional size carries the rounded dimensions of its key', () => { + assert.deepEqual(payload({ width: 300.5, height: 200 }), { + sizeKey: '301x200', + format: 'webp', + requestedWidth: 301, + requestedHeight: 200, + }); + assert.deepEqual(payload({ width: 0.4, height: 99.6 }), { + sizeKey: '100h', + format: 'webp', + requestedHeight: 100, + }); + }); + + test('a width-only or height-only size carries only that side', () => { + assert.deepEqual(payload({ width: 620 }), { + sizeKey: '620w', + format: 'webp', + requestedWidth: 620, + }); + assert.deepEqual(payload({ height: 400 }), { + sizeKey: '400h', + format: 'webp', + requestedHeight: 400, + }); + }); + + test('fit ignores width and height, so both spellings give one payload', () => { + const bare = payload({ fit: true }); + assert.deepEqual(bare, { sizeKey: 'fit', format: 'webp', fit: true }); + assert.deepEqual(payload({ fit: true, width: 2000, height: 1200 }), bare); + }); + + test('copies non-empty filters only', () => { + assert.deepEqual(payload({ width: 10, filters: { blur: 3 } }).filters, { + blur: 3, + }); + assert.equal( + Object.hasOwn(payload({ width: 10, filters: {} }), 'filters'), + false, + ); + }); }); describe('parseSizeKey', () => { @@ -163,6 +227,31 @@ describe('getFilterSig', () => { getFilterSig({ crop: { x: '1' } } as never), ); }); + + test('an own "__proto__" key (from JSON.parse) is part of the signature', () => { + const red = JSON.parse('{"__proto__":"red"}'); + const blue = JSON.parse('{"__proto__":"blue"}'); + assert.notEqual(getFilterSig(red), getFilterSig(blue)); + assert.equal(getFilterSig(red), '__proto__:"red"'); + const nestedA = JSON.parse('{"crop":{"__proto__":{"x":1}}}'); + const nestedB = JSON.parse('{"crop":{"__proto__":{"x":2}}}'); + assert.notEqual(getFilterSig(nestedA), getFilterSig(nestedB)); + }); +}); + +describe('canonicalizeFilterValue', () => { + test('keeps an own "__proto__" key as data and never changes the prototype', () => { + const canonical = canonicalizeFilterValue( + JSON.parse('{"b":1,"__proto__":{"polluted":true}}'), + ) as Record; + assert.deepEqual(Object.keys(canonical), ['__proto__', 'b']); + assert.equal(Object.getPrototypeOf(canonical), Object.prototype); + assert.equal((canonical as { polluted?: unknown }).polluted, undefined); + assert.equal( + JSON.stringify(canonical), + '{"__proto__":{"polluted":true},"b":1}', + ); + }); }); describe('getPreviewIdentity', () => { @@ -267,18 +356,6 @@ describe('expandMissingPreviews with a scope', () => { }); }); -describe('getImageContentType', () => { - test('maps each raster preview format', () => { - assert.equal(getImageContentType('jpeg'), 'image/jpeg'); - assert.equal(getImageContentType('webp'), 'image/webp'); - assert.equal(getImageContentType('avif'), 'image/avif'); - }); - - test('undefined format → undefined', () => { - assert.equal(getImageContentType(undefined), undefined); - }); -}); - describe('calculateResizedDimensions', () => { test('cover passes both target dims through unchanged', () => { const r = calculateResizedDimensions(4000, 3000, 300, 300, false); @@ -359,6 +436,67 @@ describe('calculateResizedDimensions', () => { assert.equal(r.width, 1600); assert.equal(r.height, 1200); }); + + test('fit keeps each side at least 1 on an extreme aspect ratio', () => { + assert.deepEqual( + calculateResizedDimensions(10000, 2, undefined, undefined, true), + { width: 2000, height: 1 }, + ); + assert.deepEqual( + calculateResizedDimensions(2, 10000, undefined, undefined, true), + { width: 1, height: 1200 }, + ); + }); +}); + +describe('coverDimensions', () => { + test('both sides pass through, rounded and capped per side', () => { + assert.deepEqual(coverDimensions(4000, 3000, 300, 200, 5000), { + width: 300, + height: 200, + }); + assert.deepEqual(coverDimensions(4000, 3000, 300.5, 199.6, 5000), { + width: 301, + height: 200, + }); + assert.deepEqual(coverDimensions(64, 48, 9000, 50, 100), { + width: 100, + height: 50, + }); + }); + + test('width-only keeps the aspect ratio while the derived height fits the cap', () => { + assert.deepEqual(coverDimensions(4000, 3000, 620, undefined, 5000), { + width: 620, + height: undefined, + }); + }); + + test('width-only crops to the cap when the derived height would exceed it', () => { + // 1×100 source at width 1300 would be 1300×130000. + assert.deepEqual(coverDimensions(1, 100, 1300, undefined, 5000), { + width: 1300, + height: 5000, + }); + }); + + test('height-only crops to the cap when the derived width would exceed it', () => { + assert.deepEqual(coverDimensions(100, 1, undefined, 1300, 5000), { + width: 5000, + height: 1300, + }); + assert.deepEqual(coverDimensions(4000, 3000, undefined, 400, 5000), { + width: undefined, + height: 400, + }); + }); + + test('the requested side is capped before the derived side is computed', () => { + assert.deepEqual(coverDimensions(1, 100, 9000, undefined, 200), { + width: 200, + height: 200, + }); + }); }); describe('isCatalogCovered', () => { diff --git a/src/images.ts b/src/images.ts index ce8fb40..5b71b69 100644 --- a/src/images.ts +++ b/src/images.ts @@ -6,24 +6,33 @@ import type { Filters, MediaLike, MissingPreview, - Original, Preview, PreviewFormat, PreviewScope, SizeInput, } from './types.d.ts'; +/** A size dimension rounded to whole pixels; undefined when not finite or below 1 once rounded. */ +function keyDimension(n: number | undefined): number | undefined { + if (!isPositiveFinite(n)) { + return undefined; + } + const rounded = Math.round(n); + return rounded >= 1 ? rounded : undefined; +} + /** * Canonical size key (size only — never format or filters). `fit` wins; a dimension - * counts only if finite and > 0; each is Math.round-ed so the key round-trips through - * parseSizeKey's integer regexes. Throws when nothing usable is provided. + * counts only if finite and still ≥ 1 once Math.round-ed (so the key round-trips through + * parseSizeKey's integer regexes and never asks sharp for 0 pixels). Throws when nothing + * usable is provided. */ export function getSizeKey({ width, height, fit }: SizeInput): string { if (fit) { return 'fit'; } - const w = isPositiveFinite(width) ? Math.round(width) : undefined; - const h = isPositiveFinite(height) ? Math.round(height) : undefined; + const w = keyDimension(width); + const h = keyDimension(height); if (w !== undefined && h !== undefined) { return `${w}x${h}`; } @@ -97,7 +106,15 @@ export function canonicalizeFilterValue(value: unknown): unknown { // JSON.stringify omits undefined object values. The request-key payload // does too, so omit them here before identity construction. if (record[key] !== undefined) { - result[key] = canonicalizeFilterValue(record[key]); + // Define, never assign: `result['__proto__'] = …` would hit the prototype setter + // and drop an own "__proto__" key (JSON.parse creates one), so two different + // filter objects would share one identity. + Object.defineProperty(result, key, { + value: canonicalizeFilterValue(record[key]), + enumerable: true, + writable: true, + configurable: true, + }); } } return result; @@ -238,25 +255,39 @@ export function expandPreviewRequests( continue; } seen.add(identity); - const mp: MissingPreview = { sizeKey, format }; - if (size.filters && Object.keys(size.filters).length > 0) { - mp.filters = size.filters; - } - if (isPositiveFinite(size.width)) { - mp.requestedWidth = size.width; - } - if (isPositiveFinite(size.height)) { - mp.requestedHeight = size.height; - } - if (size.fit) { - mp.fit = true; - } - requested.push(mp); + requested.push(toMissingPreview(size, sizeKey, format)); } } return requested; } +/** + * Task payload for one size × format. The dimensions come from the size key, so the payload is + * a pure function of the preview identity: a fractional input is rounded exactly as the key + * rounds it, and a `fit` size carries no dimensions (fit ignores them). + */ +export function toMissingPreview( + size: SizeInput, + sizeKey: string, + format: PreviewFormat, +): MissingPreview { + const parsed = parseSizeKey(sizeKey); + const mp: MissingPreview = { sizeKey, format }; + if (size.filters && Object.keys(size.filters).length > 0) { + mp.filters = size.filters; + } + if (parsed.width !== undefined) { + mp.requestedWidth = parsed.width; + } + if (parsed.height !== undefined) { + mp.requestedHeight = parsed.height; + } + if (parsed.fit) { + mp.fit = true; + } + return mp; +} + /** * True when every `sizes × formats` identity of `scope` (default: the default Resizer and * pipeline) is already stored on `media.previews`. Hosts use this to skip a no-op @@ -271,12 +302,6 @@ export function isCatalogCovered( return expandMissingPreviews(media, sizes, formats, scope).length === 0; } -export function isSvgOriginal(original: Original | undefined): boolean { - return ( - original?.format === 'svg' || original?.contentType === 'image/svg+xml' - ); -} - export function isUsablePreview(preview: Preview): boolean { return Boolean( preview.storageRef != null && @@ -286,16 +311,6 @@ export function isUsablePreview(preview: Preview): boolean { ); } -/** - * Content type for a raster PREVIEW format only. Never pass an original's format — - * originals carry their own `original.contentType` (e.g. 'image/svg+xml'). - */ -export function getImageContentType( - format?: PreviewFormat, -): `image/${PreviewFormat}` | undefined { - return format ? `image/${format}` : undefined; -} - export interface ResizedDimensions { width?: number; height?: number; @@ -304,7 +319,8 @@ export interface ResizedDimensions { /** * cover (!fit): pass target dims straight through (either may be undefined for a * width-/height-only key — sharp resizes by the provided side). fit: scale to fit - * INSIDE maxSize preserving aspect, never upscaling beyond the original; sides rounded. + * INSIDE maxSize preserving aspect, never upscaling beyond the original; sides rounded + * and kept ≥ 1 (an extreme aspect ratio would otherwise round one side to 0). * origW/origH MUST be DISPLAY dims (post-EXIF-orient) — see 07 · Worker. */ export function calculateResizedDimensions( @@ -320,7 +336,35 @@ export function calculateResizedDimensions( } const scale = Math.min(maxSize.width / origW, maxSize.height / origH, 1); return { - width: Math.round(origW * scale), - height: Math.round(origH * scale), + width: Math.max(1, Math.round(origW * scale)), + height: Math.max(1, Math.round(origH * scale)), }; } + +/** + * The box a cover (cropping) variant is resized to. Each requested side is rounded to whole + * pixels (an old task payload may still carry a fraction) and capped at `cap` + * (limits.resultDimension). A width-only or height-only size keeps the source aspect ratio, + * unless the derived side would exceed `cap`: then the box is the requested side × `cap` and + * the cover resize crops, so no output side is ever larger than `cap`. srcW/srcH MUST be + * DISPLAY dims of one frame. + */ +export function coverDimensions( + srcW: number, + srcH: number, + targetW: number | undefined, + targetH: number | undefined, + cap: number, +): ResizedDimensions { + const side = (n: number | undefined) => + n === undefined ? undefined : Math.min(cap, Math.max(1, Math.round(n))); + const width = side(targetW); + const height = side(targetH); + if (width !== undefined && height === undefined) { + return { width, height: (width * srcH) / srcW > cap ? cap : undefined }; + } + if (height !== undefined && width === undefined) { + return { width: (height * srcW) / srcH > cap ? cap : undefined, height }; + } + return { width, height }; +} diff --git a/src/index.test.ts b/src/index.test.ts index c02ccfd..306d4a4 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -7,8 +7,8 @@ import * as api from './index.ts'; const SRC_DIR = dirname(fileURLToPath(import.meta.url)); -// Every VALUE export the main entry is contractually required to expose (02 · §6, reconciled -// with the real file layout). Kept as an explicit, sorted list so an ACCIDENTAL new value export +// Every VALUE export the main entry is contractually required to expose. +// Kept as an explicit list so an ACCIDENTAL new value export // (or a dropped one) fails THIS test rather than silently growing the public surface. Type-only // re-exports (contract interfaces, TResizeTask, types.d.ts) are erased and never appear here. const EXPECTED_VALUE_EXPORTS = [ @@ -23,25 +23,17 @@ const EXPECTED_VALUE_EXPORTS = [ 'ResizeSecurityError', 'ResizeSetupError', 'ResizeStorageError', - 'calculateResizedDimensions', - 'consumeQueue', 'formatPictureUrls', - 'getFilterSig', - 'getImageContentType', - 'getPreviewIdentity', 'getResizer', 'getSizeKey', 'isCatalogCovered', - 'listResizers', 'ResizeStorage', 'parseSizeKey', - 'processTask', 'resetResizerForTests', 'resizeMediaPaths', 'resizeMediaSchemaFragment', 'runWorker', 'TaskQueue', - 'timingOf', ]; // Drivers are SUBPATH-ONLY (the uniform rule 02 · §6) — they must NEVER appear on the main entry. @@ -87,21 +79,13 @@ describe('public API surface (src/index.ts)', () => { } }); - test('the helper + config accessors are functions', () => { + test('the helpers + registry accessors are functions', () => { for (const name of [ 'getResizer', 'resetResizerForTests', - 'listResizers', 'runWorker', - 'processTask', - 'consumeQueue', - 'timingOf', 'getSizeKey', 'parseSizeKey', - 'getFilterSig', - 'getPreviewIdentity', - 'calculateResizedDimensions', - 'getImageContentType', 'formatPictureUrls', 'isCatalogCovered', ]) { @@ -121,14 +105,6 @@ describe('public API surface (src/index.ts)', () => { test('a pure helper actually works through the re-export', () => { assert.equal(api.getSizeKey({ width: 320, height: 200 }), '320x200'); assert.equal(api.getSizeKey({ fit: true }), 'fit'); - assert.equal( - api.getPreviewIdentity( - { resizer: 'default', pipeline: 'default' }, - 'fit', - 'webp', - ), - 'default:default:fit:webp:none', - ); }); test('NO driver value exports leak onto the main entry (subpath-only rule)', () => { diff --git a/src/index.ts b/src/index.ts index b342870..e6d10e7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -39,24 +39,18 @@ export { ResizeSetupError, ResizeStorageError, } from './errors.ts'; -// --- pure identity + dimension helpers (03 · Identity) --- +// --- public URL formatting and size catalog helpers --- export { formatPictureUrls } from './formatPictureUrls.ts'; export { - calculateResizedDimensions, - getFilterSig, - getImageContentType, - getPreviewIdentity, getSizeKey, isCatalogCovered, parseSizeKey, } from './images.ts'; -// --- optional `as const` media schema fragment the host spreads into File/Media (08 · §12) --- +// --- optional `as const` media schema fragment the host spreads into File/Media --- export { resizeMediaPaths, resizeMediaSchemaFragment, } from './mediaFragment.ts'; -// --- the core queue logic (custom workers / tests); runWorker below is the normal entry --- -export { consumeQueue, timingOf } from './queue.ts'; // --- contract types for custom-driver / pipeline / hook authors (type-only; erased at runtime) --- export type { BeforeStep, @@ -72,14 +66,10 @@ export type { VariantStep, WaterfallName, } from './resizer.ts'; -// --- core: the Resizer + its registry accessors (constructor-wired; one per name) --- -// `resetResizerForTests` is a TEST-ONLY escape hatch. 02 · §6 documents it as "not re-exported -// from index.ts docs", but HOST test suites construct Resizers in their own tests (mirroring the -// framework publicly exporting `resetAppInstance`), so it IS re-exported here — documented -// deviation from that literal note. +// --- core: the Resizer + named lookup (constructor-wired; one per name) --- +// `resetResizerForTests` is a TEST-ONLY escape hatch for host test suites. export { getResizer, - listResizers, Resizer, resetResizerForTests, } from './resizer.ts'; @@ -87,4 +77,4 @@ export { export type * from './types.d.ts'; // --- the framework-free worker (framework hosts run `runResizeWorker` from …/framework.js) --- export type { RunWorkerOptions } from './worker.ts'; -export { processTask, runWorker } from './worker.ts'; +export { runWorker } from './worker.ts'; diff --git a/src/mediaFragment.test.ts b/src/mediaFragment.test.ts index 2e9f166..f9ac45e 100644 --- a/src/mediaFragment.test.ts +++ b/src/mediaFragment.test.ts @@ -41,6 +41,7 @@ describe('resizeMediaSchemaFragment — shape', () => { const p = resizeMediaSchemaFragment.previews[0]; for (const k of [ 'storageRef', + 'identity', 'resizer', 'pipeline', 'sizeKey', @@ -63,6 +64,7 @@ describe('resizeMediaSchemaFragment — shape', () => { assert.equal(resizeMediaSchemaFragment.previews[0].fit.type, Boolean); assert.equal(resizeMediaSchemaFragment.previews[0].resizer.type, String); assert.equal(resizeMediaSchemaFragment.previews[0].pipeline.type, String); + assert.equal(resizeMediaSchemaFragment.previews[0].identity.type, String); // Mixed via the string alias (no mongoose import in the fragment source). assert.equal(resizeMediaSchemaFragment.previews[0].filters.type, 'Mixed'); }); @@ -83,6 +85,8 @@ describe('resizeMediaSchemaFragment — host usage', () => { assert.ok(schema.path('original.width')); assert.ok(schema.path('original.storageRef')); assert.ok(schema.path('previews.storageRef')); + // The database checks this path at startup: strict mode would strip an unknown one. + assert.ok(schema.path('previews.identity')); assert.equal(schema.path('original.key'), undefined); assert.equal(schema.path('original.bucket'), undefined); }); diff --git a/src/mediaFragment.ts b/src/mediaFragment.ts index 5363cc0..85ff44b 100644 --- a/src/mediaFragment.ts +++ b/src/mediaFragment.ts @@ -32,11 +32,14 @@ export const resizeMediaSchemaFragment = { width: { type: Number }, height: { type: Number }, }, - // A generated variant (the full Preview): the worker `$push`es one of these per - // (sizeKey, format, filters) identity. + // A generated variant (the full Preview): the database stores one of these per preview + // identity. previews: [ { storageRef: { type: 'Mixed' }, + // The full preview identity (resizer:pipeline:sizeKey:format:filters). The database + // stores one row per identity, so it needs this path: strict mode would strip it. + identity: { type: String }, // Resizer and pipeline that generated this preview; absent means 'default'. resizer: { type: String }, pipeline: { type: String }, diff --git a/src/prewarm.test.ts b/src/prewarm.test.ts index ab8c52e..9e83555 100644 --- a/src/prewarm.test.ts +++ b/src/prewarm.test.ts @@ -93,6 +93,7 @@ function makeResizer( name?: string; queue?: string; hooks?: ConstructorParameters[0]['hooks']; + pipelines?: ConstructorParameters[0]['pipelines']; } = {}, ) { return new Resizer({ @@ -104,6 +105,7 @@ function makeResizer( name: options.name, queue: options.queue, hooks: options.hooks, + pipelines: options.pipelines, }); } @@ -125,6 +127,7 @@ describe('prewarm — happy path', () => { storage: makeStorage(), tasks, dbLocks, + pipelines: { photo: {} }, }); const { accepted } = await r.prewarm({ media: { id: 'm1', original: { storageRef: { key: 'orig.jpg' } } }, @@ -581,8 +584,8 @@ describe('prewarm — queue and lock failures never throw', () => { }); }); -describe('prewarm — fast-path is NOT consulted', () => { - test('a size the original already fits still gets added (generation decision, not serving)', async () => { +describe('prewarm — a small original', () => { + test('a box larger than the original is queued like any other size', async () => { installFakeApp(); const { tasks, calls } = makeTaskQueue(); const { dbLocks } = makeLocks(true); @@ -593,8 +596,8 @@ describe('prewarm — fast-path is NOT consulted', () => { }); const media: MediaLike = { id: 'm1', - // original (200×150) fits inside the 300×300 box — resolve() would serve the original, - // but prewarm generates the preview regardless (11 · §11.1b step 2). + // The 200×150 original is smaller than the 300×300 box: the worker makes a preview at the + // original's own size, so the variant is queued like any other. original: { storageRef: { key: 'orig.jpg' }, contentType: 'image/jpeg', @@ -624,6 +627,7 @@ describe('prewarm — pipelines are part of identity', () => { storage: makeStorage(), tasks, dbLocks, + pipelines: { watermark: {} }, }); const media = { id: 'm1', @@ -656,3 +660,178 @@ describe('prewarm — pipelines are part of identity', () => { assert.equal(calls[0].pipeline, 'watermark'); }); }); + +describe('prewarm — unknown pipeline', () => { + test('every requested variant is unconfirmed with one non-retryable RESIZE_PIPELINE_UNKNOWN issue', async () => { + const { errors } = installFakeApp(); + const { tasks, calls } = makeTaskQueue(); + const { dbLocks, acquired } = makeLocks(true); + const r = makeResizer({ tasks, dbLocks }); + const result = await r.prewarm({ + media: { + id: 'm1', + original: { storageRef: { key: 'orig.jpg' } }, + // Stored for that pipeline name: still not reported as ready. + previews: [ + { + storageRef: { key: 'p.jpg' }, + sizeKey: '300x300', + format: 'jpeg', + contentType: 'image/jpeg', + pipeline: 'retired', + }, + ], + }, + sizes: [{ width: 300, height: 300 }], + formats: ['jpeg', 'webp'], + pipeline: 'retired', + }); + assert.equal(result.status, 'incomplete'); + assert.deepEqual( + result.requested.map((p) => `${p.sizeKey}:${p.format}`), + ['300x300:jpeg', '300x300:webp'], + ); + assert.deepEqual(result.unconfirmed, result.requested); + assert.deepEqual(result.ready, []); + assert.deepEqual(result.accepted, []); + assert.deepEqual(result.notRequired, []); + assert.deepEqual(result.tasks, []); + assert.equal(result.issues.length, 1); + assert.equal(result.issues[0].code, 'RESIZE_PIPELINE_UNKNOWN'); + assert.equal(result.issues[0].retryable, false); + assert.match(result.issues[0].message, /'retired'/); + assert.deepEqual(result.issues[0].previews, result.requested); + assert.equal(calls.length, 0); + assert.equal(acquired.length, 0); + assert.equal(errors.length, 1); + }); +}); + +describe('prewarm — per-call formats', () => { + test('formats without an encode.formats entry are unconfirmed with a non-retryable issue; the rest queue', async () => { + const { errors } = installFakeApp(); + const { tasks, calls } = makeTaskQueue(); + const { dbLocks } = makeLocks(true); + const r = makeResizer({ tasks, dbLocks }); + const result = await r.prewarm({ + media: { id: 'm1', original: { storageRef: { key: 'orig.jpg' } } }, + sizes: [{ width: 300, height: 300 }], + formats: ['jpg', 'webp'], + }); + assert.equal(result.status, 'incomplete'); + assert.deepEqual( + calls[0].previews.map((p) => p.format), + ['webp'], + ); + assert.deepEqual( + result.accepted.map((p) => p.format), + ['webp'], + ); + assert.deepEqual( + result.unconfirmed.map((p) => p.format), + ['jpg'], + ); + assert.deepEqual(result.requested.map((p) => p.format).sort(), [ + 'jpg', + 'webp', + ]); + assert.equal(result.issues.length, 1); + assert.equal(result.issues[0].code, 'RESIZE_FORMAT_NOT_CONFIGURED'); + assert.equal(result.issues[0].retryable, false); + assert.deepEqual(result.issues[0].previews, result.unconfirmed); + assert.equal(errors.length, 1); + }); + + test('only unconfigured formats: nothing queued, incomplete rather than an empty request', async () => { + installFakeApp(); + const { tasks, calls } = makeTaskQueue(); + const { dbLocks } = makeLocks(true); + const r = makeResizer({ tasks, dbLocks }); + const result = await r.prewarm({ + media: { id: 'm1', original: { storageRef: { key: 'orig.jpg' } } }, + sizes: [{ width: 300, height: 300 }], + formats: ['jpg'], + }); + assert.equal(result.status, 'incomplete'); + assert.equal(result.reason, undefined); + assert.equal(result.unconfirmed.length, 1); + assert.equal(result.issues[0].code, 'RESIZE_FORMAT_NOT_CONFIGURED'); + assert.equal(calls.length, 0); + }); +}); + +describe('prewarm — formats from a beforeEnqueue tap', () => { + const jpg300: MissingPreview = { + sizeKey: '300x300', + format: 'jpg', + requestedWidth: 300, + requestedHeight: 300, + }; + + test('an unconfigured format a tap adds is unconfirmed with a non-retryable issue; the rest queue', async () => { + const { errors } = installFakeApp(); + const { tasks, calls } = makeTaskQueue(); + const { dbLocks, acquired } = makeLocks(true); + const r = makeResizer({ + tasks, + dbLocks, + hooks: { beforeEnqueue: (missing) => [...missing, jpg300] }, + }); + const result = await r.prewarm({ + media: { id: 'm1', original: { storageRef: { key: 'orig.jpg' } } }, + sizes: [{ width: 300, height: 300 }], + formats: ['webp'], + }); + assert.equal(result.status, 'incomplete'); + assert.deepEqual( + calls[0].previews.map((p) => p.format), + ['webp'], + ); + assert.equal(acquired.length, 1); + assert.deepEqual( + result.accepted.map((p) => p.format), + ['webp'], + ); + assert.deepEqual(result.unconfirmed, [jpg300]); + assert.deepEqual(result.requested.map((p) => p.format).sort(), [ + 'jpg', + 'webp', + ]); + assert.equal(result.issues.length, 1); + assert.equal(result.issues[0].code, 'RESIZE_FORMAT_NOT_CONFIGURED'); + assert.equal(result.issues[0].retryable, false); + assert.deepEqual(result.issues[0].previews, [jpg300]); + assert.equal(errors.length, 1); + }); + + test('a tap that rewrites every variant to an unconfigured format queues nothing; a per-call duplicate is reported once', async () => { + installFakeApp(); + const { tasks, calls } = makeTaskQueue(); + const { dbLocks } = makeLocks(true); + const r = makeResizer({ + tasks, + dbLocks, + hooks: { + beforeEnqueue: (missing) => + missing.map((m) => ({ ...m, format: 'jpg' })), + }, + }); + const result = await r.prewarm({ + media: { id: 'm1', original: { storageRef: { key: 'orig.jpg' } } }, + sizes: [{ width: 300, height: 300 }], + formats: ['jpg', 'webp'], + }); + assert.equal(result.status, 'incomplete'); + assert.equal(result.reason, undefined); + assert.equal(calls.length, 0); + assert.deepEqual(result.unconfirmed, [jpg300]); + assert.deepEqual( + result.notRequired.map((p) => p.format), + ['webp'], + ); + assert.deepEqual( + result.issues.map((issue) => [issue.code, issue.previews.length]), + [['RESIZE_FORMAT_NOT_CONFIGURED', 1]], + ); + }); +}); diff --git a/src/prewarmCoverage.test.ts b/src/prewarmCoverage.test.ts index fedda1c..3a71111 100644 --- a/src/prewarmCoverage.test.ts +++ b/src/prewarmCoverage.test.ts @@ -69,6 +69,7 @@ function makeResizer( name?: string; queue?: string; hooks?: ConstructorParameters[0]['hooks']; + pipelines?: ConstructorParameters[0]['pipelines']; } = {}, ) { return new Resizer({ @@ -80,6 +81,7 @@ function makeResizer( name: options.name, queue: options.queue, hooks: options.hooks, + pipelines: options.pipelines, }); } @@ -637,6 +639,7 @@ describe('prewarm — pipeline scope is part of preview identity', () => { storage, tasks, dbLocks: locks().dbLocks, + pipelines: { watermark: {} }, }); const media = { id: 'm1', @@ -663,3 +666,163 @@ describe('prewarm — pipeline scope is part of preview identity', () => { assert.equal(watermarked.accepted.length, 1); }); }); + +describe('prewarm — issues describe only what stays unconfirmed', () => { + for (const [label, add, code] of [ + ['a null taskId', async () => ({ taskId: null }), 'UNCONFIRMED'], + [ + 'a throwing add', + async () => { + throw new Error('connection dropped after insert'); + }, + 'QUEUE_FAILED', + ], + ] as const) { + test(`${label} confirmed by findActive leaves no RESIZE_ENQUEUE_${code} issue`, async () => { + const { tasks } = makeTaskQueue({ + add, + findActive: async () => [ + { taskId: 'active-1', previews: [taskPreview('jpeg')] }, + ], + }); + const r = makeResizer({ storage, tasks, dbLocks: locks().dbLocks }); + const result = await r.prewarm({ + media: { id: 'm1', original: { storageRef: { key: 'x.jpg' } } }, + sizes: [{ width: 300, height: 300 }], + formats: ['jpeg'], + }); + assert.equal(result.status, 'accepted'); + assert.deepEqual(result.unconfirmed, []); + assert.deepEqual(result.issues, []); + assert.deepEqual(result.tasks, [ + { taskId: 'active-1', previews: [taskPreview('jpeg')] }, + ]); + }); + } + + test('a partly confirmed failure keeps its issue for the unconfirmed previews only', async () => { + const { tasks } = makeTaskQueue({ + add: async () => { + throw new Error('connection dropped after insert'); + }, + findActive: async () => [ + { taskId: 'active-1', previews: [taskPreview('jpeg')] }, + ], + }); + const r = makeResizer({ storage, tasks, dbLocks: locks().dbLocks }); + const result = await r.prewarm({ + media: { id: 'm1', original: { storageRef: { key: 'x.jpg' } } }, + sizes: [{ width: 300, height: 300 }], + formats: ['jpeg', 'webp'], + }); + assert.equal(result.status, 'incomplete'); + assert.deepEqual(result.accepted, [taskPreview('jpeg')]); + assert.deepEqual(result.unconfirmed, [taskPreview('webp')]); + assert.deepEqual(result.issues, [ + { + code: 'RESIZE_ENQUEUE_QUEUE_FAILED', + message: 'adding the task failed; outcome is unconfirmed', + retryable: true, + previews: [taskPreview('webp')], + }, + ]); + }); +}); + +describe('prewarm — an internal error after expansion', () => { + test('echoes the expanded request, all of it unconfirmed', async () => { + const { tasks, added } = makeTaskQueue(); + const r = makeResizer({ + storage, + tasks, + dbLocks: locks().dbLocks, + // A host bug: the tap returns nothing, which the policy step cannot read. + hooks: { beforeEnqueue: (() => undefined) as never }, + }); + const result = await r.prewarm({ + media: { id: 'm1', original: { storageRef: { key: 'x.jpg' } } }, + sizes: [{ width: 300, height: 300 }], + formats: ['jpeg', 'webp'], + }); + const expanded = [taskPreview('jpeg'), taskPreview('webp')]; + assert.equal(result.status, 'incomplete'); + assert.deepEqual(result.requested, expanded); + assert.deepEqual(result.unconfirmed, expanded); + assert.deepEqual(result.ready, []); + assert.deepEqual(result.accepted, []); + assert.deepEqual(result.notRequired, []); + assert.deepEqual(result.tasks, []); + assert.equal(result.issues.length, 1); + assert.equal(result.issues[0].code, 'RESIZE_ENQUEUE_INTERNAL_ERROR'); + assert.equal(result.issues[0].retryable, true); + assert.deepEqual(result.issues[0].previews, expanded); + assert.equal(added.length, 0); + }); + + test('a config error before expansion still reports an empty request', async () => { + const r = new Resizer({ + storage, + db: fakeDb(), + logger, + config: () => ({ ...makeImageConfig(), upload: null }) as never, + }); + const result = await r.prewarm({ + media: { id: 'm1', original: { storageRef: { key: 'x.jpg' } } }, + sizes: [{ width: 300, height: 300 }], + }); + assert.equal(result.status, 'incomplete'); + assert.deepEqual(result.requested, []); + assert.deepEqual(result.unconfirmed, []); + assert.equal(result.issues[0].code, 'RESIZE_ENQUEUE_INTERNAL_ERROR'); + assert.equal(result.issues[0].retryable, false); + }); +}); + +describe('prewarm — dispatch locks', () => { + test('are acquired in parallel; contended and failed locks are still reported', async () => { + const { tasks, added } = makeTaskQueue({ + add: async () => ({ taskId: 'task-1' }), + findActive: async () => [], + }); + let inFlight = 0; + let maxInFlight = 0; + const r = makeResizer({ + storage, + tasks, + dbLocks: locks(async (key) => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + await new Promise((resolve) => setImmediate(resolve)); + inFlight -= 1; + if (key.includes(':avif:')) { + throw new Error('lock backend down'); + } + return !key.includes(':webp:'); + }).dbLocks, + }); + const result = await r.prewarm({ + media: { id: 'm1', original: { storageRef: { key: 'x.jpg' } } }, + sizes: [{ width: 300, height: 300 }], + formats: ['jpeg', 'webp', 'avif'], + }); + assert.equal(maxInFlight, 3); + assert.deepEqual( + added[0]?.previews.map((p) => p.format), + ['jpeg'], + ); + assert.deepEqual( + result.accepted.map((p) => p.format), + ['jpeg'], + ); + assert.deepEqual( + result.issues.map((issue) => [ + issue.code, + issue.previews.map((p) => p.format), + ]), + [ + ['RESIZE_ENQUEUE_LOCK_CONTENDED', ['webp']], + ['RESIZE_ENQUEUE_LOCK_FAILED', ['avif']], + ], + ); + }); +}); diff --git a/src/queue.test.ts b/src/queue.test.ts index f92d28c..2ac08ed 100644 --- a/src/queue.test.ts +++ b/src/queue.test.ts @@ -1,6 +1,6 @@ // The core queue loop (consumeQueue) against the in-memory TaskQueue: the same lifecycle every // backend gets — completion, retry with backoff, dead-lettering, terminal errors, crash loops, -// timeouts, lost leases and resilient claiming. +// timeouts, lost leases, graceful shutdown and resilient claiming. import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; import type { @@ -9,12 +9,96 @@ import type { NewTask, TaskEvent, } from './contracts/taskQueue.ts'; -import { ResizeConfigError, ResizeNoOriginalError } from './errors.ts'; +import { + ResizeConfigError, + ResizeError, + ResizeGenerateError, + ResizeMediaError, + ResizeNoOriginalError, +} from './errors.ts'; import { backoffMs, consumeQueue, timingOf } from './queue.ts'; import { MemoryTaskQueue } from './testHelpers/fakes.ts'; const silent = { info() {}, warn() {}, error() {} }; +/** The in-memory queue with `release`: the task is due at once and the delivery is not counted. */ +class ReleasingTaskQueue extends MemoryTaskQueue { + readonly releaseCalls: string[] = []; + + async release(task: ClaimedTask): Promise { + this.releaseCalls.push(task.taskId); + const row = this.rows.find( + (r) => + r.id === task.taskId && + r.status === 'processing' && + r.token === task.token, + ); + if (!row) { + return false; + } + row.status = 'pending'; + row.token = null; + row.availableAt = 0; + row.attempts -= 1; + return true; + } +} + +// Like the worker on SIGTERM: the task signal aborts, the current variant ends, the remaining ones +// are skipped, and the handler rejects as incomplete. +const stoppedByShutdown = ( + _task: LeasedTask, + { signal }: { signal: AbortSignal }, +) => + new Promise((_resolve, reject) => { + signal.addEventListener( + 'abort', + () => + reject( + new ResizeGenerateError({ + mediaId: 'm1', + failed: 1, + requested: 1, + code: 'RESIZE_WORKER_INCOMPLETE', + }), + ), + { once: true }, + ); + }); + +/** Run the loop until the handler has started, then shut it down and wait for it to return. */ +async function shutDownMidTask( + tasks: MemoryTaskQueue, + handle: ( + task: LeasedTask, + opts: { signal: AbortSignal }, + ) => Promise = stoppedByShutdown, +) { + const events: { event: TaskEvent; task: LeasedTask; error?: unknown }[] = []; + const stop = new AbortController(); + let started = false; + const loop = consumeQueue(tasks, { + queue: 'default', + signal: stop.signal, + handle: (task, opts) => { + started = true; + return handle(task, opts); + }, + onEvent: (event, task, error) => { + events.push({ event, task, error }); + }, + logger: silent, + }); + const until = Date.now() + 5000; + while (!started) { + assert.ok(Date.now() < until, 'the handler never started'); + await new Promise((r) => setTimeout(r, 2)); + } + stop.abort(); + await loop; + return events; +} + const newTask = (over: Partial = {}): NewTask => ({ resizer: 'default', queue: 'default', @@ -139,6 +223,57 @@ describe('consumeQueue', () => { ); }); + for (const code of [ + 'RESIZE_SOURCE_TOO_LARGE', + 'RESIZE_SOURCE_METADATA_MISSING', + 'RESIZE_SVG_RENDER_TIMEOUT', // rendering cannot get faster on a retry + ]) { + test(`an unusable source (${code}) is dead on the first failure`, async () => { + const tasks = new MemoryTaskQueue({ timing: fastTiming }); + await tasks.add(newTask()); + const events = await runUntil( + tasks, + async () => { + throw new ResizeMediaError('resize: unusable source', { + mediaId: 'm1', + code, + }); + }, + () => tasks.rows[0].status === 'dead', + ); + assert.equal(tasks.rows[0].attempts, 1); + assert.deepEqual( + events.map((e) => e.event), + ['deadLettered'], + ); + assert.equal((events[0].error as { code?: string }).code, code); + }); + } + + for (const code of ['RESIZE_WORKER_INCOMPLETE', 'RESIZE_PIPELINE_UNKNOWN']) { + test(`${code} stays a retryable failure`, async () => { + const tasks = new MemoryTaskQueue({ + timing: { + ...fastTiming, + retryBackoffMs: { base: 60_000, max: 60_000 }, + }, + }); + await tasks.add(newTask()); + const events = await runUntil( + tasks, + async () => { + throw new ResizeError('resize worker: not this time', { code }); + }, + () => + tasks.rows[0].attempts === 1 && tasks.rows[0].status === 'pending', + ); + assert.deepEqual( + events.map((e) => e.event), + ['failed'], + ); + }); + } + test('a task delivered more than maxAttempts times (crash loop) is dead without running', async () => { const tasks = new MemoryTaskQueue({ timing: fastTiming }); await tasks.add(newTask()); @@ -367,6 +502,185 @@ describe('consumeQueue', () => { }); }); +describe('consumeQueue shutdown', () => { + for (const prior of [0, 2]) { + test(`a shutdown mid-task gives the task back unprocessed (${prior} earlier attempts of 3)`, async () => { + const tasks = new ReleasingTaskQueue({ timing: fastTiming }); + await tasks.add(newTask()); + tasks.rows[0].attempts = prior; + const events = await shutDownMidTask(tasks); + assert.deepEqual(events, []); + assert.deepEqual(tasks.releaseCalls, ['task-1']); + assert.equal(tasks.rows[0].status, 'pending'); + assert.equal(tasks.rows[0].attempts, prior); + assert.ok(tasks.rows[0].availableAt <= Date.now(), 'claimable at once'); + assert.equal(tasks.rows[0].error, undefined); + }); + } + + for (const code of [ + 'RESIZE_NO_ORIGINAL', + 'RESIZE_SOURCE_METADATA_MISSING', + 'RESIZE_SOURCE_TOO_LARGE', + 'RESIZE_SVG_RENDER_TIMEOUT', + ]) { + test(`a terminal error (${code}) during shutdown is still dead-lettered, not given back`, async () => { + const tasks = new ReleasingTaskQueue({ timing: fastTiming }); + await tasks.add(newTask()); + // The shutdown arrives while the handler inspects the source, which then proves unusable. + const events = await shutDownMidTask( + tasks, + (_task, { signal }) => + new Promise((_resolve, reject) => { + signal.addEventListener( + 'abort', + () => + reject( + new ResizeMediaError('resize: unusable source', { + mediaId: 'm1', + code, + }), + ), + { once: true }, + ); + }), + ); + assert.deepEqual( + events.map((e) => e.event), + ['deadLettered'], + ); + assert.equal((events[0].error as { code?: string }).code, code); + assert.equal(tasks.rows[0].status, 'dead'); + assert.deepEqual(tasks.releaseCalls, []); + }); + } + + test('a queue without release retries the stopped task at once, with no event and never dead', async () => { + const tasks = new MemoryTaskQueue({ timing: fastTiming }); + await tasks.add(newTask()); + tasks.rows[0].attempts = 2; // this delivery is the last one allowed + const events = await shutDownMidTask(tasks); + assert.deepEqual(events, []); + assert.equal(tasks.rows[0].status, 'pending'); + // fail() cannot give the attempt back: the delivery counts. + assert.equal(tasks.rows[0].attempts, 3); + assert.ok(tasks.rows[0].availableAt <= Date.now(), 'no backoff'); + assert.match(tasks.rows[0].error ?? '', /shutdown/); + }); + + test('a handler that completes during shutdown is completed normally', async () => { + const tasks = new ReleasingTaskQueue({ timing: fastTiming }); + await tasks.add(newTask()); + const events = await shutDownMidTask( + tasks, + (_task, { signal }) => + new Promise((resolve) => { + signal.addEventListener('abort', () => resolve(), { once: true }); + }), + ); + assert.deepEqual( + events.map((e) => e.event), + ['completed'], + ); + assert.equal(tasks.rows[0].status, 'completed'); + assert.deepEqual(tasks.releaseCalls, []); + }); + + test('a task claimed while the worker shuts down is given back without running', async () => { + const tasks = new ReleasingTaskQueue({ timing: fastTiming }); + await tasks.add(newTask()); + const stop = new AbortController(); + const realClaim = tasks.claim.bind(tasks); + // The claim was already in flight when the shutdown arrived. + tasks.claim = async (...args: Parameters) => { + const task = await realClaim(...args); + stop.abort(); + return task; + }; + let ran = false; + const events: TaskEvent[] = []; + await consumeQueue(tasks, { + queue: 'default', + signal: stop.signal, + handle: async () => { + ran = true; + }, + onEvent: (event) => { + events.push(event); + }, + logger: silent, + }); + assert.equal(ran, false); + assert.deepEqual(events, []); + assert.equal(tasks.rows[0].status, 'pending'); + assert.equal(tasks.rows[0].attempts, 0); + }); + + test('a task that times out during shutdown still fails as a timeout', async () => { + const tasks = new ReleasingTaskQueue({ + timing: { ...fastTiming, taskTimeoutMs: 30 }, + }); + await tasks.add(newTask()); + const events = await shutDownMidTask(tasks, () => new Promise(() => {})); + assert.deepEqual( + events.map((e) => e.event), + ['failed'], + ); + assert.equal( + (events[0].error as { code?: string }).code, + 'RESIZE_TASK_TIMEOUT', + ); + assert.deepEqual(tasks.releaseCalls, []); + }); + + test('a task whose lease was lost before the shutdown is not released', async () => { + const tasks = new ReleasingTaskQueue({ + timing: { + ...fastTiming, + leaseMs: 20, + lockTtlMs: { dispatch: 1000, worker: 20 }, + }, + }); + await tasks.add(newTask()); + // Another worker took the task over. + tasks.renew = async () => { + tasks.rows[0].token = 'another-worker'; + return false; + }; + const failCalls: string[] = []; + const realFail = tasks.fail.bind(tasks); + tasks.fail = async (...args: Parameters) => { + failCalls.push(args[0].taskId); + return realFail(...args); + }; + const stop = new AbortController(); + const events: TaskEvent[] = []; + await consumeQueue(tasks, { + queue: 'default', + signal: stop.signal, + handle: (_task, { signal }) => + new Promise((_resolve, reject) => { + signal.addEventListener( + 'abort', + () => { + stop.abort(); // the deploy arrives right after the lease is gone + reject(new Error('lease lost')); + }, + { once: true }, + ); + }), + onEvent: (event) => { + events.push(event); + }, + logger: silent, + }); + assert.deepEqual(tasks.releaseCalls, []); + assert.deepEqual(failCalls, ['task-1']); // fenced: the new holder keeps it + assert.deepEqual(events, []); + assert.equal(tasks.rows[0].token, 'another-worker'); + }); +}); + describe('timingOf / backoffMs', () => { test('fills defaults, keeps set values, and is read once per queue', () => { let reads = 0; @@ -382,6 +696,43 @@ describe('timingOf / backoffMs', () => { assert.equal(reads, 1); }); + test('the dead-letter cooldown lockTtlMs.failed defaults to 10 minutes, also beside set lock TTLs', () => { + assert.equal(timingOf(new MemoryTaskQueue()).lockTtlMs.failed, 600_000); + const own = timingOf( + new MemoryTaskQueue({ + timing: { lockTtlMs: { dispatch: 1000, worker: 1000 } }, + }), + ); + assert.deepEqual(own.lockTtlMs, { + dispatch: 1000, + worker: 1000, + failed: 600_000, + }); + const set = timingOf( + new MemoryTaskQueue({ + timing: { lockTtlMs: { dispatch: 1000, worker: 1000, failed: 5000 } }, + }), + ); + assert.equal(set.lockTtlMs.failed, 5000); + }); + + for (const failed of [0, -1, 1.5, '600000', null]) { + test(`lockTtlMs.failed ${JSON.stringify(failed)} is a config error`, () => { + const tasks = new MemoryTaskQueue({ + timing: { + lockTtlMs: { dispatch: 1000, worker: 1000, failed } as never, + }, + }); + assert.throws( + () => timingOf(tasks), + (err: unknown) => + err instanceof ResizeConfigError && + err.code === 'RESIZE_CONFIG_QUEUE_LOCK_TTL_INVALID' && + /lockTtlMs\.failed/.test(err.message), + ); + }); + } + test('invalid timing is a config error (worker lock must fit the lease)', () => { const tasks = new MemoryTaskQueue({ timing: { leaseMs: 1000, lockTtlMs: { dispatch: 1000, worker: 5000 } }, diff --git a/src/queue.ts b/src/queue.ts index edada68..8020236 100644 --- a/src/queue.ts +++ b/src/queue.ts @@ -1,6 +1,7 @@ // The core queue logic, the same for every TaskQueue backend: the worker loop (claim, lease -// heartbeat, task timeout), the retry policy (backoff, dead-lettering after maxAttempts, terminal -// errors) and task events. Backends only implement the atomic TaskQueue operations. +// heartbeat, task timeout, giving tasks back at shutdown), the retry policy (backoff, +// dead-lettering after maxAttempts, terminal errors) and task events. Backends only implement the +// atomic TaskQueue operations. import { defaultQueueOptions } from './config/resize.ts'; import type { ClaimedTask, @@ -9,7 +10,7 @@ import type { TaskEventHandler, TaskQueue, } from './contracts/taskQueue.ts'; -import { ResizeError } from './errors.ts'; +import { ResizeConfigError, ResizeError } from './errors.ts'; import { sleep } from './helpers/sleep.ts'; import { validateQueueTiming } from './resizeConfig.ts'; import type { QueueTimingOptions, ResizeLogger } from './types.d.ts'; @@ -23,15 +24,37 @@ const TIMING_KEYS = [ 'taskTimeoutMs', ] as const; -const timings = new WeakMap(); +/** Complete queue timing: every key set, the optional lock TTLs included. */ +export type QueueTiming = QueueTimingOptions & { + lockTtlMs: Required; +}; + +const timings = new WeakMap(); + +/** + * Check the optional dead-letter cooldown `lockTtlMs.failed` (ms) when it is set. Throws + * ResizeConfigError. + */ +export function validateFailedLockTtl(failed: unknown): void { + if ( + failed !== undefined && + !(typeof failed === 'number' && Number.isSafeInteger(failed) && failed > 0) + ) { + throw new ResizeConfigError( + 'resize queue options: lockTtlMs.failed must be a positive safe integer (ms)', + { code: 'RESIZE_CONFIG_QUEUE_LOCK_TTL_INVALID' }, + ); + } +} /** * Complete queue timing: the timing keys set in `own` over the defaults (other keys are ignored), - * validated. Throws ResizeConfigError for invalid timing. + * validated. A `lockTtlMs` without `failed` gets the default cooldown. Throws ResizeConfigError for + * invalid timing. */ export function fillTiming( own: Partial | Record, -): QueueTimingOptions { +): QueueTiming { const timing: Record = { ...defaultQueueOptions }; for (const key of TIMING_KEYS) { const value = (own as Record)[key]; @@ -39,15 +62,24 @@ export function fillTiming( timing[key] = value; } } + const lockTtlMs = timing.lockTtlMs; + if (typeof lockTtlMs === 'object' && lockTtlMs !== null) { + const failed = (lockTtlMs as { failed?: unknown }).failed; + validateFailedLockTtl(failed); + timing.lockTtlMs = { + ...lockTtlMs, + failed: failed ?? defaultQueueOptions.lockTtlMs.failed, + }; + } validateQueueTiming(timing); - return timing as unknown as QueueTimingOptions; + return timing as unknown as QueueTiming; } /** * A queue's timing: its getTiming() over the defaults, validated once per queue instance (a lazy * getTiming is read on first use). Throws ResizeConfigError for invalid timing. */ -export function timingOf(tasks: TaskQueue): QueueTimingOptions { +export function timingOf(tasks: TaskQueue): QueueTiming { const cached = timings.get(tasks); if (cached) { return cached; @@ -78,9 +110,28 @@ export function toLeasedTask(task: ClaimedTask): LeasedTask { }; } +// Errors no retry can fix: the media row has no original, its source has no dimensions or is over +// the pixel limits, or its SVG takes longer to render than allowed. Each retry would only download +// and decode the original again. Errors cross module boundaries as plain objects, so match the +// stable code, not the class. +const TERMINAL_ERROR_CODES: ReadonlySet = new Set([ + 'RESIZE_NO_ORIGINAL', + 'RESIZE_SOURCE_METADATA_MISSING', + 'RESIZE_SOURCE_TOO_LARGE', + 'RESIZE_SVG_RENDER_TIMEOUT', +]); + +const isTerminal = (error: unknown): boolean => + typeof error === 'object' && + error !== null && + 'code' in error && + TERMINAL_ERROR_CODES.has((error as { code?: unknown }).code); + export interface ConsumeQueueOptions { queue: string; - signal: AbortSignal; // stops the loop: the current task finishes or aborts, then it returns + // Stops the loop: the current task finishes, or goes back to the queue unprocessed if the stop + // aborted it; then it returns. + signal: AbortSignal; handle: ( task: LeasedTask, taskOpts: { signal: AbortSignal }, @@ -124,16 +175,7 @@ export async function consumeQueue( error: unknown, forceDead = false, ): Promise => { - // A media row without an original is terminal: retrying cannot make it appear. Errors cross - // module boundaries as plain objects, so match the stable code, not the class. - const code = - typeof error === 'object' && error !== null && 'code' in error - ? (error as { code?: unknown }).code - : undefined; - const dead = - forceDead || - code === 'RESIZE_NO_ORIGINAL' || - task.attempts >= maxAttempts; + const dead = forceDead || isTerminal(error) || task.attempts >= maxAttempts; try { const held = await tasks.fail( task, @@ -151,6 +193,31 @@ export async function consumeQueue( logger.error(`resize worker: failing task ${task.taskId} failed`, err); } }; + // The worker's shutdown stopped the task: it was not processed, so it goes back with no event, + // no backoff and, where the queue can release it, without counting the delivery. A queue without + // `release` retries it at once through `fail`; the delivery then counts, but it is never + // dead-lettered here. + const giveBack = async (task: ClaimedTask): Promise => { + try { + const held = tasks.release + ? await tasks.release(task) + : await tasks.fail( + task, + { retryAt: new Date() }, + `resize worker: task ${task.taskId} was stopped by a worker shutdown`, + ); + if (held) { + logger.info( + `resize worker: shutdown — task ${task.taskId} is back in the queue`, + ); + } + } catch (err) { + logger.error( + `resize worker: giving task ${task.taskId} back failed`, + err, + ); + } + }; let claimFailures = 0; while (!opts.signal.aborted) { @@ -193,21 +260,25 @@ export async function consumeQueue( ); continue; } + // The shutdown arrived while the claim ran (a claim that returns at once ignores the signal). + if (opts.signal.aborted) { + await giveBack(task); + break; + } // Per-task lease-loss signal; worker shutdown also aborts the current task, so it finishes - // its current variant, skips the rest, and the loop exits promptly. + // its current variant, skips the rest, goes back to the queue, and the loop exits promptly. const taskController = new AbortController(); const onShutdown = () => taskController.abort(); opts.signal.addEventListener('abort', onShutdown, { once: true }); - if (opts.signal.aborted) { - taskController.abort(); - } const claimed = task; + let leaseLost = false; const heartbeat = setInterval(() => { tasks .renew(claimed, leaseMs) .then((held) => { if (!held) { + leaseLost = true; taskController.abort(); } }) @@ -271,6 +342,10 @@ export async function consumeQueue( err, ); } + } else if (opts.signal.aborted && !leaseLost && !isTerminal(handlerError)) { + // Stopped by the shutdown, not by its own failure: not an attempt. A terminal error still + // dead-letters: the task proved it can never succeed before the shutdown stopped it. + await giveBack(task); } else { await finish(task, handlerError); } diff --git a/src/resizeTask.test.ts b/src/resizeTask.test.ts index 3366ee0..5dc8587 100644 --- a/src/resizeTask.test.ts +++ b/src/resizeTask.test.ts @@ -2,12 +2,13 @@ // generated with sharp itself; fakes for storage, database and task queues. // Fresh Resizer + fake ambient app per test (node:test = per-file process isolation). import assert from 'node:assert/strict'; +import childProcess from 'node:child_process'; import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { createServer } from 'node:http'; import type { AddressInfo } from 'node:net'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterEach, describe, test } from 'node:test'; +import { afterEach, describe, mock, test } from 'node:test'; import { resetAppInstance, setAppInstance, @@ -18,12 +19,14 @@ import type { LeasedTask, NewTask } from './contracts/taskQueue.ts'; import { ResizeConfigError, ResizeGenerateError, + ResizeMediaError, ResizeNoOriginalError, ResizeSetupError, } from './errors.ts'; import ResizeWorker from './framework/ResizeWorkerCommand.ts'; import { FrameworkResizer } from './framework/resizer.ts'; import { runResizeWorker } from './framework/worker.ts'; +import { getSizeKey, toMissingPreview } from './images.ts'; import { type Pipeline, Resizer, @@ -88,6 +91,128 @@ const orientedJpeg = await sharp({ .withMetadata({ orientation: 6 }) .toBuffer(); +/** Raw RGB frames of one solid colour each, stacked vertically (sharp's animation layout). */ +function solidFrames(colours: number[][], width: number, height: number) { + return Buffer.concat( + colours.map((colour) => { + const frame = Buffer.alloc(width * height * 3); + for (let i = 0; i < width * height; i++) { + frame.set(colour, i * 3); + } + return frame; + }), + ); +} + +const RGB = [ + [255, 0, 0], + [0, 255, 0], + [0, 0, 255], +]; + +// Three 20×20 frames: red, green, blue. +const animatedGif = await sharp(solidFrames(RGB, 20, 20), { + raw: { width: 20, height: 60, channels: 3, pageHeight: 20 }, +}) + .gif({ delay: [100, 100, 100] }) + .toBuffer(); + +// Three 20×10 frames with EXIF orientation 6 → displayed 10×20. +const orientedAnimatedWebp = await sharp(solidFrames(RGB, 20, 10), { + raw: { width: 20, height: 30, channels: 3, pageHeight: 10 }, +}) + .webp({ delay: [100, 100, 100] }) + .withMetadata({ orientation: 6 }) + .toBuffer(); + +const tallPng = await sharp({ + create: { width: 1, height: 100, channels: 3, background: '#808080' }, +}) + .png() + .toBuffer(); + +const widePng = await sharp({ + create: { width: 100, height: 1, channels: 3, background: '#808080' }, +}) + .png() + .toBuffer(); + +// A 1×5000 SVG: at the density a 300×300 cover needs, its long side would exceed librsvg's +// 32767-pixel limit. +const tallSvg = Buffer.from( + '', +); + +// Six turbulence-filtered rects: librsvg needs more than 10 s for this 488-byte SVG, inside one +// native call that sharp's `.timeout()` cannot interrupt. +const heavySvg = Buffer.from( + `${''.repeat(6)}`, +); + +// Larger than the boxes the fractional-size and cap tests request, so they still crop. +const bigPng = await sharp({ + create: { width: 400, height: 300, channels: 3, background: '#3366cc' }, +}) + .png() + .toBuffer(); + +// 100×80 photo-like JPEG carrying EXIF, including a GPS position. +const exifJpeg = await sharp(texture(100, 80), { + raw: { width: 100, height: 80, channels: 3 }, +}) + .jpeg({ quality: 90 }) + .withExif({ + IFD0: { Artist: 'Someone', Copyright: 'Someone' }, + IFD3: { + GPSLatitudeRef: 'N', + GPSLatitude: '51/1 30/1 0/1', + GPSLongitudeRef: 'W', + GPSLongitude: '0/1 7/1 0/1', + }, + }) + .toBuffer(); + +// Photo-like texture (smooth waves plus fine noise), so re-encoding loss is measurable. +function texture(width: number, height: number): Buffer { + const raw = Buffer.alloc(width * height * 3); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const i = (y * width + x) * 3; + const v = + 128 + + 50 * Math.sin(x / 7) * Math.cos(y / 11) + + 30 * Math.sin((x + y) / 3.3) + + ((((x * 73856093) ^ (y * 19349663)) >>> 0) % 21) - + 10; + raw[i] = Math.max(0, Math.min(255, v)); + raw[i + 1] = Math.max(0, Math.min(255, 255 - v * 0.8)); + raw[i + 2] = Math.max(0, Math.min(255, v * 0.5 + 60)); + } + } + return raw; +} + +// Identical stored pixels (300×200); only the EXIF orientation tag differs. +const texturedJpeg = (orientation: number) => + sharp(texture(300, 200), { raw: { width: 300, height: 200, channels: 3 } }) + .jpeg({ quality: 95 }) + .withMetadata({ orientation }) + .toBuffer(); + +/** Peak signal-to-noise ratio of two same-size images (Infinity when identical). */ +async function psnr(a: Buffer, b: Buffer): Promise { + const x = await sharp(a).removeAlpha().raw().toBuffer(); + const y = await sharp(b).removeAlpha().raw().toBuffer(); + assert.equal(x.length, y.length); + let squared = 0; + for (let i = 0; i < x.length; i++) { + squared += (x[i] - y[i]) ** 2; + } + return squared === 0 + ? Number.POSITIVE_INFINITY + : 10 * Math.log10((255 * 255 * x.length) / squared); +} + // --------------------------------------------------------------------------- // Fakes // --------------------------------------------------------------------------- @@ -143,7 +268,6 @@ function makeStorage( return { bucket: 'previews', key }; }, publicUrl: (ref) => `https://cdn/${ref.key}`, - canServeOriginalPublicly: (ref) => ref.bucket === 'previews', }; return { storage, uploads }; } @@ -239,8 +363,15 @@ const fitVariant: MissingPreview = { afterEach(() => { resetResizerForTests(); resetAppInstance(); + mock.restoreAll(); }); +/** Count SVG render processes (svgRaster spawns one node child per render). */ +function countRenders(): () => number { + const spawn = mock.method(childProcess, 'spawn'); + return () => spawn.mock.callCount(); +} + // --------------------------------------------------------------------------- // processTask — download / metadata / beforeSteps // --------------------------------------------------------------------------- @@ -684,7 +815,8 @@ describe('processTask — variants', () => { test('cover branch clamps each side to limits.resultDimension', async () => { installApp({ limits: { resultDimension: 100 } }); - const { storage } = makeStorage(redPng); + // 400×300 is larger than the requested 300×300 box, so it is cover-cropped, to the cap. + const { storage } = makeStorage(bigPng); const { db, appendCalls } = makeDatabase(mediaDoc()); new FrameworkResizer({ storage, @@ -694,9 +826,9 @@ describe('processTask — variants', () => { task({ previews: [ variant({ - sizeKey: '5000x5000', - requestedWidth: 5000, - requestedHeight: 5000, + sizeKey: '300x300', + requestedWidth: 300, + requestedHeight: 300, }), ], }), @@ -1498,174 +1630,1737 @@ describe('generate (eager)', () => { }); // --------------------------------------------------------------------------- -// runResizeWorker (07 · §11) +// Pipelines: an unregistered name never renders // --------------------------------------------------------------------------- -// Stop idle test queues deterministically after their queued work has been handled. -function observedQueue(onIdle: () => void = () => process.emit('SIGTERM')) { - const tasks = new MemoryTaskQueue({ timing: { idlePollMs: 1 } }); - const claimedQueues: string[] = []; - const claim = tasks.claim.bind(tasks); - tasks.claim = async (queue, leaseMs) => { - claimedQueues.push(queue); - const next = await claim(queue, leaseMs); - if (!next) { - onIdle(); - } - return next; - }; - return { tasks, claimedQueues }; -} +describe('unknown pipeline', () => { + function countingStorage(fixture: Buffer) { + const base = makeStorage(fixture); + let downloads = 0; + const storage: ResizeStorage = { + ...base.storage, + download: async () => { + downloads += 1; + return fixture; + }, + }; + return { storage, uploads: base.uploads, downloads: () => downloads }; + } -async function addTask(tasks: MemoryTaskQueue, over: Partial = {}) { - const { taskId: _taskId, ...payload } = task(); - await tasks.add({ ...payload, requestKey: JSON.stringify(over), ...over }); -} + const isUnknownPipeline = (err: unknown) => + err instanceof ResizeSetupError && + err.code === 'RESIZE_PIPELINE_UNKNOWN' && + err.message.includes("'watermark-v2'"); -describe('runResizeWorker', () => { - test('worker.enabled=false → clean no-op (claim NOT called); log says how to enable', async () => { - const { logs } = installApp(); - const { tasks, claimedQueues } = observedQueue(); - new FrameworkResizer({ storage: makeStorage(redPng).storage, tasks }); - await runResizeWorker(); - assert.deepEqual(claimedQueues, []); - assert.ok( - logs.info.some((l) => String(l[0]).includes('worker.enabled=true')), + test('a queued task for an unregistered pipeline fails before download; nothing is stored', async () => { + installApp(); + const { storage, uploads, downloads } = countingStorage(redPng); + const { db, appendCalls } = makeDatabase(mediaDoc()); + const { lockProvider, acquired } = makeLocks(true); + new FrameworkResizer({ + storage, + db: { ...db, ...fakeLockMethods(lockProvider) }, + pipelines: { watermark: {} }, + }); + await assert.rejects( + () => + processTask(task({ pipeline: 'watermark-v2', previews: [variant()] })), + isUnknownPipeline, ); + assert.equal(downloads(), 0); + assert.equal(uploads.length, 0); + assert.equal(appendCalls.length, 0); + assert.deepEqual(acquired, []); }); - test('no task queue → logs an error and returns without preparing framework drivers', async () => { - const { logs, getModelCalls } = installApp({ worker: { enabled: true } }); - new FrameworkResizer({ storage: makeStorage(redPng).storage }); - await runResizeWorker(); - assert.ok(logs.error.length >= 1); - assert.equal(getModelCalls(), 0); - }); - - test('a worker with no Resizers fails before leasing', async () => { - installApp({ worker: { enabled: true } }); + test('generate() with an unregistered pipeline throws before download', async () => { + installApp(); + const { storage, uploads, downloads } = countingStorage(redPng); + const { db, appendCalls } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); await assert.rejects( - () => runResizeWorker(), - (err: unknown) => - err instanceof ResizeSetupError && err.code === 'RESIZE_NO_RESIZER', + () => + r.generate({ + media: mediaDoc(), + sizes: [{ width: 20, height: 20 }], + formats: ['jpeg'], + pipeline: 'watermark-v2', + }), + isUnknownPipeline, ); + assert.equal(downloads(), 0); + assert.equal(uploads.length, 0); + assert.equal(appendCalls.length, 0); }); - test('default database + unregistered mediaModelName → throws before claiming', async () => { - setAppInstance({ - getConfig: () => - makeResizeConfig({ - mediaModelName: 'Media', - worker: { enabled: true }, - }), - getModel: () => false, - logger: { info() {}, warn() {}, error() {} }, - } as never); - const { tasks, claimedQueues } = observedQueue(); - new FrameworkResizer({ storage: makeStorage(redPng).storage, tasks }); - await assert.rejects( - () => runResizeWorker(), - (err: unknown) => - err instanceof ResizeConfigError && - err.code === 'RESIZE_CONFIG_MEDIA_MODEL_UNKNOWN', - ); - assert.deepEqual(claimedQueues, []); + test("'default' renders without being registered", async () => { + installApp(); + const { storage, uploads } = makeStorage(redPng); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db, pipelines: {} }); + const { created } = await r.generate({ + media: mediaDoc(), + sizes: [{ width: 20, height: 20 }], + formats: ['jpeg'], + }); + assert.equal(created.length, 1); + assert.equal(uploads.length, 1); }); +}); - test('custom database verify() runs before claim, and its failure stops the worker', async () => { - installApp({ worker: { enabled: true } }); - const events: string[] = []; - const { tasks } = observedQueue(() => { - events.push('claim'); - process.emit('SIGTERM'); +// --------------------------------------------------------------------------- +// Animated sources (config.animated) +// --------------------------------------------------------------------------- + +describe('animated sources', () => { + const formats = ['webp', 'gif', 'avif', 'jpeg', 'png']; + const widthOnly = (format: string) => + variant({ + sizeKey: '10w', + format, + requestedWidth: 10, + requestedHeight: undefined, + }); + + async function framesOf(body: Buffer) { + const meta = await sharp(body, { animated: true }).metadata(); + return { + pages: meta.pages ?? 1, + frameHeight: meta.pageHeight ?? meta.height, + }; + } + + async function pixelAt(body: Buffer, left: number, top: number) { + return [ + ...(await sharp(body) + .removeAlpha() + .extract({ left, top, width: 1, height: 1 }) + .raw() + .toBuffer()), + ]; + } + + const topLeftPixel = (body: Buffer) => pixelAt(body, 0, 0); + + test('a 3-frame GIF: webp and gif keep every frame, other formats get the first frame', async () => { + installApp({ + animated: true, + encode: { formats: { gif: {}, png: {} } }, }); + const { storage, uploads } = makeStorage(animatedGif); + const { db, appendCalls } = makeDatabase(mediaDoc()); new FrameworkResizer({ - storage: makeStorage(redPng).storage, - tasks, - db: fakeDb({ - // async: a rejected promise stops the worker only if verify() is awaited - async verify() { - events.push('verify'); - throw new ResizeConfigError('custom database is misconfigured', { - code: 'CUSTOM_STORE_INVALID', - }); - }, - }), + storage, + db: { ...db, ...fakeLockMethods(makeLocks().lockProvider) }, }); - await assert.rejects( - () => runResizeWorker(), - (err: unknown) => - err instanceof ResizeConfigError && err.code === 'CUSTOM_STORE_INVALID', + await processTask(task({ previews: formats.map(widthOnly) })); + + assert.equal(uploads.length, formats.length); + const bodies = new Map(uploads.map((u) => [u.key, u.body])); + const expectedPages: Record = { + webp: 3, + gif: 3, + avif: 1, + jpeg: 1, + png: 1, + }; + for (const row of appendCalls[0].previews) { + const { format } = row; + const body = bodies.get((row.storageRef as { key: string }).key); + assert.ok(body, `${format} uploaded`); + const { pages, frameHeight } = await framesOf(body); + assert.equal(pages, expectedPages[format], `${format} pages`); + assert.equal(frameHeight, 10, `${format} frame height`); + // Per-frame size recorded, not the height of all frames stacked. + assert.equal(row.actualWidth, 10, `${format} actualWidth`); + assert.equal(row.actualHeight, 10, `${format} actualHeight`); + if (expectedPages[format] === 1) { + const [r, g, b] = await topLeftPixel(body); + assert.ok( + r > 200 && g < 60 && b < 60, + `${format} is the first (red) frame`, + ); + } + } + assert.deepEqual( + appendCalls[0].previews.map((p) => p.format).sort(), + [...formats].sort(), ); - assert.deepEqual(events, ['verify']); - resetResizerForTests(); - events.length = 0; - new FrameworkResizer({ - storage: makeStorage(redPng).storage, - tasks, - db: fakeDb({ - async verify() { - events.push('verify'); - }, - }), - }); - await runResizeWorker(); - assert.deepEqual(events, ['verify', 'claim']); }); - test('enabled + task queue → a claimed task reaches processTask', async () => { - installApp({ worker: { enabled: true } }); - const { tasks } = observedQueue(); - let loads = 0; + test('an animation above limits.animationFrames keeps only the first frames', async () => { + installApp({ animated: true, limits: { animationFrames: 2 } }); + const { storage, uploads } = makeStorage(animatedGif); + const { db, appendCalls } = makeDatabase(mediaDoc()); new FrameworkResizer({ - storage: makeStorage(redPng).storage, - tasks, - db: fakeDb({ - load: async () => { - loads++; - return null; - }, - }), + storage, + db: { ...db, ...fakeLockMethods(makeLocks().lockProvider) }, }); - await addTask(tasks, { previews: [variant()] }); - await runResizeWorker(); - assert.equal(loads, 1); - assert.equal(tasks.rows[0].status, 'completed'); + await processTask( + task({ previews: [widthOnly('webp'), variant({ format: 'webp' })] }), + ); + assert.equal(uploads.length, 2); + for (const upload of uploads) { + assert.equal((await framesOf(upload.body)).pages, 2); + } + const cover = appendCalls[0].previews.find((p) => p.sizeKey === '20x20'); + assert.equal(cover?.actualWidth, 20); + assert.equal(cover?.actualHeight, 20); }); -}); -describe('one worker serves every Resizer', () => { - function register(tasks: MemoryTaskQueue, name = 'default', db = fakeDb()) { - return new FrameworkResizer({ - name, - storage: makeStorage(redPng).storage, - tasks, - db, + test('animated: false (default) decodes only the first frame', async () => { + installApp(); + const { storage, uploads } = makeStorage(animatedGif); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + await r.generate({ + media: mediaDoc(), + sizes: [{ width: 10 }], + formats: ['webp'], + }); + assert.deepEqual(await framesOf(uploads[0].body), { + pages: 1, + frameHeight: 10, }); - } - - test('one task queue serves both Resizers, and each task runs with its own Resizer', async () => { - installApp({ worker: { enabled: true } }); - const { tasks, claimedQueues } = observedQueue(); - const loadedBy: string[] = []; - const labelled = (label: string) => - fakeDb({ - load: async () => { - loadedBy.push(label); - return null; - }, - }); - register(tasks, 'default', labelled('default')); - register(tasks, 'listings', labelled('listings')); - await addTask(tasks, { resizer: 'listings' }); - await addTask(tasks); - await runResizeWorker(); - assert.deepEqual(loadedBy, ['listings', 'default']); - assert.deepEqual(claimedQueues, ['default', 'default', 'default']); - assert.ok(tasks.rows.every((row) => row.status === 'completed')); }); - test('runResizeWorker({ queue }) consumes that queue only', async () => { + test('an animation with an EXIF orientation renders upright from its first frame', async () => { + // libvips cannot rotate a multi-page image, so orientation wins over the animation. + installApp({ animated: true }); + const { storage, uploads } = makeStorage(orientedAnimatedWebp); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const { created, failed } = await r.generate({ + media: mediaDoc(), + sizes: [{ fit: true }], + formats: ['webp'], + }); + assert.equal(failed, 0); + assert.equal(created[0].actualWidth, 10); + assert.equal(created[0].actualHeight, 20); + assert.deepEqual(await framesOf(uploads[0].body), { + pages: 1, + frameHeight: 20, + }); + }); + + test('a pipeline with variantSteps renders the first frame, so a watermark lands on every output', async () => { + installApp({ animated: true, encode: { formats: { gif: {} } } }); + const mark = await sharp({ + create: { width: 6, height: 6, channels: 3, background: '#ffffff' }, + }) + .png() + .toBuffer(); + const { storage, uploads } = makeStorage(animatedGif); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ + storage, + db, + pipelines: { + default: { + variantSteps: [ + async (img) => + img.composite([{ input: mark, gravity: 'southeast' }]), + ], + }, + }, + }); + const { created, failed } = await r.generate({ + media: mediaDoc(), + sizes: [{ width: 20, height: 20 }], + formats: ['webp', 'gif'], + }); + assert.equal(failed, 0); + assert.equal(created.length, 2); + for (const upload of uploads) { + assert.deepEqual(await framesOf(upload.body), { + pages: 1, + frameHeight: 20, + }); + const [r0, g0, b0] = await topLeftPixel(upload.body); + assert.ok(r0 > 200 && g0 < 60 && b0 < 60, 'the first (red) frame'); + const [r1, g1, b1] = await pixelAt(upload.body, 18, 18); + assert.ok(r1 > 200 && g1 > 200 && b1 > 200, 'the overlay is present'); + } + }); + + test('limits.sourcePixels counts frames only when an output is animated', async () => { + // 20×20 frames: one frame (400 px) fits 800, all three (1200 px) do not. + installApp({ animated: true, limits: { sourcePixels: 800 } }); + const { storage, uploads } = makeStorage(animatedGif); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const { created, failed } = await r.generate({ + media: mediaDoc(), + sizes: [{ width: 10 }], + formats: ['jpeg', 'avif'], + }); + assert.equal(failed, 0); + assert.equal(created.length, 2); + for (const upload of uploads) { + assert.equal((await framesOf(upload.body)).pages, 1); + } + }); + + test('an animation over limits.sourcePixels is shortened to the frames that fit', async () => { + installApp({ animated: true, limits: { sourcePixels: 800 } }); + const { storage, uploads } = makeStorage(animatedGif); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const { created, failed } = await r.generate({ + media: mediaDoc(), + sizes: [{ width: 10 }], + formats: ['webp'], + }); + assert.equal(failed, 0); + assert.equal(created[0].actualHeight, 10); + assert.deepEqual(await framesOf(uploads[0].body), { + pages: 2, + frameHeight: 10, + }); + }); + + test('a single animation frame over limits.sourcePixels is still refused', async () => { + installApp({ animated: true, limits: { sourcePixels: 300 } }); + const { storage, uploads } = makeStorage(animatedGif); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + await assert.rejects( + () => + r.generate({ + media: mediaDoc(), + sizes: [{ width: 10 }], + formats: ['webp'], + }), + (err: unknown) => + err instanceof ResizeMediaError && + err.code === 'RESIZE_SOURCE_TOO_LARGE', + ); + assert.equal(uploads.length, 0); + }); +}); + +// --------------------------------------------------------------------------- +// Cover sizes: the derived side, fractional sizes +// --------------------------------------------------------------------------- + +describe('cover sizes', () => { + test('width-only on a 1×100 source: the derived height is cropped to limits.resultDimension', async () => { + installApp({ limits: { resultDimension: 200 } }); + const { storage, uploads } = makeStorage(tallPng); + const { db, appendCalls } = makeDatabase(mediaDoc()); + new FrameworkResizer({ + storage, + db: { ...db, ...fakeLockMethods(makeLocks().lockProvider) }, + }); + await processTask( + task({ + previews: [ + variant({ + sizeKey: '100w', + requestedWidth: 100, + requestedHeight: undefined, + }), + ], + }), + ); + const meta = await sharp(uploads[0].body).metadata(); + assert.deepEqual([meta.width, meta.height], [100, 200]); + assert.equal(appendCalls[0].previews[0].actualWidth, 100); + assert.equal(appendCalls[0].previews[0].actualHeight, 200); + }); + + test('height-only on a 100×1 source: the derived width is cropped to limits.resultDimension', async () => { + installApp({ limits: { resultDimension: 200 } }); + const { storage, uploads } = makeStorage(widePng); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const { created } = await r.generate({ + media: mediaDoc(), + sizes: [{ height: 100 }], + formats: ['jpeg'], + }); + const meta = await sharp(uploads[0].body).metadata(); + assert.deepEqual([meta.width, meta.height], [200, 100]); + assert.equal(created[0].sizeKey, '100h'); + }); + + test('width-only within the cap keeps the source aspect ratio', async () => { + installApp(); + const { storage } = makeStorage(redPng); // 64×48 + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const { created } = await r.generate({ + media: mediaDoc(), + sizes: [{ width: 32 }], + formats: ['jpeg'], + }); + assert.equal(created[0].actualWidth, 32); + assert.equal(created[0].actualHeight, 24); + }); + + test('a fractional size generates through generate()', async () => { + installApp(); + const { storage } = makeStorage(bigPng); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const { created, failed } = await r.generate({ + media: mediaDoc(), + sizes: [{ width: 300.5, height: 200 }], + formats: ['jpeg'], + }); + assert.equal(failed, 0); + assert.equal(created[0].sizeKey, '301x200'); + assert.equal(created[0].actualWidth, 301); + assert.equal(created[0].actualHeight, 200); + }); + + test('a fractional size generates through the queued path', async () => { + installApp(); + const { storage } = makeStorage(bigPng); + const { db, appendCalls } = makeDatabase(mediaDoc()); + new FrameworkResizer({ + storage, + db: { ...db, ...fakeLockMethods(makeLocks().lockProvider) }, + }); + const size = { width: 300.5, height: 200 }; + await processTask( + task({ previews: [toMissingPreview(size, getSizeKey(size), 'jpeg')] }), + ); + const row = appendCalls[0].previews[0]; + assert.equal(row.sizeKey, '301x200'); + assert.equal(row.actualWidth, 301); + assert.equal(row.actualHeight, 200); + }); + + test('an old task payload with fractional dimensions is rounded before sharp', async () => { + installApp(); + const { storage } = makeStorage(bigPng); + const { db, appendCalls } = makeDatabase(mediaDoc()); + new FrameworkResizer({ + storage, + db: { ...db, ...fakeLockMethods(makeLocks().lockProvider) }, + }); + await processTask( + task({ + previews: [ + variant({ + sizeKey: '301x200', + requestedWidth: 300.5, + requestedHeight: 199.8, + }), + ], + }), + ); + const row = appendCalls[0].previews[0]; + assert.equal(row.actualWidth, 301); + assert.equal(row.actualHeight, 200); + assert.equal(row.requestedWidth, 301); + assert.equal(row.requestedHeight, 200); + }); + + test('fit on an extreme aspect ratio keeps each side at least 1 pixel', async () => { + installApp(); + // 1×5000 into the 2000×1200 box: scale 0.24 rounds the width to 0. + const needle = await sharp({ + create: { width: 1, height: 5000, channels: 3, background: '#808080' }, + }) + .png() + .toBuffer(); + const { storage } = makeStorage(needle); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const { created, failed } = await r.generate({ + media: mediaDoc(), + sizes: [{ fit: true }], + formats: ['jpeg'], + }); + assert.equal(failed, 0); + assert.equal(created[0].actualWidth, 1); + assert.equal(created[0].actualHeight, 1200); + }); +}); + +// --------------------------------------------------------------------------- +// Per-call formats must be configured encoders +// --------------------------------------------------------------------------- + +describe('unconfigured formats', () => { + test('generate() rejects formats without an encode.formats entry before any work', async () => { + installApp(); + let downloads = 0; + const base = makeStorage(alphaPng); + const storage: ResizeStorage = { + ...base.storage, + download: async () => { + downloads += 1; + return alphaPng; + }, + }; + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + await assert.rejects( + () => + r.generate({ + media: mediaDoc(), + sizes: [{ width: 20, height: 20 }], + formats: ['jpeg', 'jpg', 'raw', 'toString'], + }), + (err: unknown) => { + // A wrong per-call argument is a wiring error at the call site, not a boot-time config. + assert.ok(err instanceof ResizeSetupError); + assert.equal(err.code, 'RESIZE_FORMAT_NOT_CONFIGURED'); + assert.match(err.message, /\[jpg, raw, toString\]/); + return true; + }, + ); + assert.equal(downloads, 0); + assert.equal(base.uploads.length, 0); + }); + + test('a queued variant with an unconfigured format fails alone and is logged', async () => { + const { logs } = installApp(); + const { storage, uploads } = makeStorage(alphaPng); + const { db, appendCalls } = makeDatabase(mediaDoc()); + new FrameworkResizer({ + storage, + db: { ...db, ...fakeLockMethods(makeLocks().lockProvider) }, + }); + await assert.rejects( + () => + processTask( + task({ previews: [variant(), variant({ format: 'jpg' })] }), + ), + (err: unknown) => + err instanceof ResizeGenerateError && + err.missing.includes('default:default:20x20:jpg:none'), + ); + assert.equal(uploads.length, 1); + assert.equal(uploads[0].contentType, 'image/jpeg'); + assert.deepEqual( + appendCalls[0].previews.map((p) => p.format), + ['jpeg'], + ); + assert.ok( + logs.error.some( + (entry) => + entry[1] instanceof ResizeSetupError && + entry[1].code === 'RESIZE_FORMAT_NOT_CONFIGURED', + ), + ); + }); +}); + +// --------------------------------------------------------------------------- +// EXIF orientation: no lossy re-encode of the original +// --------------------------------------------------------------------------- + +describe('EXIF orientation quality', () => { + const sizes = [{ width: 100, height: 120 }, { fit: true }, { width: 100 }]; + + /** Generate PNG (lossless) previews and score each against a rotate-first reference. */ + async function score(source: Buffer, pipeline?: Pipeline) { + resetResizerForTests(); + resetAppInstance(); + installApp({ encode: { formats: { png: {} } } }); + const { storage, uploads } = makeStorage(source); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ + storage, + db, + ...(pipeline ? { pipelines: { default: pipeline } } : {}), + }); + const { created } = await r.generate({ + media: mediaDoc(), + sizes, + formats: ['png'], + }); + const scores = new Map(); + for (const preview of created) { + const body = uploads.find( + (u) => u.key === (preview.storageRef as { key: string }).key, + )?.body as Buffer; + const reference = await (preview.fit + ? sharp(source) + .rotate() + .resize(2000, 1200, { fit: 'inside', withoutEnlargement: true }) + .toColorspace('srgb') + : sharp(source) + .rotate() + .resize(preview.requestedWidth, preview.requestedHeight, { + fit: 'cover', + position: 'center', + }) + .toColorspace('srgb') + .sharpen() + ) + .png() + .toBuffer(); + const out = await sharp(body).metadata(); + const ref = await sharp(reference).metadata(); + assert.deepEqual([out.width, out.height], [ref.width, ref.height]); + scores.set(preview.sizeKey, { + dims: `${out.width}x${out.height}`, + psnr: await psnr(body, reference), + }); + } + return scores; + } + + test('orientation 6 without beforeSteps: same dims and pixels as rotating first, no added loss', async () => { + const rotated = await score(await texturedJpeg(6)); + const upright = await score(await texturedJpeg(1)); + assert.deepEqual( + [...rotated].map(([key, s]) => `${key}=${s.dims}`).sort(), + ['100w=100x150', '100x120=100x120', 'fit=200x300'], + ); + for (const [key, { psnr: db }] of rotated) { + assert.ok(db >= 50, `${key}: ${db.toFixed(2)} dB vs rotate-first`); + const baseline = upright.get(key)?.psnr ?? 0; + assert.ok( + db >= baseline - 0.5, + `${key}: ${db.toFixed(2)} dB vs orientation-1 ${baseline.toFixed(2)} dB`, + ); + } + }); + + test('orientation 6 with beforeSteps: normalised first, without visible loss', async () => { + let seen: { width?: number; height?: number } = {}; + const scores = await score(await texturedJpeg(6), { + beforeSteps: [ + async (buf) => { + const meta = await sharp(buf).metadata(); + seen = { width: meta.width, height: meta.height }; + return buf; + }, + ], + }); + assert.deepEqual(seen, { width: 200, height: 300 }); + assert.equal(scores.size, 3); + for (const [key, { psnr: db }] of scores) { + // A default-quality JPEG round trip of this texture scores about 35 dB. + assert.ok(db >= 45, `${key}: ${db.toFixed(2)} dB vs rotate-first`); + } + }); + + test('an oriented WebP with beforeSteps is normalised with a fast lossy encode', async () => { + // Lossless WebP of a large photo takes seconds and tens of MB; quality 95 is enough for + // an intermediate. + const source = await sharp(texture(300, 200), { + raw: { width: 300, height: 200, channels: 3 }, + }) + .webp({ quality: 95 }) + .withMetadata({ orientation: 6 }) + .toBuffer(); + let chunk = ''; + let seen: { width?: number; height?: number } = {}; + installApp({ encode: { formats: { png: {} } } }); + const { storage } = makeStorage(source); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ + storage, + db, + pipelines: { + default: { + beforeSteps: [ + async (buf) => { + // Simple-format WebP: the first chunk names the codec (VP8 lossy, VP8L lossless). + chunk = Buffer.from(buf).toString('ascii', 12, 16); + const meta = await sharp(buf).metadata(); + seen = { width: meta.width, height: meta.height }; + return buf; + }, + ], + }, + }, + }); + const { created } = await r.generate({ + media: mediaDoc(), + sizes: [{ width: 100 }], + formats: ['png'], + }); + assert.equal(chunk, 'VP8 '); + assert.deepEqual(seen, { width: 200, height: 300 }); + assert.equal(created[0].actualHeight, 150); + }); +}); + +// --------------------------------------------------------------------------- +// SVG rasterization stays within librsvg's side limit +// --------------------------------------------------------------------------- + +describe('SVG raster size', () => { + test('a 1×5000 SVG renders cover and fit variants', async () => { + installApp(); + const { storage } = makeStorage(tallSvg); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const { created, failed } = await r.generate({ + media: mediaDoc({ + original: { storageRef: { key: 'uploads/x.svg' }, format: 'svg' }, + }), + sizes: [{ width: 300, height: 300 }, { fit: true }], + formats: ['jpeg'], + }); + assert.equal(failed, 0); + const dims = Object.fromEntries( + created.map((p) => [p.sizeKey, `${p.actualWidth}x${p.actualHeight}`]), + ); + assert.deepEqual(dims, { '300x300': '300x300', fit: '1x1200' }); + }); + + test('an SVG wider than the side limit keeps its exact fit size', async () => { + // 40000 px wide at 72 dpi: the cover decode drops below 72 dpi, fit must not. + installApp(); + const wideSvg = Buffer.from( + '', + ); + const { storage } = makeStorage(wideSvg); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const { created, failed } = await r.generate({ + media: mediaDoc({ + original: { storageRef: { key: 'uploads/x.svg' }, format: 'svg' }, + }), + sizes: [{ fit: true }, { width: 300, height: 300 }, { height: 25 }], + formats: ['jpeg'], + }); + assert.equal(failed, 0); + const dims = Object.fromEntries( + created.map((p) => [p.sizeKey, `${p.actualWidth}x${p.actualHeight}`]), + ); + // 25 × 40000 / 300 = 3333.3: exact although the pixel limit allows only a coarse size read. + assert.deepEqual(dims, { + fit: '2000x15', + '300x300': '300x300', + '25h': '3333x25', + }); + }); + + test('a 1×40000 SVG renders its fit variant (no 72 dpi decode over the side limit)', async () => { + installApp(); + const needleSvg = Buffer.from( + '', + ); + const { storage } = makeStorage(needleSvg); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const { created, failed } = await r.generate({ + media: mediaDoc({ + original: { storageRef: { key: 'uploads/x.svg' }, format: 'svg' }, + }), + sizes: [{ fit: true }, { width: 300, height: 300 }], + formats: ['jpeg'], + }); + assert.equal(failed, 0); + const dims = Object.fromEntries( + created.map((p) => [p.sizeKey, `${p.actualWidth}x${p.actualHeight}`]), + ); + assert.deepEqual(dims, { fit: '1x1200', '300x300': '300x300' }); + }); +}); + +// --------------------------------------------------------------------------- +// A failed persist names what was uploaded +// --------------------------------------------------------------------------- + +describe('persist failure', () => { + test('logs the uploaded but unrecorded storage refs, then rethrows', async () => { + const { logs } = installApp(); + const { storage, uploads } = makeStorage(redPng); + const dbDown = new Error('db down'); + const db = fakeDb({ + appendPreviews: async () => { + throw dbDown; + }, + }); + const r = new FrameworkResizer({ storage, db }); + await assert.rejects( + () => + r.generate({ + media: mediaDoc(), + sizes: [{ width: 20, height: 20 }], + formats: ['jpeg', 'webp'], + }), + (err: unknown) => err === dbDown, + ); + assert.equal(uploads.length, 2); + const entry = logs.error.find((l) => l[1] === dbDown); + assert.ok(entry, 'the persist failure is logged'); + for (const upload of uploads) { + assert.ok( + String(entry[0]).includes(upload.key), + `${upload.key} is named in the log`, + ); + } + }); + + /** A database that stores the first preview, then fails (a write of one row at a time). */ + function failsAfterFirst(reload: () => Promise) { + const media = mediaDoc(); + const dbDown = new Error('db down after one row'); + const db: ResizeDatabase = { + ...fakeDb(), + loadMedia: reload, + appendPreviews: async (_mediaId, previews) => { + media.previews = [previews[0]]; + throw dbDown; + }, + }; + return { db, media, dbDown }; + } + + test('a write that stopped part-way names only the uploads that are not stored', async () => { + const { logs } = installApp(); + const { storage, uploads } = makeStorage(redPng); + const state: { media?: MediaLike } = {}; + const { db, media, dbDown } = failsAfterFirst( + async () => state.media ?? null, + ); + state.media = media; + const r = new FrameworkResizer({ storage, db }); + await assert.rejects( + () => + r.generate({ + media: mediaDoc(), + sizes: [{ width: 20, height: 20 }], + formats: ['jpeg', 'webp'], + }), + (err: unknown) => err === dbDown, + ); + const entry = logs.error.find((l) => l[1] === dbDown); + assert.ok(entry); + const storedRef = media.previews?.[0]?.storageRef as + | { key: string } + | undefined; + assert.ok(storedRef, 'the first row was stored'); + const storedKey = storedRef.key; + const unstored = uploads.filter((u) => u.key !== storedKey); + assert.equal(unstored.length, 1); + assert.ok(String(entry[0]).includes(unstored[0].key), 'unstored is named'); + assert.ok( + !String(entry[0]).includes(storedKey), + 'a stored row is never offered for deletion', + ); + }); + + test('a failing reload after a failed write names every upload', async () => { + const { logs } = installApp(); + const { storage, uploads } = makeStorage(redPng); + const { db, dbDown } = failsAfterFirst(async () => { + throw new Error('reload failed too'); + }); + const r = new FrameworkResizer({ storage, db }); + await assert.rejects( + () => + r.generate({ + media: mediaDoc(), + sizes: [{ width: 20, height: 20 }], + formats: ['jpeg', 'webp'], + }), + (err: unknown) => err === dbDown, + ); + const entry = logs.error.find((l) => l[1] === dbDown); + assert.ok(entry); + for (const upload of uploads) { + assert.ok(String(entry[0]).includes(upload.key), upload.key); + } + }); +}); + +// --------------------------------------------------------------------------- +// SVG: rendered once per task, in a child process with a hard time limit +// --------------------------------------------------------------------------- + +describe('SVG rendering', () => { + const svgDoc = () => + mediaDoc({ + original: { storageRef: { key: 'uploads/x.svg' }, format: 'svg' }, + }); + + /** True while a process with `pid` exists (signal 0 only checks). */ + function isRunning(pid: number | undefined): boolean { + if (pid === undefined) { + return false; + } + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + } + + test('eager: one render however many sizes and formats are requested', async () => { + installApp(); + const renders = countRenders(); + const { storage, uploads } = makeStorage(smallSvg); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const { created, failed } = await r.generate({ + media: svgDoc(), + sizes: [{ width: 200, height: 200 }, { width: 50 }, { fit: true }], + formats: ['jpeg', 'webp', 'avif'], + }); + assert.equal(failed, 0); + assert.equal(created.length, 9); + assert.equal(uploads.length, 9); + assert.equal(renders(), 1); + }); + + test('queued: one render per task', async () => { + installApp(); + const renders = countRenders(); + const { storage, uploads } = makeStorage(smallSvg); + const { db, appendCalls } = makeDatabase(svgDoc()); + new FrameworkResizer({ + storage, + db: { ...db, ...fakeLockMethods(makeLocks().lockProvider) }, + }); + await processTask( + task({ + previews: [ + variant(), + variant({ format: 'webp' }), + variant({ + sizeKey: '40w', + requestedWidth: 40, + requestedHeight: undefined, + }), + fitVariant, + ], + }), + ); + assert.equal(uploads.length, 4); + assert.equal(appendCalls[0].previews.length, 4); + assert.equal(renders(), 1); + }); + + test('eager: a render over limits.processingTimeoutSeconds is killed (RESIZE_SVG_RENDER_TIMEOUT)', { + timeout: 8000, + }, async () => { + installApp({ limits: { processingTimeoutSeconds: 1 } }); + const spawn = mock.method(childProcess, 'spawn'); + const { storage, uploads } = makeStorage(heavySvg); + const { db, appendCalls } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const started = Date.now(); + await assert.rejects( + () => + r.generate({ + media: svgDoc(), + sizes: [{ width: 300, height: 300 }, { fit: true }], + formats: ['jpeg', 'webp'], + }), + (err: unknown) => + err instanceof ResizeMediaError && + err.code === 'RESIZE_SVG_RENDER_TIMEOUT' && + err.mediaId === 'm1', + ); + const elapsed = Date.now() - started; + assert.ok(elapsed < 3500, `settled after ${elapsed} ms`); + assert.equal(spawn.mock.callCount(), 1); + assert.equal( + isRunning((spawn.mock.calls[0].result as { pid?: number }).pid), + false, + ); + assert.equal(uploads.length, 0); + assert.equal(appendCalls.length, 0); + }); + + test('queued: a render over the time limit fails the task (RESIZE_SVG_RENDER_TIMEOUT); nothing is stored', { + timeout: 8000, + }, async () => { + installApp({ limits: { processingTimeoutSeconds: 1 } }); + const { storage, uploads } = makeStorage(heavySvg); + const { db, appendCalls } = makeDatabase(svgDoc()); + const { lockProvider, acquired } = makeLocks(true); + new FrameworkResizer({ + storage, + db: { ...db, ...fakeLockMethods(lockProvider) }, + }); + await assert.rejects( + () => processTask(task({ previews: [variant(), fitVariant] })), + (err: unknown) => + err instanceof ResizeMediaError && + err.code === 'RESIZE_SVG_RENDER_TIMEOUT', + ); + assert.equal(uploads.length, 0); + assert.equal(appendCalls.length, 0); + assert.deepEqual(acquired, []); + }); + + /** Red, green and blue of the pixel at x, y. */ + async function rgbAt(body: Buffer, x: number, y: number) { + return [ + ...(await sharp(body) + .removeAlpha() + .extract({ left: x, top: y, width: 1, height: 1 }) + .raw() + .toBuffer()), + ]; + } + const isRed = ([r, g, b]: number[]) => r > 200 && g < 60 && b < 60; + const isBlue = ([r, g, b]: number[]) => b > 200 && r < 60 && g < 60; + + test('beforeSteps receive the rendered PNG: one render, and a large size stays sharp', async () => { + installApp({ encode: { formats: { png: {} } } }); + const renders = countRenders(); + // Left half red, right half blue. A 2000 px preview upscaled from the 20×10 natural size + // would blur the edge over about 100 px. + const { storage, uploads } = makeStorage( + Buffer.from( + '', + ), + ); + const { db } = makeDatabase(null); + const seen: Array<{ head: string; format?: string; width?: number }> = []; + const r = new FrameworkResizer({ + storage, + db, + pipelines: { + default: { + beforeSteps: [ + async (buf, { metadata }) => { + seen.push({ + head: Buffer.from(buf).subarray(0, 8).toString('latin1'), + format: metadata.format, + width: metadata.width, + }); + // A step that decodes: it must get pixels, not SVG markup to render in-process. + return sharp(buf).toBuffer(); + }, + ], + }, + }, + }); + const { created, failed } = await r.generate({ + media: svgDoc(), + sizes: [{ width: 2000 }], + formats: ['png'], + }); + assert.equal(failed, 0); + assert.equal(renders(), 1); + assert.equal(seen.length, 1); + assert.equal(seen[0].head, '\x89PNG\r\n\x1a\n'); + assert.equal(seen[0].format, 'png'); + assert.equal(seen[0].width, 2000); + assert.deepEqual( + [created[0].actualWidth, created[0].actualHeight], + [2000, 1000], + ); + assert.ok(isRed(await rgbAt(uploads[0].body, 990, 500)), 'sharp edge'); + assert.ok(isBlue(await rgbAt(uploads[0].body, 1010, 500)), 'sharp edge'); + }); + + test('a derived side over the cap is cropped, even when the SVG size rounds it under', async () => { + // 1000×0.6 reports as 1000×1: at height 5 the real width is 8333, over the cap of 5000, so + // the preview is a 5000×5 crop of the middle, not the whole SVG squeezed into 5000×5. + installApp({ encode: { formats: { png: {} } } }); + const { storage, uploads } = makeStorage( + Buffer.from( + '', + ), + ); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const { created, failed } = await r.generate({ + media: svgDoc(), + sizes: [{ height: 5 }], + formats: ['png'], + }); + assert.equal(failed, 0); + assert.deepEqual( + [created[0].actualWidth, created[0].actualHeight], + [5000, 5], + ); + // Cropped: x = 1200 shows SVG x ≈ 344 (blue). Squeezed it would show x = 240 (red). + assert.ok(isBlue(await rgbAt(uploads[0].body, 1200, 2))); + }); + + test('a derived side follows the SVG, not the rounded raster', async () => { + // 620w alone renders at density 223: the raster is 619×310 (619.4×309.7), from which a + // width-only resize would derive 310.5 → 311. + installApp(); + const { storage } = makeStorage( + Buffer.from( + '', + ), + ); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const { created, failed } = await r.generate({ + media: svgDoc(), + sizes: [{ width: 620 }, { height: 25 }], + formats: ['jpeg'], + }); + assert.equal(failed, 0); + const dims = Object.fromEntries( + created.map((p) => [p.sizeKey, `${p.actualWidth}x${p.actualHeight}`]), + ); + assert.deepEqual(dims, { '620w': '620x310', '25h': '50x25' }); + }); + + test('a normal SVG keeps its output sizes for cover, fit, width-only and height-only sizes', async () => { + // A derived side (width-only, height-only) is the nearest whole pixel to the SVG's real + // aspect ratio, whatever density the shared raster was rendered at. The per-variant + // re-render this replaces usually agreed; where it was off by one, the comment says so. + const cases: Array<[string, Record]> = [ + [ + 'width="200" height="100"', + { + fit: '200x100', + // 620 × 100 / 200 = 310 exactly (the raster is 619×310 at this density). + '620w': '620x310', + '100w': '100x50', + '400h': '800x400', + '25h': '50x25', + '300x300': '300x300', + }, + ], + [ + 'width="300" height="200"', + { + fit: '300x200', + '620w': '620x413', + '100w': '100x67', + '400h': '600x400', + '25h': '38x25', + '300x300': '300x300', + }, + ], + [ + 'width="8" height="8"', + { + fit: '8x8', + '620w': '620x620', + '100w': '100x100', + '400h': '400x400', + '25h': '25x25', + '300x300': '300x300', + }, + ], + [ + 'width="1000" height="10"', + { + fit: '1000x10', + '620w': '620x6', + '100w': '100x1', + '400h': '5000x400', + '25h': '2500x25', // 25 × 1000 / 10; the re-render gave 2497 + + '300x300': '300x300', + }, + ], + [ + 'width="37" height="91"', + { + fit: '37x91', + '620w': '620x1525', // 620 × 91 / 37 = 1524.86; the re-render gave 1524 + + '100w': '100x246', + '400h': '163x400', + '25h': '10x25', + '300x300': '300x300', + }, + ], + [ + 'width="333.3" height="77.7"', + { + fit: '333x78', + '620w': '620x145', // 620 × 77.7 / 333.3 = 144.54 + '100w': '100x23', + '400h': '1716x400', // 400 × 333.3 / 77.7 = 1715.83; the re-render gave 1717 + '25h': '107x25', + '300x300': '300x300', + }, + ], + [ + 'viewBox="0 0 100 50"', + { + fit: '100x50', + '620w': '620x310', + '100w': '100x50', + '400h': '800x400', + '25h': '50x25', + '300x300': '300x300', + }, + ], + [ + 'width="3000" height="2000"', + { + fit: '1800x1200', + '620w': '620x413', + '100w': '100x67', + '400h': '600x400', + '25h': '38x25', + '300x300': '300x300', + }, + ], + ]; + for (const [attributes, expected] of cases) { + resetResizerForTests(); + resetAppInstance(); + installApp(); + const { storage, uploads } = makeStorage( + Buffer.from( + ``, + ), + ); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const { created, failed } = await r.generate({ + media: svgDoc(), + sizes: [ + { fit: true }, + { width: 620 }, + { width: 100 }, + { height: 400 }, + { height: 25 }, + { width: 300, height: 300 }, + ], + formats: ['jpeg'], + }); + assert.equal(failed, 0, attributes); + const dims = Object.fromEntries( + created.map((p) => [p.sizeKey, `${p.actualWidth}x${p.actualHeight}`]), + ); + assert.deepEqual(dims, expected, attributes); + for (const upload of uploads) { + assert.equal((await sharp(upload.body).metadata()).format, 'jpeg'); + } + } + }); +}); + +// --------------------------------------------------------------------------- +// One preview row per identity: the database reports what it stored +// --------------------------------------------------------------------------- + +describe('one preview row per identity', () => { + /** + * A database that stores only the previews `keep` accepts, as if another worker won the rest. + * `load` answers every loadMedia (the worker's first read, then its reloads). + */ + function partialDb( + load: () => MediaLike | null, + keep: (p: Preview) => boolean, + ) { + const appended: Preview[][] = []; + const db: ResizeDatabase = { + ...fakeDb({ load: async () => load() }), + appendPreviews: async (_mediaId, previews) => { + appended.push(previews); + return previews.filter(keep); + }, + }; + return { db, appended }; + } + + // The webp row another worker stored first. + const otherWorkersWebp: Preview = { + storageRef: { bucket: 'previews', key: 'other-worker.webp' }, + identity: 'default:default:20x20:webp:none', + sizeKey: '20x20', + format: 'webp', + contentType: 'image/webp', + }; + + /** The media without previews on the first read, with the other worker's row afterwards. */ + function racedMedia() { + let loads = 0; + return () => { + loads += 1; + return mediaDoc({ previews: loads === 1 ? [] : [otherWorkersWebp] }); + }; + } + + test('every generated preview carries its identity', async () => { + installApp(); + const { storage } = makeStorage(redPng); + const { db, appendCalls } = makeDatabase(mediaDoc()); + new FrameworkResizer({ + storage, + db: { ...db, ...fakeLockMethods(makeLocks().lockProvider) }, + pipelines: { watermark: {} }, + }); + await processTask( + task({ + pipeline: 'watermark', + previews: [variant({ filters: { blur: 3 } }), fitVariant], + }), + ); + assert.deepEqual(appendCalls[0].previews.map((p) => p.identity).sort(), [ + 'default:watermark:20x20:jpeg:blur:3', + 'default:watermark:fit:jpeg:none', + ]); + + resetResizerForTests(); + const r = new FrameworkResizer({ storage, db, name: 'listings' }); + const { created } = await r.generate({ + media: mediaDoc(), + sizes: [{ width: 20, height: 20 }], + formats: ['webp'], + }); + assert.equal(created[0].identity, 'listings:default:20x20:webp:none'); + }); + + test('eager: rows the database left out are not created, not appended and logged once', async () => { + const { logs } = installApp(); + const { storage, uploads } = makeStorage(redPng); + // Eager generate never loads first: every read is the reload after the write. + const { db } = partialDb( + () => mediaDoc({ previews: [otherWorkersWebp] }), + (p) => p.format === 'jpeg', + ); + const fired: Preview[] = []; + const r = new FrameworkResizer({ + storage, + db, + hooks: { + onPreviewGenerated: (preview: unknown) => { + fired.push(preview as Preview); + }, + }, + }); + const media = mediaDoc(); + const { created, failed } = await r.generate({ + media, + sizes: [{ width: 20, height: 20 }], + formats: ['jpeg', 'webp'], + }); + assert.equal(failed, 0); + assert.deepEqual( + created.map((p) => p.format), + ['jpeg'], + ); + assert.deepEqual( + media.previews?.map((p) => p.format), + ['jpeg'], + ); + assert.deepEqual( + fired.map((p) => p.format), + ['jpeg'], + ); + const webpKey = uploads.find((u) => u.contentType === 'image/webp')?.key; + assert.ok(webpKey); + const mentions = [...logs.warn, ...logs.error, ...logs.info].filter((l) => + String(l[0]).includes(webpKey), + ); + assert.equal(mentions.length, 1, 'the unrecorded upload is logged once'); + assert.match(String(mentions[0][0]), /another worker/); + assert.equal(logs.error.length, 0); + }); + + test('queued: an identity another worker stored first counts as covered', async () => { + installApp(); + const { storage } = makeStorage(redPng); + const { db, appended } = partialDb( + racedMedia(), + (p) => p.format === 'jpeg', + ); + new FrameworkResizer({ + storage, + db: { ...db, ...fakeLockMethods(makeLocks().lockProvider) }, + }); + await processTask( + task({ previews: [variant(), variant({ format: 'webp' })] }), + ); + assert.equal(appended.length, 1); + assert.equal(appended[0].length, 2); + }); + + test('queued: a row left out that the reload does not show is not stored: the task is incomplete', async () => { + const { logs } = installApp(); + const { storage, uploads } = makeStorage(redPng); + const { db } = partialDb( + () => mediaDoc(), + (p) => p.format === 'jpeg', + ); + new FrameworkResizer({ + storage, + db: { ...db, ...fakeLockMethods(makeLocks().lockProvider) }, + }); + await assert.rejects( + () => + processTask( + task({ previews: [variant(), variant({ format: 'webp' })] }), + ), + (err: unknown) => + err instanceof ResizeGenerateError && + err.code === 'RESIZE_WORKER_INCOMPLETE' && + err.missing.includes('default:default:20x20:webp:none'), + ); + const webpKey = uploads.find((u) => u.contentType === 'image/webp')?.key; + assert.ok(webpKey); + const entry = logs.error.find((l) => String(l[0]).includes(webpKey)); + assert.ok(entry, 'the unrecorded upload is named'); + assert.doesNotMatch(String(entry[0]), /another worker/); + }); + + test('rows left out because the media no longer exists are reported as such', async () => { + const { logs } = installApp(); + const { storage, uploads } = makeStorage(redPng); + // A driver resolves with an empty list when the media document is gone. + const { db } = partialDb( + () => null, + () => false, + ); + const r = new FrameworkResizer({ storage, db }); + const { created, failed } = await r.generate({ + media: mediaDoc(), + sizes: [{ width: 20, height: 20 }], + formats: ['jpeg', 'webp'], + }); + assert.equal(failed, 0); + assert.deepEqual(created, []); + const entries = logs.warn.filter((l) => + /no longer exists/.test(String(l[0])), + ); + assert.equal(entries.length, 1); + for (const upload of uploads) { + assert.ok(String(entries[0][0]).includes(upload.key), upload.key); + } + assert.ok( + ![...logs.warn, ...logs.error].some((l) => + /another worker/.test(String(l[0])), + ), + ); + }); + + test('a database that resolves with nothing stored every row', async () => { + installApp(); + const { storage } = makeStorage(redPng); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const media = mediaDoc(); + const { created } = await r.generate({ + media, + sizes: [{ width: 20, height: 20 }], + formats: ['jpeg', 'webp'], + }); + assert.equal(created.length, 2); + assert.equal(media.previews?.length, 2); + }); +}); + +// --------------------------------------------------------------------------- +// A source no larger than a WxH box: a cleaned preview at its own size +// --------------------------------------------------------------------------- + +describe('a source smaller than the box', () => { + test('a 100×80 JPEG with EXIF and GPS at 300×300 gives 100×80 in every format, without EXIF', async () => { + installApp(); + assert.ok((await sharp(exifJpeg).metadata()).exif, 'fixture has EXIF'); + const { storage, uploads } = makeStorage(exifJpeg); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const { created, failed } = await r.generate({ + media: mediaDoc(), + sizes: [{ width: 300, height: 300 }], + formats: ['jpeg', 'webp', 'avif'], + }); + assert.equal(failed, 0); + assert.equal(created.length, 3); + for (const preview of created) { + assert.equal(preview.sizeKey, '300x300'); + assert.equal(preview.actualWidth, 100, preview.format); + assert.equal(preview.actualHeight, 80, preview.format); + } + for (const upload of uploads) { + const meta = await sharp(upload.body).metadata(); + assert.deepEqual([meta.width, meta.height], [100, 80], upload.key); + assert.equal(meta.exif, undefined, `${upload.key} has no EXIF`); + } + }); + + test('a pipeline with variantSteps keeps the full box its steps were written for', async () => { + // A 200×50 watermark composited onto a 150×150 own-size image would fail every time. + installApp(); + const mark = await sharp({ + create: { width: 200, height: 50, channels: 3, background: '#ffffff' }, + }) + .png() + .toBuffer(); + const small = await sharp(texture(150, 150), { + raw: { width: 150, height: 150, channels: 3 }, + }) + .jpeg() + .withExif({ IFD0: { Artist: 'Someone' } }) + .toBuffer(); + const { storage, uploads } = makeStorage(small); + const { db } = makeDatabase(null); + let steps = 0; + const r = new FrameworkResizer({ + storage, + db, + pipelines: { + default: { + variantSteps: [ + async (img) => { + steps += 1; + return img.composite([{ input: mark, gravity: 'southeast' }]); + }, + ], + }, + }, + }); + const { created, failed } = await r.generate({ + media: mediaDoc(), + sizes: [{ width: 300, height: 300 }], + formats: ['jpeg', 'webp'], + }); + assert.equal(failed, 0); + assert.equal(steps, 2); + for (const preview of created) { + assert.deepEqual([preview.actualWidth, preview.actualHeight], [300, 300]); + } + for (const upload of uploads) { + const meta = await sharp(upload.body).metadata(); + assert.deepEqual([meta.width, meta.height], [300, 300]); + assert.equal(meta.exif, undefined); + const corner = await sharp(upload.body) + .extract({ left: 290, top: 290, width: 1, height: 1 }) + .raw() + .toBuffer(); + assert.ok( + corner[0] > 200 && corner[1] > 200 && corner[2] > 200, + 'overlay', + ); + } + }); + + test('fitting is decided against the requested box, then the cap still applies', async () => { + // 120×60 fits the requested 200×200 box; the cap of 100 then scales it to 100×50 instead of + // cropping it to the capped 100×100. + installApp({ limits: { resultDimension: 100 } }); + const source = await sharp({ + create: { width: 120, height: 60, channels: 3, background: '#808080' }, + }) + .png() + .toBuffer(); + const { storage } = makeStorage(source); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const { created } = await r.generate({ + media: mediaDoc(), + sizes: [{ width: 200, height: 200 }], + formats: ['jpeg'], + }); + assert.deepEqual( + [created[0].actualWidth, created[0].actualHeight], + [100, 50], + ); + }); + + test('an image that is not scaled is not sharpened either', async () => { + installApp({ encode: { formats: { png: {} } } }); + const { storage, uploads } = makeStorage(exifJpeg); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + await r.generate({ + media: mediaDoc(), + sizes: [{ width: 300, height: 300 }], + formats: ['png'], + }); + // Lossless output of an unscaled, unsharpened image: the decoded source pixels. + assert.equal( + await psnr(uploads[0].body, exifJpeg), + Number.POSITIVE_INFINITY, + ); + }); + + test('a source larger than the box in one direction is still cover-cropped', async () => { + installApp(); + const wide = await sharp({ + create: { width: 1000, height: 200, channels: 3, background: '#808080' }, + }) + .png() + .toBuffer(); + const { storage } = makeStorage(wide); + const { db } = makeDatabase(null); + const r = new FrameworkResizer({ storage, db }); + const { created } = await r.generate({ + media: mediaDoc(), + sizes: [{ width: 300, height: 300 }], + formats: ['jpeg'], + }); + assert.deepEqual( + [created[0].actualWidth, created[0].actualHeight], + [300, 300], + ); + }); +}); + +// --------------------------------------------------------------------------- +// runResizeWorker (07 · §11) +// --------------------------------------------------------------------------- + +// Stop idle test queues deterministically after their queued work has been handled. +function observedQueue(onIdle: () => void = () => process.emit('SIGTERM')) { + const tasks = new MemoryTaskQueue({ timing: { idlePollMs: 1 } }); + const claimedQueues: string[] = []; + const claim = tasks.claim.bind(tasks); + tasks.claim = async (queue, leaseMs) => { + claimedQueues.push(queue); + const next = await claim(queue, leaseMs); + if (!next) { + onIdle(); + } + return next; + }; + return { tasks, claimedQueues }; +} + +async function addTask(tasks: MemoryTaskQueue, over: Partial = {}) { + const { taskId: _taskId, ...payload } = task(); + await tasks.add({ ...payload, requestKey: JSON.stringify(over), ...over }); +} + +describe('runResizeWorker', () => { + test('worker.enabled=false → clean no-op (claim NOT called); log says how to enable', async () => { + const { logs } = installApp(); + const { tasks, claimedQueues } = observedQueue(); + new FrameworkResizer({ storage: makeStorage(redPng).storage, tasks }); + await runResizeWorker(); + assert.deepEqual(claimedQueues, []); + assert.ok( + logs.info.some((l) => String(l[0]).includes('worker.enabled=true')), + ); + }); + + test('no task queue → logs an error and returns without preparing framework drivers', async () => { + const { logs, getModelCalls } = installApp({ worker: { enabled: true } }); + new FrameworkResizer({ storage: makeStorage(redPng).storage }); + await runResizeWorker(); + assert.ok(logs.error.length >= 1); + assert.equal(getModelCalls(), 0); + }); + + test('a worker with no Resizers fails before leasing', async () => { + installApp({ worker: { enabled: true } }); + await assert.rejects( + () => runResizeWorker(), + (err: unknown) => + err instanceof ResizeSetupError && err.code === 'RESIZE_NO_RESIZER', + ); + }); + + test('default database + unregistered mediaModelName → throws before claiming', async () => { + setAppInstance({ + getConfig: () => + makeResizeConfig({ + mediaModelName: 'Media', + worker: { enabled: true }, + }), + getModel: () => false, + logger: { info() {}, warn() {}, error() {} }, + } as never); + const { tasks, claimedQueues } = observedQueue(); + new FrameworkResizer({ storage: makeStorage(redPng).storage, tasks }); + await assert.rejects( + () => runResizeWorker(), + (err: unknown) => + err instanceof ResizeConfigError && + err.code === 'RESIZE_CONFIG_MEDIA_MODEL_UNKNOWN', + ); + assert.deepEqual(claimedQueues, []); + }); + + test('custom database verify() runs before claim, and its failure stops the worker', async () => { + installApp({ worker: { enabled: true } }); + const events: string[] = []; + const { tasks } = observedQueue(() => { + events.push('claim'); + process.emit('SIGTERM'); + }); + new FrameworkResizer({ + storage: makeStorage(redPng).storage, + tasks, + db: fakeDb({ + // async: a rejected promise stops the worker only if verify() is awaited + async verify() { + events.push('verify'); + throw new ResizeConfigError('custom database is misconfigured', { + code: 'CUSTOM_STORE_INVALID', + }); + }, + }), + }); + await assert.rejects( + () => runResizeWorker(), + (err: unknown) => + err instanceof ResizeConfigError && err.code === 'CUSTOM_STORE_INVALID', + ); + assert.deepEqual(events, ['verify']); + resetResizerForTests(); + events.length = 0; + new FrameworkResizer({ + storage: makeStorage(redPng).storage, + tasks, + db: fakeDb({ + async verify() { + events.push('verify'); + }, + }), + }); + await runResizeWorker(); + assert.deepEqual(events, ['verify', 'claim']); + }); + + test('enabled + task queue → a claimed task reaches processTask', async () => { + installApp({ worker: { enabled: true } }); + const { tasks } = observedQueue(); + let loads = 0; + new FrameworkResizer({ + storage: makeStorage(redPng).storage, + tasks, + db: fakeDb({ + load: async () => { + loads++; + return null; + }, + }), + }); + await addTask(tasks, { previews: [variant()] }); + await runResizeWorker(); + assert.equal(loads, 1); + assert.equal(tasks.rows[0].status, 'completed'); + }); +}); + +describe('one worker serves every Resizer', () => { + function register(tasks: MemoryTaskQueue, name = 'default', db = fakeDb()) { + return new FrameworkResizer({ + name, + storage: makeStorage(redPng).storage, + tasks, + db, + }); + } + + test('one task queue serves both Resizers, and each task runs with its own Resizer', async () => { + installApp({ worker: { enabled: true } }); + const { tasks, claimedQueues } = observedQueue(); + const loadedBy: string[] = []; + const labelled = (label: string) => + fakeDb({ + load: async () => { + loadedBy.push(label); + return null; + }, + }); + register(tasks, 'default', labelled('default')); + register(tasks, 'listings', labelled('listings')); + await addTask(tasks, { resizer: 'listings' }); + await addTask(tasks); + await runResizeWorker(); + assert.deepEqual(loadedBy, ['listings', 'default']); + assert.deepEqual(claimedQueues, ['default', 'default', 'default']); + assert.ok(tasks.rows.every((row) => row.status === 'completed')); + }); + + test('runResizeWorker({ queue }) consumes that queue only', async () => { installApp({ worker: { enabled: true } }); const { tasks, claimedQueues } = observedQueue(); register(tasks); @@ -2042,6 +3737,7 @@ describe('scoped generation', () => { name: 'listings', storage, db, + pipelines: { watermark: {} }, }); const watermarked = await listings.generate({ media: mediaDoc(), @@ -2057,7 +3753,11 @@ describe('scoped generation', () => { installApp(); const { storage, uploads } = makeStorage(redPng); const { db } = makeDatabase(null); - const r = new FrameworkResizer({ storage, db }); + const r = new FrameworkResizer({ + storage, + db, + pipelines: { watermark: {} }, + }); const media = mediaDoc({ previews: [cleanPreview] }); const same = await r.generate({ media, @@ -2084,6 +3784,7 @@ describe('scoped generation', () => { new FrameworkResizer({ storage, db: { ...db, ...fakeLockMethods(lockProvider) }, + pipelines: { watermark: {} }, }); await processTask(task({ pipeline: 'watermark', previews: [variant()] })); assert.ok(acquired.some((key) => key.includes(':watermark:'))); @@ -2100,6 +3801,7 @@ describe('scoped generation', () => { new FrameworkResizer({ storage, db: { ...db, ...fakeLockMethods(lockProvider) }, + pipelines: { watermark: {} }, }); await assert.rejects( () => processTask(task({ pipeline: 'watermark', previews: [variant()] })), diff --git a/src/resizeTask.ts b/src/resizeTask.ts index 91144ca..a8a6148 100644 --- a/src/resizeTask.ts +++ b/src/resizeTask.ts @@ -2,26 +2,35 @@ // both generation modes: // - processTaskWith() = the core + lock bookkeeping (queued/lazy worker) // - generateImpl() = the core WITHOUT locks (eager `resizer.generate`) -// Steps 2–8 (download once → metadata guards + orientation normalize → beforeSteps once → -// decode once + bounded per-variant resize/encode/upload → one appendPreviews) live in -// generatePreviews(). Every function receives its Resizer as an argument, and resizer.ts is -// imported for types only, so the resizer↔resizeTask cycle is runtime-free. sharp is a hard -// dep; this is the only place besides worker.ts that decodes. -import sharp, { type FormatEnum, type OutputOptions } from 'sharp'; +// Steps 2–8 (download once → metadata guards → beforeSteps once, after an orientation +// normalize → an SVG rendered once to a raster → decode once + bounded per-variant +// resize/encode/upload → one appendPreviews) live in generatePreviews(). Every function receives +// its Resizer as an argument, and resizer.ts is imported for types only, so the +// resizer↔resizeTask cycle is runtime-free. sharp is a hard dep; this is the only place besides +// worker.ts that decodes (SVG rendering runs in a child process: helpers/svgRaster.ts). +import sharp, { + type FormatEnum, + type Metadata, + type OutputOptions, + type Sharp, +} from 'sharp'; import { defaultQueueOptions } from './config/resize.ts'; import type { TaskQueue } from './contracts/taskQueue.ts'; -import { canonicalizeVariants } from './enqueue.ts'; +import { canonicalizeVariants, dispatchLockKey } from './enqueue.ts'; import { ResizeGenerateError, ResizeMediaError, ResizeNoOriginalError, + ResizeSetupError, ResizeStorageError, } from './errors.ts'; import { runBounded } from './helpers/concurrency.ts'; -import { isAvifBuffer } from './helpers/imageFormat.ts'; +import { isAnimatedFormat, isAvifBuffer } from './helpers/imageFormat.ts'; import { randomHex } from './helpers/random.ts'; +import { rasterizeSvg } from './helpers/svgRaster.ts'; import { calculateResizedDimensions, + coverDimensions, expandMissingPreviews, getPreviewIdentity, isUsablePreview, @@ -47,6 +56,138 @@ import type { const asBuffer = (b: Buffer | Uint8Array): Buffer => Buffer.isBuffer(b) ? b : Buffer.from(b); +// librsvg refuses to render an SVG whose raster side is larger than this. +const SVG_MAX_SIDE = 32767; + +/** + * An unregistered pipeline must not render: its previews would be stored under that pipeline's + * identity without its steps, and `resolve` would serve them for good (a renamed watermark + * pipeline during a rolling deploy would lose its watermark). `default` always exists. + */ +function requirePipeline(resizer: Resizer, name: string): void { + if (!resizer.hasPipeline(name)) { + throw new ResizeSetupError( + `resize: pipeline '${name}' is not registered on Resizer '${resizer.name}' — register it in every process that generates previews (API and worker) before using it`, + { code: 'RESIZE_PIPELINE_UNKNOWN' }, + ); + } +} + +/** + * Formats that are not an own key of `encode.formats`: an alias such as 'jpg' would skip the + * JPEG options and the flatten step, and an arbitrary Sharp id such as 'raw' would publish a + * pixel dump. + */ +function unconfiguredFormats( + encoders: Record, + formats: readonly string[], +): string[] { + return [...new Set(formats)].filter( + (format) => !Object.hasOwn(encoders, format), + ); +} + +/** A per-call argument names an unconfigured format: a wiring error at the call site. */ +function formatNotConfiguredError(formats: string[]): ResizeSetupError { + return new ResizeSetupError( + `resize: formats [${formats.join(', ')}] have no encode.formats entry — request configured Sharp format ids such as 'jpeg', not aliases such as 'jpg'`, + { code: 'RESIZE_FORMAT_NOT_CONFIGURED' }, + ); +} + +/** + * Encode an orientation-normalized original in its own format without visible loss. Sharp's + * defaults are lossy for some formats (JPEG quality 80 with 4:2:0, TIFF with JPEG compression), + * and every variant is made from this buffer. The buffer is an in-memory intermediate, so encode + * time matters more than size: WebP uses quality 95 at the lowest effort (lossless WebP of a + * 49 MP photo took 16.5 s and 63 MB on one thread, close to the processing timeout), and the + * lossless AVIF path uses the lowest effort too. + */ +function withoutVisibleLoss(img: Sharp, format: string | undefined): Sharp { + switch (format) { + case 'jpeg': + return img.jpeg({ quality: 100, chromaSubsampling: '4:4:4' }); + case 'webp': + return img.webp({ quality: 95, effort: 0 }); + case 'heif': // AVIF reports its HEIF container + return img.heif({ compression: 'av1', lossless: true, effort: 0 }); + case 'tiff': + return img.tiff({ compression: 'lzw' }); + default: + return img; // PNG and GIF are lossless at Sharp's defaults + } +} + +/** + * An SVG's own size to a fraction of a pixel. Its 72 dpi size (roundedW × roundedH) and any + * raster are rounded to whole pixels, so a side derived from them can be a pixel off (200×100 + * rendered at 619×310 gives 620×311 for a 620 width), and a 1000×0.6 SVG reads as 1000×1. This + * is a header-only read (nothing is rendered) at the highest density whose size still fits + * limitInputPixels, which Sharp checks on the header too; the +2 margins absorb its rounding. + */ +async function svgNaturalSize( + svg: Buffer, + roundedW: number, + roundedH: number, + config: Resizer['config'], +): Promise<{ width: number; height: number }> { + const density = Math.max( + 72, + Math.min( + 100_000, + Math.floor( + 72 * + Math.sqrt( + config.limits.inputPixels / ((roundedW + 2) * (roundedH + 2)), + ), + ), + ), + ); + const meta = await sharp(svg, { + density, + limitInputPixels: config.limits.inputPixels, + }) + .timeout({ seconds: config.limits.processingTimeoutSeconds }) + .metadata(); + // Each side to within 1/scale of a pixel. A whole-number size inside that range is taken as + // exact: the read is coarse for a long strip (40000×300 allows only about 4.7×). + const scale = density / 72; + const side = (scaled: number | undefined, rounded: number) => { + const estimate = scaled === undefined ? rounded : scaled / scale; + return Math.abs(estimate - rounded) <= 1 / scale ? rounded : estimate; + }; + return { + width: side(meta.width, roundedW), + height: side(meta.height, roundedH), + }; +} + +/** The identity a stored row answers for (rows written before previews carried one: derived). */ +function identityOf(preview: Preview): string { + return ( + preview.identity ?? + getPreviewIdentity( + previewScope(preview), + preview.sizeKey, + preview.format, + preview.filters, + ) + ); +} + +/** Uploaded locators for a log line; an opaque ref that JSON cannot encode is still named. */ +function describeRefs(previews: Preview[]): string { + return previews + .map((preview) => { + try { + return JSON.stringify(preview.storageRef) ?? String(preview.storageRef); + } catch { + return String(preview.storageRef); + } + }) + .join(', '); +} + // --------------------------------------------------------------------------- // The shared core (07 steps 2–8; 11 · §11.1 step 4). Both modes expand their inputs into a // `requested` MissingPreview[] and call this. `locks` (queued mode only: the database's locks) @@ -67,8 +208,11 @@ export interface GenerateCoreArgs { } export interface GenerateCoreResult { + // Previews this call stored (or, with persist:false, produced). generated: Preview[]; failedCount: number; + // Identities the database already held from another worker: covered, not failed. + alreadyStored: string[]; } export async function generatePreviews( @@ -88,27 +232,63 @@ export async function generatePreviews( const { config, logger } = resizer; const storage = resizer.storage; - const generated: Preview[] = []; + // Before any download or decode, in both modes; the queue retries it like any failure. + requirePipeline(resizer, pipelineName); + + let generated: Preview[] = []; let failedCount = 0; + const alreadyStored: string[] = []; // Nothing requested (e.g. eager re-run where everything already exists) → no download. if (requested.length === 0) { - return { generated, failedCount }; + return { generated, failedCount, alreadyStored }; } const original = media.original; if (original?.storageRef == null) { // Callers guard this, but never assume — a media without an original has nothing to // resize from. - return { generated, failedCount }; + return { generated, failedCount, alreadyStored }; + } + + // Existing-preview set (the DB check that makes re-runs idempotent — 07 step 6). Stored + // previews keep their own scope, so another pipeline's rendering never counts as done here. + const scope: PreviewScope = { resizer: resizer.name, pipeline: pipelineName }; + const existing = new Set(); + for (const p of media.previews ?? []) { + if (isUsablePreview(p)) { + existing.add( + getPreviewIdentity(previewScope(p), p.sizeKey, p.format, p.filters), + ); + } + } + const pending = requested.filter( + (v) => + !existing.has(getPreviewIdentity(scope, v.sizeKey, v.format, v.filters)), + ); + // Everything requested is already stored: no download. In queued mode drop the dispatch locks + // so a later read can re-enqueue a sibling promptly. + if (pending.length === 0) { + if (locks) { + for (const v of requested) { + await releaseLock( + resizer, + dispatchLockKey( + mediaId, + getPreviewIdentity(scope, v.sizeKey, v.format, v.filters), + ), + ); + } + } + return { generated, failedCount, alreadyStored }; } // 2. Download the original ONCE. let buf = asBuffer(await storage.download(original.storageRef)); - // 3. Metadata + decode-bomb guards + orientation normalization (07 · §11 step 3). EVERY worker - // sharp() call carries limitInputPixels (01 · §16), so an oversized-for-inputPixels source is - // rejected consistently at this first probe rather than slipping through to a per-variant decode. + // 3. Metadata + decode-bomb guards (07 · §11 step 3). EVERY worker sharp() call carries + // limitInputPixels (01 · §16), so an oversized-for-inputPixels source is rejected + // consistently at this first probe rather than slipping through to a per-variant decode. const origMeta = await sharp(buf, { limitInputPixels: config.limits.inputPixels, }) @@ -126,129 +306,167 @@ export async function generatePreviews( orientation >= 5 ? [origMeta.height, origMeta.width] : [origMeta.width, origMeta.height]; - const frames = config.animated - ? Math.min(origMeta.pages ?? 1, config.limits.animationFrames) - : 1; - if (origMeta.width * origMeta.height * frames > config.limits.sourcePixels) { - throw new ResizeMediaError( - `resize: source ${origMeta.width}×${origMeta.height}×${frames}f exceeds limits.sourcePixels (${config.limits.sourcePixels}) for media ${mediaId}`, - { mediaId, code: 'RESIZE_SOURCE_TOO_LARGE' }, - ); - } - // Normalize orientation ONCE, before beforeSteps, so every step + variant sees DISPLAY- - // orientation pixels (and a beforeStep that round-trips sharp() cannot re-strip a live EXIF - // orientation and desync the result). Per-variant `.rotate()` below is then defense-in-depth. - let displayMeta = origMeta; - if (orientation > 1) { - buf = await sharp(buf, { - failOn: 'none', - limitInputPixels: config.limits.inputPixels, - }) - .rotate() - .toBuffer(); - displayMeta = await sharp(buf, { - limitInputPixels: config.limits.inputPixels, - }).metadata(); - } - - // 4. beforeSteps — the ordered, awaited chain, ONCE, over the display-orientation buffer. - const pipeline = resizer.getPipeline(pipelineName); - for (const step of pipeline.beforeSteps ?? []) { - buf = asBuffer(await step(buf, { media, metadata: displayMeta, ctx })); - } - - // 5. Post-beforeSteps metadata. Buffer is already display-oriented → NO swap logic here. - const procMeta = await sharp(buf, { - limitInputPixels: config.limits.inputPixels, - }) - .timeout({ seconds: config.limits.processingTimeoutSeconds }) - .metadata(); - const procW = procMeta.width ?? dispW; - const procH = procMeta.height ?? dispH; - if ( - procW * procH > - Math.min(config.limits.sourcePixels, config.limits.inputPixels) - ) { + // One frame must fit; how many frames are decoded is bounded below. + if (origMeta.width * origMeta.height > config.limits.sourcePixels) { throw new ResizeMediaError( - `resize: processed source exceeds pixel limits for media ${mediaId}`, + `resize: source ${origMeta.width}×${origMeta.height} exceeds limits.sourcePixels (${config.limits.sourcePixels}) for media ${mediaId}`, { mediaId, code: 'RESIZE_SOURCE_TOO_LARGE' }, ); } - - // 6. Existing-preview set (the DB check that makes re-runs idempotent — 07 step 6). Stored - // previews keep their own scope, so another pipeline's rendering never counts as done here. - const scope: PreviewScope = { resizer: resizer.name, pipeline: pipelineName }; - const existing = new Set(); - for (const p of media.previews ?? []) { - if (isUsablePreview(p)) { - existing.add( - getPreviewIdentity(previewScope(p), p.sizeKey, p.format, p.filters), - ); - } - } - - // 7. Decode the original ONCE; clone the base per variant so the decode is shared. - // SVG is decoded at the largest requested scale before the shared preview pipeline. - // Cap the intermediate raster by the same source/input pixel budgets as other inputs. const pixelBudget = Math.min( config.limits.sourcePixels, config.limits.inputPixels, ); - const largestScale = - procMeta.format === 'svg' + + const pipeline = resizer.getPipeline(pipelineName); + // A preview keeps the animation only in an animated format and only without variantSteps: a + // step sees every frame stacked into one tall image, so a composited watermark would land on + // a single frame. Everything else is rendered from the first frame. + const animates = + config.animated && + (pipeline.variantSteps ?? []).length === 0 && + requested.some((v) => isAnimatedFormat(v.format)); + // Frames to decode: the source's own count (asking for more pages than exist fails the + // decode), up to limits.animationFrames and to what fits the pixel budget (Sharp counts every + // decoded frame against limitInputPixels). libvips cannot rotate a multi-page image, so an + // animation that carries an EXIF orientation is rendered from its first frame. + const framesOf = (meta: Metadata, frameW: number, frameH: number): number => + animates && (meta.orientation ?? 1) <= 1 ? Math.max( 1, - ...requested - .filter((variant) => !variant.fit) - .map((variant) => - Math.max( - (variant.requestedWidth ?? 0) / procW, - (variant.requestedHeight ?? 0) / procH, - ), - ), - ) - : 1; - const density = - procMeta.format === 'svg' - ? Math.max( - 72, Math.min( - 100_000, - Math.floor( - 72 * - Math.min( - largestScale, - Math.sqrt(pixelBudget / (procW * procH)), - ), - ), + meta.pages ?? 1, + config.limits.animationFrames, + Math.floor(pixelBudget / (frameW * frameH)), ), ) - : undefined; - const base = sharp(buf, { - failOn: procMeta.format === 'svg' ? 'warning' : 'none', - sequentialRead: true, - limitInputPixels: config.limits.inputPixels, - animated: config.animated, - pages: config.animated ? config.limits.animationFrames : 1, - ...(density !== undefined ? { density } : {}), - }); - const fitBase = - density !== undefined && density > 72 && requested.some((v) => v.fit) - ? sharp(buf, { - failOn: 'warning', - sequentialRead: true, + : 1; + + // 4. An SVG is rendered ONCE, before anything else touches it, in a child process with a hard + // time limit, to a lossless raster at the largest scale the pending variants need (capped by + // the pixel budget and librsvg's side limit). From then on it is a regular image: beforeSteps + // receive the PNG, never SVG markup they could render in-process without a time limit. + // procW/procH stay the SVG's own size (fit sizes are computed from it and never exceed it); + // svgAspect is its real aspect ratio, for derived sides and the cap decision. + const fromSvg = origMeta.format === 'svg'; + let svgAspect: number | undefined; + if (fromSvg) { + const natural = await svgNaturalSize(buf, dispW, dispH, config); + svgAspect = natural.width / natural.height; + const largestScale = Math.max( + 1, + ...pending + .filter((variant) => !variant.fit) + .map((variant) => + Math.max( + (variant.requestedWidth ?? 0) / natural.width, + (variant.requestedHeight ?? 0) / natural.height, + ), + ), + ); + const scale = Math.min( + Math.max( + 1, + Math.min( + largestScale, + Math.sqrt(pixelBudget / (natural.width * natural.height)), + ), + ), + // librsvg's side limit holds whatever the pixel budget allows (a 1×5000 strip). + SVG_MAX_SIDE / Math.max(natural.width, natural.height), + ); + buf = await rasterizeSvg(buf, { + density: Math.max(1, Math.min(100_000, Math.floor(72 * scale))), + limitInputPixels: config.limits.inputPixels, + timeoutMs: config.limits.processingTimeoutSeconds * 1000, + signal, + mediaId, + }); + } + + // 5. beforeSteps — the ordered, awaited chain, ONCE, over display-orientation pixels. + // Without steps the original is used as downloaded: each variant's `.rotate()` applies the + // EXIF orientation, so nothing is re-encoded before the variants are made. + const beforeSteps = pipeline.beforeSteps ?? []; + let procMeta = fromSvg + ? await sharp(buf, { limitInputPixels: config.limits.inputPixels }) + .timeout({ seconds: config.limits.processingTimeoutSeconds }) + .metadata() + : origMeta; + let procW = dispW; + let procH = dispH; + // The SVG sizing rules above apply while the pixels are the SVG's own raster. + let svgSizing = fromSvg; + if (beforeSteps.length > 0) { + // Normalize orientation ONCE so every step sees DISPLAY-orientation pixels (and a step that + // round-trips sharp() cannot re-strip a live EXIF orientation and desync the result), + // re-encoded in the same format without visible loss. + let stepMeta = procMeta; + if (orientation > 1) { + buf = await withoutVisibleLoss( + sharp(buf, { + failOn: 'none', limitInputPixels: config.limits.inputPixels, - animated: config.animated, - pages: config.animated ? config.limits.animationFrames : 1, - }) - : undefined; + }).rotate(), + origMeta.format, + ) + .timeout({ seconds: config.limits.processingTimeoutSeconds }) + .toBuffer(); + stepMeta = await sharp(buf, { + limitInputPixels: config.limits.inputPixels, + }).metadata(); + } + for (const step of beforeSteps) { + buf = asBuffer(await step(buf, { media, metadata: stepMeta, ctx })); + } + + // Post-beforeSteps metadata. Buffer is already display-oriented → NO swap logic here. + const after = await sharp(buf, { + limitInputPixels: config.limits.inputPixels, + }) + .timeout({ seconds: config.limits.processingTimeoutSeconds }) + .metadata(); + if ( + !fromSvg || + after.width !== procMeta.width || + after.height !== procMeta.height + ) { + // A step that resized the SVG's raster made a new image: from here it is sized by its + // own pixels, like any raster. + svgSizing = false; + svgAspect = undefined; + procW = after.width ?? dispW; + procH = after.height ?? dispH; + } + procMeta = after; + } + if (procW * procH > pixelBudget) { + throw new ResizeMediaError( + `resize: processed source exceeds pixel limits for media ${mediaId}`, + { mediaId, code: 'RESIZE_SOURCE_TOO_LARGE' }, + ); + } + + // Decode ONCE per kind; clone the base per variant so the decode is shared. Formats that + // cannot hold an animation decode only the first frame (otherwise Sharp stacks every frame + // into one tall image). + const decode = (pages: number): Sharp => + sharp(buf, { + failOn: 'none', + sequentialRead: true, + limitInputPixels: config.limits.inputPixels, + pages, + }); + const frames = framesOf(procMeta, procW, procH); + const base = decode(1); + const animatedBase = frames > 1 ? decode(frames) : undefined; // Locks held for processed variants; released once after the pool (success AND error). const heldLocks = new Set(); const processVariant = async (v: MissingPreview): Promise => { const identity = getPreviewIdentity(scope, v.sizeKey, v.format, v.filters); - const dispatchKey = `resize_dispatch:${mediaId}:${identity}`; + const dispatchKey = dispatchLockKey(mediaId, identity); const workerKey = `resize_worker:${mediaId}:${identity}`; // Skip anything already generated; in queued mode drop its dispatch lock so a later read @@ -284,43 +502,89 @@ export async function generatePreviews( } try { - // Cover: dims pass straight through, clamped per provided side to limits.resultDimension. - // Fit: already capped to config.maxSize by calculateResizedDimensions. - const dims = calculateResizedDimensions( - procW, - procH, - v.requestedWidth, - v.requestedHeight, - v.fit ?? false, - config.maxSize, - ); - let width = dims.width; - let height = dims.height; - if (!v.fit) { - const cap = config.limits.resultDimension; - if (typeof width === 'number' && width > cap) { - width = cap; - } - if (typeof height === 'number' && height > cap) { - height = cap; + // A task payload can name any format; only configured encoders run (no sharp work here). + if (!Object.hasOwn(config.encode.formats, v.format)) { + throw formatNotConfiguredError([v.format]); + } + // Fit: capped to config.maxSize by calculateResizedDimensions. Cover: requested sides + // rounded and capped to limits.resultDimension, a derived side included. For an SVG the + // cap decision uses its real aspect ratio (a 1000×0.6 SVG reads as 1000×1). + let { width, height } = v.fit + ? calculateResizedDimensions( + procW, + procH, + undefined, + undefined, + true, + config.maxSize, + ) + : svgAspect !== undefined + ? coverDimensions( + svgAspect, + 1, + v.requestedWidth, + v.requestedHeight, + config.limits.resultDimension, + ) + : coverDimensions( + procW, + procH, + v.requestedWidth, + v.requestedHeight, + config.limits.resultDimension, + ); + // An SVG's width-only or height-only size under the cap: the other side from the SVG's + // real aspect ratio, and the raster resized to exactly that box (no crop). + const svgDerived = + svgAspect !== undefined && + !v.fit && + (width === undefined) !== (height === undefined); + if (svgDerived && svgAspect !== undefined) { + if (width !== undefined) { + height = Math.max(1, Math.round(width / svgAspect)); + } else if (height !== undefined) { + width = Math.max(1, Math.round(height * svgAspect)); } } - // Clone the shared decode; `.rotate()` on EVERY branch (defense-in-depth); normalize the - // working colorspace BEFORE variantSteps so composited overlay colors are predictable. - let img = (v.fit && fitBase ? fitBase : base) + // A raster source that fits inside the requested WxH box is the whole image at its own + // size: no upscaling and no cropping (only the resultDimension cap may scale it down); the + // encode strips metadata. Only without variantSteps: a pipeline's steps keep the full box + // they were written for (a 200×50 watermark cannot be composited onto 150×150). + const ownSize = + !svgSizing && + !v.fit && + (pipeline.variantSteps ?? []).length === 0 && + v.requestedWidth !== undefined && + v.requestedHeight !== undefined && + procW <= Math.round(v.requestedWidth) && + procH <= Math.round(v.requestedHeight); + // Not sharpened when nothing was scaled. + const unscaled = + ownSize && + (width === undefined || procW <= width) && + (height === undefined || procH <= height); + + // Clone the shared decode; `.rotate()` on EVERY branch applies the EXIF orientation; + // normalize the working colorspace BEFORE variantSteps so composited overlay colors are + // predictable. An SVG raster is filled into the exact box computed from the SVG itself + // (its rounded raster aspect ratio would shift an `inside` resize by a pixel). + const animate = animatedBase !== undefined && isAnimatedFormat(v.format); + let img = (animate ? animatedBase : base) .clone() .rotate() .resize( width, height, - v.fit - ? { fit: 'inside', withoutEnlargement: true } - : { fit: 'cover', position: 'center' }, + (v.fit && svgSizing) || svgDerived + ? { fit: 'fill' } + : v.fit || ownSize + ? { fit: 'inside', withoutEnlargement: true } + : { fit: 'cover', position: 'center' }, ) .toColorspace('srgb'); const s = config.encode.sharpen; - const sharpenOn = s && (v.fit ? s.fit : s.cover); + const sharpenOn = s && !unscaled && (v.fit ? s.fit : s.cover); if (sharpenOn) { img = img.sharpen(); } @@ -375,22 +639,24 @@ export async function generatePreviews( const preview: Preview = { storageRef: ref, + identity, resizer: resizer.name, pipeline: pipelineName, sizeKey: v.sizeKey, format: v.format, contentType, + // One frame: an animated output reports the height of all frames stacked. actualWidth: info.width, - actualHeight: info.height, + actualHeight: info.pageHeight ?? info.height, }; if (v.filters) { preview.filters = v.filters; } if (v.requestedWidth !== undefined) { - preview.requestedWidth = v.requestedWidth; + preview.requestedWidth = Math.round(v.requestedWidth); } if (v.requestedHeight !== undefined) { - preview.requestedHeight = v.requestedHeight; + preview.requestedHeight = Math.round(v.requestedHeight); } if (v.fit) { preview.fit = true; @@ -422,7 +688,75 @@ export async function generatePreviews( original.width === undefined || original.height === undefined ? { width: dispW, height: dispH } : undefined; - await resizer.db.appendPreviews(mediaId, generated, backfillDims); + // What the database holds now, read once after a failed or partial write: the identities + // stored on the media, null when the media is gone, undefined when the reload failed too. + const storedIdentities = async (): Promise< + Set | null | undefined + > => { + try { + const reloaded = await resizer.db.loadMedia(mediaId); + return reloaded + ? new Set( + (reloaded.previews ?? []) + .filter(isUsablePreview) + .map(identityOf), + ) + : null; + } catch { + return undefined; + } + }; + const stored = await resizer.db + .appendPreviews(mediaId, generated, backfillDims) + .catch(async (err: unknown) => { + // The write may have stopped part-way (a driver can store one row at a time). Name + // only the uploads no row points at, so an operator can find (or delete) them; a file + // a stored row references is never offered. + const recorded = await storedIdentities(); + const unrecorded = recorded + ? generated.filter((p) => !recorded.has(identityOf(p))) + : generated; + logger.error( + `resize: recording previews failed for media ${mediaId}; ${unrecorded.length} of ${generated.length} uploaded preview(s) are not recorded${recorded === undefined ? ' (the media could not be reloaded to check)' : ''}: storage refs ${describeRefs(unrecorded)}`, + err, + ); + throw err; + }); + // The database keeps one row per identity and reports the rows it stored (nothing = all). + // The reload says why a row was left out: the media is gone, or another worker stored the + // same identity first (covered, not a failure). + if (Array.isArray(stored)) { + const kept = new Set(stored.map(identityOf)); + const leftOut = generated.filter((p) => !kept.has(identityOf(p))); + if (leftOut.length > 0) { + generated = generated.filter((p) => kept.has(identityOf(p))); + const recorded = await storedIdentities(); + if (recorded === null) { + logger.warn( + `resize: media ${mediaId} no longer exists; ${leftOut.length} uploaded preview(s) are not recorded: storage refs ${describeRefs(leftOut)}`, + ); + } else { + // A failed reload trusts the database's answer: the rows are another worker's. + const duplicates = recorded + ? leftOut.filter((p) => recorded.has(identityOf(p))) + : leftOut; + const lost = recorded + ? leftOut.filter((p) => !recorded.has(identityOf(p))) + : []; + if (duplicates.length > 0) { + logger.warn( + `resize: ${duplicates.length} preview(s) for media ${mediaId} were already stored by another worker; uploaded but not recorded storage refs: ${describeRefs(duplicates)}`, + ); + alreadyStored.push(...duplicates.map(identityOf)); + } + if (lost.length > 0) { + logger.error( + `resize: the database left out ${lost.length} preview(s) for media ${mediaId} that it does not hold; uploaded but not recorded storage refs: ${describeRefs(lost)}`, + ); + } + } + } + } for (const preview of generated) { await resizer.runObservers('onPreviewGenerated', preview, {}); } @@ -436,7 +770,7 @@ export async function generatePreviews( } } - return { generated, failedCount }; + return { generated, failedCount, alreadyStored }; } /** Best-effort lock release; a failing release is logged, never thrown. */ @@ -496,24 +830,28 @@ export async function processTaskWith( } } const requested = [...requestedByIdentity.values()]; - const { generated, failedCount } = await generatePreviews(resizer, { - media, - mediaId: task.mediaId, - requested, - pipeline: task.pipeline, - ctx, - locks: { - workerTtlMs: tasks - ? timingOf(tasks).lockTtlMs.worker - : defaultQueueOptions.lockTtlMs.worker, + const { generated, failedCount, alreadyStored } = await generatePreviews( + resizer, + { + media, + mediaId: task.mediaId, + requested, + pipeline: task.pipeline, + ctx, + locks: { + workerTtlMs: tasks + ? timingOf(tasks).lockTtlMs.worker + : defaultQueueOptions.lockTtlMs.worker, + }, + persist: true, + signal: taskOpts?.signal, }, - persist: true, - signal: taskOpts?.signal, - }); + ); // 10. A queued task is complete only when every requested identity is now persisted. Re-read // once so a worker-lock loser can observe a concurrent worker's write. Our own generated rows - // are included too: appendPreviews returned successfully before generatePreviews returned. + // are included too: appendPreviews returned successfully before generatePreviews returned, + // and so are the identities the database reported as already stored by another worker. const refreshed = await resizer.db.loadMedia(task.mediaId); if (!refreshed) { logger.info( @@ -521,7 +859,7 @@ export async function processTaskWith( ); return; } - const covered = new Set(); + const covered = new Set(alreadyStored); for (const preview of [ ...(media.previews ?? []), ...(refreshed.previews ?? []), @@ -590,14 +928,20 @@ export async function generateImpl( if (original?.storageRef == null) { throw new ResizeNoOriginalError(mediaId); } + // Caller bugs fail before any host hook, download or decode. + requirePipeline(resizer, pipeline); + const formats = opts.formats ?? config.formats; + const unconfigured = unconfiguredFormats(config.encode.formats, formats); + if (unconfigured.length > 0) { + throw formatNotConfiguredError(unconfigured); + } - // Host size magic (real ctx in eager mode), then the active format list. + // Host size magic (real ctx in eager mode). const sizes = (await resizer.runWaterfall( 'resolveSizes', opts.sizes, ctx, )) as SizeInput[]; - const formats = opts.formats ?? config.formats; // Expand sizes × formats; skip unbuildable sizes + existing identities (idempotent). const requested = expandMissingPreviews(media, sizes, formats, { diff --git a/src/resizer.test.ts b/src/resizer.test.ts index 48327c7..653ee9c 100644 --- a/src/resizer.test.ts +++ b/src/resizer.test.ts @@ -128,7 +128,9 @@ describe('Resizer constructor — driver wiring', () => { () => new Resizer({ db: fakeDb() } as never), (err: unknown) => err instanceof ResizeSetupError && - err.code === 'RESIZE_STORAGE_REQUIRED', + err.code === 'RESIZE_STORAGE_REQUIRED' && + // The message points at nothing the package does not ship. + !err.message.includes('§'), ); // The bad construction must NOT have claimed the active slot. assert.throws(() => getResizer(), /no Resizer named 'default'/); @@ -298,6 +300,61 @@ describe('drivers given as functions', () => { assert.equal(r.storage, storage); }); + test('a part that loaded is kept when a sibling fails; only the failed part is retried', async () => { + let storageCalls = 0; + let dbCalls = 0; + const storage = fakeStorage(); + const db = fakeDb(); + const r = new Resizer({ + logger: silent, + storage: async () => { + storageCalls += 1; + return storage; + }, + db: async () => { + dbCalls += 1; + if (dbCalls === 1) { + throw new Error('db not up'); + } + return db; + }, + }); + await assert.rejects(() => r.ready(), /db not up/); + // Concurrent calls after the failure share one retry of the failed part. + await Promise.all([r.ready(), r.ready()]); + await r.ready(); + assert.equal(storageCalls, 1); + assert.equal(dbCalls, 2); + assert.equal(r.storage, storage); + assert.equal(r.db, db); + }); + + test('a failed lazy task queue stays not ready until it loads', async () => { + let taskCalls = 0; + const tasks = new MemoryTaskQueue(); + const r = new Resizer({ + logger: silent, + storage: fakeStorage(), + db: fakeDb(), + tasks: async () => { + taskCalls += 1; + if (taskCalls === 1) { + throw new Error('queue not up'); + } + return tasks; + }, + }); + await assert.rejects(() => r.ready(), /queue not up/); + assert.throws( + () => r.tasks, + (err: unknown) => + err instanceof ResizeSetupError && err.code === 'RESIZE_NOT_READY', + ); + await r.ready(); + assert.equal(r.tasks, tasks); + assert.equal(taskCalls, 2); + }); + test('a loader that returns nothing for a required part is a setup error', async () => { const noStorage = new Resizer({ name: 'a', diff --git a/src/resizer.ts b/src/resizer.ts index 9de6ec1..1b64580 100644 --- a/src/resizer.ts +++ b/src/resizer.ts @@ -194,13 +194,15 @@ const EMPTY_PIPELINE: Pipeline = Object.freeze({}); // core code always receives its Resizer as an argument. const resizers = new Map(); +type PartName = 'storage' | 'db' | 'tasks'; + // How framework hosts get the parts below filled in from their app. const FRAMEWORK_HINT = "framework hosts: use new FrameworkResizer() from '@adaptivestone/framework-module-resize/framework.js'"; const storageRequired = () => new ResizeSetupError( - 'resize: `storage` is required — construct `new Resizer({ storage: … })` with a ResizeStorage driver (e.g. new S3Storage({ … })); both the read path (publicUrl) and the worker (download/upload) need it (05 · §10.4)', + 'resize: `storage` is required — construct `new Resizer({ storage: … })` with a ResizeStorage driver (e.g. new S3Storage({ … })); both the read path (publicUrl) and the worker (download/upload) need it', { code: 'RESIZE_STORAGE_REQUIRED' }, ); @@ -228,8 +230,10 @@ export class Resizer { #storage: ResizeStorage | undefined; #db: ResizeDatabase | undefined; #tasks: TaskQueue | undefined; - #loaded = false; - #loading: Promise | undefined; + // Each part loads once: the parts usable now (given as objects, or loaded) and the loads in + // flight. A part whose function failed is retried alone; the parts that loaded are kept. + readonly #loadedParts = new Set(); + readonly #loadingParts = new Map>(); // Named pipelines: last-wins per name (04 · §8). readonly #pipelines: Map; // Hook bus: taps run in REGISTRATION order, awaited sequentially (04 · §9). @@ -283,16 +287,16 @@ export class Resizer { // Parts given as objects are usable at once; functions wait for ready(). if (typeof opts.storage !== 'function') { this.#storage = opts.storage; + this.#loadedParts.add('storage'); } if (typeof opts.db !== 'function') { this.#db = opts.db; + this.#loadedParts.add('db'); } if (typeof opts.tasks !== 'function') { this.#tasks = opts.tasks; + this.#loadedParts.add('tasks'); } - this.#loaded = ![opts.storage, opts.db, opts.tasks].some( - (part) => typeof part === 'function', - ); this.#pipelines = new Map(Object.entries(opts.pipelines ?? {})); // A seeded hooks value may be a single fn or an array — normalize to arrays and // COPY them, so a caller mutating its own array later cannot bypass hook(). @@ -329,7 +333,7 @@ export class Resizer { /** Where tasks wait (undefined: eager only). Available once the parts are loaded. */ get tasks(): TaskQueue | undefined { - if (typeof this.#sources.tasks === 'function' && !this.#loaded) { + if (!this.#loadedParts.has('tasks')) { this.#notReady('tasks'); } return this.#tasks; @@ -343,39 +347,53 @@ export class Resizer { } /** - * Load the parts given as functions, once. Every method of the Resizer (and the worker) awaits - * it first, so hosts only need it before reading `storage`, `db` or `tasks` directly. A failed - * load is retried by the next call. + * Load the parts given as functions, each once. Every method of the Resizer (and the worker) + * awaits it first, so hosts only need it before reading `storage`, `db` or `tasks` directly. A + * part whose function fails is retried by the next call; the parts that loaded are kept. */ async ready(): Promise { - if (this.#loaded) { + if (this.#loadedParts.size === 3) { return; } - this.#loading ??= this.#load().catch((err: unknown) => { - this.#loading = undefined; - throw err; - }); - await this.#loading; + await Promise.all([ + this.#loadPart('storage', this.#sources.storage, (storage) => { + if (!storage) { + throw storageRequired(); + } + this.#storage = storage; + }), + this.#loadPart('db', this.#sources.db, (db) => { + if (!db) { + throw databaseRequired(); + } + this.#db = db; + }), + this.#loadPart('tasks', this.#sources.tasks, (tasks) => { + this.#tasks = tasks ?? undefined; + }), + ]); } - async #load(): Promise { - const call = async (part: T | LazyPart): Promise => - typeof part === 'function' ? (part as LazyPart)() : part; - const [storage, db, tasks] = await Promise.all([ - call(this.#sources.storage), - call(this.#sources.db), - call(this.#sources.tasks), - ]); - if (!storage) { - throw storageRequired(); + /** Load one part given as a function; concurrent callers share its load while it runs. */ + #loadPart( + part: PartName, + source: T | LazyPart, + keep: (value: T) => void, + ): Promise { + if (this.#loadedParts.has(part)) { + return Promise.resolve(); } - if (!db) { - throw databaseRequired(); + let loading = this.#loadingParts.get(part); + if (!loading) { + loading = (async () => { + keep(await (source as LazyPart)()); + this.#loadedParts.add(part); + })().finally(() => { + this.#loadingParts.delete(part); + }); + this.#loadingParts.set(part, loading); } - this.#storage = storage; - this.#db = db; - this.#tasks = tasks ?? undefined; - this.#loaded = true; + return loading; } /** The validated image config; a lazy config (a function) is read and validated on first use. */ @@ -417,6 +435,11 @@ export class Resizer { return this.#pipelines.get(name) ?? EMPTY_PIPELINE; } + /** True when `name` is registered. `default` always exists: unregistered, it has no steps. */ + hasPipeline(name: string): boolean { + return name === 'default' || this.#pipelines.has(name); + } + /** * Thread `value` through the name's taps in order, awaiting each. Taps are HOST code * on the read path, so each is GUARDED: on throw, log and keep the prior value (treat @@ -528,7 +551,19 @@ export function listResizers(): Resizer[] { return [...resizers.values()]; } +// Per-process state an adapter keeps beside the registry (the framework adapter's shared task +// queues), cleared with it. +const resetHooks = new Set<() => void>(); + +/** Run `reset` on every resetResizerForTests() (for an adapter's per-process state). */ +export function onResetResizerForTests(reset: () => void): void { + resetHooks.add(reset); +} + /** TEST-ONLY: forget every constructed Resizer so a test can construct fresh ones. */ export function resetResizerForTests(): void { resizers.clear(); + for (const reset of resetHooks) { + reset(); + } } diff --git a/src/testHelpers/contractShapes.ts b/src/testHelpers/contractShapes.ts index 81f2428..bebcd63 100644 --- a/src/testHelpers/contractShapes.ts +++ b/src/testHelpers/contractShapes.ts @@ -26,6 +26,7 @@ export const fullTasks: TaskQueue = { renew: async () => true, complete: async () => true, fail: async () => true, + release: async () => true, findActive: async () => [], servesQueue: () => true, getTiming: () => ({ leaseMs: 1000 }), diff --git a/src/testHelpers/fakes.test.ts b/src/testHelpers/fakes.test.ts new file mode 100644 index 0000000..f80eb5f --- /dev/null +++ b/src/testHelpers/fakes.test.ts @@ -0,0 +1,43 @@ +// The in-memory database follows the database contract tests rely on: one stored preview row per +// preview identity, resolving with the previews it stored. +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import type { Preview } from '../types.d.ts'; +import { fakeDb } from './fakes.ts'; + +const preview = (identity: string | undefined, path: string): Preview => + ({ + storageRef: { path }, + ...(identity === undefined ? {} : { identity }), + sizeKey: '16x16', + format: 'webp', + }) as Preview; + +test('fakeDb keeps one preview per identity and resolves with the stored ones', async () => { + const previews = new Map(); + const db = fakeDb({ previews }); + const a = preview('a', 'a1'); + const b = preview('b', 'b1'); + + assert.deepEqual(await db.appendPreviews('m1', [a, b, preview('a', 'a2')]), [ + a, + b, + ]); + assert.deepEqual(await db.appendPreviews('m1', [preview('a', 'a3')]), []); + // Identities are per media. + assert.deepEqual(await db.appendPreviews('m2', [a]), [a]); + // A preview without an identity is always stored. + const plain = preview(undefined, 'plain'); + assert.deepEqual(await db.appendPreviews('m1', [plain, plain]), [ + plain, + plain, + ]); + assert.deepEqual(previews.get('m1'), [a, b, plain, plain]); + assert.deepEqual(previews.get('m2'), [a]); +}); + +test('fakeDb treats seeded previews as already stored', async () => { + const seeded = preview('a', 'other-worker'); + const db = fakeDb({ previews: new Map([['m1', [seeded]]]) }); + assert.deepEqual(await db.appendPreviews('m1', [preview('a', 'late')]), []); +}); diff --git a/src/testHelpers/fakes.ts b/src/testHelpers/fakes.ts index d45d368..9597e91 100644 --- a/src/testHelpers/fakes.ts +++ b/src/testHelpers/fakes.ts @@ -36,24 +36,43 @@ export function memoryLocks(): FakeLocks { }; } -/** A ResizeDatabase from optional parts; missing parts are harmless no-ops. */ +/** + * A ResizeDatabase from optional parts; missing parts are harmless in-memory stand-ins. The default + * appendPreviews follows the database contract: it keeps one row per preview identity in + * `previews` (per media id; pass a map to seed or inspect it) and resolves with the previews it + * stored. A preview without an identity is always stored. + */ export function fakeDb( parts: { load?: (mediaId: string) => Promise; - appendPreviews?: ( - mediaId: string, - previews: Preview[], - backfillDims?: { width: number; height: number }, - ) => Promise; + appendPreviews?: ResizeDatabase['appendPreviews']; + previews?: Map; locks?: FakeLocks; tasks?: TaskQueue; verify?: () => void | Promise; } = {}, ): ResizeDatabase { const locks = parts.locks ?? memoryLocks(); + const rows = parts.previews ?? new Map(); + const appendPreviews = async (mediaId: string, previews: Preview[]) => { + const media = rows.get(mediaId) ?? []; + rows.set(mediaId, media); + const stored: Preview[] = []; + for (const preview of previews) { + if ( + preview.identity !== undefined && + media.some((row) => row.identity === preview.identity) + ) { + continue; + } + media.push(preview); + stored.push(preview); + } + return stored; + }; return { loadMedia: parts.load ?? (async () => null), - appendPreviews: parts.appendPreviews ?? (async () => {}), + appendPreviews: parts.appendPreviews ?? appendPreviews, acquireLock: (key, ttlMs) => locks.acquire(key, ttlMs), releaseLock: (key) => locks.release(key), ...(parts.tasks ? { tasks: parts.tasks } : {}), diff --git a/src/types.d.ts b/src/types.d.ts index a465ee1..c5f0079 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -67,6 +67,9 @@ export interface PreviewScope { export interface Preview { storageRef: StorageRef; + // The full preview identity (resizer:pipeline:sizeKey:format:filters). The database stores one + // row per identity; absent on rows written before previews carried it. + identity?: string; resizer?: string; // Resizer that generated it; absent → 'default' pipeline?: string; // pipeline that generated it; absent → 'default' sizeKey: string; // canonical size key — see 03 · Identity @@ -110,9 +113,8 @@ export interface ReadyEntry { format: PreviewFormat; filters?: Filters; url: string; - preview?: Preview; // present for generated previews; ABSENT for original-backed entries - isOriginal?: boolean; // true when `url` points at the untouched original - contentType?: string; // preview.contentType, or original.contentType when isOriginal + preview: Preview; // the stored preview this entry serves + contentType: string; // preview.contentType } export interface ReadDecision { @@ -141,6 +143,8 @@ export interface EnqueueIssue { | 'RESIZE_ENQUEUE_UNCONFIRMED' | 'RESIZE_ENQUEUE_CONFIRM_FAILED' | 'RESIZE_ENQUEUE_VARIANT_CONFLICT' + | 'RESIZE_PIPELINE_UNKNOWN' // the pipeline is not registered on this Resizer + | 'RESIZE_FORMAT_NOT_CONFIGURED' // a requested format has no encode.formats entry | 'RESIZE_ENQUEUE_INTERNAL_ERROR'; // prewarm caught an unexpected error (see message) message: string; retryable: boolean; @@ -218,7 +222,11 @@ export interface ResizeConfig { // Queue timing and lock TTLs: a task queue's timing (MongoTaskQueue / SqsTaskQueue `timing`; the // framework config file's `queue` section feeds the framework's queue). export interface QueueTimingOptions { - lockTtlMs: { dispatch: number; worker: number }; // worker must be ≤ leaseMs + // dispatch: default 60000 — one read's enqueue holds each variant this long; + // worker: default 60000, must be ≤ leaseMs; + // failed: default 600000 — after a task is dead-lettered, reads do not queue its variants again + // for this long. + lockTtlMs: { dispatch: number; worker: number; failed?: number }; leaseMs: number; // default 60000 — the heartbeat renews at leaseMs / 2 retryBackoffMs: { base: number; max: number }; // default { base: 5000, max: 300000 } maxAttempts: number; // default 5 — deliveries before dead-letter (every lease counts, incl. reclaims) diff --git a/src/worker.test.ts b/src/worker.test.ts new file mode 100644 index 0000000..2bf07bb --- /dev/null +++ b/src/worker.test.ts @@ -0,0 +1,296 @@ +// The worker's task handler (processTask) inside the core queue loop: a media error thrown while +// reading the source reaches the retry policy unwrapped, so an unusable source is dead-lettered +// on its first delivery instead of being downloaded and decoded again on every retry. After a +// dead-letter the worker holds the variants' dispatch locks for a cooldown, so reads stop queueing +// the same failing work. +import assert from 'node:assert/strict'; +import { afterEach, describe, test } from 'node:test'; +import sharp from 'sharp'; +import type { LeasedTask, TaskEvent } from './contracts/taskQueue.ts'; +import { consumeQueue } from './queue.ts'; +import { Resizer, resetResizerForTests } from './resizer.ts'; +import { + type FakeLocks, + fakeDb, + MemoryTaskQueue, +} from './testHelpers/fakes.ts'; +import { makeImageConfig } from './testHelpers/resizeConfig.ts'; +import type { MediaLike, QueueTimingOptions, SizeInput } from './types.d.ts'; +import { processTask, runWorker } from './worker.ts'; + +const silent = { info() {}, warn() {}, error() {} }; + +const png = await sharp({ + create: { + width: 64, + height: 48, + channels: 3, + background: { r: 255, g: 0, b: 0 }, + }, +}) + .png() + .toBuffer(); + +afterEach(resetResizerForTests); + +test('a source over limits.sourcePixels is dead-lettered on its first delivery', async () => { + const tasks = new MemoryTaskQueue({ + timing: { idlePollMs: 5, retryBackoffMs: { base: 5, max: 5 } }, + }); + let downloads = 0; + new Resizer({ + config: makeImageConfig({ + formats: ['webp'], + limits: { sourcePixels: 100 }, + }), + logger: silent, + storage: { + download: async () => { + downloads += 1; + return png; + }, + upload: async () => ({ key: 'k' }), + publicUrl: () => '', + }, + db: fakeDb({ + load: async () => ({ + id: 'm1', + original: { storageRef: { key: 'o' } }, + previews: [], + }), + }), + tasks, + }); + await tasks.add({ + resizer: 'default', + queue: 'default', + mediaId: 'm1', + pipeline: 'default', + previews: [ + { + sizeKey: '10x10', + format: 'webp', + requestedWidth: 10, + requestedHeight: 10, + }, + ], + requestKey: 'k1', + }); + const events: { event: TaskEvent; task: LeasedTask; error?: unknown }[] = []; + const stop = new AbortController(); + const timeout = setTimeout(() => stop.abort(), 5000); + await consumeQueue(tasks, { + queue: 'default', + signal: stop.signal, + handle: (task, opts) => processTask(task, opts, tasks), + onEvent: (event, task, error) => { + events.push({ event, task, error }); + stop.abort(); + }, + logger: silent, + }); + clearTimeout(timeout); + assert.deepEqual( + events.map((e) => e.event), + ['deadLettered'], + ); + assert.equal( + (events[0].error as { code?: string }).code, + 'RESIZE_SOURCE_TOO_LARGE', + ); + assert.equal(tasks.rows[0].status, 'dead'); + assert.equal(tasks.rows[0].attempts, 1); + assert.equal(downloads, 1); +}); + +describe('dead-letter cooldown', () => { + /** Locks that expire, on a clock the test moves forward: `clock.ms` is added to real time. */ + function expiringLocks(clock: { ms: number }) { + const expiry = new Map(); + const calls: string[] = []; + const locks: FakeLocks & { calls: string[] } = { + calls, + acquire: async (key, ttlMs) => { + calls.push(`acquire ${key} ${ttlMs}`); + const now = Date.now() + clock.ms; + if ((expiry.get(key) ?? 0) > now) { + return false; + } + expiry.set(key, now + ttlMs); + return true; + }, + release: async (key) => { + calls.push(`release ${key}`); + expiry.delete(key); + }, + }; + return locks; + } + + const size: SizeInput = { width: 10, height: 10 }; + + // A Resizer whose sources are all over limits.sourcePixels, so every task is dead-lettered on + // its first delivery. Dispatch locks last 1 s, the cooldown 10 s. + function setup(locks: FakeLocks, timing: Partial = {}) { + const tasks = new MemoryTaskQueue({ + timing: { + idlePollMs: 5, + lockTtlMs: { dispatch: 1000, worker: 1000, failed: 10_000 }, + ...timing, + }, + }); + const docs = new Map(); + const resizer = new Resizer({ + config: makeImageConfig({ + formats: ['webp'], + limits: { sourcePixels: 100 }, + }), + logger: silent, + storage: { + download: async () => png, + upload: async () => ({ key: 'k' }), + publicUrl: () => '', + }, + db: fakeDb({ load: async (id) => docs.get(id) ?? null, locks }), + tasks, + }); + const errors: unknown[][] = []; + const deadLettered: LeasedTask[] = []; + const stop = new AbortController(); + resizer.hook('onTaskDeadLettered', (task) => { + deadLettered.push(task); + stop.abort(); + }); + return { + tasks, + resizer, + errors, + deadLettered, + stop, + media(id: string): MediaLike { + const doc: MediaLike = { + id, + original: { + storageRef: { key: `o-${id}` }, + contentType: 'image/png', + }, + previews: [], + }; + docs.set(id, doc); + return doc; + }, + read: (media: MediaLike, sizes: SizeInput[] = [size]) => + resizer.resolve({ media, sizes }), + queued: () => tasks.added.length, + // Run the worker until the first dead-letter (or 5 s). + async work() { + const timer = setTimeout(() => stop.abort(), 5000); + try { + await runWorker({ + signal: stop.signal, + logger: { ...silent, error: (...args) => errors.push(args) }, + }); + } finally { + clearTimeout(timer); + } + }, + }; + } + + test('after a dead-letter, reads do not queue that variant again until the cooldown ends', async () => { + const clock = { ms: 0 }; + const t = setup(expiringLocks(clock)); + const m1 = t.media('m1'); + await t.read(m1); + assert.equal(t.queued(), 1); + await t.work(); + assert.equal(t.deadLettered.length, 1); + // Past the 1 s dispatch lock, inside the 10 s cooldown. + clock.ms = 5000; + await t.read(m1); + assert.equal(t.queued(), 1, 'the dead variant is not queued again'); + // Another variant of that media, and the same variant of another media, are not held. + await t.read(m1, [{ width: 20, height: 20 }]); + await t.read(t.media('m2')); + assert.equal(t.queued(), 3); + clock.ms = 10_001; + await t.read(m1); + assert.equal(t.queued(), 4, 'queued again once the cooldown ends'); + }); + + test('a failing lock call is logged; nothing throws and the observer still fires', async () => { + const locks = expiringLocks({ ms: 0 }); + const t = setup(locks); + await t.read(t.media('m1')); + locks.release = async () => { + throw new Error('lock store down'); + }; + await t.work(); + assert.equal(t.deadLettered.length, 1); + assert.ok( + t.errors.some((args) => /cooldown/.test(String(args[0]))), + 'the failed cooldown is logged', + ); + }); + + test('the observer does not wait for the cooldown; the worker returns only after it', async () => { + const locks = expiringLocks({ ms: 0 }); + const t = setup(locks); + await t.read(t.media('m1')); + let openGate!: () => void; + const gate = new Promise((resolve) => { + openGate = resolve; + }); + let released = false; + const release = locks.release; + locks.release = async (key) => { + await gate; + released = true; + await release(key); + }; + let returned = false; + const working = t.work().then(() => { + returned = true; + }); + while (t.deadLettered.length === 0) { + await new Promise((r) => setTimeout(r, 5)); + } + assert.equal( + released, + false, + 'the observer fired before the lock calls ended', + ); + await new Promise((r) => setTimeout(r, 30)); + assert.equal( + returned, + false, + 'the worker waits for the cooldown to be written', + ); + openGate(); + await working; + assert.equal(released, true); + }); + + test('a dead task of a Resizer unknown in this process takes no lock', async () => { + const locks = expiringLocks({ ms: 0 }); + const t = setup(locks, { maxAttempts: 1 }); + await t.tasks.add({ + resizer: 'ghost', + queue: 'default', + mediaId: 'm1', + pipeline: 'default', + previews: [{ sizeKey: '10x10', format: 'webp' }], + requestKey: 'ghost-request', + }); + const working = t.work(); + while (t.tasks.rows[0].status !== 'dead') { + await new Promise((r) => setTimeout(r, 5)); + } + t.stop.abort(); + await working; + assert.deepEqual(locks.calls, []); + assert.ok( + t.errors.some((args) => /unknown Resizer 'ghost'/.test(String(args[0]))), + ); + }); +}); diff --git a/src/worker.ts b/src/worker.ts index 0eb246d..68135bd 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -1,9 +1,9 @@ -// The worker (07 · Worker §11), framework-free. `runWorker()` serves every registered Resizer for -// ONE named queue: it runs the core queue loop (src/queue.ts) once per distinct TaskQueue the -// Resizers use, after verifying every Resizer. Each task runs with the Resizer named in it, and its -// events go to that Resizer's observers. Framework hosts start it through `runResizeWorker()` -// (src/framework/worker.ts), which adds the `worker.enabled` switch, process signals and the app -// logger. +// The worker, framework-free. `runWorker()` serves every registered Resizer for ONE named queue: it +// runs the core queue loop (src/queue.ts) once per distinct TaskQueue the Resizers use, after +// verifying every Resizer. Each task runs with the Resizer named in it, and its events go to that +// Resizer's observers. After a dead-letter it holds the task's dispatch locks for a cooldown. +// Framework hosts start it through `runResizeWorker()` (src/framework/worker.ts), which adds the +// `worker.enabled` switch, process signals and the app logger. import sharp from 'sharp'; import type { LeasedTask, @@ -11,9 +11,16 @@ import type { TaskEventHandler, TaskQueue, } from './contracts/taskQueue.ts'; +import { canonicalizeVariants, dispatchLockKey } from './enqueue.ts'; import { ResizeSetupError } from './errors.ts'; -import { consumeQueue } from './queue.ts'; -import { getResizer, listResizers, type ObserverName } from './resizer.ts'; +import { getPreviewIdentity } from './images.ts'; +import { consumeQueue, timingOf } from './queue.ts'; +import { + getResizer, + listResizers, + type ObserverName, + type Resizer, +} from './resizer.ts'; import { processTaskWith } from './resizeTask.ts'; import type { ResizeLogger } from './types.d.ts'; @@ -37,6 +44,56 @@ export async function processTask( return processTaskWith(resizer, task, taskOpts, tasks ?? resizer.tasks); } +/** + * After a dead-letter, hold the dispatch lock of every variant in the task for the delivering + * queue's lockTtlMs.failed, so reads (resolve, prewarm) do not queue the same failing work again + * until it ends. Release first, then acquire: the read path's shorter dispatch lock may still be + * held. Best effort and never rejects: a failing lock call is logged, and a read that takes the + * lock between the two calls simply queues the work once more. + */ +async function holdFailedCooldown( + resizer: Resizer, + task: LeasedTask, + tasks: TaskQueue, + logger: ResizeLogger, +): Promise { + try { + const ttlMs = timingOf(tasks).lockTtlMs.failed; + const scope = { resizer: resizer.name, pipeline: task.pipeline }; + const keys = new Set( + canonicalizeVariants(task.previews).map((variant) => + dispatchLockKey( + task.mediaId, + getPreviewIdentity( + scope, + variant.sizeKey, + variant.format, + variant.filters, + ), + ), + ), + ); + await Promise.all( + [...keys].map(async (key) => { + try { + await resizer.db.releaseLock(key); + await resizer.db.acquireLock(key, ttlMs); + } catch (err) { + logger.error( + `resize worker: holding the dead-letter cooldown lock ${key} failed`, + err, + ); + } + }), + ); + } catch (err) { + logger.error( + `resize worker: the dead-letter cooldown of task ${task.taskId} failed`, + err, + ); + } +} + export interface RunWorkerOptions { queue?: string; // queue to consume; default 'default' signal: AbortSignal; // stops the worker: finish in-flight tasks, then return @@ -92,21 +149,31 @@ export async function runWorker(opts: RunWorkerOptions): Promise { await resizer.verify(); } + // Dead-letter cooldowns run beside the loop, so they never delay it or the observers; the worker + // waits for the pending ones before it returns. + const cooldowns = new Set>(); // Events are routed by the task's Resizer name, looked up when the event arrives. - const onEvent: TaskEventHandler = async (event, task, error) => { - const owner = listResizers().find((r) => r.name === task.resizer); - if (!owner) { - logger.error( - `resize worker: ${event} for task ${task.taskId} of unknown Resizer '${task.resizer}'`, - ); - return; - } - if (event === 'completed') { - await owner.runObservers(OBSERVER[event], task, {}); - } else { - await owner.runObservers(OBSERVER[event], task, error, {}); - } - }; + const eventsOf = + (tasks: TaskQueue): TaskEventHandler => + async (event, task, error) => { + const owner = listResizers().find((r) => r.name === task.resizer); + if (!owner) { + logger.error( + `resize worker: ${event} for task ${task.taskId} of unknown Resizer '${task.resizer}'`, + ); + return; + } + if (event === 'deadLettered') { + const cooldown = holdFailedCooldown(owner, task, tasks, logger); + cooldowns.add(cooldown); + void cooldown.then(() => cooldowns.delete(cooldown)); + } + if (event === 'completed') { + await owner.runObservers(OBSERVER[event], task, {}); + } else { + await owner.runObservers(OBSERVER[event], task, error, {}); + } + }; if (opts.sharp) { sharp.concurrency(opts.sharp.concurrency); @@ -130,7 +197,7 @@ export async function runWorker(opts: RunWorkerOptions): Promise { queue, signal: stop.signal, handle: (task, taskOpts) => processTask(task, taskOpts, tasks), - onEvent, + onEvent: eventsOf(tasks), logger, }); } catch (err) { @@ -142,6 +209,7 @@ export async function runWorker(opts: RunWorkerOptions): Promise { } finally { opts.signal.removeEventListener('abort', onAbort); } + await Promise.all(cooldowns); if (firstFailure) { throw firstFailure.error; }