From a508bd52e9fde7e60db8517c8e2f26e528482e6c Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Fri, 16 Oct 2026 19:24:05 +0200 Subject: [PATCH 01/32] feat(datasource-dynamodb): add a DynamoDB datasource DynamoDB reads fast by key and scans otherwise, and it cannot sort, count or aggregate server-side. The datasource picks the cheapest access path for each request (Query on the table or an index, BatchGetItem for whole keys, parallel Scan otherwise) and does the rest on the items it reads: operators DynamoDB lacks, sorting on non-key attributes, aggregations and charts. It introspects tables or takes a manual schema, exposes the partition and sort keys as a composite primary key, shares one table between collections (single-table design), exposes arrays of maps as collections, and runs PartiQL SELECT native queries. `scanPolicy` and `maxScannedItems` keep a request that no key serves from reading a whole table unnoticed. The unit tests use a fake client; an integration suite runs against DynamoDB Local, started by the CI like the other datasources' databases. Refs: PRD-1670 Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/build.yml | 5 +- CLAUDE.md | 1 + packages/datasource-dynamodb/LICENSE | 202 +++++ packages/datasource-dynamodb/README.md | 172 +++++ .../datasource-dynamodb/docker-compose.yml | 8 + packages/datasource-dynamodb/jest.config.ts | 8 + packages/datasource-dynamodb/package.json | 36 + .../src/array-collection.ts | 326 ++++++++ packages/datasource-dynamodb/src/client.ts | 36 + .../datasource-dynamodb/src/collection.ts | 698 ++++++++++++++++++ .../datasource-dynamodb/src/datasource.ts | 94 +++ packages/datasource-dynamodb/src/errors.ts | 19 + packages/datasource-dynamodb/src/index.ts | 150 ++++ .../src/introspection/builder.ts | 61 ++ .../src/introspection/introspector.ts | 290 ++++++++ .../src/schema/manual-schema.ts | 222 ++++++ .../src/schema/schema-converter.ts | 51 ++ packages/datasource-dynamodb/src/settings.ts | 25 + packages/datasource-dynamodb/src/types.ts | 163 ++++ .../src/utils/access-path.ts | 299 ++++++++ .../src/utils/aggregator.ts | 174 +++++ .../src/utils/concurrency.ts | 35 + .../datasource-dynamodb/src/utils/executor.ts | 260 +++++++ .../src/utils/expression-builder.ts | 246 ++++++ .../src/utils/pagination-cache.ts | 97 +++ .../src/utils/serializer.ts | 117 +++ .../datasource-dynamodb/src/utils/sort.ts | 20 + .../src/utils/type-converter.ts | 106 +++ .../src/virtual-collections.ts | 76 ++ .../test/array-collection.test.ts | 312 ++++++++ .../test/collection-helpers.ts | 26 + .../test/collection.aggregate.test.ts | 462 ++++++++++++ .../test/collection.list.test.ts | 604 +++++++++++++++ .../test/collection.schema.test.ts | 108 +++ .../test/collection.write.test.ts | 343 +++++++++ .../test/datasource.test.ts | 248 +++++++ .../datasource-dynamodb/test/fake-client.ts | 42 ++ packages/datasource-dynamodb/test/helpers.ts | 47 ++ .../test/integration/dynamodb-local.test.ts | 503 +++++++++++++ .../test/introspection/introspector.test.ts | 377 ++++++++++ .../test/schema/manual-schema.test.ts | 217 ++++++ .../test/utils/access-path.test.ts | 294 ++++++++ .../test/utils/aggregator.test.ts | 196 +++++ .../test/utils/expression-builder.test.ts | 292 ++++++++ .../test/utils/pagination-cache.test.ts | 74 ++ .../test/utils/serializer.test.ts | 72 ++ .../test/utils/type-converter.test.ts | 111 +++ .../test/virtual-collections.test.ts | 102 +++ .../datasource-dynamodb/tsconfig.eslint.json | 3 + packages/datasource-dynamodb/tsconfig.json | 7 + yarn.lock | 259 +++++++ 51 files changed, 8695 insertions(+), 1 deletion(-) create mode 100644 packages/datasource-dynamodb/LICENSE create mode 100644 packages/datasource-dynamodb/README.md create mode 100644 packages/datasource-dynamodb/docker-compose.yml create mode 100644 packages/datasource-dynamodb/jest.config.ts create mode 100644 packages/datasource-dynamodb/package.json create mode 100644 packages/datasource-dynamodb/src/array-collection.ts create mode 100644 packages/datasource-dynamodb/src/client.ts create mode 100644 packages/datasource-dynamodb/src/collection.ts create mode 100644 packages/datasource-dynamodb/src/datasource.ts create mode 100644 packages/datasource-dynamodb/src/errors.ts create mode 100644 packages/datasource-dynamodb/src/index.ts create mode 100644 packages/datasource-dynamodb/src/introspection/builder.ts create mode 100644 packages/datasource-dynamodb/src/introspection/introspector.ts create mode 100644 packages/datasource-dynamodb/src/schema/manual-schema.ts create mode 100644 packages/datasource-dynamodb/src/schema/schema-converter.ts create mode 100644 packages/datasource-dynamodb/src/settings.ts create mode 100644 packages/datasource-dynamodb/src/types.ts create mode 100644 packages/datasource-dynamodb/src/utils/access-path.ts create mode 100644 packages/datasource-dynamodb/src/utils/aggregator.ts create mode 100644 packages/datasource-dynamodb/src/utils/concurrency.ts create mode 100644 packages/datasource-dynamodb/src/utils/executor.ts create mode 100644 packages/datasource-dynamodb/src/utils/expression-builder.ts create mode 100644 packages/datasource-dynamodb/src/utils/pagination-cache.ts create mode 100644 packages/datasource-dynamodb/src/utils/serializer.ts create mode 100644 packages/datasource-dynamodb/src/utils/sort.ts create mode 100644 packages/datasource-dynamodb/src/utils/type-converter.ts create mode 100644 packages/datasource-dynamodb/src/virtual-collections.ts create mode 100644 packages/datasource-dynamodb/test/array-collection.test.ts create mode 100644 packages/datasource-dynamodb/test/collection-helpers.ts create mode 100644 packages/datasource-dynamodb/test/collection.aggregate.test.ts create mode 100644 packages/datasource-dynamodb/test/collection.list.test.ts create mode 100644 packages/datasource-dynamodb/test/collection.schema.test.ts create mode 100644 packages/datasource-dynamodb/test/collection.write.test.ts create mode 100644 packages/datasource-dynamodb/test/datasource.test.ts create mode 100644 packages/datasource-dynamodb/test/fake-client.ts create mode 100644 packages/datasource-dynamodb/test/helpers.ts create mode 100644 packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts create mode 100644 packages/datasource-dynamodb/test/introspection/introspector.test.ts create mode 100644 packages/datasource-dynamodb/test/schema/manual-schema.test.ts create mode 100644 packages/datasource-dynamodb/test/utils/access-path.test.ts create mode 100644 packages/datasource-dynamodb/test/utils/aggregator.test.ts create mode 100644 packages/datasource-dynamodb/test/utils/expression-builder.test.ts create mode 100644 packages/datasource-dynamodb/test/utils/pagination-cache.test.ts create mode 100644 packages/datasource-dynamodb/test/utils/serializer.test.ts create mode 100644 packages/datasource-dynamodb/test/utils/type-converter.test.ts create mode 100644 packages/datasource-dynamodb/test/virtual-collections.test.ts create mode 100644 packages/datasource-dynamodb/tsconfig.eslint.json create mode 100644 packages/datasource-dynamodb/tsconfig.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7d3bf6a6b7..77e2591ff0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -72,6 +72,7 @@ jobs: - datasource-customizer - datasource-demo-fintech - datasource-dummy + - datasource-dynamodb - datasource-mongoose - datasource-replica - datasource-mongo @@ -90,7 +91,7 @@ jobs: with: node-version-file: ".nvmrc" - name: Start docker containers - if: ${{ matrix.package == 'datasource-mongo' || matrix.package == 'datasource-mongoose' || matrix.package == 'datasource-sql' || matrix.package == 'datasource-sequelize' }} + if: ${{ matrix.package == 'datasource-mongo' || matrix.package == 'datasource-mongoose' || matrix.package == 'datasource-sql' || matrix.package == 'datasource-sequelize' || matrix.package == 'datasource-dynamodb' }} run: docker compose -f ./packages/${{ matrix.package }}/docker-compose.yml up -d; sleep 5 - name: Restore dependencies from cache uses: actions/cache/restore@v4 @@ -112,6 +113,8 @@ jobs: # broadly: most packages keep genuine unit tests under `test/integration*/`, and a wide # pattern would silently stop running hundreds of them. - name: Test code (excluding integration suites with their own job) + env: + DYNAMODB_ENDPOINT: ${{ matrix.package == 'datasource-dynamodb' && 'http://localhost:8000' || '' }} run: cd packages/${{ matrix.package }} && yarn test --coverage --testPathIgnorePatterns='llm.integration|bedrock.integration|embedded-bff.e2e' && cd - - name: Upload coverage uses: actions/upload-artifact@v4 diff --git a/CLAUDE.md b/CLAUDE.md index 91a18bc97b..398139d614 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,6 +34,7 @@ yarn test:coverage # Run tests with coverage - `datasource-sequelize` - Sequelize datasource - `datasource-mongo` - MongoDB datasource - `datasource-mongoose` - Mongoose datasource +- `datasource-dynamodb` - DynamoDB datasource **Other:** - `mcp-server` - MCP (Model Context Protocol) server for AI integration diff --git a/packages/datasource-dynamodb/LICENSE b/packages/datasource-dynamodb/LICENSE new file mode 100644 index 0000000000..acb12af3ef --- /dev/null +++ b/packages/datasource-dynamodb/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2021-2026 Forest Admin + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/datasource-dynamodb/README.md b/packages/datasource-dynamodb/README.md new file mode 100644 index 0000000000..5c7917341c --- /dev/null +++ b/packages/datasource-dynamodb/README.md @@ -0,0 +1,172 @@ +# @forestadmin/datasource-dynamodb + +Forest Admin datasource for Amazon DynamoDB: introspection or manual schema, composite keys, +relations, search, aggregations and charts, single-table design. + +```bash +yarn add @forestadmin/datasource-dynamodb @aws-sdk/client-dynamodb @aws-sdk/lib-dynamodb +``` + +```ts +import { createAgent } from '@forestadmin/agent'; +import { createDynamoDBDataSource } from '@forestadmin/datasource-dynamodb'; + +createAgent({ authSecret, envSecret, isProduction }) + // Every table of the account and region, introspected + .addDataSource(createDynamoDBDataSource({ region: 'eu-west-1' })); +``` + +Credentials come from the usual AWS chain (environment, profile, role). Pass `credentials`, +`endpoint` (DynamoDB Local) or `clientConfig` to override them, or `client` to reuse a +`DynamoDBClient` or `DynamoDBDocumentClient` you already have. + +## Why DynamoDB is not like the other datasources + +DynamoDB reads fast by key and scans otherwise, and it cannot sort, count or aggregate on the +server. The datasource uses the key whenever a filter allows it and does the rest on the items it +reads. What that means for each feature: + +| Feature | How it works | +| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Filters | Pushed down when DynamoDB can evaluate them. `Like`, `ILike`, `IContains`, `EndsWith`, date intervals and the like are applied on the returned items, so they read more than they return. | +| Access path | A `Query` on the table, a global or local secondary index when an equality pins its partition key (with the sort key range when there is one), a `BatchGetItem` when the filter names whole keys, a `Scan` otherwise. | +| Sort | Free on the sort key of the table or index being read. On any other attribute, the matching items are read, then sorted in memory. | +| Pagination | There is no offset: the datasource remembers the key where earlier pages stopped and resumes from the closest one. `maxOffset` (100 000) caps how deep a page can be. | +| Count and charts | Computed by reading the items (a counting `Scan`/`Query` for a plain count), in parallel segments for a table. Results are kept in memory for `chartCacheTtlMs` (30 s). | +| Search | Evaluated on the items: it scans. Case-insensitive, accents kept. | +| Relations | None in DynamoDB. Declare them with `addManyToOneRelation` and friends: a lookup by key is one `BatchGetItem`, a relation backed by a secondary index is a `Query`. | + +## Controlling cost: `scanPolicy` + +A request that no key or index serves reads the whole table. `scanPolicy` decides what happens: + +| Value | Behaviour | +| ------------------ | ------------------------------------------------------------------------------------ | +| `'warn'` (default) | Scans and logs a warning naming the table and the filtered fields, never the values. | +| `'allow'` | Scans silently. | +| `'deny'` | Refuses with an explicit error. | + +`maxScannedItems` (1 000 000) stops any single request that reads more items than that, with an error +rather than a partial result. Both can be set on the datasource or on a collection. + +## Choosing the tables and naming collections + +```ts +createDynamoDBDataSource({ + region: 'eu-west-1', + builder: configurator => + configurator + .addCollectionFromTable({ name: 'customers', tableName: 'Customers', sampleSize: 200 }) + .addCollectionFromTable({ name: 'orders', tableName: 'Orders', scanPolicy: 'deny' }), +}); +``` + +`tables: ['Customers', 'Orders']` keeps the default naming (collections are named after tables) and +only introspects the listed tables. + +## Introspection + +`DescribeTable` gives the keys and the indexes. The other attributes are typed from a sample of +`sampleSize` items (default 100): DynamoDB does not type them. A scan has no order, so the sample is +not the latest items. Nested maps become `address->city` columns up to `maxDepth` (5); an attribute +that is absent or null in part of the sample is nullable; mixed types become Json. + +## Manual schema + +```ts +createDynamoDBDataSource({ + region: 'eu-west-1', + disableIntrospection: true, + schema: { + collections: [ + { + name: 'orders', + tableName: 'Orders', + partitionKey: { name: 'customerId', type: 'string' }, + sortKey: { name: 'orderId', type: 'string' }, + indexes: [ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'string' }, + sortKey: { name: 'createdAt', type: 'string' }, + }, + ], + fields: [ + { name: 'total', type: 'number' }, + { name: 'status', type: 'string', nullable: true }, + { name: 'address', type: 'object', fields: [{ name: 'city', type: 'string' }] }, + ], + }, + ], + }, +}); +``` + +Types: `string`, `number`, `boolean`, `date` and `dateonly` (ISO 8601 strings), `timeonly`, `object`, +`array`, `point` (GeoJSON), `binary` (base64). Keys are strings or numbers. + +## Composite keys + +The partition key and the sort key are both primary key columns of the collection, so Forest +identifies a record by the two. A record created without a string partition key gets a UUID. +Keys cannot be updated. + +## Single-table design + +Several collections can share a table, each restricted to its items: + +```ts +configurator + .addCollectionFromTable({ + name: 'users', + tableName: 'App', + entity: { sortKeyPrefix: 'USER#' }, // or { attribute: 'entityType', value: 'USER' } + }) + .addCollectionFromTable({ + name: 'orders', + tableName: 'App', + entity: { sortKeyPrefix: 'ORDER#' }, + }); +``` + +A sort key prefix is also used as the sort key condition of queries, and checked on create. + +## Arrays as collections + +```ts +createDynamoDBDataSource({ + virtualCollections: [ + { parentCollection: 'orders', collectionName: 'order_lines', arrayField: 'lines' }, + ], +}); +``` + +Each element of the array becomes a record, identified by the key of its parent and its `index` in +the array. Creating appends, updating sets the element's attributes, and deleting removes it, so the +index of the following elements shifts. + +## Native queries + +`liveQueryConnection: 'dynamo'` offers PartiQL `SELECT` statements as native queries, with `$name` +placeholders: `SELECT * FROM "Orders" WHERE customerId = $customer`. PartiQL cannot aggregate, so +charts are built with the collection's aggregations instead. + +## Observability + +`logQueries` logs the expressions sent to DynamoDB, `logConsumedCapacity` the capacity units each +request consumed. Throttled requests are retried by the AWS SDK (`adaptive`, 10 attempts). + +## Not supported yet + +Collections backed by a secondary index, optimistic updates, binary keys, exact counts without +reading the items. + +## Testing + +```bash +docker compose up -d +DYNAMODB_ENDPOINT=http://localhost:8000 yarn test +``` + +Without `DYNAMODB_ENDPOINT` the integration suite against DynamoDB Local is skipped. diff --git a/packages/datasource-dynamodb/docker-compose.yml b/packages/datasource-dynamodb/docker-compose.yml new file mode 100644 index 0000000000..873d974da8 --- /dev/null +++ b/packages/datasource-dynamodb/docker-compose.yml @@ -0,0 +1,8 @@ +version: '3.1' +services: + dynamodb: + image: amazon/dynamodb-local + container_name: forest_datasource_dynamodb_test + command: -jar DynamoDBLocal.jar -inMemory -sharedDb + ports: + - 8000:8000 diff --git a/packages/datasource-dynamodb/jest.config.ts b/packages/datasource-dynamodb/jest.config.ts new file mode 100644 index 0000000000..d622773e8a --- /dev/null +++ b/packages/datasource-dynamodb/jest.config.ts @@ -0,0 +1,8 @@ +/* eslint-disable import/no-relative-packages */ +import jestConfig from '../../jest.config'; + +export default { + ...jestConfig, + collectCoverageFrom: ['/src/**/*.ts'], + testMatch: ['/test/**/*.test.ts'], +}; diff --git a/packages/datasource-dynamodb/package.json b/packages/datasource-dynamodb/package.json new file mode 100644 index 0000000000..4a01cf86de --- /dev/null +++ b/packages/datasource-dynamodb/package.json @@ -0,0 +1,36 @@ +{ + "name": "@forestadmin/datasource-dynamodb", + "version": "0.0.0", + "main": "dist/index.js", + "license": "Apache-2.0", + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/ForestAdmin/agent-nodejs.git", + "directory": "packages/datasource-dynamodb" + }, + "dependencies": { + "@forestadmin/datasource-toolkit": "1.56.1" + }, + "peerDependencies": { + "@aws-sdk/client-dynamodb": "^3.750.0", + "@aws-sdk/lib-dynamodb": "^3.750.0" + }, + "devDependencies": { + "@aws-sdk/client-dynamodb": "^3.750.0", + "@aws-sdk/lib-dynamodb": "^3.750.0" + }, + "files": [ + "dist/**/*.js", + "dist/**/*.d.ts" + ], + "scripts": { + "build": "tsc", + "build:watch": "tsc --watch", + "clean": "rm -rf coverage dist", + "lint": "eslint src test", + "test": "jest" + } +} diff --git a/packages/datasource-dynamodb/src/array-collection.ts b/packages/datasource-dynamodb/src/array-collection.ts new file mode 100644 index 0000000000..80a0fa80b4 --- /dev/null +++ b/packages/datasource-dynamodb/src/array-collection.ts @@ -0,0 +1,326 @@ +import type DynamoDBCollection from './collection'; +import type { AttributeSchema, TableModel } from './types'; +import type { + AggregateResult, + Aggregation, + Caller, + ColumnSchema, + ConditionTree, + DataSource, + Filter, + PaginatedFilter, + RecordData, +} from '@forestadmin/datasource-toolkit'; + +import { + BaseCollection, + ConditionTreeBranch, + ConditionTreeFactory, + ConditionTreeLeaf, + PaginatedFilter as PaginatedFilterClass, + Projection, + ValidationError, +} from '@forestadmin/datasource-toolkit'; + +import convertModelToFields from './schema/schema-converter'; +import StreamingAggregator from './utils/aggregator'; +import ExpressionBuilder from './utils/expression-builder'; +import Serializer from './utils/serializer'; +import { compareRecords } from './utils/sort'; + +export const INDEX_FIELD = 'index'; + +export type ArrayCollectionConfig = { + name: string; + /** Attribute of the parent holding the array, `a->b` for a nested one. */ + arrayField: string; + /** Attributes of an item, by flattened name. */ + itemSchema: Record; +}; + +type Item = Record; + +function conjuncts(tree?: ConditionTree): ConditionTree[] { + if (!tree) return []; + + if (tree instanceof ConditionTreeBranch && tree.aggregator === 'And') { + return tree.conditions.flatMap(condition => conjuncts(condition)); + } + + return [tree]; +} + +/** + * Exposes an array of maps stored in the items of another collection as a collection of its own. + * A record is identified by the key of its parent and its position in the array, which is why + * those positions move when an earlier item of the same array is removed. + */ +export default class ArrayCollection extends BaseCollection { + private readonly parent: DynamoDBCollection; + private readonly arrayField: string; + private readonly parentKeys: string[]; + private readonly itemFields: Set; + + constructor(datasource: DataSource, parent: DynamoDBCollection, config: ArrayCollectionConfig) { + super(config.name, datasource, parent.nativeDriver); + + this.parent = parent; + this.arrayField = config.arrayField; + + const { partitionKey, sortKey } = parent.model; + const keys = [partitionKey, sortKey].filter(Boolean); + this.parentKeys = keys.map(key => key.name); + + const schema: Record = { + ...config.itemSchema, + ...Object.fromEntries( + keys.map(key => [key.name, { type: key.type === 'N' ? 'number' : 'string' }]), + ), + [INDEX_FIELD]: { type: 'number' }, + }; + const model: TableModel = { + ...parent.model, + name: config.name, + indexes: [], + schema, + overrideTypeConverter: undefined, + }; + const fields = convertModelToFields(model); + + for (const name of [...this.parentKeys, INDEX_FIELD]) { + Object.assign(fields[name], { isPrimaryKey: true, isReadOnly: true }); + } + + this.addFields(fields); + this.enableCount(); + + this.itemFields = new Set(Object.keys(config.itemSchema)); + } + + async create(caller: Caller, data: RecordData[]): Promise { + const created: RecordData[] = []; + + // Appended one after the other: each append reads the length the previous one left. + for (const record of data) { + // eslint-disable-next-line no-await-in-loop + created.push(await this.append(record)); + } + + this.parent.invalidateCaches(); + + return created; + } + + async list( + caller: Caller, + filter: PaginatedFilter, + projection: Projection, + ): Promise { + const records = await this.readRecords(caller, filter.conditionTree); + const sort = filter.sort?.length ? filter.sort : undefined; + + if (sort) records.sort((a, b) => compareRecords(a, b, sort)); + + const skip = filter.page?.skip ?? 0; + const limit = filter.page?.limit || undefined; + const columns = projection.columns; + + return records + .slice(skip, limit === undefined ? undefined : skip + limit) + .map(record => + Object.fromEntries(columns.filter(field => field in record).map(f => [f, record[f]])), + ); + } + + async update(caller: Caller, filter: Filter, patch: RecordData): Promise { + for (const field of Object.keys(patch)) { + if (field === INDEX_FIELD || this.parentKeys.includes(field)) { + throw new ValidationError(`The attribute '${field}' of '${this.name}' cannot be updated`); + } + } + + const entries = Object.entries(patch).filter(([, value]) => value !== undefined); + + if (entries.length === 0) return; + + await this.writePerParent(caller, filter, (builder, indexes) => { + const values = entries.map(([, value]) => builder.value(value)); + const array = builder.attribute(this.arrayField); + + return { + UpdateExpression: `SET ${indexes + .flatMap(index => + entries.map( + ([field], position) => + `${array}[${index}].${builder.attribute(field)} = ${values[position]}`, + ), + ) + .join(', ')}`, + }; + }); + } + + async delete(caller: Caller, filter: Filter): Promise { + await this.writePerParent(caller, filter, (builder, indexes) => { + const array = builder.attribute(this.arrayField); + const descending = [...indexes].sort((a, b) => b - a); + + return { UpdateExpression: `REMOVE ${descending.map(i => `${array}[${i}]`).join(', ')}` }; + }); + } + + async aggregate( + caller: Caller, + filter: Filter, + aggregation: Aggregation, + limit?: number, + ): Promise { + const aggregator = new StreamingAggregator(aggregation, caller.timezone); + + for (const record of await this.readRecords(caller, filter.conditionTree)) { + aggregator.add(record); + } + + return aggregator.result(limit); + } + + /** Reads the parents, then turns each array item into a record and applies the filter to it. */ + private async readRecords(caller: Caller, tree?: ConditionTree): Promise { + // Conditions on the parent key narrow the parents read; the rest applies to the items. + const onParentKey = conjuncts(tree).filter( + condition => + condition instanceof ConditionTreeLeaf && this.parentKeys.includes(condition.field), + ); + const parents = await this.parent.list( + caller, + new PaginatedFilterClass({ + conditionTree: onParentKey.length + ? ConditionTreeFactory.intersect(...onParentKey) + : undefined, + }), + new Projection(...this.parentKeys, this.arrayField), + ); + + const records: RecordData[] = []; + + for (const parent of parents) { + const items = parent[this.arrayField]; + + if (Array.isArray(items)) { + items.forEach((item, index) => { + if (item && typeof item === 'object') { + records.push({ + ...Object.fromEntries(this.parentKeys.map(key => [key, parent[key]])), + [INDEX_FIELD]: index, + ...Serializer.flatten(item as Item, this.itemFields), + }); + } + }); + } + } + + return tree ? records.filter(record => tree.match(record, this, caller.timezone)) : records; + } + + private async append(record: RecordData): Promise { + const key: Item = {}; + + for (const name of this.parentKeys) { + if (record[name] === undefined || record[name] === null) { + throw new ValidationError( + `The key attribute '${name}' of the parent '${this.parent.name}' is required to create ` + + `a record in '${this.name}'`, + ); + } + + key[name] = record[name]; + } + + const fields = Object.fromEntries( + Object.entries(record).filter( + ([name]) => !this.parentKeys.includes(name) && name !== INDEX_FIELD, + ), + ); + const builder = new ExpressionBuilder(); + const keyBuilder = new ExpressionBuilder('k', 'kv'); + const arrayAlias = builder.attribute(this.arrayField); + const emptyList = builder.value([]); + const appended = builder.value([Serializer.unflatten(fields)]); + const expression = `SET ${arrayAlias} = list_append(if_not_exists(${arrayAlias}, ${emptyList}), ${appended})`; + + let attributes: Item; + + try { + attributes = await this.parent.executor.update({ + Key: key, + UpdateExpression: expression, + ConditionExpression: `attribute_exists(${keyBuilder.attribute(this.parentKeys[0])})`, + ExpressionAttributeNames: { ...builder.names, ...keyBuilder.names }, + ExpressionAttributeValues: builder.values, + ReturnValues: 'UPDATED_NEW', + }); + } catch (error) { + if (error.name === 'ConditionalCheckFailedException') { + throw new ValidationError( + `The parent record of '${this.name}' does not exist in '${this.parent.name}'`, + ); + } + + throw error; + } + + const array = Serializer.toPathSegments(this.arrayField).reduce( + (value, segment) => (value as Item)?.[segment], + attributes, + ) as unknown[]; + + return { ...key, [INDEX_FIELD]: array.length - 1, ...fields }; + } + + /** Finds the matching items, then writes once per parent whatever the items need. */ + private async writePerParent( + caller: Caller, + filter: Filter, + build: (builder: ExpressionBuilder, indexes: number[]) => { UpdateExpression: string }, + ): Promise { + const records = await this.list( + caller, + new PaginatedFilterClass({ conditionTree: filter.conditionTree }), + new Projection(...this.parentKeys, INDEX_FIELD), + ); + const perParent = new Map(); + + for (const record of records) { + const key = Object.fromEntries(this.parentKeys.map(name => [name, record[name]])); + const id = JSON.stringify(key); + const entry = perParent.get(id) ?? { key, indexes: [] }; + entry.indexes.push(record[INDEX_FIELD] as number); + perParent.set(id, entry); + } + + await this.parent.executor.mapWrites([...perParent.values()], async ({ key, indexes }) => { + const builder = new ExpressionBuilder(); + const keyBuilder = new ExpressionBuilder('k', 'kv'); + const { UpdateExpression } = build( + builder, + [...indexes].sort((a, b) => a - b), + ); + + try { + await this.parent.executor.update({ + Key: key, + UpdateExpression, + ConditionExpression: `attribute_exists(${keyBuilder.attribute(this.parentKeys[0])})`, + ExpressionAttributeNames: { ...builder.names, ...keyBuilder.names }, + ...(Object.keys(builder.values).length + ? { ExpressionAttributeValues: builder.values } + : {}), + }); + } catch (error) { + if (error.name !== 'ConditionalCheckFailedException') throw error; + } + }); + + this.parent.invalidateCaches(); + } +} diff --git a/packages/datasource-dynamodb/src/client.ts b/packages/datasource-dynamodb/src/client.ts new file mode 100644 index 0000000000..fce7d4c108 --- /dev/null +++ b/packages/datasource-dynamodb/src/client.ts @@ -0,0 +1,36 @@ +import type { DynamoDBClientOptions } from './types'; + +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; + +/** The AWS SDK default is 3 attempts: throttling on a busy table is worth more patience. */ +const DEFAULT_MAX_ATTEMPTS = 10; + +const TRANSLATE_CONFIG = { + marshallOptions: { removeUndefinedValues: true }, + unmarshallOptions: { wrapNumbers: false }, +}; + +function isLowLevelClient(client: unknown): client is DynamoDBClient { + return client instanceof DynamoDBClient || client?.constructor?.name === 'DynamoDBClient'; +} + +export function createDocumentClient(options: DynamoDBClientOptions): DynamoDBDocumentClient { + if (options.client) { + return isLowLevelClient(options.client) + ? DynamoDBDocumentClient.from(options.client, TRANSLATE_CONFIG) + : (options.client as DynamoDBDocumentClient); + } + + return DynamoDBDocumentClient.from( + new DynamoDBClient({ + maxAttempts: DEFAULT_MAX_ATTEMPTS, + retryMode: 'adaptive', + region: options.region, + endpoint: options.endpoint, + credentials: options.credentials, + ...options.clientConfig, + }), + TRANSLATE_CONFIG, + ); +} diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts new file mode 100644 index 0000000000..704a5af50d --- /dev/null +++ b/packages/datasource-dynamodb/src/collection.ts @@ -0,0 +1,698 @@ +import type { CollectionSettings, TableModel } from './types'; +import type { AccessPath, KeyLookup } from './utils/access-path'; +import type { Item, ReadInput, ReadOperation, ReadBudget } from './utils/executor'; +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import type { + AggregateResult, + Aggregation, + Caller, + Collection, + ConditionTree, + ConditionTreeLeaf, + DataSource, + Filter, + Logger, + PaginatedFilter, + Projection, + RecordData, + Sort, +} from '@forestadmin/datasource-toolkit'; + +import { + BaseCollection, + ConditionTreeFactory, + PaginatedFilter as PaginatedFilterClass, + Projection as ProjectionClass, + ValidationError, +} from '@forestadmin/datasource-toolkit'; +import { randomUUID } from 'crypto'; + +import { DynamoDBOperationError, OffsetTooDeepError, ScanDeniedError } from './errors'; +import convertModelToFields from './schema/schema-converter'; +import { planAccessPath, planKeyLookup } from './utils/access-path'; +import StreamingAggregator from './utils/aggregator'; +import DynamoExecutor from './utils/executor'; +import ExpressionBuilder, { splitCondition } from './utils/expression-builder'; +import Serializer from './utils/serializer'; +import { compareRecords } from './utils/sort'; + +type ReadPlan = { lookup?: KeyLookup; access?: AccessPath; residual?: ConditionTree }; + +function intersect(...trees: ConditionTree[]): ConditionTree | undefined { + const present = trees.filter(Boolean); + + return present.length ? ConditionTreeFactory.intersect(...present) : undefined; +} + +/** Values a single expression may carry before it is evaluated on the returned items instead. */ +const MAX_PUSHED_LIST_VALUES = 200; +const LIST_OPERATORS = ['In', 'NotIn', 'IncludesAll', 'IncludesNone']; +/** Charts remembered per collection, the oldest dropped first. */ +const MAX_CACHED_CHARTS = 100; + +const hasOversizedList = (leaf: ConditionTreeLeaf): boolean => + LIST_OPERATORS.includes(leaf.operator) && + Array.isArray(leaf.value) && + leaf.value.length > MAX_PUSHED_LIST_VALUES; + +const isEmptyKey = (value: unknown): boolean => + value === undefined || value === null || value === ''; + +function describeFilter(tree?: ConditionTree): string { + const parts = new Set(); + tree?.forEachLeaf(leaf => parts.add(`${leaf.field} ${leaf.operator}`)); + + return [...parts].join(', '); +} + +function keyConditionExpression( + builder: ExpressionBuilder, + { partition, sort }: AccessPath['keyCondition'], +): string { + const expressions = [ + `${builder.attribute(partition.attribute)} = ${builder.value(partition.value)}`, + ]; + + if (sort) { + const attribute = builder.attribute(sort.attribute); + const comparisons = { + Equal: '=', + LessThan: '<', + LessThanOrEqual: '<=', + GreaterThan: '>', + GreaterThanOrEqual: '>=', + }; + + if (sort.operator === 'StartsWith') { + expressions.push(`begins_with(${attribute}, ${builder.value(sort.value)})`); + } else if (sort.operator === 'Between') { + const [lower, upper] = sort.value as unknown[]; + expressions.push(`${attribute} BETWEEN ${builder.value(lower)} AND ${builder.value(upper)}`); + } else { + expressions.push(`${attribute} ${comparisons[sort.operator]} ${builder.value(sort.value)}`); + } + } + + return expressions.join(' AND '); +} + +export default class DynamoDBCollection extends BaseCollection { + readonly model: TableModel; + readonly executor: DynamoExecutor; + readonly settings: CollectionSettings; + private readonly logger: Logger; + private readonly fieldNames: Set; + private readonly matchContext: Collection; + private readonly chartCache = new Map(); + + constructor( + datasource: DataSource, + model: TableModel, + client: DynamoDBDocumentClient, + logger: Logger, + settings: CollectionSettings, + ) { + super(model.name, datasource, client); + + this.model = model; + this.settings = settings; + this.logger = logger; + this.executor = new DynamoExecutor(client, model, settings, logger); + + this.addFields(convertModelToFields(model, logger)); + if (model.enableCount ?? true) this.enableCount(); + + this.fieldNames = new Set(Object.keys(this.schema.fields)); + + const entityAttribute = model.entity?.attribute; + this.matchContext = { + schema: { + fields: { + ...this.schema.fields, + ...(entityAttribute && !this.schema.fields[entityAttribute] + ? { [entityAttribute]: { type: 'Column', columnType: 'String' } } + : {}), + }, + }, + } as unknown as Collection; + } + + invalidateCaches(): void { + this.settings.paginationCache.invalidateTable(this.model.tableName); + this.chartCache.clear(); + } + + async create(caller: Caller, data: RecordData[]): Promise { + try { + const created = await this.executor.mapWrites(data, record => this.createOne(record)); + this.invalidateCaches(); + + return created; + } catch (error) { + throw this.wrap(error, 'create'); + } + } + + async list( + caller: Caller, + filter: PaginatedFilter, + projection: Projection, + ): Promise { + try { + return await this.listRecords(caller, filter, projection); + } catch (error) { + throw this.wrap(error, 'list'); + } + } + + async update(caller: Caller, filter: Filter, patch: RecordData): Promise { + try { + const entries = Object.entries(patch).filter(([, value]) => value !== undefined); + + for (const [field] of entries) { + if (this.keyNames().includes(field)) { + throw new ValidationError( + `The key attribute '${field}' of '${this.name}' cannot be updated`, + ); + } + } + + if (entries.length === 0) return; + + const keys = await this.resolveKeys(caller, filter); + await this.executor.mapWrites(keys, key => this.updateOne(key, entries)); + this.invalidateCaches(); + } catch (error) { + throw this.wrap(error, 'update'); + } + } + + async delete(caller: Caller, filter: Filter): Promise { + try { + const keys = await this.resolveKeys(caller, filter); + + if (keys.length === 0) return; + + await this.executor.batchDelete(keys); + this.invalidateCaches(); + } catch (error) { + throw this.wrap(error, 'delete'); + } + } + + async aggregate( + caller: Caller, + filter: Filter, + aggregation: Aggregation, + limit?: number, + ): Promise { + try { + return await this.aggregateRecords(caller, filter, aggregation, limit); + } catch (error) { + throw this.wrap(error, 'aggregate'); + } + } + + private wrap(error: Error & { isBusinessError?: boolean }, verb: string): Error { + if (error?.isBusinessError && !(error instanceof DynamoDBOperationError)) return error; + + return new DynamoDBOperationError( + `Failed to ${verb} records of '${this.name}': ${error.message}`, + error, + ); + } + + private keyNames(): string[] { + return [this.model.partitionKey, this.model.sortKey].filter(Boolean).map(key => key.name); + } + + /** The attributes a position in the table or index read is made of. */ + private positionAttributes(access?: AccessPath): string[] { + return [...new Set([...this.keyNames(), ...(access?.keyAttributes ?? [])])]; + } + + private toRecord(item: Item): RecordData { + return Serializer.flatten(item, this.fieldNames); + } + + private matches(tree: ConditionTree, item: Item, timezone: string): boolean { + return tree.match(this.toRecord(item), this.matchContext, timezone); + } + + // ---- reads ------------------------------------------------------------------------------ + + private plan(tree?: ConditionTree, sort?: Sort): ReadPlan { + const { pushable, residual } = splitCondition(tree); + const lookup = planKeyLookup(this.model, pushable); + + if (lookup) return { lookup, residual: intersect(lookup.remaining, residual) }; + + const access = planAccessPath(this.model, pushable, sort); + + if (!access.remaining) return { access, residual }; + + // A Query filter cannot reference the key attributes it reads by, and an expression has a size + // limit that a long list of values would exceed: both are applied on the returned items. + const keys = new Set(access.kind === 'query' ? access.keyAttributes : []); + const { pushable: filter, residual: moved } = splitCondition( + access.remaining, + leaf => !keys.has(leaf.field) && !hasOversizedList(leaf), + ); + + return { access: { ...access, remaining: filter }, residual: intersect(moved, residual) }; + } + + private guardScan(plan: ReadPlan, tree?: ConditionTree): void { + if (plan.access?.kind !== 'scan') return; + + const policy = this.model.scanPolicy ?? this.settings.scanPolicy; + + if (policy === 'allow') return; + + const filter = describeFilter(tree); + const reason = `no key or index serves the filter${filter ? ` (${filter})` : ''}`; + + if (policy === 'deny') { + throw new ScanDeniedError( + `Scan on table '${this.model.tableName}' refused by scanPolicy 'deny': ${reason}`, + ); + } + + this.logger( + 'Warn', + `Scan on table '${this.model.tableName}' for collection '${this.name}': ${reason}`, + ); + } + + private buildReadInput( + access: AccessPath, + fields: string[], + extra: Partial = {}, + ): ReadInput { + const builder = new ExpressionBuilder(); + const input: ReadInput = {}; + + if (access.kind === 'query') { + input.IndexName = access.indexName; + input.KeyConditionExpression = keyConditionExpression(builder, access.keyCondition); + if (!access.scanIndexForward) input.ScanIndexForward = false; + } + + if (access.remaining) input.FilterExpression = builder.condition(access.remaining); + if (fields.length) input.ProjectionExpression = builder.projection(fields); + + return { ...input, ...builder.result(), ...extra } as ReadInput; + } + + private projectionParams(fields: string[]): Partial { + if (!fields.length) return {}; + + const builder = new ExpressionBuilder(); + + return { ProjectionExpression: builder.projection(fields), ...builder.result() } as never; + } + + /** Reads the items DynamoDB selects, as the batches it returns them in. */ + private async *readBatches( + plan: ReadPlan, + fields: string[], + budget: ReadBudget, + segment?: { index: number; total: number }, + ): AsyncGenerator { + if (plan.lookup) { + yield await this.executor.batchGet(plan.lookup.keys, this.projectionParams(fields) as never); + + return; + } + + const operation: ReadOperation = plan.access.kind; + const input = this.buildReadInput(plan.access, fields, { + ...(segment ? { Segment: segment.index, TotalSegments: segment.total } : {}), + }); + + for await (const page of this.executor.readPages(operation, input, budget)) yield page.items; + } + + private async readAll(plan: ReadPlan, fields: string[], timezone: string): Promise { + const items: Item[] = []; + + for await (const batch of this.readBatches(plan, fields, this.executor.newBudget())) { + for (const item of batch) { + if (!plan.residual || this.matches(plan.residual, item, timezone)) items.push(item); + } + } + + return items; + } + + /** Resumes from the closest position already seen, since a page cannot be reached by offset. */ + private async readPage( + plan: ReadPlan, + fields: string[], + skip: number, + limit: number | undefined, + timezone: string, + ): Promise { + const { access, residual } = plan; + const cache = this.settings.paginationCache; + const maxOffset = this.settings.maxOffset; + + if (skip > maxOffset) { + throw new OffsetTooDeepError( + `Offset ${skip} exceeds the maximum allowed offset of ${maxOffset}. ` + + `Narrow the results with filters.`, + ); + } + + const base = this.buildReadInput(access, fields); + const hash = cache.hash({ + base, + residual: residual?.toPlainObject(), + table: this.model.tableName, + }); + const cached = limit === undefined ? undefined : cache.findBestToken(hash, skip); + let matched = cached?.offset ?? 0; + let lastStored = matched; + const unfiltered = !base.FilterExpression && !residual; + const input: ReadInput = { + ...base, + ...(cached ? { ExclusiveStartKey: cached.startKey } : {}), + ...(unfiltered && limit !== undefined ? { Limit: skip + limit - matched } : {}), + }; + const collected: Item[] = []; + + for await (const page of this.executor.readPages(access.kind, input)) { + for (const item of page.items) { + if (!residual || this.matches(residual, item, timezone)) { + if (matched >= skip) collected.push(item); + matched += 1; + + if (limit !== undefined && collected.length >= limit) { + cache.storeToken(hash, matched, this.positionOf(item, access), this.model.tableName); + + return collected; + } + } + } + + if (page.lastKey && matched - lastStored >= cache.getCacheInterval()) { + cache.storeToken(hash, matched, page.lastKey, this.model.tableName); + lastStored = matched; + } + } + + return collected; + } + + private positionOf(item: Item, access: AccessPath): Record { + return Object.fromEntries( + this.positionAttributes(access) + .filter(attribute => item[attribute] !== undefined) + .map(attribute => [attribute, item[attribute]]), + ); + } + + private async listRecords( + caller: Caller, + filter: PaginatedFilter, + projection: Projection, + ): Promise { + const sort = filter.sort?.length ? filter.sort : undefined; + const skip = filter.page?.skip ?? 0; + const limit = filter.page?.limit || undefined; + const plan = this.plan(filter.conditionTree, sort); + + this.guardScan(plan, filter.conditionTree); + + const columns = projection.columns; + const fields = [ + ...new Set([ + ...columns, + ...(plan.residual?.projection ?? []).filter(field => !field.includes(':')), + ...(sort ?? []).map(clause => clause.field), + ...this.positionAttributes(plan.access), + ]), + ]; + const toResult = (item: Item) => { + const record = this.toRecord(item); + + return Object.fromEntries( + columns.filter(field => field in record).map(field => [field, record[field]]), + ); + }; + + if (!plan.lookup && (!sort || plan.access.sortSatisfied)) { + return (await this.readPage(plan, fields, skip, limit, caller.timezone)).map(toResult); + } + + const items = await this.readAll(plan, fields, caller.timezone); + + if (sort && !plan.access?.sortSatisfied) { + const rows = items.map(item => ({ item, record: this.toRecord(item) })); + rows.sort((a, b) => compareRecords(a.record, b.record, sort)); + items.splice(0, items.length, ...rows.map(row => row.item)); + } + + return items.slice(skip, limit === undefined ? undefined : skip + limit).map(toResult); + } + + private async resolveKeys(caller: Caller, filter: Filter): Promise { + const plan = this.plan(filter.conditionTree); + + if (plan.lookup && !plan.residual) return plan.lookup.keys; + + const keyNames = this.keyNames(); + const records = await this.listRecords( + caller, + new PaginatedFilterClass({ conditionTree: filter.conditionTree }), + new ProjectionClass(...keyNames), + ); + + return records.map(record => Object.fromEntries(keyNames.map(name => [name, record[name]]))); + } + + // ---- writes ----------------------------------------------------------------------------- + + private async createOne(record: RecordData): Promise { + const { partitionKey, sortKey, entity } = this.model; + const created: RecordData = { ...record }; + + if (isEmptyKey(created[partitionKey.name])) { + if (partitionKey.type !== 'S') { + throw new ValidationError( + `The partition key '${partitionKey.name}' is required to create a record in '${this.name}'`, + ); + } + + created[partitionKey.name] = randomUUID(); + } + + if (sortKey && isEmptyKey(created[sortKey.name])) { + throw new ValidationError( + `The sort key '${sortKey.name}' is required to create a record in '${this.name}'`, + ); + } + + if ( + entity?.sortKeyPrefix !== undefined && + !String(created[sortKey.name]).startsWith(entity.sortKeyPrefix) + ) { + throw new ValidationError( + `The sort key '${sortKey.name}' of '${this.name}' must start with '${entity.sortKeyPrefix}'`, + ); + } + + const item = Serializer.unflatten(created); + if (entity?.attribute !== undefined) item[entity.attribute] ??= entity.value; + + const keyBuilder = new ExpressionBuilder('k', 'kv'); + + try { + await this.executor.put({ + Item: item, + ConditionExpression: `attribute_not_exists(${keyBuilder.attribute(partitionKey.name)})`, + ...keyBuilder.result(), + }); + } catch (error) { + if (error.name === 'ConditionalCheckFailedException') { + throw new ValidationError(`A record with the same key already exists in '${this.name}'`); + } + + throw error; + } + + return created; + } + + private async updateOne(key: Item, entries: Array<[string, unknown]>): Promise { + const builder = new ExpressionBuilder(); + const keyBuilder = new ExpressionBuilder('k', 'kv'); + const sets = entries.map( + ([field, value]) => `${builder.attribute(field)} = ${builder.value(value)}`, + ); + const condition = `attribute_exists(${keyBuilder.attribute(this.model.partitionKey.name)})`; + + try { + await this.executor.update({ + Key: key, + UpdateExpression: `SET ${sets.join(', ')}`, + ConditionExpression: condition, + ExpressionAttributeNames: { ...builder.names, ...keyBuilder.names }, + ExpressionAttributeValues: builder.values, + }); + } catch (error) { + if (error.name === 'ConditionalCheckFailedException') return; + + // A nested path can only be set under a map that already exists: merge client-side instead. + if (error.name === 'ValidationException' && /document path/i.test(error.message)) { + await this.replaceOne(key, entries); + + return; + } + + throw error; + } + } + + private async replaceOne(key: Item, entries: Array<[string, unknown]>): Promise { + const existing = await this.executor.get(key); + + if (!existing) return; + + const keyBuilder = new ExpressionBuilder('k', 'kv'); + + try { + await this.executor.put({ + Item: Serializer.deepMerge(existing, Serializer.unflatten(Object.fromEntries(entries))), + ConditionExpression: `attribute_exists(${keyBuilder.attribute( + this.model.partitionKey.name, + )})`, + ...keyBuilder.result(), + }); + } catch (error) { + if (error.name !== 'ConditionalCheckFailedException') throw error; + } + } + + // ---- aggregation ------------------------------------------------------------------------ + + private async aggregateRecords( + caller: Caller, + filter: Filter, + aggregation: Aggregation, + limit?: number, + ): Promise { + const { timezone } = caller; + const { paginationCache: cache, chartCacheTtlMs, now } = this.settings; + const cacheKey = cache.hash({ + filter: filter.conditionTree?.toPlainObject(), + aggregation: { + field: aggregation.field, + operation: aggregation.operation, + groups: aggregation.groups, + }, + limit, + timezone, + }); + const cached = this.chartCache.get(cacheKey); + + if (cached && now() - cached.at < chartCacheTtlMs) { + return cached.value.map(row => ({ ...row, group: { ...row.group } })); + } + + const value = await this.computeAggregation(filter, aggregation, limit, timezone); + + if (chartCacheTtlMs > 0) { + this.chartCache.delete(cacheKey); + this.chartCache.set(cacheKey, { at: now(), value }); + + if (this.chartCache.size > MAX_CACHED_CHARTS) { + this.chartCache.delete(this.chartCache.keys().next().value); + } + } + + return value; + } + + /** Reads the segments of a parallel scan at once; a failure stops the others from reading on. */ + private async inSegments( + budget: ReadBudget, + segments: T[], + read: (segment: T) => Promise, + ): Promise { + try { + await Promise.all(segments.map(read)); + } catch (error) { + budget.abort(); + + throw error; + } + } + + private async computeAggregation( + filter: Filter, + aggregation: Aggregation, + limit: number | undefined, + timezone: string, + ): Promise { + const plan = this.plan(filter.conditionTree); + const isCount = aggregation.operation === 'Count' && !aggregation.field; + const ungrouped = !aggregation.groups?.length; + + if ( + isCount && + ungrouped && + this.settings.approximateCount && + !filter.conditionTree && + !this.model.entity + ) { + return [{ value: await this.executor.describeItemCount(), group: {} }]; + } + + this.guardScan(plan, filter.conditionTree); + + const budget = this.executor.newBudget(); + const segments = plan.access?.kind === 'scan' ? Math.max(this.settings.scanConcurrency, 1) : 1; + const segmentsOf = (): Array<{ index: number; total: number } | undefined> => + segments > 1 + ? Array.from({ length: segments }, (_, index) => ({ index, total: segments })) + : [undefined]; + + if (isCount && ungrouped && !plan.lookup && !plan.residual) { + let total = 0; + + await this.inSegments(budget, segmentsOf(), async segment => { + const input = this.buildReadInput(plan.access, [], { + Select: 'COUNT', + ...(segment ? { Segment: segment.index, TotalSegments: segment.total } : {}), + }); + + for await (const page of this.executor.readPages(plan.access.kind, input, budget)) { + total += page.count; + } + }); + + return [{ value: total, group: {} }]; + } + + const fields = [ + ...new Set([ + ...aggregation.projection.filter(field => !field.includes(':')), + ...(plan.residual?.projection ?? []).filter(field => !field.includes(':')), + ]), + ]; + const aggregator = new StreamingAggregator(aggregation, timezone); + + await this.inSegments(budget, segmentsOf(), async segment => { + for await (const batch of this.readBatches(plan, fields, budget, segment)) { + for (const item of batch) { + if (!plan.residual || this.matches(plan.residual, item, timezone)) { + aggregator.add(this.toRecord(item)); + } + } + } + }); + + return aggregator.result(limit); + } +} diff --git a/packages/datasource-dynamodb/src/datasource.ts b/packages/datasource-dynamodb/src/datasource.ts new file mode 100644 index 0000000000..f4bc9a5259 --- /dev/null +++ b/packages/datasource-dynamodb/src/datasource.ts @@ -0,0 +1,94 @@ +import type ArrayCollection from './array-collection'; +import type { CollectionSettings, TableModel } from './types'; +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import type { Logger } from '@forestadmin/datasource-toolkit'; + +import { ExecuteStatementCommand } from '@aws-sdk/lib-dynamodb'; +import { BaseDataSource, ValidationError } from '@forestadmin/datasource-toolkit'; + +import DynamoDBCollection from './collection'; +import { ScanLimitExceededError } from './errors'; + +const PLACEHOLDER = /\$(\w+)/g; +const READ_ONLY_STATEMENT = /^\s*select\b/i; + +type NativeQueryConnection = { client: DynamoDBDocumentClient }; + +export default class DynamoDBDataSource extends BaseDataSource< + DynamoDBCollection | ArrayCollection +> { + constructor( + private readonly client: DynamoDBDocumentClient, + models: TableModel[], + logger: Logger, + private readonly settings: CollectionSettings, + options?: { liveQueryConnection?: string }, + ) { + super(); + + [...models] + // keeps the schema from being reordered between two starts + .sort((a, b) => (a.name > b.name ? 1 : -1)) + .forEach(model => + this.addCollection(new DynamoDBCollection(this, model, client, logger, settings)), + ); + + if (options?.liveQueryConnection) { + this.addNativeQueryConnection(options.liveQueryConnection, { client }); + } + + logger('Info', 'DynamoDBDataSource - Built'); + } + + /** Runs a PartiQL SELECT: `$name` placeholders become positional parameters. */ + override async executeNativeQuery( + connectionName: string, + query: string, + contextVariables: Record = {}, + ): Promise[]> { + const connection = this.nativeQueryConnections[connectionName] as NativeQueryConnection; + + if (!connection) throw new ValidationError(`Unknown connection name '${connectionName}'`); + + if (!READ_ONLY_STATEMENT.test(query)) { + throw new ValidationError('Only SELECT statements can be run as native queries'); + } + + const parameters: unknown[] = []; + const statement = query.replace(PLACEHOLDER, (_match, name: string) => { + if (!(name in contextVariables)) { + throw new ValidationError(`No value was given for the placeholder $${name}`); + } + + parameters.push(contextVariables[name]); + + return '?'; + }); + + const rows: Record[] = []; + let nextToken: string; + + do { + // eslint-disable-next-line no-await-in-loop + const response = await connection.client.send( + new ExecuteStatementCommand({ + Statement: statement, + ...(parameters.length ? { Parameters: parameters as never } : {}), + ...(nextToken ? { NextToken: nextToken } : {}), + }), + ); + + rows.push(...((response.Items ?? []) as Record[])); + nextToken = response.NextToken; + + if (rows.length > this.settings.maxScannedItems) { + throw new ScanLimitExceededError( + `The native query returned more than ${this.settings.maxScannedItems} items, which is ` + + `maxScannedItems. Narrow the results with a WHERE clause.`, + ); + } + } while (nextToken); + + return rows; + } +} diff --git a/packages/datasource-dynamodb/src/errors.ts b/packages/datasource-dynamodb/src/errors.ts new file mode 100644 index 0000000000..d241ad64bc --- /dev/null +++ b/packages/datasource-dynamodb/src/errors.ts @@ -0,0 +1,19 @@ +import { + BadRequestError, + ForbiddenError, + InternalServerError, + UnprocessableError, +} from '@forestadmin/datasource-toolkit'; + +export class ScanDeniedError extends ForbiddenError {} + +export class ScanLimitExceededError extends UnprocessableError {} + +export class OffsetTooDeepError extends BadRequestError {} + +export class DynamoDBOperationError extends InternalServerError { + constructor(message: string, cause: unknown) { + super(message); + Object.assign(this, { cause }); + } +} diff --git a/packages/datasource-dynamodb/src/index.ts b/packages/datasource-dynamodb/src/index.ts new file mode 100644 index 0000000000..cc0163ab63 --- /dev/null +++ b/packages/datasource-dynamodb/src/index.ts @@ -0,0 +1,150 @@ +import type { ConfigurationOptions } from './introspection/builder'; +import type { + DynamoDBClientOptions, + DynamoDBSettingsOptions, + IntrospectionConfig, + ManualSchemaConfig, + TableModel, + VirtualCollectionConfig, +} from './types'; +import type { DataSourceFactory } from '@forestadmin/datasource-toolkit'; + +import { ValidationError } from '@forestadmin/datasource-toolkit'; + +import { createDocumentClient } from './client'; +import DynamoDBDataSource from './datasource'; +import { DynamoDBDatasourceBuilder } from './introspection/builder'; +import { + DEFAULT_MAX_DEPTH, + DEFAULT_SAMPLE_SIZE, + introspectTables, +} from './introspection/introspector'; +import { convertManualSchema } from './schema/manual-schema'; +import { resolveSettings } from './settings'; +import PaginationCache from './utils/pagination-cache'; +import addVirtualCollections from './virtual-collections'; + +export { default as DynamoDBCollection } from './collection'; +export { default as ArrayCollection } from './array-collection'; +export { default as DynamoDBDataSource } from './datasource'; +export { ScanDeniedError, ScanLimitExceededError, OffsetTooDeepError } from './errors'; +export type { + ConfigurationOptions, + DynamoDBDatasourceOptionsBuilder, + TableCollectionOptions, +} from './introspection/builder'; +export type { + CollectionDefinition, + DynamoDBClientOptions, + DynamoDBSettingsOptions, + EntityDefinition, + FieldDefinition, + IndexDefinition, + IntrospectionConfig, + ManualSchemaConfig, + ScanPolicy, + VirtualCollectionConfig, +} from './types'; + +export type DynamoDBDataSourceOptions = DynamoDBClientOptions & + DynamoDBSettingsOptions & { + /** Tables to introspect, instead of every table of the account and region. */ + tables?: string[]; + /** Adds collections one table at a time, to name them or to share a table between several. */ + builder?: ConfigurationOptions; + /** Declares the collections instead of discovering them. Needs `disableIntrospection`. */ + schema?: ManualSchemaConfig; + disableIntrospection?: boolean; + introspectionConfig?: IntrospectionConfig; + /** Exposes arrays of maps as collections of their own. */ + virtualCollections?: VirtualCollectionConfig[]; + /** Name of the connection that runs PartiQL native queries. Without it, none is offered. */ + liveQueryConnection?: string; + }; + +const withoutUndefined = (value: T): T => + Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined)) as T; + +async function loadModels( + options: DynamoDBDataSourceOptions, + client: ReturnType, + logger: Parameters[0], +): Promise { + const sampleSize = options.introspectionConfig?.sampleSize ?? DEFAULT_SAMPLE_SIZE; + const maxDepth = options.introspectionConfig?.maxDepth ?? DEFAULT_MAX_DEPTH; + + if (options.disableIntrospection) { + if (!options.schema) { + throw new ValidationError( + 'When disableIntrospection is true, you must provide a schema with collection definitions', + ); + } + + logger('Info', 'Using the manual schema (introspection disabled)'); + + return convertManualSchema(options.schema); + } + + if (options.builder) { + const builder = options.builder( + new DynamoDBDatasourceBuilder(client, { sampleSize, maxDepth }), + ) as DynamoDBDatasourceBuilder; + + return builder.createCollectionsFromConfiguration(); + } + + const models = await introspectTables( + client, + { tables: options.tables, sampleSize, maxDepth }, + logger, + ); + + if (models.length === 0) logger('Warn', 'No collections were introspected.'); + + return models; +} + +/** + * Creates a datasource over DynamoDB tables. + * + * @example + * agent.addDataSource( + * createDynamoDBDataSource({ region: 'eu-west-1', tables: ['Orders'], scanPolicy: 'warn' }), + * ); + */ +export function createDynamoDBDataSource( + options: DynamoDBDataSourceOptions = {}, +): DataSourceFactory { + return async logger => { + const client = createDocumentClient(options); + const settings = resolveSettings( + withoutUndefined({ + scanPolicy: options.scanPolicy, + maxScannedItems: options.maxScannedItems, + scanConcurrency: options.scanConcurrency, + approximateCount: options.approximateCount, + chartCacheTtlMs: options.chartCacheTtlMs, + maxOffset: options.maxOffset, + logConsumedCapacity: options.logConsumedCapacity, + logQueries: options.logQueries, + paginationCache: new PaginationCache(options.paginationCache), + }), + ); + + const models = await loadModels(options, client, logger); + const dataSource = new DynamoDBDataSource(client, models, logger, settings, { + liveQueryConnection: options.liveQueryConnection, + }); + + if (options.virtualCollections?.length) { + await addVirtualCollections( + dataSource, + options.virtualCollections, + options.introspectionConfig?.sampleSize, + logger, + ); + } + + return dataSource; + }; +} diff --git a/packages/datasource-dynamodb/src/introspection/builder.ts b/packages/datasource-dynamodb/src/introspection/builder.ts new file mode 100644 index 0000000000..9d84010b7f --- /dev/null +++ b/packages/datasource-dynamodb/src/introspection/builder.ts @@ -0,0 +1,61 @@ +import type { + EntityDefinition, + IntrospectionConfig, + OverrideTypeConverter, + ScanPolicy, + TableModel, +} from '../types'; +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; + +import { introspectTable } from './introspector'; + +export type TableCollectionOptions = { + /** Name of the collection in Forest Admin. */ + name: string; + tableName: string; + sampleSize?: number; + maxDepth?: number; + /** Keeps the collection to a part of a table shared with other collections. */ + entity?: EntityDefinition; + /** Allows the pagination to show the number of pages, at the cost of counting. Default: true. */ + enableCount?: boolean; + scanPolicy?: ScanPolicy; + maxScannedItems?: number; + overrideTypeConverter?: OverrideTypeConverter; +}; + +export interface DynamoDBDatasourceOptionsBuilder { + addCollectionFromTable(options: TableCollectionOptions): this; +} + +export type ConfigurationOptions = ( + configurator: DynamoDBDatasourceOptionsBuilder, +) => DynamoDBDatasourceOptionsBuilder; + +/** Adds collections one table at a time, to name them or to share a table between several. */ +export class DynamoDBDatasourceBuilder implements DynamoDBDatasourceOptionsBuilder { + private readonly pending: Array> = []; + + constructor( + private readonly client: DynamoDBDocumentClient, + private readonly defaults: IntrospectionConfig = {}, + ) {} + + addCollectionFromTable(options: TableCollectionOptions): this { + const { name, ...rest } = options; + + this.pending.push( + introspectTable(this.client, { + ...this.defaults, + ...rest, + collectionName: name, + }), + ); + + return this; + } + + createCollectionsFromConfiguration(): Promise { + return Promise.all(this.pending); + } +} diff --git a/packages/datasource-dynamodb/src/introspection/introspector.ts b/packages/datasource-dynamodb/src/introspection/introspector.ts new file mode 100644 index 0000000000..35d7738882 --- /dev/null +++ b/packages/datasource-dynamodb/src/introspection/introspector.ts @@ -0,0 +1,290 @@ +import type { + AttributeSchema, + DynamoDataType, + EntityDefinition, + IndexModel, + KeyAttribute, + OverrideTypeConverter, + ScanPolicy, + TableModel, +} from '../types'; +import type { TableDescription } from '@aws-sdk/client-dynamodb'; +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import type { Logger } from '@forestadmin/datasource-toolkit'; + +import { DescribeTableCommand, ListTablesCommand } from '@aws-sdk/client-dynamodb'; +import { ScanCommand } from '@aws-sdk/lib-dynamodb'; +import { ValidationError } from '@forestadmin/datasource-toolkit'; + +import { mapWithConcurrency } from '../utils/concurrency'; +import ExpressionBuilder from '../utils/expression-builder'; +import { PATH_SEPARATOR } from '../utils/serializer'; +import TypeConverter from '../utils/type-converter'; + +export const DEFAULT_SAMPLE_SIZE = 100; +export const DEFAULT_MAX_DEPTH = 5; +const INTROSPECTION_CONCURRENCY = 5; +/** A sample restricted to an entity can need several scans, as the limit counts unfiltered items. */ +const MAX_SAMPLE_SCANS = 20; +const KEY_TYPE_FOR_ATTRIBUTE = { S: 'S', N: 'N' } as const; + +export type TableIntrospectionOptions = { + collectionName: string; + tableName: string; + sampleSize?: number; + maxDepth?: number; + entity?: EntityDefinition; + enableCount?: boolean; + scanPolicy?: ScanPolicy; + maxScannedItems?: number; + overrideTypeConverter?: OverrideTypeConverter; +}; + +type Item = Record; + +function toKeyAttribute(table: TableDescription, name: string): KeyAttribute { + const attributeType = table.AttributeDefinitions?.find( + definition => definition.AttributeName === name, + )?.AttributeType; + + if (attributeType === 'B') { + throw new ValidationError( + `Key '${name}' of table '${table.TableName}' is binary, which is not supported`, + ); + } + + return { name, type: KEY_TYPE_FOR_ATTRIBUTE[attributeType as 'S' | 'N'] ?? 'S' }; +} + +function keysOf( + table: TableDescription, + keySchema: TableDescription['KeySchema'], +): { partitionKey: KeyAttribute; sortKey?: KeyAttribute } { + const partition = keySchema.find(key => key.KeyType === 'HASH'); + const sort = keySchema.find(key => key.KeyType === 'RANGE'); + + return { + partitionKey: toKeyAttribute(table, partition.AttributeName), + sortKey: sort ? toKeyAttribute(table, sort.AttributeName) : undefined, + }; +} + +function indexesOf(table: TableDescription): IndexModel[] { + const describe = ( + indexes: TableDescription['GlobalSecondaryIndexes'], + type: IndexModel['type'], + ): IndexModel[] => + (indexes ?? []).map(index => ({ + name: index.IndexName, + type, + ...keysOf(table, index.KeySchema), + projectsAllAttributes: index.Projection?.ProjectionType === 'ALL', + })); + + return [ + ...describe(table.GlobalSecondaryIndexes, 'GSI'), + ...describe(table.LocalSecondaryIndexes, 'LSI'), + ]; +} + +/** Records the type of every attribute of an item, expanding nested maps up to `maxDepth`. */ +function collectTypes( + value: unknown, + path: string, + depth: number, + maxDepth: number, + types: Record, + presentInItem: Set, +): void { + const record = (type: DynamoDataType) => { + types[path] ??= []; + types[path].push(type); + presentInItem.add(path); + }; + + const inferred = TypeConverter.inferTypeFromValue(value); + + if (inferred !== 'object') { + if (path) record(inferred); + + return; + } + + if (depth >= maxDepth) { + record('object'); + + return; + } + + const entries = Object.entries(value as Item); + + if (entries.length === 0) record('object'); + + for (const [key, nested] of entries) { + collectTypes( + nested, + path ? `${path}${PATH_SEPARATOR}${key}` : key, + depth + 1, + maxDepth, + types, + presentInItem, + ); + } +} + +export function inferSchema( + items: Item[], + keys: KeyAttribute[], + maxDepth: number, +): Record { + const types: Record = {}; + const presence: Record = {}; + + for (const item of items) { + const present = new Set(); + collectTypes(item, '', 0, maxDepth, types, present); + present.forEach(path => { + presence[path] = (presence[path] ?? 0) + 1; + }); + } + + const schema: Record = {}; + + for (const [path, observed] of Object.entries(types)) { + schema[path] = { + type: TypeConverter.getMostSpecificType(observed), + nullable: observed.includes('null') || presence[path] < items.length, + }; + } + + for (const key of keys) { + schema[key.name] = { type: key.type === 'N' ? 'number' : 'string', nullable: false }; + } + + return schema; +} + +async function sampleItems( + client: DynamoDBDocumentClient, + table: TableDescription, + sortKey: KeyAttribute | undefined, + options: TableIntrospectionOptions, +): Promise { + const sampleSize = options.sampleSize ?? DEFAULT_SAMPLE_SIZE; + const { entity } = options; + const builder = new ExpressionBuilder(); + let filter: string; + + if (entity?.sortKeyPrefix !== undefined) { + filter = `begins_with(${builder.attribute(sortKey.name)}, ${builder.value( + entity.sortKeyPrefix, + )})`; + } else if (entity?.attribute !== undefined) { + filter = `${builder.attribute(entity.attribute)} = ${builder.value(entity.value)}`; + } + + const items: Item[] = []; + let startKey: Item; + + for (let scans = 0; scans < MAX_SAMPLE_SCANS && items.length < sampleSize; scans += 1) { + // eslint-disable-next-line no-await-in-loop + const response = await client.send( + new ScanCommand({ + TableName: table.TableName, + Limit: sampleSize, + ...(filter ? { FilterExpression: filter, ...builder.result() } : {}), + ...(startKey ? { ExclusiveStartKey: startKey } : {}), + }), + ); + + items.push(...((response.Items ?? []) as Item[])); + startKey = response.LastEvaluatedKey; + + if (!startKey) break; + } + + return items.slice(0, sampleSize); +} + +/** Reads the key schema and indexes of a table, and infers its attributes from a sample. */ +export async function introspectTable( + client: DynamoDBDocumentClient, + options: TableIntrospectionOptions, +): Promise { + const { Table: table } = await client.send( + new DescribeTableCommand({ TableName: options.tableName }), + ); + const { partitionKey, sortKey } = keysOf(table, table.KeySchema); + + if (options.entity?.sortKeyPrefix !== undefined && !sortKey) { + throw new ValidationError( + `Table '${options.tableName}' has no sort key, so 'sortKeyPrefix' cannot be used`, + ); + } + + const items = await sampleItems(client, table, sortKey, options); + + return { + name: options.collectionName, + tableName: options.tableName, + partitionKey, + sortKey, + indexes: indexesOf(table), + schema: inferSchema( + items, + [partitionKey, sortKey].filter(Boolean), + options.maxDepth ?? DEFAULT_MAX_DEPTH, + ), + entity: options.entity, + enableCount: options.enableCount, + scanPolicy: options.scanPolicy, + maxScannedItems: options.maxScannedItems, + overrideTypeConverter: options.overrideTypeConverter, + }; +} + +async function listTables(client: DynamoDBDocumentClient): Promise { + const names: string[] = []; + let startName: string; + + do { + // eslint-disable-next-line no-await-in-loop + const response = await client.send( + new ListTablesCommand(startName ? { ExclusiveStartTableName: startName } : {}), + ); + names.push(...(response.TableNames ?? [])); + startName = response.LastEvaluatedTableName; + } while (startName); + + return names; +} + +/** Introspects the given tables, or every table of the account and region. */ +export async function introspectTables( + client: DynamoDBDocumentClient, + options: { tables?: string[]; sampleSize?: number; maxDepth?: number }, + logger: Logger, +): Promise { + const tableNames = options.tables ?? (await listTables(client)); + + const models = await mapWithConcurrency( + tableNames, + INTROSPECTION_CONCURRENCY, + async tableName => { + try { + return await introspectTable(client, { + collectionName: tableName, + tableName, + sampleSize: options.sampleSize, + maxDepth: options.maxDepth, + }); + } catch (error) { + logger('Warn', `Introspector - Skipping table '${tableName}': ${error.message}`); + + return null; + } + }, + ); + + return models.filter(Boolean); +} diff --git a/packages/datasource-dynamodb/src/schema/manual-schema.ts b/packages/datasource-dynamodb/src/schema/manual-schema.ts new file mode 100644 index 0000000000..eadc11d887 --- /dev/null +++ b/packages/datasource-dynamodb/src/schema/manual-schema.ts @@ -0,0 +1,222 @@ +import type { + AttributeSchema, + CollectionDefinition, + DynamoDataType, + FieldDefinition, + IndexModel, + KeyAttribute, + ManualSchemaConfig, + TableModel, +} from '../types'; + +import { ValidationError } from '@forestadmin/datasource-toolkit'; + +import { PATH_SEPARATOR } from '../utils/serializer'; + +const FIELD_TYPES: DynamoDataType[] = [ + 'string', + 'number', + 'boolean', + 'date', + 'dateonly', + 'timeonly', + 'array', + 'object', + 'null', + 'point', + 'binary', +]; +const SCAN_POLICIES = ['allow', 'warn', 'deny']; +const KEY_TYPES = ['string', 'number']; + +function validateFields(fields: FieldDefinition[], collectionName: string, path = ''): void { + const names = new Set(); + + for (const field of fields) { + const fullPath = path ? `${path}${PATH_SEPARATOR}${field.name}` : field.name; + + if (!field.name) { + throw new ValidationError(`A field of collection '${collectionName}' has no name`); + } + + if (names.has(field.name)) { + throw new ValidationError( + `Duplicate field name: '${fullPath}' in collection '${collectionName}'`, + ); + } + + names.add(field.name); + + if (!FIELD_TYPES.includes(field.type)) { + throw new ValidationError( + `Field '${fullPath}' in collection '${collectionName}' has invalid type '${field.type}'. ` + + `Valid types: ${FIELD_TYPES.join(', ')}`, + ); + } + + if (field.type === 'array') { + if (!field.subType) { + throw new ValidationError( + `Array field '${fullPath}' in collection '${collectionName}' must have a subType`, + ); + } + + if (field.subType === 'object' && !field.fields?.length) { + throw new ValidationError( + `Array field '${fullPath}' in collection '${collectionName}' of objects must have ` + + `nested field definitions`, + ); + } + } + + if (field.type === 'object' && !field.fields?.length) { + throw new ValidationError( + `Object field '${fullPath}' in collection '${collectionName}' must have nested field ` + + `definitions`, + ); + } + + if (field.fields) validateFields(field.fields, collectionName, fullPath); + } +} + +function validateCollection(collection: CollectionDefinition): void { + const { name } = collection; + + if (!name) throw new ValidationError('Collection must have a name'); + + if (!collection.tableName) { + throw new ValidationError(`Collection '${name}' must have a tableName`); + } + + if (!collection.partitionKey?.name) { + throw new ValidationError(`Collection '${name}' must have a partitionKey`); + } + + for (const key of [collection.partitionKey, collection.sortKey].filter(Boolean)) { + if (!KEY_TYPES.includes(key.type)) { + throw new ValidationError( + `Key '${key.name}' of collection '${name}' must be a string or a number`, + ); + } + } + + if (collection.sortKey?.name === collection.partitionKey.name) { + throw new ValidationError( + `Collection '${name}' uses '${collection.partitionKey.name}' as both partition key and sort key`, + ); + } + + const { entity } = collection; + + if (entity) { + const hasDiscriminator = entity.attribute !== undefined || entity.value !== undefined; + + if (hasDiscriminator === (entity.sortKeyPrefix !== undefined)) { + throw new ValidationError( + `Collection '${name}' entity must use either 'attribute'/'value' or 'sortKeyPrefix'`, + ); + } + + if (entity.sortKeyPrefix !== undefined && !collection.sortKey) { + throw new ValidationError( + `Collection '${name}' has no sortKey, so 'entity.sortKeyPrefix' cannot be used`, + ); + } + } + + if (collection.scanPolicy !== undefined && !SCAN_POLICIES.includes(collection.scanPolicy)) { + throw new ValidationError( + `Collection '${name}' has an invalid scanPolicy '${collection.scanPolicy}'`, + ); + } + + validateFields(collection.fields ?? [], name); +} + +export function validateManualSchema(schema: ManualSchemaConfig): void { + if (!Array.isArray(schema?.collections)) { + throw new ValidationError('Manual schema must have a collections array'); + } + + if (schema.collections.length === 0) { + throw new ValidationError('Manual schema must have at least one collection'); + } + + const names = new Set(); + + for (const collection of schema.collections) { + validateCollection(collection); + + if (names.has(collection.name)) { + throw new ValidationError(`Duplicate collection name: '${collection.name}'`); + } + + names.add(collection.name); + } +} + +function flattenFields(fields: FieldDefinition[], prefix = ''): Record { + const attributes: Record = {}; + + for (const field of fields) { + const path = prefix ? `${prefix}${PATH_SEPARATOR}${field.name}` : field.name; + + if (field.type === 'object') { + Object.assign(attributes, flattenFields(field.fields, path)); + } else { + attributes[path] = { + type: field.type, + nullable: !!field.nullable, + ...(field.type === 'array' && field.subType === 'object' + ? { items: flattenFields(field.fields) } + : {}), + }; + } + } + + return attributes; +} + +const toKey = (key: { name: string; type: 'string' | 'number' }): KeyAttribute => ({ + name: key.name, + type: key.type === 'number' ? 'N' : 'S', +}); + +function toModel(collection: CollectionDefinition): TableModel { + const partitionKey = toKey(collection.partitionKey); + const sortKey = collection.sortKey ? toKey(collection.sortKey) : undefined; + const keys: Record = {}; + + for (const key of [partitionKey, sortKey].filter(Boolean)) { + keys[key.name] = { type: key.type === 'N' ? 'number' : 'string', nullable: false }; + } + + const indexes: IndexModel[] = (collection.indexes ?? []).map(index => ({ + name: index.name, + type: index.type, + partitionKey: toKey(index.partitionKey), + sortKey: index.sortKey ? toKey(index.sortKey) : undefined, + projectsAllAttributes: index.projectsAllAttributes ?? true, + })); + + return { + name: collection.name, + tableName: collection.tableName, + partitionKey, + sortKey, + indexes, + schema: { ...keys, ...flattenFields(collection.fields) }, + entity: collection.entity, + enableCount: collection.enableCount, + scanPolicy: collection.scanPolicy, + maxScannedItems: collection.maxScannedItems, + overrideTypeConverter: collection.overrideTypeConverter, + }; +} + +export function convertManualSchema(schema: ManualSchemaConfig): TableModel[] { + validateManualSchema(schema); + + return schema.collections.map(toModel); +} diff --git a/packages/datasource-dynamodb/src/schema/schema-converter.ts b/packages/datasource-dynamodb/src/schema/schema-converter.ts new file mode 100644 index 0000000000..1be125a02b --- /dev/null +++ b/packages/datasource-dynamodb/src/schema/schema-converter.ts @@ -0,0 +1,51 @@ +import type { AttributeSchema, TableModel } from '../types'; +import type { ColumnSchema, FieldSchema, Logger } from '@forestadmin/datasource-toolkit'; + +import TypeConverter from '../utils/type-converter'; + +function convertAttribute(attribute: AttributeSchema, isPrimaryKey: boolean): ColumnSchema { + const columnType = TypeConverter.getColumnTypeFromDataType(attribute.type); + + return { + type: 'Column', + columnType, + filterOperators: TypeConverter.operatorsForColumnType(columnType), + isPrimaryKey, + isReadOnly: false, + isSortable: TypeConverter.isSortable(attribute.type), + validation: [], + }; +} + +/** Turns the attributes of a table into the columns of its collection, keys being the primary key. */ +export default function convertModelToFields( + model: TableModel, + logger?: Logger, +): Record { + const keys = [model.partitionKey, model.sortKey].filter(Boolean); + const attributes: Record = { ...model.schema }; + + for (const key of keys) { + attributes[key.name] ??= { type: key.type === 'N' ? 'number' : 'string', nullable: false }; + } + + const fields: Record = {}; + + for (const [name, attribute] of Object.entries(attributes)) { + try { + const isPrimaryKey = keys.some(key => key.name === name); + const generated = convertAttribute(attribute, isPrimaryKey); + const overridden = model.overrideTypeConverter?.({ + fieldName: name, + attribute, + generatedFieldSchema: generated, + }); + + fields[name] = overridden || generated; + } catch (error) { + logger?.('Warn', `Skipping column '${model.name}.${name}' (${error.message})`); + } + } + + return fields; +} diff --git a/packages/datasource-dynamodb/src/settings.ts b/packages/datasource-dynamodb/src/settings.ts new file mode 100644 index 0000000000..59293698b6 --- /dev/null +++ b/packages/datasource-dynamodb/src/settings.ts @@ -0,0 +1,25 @@ +import type { CollectionSettings } from './types'; + +import PaginationCache from './utils/pagination-cache'; + +export const DEFAULT_MAX_SCANNED_ITEMS = 1000000; + +export function resolveSettings(settings: Partial = {}): CollectionSettings { + const now = settings.now ?? (() => Date.now()); + + return { + scanPolicy: 'warn', + maxScannedItems: DEFAULT_MAX_SCANNED_ITEMS, + scanConcurrency: 4, + approximateCount: false, + chartCacheTtlMs: 30000, + maxOffset: 100000, + maxWriteRetries: 5, + writeRetryBaseDelayMs: 50, + logConsumedCapacity: false, + logQueries: false, + ...settings, + now, + paginationCache: settings.paginationCache ?? new PaginationCache({ now }), + }; +} diff --git a/packages/datasource-dynamodb/src/types.ts b/packages/datasource-dynamodb/src/types.ts new file mode 100644 index 0000000000..a609ecbfd9 --- /dev/null +++ b/packages/datasource-dynamodb/src/types.ts @@ -0,0 +1,163 @@ +import type PaginationCache from './utils/pagination-cache'; +import type { PaginationCacheOptions } from './utils/pagination-cache'; +import type { DynamoDBClientConfig } from '@aws-sdk/client-dynamodb'; +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import type { FieldSchema } from '@forestadmin/datasource-toolkit'; + +export type ScanPolicy = 'allow' | 'warn' | 'deny'; + +export type KeyType = 'S' | 'N'; + +export type KeyAttribute = { name: string; type: KeyType }; + +export type DynamoDataType = + | 'string' + | 'number' + | 'boolean' + | 'date' + | 'dateonly' + | 'timeonly' + | 'object' + | 'array' + | 'null' + | 'binary' + | 'point'; + +export type AttributeSchema = { + type: DynamoDataType; + nullable?: boolean; + /** Schema of the items of an array of maps, used to expose it as a virtual collection. */ + items?: Record; +}; + +export type IndexModel = { + name: string; + type: 'GSI' | 'LSI'; + partitionKey: KeyAttribute; + sortKey?: KeyAttribute; + projectsAllAttributes: boolean; +}; + +/** + * Tells which items of a table belong to a collection, so that several collections can share one + * table (single-table design): either a discriminator attribute or a prefix of the sort key. + */ +export type EntityDefinition = { attribute?: string; value?: string; sortKeyPrefix?: string }; + +export type OverrideTypeConverter = (field: { + fieldName: string; + attribute: AttributeSchema; + generatedFieldSchema: FieldSchema; +}) => void | FieldSchema; + +export type TableModel = { + /** Name of the collection in Forest Admin. */ + name: string; + tableName: string; + partitionKey: KeyAttribute; + sortKey?: KeyAttribute; + indexes: IndexModel[]; + /** Attributes by flattened name: nested maps are `a->b`. */ + schema: Record; + entity?: EntityDefinition; + enableCount?: boolean; + scanPolicy?: ScanPolicy; + maxScannedItems?: number; + overrideTypeConverter?: OverrideTypeConverter; +}; + +export type FieldDefinition = { + name: string; + type: DynamoDataType; + nullable?: boolean; + /** For arrays: the type of the items. */ + subType?: DynamoDataType; + /** For objects, and arrays of objects: the nested attributes. */ + fields?: FieldDefinition[]; +}; + +export type IndexDefinition = { + name: string; + type: 'GSI' | 'LSI'; + partitionKey: { name: string; type: 'string' | 'number' }; + sortKey?: { name: string; type: 'string' | 'number' }; + /** Whether the index projects every attribute, which a query on it needs. Default: true. */ + projectsAllAttributes?: boolean; +}; + +export type CollectionDefinition = { + name: string; + tableName: string; + partitionKey: { name: string; type: 'string' | 'number' }; + sortKey?: { name: string; type: 'string' | 'number' }; + fields: FieldDefinition[]; + indexes?: IndexDefinition[]; + entity?: EntityDefinition; + enableCount?: boolean; + scanPolicy?: ScanPolicy; + maxScannedItems?: number; + overrideTypeConverter?: OverrideTypeConverter; +}; + +export type ManualSchemaConfig = { collections: CollectionDefinition[] }; + +export type VirtualCollectionConfig = { + /** Name of the collection holding the array. */ + parentCollection: string; + /** Name of the new collection. */ + collectionName: string; + /** Attribute holding the array, `a->b` for a nested one. */ + arrayField: string; +}; + +export type CollectionSettings = { + scanPolicy: ScanPolicy; + maxScannedItems: number; + scanConcurrency: number; + approximateCount: boolean; + chartCacheTtlMs: number; + maxOffset: number; + maxWriteRetries: number; + writeRetryBaseDelayMs: number; + logConsumedCapacity: boolean; + logQueries: boolean; + paginationCache: PaginationCache; + now: () => number; +}; + +export type DynamoDBClientOptions = { + /** An existing client, low-level or document client. When set, the options below are ignored. */ + client?: DynamoDBDocumentClient | { send: (command: never) => Promise }; + region?: string; + endpoint?: string; + credentials?: DynamoDBClientConfig['credentials']; + /** Anything else the AWS SDK client accepts. */ + clientConfig?: DynamoDBClientConfig; +}; + +export type DynamoDBSettingsOptions = { + /** What happens when a request has to scan the table. Default: 'warn'. */ + scanPolicy?: ScanPolicy; + /** Items a single request may read before it is stopped. Default: 1 000 000. */ + maxScannedItems?: number; + /** Segments of the parallel scans run for charts. Default: 4. */ + scanConcurrency?: number; + /** Answer an unfiltered count from the table description (refreshed about every 6 hours). */ + approximateCount?: boolean; + /** How long a chart is served from memory, in milliseconds. 0 disables it. Default: 30 000. */ + chartCacheTtlMs?: number; + /** Deepest page that can be requested. Default: 100 000. */ + maxOffset?: number; + /** Log the capacity units each request consumed. */ + logConsumedCapacity?: boolean; + /** Log the expressions sent to DynamoDB. */ + logQueries?: boolean; + paginationCache?: PaginationCacheOptions; +}; + +export type IntrospectionConfig = { + /** Items read per table to infer its attributes. Default: 100. */ + sampleSize?: number; + /** Deepest nested map exposed as separate attributes. Default: 5. */ + maxDepth?: number; +}; diff --git a/packages/datasource-dynamodb/src/utils/access-path.ts b/packages/datasource-dynamodb/src/utils/access-path.ts new file mode 100644 index 0000000000..89fb541bc6 --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/access-path.ts @@ -0,0 +1,299 @@ +import type { KeyAttribute, TableModel } from '../types'; +import type { ConditionTree, Sort } from '@forestadmin/datasource-toolkit'; + +import { + ConditionTreeBranch, + ConditionTreeFactory, + ConditionTreeLeaf, +} from '@forestadmin/datasource-toolkit'; + +export type SortKeyOperator = + | 'Equal' + | 'LessThan' + | 'LessThanOrEqual' + | 'GreaterThan' + | 'GreaterThanOrEqual' + | 'StartsWith' + | 'Between'; + +export type SortKeyCondition = { attribute: string; operator: SortKeyOperator; value: unknown }; + +export type AccessPath = { + kind: 'query' | 'scan'; + /** Undefined for the table itself. */ + indexName?: string; + /** Key attributes of the table or index read: a Query filter cannot reference them. */ + keyAttributes?: string[]; + keyCondition?: { + partition: { attribute: string; value: string | number }; + sort?: SortKeyCondition; + }; + /** What the key condition leaves for a filter expression. */ + remaining?: ConditionTree; + scanIndexForward: boolean; + /** Whether reading in key order already gives the requested sort. */ + sortSatisfied: boolean; +}; + +export type KeyLookup = { keys: Array>; remaining?: ConditionTree }; + +type Candidate = { indexName?: string; partitionKey: KeyAttribute; sortKey?: KeyAttribute }; + +const RANGE_OPERATORS = ['LessThan', 'LessThanOrEqual', 'GreaterThan', 'GreaterThanOrEqual']; + +const isKeyValue = (value: unknown): value is string | number => + typeof value === 'string' || typeof value === 'number'; + +/** DynamoDB tells 1 from "1": a foreign key stored as the other type would find nothing. */ +function coerceKeyValue(attribute: KeyAttribute, value: T): T | string | number { + if (attribute.type === 'N' && typeof value === 'string' && value.trim() !== '') { + return Number.isNaN(Number(value)) ? value : Number(value); + } + + if (attribute.type === 'S' && typeof value === 'number') return String(value); + + return value; +} + +/** The restriction a single-table entity puts on the items of its table. */ +function withEntity(model: TableModel, tree?: ConditionTree): ConditionTree | undefined { + const { entity } = model; + + if (!entity) return tree; + + const restriction = + entity.sortKeyPrefix !== undefined && model.sortKey + ? new ConditionTreeLeaf(model.sortKey.name, 'StartsWith', entity.sortKeyPrefix) + : entity.attribute !== undefined + ? new ConditionTreeLeaf(entity.attribute, 'Equal', entity.value) + : undefined; + + if (!restriction) return tree; + + return tree ? ConditionTreeFactory.intersect(tree, restriction) : restriction; +} + +/** The conditions that must all hold, as the root of an AND or alone. */ +function conjuncts(tree?: ConditionTree): ConditionTree[] { + if (!tree) return []; + + if (tree instanceof ConditionTreeBranch && tree.aggregator === 'And') { + return tree.conditions.flatMap(condition => conjuncts(condition)); + } + + return [tree]; +} + +const leavesOf = (parts: ConditionTree[]): ConditionTreeLeaf[] => + parts.filter((part): part is ConditionTreeLeaf => part instanceof ConditionTreeLeaf); + +function rebuild(parts: ConditionTree[]): ConditionTree | undefined { + return parts.length ? ConditionTreeFactory.intersect(...parts) : undefined; +} + +function pickSortCondition( + sortKey: KeyAttribute, + parts: ConditionTree[], +): { condition: SortKeyCondition; consumed: ConditionTree[] } | undefined { + const attribute = sortKey.name; + const onSortKey = leavesOf(parts).filter( + leaf => leaf.field === attribute && isKeyValue(leaf.value), + ); + const find = (operators: string[]) => onSortKey.filter(leaf => operators.includes(leaf.operator)); + const valueOf = (leaf: ConditionTreeLeaf) => coerceKeyValue(sortKey, leaf.value); + + const [equal] = find(['Equal']); + + if (equal) { + return { + condition: { attribute, operator: 'Equal', value: valueOf(equal) }, + consumed: [equal], + }; + } + + const [lower] = find(['GreaterThanOrEqual']); + const [upper] = find(['LessThanOrEqual']); + + if (lower && upper) { + return { + condition: { attribute, operator: 'Between', value: [valueOf(lower), valueOf(upper)] }, + consumed: [lower, upper], + }; + } + + const [range] = find(RANGE_OPERATORS); + + if (range) { + return { + condition: { attribute, operator: range.operator as SortKeyOperator, value: valueOf(range) }, + consumed: [range], + }; + } + + const [prefix] = find(['StartsWith']).sort( + (a, b) => String(b.value).length - String(a.value).length, + ); + + if (prefix) { + return { + condition: { attribute, operator: 'StartsWith', value: valueOf(prefix) }, + consumed: [prefix], + }; + } + + return undefined; +} + +/** + * Picks the cheapest way to read the items a condition tree selects: a Query on the table or on + * an index whenever an equality pins a partition key, a Scan otherwise. + */ +export function planAccessPath( + model: TableModel, + conditionTree?: ConditionTree, + sort?: Sort, +): AccessPath { + const tree = withEntity(model, conditionTree); + const parts = conjuncts(tree); + const leaves = leavesOf(parts); + + const candidates: Candidate[] = [ + { partitionKey: model.partitionKey, sortKey: model.sortKey }, + ...model.indexes + .filter(index => index.projectsAllAttributes) + .map(index => ({ + indexName: index.name, + partitionKey: index.partitionKey, + sortKey: index.sortKey, + })), + ]; + + let best: { path: AccessPath; score: number } | undefined; + + for (const candidate of candidates) { + const partition = leaves.find( + leaf => + leaf.field === candidate.partitionKey.name && + leaf.operator === 'Equal' && + isKeyValue(leaf.value), + ); + + if (partition) { + const picked = candidate.sortKey ? pickSortCondition(candidate.sortKey, parts) : undefined; + const sortClause = sort?.length === 1 ? sort[0] : undefined; + const sortSatisfied = !!sortClause && sortClause.field === candidate.sortKey?.name; + const consumed = new Set([partition, ...(picked?.consumed ?? [])]); + const score = (picked ? 2 : 0) + (sortSatisfied ? 1 : 0); + + if (!best || score > best.score) { + best = { + score, + path: { + kind: 'query', + indexName: candidate.indexName, + keyAttributes: [candidate.partitionKey.name, candidate.sortKey?.name].filter(Boolean), + keyCondition: { + partition: { + attribute: candidate.partitionKey.name, + value: coerceKeyValue(candidate.partitionKey, partition.value) as string | number, + }, + sort: picked?.condition, + }, + remaining: rebuild(parts.filter(part => !consumed.has(part))), + scanIndexForward: sortSatisfied ? sortClause.ascending : true, + sortSatisfied, + }, + }; + } + } + } + + return ( + best?.path ?? { + kind: 'scan', + remaining: tree, + scanIndexForward: true, + sortSatisfied: false, + } + ); +} + +function keysFromOr(model: TableModel, part: ConditionTree): KeyLookup['keys'] | null { + if (!(part instanceof ConditionTreeBranch) || part.aggregator !== 'Or') return null; + + const keys: KeyLookup['keys'] = []; + + for (const condition of part.conditions) { + const equalities = leavesOf(conjuncts(condition)); + const key: Record = {}; + + for (const attribute of [model.partitionKey, model.sortKey].filter(Boolean)) { + const equal = equalities.find( + leaf => + leaf.field === attribute.name && leaf.operator === 'Equal' && isKeyValue(leaf.value), + ); + + if (!equal) return null; + key[attribute.name] = coerceKeyValue(attribute, equal.value); + } + + if (equalities.length !== Object.keys(key).length) return null; + keys.push(key); + } + + return keys; +} + +/** + * Detects the filters that name whole keys, which BatchGetItem reads without scanning or + * querying: a single key, a list of keys, or the OR of key equalities the relation emulation sends. + */ +export function planKeyLookup(model: TableModel, conditionTree?: ConditionTree): KeyLookup | null { + const tree = withEntity(model, conditionTree); + + if (!tree) return null; + + const parts = conjuncts(tree); + const keyAttributes = [model.partitionKey, model.sortKey].filter(Boolean); + + const valuesOf = (attribute: KeyAttribute) => { + const leaf = leavesOf(parts).find( + candidate => + candidate.field === attribute.name && + (candidate.operator === 'Equal' || candidate.operator === 'In') && + (candidate.operator === 'Equal' + ? isKeyValue(candidate.value) + : Array.isArray(candidate.value) && candidate.value.every(isKeyValue)), + ); + + if (!leaf) return undefined; + + return { + leaf, + values: ( + (leaf.operator === 'Equal' ? [leaf.value] : leaf.value) as Array + ).map(value => coerceKeyValue(attribute, value)), + }; + }; + + const resolved = keyAttributes.map(attribute => ({ attribute, found: valuesOf(attribute) })); + + if (resolved.every(({ found }) => found)) { + const keys = resolved.reduce>>( + (combined, { attribute, found }) => + combined.flatMap(key => found.values.map(value => ({ ...key, [attribute.name]: value }))), + [{}], + ); + const consumed = new Set(resolved.map(({ found }) => found.leaf)); + + return { keys, remaining: rebuild(parts.filter(part => !consumed.has(part))) }; + } + + for (const part of parts) { + const keys = keysFromOr(model, part); + + if (keys) return { keys, remaining: rebuild(parts.filter(other => other !== part)) }; + } + + return null; +} diff --git a/packages/datasource-dynamodb/src/utils/aggregator.ts b/packages/datasource-dynamodb/src/utils/aggregator.ts new file mode 100644 index 0000000000..c94267629a --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/aggregator.ts @@ -0,0 +1,174 @@ +import type { AggregateResult, Aggregation, DateOperation } from '@forestadmin/datasource-toolkit'; +import type { RecordData } from '@forestadmin/datasource-toolkit'; + +/** Numbers below this are epoch seconds: in milliseconds they would be before 1973. */ +const EPOCH_SECONDS_CEILING = 1e11; +const MONTHS_PER_QUARTER = 3; +const DAYS_PER_WEEK = 7; + +type Accumulator = { + group: RecordData; + records: number; + values: number; + numbers: number; + sum: number; + min: unknown; + max: unknown; +}; + +function toDate(value: unknown): Date | null { + let date: Date; + + if (value instanceof Date) date = value; + else if (typeof value === 'number') { + date = new Date(value < EPOCH_SECONDS_CEILING ? value * 1000 : value); + } else if (typeof value === 'string') date = new Date(value); + else return null; + + return Number.isNaN(date.getTime()) ? null : date; +} + +const formatters = new Map(); + +function localDay(date: Date, timezone: string): { year: number; month: number; day: number } { + let formatter = formatters.get(timezone); + + if (!formatter) { + formatter = new Intl.DateTimeFormat('en-CA', { + timeZone: timezone, + year: 'numeric', + month: '2-digit', + day: '2-digit', + }); + formatters.set(timezone, formatter); + } + + const parts = Object.fromEntries(formatter.formatToParts(date).map(p => [p.type, p.value])); + + return { year: Number(parts.year), month: Number(parts.month), day: Number(parts.day) }; +} + +const two = (n: number) => String(n).padStart(2, '0'); + +function dateBucket(value: unknown, operation: DateOperation, timezone: string): string | null { + const date = toDate(value); + + if (!date) return null; + + const { year, month, day } = localDay(date, timezone); + + switch (operation) { + case 'Year': + return `${year}-01-01`; + case 'Quarter': + return `${year}-${two( + Math.floor((month - 1) / MONTHS_PER_QUARTER) * MONTHS_PER_QUARTER + 1, + )}-01`; + case 'Month': + return `${year}-${two(month)}-01`; + case 'Week': { + const monday = new Date(Date.UTC(year, month - 1, day)); + monday.setUTCDate( + monday.getUTCDate() - ((monday.getUTCDay() + DAYS_PER_WEEK - 1) % DAYS_PER_WEEK), + ); + + return monday.toISOString().slice(0, 10); + } + default: + return `${year}-${two(month)}-${two(day)}`; + } +} + +/** + * DynamoDB cannot aggregate, so records are reduced here as they are read, group by group, + * without ever holding the whole table. + */ +export default class StreamingAggregator { + private readonly accumulators = new Map(); + + constructor(private readonly aggregation: Aggregation, private readonly timezone: string) {} + + add(record: RecordData): void { + const group: RecordData = {}; + + for (const { field, operation } of this.aggregation.groups ?? []) { + const value = record[field] ?? null; + group[field] = operation ? dateBucket(value, operation, this.timezone) : value; + } + + const key = JSON.stringify(Object.values(group)); + let accumulator = this.accumulators.get(key); + + if (!accumulator) { + accumulator = this.emptyAccumulator(group); + this.accumulators.set(key, accumulator); + } + + accumulator.records += 1; + + if (!this.aggregation.field) return; + + const value = record[this.aggregation.field]; + + if (value === undefined || value === null) return; + + accumulator.values += 1; + if (accumulator.min === undefined || value < accumulator.min) accumulator.min = value; + if (accumulator.max === undefined || value > accumulator.max) accumulator.max = value; + + if (typeof value === 'number' && !Number.isNaN(value)) { + accumulator.numbers += 1; + accumulator.sum += value; + } + } + + result(limit?: number): AggregateResult[] { + const accumulators = [...this.accumulators.values()]; + + if (accumulators.length === 0 && !this.aggregation.groups?.length) { + accumulators.push(this.emptyAccumulator({})); + } + + const rows = accumulators.map(accumulator => ({ + group: accumulator.group, + value: this.valueOf(accumulator), + })); + + rows.sort((a, b) => { + if (a.value === b.value) return 0; + if (a.value === null) return 1; + if (b.value === null) return -1; + + return a.value < b.value ? 1 : -1; + }); + + return limit && rows.length > limit ? rows.slice(0, limit) : rows; + } + + private emptyAccumulator(group: RecordData): Accumulator { + return { + group, + records: 0, + values: 0, + numbers: 0, + sum: 0, + min: undefined, + max: undefined, + }; + } + + private valueOf(accumulator: Accumulator): unknown { + switch (this.aggregation.operation) { + case 'Count': + return this.aggregation.field ? accumulator.values : accumulator.records; + case 'Sum': + return accumulator.sum; + case 'Avg': + return accumulator.numbers ? accumulator.sum / accumulator.numbers : null; + case 'Min': + return accumulator.min ?? null; + default: + return accumulator.max ?? null; + } + } +} diff --git a/packages/datasource-dynamodb/src/utils/concurrency.ts b/packages/datasource-dynamodb/src/utils/concurrency.ts new file mode 100644 index 0000000000..93eda46fd9 --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/concurrency.ts @@ -0,0 +1,35 @@ +/** Maps with at most `limit` calls in flight, keeping the order of the results. */ +export async function mapWithConcurrency( + items: T[], + limit: number, + handler: (item: T, index: number) => Promise, +): Promise { + const results = new Array(items.length); + let next = 0; + + const worker = async () => { + while (next < items.length) { + const index = next; + next += 1; + // eslint-disable-next-line no-await-in-loop + results[index] = await handler(items[index], index); + } + }; + + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)); + + return results; +} + +export function chunk(items: T[], size: number): T[][] { + const chunks: T[][] = []; + + for (let i = 0; i < items.length; i += size) chunks.push(items.slice(i, i + size)); + + return chunks; +} + +export const sleep = (ms: number): Promise => + new Promise(resolve => { + setTimeout(resolve, ms); + }); diff --git a/packages/datasource-dynamodb/src/utils/executor.ts b/packages/datasource-dynamodb/src/utils/executor.ts new file mode 100644 index 0000000000..43a1fa1cb4 --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/executor.ts @@ -0,0 +1,260 @@ +import type { CollectionSettings, TableModel } from '../types'; +import type { StartKey } from './pagination-cache'; +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import type { Logger } from '@forestadmin/datasource-toolkit'; + +import { DescribeTableCommand } from '@aws-sdk/client-dynamodb'; +import { + BatchGetCommand, + BatchWriteCommand, + GetCommand, + PutCommand, + QueryCommand, + ScanCommand, + UpdateCommand, +} from '@aws-sdk/lib-dynamodb'; + +import { DynamoDBOperationError, ScanLimitExceededError } from '../errors'; +import { chunk, mapWithConcurrency, sleep } from './concurrency'; + +export type Item = Record; + +export type ReadOperation = 'query' | 'scan'; + +export type ReadInput = { + IndexName?: string; + KeyConditionExpression?: string; + FilterExpression?: string; + ProjectionExpression?: string; + ExpressionAttributeNames?: Record; + ExpressionAttributeValues?: Record; + ScanIndexForward?: boolean; + Limit?: number; + Select?: 'COUNT'; + Segment?: number; + TotalSegments?: number; + ExclusiveStartKey?: StartKey; +}; + +export type Page = { items: Item[]; count: number; lastKey?: StartKey }; + +const BATCH_GET_SIZE = 100; +const BATCH_WRITE_SIZE = 25; +const WRITE_CONCURRENCY = 10; + +/** Counts the items a request reads, across the segments of a parallel scan, and caps them. */ +export class ReadBudget { + private scanned = 0; + private aborted = false; + + constructor(private readonly tableName: string, private readonly max: number) {} + + /** Stops the other segments of a parallel read once one of them has failed. */ + abort(): void { + this.aborted = true; + } + + get isAborted(): boolean { + return this.aborted; + } + + charge(items: number, operation: ReadOperation): void { + this.scanned += items; + + if (this.scanned > this.max) { + throw new ScanLimitExceededError( + `${operation === 'scan' ? 'Scan' : 'Query'} on table '${this.tableName}' stopped after ` + + `${this.scanned} items: maxScannedItems is ${this.max}. Narrow the results with filters.`, + ); + } + } +} + +/** Sends the requests of one collection: paging, batching, retries, and what gets logged. */ +export default class DynamoExecutor { + constructor( + private readonly client: DynamoDBDocumentClient, + private readonly model: TableModel, + private readonly settings: CollectionSettings, + private readonly logger: Logger, + ) {} + + get maxScannedItems(): number { + return this.model.maxScannedItems ?? this.settings.maxScannedItems; + } + + newBudget(): ReadBudget { + return new ReadBudget(this.model.tableName, this.maxScannedItems); + } + + async *readPages( + operation: ReadOperation, + input: ReadInput, + budget: ReadBudget = this.newBudget(), + ): AsyncGenerator { + let startKey = input.ExclusiveStartKey; + + this.logQuery(operation, input); + + do { + if (budget.isAborted) return; + + const request = { + TableName: this.model.tableName, + ...input, + ...this.capacityRequest(), + ...(startKey ? { ExclusiveStartKey: startKey } : {}), + }; + // eslint-disable-next-line no-await-in-loop + const response = await this.client.send( + operation === 'query' ? new QueryCommand(request) : new ScanCommand(request), + ); + + this.logCapacity(operation, response.ConsumedCapacity?.CapacityUnits); + budget.charge( + response.ScannedCount ?? response.Count ?? response.Items?.length ?? 0, + operation, + ); + startKey = response.LastEvaluatedKey; + + yield { + items: (response.Items ?? []) as Item[], + count: response.Count ?? response.Items?.length ?? 0, + lastKey: startKey, + }; + } while (startKey); + } + + /** Reads items by key, 100 per request, asking again for the ones DynamoDB did not process. */ + async batchGet( + keys: Item[], + projection: Pick = {}, + ): Promise { + const items: Item[] = []; + + for (const keysChunk of chunk(keys, BATCH_GET_SIZE)) { + let pending: Item[] = keysChunk; + + for (let attempt = 0; pending.length > 0; attempt += 1) { + if (attempt > 0) { + // eslint-disable-next-line no-await-in-loop + await this.backoff(attempt, pending.length); + } + + // eslint-disable-next-line no-await-in-loop + const response = await this.client.send( + new BatchGetCommand({ + RequestItems: { [this.model.tableName]: { Keys: pending, ...projection } }, + ...this.capacityRequest(), + }), + ); + + items.push(...((response.Responses?.[this.model.tableName] ?? []) as Item[])); + pending = (response.UnprocessedKeys?.[this.model.tableName]?.Keys ?? []) as Item[]; + } + } + + return items; + } + + async batchDelete(keys: Item[]): Promise { + await mapWithConcurrency(chunk(keys, BATCH_WRITE_SIZE), WRITE_CONCURRENCY, async keysChunk => { + let pending = keysChunk.map(Key => ({ DeleteRequest: { Key } })); + + for (let attempt = 0; pending.length > 0; attempt += 1) { + if (attempt > 0) { + // eslint-disable-next-line no-await-in-loop + await this.backoff(attempt, pending.length); + } + + // eslint-disable-next-line no-await-in-loop + const response = await this.client.send( + new BatchWriteCommand({ + RequestItems: { [this.model.tableName]: pending }, + ...this.capacityRequest(), + }), + ); + + pending = (response.UnprocessedItems?.[this.model.tableName] ?? []) as typeof pending; + } + }); + } + + async mapWrites(items: T[], handler: (item: T) => Promise): Promise { + return mapWithConcurrency(items, WRITE_CONCURRENCY, handler); + } + + async put(input: Item): Promise { + const response = await this.client.send( + new PutCommand({ + TableName: this.model.tableName, + ...input, + ...this.capacityRequest(), + } as never), + ); + this.logCapacity('put', response.ConsumedCapacity?.CapacityUnits); + } + + async update(input: Item): Promise { + const response = await this.client.send( + new UpdateCommand({ + TableName: this.model.tableName, + ...input, + ...this.capacityRequest(), + } as never), + ); + this.logCapacity('update', response.ConsumedCapacity?.CapacityUnits); + + return response.Attributes; + } + + async get(key: Item): Promise { + const response = await this.client.send( + new GetCommand({ TableName: this.model.tableName, Key: key, ...this.capacityRequest() }), + ); + this.logCapacity('get', response.ConsumedCapacity?.CapacityUnits); + + return response.Item; + } + + /** The item count DynamoDB keeps for the table, refreshed about every six hours. */ + async describeItemCount(): Promise { + const { Table: table } = await this.client.send( + new DescribeTableCommand({ TableName: this.model.tableName }), + ); + + return table.ItemCount ?? 0; + } + + private async backoff(attempt: number, unprocessed: number): Promise { + if (attempt > this.settings.maxWriteRetries) { + throw new DynamoDBOperationError( + `${unprocessed} item(s) are still unprocessed after ${this.settings.maxWriteRetries} retries`, + undefined, + ); + } + + await sleep(this.settings.writeRetryBaseDelayMs * 2 ** (attempt - 1)); + } + + private capacityRequest(): { ReturnConsumedCapacity?: 'TOTAL' } { + return this.settings.logConsumedCapacity ? { ReturnConsumedCapacity: 'TOTAL' } : {}; + } + + private logCapacity(operation: string, units?: number): void { + if (this.settings.logConsumedCapacity && units !== undefined) { + this.logger('Info', `[DynamoDB Capacity] ${this.model.name}.${operation}: ${units} units`); + } + } + + private logQuery(operation: ReadOperation, input: ReadInput): void { + if (!this.settings.logQueries) return; + + this.logger( + 'Info', + `[DynamoDB Query] ${this.model.name}.${operation}: ` + + `KeyCondition=${input.KeyConditionExpression ?? '-'} | ` + + `Filter=${input.FilterExpression ?? '-'} | Index=${input.IndexName ?? 'table'}`, + ); + } +} diff --git a/packages/datasource-dynamodb/src/utils/expression-builder.ts b/packages/datasource-dynamodb/src/utils/expression-builder.ts new file mode 100644 index 0000000000..2edd9b212c --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/expression-builder.ts @@ -0,0 +1,246 @@ +import type { ConditionTree, Operator } from '@forestadmin/datasource-toolkit'; + +import { + ConditionTreeBranch, + ConditionTreeFactory, + ConditionTreeLeaf, + ValidationError, +} from '@forestadmin/datasource-toolkit'; + +import Serializer from './serializer'; + +/** DynamoDB accepts at most 100 operands in an IN. */ +const MAX_IN_OPERANDS = 100; +const NULL_TYPE = 'NULL'; + +const COMPARISONS: Partial> = { + NotEqual: '<>', + LessThan: '<', + LessThanOrEqual: '<=', + GreaterThan: '>', + GreaterThanOrEqual: '>=', + Before: '<', + After: '>', +}; + +/** What DynamoDB evaluates itself; any other operator is applied on the items it returns. */ +export const PUSHABLE_OPERATORS = new Set([ + 'Equal', + 'In', + 'NotIn', + 'Present', + 'Missing', + 'Blank', + 'StartsWith', + 'Contains', + 'NotContains', + 'IncludesAll', + 'IncludesNone', + ...(Object.keys(COMPARISONS) as Operator[]), +]); + +type LeafFilter = (leaf: ConditionTreeLeaf) => boolean; + +function isPushable(leaf: ConditionTreeLeaf, accept: LeafFilter): boolean { + return PUSHABLE_OPERATORS.has(leaf.operator) && !leaf.field.includes(':') && accept(leaf); +} + +function intersect(trees: ConditionTree[]): ConditionTree | undefined { + return trees.length ? ConditionTreeFactory.intersect(...trees) : undefined; +} + +/** + * Splits a condition tree between what DynamoDB can evaluate and what must be applied on the + * returned items. An OR can only be pushed whole: half of it evaluated server-side would drop + * items the other half still matches. + */ +export function splitCondition( + tree?: ConditionTree, + accept: LeafFilter = () => true, +): { + pushable?: ConditionTree; + residual?: ConditionTree; +} { + if (!tree) return { pushable: undefined, residual: undefined }; + + if (tree instanceof ConditionTreeLeaf) { + return isPushable(tree, accept) ? { pushable: tree } : { residual: tree }; + } + + const branch = tree as ConditionTreeBranch; + + if (branch.aggregator === 'Or') { + return branch.conditions.every( + condition => splitCondition(condition, accept).residual === undefined, + ) + ? { pushable: tree } + : { residual: tree }; + } + + const parts = branch.conditions.map(condition => splitCondition(condition, accept)); + const pushable = parts.map(part => part.pushable).filter(Boolean); + const residual = parts.map(part => part.residual).filter(Boolean); + + if (residual.length === 0) return { pushable: tree }; + if (pushable.length === 0) return { residual: tree }; + + return { pushable: intersect(pushable), residual: intersect(residual) }; +} + +/** + * Builds expressions that reference attributes and values by alias, which is both what + * DynamoDB's reserved words require and what keeps user values out of the expression text. + */ +export default class ExpressionBuilder { + readonly names: Record = {}; + readonly values: Record = {}; + private readonly nameAliases = new Map(); + + constructor(private readonly namePrefix = 'n', private readonly valuePrefix = 'v') {} + + /** Alias of an attribute path: `a->b` becomes `#n0.#n1`. */ + attribute(field: string): string { + if (field.includes(':')) { + throw new ValidationError(`Filtering on the relation field "${field}" is not supported`); + } + + return Serializer.toPathSegments(field) + .map(segment => this.name(segment)) + .join('.'); + } + + name(attributeName: string): string { + let alias = this.nameAliases.get(attributeName); + + if (!alias) { + alias = `#${this.namePrefix}${this.nameAliases.size}`; + this.nameAliases.set(attributeName, alias); + this.names[alias] = attributeName; + } + + return alias; + } + + value(value: unknown): string { + const alias = `:${this.valuePrefix}${Object.keys(this.values).length}`; + this.values[alias] = value instanceof Date ? value.toISOString() : value; + + return alias; + } + + /** The top-level attributes a projection needs. */ + projection(fields: string[]): string { + return [...new Set(fields.map(field => Serializer.toPathSegments(field)[0]))] + .map(attribute => this.name(attribute)) + .join(', '); + } + + result(): { + ExpressionAttributeNames?: Record; + ExpressionAttributeValues?: Record; + } { + return { + ...(Object.keys(this.names).length ? { ExpressionAttributeNames: { ...this.names } } : {}), + ...(Object.keys(this.values).length ? { ExpressionAttributeValues: { ...this.values } } : {}), + }; + } + + condition(tree: ConditionTree): string { + if (tree instanceof ConditionTreeLeaf) return this.leaf(tree); + + const { aggregator, conditions } = tree as ConditionTreeBranch; + const expressions = conditions.map(condition => this.condition(condition)); + + if (expressions.length === 1) return expressions[0]; + + return `(${expressions.join(aggregator === 'And' ? ' AND ' : ' OR ')})`; + } + + private isNull(path: string): string { + return `attribute_type(${path}, ${this.value(NULL_TYPE)})`; + } + + private missing(path: string): string { + return `(attribute_not_exists(${path}) OR ${this.isNull(path)})`; + } + + private never(path: string): string { + return `(attribute_exists(${path}) AND attribute_not_exists(${path}))`; + } + + private always(path: string): string { + return `(attribute_exists(${path}) OR attribute_not_exists(${path}))`; + } + + private in(path: string, values: unknown[]): string { + const nonNull = values.filter(value => value !== null && value !== undefined); + const hasNull = nonNull.length !== values.length; + + if (nonNull.length === 0) return hasNull ? this.missing(path) : this.never(path); + + const groups: string[] = []; + + for (let i = 0; i < nonNull.length; i += MAX_IN_OPERANDS) { + const operands = nonNull.slice(i, i + MAX_IN_OPERANDS).map(value => this.value(value)); + groups.push(`${path} IN (${operands.join(', ')})`); + } + + const matches = groups.length === 1 ? groups[0] : `(${groups.join(' OR ')})`; + + return hasNull ? `(${matches} OR ${this.missing(path)})` : matches; + } + + private leaf(leaf: ConditionTreeLeaf): string { + const path = this.attribute(leaf.field); + const { operator, value } = leaf; + const values = Array.isArray(value) ? value : [value]; + + switch (operator) { + case 'Equal': + return value === null || value === undefined + ? this.missing(path) + : `${path} = ${this.value(value)}`; + case 'Present': + return `(attribute_exists(${path}) AND NOT ${this.isNull(path)})`; + case 'Missing': + return this.missing(path); + case 'Blank': + return `(attribute_not_exists(${path}) OR ${this.isNull(path)} OR ${path} = ${this.value( + '', + )})`; + case 'In': + return values.length === 0 ? this.never(path) : this.in(path, values); + case 'NotIn': { + if (values.length === 0) return this.always(path); + const matches = this.in(path, values); + + return matches.startsWith('(') ? `NOT ${matches}` : `NOT (${matches})`; + } + case 'StartsWith': + return `begins_with(${path}, ${this.value(value)})`; + case 'Contains': + return `contains(${path}, ${this.value(value)})`; + case 'NotContains': + return `NOT contains(${path}, ${this.value(value)})`; + case 'IncludesAll': + if (values.length === 0) return this.always(path); + + return `(${values.map(item => `contains(${path}, ${this.value(item)})`).join(' AND ')})`; + case 'IncludesNone': + if (values.length === 0) return this.always(path); + + return `(${values + .map(item => `NOT contains(${path}, ${this.value(item)})`) + .join(' AND ')})`; + default: { + const comparison = COMPARISONS[operator]; + + if (!comparison) { + throw new ValidationError(`Operator "${operator}" cannot be evaluated by DynamoDB`); + } + + return `${path} ${comparison} ${this.value(value)}`; + } + } + } +} diff --git a/packages/datasource-dynamodb/src/utils/pagination-cache.ts b/packages/datasource-dynamodb/src/utils/pagination-cache.ts new file mode 100644 index 0000000000..1ce66df51b --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/pagination-cache.ts @@ -0,0 +1,97 @@ +import { createHash } from 'crypto'; + +export type StartKey = Record; + +type Entry = { offset: number; startKey: StartKey; createdAt: number }; + +export type PaginationCacheOptions = { + /** Positions remembered per query. Default: 100. */ + maxEntriesPerQuery?: number; + /** Queries remembered. Default: 1000. */ + maxQueries?: number; + /** How long a position stays usable, in milliseconds. Default: 5 minutes. */ + ttlMs?: number; + /** Records between two positions remembered while reading. Default: 1000. */ + cacheInterval?: number; + now?: () => number; +}; + +const DEFAULTS = { + maxEntriesPerQuery: 100, + maxQueries: 1000, + ttlMs: 5 * 60 * 1000, + cacheInterval: 1000, +}; + +function stableStringify(value: unknown): string { + return JSON.stringify(value, (_key, nested) => { + if (nested && typeof nested === 'object' && !Array.isArray(nested)) { + return Object.fromEntries(Object.entries(nested).sort(([a], [b]) => (a < b ? -1 : 1))); + } + + return nested; + }); +} + +/** + * DynamoDB has no OFFSET: a page is reached by resuming from the key where an earlier read of + * the same query stopped. This remembers those keys, by the number of matching records before them. + */ +export default class PaginationCache { + private readonly queries = new Map(); + private readonly options: Required; + + constructor(options: PaginationCacheOptions = {}) { + this.options = { now: () => Date.now(), ...DEFAULTS, ...options }; + } + + hash(query: unknown): string { + return createHash('sha1').update(stableStringify(query)).digest('hex'); + } + + getCacheInterval(): number { + return this.options.cacheInterval; + } + + storeToken(queryHash: string, offset: number, startKey: StartKey, table?: string): void { + const query = this.queries.get(queryHash) ?? { table, entries: [] }; + this.queries.delete(queryHash); + this.queries.set(queryHash, query); + + query.entries = query.entries.filter(entry => entry.offset !== offset); + query.entries.push({ offset, startKey, createdAt: this.options.now() }); + + if (query.entries.length > this.options.maxEntriesPerQuery) query.entries.shift(); + + if (this.queries.size > this.options.maxQueries) { + this.queries.delete(this.queries.keys().next().value); + } + } + + findBestToken( + queryHash: string, + offset: number, + ): { offset: number; startKey: StartKey } | undefined { + const query = this.queries.get(queryHash); + + if (!query) return undefined; + + const now = this.options.now(); + query.entries = query.entries.filter(entry => now - entry.createdAt <= this.options.ttlMs); + + const best = query.entries + .filter(entry => entry.offset <= offset) + .reduce( + (closest, entry) => (!closest || entry.offset > closest.offset ? entry : closest), + undefined, + ); + + return best && { offset: best.offset, startKey: best.startKey }; + } + + invalidateTable(table: string): void { + for (const [hash, query] of this.queries) { + if (query.table === table) this.queries.delete(hash); + } + } +} diff --git a/packages/datasource-dynamodb/src/utils/serializer.ts b/packages/datasource-dynamodb/src/utils/serializer.ts new file mode 100644 index 0000000000..3b1f2db4af --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/serializer.ts @@ -0,0 +1,117 @@ +import type { RecordData } from '@forestadmin/datasource-toolkit'; + +import TypeConverter from './type-converter'; + +export const PATH_SEPARATOR = '->'; + +const nestedPrefixesCache = new WeakMap, Set>(); + +/** Paths that have columns declared under them, hence are expanded rather than kept as Json. */ +function nestedPrefixes(fields: Set): Set { + let prefixes = nestedPrefixesCache.get(fields); + + if (!prefixes) { + prefixes = new Set(); + + for (const field of fields) { + const parts = field.split(PATH_SEPARATOR); + + for (let i = 1; i < parts.length; i += 1) { + prefixes.add(parts.slice(0, i).join(PATH_SEPARATOR)); + } + } + + nestedPrefixesCache.set(fields, prefixes); + } + + return prefixes; +} + +function isPlainObject(value: unknown): value is Record { + return ( + typeof value === 'object' && + value !== null && + !Array.isArray(value) && + !(value instanceof Date) && + !(value instanceof Set) && + !(value instanceof Uint8Array) && + !TypeConverter.isGeoPoint(value) + ); +} + +export default class Serializer { + /** Makes a value out of what the DocumentClient reads: no set, no date object, no buffer. */ + static normalize(value: unknown): unknown { + if (value instanceof Set) return [...value].map(item => Serializer.normalize(item)); + if (value instanceof Date) return value.toISOString(); + if (value instanceof Uint8Array) return Buffer.from(value).toString('base64'); + if (Array.isArray(value)) return value.map(item => Serializer.normalize(item)); + + if (isPlainObject(value)) { + return Object.fromEntries( + Object.entries(value).map(([key, nested]) => [key, Serializer.normalize(nested)]), + ); + } + + return value; + } + + /** Turns an item into a record: maps that have declared columns become `a->b` fields. */ + static flatten(item: Record, fields: Set): RecordData { + const prefixes = nestedPrefixes(fields); + const record: RecordData = {}; + + const visit = (value: Record, prefix: string) => { + for (const [key, nested] of Object.entries(value)) { + const path = prefix ? `${prefix}${PATH_SEPARATOR}${key}` : key; + + if (isPlainObject(nested) && prefixes.has(path)) visit(nested, path); + else record[path] = Serializer.normalize(nested); + } + }; + + visit(item, ''); + + return record; + } + + /** Turns `a->b` fields back into nested maps, ready to be written. */ + static unflatten(record: RecordData): Record { + const item: Record = {}; + + for (const [field, value] of Object.entries(record)) { + const parts = Serializer.toPathSegments(field); + let target = item; + + for (const part of parts.slice(0, -1)) { + if (!isPlainObject(target[part])) target[part] = {}; + target = target[part] as Record; + } + + target[parts[parts.length - 1]] = value instanceof Date ? value.toISOString() : value; + } + + return item; + } + + static toPathSegments(field: string): string[] { + return field.split(PATH_SEPARATOR); + } + + /** Merges `patch` into `item`, recursing into maps only. */ + static deepMerge( + item: Record, + patch: Record, + ): Record { + const merged = { ...item }; + + for (const [key, value] of Object.entries(patch)) { + merged[key] = + isPlainObject(value) && isPlainObject(merged[key]) + ? Serializer.deepMerge(merged[key] as Record, value) + : value; + } + + return merged; + } +} diff --git a/packages/datasource-dynamodb/src/utils/sort.ts b/packages/datasource-dynamodb/src/utils/sort.ts new file mode 100644 index 0000000000..6111cc0604 --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/sort.ts @@ -0,0 +1,20 @@ +import type { RecordData, Sort } from '@forestadmin/datasource-toolkit'; + +/** Missing values come last whatever the direction, as DynamoDB has no notion of nulls order. */ +export function compareRecords(a: RecordData, b: RecordData, sort: Sort): number { + for (const { field, ascending } of sort) { + const [left, right] = [a[field], b[field]]; + const leftMissing = left === undefined || left === null; + const rightMissing = right === undefined || right === null; + + if (leftMissing !== rightMissing) return leftMissing ? 1 : -1; + + if (!leftMissing && left !== right) { + const order = (left as number) < (right as number) ? -1 : 1; + + return ascending ? order : -order; + } + } + + return 0; +} diff --git a/packages/datasource-dynamodb/src/utils/type-converter.ts b/packages/datasource-dynamodb/src/utils/type-converter.ts new file mode 100644 index 0000000000..40b3646c4c --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/type-converter.ts @@ -0,0 +1,106 @@ +import type { DynamoDataType } from '../types'; +import type { ColumnType, Operator, PrimitiveTypes } from '@forestadmin/datasource-toolkit'; + +import { allowedOperatorsForColumnType } from '@forestadmin/datasource-toolkit'; + +const ISO_DATE_TIME = /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:?\d{2})?$/; +const DATE_ONLY = /^\d{4}-\d{2}-\d{2}$/; +const TIME_ONLY = /^([01]\d|2[0-3]):([0-5]\d):([0-5]\d)(\.\d+)?$/; +const NON_SORTABLE_TYPES: DynamoDataType[] = ['object', 'array', 'point', 'null']; + +function isGeoPoint(value: unknown): boolean { + const point = value as { type?: unknown; coordinates?: unknown }; + + return ( + typeof value === 'object' && + point.type === 'Point' && + Array.isArray(point.coordinates) && + point.coordinates.length >= 2 && + point.coordinates.every(coordinate => typeof coordinate === 'number') + ); +} + +function isValidDate(value: string): boolean { + return !Number.isNaN(new Date(value).getTime()); +} + +export default class TypeConverter { + static getColumnTypeFromDataType(dataType: DynamoDataType): PrimitiveTypes { + switch (dataType) { + case 'boolean': + return 'Boolean'; + case 'date': + return 'Date'; + case 'dateonly': + return 'Dateonly'; + case 'timeonly': + return 'Timeonly'; + case 'number': + return 'Number'; + case 'string': + case 'binary': + return 'String'; + case 'point': + return 'Point'; + case 'object': + case 'array': + case 'null': + return 'Json'; + default: + throw new Error(`Unsupported data type: "${dataType}"`); + } + } + + /** DynamoDB only types its keys, so the other attributes are typed from their values. */ + static inferTypeFromValue(value: unknown): DynamoDataType { + if (value === null || value === undefined) return 'null'; + if (typeof value === 'boolean') return 'boolean'; + if (typeof value === 'number') return 'number'; + + if (typeof value === 'string') { + if (TIME_ONLY.test(value)) return 'timeonly'; + if (DATE_ONLY.test(value) && isValidDate(value)) return 'dateonly'; + if (ISO_DATE_TIME.test(value) && isValidDate(value)) return 'date'; + + return 'string'; + } + + if (Array.isArray(value) || value instanceof Set) return 'array'; + if (value instanceof Date) return 'date'; + if (value instanceof Uint8Array) return 'binary'; + if (typeof value === 'object') return isGeoPoint(value) ? 'point' : 'object'; + + return 'string'; + } + + static isGeoPoint(value: unknown): boolean { + return isGeoPoint(value); + } + + static getMostSpecificType(types: DynamoDataType[]): DynamoDataType { + const unique = [...new Set(types)]; + const nonNull = unique.filter(type => type !== 'null'); + + if (unique.length === 0) return 'null'; + if (nonNull.length === 0) return 'null'; + if (nonNull.length === 1) return nonNull[0]; + + if (nonNull.every(type => type === 'date' || type === 'dateonly')) return 'date'; + + return 'object'; + } + + static isSortable(dataType: DynamoDataType): boolean { + return !NON_SORTABLE_TYPES.includes(dataType); + } + + /** + * Every operator Forest allows for the type: the ones DynamoDB cannot evaluate are applied + * on the items it returns. + */ + static operatorsForColumnType(columnType: ColumnType): Set { + return new Set( + typeof columnType === 'string' ? allowedOperatorsForColumnType[columnType] : ['Present'], + ); + } +} diff --git a/packages/datasource-dynamodb/src/virtual-collections.ts b/packages/datasource-dynamodb/src/virtual-collections.ts new file mode 100644 index 0000000000..96aeabf595 --- /dev/null +++ b/packages/datasource-dynamodb/src/virtual-collections.ts @@ -0,0 +1,76 @@ +import type DynamoDBCollection from './collection'; +import type DynamoDBDataSource from './datasource'; +import type { AttributeSchema, VirtualCollectionConfig } from './types'; +import type { Caller, Logger } from '@forestadmin/datasource-toolkit'; + +import { + ConditionTreeLeaf, + Page, + PaginatedFilter, + Projection, +} from '@forestadmin/datasource-toolkit'; + +import ArrayCollection from './array-collection'; +import { DEFAULT_MAX_DEPTH, DEFAULT_SAMPLE_SIZE, inferSchema } from './introspection/introspector'; + +const SAMPLING_CALLER = { timezone: 'UTC' } as Caller; + +/** The attributes of an item come from the schema when it declares them, else from a sample. */ +async function itemSchemaOf( + parent: DynamoDBCollection, + arrayField: string, + sampleSize: number, +): Promise> { + const declared = parent.model.schema[arrayField]?.items; + + if (declared) return declared; + + const parents = await parent.list( + SAMPLING_CALLER, + new PaginatedFilter({ + conditionTree: new ConditionTreeLeaf(arrayField, 'Present'), + page: new Page(0, sampleSize), + }), + new Projection(arrayField), + ); + const items = parents + .flatMap(record => (Array.isArray(record[arrayField]) ? (record[arrayField] as unknown[]) : [])) + .filter((item): item is Record => !!item && typeof item === 'object'); + + return inferSchema(items, [], DEFAULT_MAX_DEPTH); +} + +/** Adds one collection per configured array, each skipped with a warning if it cannot be built. */ +export default async function addVirtualCollections( + dataSource: DynamoDBDataSource, + configs: VirtualCollectionConfig[], + sampleSize: number = DEFAULT_SAMPLE_SIZE, + logger?: Logger, +): Promise { + for (const config of configs) { + try { + const parent = dataSource.getCollection(config.parentCollection); + + if (parent instanceof ArrayCollection) { + throw new Error('an array of an array cannot be exposed'); + } + + // eslint-disable-next-line no-await-in-loop + const itemSchema = await itemSchemaOf(parent, config.arrayField, sampleSize); + + dataSource.addCollection( + new ArrayCollection(dataSource, parent, { + name: config.collectionName, + arrayField: config.arrayField, + itemSchema, + }), + ); + } catch (error) { + logger?.( + 'Warn', + `Skipping virtual collection '${config.collectionName}' for array ` + + `'${config.parentCollection}.${config.arrayField}': ${error.message}`, + ); + } + } +} diff --git a/packages/datasource-dynamodb/test/array-collection.test.ts b/packages/datasource-dynamodb/test/array-collection.test.ts new file mode 100644 index 0000000000..964d278386 --- /dev/null +++ b/packages/datasource-dynamodb/test/array-collection.test.ts @@ -0,0 +1,312 @@ +import { + Aggregation, + Filter, + PaginatedFilter, + Projection, + Sort, +} from '@forestadmin/datasource-toolkit'; +import type { DataSource } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { and, buildModel, leaf, plain } from './helpers'; +import ArrayCollection from '../src/array-collection'; + +const caller = { timezone: 'UTC' } as never; + +const orders = [ + { + customerId: 'c1', + orderId: 'o1', + lines: [ + { sku: 'A', qty: 1 }, + { sku: 'B', qty: 5 }, + ], + }, + { customerId: 'c2', orderId: 'o2', lines: [{ sku: 'C', qty: 3 }] }, + { customerId: 'c3', orderId: 'o3' }, +]; + +function build(handlers: Record) { + const parent = buildCollection( + buildModel({ + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + lines: { type: 'array' }, + }, + }), + handlers, + { scanConcurrency: 1 }, + ); + const collection = new ArrayCollection({} as DataSource, parent.collection, { + name: 'order_lines', + arrayField: 'lines', + itemSchema: { sku: { type: 'string' }, qty: { type: 'number' } }, + }); + + return { collection, client: parent.client }; +} + +describe('ArrayCollection', () => { + describe('schema', () => { + it('is keyed by the parent key and the position in the array', () => { + const { collection } = build({}); + + expect(collection.name).toBe('order_lines'); + expect( + Object.entries(collection.schema.fields) + .filter(([, field]) => field.type === 'Column' && field.isPrimaryKey) + .map(([name]) => name), + ).toEqual(['customerId', 'orderId', 'index']); + expect(collection.schema.fields.index).toMatchObject({ + columnType: 'Number', + isReadOnly: true, + }); + expect(collection.schema.fields.customerId).toMatchObject({ isReadOnly: true }); + }); + + it('exposes the item attributes as columns', () => { + const { collection } = build({}); + + expect(collection.schema.fields.sku).toMatchObject({ columnType: 'String' }); + expect(collection.schema.fields.qty).toMatchObject({ columnType: 'Number' }); + }); + }); + + describe('list', () => { + it('turns each array item of each parent into a record', async () => { + const { collection } = build({ Scan: () => ({ Items: orders }) }); + + const records = await collection.list( + caller, + new PaginatedFilter({}), + new Projection('customerId', 'orderId', 'index', 'sku', 'qty'), + ); + + expect(records).toEqual([ + { customerId: 'c1', orderId: 'o1', index: 0, sku: 'A', qty: 1 }, + { customerId: 'c1', orderId: 'o1', index: 1, sku: 'B', qty: 5 }, + { customerId: 'c2', orderId: 'o2', index: 0, sku: 'C', qty: 3 }, + ]); + }); + + it('reads the parents through their key when the filter pins it', async () => { + const { collection, client } = build({ Query: () => ({ Items: [orders[0]] }) }); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + new Projection('index', 'sku'), + ); + + expect(records).toHaveLength(2); + expect(client.calls('Query')[0]).toMatchObject({ KeyConditionExpression: '#n0 = :v0' }); + expect(client.calls('Scan')).toHaveLength(0); + }); + + it('evaluates the conditions on the items themselves', async () => { + const { collection } = build({ Scan: () => ({ Items: orders }) }); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('qty', 'GreaterThan', 2) }), + new Projection('sku'), + ); + + expect(records).toEqual([{ sku: 'B' }, { sku: 'C' }]); + }); + + it('sorts and paginates the records', async () => { + const { collection } = build({ Scan: () => ({ Items: orders }) }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + sort: new Sort({ field: 'qty', ascending: false }), + page: { limit: 1, skip: 1 } as never, + }), + new Projection('sku'), + ); + + expect(records).toEqual([{ sku: 'C' }]); + }); + + it('selects the item by its position', async () => { + const { collection } = build({ BatchGet: () => ({ Responses: { Orders: [orders[0]] } }) }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: and( + plain('customerId', 'Equal', 'c1'), + plain('orderId', 'Equal', 'o1'), + plain('index', 'Equal', 1), + ), + }), + new Projection('sku'), + ); + + expect(records).toEqual([{ sku: 'B' }]); + }); + }); + + describe('create', () => { + it('appends the item to the array of its parent and returns its position', async () => { + const { collection, client } = build({ + Update: () => ({ + Attributes: { + lines: [ + { sku: 'A', qty: 1 }, + { sku: 'N', qty: 2 }, + ], + }, + }), + }); + + const created = await collection.create(caller, [ + { customerId: 'c1', orderId: 'o1', sku: 'N', qty: 2 }, + ]); + + expect(created).toEqual([{ customerId: 'c1', orderId: 'o1', index: 1, sku: 'N', qty: 2 }]); + expect(client.calls('Update')[0]).toEqual({ + TableName: 'Orders', + Key: { customerId: 'c1', orderId: 'o1' }, + UpdateExpression: 'SET #n0 = list_append(if_not_exists(#n0, :v0), :v1)', + ConditionExpression: 'attribute_exists(#k0)', + ExpressionAttributeNames: { '#n0': 'lines', '#k0': 'customerId' }, + ExpressionAttributeValues: { ':v0': [], ':v1': [{ sku: 'N', qty: 2 }] }, + ReturnValues: 'UPDATED_NEW', + }); + }); + + it('requires the whole key of the parent', async () => { + const { collection } = build({}); + + await expect(collection.create(caller, [{ customerId: 'c1', sku: 'N' }])).rejects.toThrow( + "The key attribute 'orderId' of the parent 'orders' is required to create a record in 'order_lines'", + ); + }); + }); + + describe('update', () => { + it('sets the attributes of the item at its position', async () => { + const { collection, client } = build({ + BatchGet: () => ({ Responses: { Orders: [orders[0]] } }), + Update: () => ({}), + }); + + await collection.update( + caller, + new Filter({ + conditionTree: and( + plain('customerId', 'Equal', 'c1'), + plain('orderId', 'Equal', 'o1'), + plain('index', 'Equal', 1), + ), + }), + { qty: 9 }, + ); + + expect(client.calls('Update')).toEqual([ + { + TableName: 'Orders', + Key: { customerId: 'c1', orderId: 'o1' }, + UpdateExpression: 'SET #n0[1].#n1 = :v0', + ConditionExpression: 'attribute_exists(#k0)', + ExpressionAttributeNames: { '#n0': 'lines', '#n1': 'qty', '#k0': 'customerId' }, + ExpressionAttributeValues: { ':v0': 9 }, + }, + ]); + }); + + it('groups the items of the same parent in one update', async () => { + const { collection, client } = build({ + Query: () => ({ Items: [orders[0]] }), + Update: () => ({}), + }); + + await collection.update( + caller, + new Filter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + { qty: 0 }, + ); + + expect(client.calls('Update')).toHaveLength(1); + expect(client.calls('Update')[0].UpdateExpression).toBe( + 'SET #n0[0].#n1 = :v0, #n0[1].#n1 = :v0', + ); + }); + + it('refuses to change the parent key or the position', async () => { + const { collection } = build({}); + + await expect(collection.update(caller, new Filter({}), { customerId: 'x' })).rejects.toThrow( + "The attribute 'customerId' of 'order_lines' cannot be updated", + ); + await expect(collection.update(caller, new Filter({}), { index: 3 })).rejects.toThrow( + "The attribute 'index' of 'order_lines' cannot be updated", + ); + }); + }); + + describe('delete', () => { + it('removes the matching items from their parent, highest position first', async () => { + const { collection, client } = build({ + Query: () => ({ Items: [orders[0]] }), + Update: () => ({}), + }); + + await collection.delete( + caller, + new Filter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + ); + + expect(client.calls('Update')).toEqual([ + { + TableName: 'Orders', + Key: { customerId: 'c1', orderId: 'o1' }, + UpdateExpression: 'REMOVE #n0[1], #n0[0]', + ConditionExpression: 'attribute_exists(#k0)', + ExpressionAttributeNames: { '#n0': 'lines', '#k0': 'customerId' }, + }, + ]); + }); + + it('does nothing when no item matches', async () => { + const { collection, client } = build({ Scan: () => ({ Items: [orders[2]] }) }); + + await collection.delete(caller, new Filter({})); + + expect(client.calls('Update')).toHaveLength(0); + }); + }); + + describe('aggregate', () => { + it('aggregates the items', async () => { + const { collection } = build({ Scan: () => ({ Items: orders }) }); + + const result = await collection.aggregate( + caller, + new Filter({}), + new Aggregation({ operation: 'Sum', field: 'qty' }), + ); + + expect(result).toEqual([{ value: 9, group: {} }]); + }); + + it('groups the items by an attribute of their parent', async () => { + const { collection } = build({ Scan: () => ({ Items: orders }) }); + + const result = await collection.aggregate( + caller, + new Filter({}), + new Aggregation({ operation: 'Count', groups: [{ field: 'customerId' }] }), + ); + + expect(result).toEqual([ + { value: 2, group: { customerId: 'c1' } }, + { value: 1, group: { customerId: 'c2' } }, + ]); + }); + }); +}); diff --git a/packages/datasource-dynamodb/test/collection-helpers.ts b/packages/datasource-dynamodb/test/collection-helpers.ts new file mode 100644 index 0000000000..3bf7813d33 --- /dev/null +++ b/packages/datasource-dynamodb/test/collection-helpers.ts @@ -0,0 +1,26 @@ +import type { CollectionSettings, TableModel } from '../src/types'; +import type { DataSource } from '@forestadmin/datasource-toolkit'; + +import { fakeClient } from './fake-client'; +import DynamoDBCollection from '../src/collection'; +import { resolveSettings } from '../src/settings'; + +export function buildCollection( + model: TableModel, + handlers: Record) => unknown) | unknown[]>, + settings: Partial = {}, +) { + const client = fakeClient(handlers); + const logger = jest.fn(); + const collection = new DynamoDBCollection( + {} as DataSource, + model, + client, + logger, + resolveSettings(settings), + ); + + return { collection, client, logger }; +} + +export const caller = { timezone: 'UTC' } as never; diff --git a/packages/datasource-dynamodb/test/collection.aggregate.test.ts b/packages/datasource-dynamodb/test/collection.aggregate.test.ts new file mode 100644 index 0000000000..7dbb838172 --- /dev/null +++ b/packages/datasource-dynamodb/test/collection.aggregate.test.ts @@ -0,0 +1,462 @@ +import { Aggregation, Filter } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { and, buildModel, buildSimpleModel, leaf, plain } from './helpers'; + +const caller = { timezone: 'UTC' } as never; +const aggregation = (plainAggregation: ConstructorParameters[0]) => + new Aggregation(plainAggregation); + +describe('DynamoDBCollection.aggregate', () => { + describe('count', () => { + it('counts a query with Select COUNT, reading no item', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: [ + { Count: 40, LastEvaluatedKey: { customerId: 'c1', orderId: 'o40' } }, + { Count: 2 }, + ], + }); + + const result = await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + aggregation({ operation: 'Count' }), + ); + + expect(result).toEqual([{ value: 42, group: {} }]); + expect(client.calls('Query')[0]).toMatchObject({ Select: 'COUNT' }); + expect(client.calls('Query')[1]).toMatchObject({ + ExclusiveStartKey: { customerId: 'c1', orderId: 'o40' }, + }); + }); + + it('counts a table with a parallel Scan and sums the segments', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: (input: any) => ({ Count: input.Segment + 1 }) }, + { scanConcurrency: 3 }, + ); + + const result = await collection.aggregate( + caller, + new Filter({}), + aggregation({ operation: 'Count' }), + ); + + expect(result).toEqual([{ value: 1 + 2 + 3, group: {} }]); + expect(client.calls('Scan').map(c => [c.Segment, c.TotalSegments])).toEqual([ + [0, 3], + [1, 3], + [2, 3], + ]); + expect(client.calls('Scan')[0]).toMatchObject({ Select: 'COUNT' }); + }); + + it('does not use segments for a single-segment scan', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Count: 7 }) }, + { scanConcurrency: 1 }, + ); + + await collection.aggregate(caller, new Filter({}), aggregation({ operation: 'Count' })); + + expect(client.calls('Scan')[0].Segment).toBeUndefined(); + expect(client.calls('Scan')[0].TotalSegments).toBeUndefined(); + }); + + it('pushes the filter into the counting request', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Count: 1 }) }, + { scanConcurrency: 1 }, + ); + + await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('age', 'GreaterThan', 3) }), + aggregation({ operation: 'Count' }), + ); + + expect(client.calls('Scan')[0]).toMatchObject({ + FilterExpression: '#n0 > :v0', + Select: 'COUNT', + }); + }); + + it('reads the items when part of the filter cannot be pushed down', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { + Scan: () => ({ + Items: [ + { id: 'a', name: 'xc' }, + { id: 'b', name: 'xd' }, + ], + }), + }, + { scanConcurrency: 1 }, + ); + + const result = await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('name', 'EndsWith', 'c') }), + aggregation({ operation: 'Count' }), + ); + + expect(result).toEqual([{ value: 1, group: {} }]); + expect(client.calls('Scan')[0].Select).toBeUndefined(); + }); + + it('answers an unfiltered count from the table description when approximate counts are on', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { DescribeTable: () => ({ Table: { ItemCount: 1234 } }) }, + { approximateCount: true }, + ); + + const result = await collection.aggregate( + caller, + new Filter({}), + aggregation({ operation: 'Count' }), + ); + + expect(result).toEqual([{ value: 1234, group: {} }]); + expect(client.calls('Scan')).toHaveLength(0); + }); + + it('still counts exactly a filtered request when approximate counts are on', async () => { + const { collection } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Count: 3 }) }, + { approximateCount: true, scanConcurrency: 1 }, + ); + + const result = await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('age', 'Equal', 3) }), + aggregation({ operation: 'Count' }), + ); + + expect(result).toEqual([{ value: 3, group: {} }]); + }); + }); + + describe('reducing operations', () => { + it('reads only the attributes it needs and reduces them', async () => { + const { collection, client } = buildCollection( + buildModel(), + { + Query: () => ({ + Items: [ + { total: 10, status: 'paid' }, + { total: 5, status: 'paid' }, + { total: 1, status: 'open' }, + ], + }), + }, + { scanConcurrency: 1 }, + ); + + const result = await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + aggregation({ operation: 'Sum', field: 'total', groups: [{ field: 'status' }] }), + ); + + expect(result).toEqual([ + { value: 15, group: { status: 'paid' } }, + { value: 1, group: { status: 'open' } }, + ]); + expect(client.calls('Query')[0]).toMatchObject({ + ProjectionExpression: '#n1, #n2', + ExpressionAttributeNames: { '#n0': 'customerId', '#n1': 'total', '#n2': 'status' }, + }); + }); + + it('groups on several fields', async () => { + const { collection } = buildCollection( + buildSimpleModel(), + { + Scan: () => ({ + Items: [ + { a: 1, b: 'x' }, + { a: 1, b: 'x' }, + { a: 2, b: 'x' }, + ], + }), + }, + { scanConcurrency: 1 }, + ); + + const result = await collection.aggregate( + caller, + new Filter({}), + aggregation({ operation: 'Count', groups: [{ field: 'a' }, { field: 'b' }] }), + ); + + expect(result).toEqual([ + { value: 2, group: { a: 1, b: 'x' } }, + { value: 1, group: { a: 2, b: 'x' } }, + ]); + }); + + it('groups by a date part in the caller timezone', async () => { + const { collection } = buildCollection( + buildModel(), + { Scan: () => ({ Items: [{ createdAt: '2024-01-20T23:30:00Z' }] }) }, + { scanConcurrency: 1 }, + ); + + const result = await collection.aggregate( + { timezone: 'Asia/Tokyo' } as never, + new Filter({}), + aggregation({ operation: 'Count', groups: [{ field: 'createdAt', operation: 'Day' }] }), + ); + + expect(result).toEqual([{ value: 1, group: { createdAt: '2024-01-21' } }]); + }); + + it('applies the limit to the groups', async () => { + const { collection } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Items: [{ a: 1 }, { a: 2 }, { a: 2 }, { a: 3 }, { a: 3 }, { a: 3 }] }) }, + { scanConcurrency: 1 }, + ); + + const result = await collection.aggregate( + caller, + new Filter({}), + aggregation({ operation: 'Count', groups: [{ field: 'a' }] }), + 2, + ); + + expect(result.map(r => r.group.a)).toEqual([3, 2]); + }); + + it('merges the segments of a parallel scan', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { + Scan: (input: any) => ({ + Items: [{ age: input.Segment + 1 }], + }), + }, + { scanConcurrency: 2 }, + ); + + const result = await collection.aggregate( + caller, + new Filter({}), + aggregation({ operation: 'Sum', field: 'age' }), + ); + + expect(result).toEqual([{ value: 3, group: {} }]); + expect(client.calls('Scan')).toHaveLength(2); + }); + + it('follows LastEvaluatedKey inside each segment', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { + Scan: [{ Items: [{ age: 1 }], LastEvaluatedKey: { id: 'a' } }, { Items: [{ age: 2 }] }], + }, + { scanConcurrency: 1 }, + ); + + const result = await collection.aggregate( + caller, + new Filter({}), + aggregation({ operation: 'Sum', field: 'age' }), + ); + + expect(result).toEqual([{ value: 3, group: {} }]); + expect(client.calls('Scan')[1]).toMatchObject({ ExclusiveStartKey: { id: 'a' } }); + }); + + it('stops reading beyond maxScannedItems', async () => { + const { collection } = buildCollection( + buildSimpleModel({ maxScannedItems: 2 }), + { + Scan: () => ({ + Items: [{ age: 1 }, { age: 1 }], + ScannedCount: 2, + LastEvaluatedKey: { id: 'a' }, + }), + }, + { scanConcurrency: 1 }, + ); + + await expect( + collection.aggregate( + caller, + new Filter({}), + aggregation({ operation: 'Sum', field: 'age' }), + ), + ).rejects.toThrow("Scan on table 'Users' stopped after 4 items: maxScannedItems is 2"); + }); + + it('stops the other segments from reading on when one fails', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { + Scan: (input: any) => { + if (input.Segment === 0) throw new Error('boom'); + + return { Items: [{ age: 1 }], LastEvaluatedKey: { id: 'a' } }; + }, + }, + { scanConcurrency: 2 }, + ); + + await expect( + collection.aggregate( + caller, + new Filter({}), + aggregation({ operation: 'Sum', field: 'age' }), + ), + ).rejects.toThrow("Failed to aggregate records of 'users': boom"); + await new Promise(resolve => { + setImmediate(resolve); + }); + + expect(client.calls('Scan').filter(call => call.Segment === 1).length).toBeLessThanOrEqual(2); + }); + + it('refuses to scan under a deny policy', async () => { + const { collection } = buildCollection(buildSimpleModel({ scanPolicy: 'deny' }), {}); + + await expect( + collection.aggregate(caller, new Filter({}), aggregation({ operation: 'Count' })), + ).rejects.toThrow("Scan on table 'Users' refused by scanPolicy 'deny'"); + }); + + it('wraps a failure with the collection name', async () => { + const { collection } = buildCollection(buildSimpleModel(), { + Scan: () => { + throw new Error('boom'); + }, + }); + + await expect( + collection.aggregate(caller, new Filter({}), aggregation({ operation: 'Count' })), + ).rejects.toThrow("Failed to aggregate records of 'users': boom"); + }); + }); + + describe('result cache', () => { + it('serves the same chart from memory within the time to live', async () => { + let now = 1000; + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Count: 5 }) }, + { chartCacheTtlMs: 1000, scanConcurrency: 1, now: () => now }, + ); + const run = () => + collection.aggregate(caller, new Filter({}), aggregation({ operation: 'Count' })); + + await run(); + now += 999; + const second = await run(); + + expect(second).toEqual([{ value: 5, group: {} }]); + expect(client.calls('Scan')).toHaveLength(1); + }); + + it('reads again once the time to live has passed', async () => { + let now = 1000; + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Count: 5 }) }, + { chartCacheTtlMs: 1000, scanConcurrency: 1, now: () => now }, + ); + const run = () => + collection.aggregate(caller, new Filter({}), aggregation({ operation: 'Count' })); + + await run(); + now += 1001; + await run(); + + expect(client.calls('Scan')).toHaveLength(2); + }); + + it('does not mix charts of different filters', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Count: 5 }) }, + { chartCacheTtlMs: 1000, scanConcurrency: 1 }, + ); + + await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('age', 'Equal', 1) }), + aggregation({ operation: 'Count' }), + ); + await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('age', 'Equal', 2) }), + aggregation({ operation: 'Count' }), + ); + + expect(client.calls('Scan')).toHaveLength(2); + }); + + it('remembers at most 100 charts, dropping the oldest', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Count: 5 }) }, + { chartCacheTtlMs: 60000, scanConcurrency: 1 }, + ); + const run = (age: number) => + collection.aggregate( + caller, + new Filter({ conditionTree: leaf('age', 'Equal', age) }), + aggregation({ operation: 'Count' }), + ); + + for (let age = 0; age <= 100; age += 1) await run(age); + client.send.mockClear(); + + await run(100); + await run(0); + + expect(client.calls('Scan')).toHaveLength(1); + }); + + it('is off when the time to live is zero', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Count: 5 }) }, + { chartCacheTtlMs: 0, scanConcurrency: 1 }, + ); + const run = () => + collection.aggregate(caller, new Filter({}), aggregation({ operation: 'Count' })); + + await run(); + await run(); + + expect(client.calls('Scan')).toHaveLength(2); + }); + }); + + it('uses the access path of the filter for the aggregation', async () => { + const { collection, client } = buildCollection( + buildModel(), + { Query: () => ({ Count: 2 }) }, + { scanConcurrency: 1 }, + ); + + await collection.aggregate( + caller, + new Filter({ + conditionTree: and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'StartsWith', 'o')), + }), + aggregation({ operation: 'Count' }), + ); + + expect(client.calls('Query')[0]).toMatchObject({ + KeyConditionExpression: '#n0 = :v0 AND begins_with(#n1, :v1)', + Select: 'COUNT', + }); + }); +}); diff --git a/packages/datasource-dynamodb/test/collection.list.test.ts b/packages/datasource-dynamodb/test/collection.list.test.ts new file mode 100644 index 0000000000..cdbda404b1 --- /dev/null +++ b/packages/datasource-dynamodb/test/collection.list.test.ts @@ -0,0 +1,604 @@ +import { PaginatedFilter, Projection, Sort } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { and, buildModel, buildSimpleModel, leaf, or, plain } from './helpers'; + +const caller = { timezone: 'UTC' } as never; +const order = (n: number, extra = {}) => ({ + customerId: 'c1', + orderId: `o${n}`, + total: n, + ...extra, +}); + +describe('DynamoDBCollection.list', () => { + describe('access path', () => { + it('queries the table with a key condition when the partition key is pinned', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: () => ({ Items: [order(1)] }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + new Projection('customerId', 'orderId', 'total'), + ); + + expect(records).toEqual([{ customerId: 'c1', orderId: 'o1', total: 1 }]); + expect(client.calls('Query')[0]).toMatchObject({ + TableName: 'Orders', + KeyConditionExpression: '#n0 = :v0', + ProjectionExpression: '#n0, #n1, #n2', + ExpressionAttributeNames: { '#n0': 'customerId', '#n1': 'orderId', '#n2': 'total' }, + ExpressionAttributeValues: { ':v0': 'c1' }, + }); + expect(client.calls('Query')[0].FilterExpression).toBeUndefined(); + }); + + it('puts the remaining conditions in a filter expression', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: () => ({ Items: [] }), + }); + + await collection.list( + caller, + new PaginatedFilter({ + conditionTree: and(plain('customerId', 'Equal', 'c1'), plain('status', 'Equal', 'paid')), + }), + new Projection('orderId'), + ); + + expect(client.calls('Query')[0]).toMatchObject({ + KeyConditionExpression: '#n0 = :v0', + FilterExpression: '#n1 = :v1', + ExpressionAttributeValues: { ':v0': 'c1', ':v1': 'paid' }, + }); + }); + + it('queries a secondary index when its partition key is pinned', async () => { + const { collection, client } = buildCollection( + buildModel({ + indexes: [ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'S' }, + projectsAllAttributes: true, + }, + ], + }), + { Query: () => ({ Items: [] }) }, + ); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('status', 'Equal', 'paid') }), + new Projection('orderId'), + ); + + expect(client.calls('Query')[0]).toMatchObject({ IndexName: 'by-status' }); + }); + + it('scans when no key is pinned', async () => { + const { collection, client } = buildCollection(buildModel(), { + Scan: () => ({ Items: [order(1)] }), + }); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('status', 'Equal', 'paid') }), + new Projection('orderId'), + ); + + expect(client.calls('Scan')[0]).toMatchObject({ + TableName: 'Orders', + FilterExpression: '#n0 = :v0', + }); + }); + + it('reads the sort key backwards for a descending sort', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: () => ({ Items: [] }), + }); + + await collection.list( + caller, + new PaginatedFilter({ + conditionTree: leaf('customerId', 'Equal', 'c1'), + sort: new Sort({ field: 'orderId', ascending: false }), + }), + new Projection('orderId'), + ); + + expect(client.calls('Query')[0].ScanIndexForward).toBe(false); + }); + + it('looks keys up with BatchGetItem when the filter pins whole keys', async () => { + const { collection, client } = buildCollection(buildModel(), { + BatchGet: () => ({ + Responses: { Orders: [order(1), order(2)] }, + }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: or( + and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'Equal', 'o1')), + and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'Equal', 'o2')), + ), + }), + new Projection('customerId', 'orderId', 'total'), + ); + + expect(records).toHaveLength(2); + expect(client.calls('BatchGet')[0].RequestItems.Orders.Keys).toEqual([ + { customerId: 'c1', orderId: 'o1' }, + { customerId: 'c1', orderId: 'o2' }, + ]); + expect(client.calls('Query')).toHaveLength(0); + expect(client.calls('Scan')).toHaveLength(0); + }); + + it('splits a BatchGetItem in chunks of 100 keys and retries unprocessed keys', async () => { + const ids = Array.from({ length: 120 }, (_, i) => `u${i}`); + let call = 0; + const { collection, client } = buildCollection(buildSimpleModel(), { + BatchGet: (input: any) => { + call += 1; + const keys = input.RequestItems.Users.Keys as Array<{ id: string }>; + + if (call === 1) { + return { + Responses: { Users: keys.slice(0, 99).map(k => ({ id: k.id })) }, + UnprocessedKeys: { Users: { Keys: [keys[99]] } }, + }; + } + + return { Responses: { Users: keys.map(k => ({ id: k.id })) } }; + }, + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('id', 'In', ids) }), + new Projection('id'), + ); + + expect(records).toHaveLength(120); + expect(client.calls('BatchGet').map(c => c.RequestItems.Users.Keys.length)).toEqual([ + 100, 1, 20, + ]); + }); + + it('applies the remaining filter to a key lookup', async () => { + const { collection } = buildCollection(buildSimpleModel(), { + BatchGet: () => ({ + Responses: { + Users: [ + { id: 'u1', age: 10 }, + { id: 'u2', age: 50 }, + ], + }, + }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: and(plain('id', 'In', ['u1', 'u2']), plain('age', 'GreaterThan', 20)), + }), + new Projection('id', 'age'), + ); + + expect(records).toEqual([{ id: 'u2', age: 50 }]); + }); + }); + + describe('scan policy', () => { + it('warns by default when it has to scan, without leaking filter values', async () => { + const { collection, logger } = buildCollection(buildModel(), { + Scan: () => ({ Items: [] }), + }); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('status', 'Equal', 'secret-value') }), + new Projection('orderId'), + ); + + expect(logger).toHaveBeenCalledWith( + 'Warn', + "Scan on table 'Orders' for collection 'orders': no key or index serves the filter (status Equal)", + ); + expect(JSON.stringify(logger.mock.calls)).not.toContain('secret-value'); + }); + + it('does not warn when the scan policy is allow', async () => { + const { collection, logger } = buildCollection(buildModel({ scanPolicy: 'allow' }), { + Scan: () => ({ Items: [] }), + }); + + await collection.list(caller, new PaginatedFilter({}), new Projection('orderId')); + + expect(logger).not.toHaveBeenCalledWith('Warn', expect.stringContaining('Scan on table')); + }); + + it('refuses to scan when the scan policy is deny', async () => { + const { collection, client } = buildCollection(buildModel({ scanPolicy: 'deny' }), { + Scan: () => ({ Items: [] }), + }); + + await expect( + collection.list(caller, new PaginatedFilter({}), new Projection('orderId')), + ).rejects.toThrow( + "Scan on table 'Orders' refused by scanPolicy 'deny': no key or index serves the filter", + ); + expect(client.calls('Scan')).toHaveLength(0); + }); + + it('lets the collection override the datasource policy', async () => { + const { collection } = buildCollection( + buildModel({ scanPolicy: 'allow' }), + { Scan: () => ({ Items: [] }) }, + { scanPolicy: 'deny' }, + ); + + await expect( + collection.list(caller, new PaginatedFilter({}), new Projection('orderId')), + ).resolves.toEqual([]); + }); + + it('still queries under a deny policy', async () => { + const { collection } = buildCollection(buildModel({ scanPolicy: 'deny' }), { + Query: () => ({ Items: [order(1)] }), + }); + + await expect( + collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + new Projection('orderId'), + ), + ).resolves.toHaveLength(1); + }); + + it('stops a scan that reads more items than maxScannedItems', async () => { + const { collection } = buildCollection(buildModel({ maxScannedItems: 3 }), { + Scan: () => ({ + Items: [order(1), order(2)], + ScannedCount: 2, + LastEvaluatedKey: { customerId: 'c1', orderId: 'o2' }, + }), + }); + + await expect( + collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('status', 'Equal', 'x') }), + new Projection('orderId'), + ), + ).rejects.toThrow("Scan on table 'Orders' stopped after 4 items: maxScannedItems is 3"); + }); + }); + + describe('client-side evaluation', () => { + it('evaluates operators DynamoDB lacks on the returned items', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Scan: () => ({ + Items: [ + { id: 'u1', name: 'Émile' }, + { id: 'u2', name: 'Bob' }, + ], + }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('name', 'ILike', '%émi%') }), + new Projection('id'), + ); + + expect(records).toEqual([{ id: 'u1' }]); + expect(client.calls('Scan')[0].FilterExpression).toBeUndefined(); + expect(client.calls('Scan')[0].ProjectionExpression).toBe('#n0, #n1'); + }); + + it('pushes the supported part of the filter and evaluates the rest on the items', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Scan: () => ({ + Items: [ + { id: 'u1', name: 'abc', age: 30 }, + { id: 'u2', name: 'xbz', age: 30 }, + ], + }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: and(plain('age', 'Equal', 30), plain('name', 'EndsWith', 'c')), + }), + new Projection('id'), + ); + + expect(records).toEqual([{ id: 'u1' }]); + expect(client.calls('Scan')[0].FilterExpression).toBe('#n0 = :v0'); + }); + }); + + describe('sort', () => { + it('sorts on a non-key attribute after reading the matching items', async () => { + const { collection } = buildCollection(buildModel(), { + Query: () => ({ Items: [order(3), order(1), order(2)] }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: leaf('customerId', 'Equal', 'c1'), + sort: new Sort({ field: 'total', ascending: false }), + }), + new Projection('orderId', 'total'), + ); + + expect(records.map(r => r.total)).toEqual([3, 2, 1]); + }); + + it('sorts null and missing values last when ascending', async () => { + const { collection } = buildCollection(buildSimpleModel(), { + Scan: () => ({ Items: [{ id: 'a' }, { id: 'b', age: 2 }, { id: 'c', age: 1 }] }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ sort: new Sort({ field: 'age', ascending: true }) }), + new Projection('id', 'age'), + ); + + expect(records.map(r => r.id)).toEqual(['c', 'b', 'a']); + }); + }); + + describe('pagination', () => { + it('limits the items read when nothing filters them', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: () => ({ Items: [order(1), order(2)] }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: leaf('customerId', 'Equal', 'c1'), + page: { limit: 2, skip: 0 } as never, + }), + new Projection('orderId'), + ); + + expect(records).toHaveLength(2); + expect(client.calls('Query')[0].Limit).toBe(2); + }); + + it('skips records and follows LastEvaluatedKey until the page is full', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: [ + { Items: [order(1), order(2)], LastEvaluatedKey: { customerId: 'c1', orderId: 'o2' } }, + { Items: [order(3), order(4)], LastEvaluatedKey: { customerId: 'c1', orderId: 'o4' } }, + { Items: [order(5)] }, + ], + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: and(plain('customerId', 'Equal', 'c1'), plain('status', 'Equal', 'paid')), + page: { limit: 2, skip: 3 } as never, + }), + new Projection('orderId'), + ); + + expect(records.map(r => r.orderId)).toEqual(['o4', 'o5']); + expect(client.calls('Query')[1]).toMatchObject({ + ExclusiveStartKey: { customerId: 'c1', orderId: 'o2' }, + }); + }); + + it('resumes from a cached key when jumping to a deeper page', async () => { + const first = buildCollection(buildModel(), { + Query: [ + { Items: [order(1), order(2)], LastEvaluatedKey: { customerId: 'c1', orderId: 'o2' } }, + { Items: [order(3), order(4)] }, + ], + }); + const filter = (skip: number) => + new PaginatedFilter({ + conditionTree: and(plain('customerId', 'Equal', 'c1'), plain('status', 'Equal', 'paid')), + page: { limit: 2, skip } as never, + }); + + await first.collection.list(caller, filter(0), new Projection('orderId')); + first.client.send.mockClear(); + await first.collection.list(caller, filter(2), new Projection('orderId')); + + expect(first.client.calls('Query')[0]).toMatchObject({ + ExclusiveStartKey: { customerId: 'c1', orderId: 'o2' }, + }); + }); + + it('refuses an offset deeper than the maximum', async () => { + const { collection } = buildCollection(buildModel(), {}, { maxOffset: 10 }); + + await expect( + collection.list( + caller, + new PaginatedFilter({ + conditionTree: leaf('customerId', 'Equal', 'c1'), + page: { limit: 5, skip: 11 } as never, + }), + new Projection('orderId'), + ), + ).rejects.toThrow( + 'Offset 11 exceeds the maximum allowed offset of 10. Narrow the results with filters.', + ); + }); + + it('returns every matching record when no page is requested', async () => { + const { collection } = buildCollection(buildModel(), { + Query: [ + { Items: [order(1)], LastEvaluatedKey: { customerId: 'c1', orderId: 'o1' } }, + { Items: [order(2)] }, + ], + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + new Projection('orderId'), + ); + + expect(records).toHaveLength(2); + }); + }); + + describe('records', () => { + it('serialises sets, nested maps and dates', async () => { + const { collection } = buildCollection( + buildSimpleModel({ + schema: { + id: { type: 'string' }, + tags: { type: 'array' }, + 'address->city': { type: 'string' }, + }, + }), + { + Scan: () => ({ + Items: [{ id: 'u1', tags: new Set(['a']), address: { city: 'Paris', zip: '75' } }], + }), + }, + ); + + const records = await collection.list( + caller, + new PaginatedFilter({}), + new Projection('id', 'tags', 'address->city'), + ); + + expect(records).toEqual([{ id: 'u1', tags: ['a'], 'address->city': 'Paris' }]); + }); + + it('fetches the top-level attribute of a nested projection, with the key', async () => { + const { collection, client } = buildCollection( + buildSimpleModel({ + schema: { id: { type: 'string' }, 'address->city': { type: 'string' } }, + }), + { Scan: () => ({ Items: [] }) }, + ); + + await collection.list(caller, new PaginatedFilter({}), new Projection('address->city')); + + expect(client.calls('Scan')[0]).toMatchObject({ + ProjectionExpression: '#n0, #n1', + ExpressionAttributeNames: { '#n0': 'address', '#n1': 'id' }, + }); + }); + }); + + describe('single-table entity', () => { + it('restricts reads to the entity discriminator', async () => { + const { collection, client } = buildCollection( + buildModel({ entity: { attribute: 'entityType', value: 'ORDER' } }), + { Query: () => ({ Items: [] }) }, + ); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + new Projection('orderId'), + ); + + expect(client.calls('Query')[0]).toMatchObject({ + FilterExpression: '#n1 = :v1', + ExpressionAttributeNames: expect.objectContaining({ '#n1': 'entityType' }), + ExpressionAttributeValues: { ':v0': 'c1', ':v1': 'ORDER' }, + }); + }); + }); + + describe('observability', () => { + it('logs the generated expressions when asked', async () => { + const { collection, logger } = buildCollection( + buildModel(), + { Query: () => ({ Items: [] }) }, + { logQueries: true }, + ); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + new Projection('orderId'), + ); + + expect(logger).toHaveBeenCalledWith( + 'Info', + '[DynamoDB Query] orders.query: KeyCondition=#n0 = :v0 | Filter=- | Index=table', + ); + }); + + it('logs the consumed capacity when asked', async () => { + const { collection, logger, client } = buildCollection( + buildModel(), + { Query: () => ({ Items: [], ConsumedCapacity: { CapacityUnits: 1.5 } }) }, + { logConsumedCapacity: true }, + ); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + new Projection('orderId'), + ); + + expect(client.calls('Query')[0].ReturnConsumedCapacity).toBe('TOTAL'); + expect(logger).toHaveBeenCalledWith('Info', '[DynamoDB Capacity] orders.query: 1.5 units'); + }); + }); +}); + +describe('DynamoDBCollection.list with long lists of values', () => { + const values = Array.from({ length: 250 }, (_, i) => `v${i}`); + + it('evaluates a long In list on the returned items instead of overflowing the expression', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Scan: () => ({ + Items: [ + { id: 'a', name: 'v7' }, + { id: 'b', name: 'other' }, + ], + }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('name', 'In', values) }), + new Projection('id'), + ); + + expect(records).toEqual([{ id: 'a' }]); + expect(client.calls('Scan')[0].FilterExpression).toBeUndefined(); + }); + + it('still reads a long list of keys with BatchGetItem', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + BatchGet: () => ({ Responses: { Users: [] } }), + }); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('id', 'In', values) }), + new Projection('id'), + ); + + expect(client.calls('BatchGet').map(call => call.RequestItems.Users.Keys.length)).toEqual([ + 100, 100, 50, + ]); + }); +}); diff --git a/packages/datasource-dynamodb/test/collection.schema.test.ts b/packages/datasource-dynamodb/test/collection.schema.test.ts new file mode 100644 index 0000000000..5d1fefd13f --- /dev/null +++ b/packages/datasource-dynamodb/test/collection.schema.test.ts @@ -0,0 +1,108 @@ +import { buildCollection } from './collection-helpers'; +import { buildModel, buildSimpleModel } from './helpers'; + +describe('DynamoDBCollection schema', () => { + it('marks the partition key and the sort key as the composite primary key', () => { + const { collection } = buildCollection(buildModel(), {}); + + expect(collection.schema.fields.customerId).toMatchObject({ + type: 'Column', + columnType: 'String', + isPrimaryKey: true, + }); + expect(collection.schema.fields.orderId).toMatchObject({ isPrimaryKey: true }); + expect(collection.schema.fields.status).toMatchObject({ isPrimaryKey: false }); + }); + + it('marks a lone partition key as the primary key', () => { + const { collection } = buildCollection(buildSimpleModel(), {}); + + expect(collection.schema.fields.id).toMatchObject({ isPrimaryKey: true }); + }); + + it('types a numeric key as a Number', () => { + const { collection } = buildCollection( + buildSimpleModel({ + partitionKey: { name: 'id', type: 'N' }, + schema: { id: { type: 'number' } }, + }), + {}, + ); + + expect(collection.schema.fields.id).toMatchObject({ columnType: 'Number', isPrimaryKey: true }); + }); + + it('maps inferred types to column types', () => { + const { collection } = buildCollection(buildModel(), {}); + + expect(collection.schema.fields.total).toMatchObject({ columnType: 'Number' }); + expect(collection.schema.fields.createdAt).toMatchObject({ columnType: 'Date' }); + }); + + it('exposes nested attributes as arrow-notation columns', () => { + const { collection } = buildCollection( + buildSimpleModel({ + schema: { id: { type: 'string' }, 'address->city': { type: 'string' } }, + }), + {}, + ); + + expect(collection.schema.fields['address->city']).toMatchObject({ columnType: 'String' }); + }); + + it('does not allow sorting on Json columns', () => { + const { collection } = buildCollection( + buildSimpleModel({ schema: { id: { type: 'string' }, meta: { type: 'object' } } }), + {}, + ); + + expect(collection.schema.fields.meta).toMatchObject({ columnType: 'Json', isSortable: false }); + }); + + it('is countable by default and not searchable natively', () => { + const { collection } = buildCollection(buildModel(), {}); + + expect(collection.schema.countable).toBe(true); + expect(collection.schema.searchable).toBe(false); + }); + + it('can disable the count', () => { + const { collection } = buildCollection(buildModel({ enableCount: false }), {}); + + expect(collection.schema.countable).toBe(false); + }); + + it('lets the user override a generated column', () => { + const { collection } = buildCollection( + buildModel({ + overrideTypeConverter: ({ fieldName, generatedFieldSchema }) => + fieldName === 'status' + ? ({ + ...generatedFieldSchema, + columnType: 'Enum', + enumValues: ['paid', 'open'], + } as never) + : undefined, + }), + {}, + ); + + expect(collection.schema.fields.status).toMatchObject({ + columnType: 'Enum', + enumValues: ['paid', 'open'], + }); + }); + + it('skips a column it cannot type and logs it', () => { + const { collection, logger } = buildCollection( + buildSimpleModel({ schema: { id: { type: 'string' }, weird: { type: 'nope' as never } } }), + {}, + ); + + expect(collection.schema.fields.weird).toBeUndefined(); + expect(logger).toHaveBeenCalledWith( + 'Warn', + expect.stringContaining("Skipping column 'users.weird'"), + ); + }); +}); diff --git a/packages/datasource-dynamodb/test/collection.write.test.ts b/packages/datasource-dynamodb/test/collection.write.test.ts new file mode 100644 index 0000000000..a97acf7a4e --- /dev/null +++ b/packages/datasource-dynamodb/test/collection.write.test.ts @@ -0,0 +1,343 @@ +import { Aggregation, Filter } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { and, buildModel, buildSimpleModel, leaf, plain } from './helpers'; + +const caller = { timezone: 'UTC' } as never; +const conditional = () => + Object.assign(new Error('The conditional request failed'), { + name: 'ConditionalCheckFailedException', + }); + +describe('DynamoDBCollection.create', () => { + it('puts each record unless it would overwrite an existing item', async () => { + const { collection, client } = buildCollection(buildModel(), { Put: () => ({}) }); + + const created = await collection.create(caller, [ + { customerId: 'c1', orderId: 'o1', total: 5, 'address->city': 'Paris' }, + ]); + + expect(created).toEqual([ + { customerId: 'c1', orderId: 'o1', total: 5, 'address->city': 'Paris' }, + ]); + expect(client.calls('Put')[0]).toEqual({ + TableName: 'Orders', + Item: { customerId: 'c1', orderId: 'o1', total: 5, address: { city: 'Paris' } }, + ConditionExpression: 'attribute_not_exists(#k0)', + ExpressionAttributeNames: { '#k0': 'customerId' }, + }); + }); + + it('generates a partition key for a string key left empty', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { Put: () => ({}) }); + + const [created] = await collection.create(caller, [{ name: 'Bob' }]); + + expect(created.id).toMatch(/^[0-9a-f-]{36}$/); + expect(client.calls('Put')[0].Item.id).toBe(created.id); + }); + + it('refuses a record without a sort key', async () => { + const { collection, client } = buildCollection(buildModel(), { Put: () => ({}) }); + + await expect(collection.create(caller, [{ customerId: 'c1' }])).rejects.toThrow( + "The sort key 'orderId' is required to create a record in 'orders'", + ); + expect(client.calls('Put')).toHaveLength(0); + }); + + it('treats an empty string key as missing', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { Put: () => ({}) }); + + const [created] = await collection.create(caller, [{ id: '', name: 'Bob' }]); + + expect(created.id).toMatch(/^[0-9a-f-]{36}$/); + expect(client.calls('Put')[0].Item.id).toBe(created.id); + }); + + it('refuses a numeric partition key left empty', async () => { + const { collection } = buildCollection( + buildSimpleModel({ + partitionKey: { name: 'id', type: 'N' }, + schema: { id: { type: 'number' } }, + }), + { Put: () => ({}) }, + ); + + await expect(collection.create(caller, [{}])).rejects.toThrow( + "The partition key 'id' is required to create a record in 'users'", + ); + }); + + it('reports an existing item as a validation error', async () => { + const { collection } = buildCollection(buildModel(), { + Put: () => { + throw conditional(); + }, + }); + + await expect(collection.create(caller, [{ customerId: 'c1', orderId: 'o1' }])).rejects.toThrow( + "A record with the same key already exists in 'orders'", + ); + }); + + it('stamps the entity discriminator', async () => { + const { collection, client } = buildCollection( + buildModel({ entity: { attribute: 'entityType', value: 'ORDER' } }), + { Put: () => ({}) }, + ); + + await collection.create(caller, [{ customerId: 'c1', orderId: 'o1' }]); + + expect(client.calls('Put')[0].Item.entityType).toBe('ORDER'); + }); + + it('refuses a sort key outside the entity prefix', async () => { + const { collection } = buildCollection(buildModel({ entity: { sortKeyPrefix: 'ORDER#' } }), { + Put: () => ({}), + }); + + await expect( + collection.create(caller, [{ customerId: 'c1', orderId: 'INVOICE#1' }]), + ).rejects.toThrow("The sort key 'orderId' of 'orders' must start with 'ORDER#'"); + }); + + it('forgets the cached pages and charts of the table', async () => { + const { collection, client } = buildCollection( + buildModel(), + { Put: () => ({}), Scan: () => ({ Items: [{ total: 1 }] }) }, + { chartCacheTtlMs: 60000, scanConcurrency: 4 }, + ); + const aggregation = new Aggregation({ operation: 'Sum', field: 'total' }); + + await collection.aggregate(caller, new Filter({}), aggregation); + await collection.create(caller, [{ customerId: 'c1', orderId: 'o1' }]); + await collection.aggregate(caller, new Filter({}), aggregation); + + expect(client.calls('Scan')).toHaveLength(8); + }); +}); + +describe('DynamoDBCollection.update', () => { + it('updates by key without reading when the filter pins the key', async () => { + const { collection, client } = buildCollection(buildModel(), { Update: () => ({}) }); + + await collection.update( + caller, + new Filter({ + conditionTree: and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'Equal', 'o1')), + }), + { total: 9, status: 'paid' }, + ); + + expect(client.calls('Update')).toEqual([ + { + TableName: 'Orders', + Key: { customerId: 'c1', orderId: 'o1' }, + UpdateExpression: 'SET #n0 = :v0, #n1 = :v1', + ConditionExpression: 'attribute_exists(#k0)', + ExpressionAttributeNames: { '#n0': 'total', '#n1': 'status', '#k0': 'customerId' }, + ExpressionAttributeValues: { ':v0': 9, ':v1': 'paid' }, + }, + ]); + expect(client.calls('Scan')).toHaveLength(0); + expect(client.calls('Query')).toHaveLength(0); + }); + + it('sets a nested attribute through its document path', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { Update: () => ({}) }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + 'address->city': 'Lyon', + }); + + expect(client.calls('Update')[0]).toMatchObject({ + UpdateExpression: 'SET #n0.#n1 = :v0', + ExpressionAttributeNames: expect.objectContaining({ '#n0': 'address', '#n1': 'city' }), + }); + }); + + it('stores a null as an explicit NULL', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { Update: () => ({}) }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + name: null, + }); + + expect(client.calls('Update')[0].ExpressionAttributeValues).toEqual({ ':v0': null }); + }); + + it('resolves the keys of the matching items when the filter does not pin them', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Scan: () => ({ Items: [{ id: 'u1' }, { id: 'u2' }] }), + Update: () => ({}), + }); + + await collection.update(caller, new Filter({ conditionTree: leaf('age', 'Equal', 5) }), { + name: 'x', + }); + + expect(client.calls('Scan')[0]).toMatchObject({ ProjectionExpression: '#n1' }); + expect(client.calls('Update').map(c => c.Key)).toEqual([{ id: 'u1' }, { id: 'u2' }]); + }); + + it('does nothing when no item matches', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Scan: () => ({ Items: [] }), + }); + + await collection.update(caller, new Filter({ conditionTree: leaf('age', 'Equal', 5) }), { + name: 'x', + }); + + expect(client.calls('Update')).toHaveLength(0); + }); + + it('refuses to change a key', async () => { + const { collection } = buildCollection(buildSimpleModel(), {}); + + await expect( + collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + id: 'u2', + }), + ).rejects.toThrow("The key attribute 'id' of 'users' cannot be updated"); + }); + + it('merges client-side when the parent map of a nested path does not exist yet', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Update: () => { + throw Object.assign( + new Error('The document path provided in the update expression is invalid for update'), + { name: 'ValidationException' }, + ); + }, + Get: () => ({ Item: { id: 'u1', name: 'a' } }), + Put: () => ({}), + }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + 'address->city': 'Lyon', + }); + + expect(client.calls('Put')[0]).toMatchObject({ + TableName: 'Users', + Item: { id: 'u1', name: 'a', address: { city: 'Lyon' } }, + ConditionExpression: 'attribute_exists(#k0)', + }); + }); + + it('does not create an item that does not exist', async () => { + const { collection } = buildCollection(buildSimpleModel(), { + Update: () => { + throw conditional(); + }, + }); + + await expect( + collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'ghost') }), { + name: 'x', + }), + ).resolves.toBeUndefined(); + }); + + it('wraps a failure with the collection name and keeps the cause', async () => { + const boom = new Error('boom'); + const { collection } = buildCollection(buildSimpleModel(), { + Update: () => { + throw boom; + }, + }); + + const error = await collection + .update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { name: 'x' }) + .catch(e => e); + + expect(error.message).toBe("Failed to update records of 'users': boom"); + expect(error.cause).toBe(boom); + }); +}); + +describe('DynamoDBCollection.delete', () => { + it('deletes by key without reading when the filter pins the keys', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { BatchWrite: () => ({}) }); + + await collection.delete(caller, new Filter({ conditionTree: leaf('id', 'In', ['u1', 'u2']) })); + + expect(client.calls('BatchWrite')).toEqual([ + { + RequestItems: { + Users: [ + { DeleteRequest: { Key: { id: 'u1' } } }, + { DeleteRequest: { Key: { id: 'u2' } } }, + ], + }, + }, + ]); + expect(client.calls('Scan')).toHaveLength(0); + }); + + it('resolves the keys of the matching items', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: () => ({ Items: [{ customerId: 'c1', orderId: 'o1' }] }), + BatchWrite: () => ({}), + }); + + await collection.delete( + caller, + new Filter({ + conditionTree: and(plain('customerId', 'Equal', 'c1'), plain('status', 'Equal', 'x')), + }), + ); + + expect(client.calls('BatchWrite')[0].RequestItems.Orders).toEqual([ + { DeleteRequest: { Key: { customerId: 'c1', orderId: 'o1' } } }, + ]); + }); + + it('writes in batches of 25 and retries unprocessed items', async () => { + const ids = Array.from({ length: 30 }, (_, i) => `u${i}`); + let first = true; + const { collection, client } = buildCollection(buildSimpleModel(), { + BatchWrite: (input: any) => { + if (first) { + first = false; + + return { UnprocessedItems: { Users: [input.RequestItems.Users[0]] } }; + } + + return {}; + }, + }); + + await collection.delete(caller, new Filter({ conditionTree: leaf('id', 'In', ids) })); + + const sizes = client.calls('BatchWrite').map(c => c.RequestItems.Users.length); + expect(sizes.sort((a, b) => a - b)).toEqual([1, 5, 25]); + }); + + it('gives up when items stay unprocessed', async () => { + const { collection } = buildCollection( + buildSimpleModel(), + { + BatchWrite: (input: any) => ({ UnprocessedItems: { Users: input.RequestItems.Users } }), + }, + { maxWriteRetries: 2 }, + ); + + await expect( + collection.delete(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') })), + ).rejects.toThrow( + "Failed to delete records of 'users': 1 item(s) are still unprocessed after 2 retries", + ); + }); + + it('does nothing when no item matches', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Scan: () => ({ Items: [] }), + }); + + await collection.delete(caller, new Filter({ conditionTree: leaf('age', 'Equal', 1) })); + + expect(client.calls('BatchWrite')).toHaveLength(0); + }); +}); diff --git a/packages/datasource-dynamodb/test/datasource.test.ts b/packages/datasource-dynamodb/test/datasource.test.ts new file mode 100644 index 0000000000..06b80e4df0 --- /dev/null +++ b/packages/datasource-dynamodb/test/datasource.test.ts @@ -0,0 +1,248 @@ +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; + +import { createDocumentClient } from '../src/client'; +import { createDynamoDBDataSource } from '../src/index'; +import { fakeClient } from './fake-client'; + +const table = (name: string, keyType = 'S') => ({ + Table: { + TableName: name, + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: keyType }], + }, +}); + +const handlers = () => ({ + ListTables: () => ({ TableNames: ['Users', 'Orders'] }), + DescribeTable: (input: { TableName: string }) => table(input.TableName), + Scan: () => ({ Items: [{ id: 'a', name: 'Bob' }] }), +}); + +describe('createDynamoDBDataSource', () => { + it('introspects every table into a collection, sorted by name', async () => { + const dataSource = await createDynamoDBDataSource({ client: fakeClient(handlers()) })( + jest.fn(), + jest.fn(), + ); + + expect(dataSource.collections.map(c => c.name)).toEqual(['Orders', 'Users']); + expect(dataSource.getCollection('Users').schema.fields.name).toMatchObject({ + columnType: 'String', + }); + }); + + it('only introspects the listed tables', async () => { + const client = fakeClient(handlers()); + + const dataSource = await createDynamoDBDataSource({ client, tables: ['Users'] })( + jest.fn(), + jest.fn(), + ); + + expect(dataSource.collections.map(c => c.name)).toEqual(['Users']); + expect(client.calls('ListTables')).toHaveLength(0); + }); + + it('builds collections from the builder, with a custom name and an entity', async () => { + const client = fakeClient({ + DescribeTable: (input: { TableName: string }) => table(input.TableName), + Scan: () => ({ Items: [] }), + }); + + const dataSource = await createDynamoDBDataSource({ + client, + builder: configurator => + configurator.addCollectionFromTable({ + name: 'customers', + tableName: 'AppTable', + entity: { attribute: 'entityType', value: 'CUSTOMER' }, + sampleSize: 20, + scanPolicy: 'deny', + }), + })(jest.fn(), jest.fn()); + + expect(dataSource.collections.map(c => c.name)).toEqual(['customers']); + expect(client.calls('Scan')[0]).toMatchObject({ TableName: 'AppTable', Limit: 20 }); + expect(client.calls('ListTables')).toHaveLength(0); + }); + + it('builds collections from a manual schema without calling AWS', async () => { + const client = fakeClient({}); + + const dataSource = await createDynamoDBDataSource({ + client, + disableIntrospection: true, + schema: { + collections: [ + { + name: 'users', + tableName: 'Users', + partitionKey: { name: 'id', type: 'string' }, + fields: [{ name: 'name', type: 'string' }], + }, + ], + }, + })(jest.fn(), jest.fn()); + + expect(dataSource.getCollection('users').schema.fields.name).toMatchObject({ + columnType: 'String', + }); + expect(client.send).not.toHaveBeenCalled(); + }); + + it('requires a schema when introspection is disabled', async () => { + const factory = createDynamoDBDataSource({ + client: fakeClient({}), + disableIntrospection: true, + }); + + await expect(factory(jest.fn(), jest.fn())).rejects.toThrow( + 'When disableIntrospection is true, you must provide a schema', + ); + }); + + it('rejects an invalid manual schema', async () => { + const factory = createDynamoDBDataSource({ + client: fakeClient({}), + disableIntrospection: true, + schema: { collections: [] }, + }); + + await expect(factory(jest.fn(), jest.fn())).rejects.toThrow( + 'Manual schema must have at least one collection', + ); + }); + + it('warns when nothing could be introspected', async () => { + const logger = jest.fn(); + + await createDynamoDBDataSource({ + client: fakeClient({ ListTables: () => ({ TableNames: [] }) }), + })(logger, jest.fn()); + + expect(logger).toHaveBeenCalledWith('Warn', 'No collections were introspected.'); + }); + + it('applies the datasource settings to its collections', async () => { + const client = fakeClient({ + ...handlers(), + Scan: () => ({ Items: [] }), + }); + + const dataSource = await createDynamoDBDataSource({ client, scanPolicy: 'deny' })( + jest.fn(), + jest.fn(), + ); + + await expect( + dataSource + .getCollection('Users') + .list({ timezone: 'UTC' } as never, { page: undefined } as never, ['id'] as never), + ).rejects.toThrow("refused by scanPolicy 'deny'"); + }); + + describe('native queries (PartiQL)', () => { + const build = (extra: Record = {}) => + createDynamoDBDataSource({ + client: fakeClient({ + ...handlers(), + ExecuteStatement: [{ Items: [{ id: 'a' }], NextToken: 'next' }, { Items: [{ id: 'b' }] }], + ...extra, + }), + tables: ['Users'], + liveQueryConnection: 'dynamo', + }); + + it('runs a statement with positional parameters and follows NextToken', async () => { + const client = fakeClient({ + ...handlers(), + ExecuteStatement: [{ Items: [{ id: 'a' }], NextToken: 'next' }, { Items: [{ id: 'b' }] }], + }); + const dataSource = await createDynamoDBDataSource({ + client, + tables: ['Users'], + liveQueryConnection: 'dynamo', + })(jest.fn(), jest.fn()); + + const rows = await dataSource.executeNativeQuery( + 'dynamo', + 'SELECT * FROM "Users" WHERE id = $id AND age > $age AND id <> $id', + { id: 'a', age: 3 }, + ); + + expect(rows).toEqual([{ id: 'a' }, { id: 'b' }]); + expect(client.calls('ExecuteStatement')).toEqual([ + { + Statement: 'SELECT * FROM "Users" WHERE id = ? AND age > ? AND id <> ?', + Parameters: ['a', 3, 'a'], + }, + { + Statement: 'SELECT * FROM "Users" WHERE id = ? AND age > ? AND id <> ?', + Parameters: ['a', 3, 'a'], + NextToken: 'next', + }, + ]); + }); + + it('refuses a placeholder without value', async () => { + const dataSource = await build()(jest.fn(), jest.fn()); + + await expect( + dataSource.executeNativeQuery('dynamo', 'SELECT * FROM "Users" WHERE id = $id', {}), + ).rejects.toThrow('No value was given for the placeholder $id'); + }); + + it('refuses an unknown connection', async () => { + const dataSource = await build()(jest.fn(), jest.fn()); + + await expect(dataSource.executeNativeQuery('nope', 'SELECT 1', {})).rejects.toThrow( + "Unknown connection name 'nope'", + ); + }); + + it('is not available unless a connection name is configured', async () => { + const dataSource = await createDynamoDBDataSource({ client: fakeClient(handlers()) })( + jest.fn(), + jest.fn(), + ); + + expect(Object.keys(dataSource.nativeQueryConnections)).toEqual([]); + }); + }); +}); + +describe('createDocumentClient', () => { + it('builds a client from region, endpoint and credentials', async () => { + const client = createDocumentClient({ + region: 'eu-west-1', + endpoint: 'http://localhost:8000', + credentials: { accessKeyId: 'k', secretAccessKey: 's' }, + }); + + expect(client).toBeInstanceOf(DynamoDBDocumentClient); + expect(await client.config.region()).toBe('eu-west-1'); + expect(await client.config.maxAttempts()).toBe(10); + }); + + it('lets the user tune the underlying client', async () => { + const client = createDocumentClient({ + region: 'eu-west-1', + clientConfig: { maxAttempts: 3 }, + }); + + expect(await client.config.maxAttempts()).toBe(3); + }); + + it('wraps an existing low-level client', () => { + const client = createDocumentClient({ client: new DynamoDBClient({ region: 'eu-west-1' }) }); + + expect(client).toBeInstanceOf(DynamoDBDocumentClient); + }); + + it('uses an existing document client as is', () => { + const existing = DynamoDBDocumentClient.from(new DynamoDBClient({ region: 'eu-west-1' })); + + expect(createDocumentClient({ client: existing })).toBe(existing); + }); +}); diff --git a/packages/datasource-dynamodb/test/fake-client.ts b/packages/datasource-dynamodb/test/fake-client.ts new file mode 100644 index 0000000000..7c7a709a50 --- /dev/null +++ b/packages/datasource-dynamodb/test/fake-client.ts @@ -0,0 +1,42 @@ +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; + +export type Handler = (input: Record, name: string) => unknown; + +export type FakeClient = DynamoDBDocumentClient & { + send: jest.Mock; + calls: (name?: string) => Array>; +}; + +/** A DocumentClient whose `send` is routed by command name to the given handlers. */ +export function fakeClient(handlers: Record): FakeClient { + const queues: Record = {}; + + const send = jest.fn(async (command: { constructor: { name: string }; input: object }) => { + const name = command.constructor.name.replace(/Command$/, ''); + const handler = handlers[name]; + + if (handler === undefined) throw new Error(`Unexpected command ${name}`); + + if (Array.isArray(handler)) { + queues[name] ??= [...handler]; + if (queues[name].length === 0) throw new Error(`No more response queued for ${name}`); + + const next = queues[name].length > 1 ? queues[name].shift() : queues[name][0]; + + return next instanceof Error ? Promise.reject(next) : next; + } + + return handler(command.input as Record, name); + }); + + const client = { send } as unknown as FakeClient; + client.calls = (name?: string) => + send.mock.calls + .map(([command]) => command) + .filter(command => !name || command.constructor.name === `${name}Command`) + .map(command => command.input); + + return client; +} + +export const silentLogger = jest.fn(); diff --git a/packages/datasource-dynamodb/test/helpers.ts b/packages/datasource-dynamodb/test/helpers.ts new file mode 100644 index 0000000000..9d84f144cc --- /dev/null +++ b/packages/datasource-dynamodb/test/helpers.ts @@ -0,0 +1,47 @@ +import type { TableModel } from '../src/types'; + +import { ConditionTreeFactory } from '@forestadmin/datasource-toolkit'; + +export const leaf = (field: string, operator: string, value?: unknown) => + ConditionTreeFactory.fromPlainObject({ field, operator, value } as never); + +export const and = (...conditions: unknown[]) => + ConditionTreeFactory.fromPlainObject({ aggregator: 'And', conditions } as never); + +export const or = (...conditions: unknown[]) => + ConditionTreeFactory.fromPlainObject({ aggregator: 'Or', conditions } as never); + +export const plain = (field: string, operator: string, value?: unknown) => ({ + field, + operator, + value, +}); + +export function buildModel(overrides: Partial = {}): TableModel { + return { + name: 'orders', + tableName: 'Orders', + partitionKey: { name: 'customerId', type: 'S' }, + sortKey: { name: 'orderId', type: 'S' }, + indexes: [], + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + status: { type: 'string', nullable: true }, + total: { type: 'number' }, + createdAt: { type: 'date' }, + }, + ...overrides, + }; +} + +export function buildSimpleModel(overrides: Partial = {}): TableModel { + return buildModel({ + name: 'users', + tableName: 'Users', + partitionKey: { name: 'id', type: 'S' }, + sortKey: undefined, + schema: { id: { type: 'string' }, name: { type: 'string' }, age: { type: 'number' } }, + ...overrides, + }); +} diff --git a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts new file mode 100644 index 0000000000..601f71407a --- /dev/null +++ b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts @@ -0,0 +1,503 @@ +import type { DataSource } from '@forestadmin/datasource-toolkit'; + +import { CreateTableCommand, DeleteTableCommand, DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { BatchWriteCommand, DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import { + Aggregation, + ConditionTreeFactory, + Filter, + PaginatedFilter, + Projection, + Sort, +} from '@forestadmin/datasource-toolkit'; + +import { createDynamoDBDataSource } from '../../src'; + +/** + * Runs against DynamoDB Local (`docker compose up -d`, then DYNAMODB_ENDPOINT=http://localhost:8000): + * the unit tests use a fake client, which cannot tell whether DynamoDB accepts the expressions. + */ +const endpoint = process.env.DYNAMODB_ENDPOINT; +const run = endpoint ? describe : describe.skip; + +const caller = { timezone: 'UTC' } as never; +const suffix = `${Date.now()}`; +const ORDERS = `orders-${suffix}`; +const SHARED = `shared-${suffix}`; +const credentials = { accessKeyId: 'local', secretAccessKey: 'local' }; + +const tree = (plainTree: unknown) => ConditionTreeFactory.fromPlainObject(plainTree as never); +const leaf = (field: string, operator: string, value?: unknown) => tree({ field, operator, value }); + +run('DynamoDB Local', () => { + let client: DynamoDBClient; + let documents: DynamoDBDocumentClient; + let dataSource: DataSource; + let logger: jest.Mock; + + const orders = () => dataSource.getCollection('orders'); + + beforeAll(async () => { + client = new DynamoDBClient({ region: 'local', endpoint, credentials }); + documents = DynamoDBDocumentClient.from(client); + + await client.send( + new CreateTableCommand({ + TableName: ORDERS, + BillingMode: 'PAY_PER_REQUEST', + KeySchema: [ + { AttributeName: 'customerId', KeyType: 'HASH' }, + { AttributeName: 'orderId', KeyType: 'RANGE' }, + ], + AttributeDefinitions: [ + { AttributeName: 'customerId', AttributeType: 'S' }, + { AttributeName: 'orderId', AttributeType: 'S' }, + { AttributeName: 'status', AttributeType: 'S' }, + { AttributeName: 'createdAt', AttributeType: 'S' }, + ], + GlobalSecondaryIndexes: [ + { + IndexName: 'by-status', + KeySchema: [ + { AttributeName: 'status', KeyType: 'HASH' }, + { AttributeName: 'createdAt', KeyType: 'RANGE' }, + ], + Projection: { ProjectionType: 'ALL' }, + }, + ], + }), + ); + await client.send( + new CreateTableCommand({ + TableName: SHARED, + BillingMode: 'PAY_PER_REQUEST', + KeySchema: [ + { AttributeName: 'pk', KeyType: 'HASH' }, + { AttributeName: 'sk', KeyType: 'RANGE' }, + ], + AttributeDefinitions: [ + { AttributeName: 'pk', AttributeType: 'S' }, + { AttributeName: 'sk', AttributeType: 'S' }, + ], + }), + ); + + const items = Array.from({ length: 30 }, (_, i) => ({ + customerId: i < 20 ? 'c1' : 'c2', + orderId: `o${String(i).padStart(2, '0')}`, + status: i % 3 === 0 ? 'paid' : 'open', + total: i + 1, + createdAt: `2024-0${(i % 3) + 1}-15T10:00:00Z`, + tags: new Set(['a']), + ...(i === 0 ? { address: { city: 'Paris' } } : {}), + ...(i === 1 + ? { + lines: [ + { sku: 'A', qty: 1 }, + { sku: 'B', qty: 2 }, + ], + } + : {}), + })); + + for (let i = 0; i < items.length; i += 25) { + // eslint-disable-next-line no-await-in-loop + await documents.send( + new BatchWriteCommand({ + RequestItems: { + [ORDERS]: items.slice(i, i + 25).map(Item => ({ PutRequest: { Item } })), + }, + }), + ); + } + + await documents.send( + new BatchWriteCommand({ + RequestItems: { + [SHARED]: [ + { PutRequest: { Item: { pk: 'ACC#1', sk: 'USER#1', name: 'Ann' } } }, + { PutRequest: { Item: { pk: 'ACC#1', sk: 'USER#2', name: 'Bob' } } }, + { PutRequest: { Item: { pk: 'ACC#1', sk: 'ORDER#1', total: 5 } } }, + ], + }, + }), + ); + + logger = jest.fn(); + dataSource = await createDynamoDBDataSource({ + client: documents, + builder: configurator => + configurator + .addCollectionFromTable({ name: 'orders', tableName: ORDERS }) + .addCollectionFromTable({ + name: 'users', + tableName: SHARED, + entity: { sortKeyPrefix: 'USER#' }, + }) + .addCollectionFromTable({ + name: 'shared_orders', + tableName: SHARED, + entity: { sortKeyPrefix: 'ORDER#' }, + }), + virtualCollections: [ + { parentCollection: 'orders', collectionName: 'order_lines', arrayField: 'lines' }, + ], + liveQueryConnection: 'dynamo', + chartCacheTtlMs: 0, + scanConcurrency: 2, + })(logger, jest.fn()); + }); + + afterAll(async () => { + for (const TableName of [ORDERS, SHARED]) { + // eslint-disable-next-line no-await-in-loop + await client.send(new DeleteTableCommand({ TableName })).catch(() => undefined); + } + }); + + describe('introspection', () => { + it('exposes the key as the primary key and infers the attributes', () => { + expect(orders().schema.fields).toMatchObject({ + customerId: { isPrimaryKey: true, columnType: 'String' }, + orderId: { isPrimaryKey: true }, + total: { columnType: 'Number' }, + createdAt: { columnType: 'Date' }, + status: { columnType: 'String' }, + }); + }); + }); + + describe('list', () => { + it('queries a partition in sort key order', async () => { + const records = await orders().list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + new Projection('orderId'), + ); + + expect(records).toHaveLength(20); + expect(records[0].orderId).toBe('o00'); + expect(records[19].orderId).toBe('o19'); + }); + + it('reads backwards and pages through a sort key range', async () => { + const records = await orders().list( + caller, + new PaginatedFilter({ + conditionTree: tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c1' }, + { field: 'orderId', operator: 'GreaterThan', value: 'o02' }, + { field: 'orderId', operator: 'LessThan', value: 'o15' }, + ], + }), + sort: new Sort({ field: 'orderId', ascending: false }), + page: { skip: 2, limit: 3 } as never, + }), + new Projection('orderId'), + ); + + expect(records.map(r => r.orderId)).toEqual(['o12', 'o11', 'o10']); + }); + + it('queries a global secondary index', async () => { + logger.mockClear(); + + const records = await orders().list( + caller, + new PaginatedFilter({ conditionTree: leaf('status', 'Equal', 'paid') }), + new Projection('orderId'), + ); + + expect(records).toHaveLength(10); + expect(logger).not.toHaveBeenCalledWith('Warn', expect.stringContaining('Scan on table')); + }); + + it('scans, with a warning, for a filter no key serves', async () => { + const records = await orders().list( + caller, + new PaginatedFilter({ conditionTree: leaf('total', 'GreaterThan', 28) }), + new Projection('orderId', 'total'), + ); + + expect(records.map(r => r.total).sort()).toEqual([29, 30]); + expect(logger).toHaveBeenCalledWith('Warn', expect.stringContaining('Scan on table')); + }); + + it('evaluates operators DynamoDB lacks on the returned items', async () => { + const records = await orders().list( + caller, + new PaginatedFilter({ conditionTree: leaf('orderId', 'ILike', '%O1_') }), + new Projection('orderId'), + ); + + expect(records.map(r => r.orderId).sort()).toEqual( + Array.from({ length: 10 }, (_, i) => `o1${i}`), + ); + }); + + it('sorts on a non-key attribute', async () => { + const records = await orders().list( + caller, + new PaginatedFilter({ + sort: new Sort({ field: 'total', ascending: false }), + page: { skip: 0, limit: 2 } as never, + }), + new Projection('total'), + ); + + expect(records.map(r => r.total)).toEqual([30, 29]); + }); + + it('reads a global secondary index in its sort order', async () => { + const records = await orders().list( + caller, + new PaginatedFilter({ + conditionTree: leaf('status', 'Equal', 'open'), + sort: new Sort({ field: 'createdAt', ascending: false }), + page: { skip: 0, limit: 3 } as never, + }), + new Projection('createdAt'), + ); + + expect(records.map(r => r.createdAt)).toEqual([ + '2024-03-15T10:00:00Z', + '2024-03-15T10:00:00Z', + '2024-03-15T10:00:00Z', + ]); + }); + + it('pages through a scan, resuming from the keys it remembers', async () => { + const page = async (skip: number) => + ( + await orders().list( + caller, + new PaginatedFilter({ + conditionTree: leaf('total', 'GreaterThan', 4), + page: { skip, limit: 7 } as never, + }), + new Projection('orderId'), + ) + ).map(r => r.orderId); + + const pages = [await page(0), await page(7), await page(14), await page(21), await page(7)]; + + expect(pages.slice(0, 4).flat()).toHaveLength(26); + expect(new Set(pages.slice(0, 4).flat()).size).toBe(26); + expect(pages[4]).toEqual(pages[1]); + }); + + it('keeps a sort key bound out of the filter expression, where DynamoDB forbids it', async () => { + const records = await dataSource.getCollection('users').list( + caller, + new PaginatedFilter({ + conditionTree: tree({ + aggregator: 'And', + conditions: [ + { field: 'pk', operator: 'Equal', value: 'ACC#1' }, + { field: 'sk', operator: 'GreaterThan', value: 'USER#1' }, + ], + }), + }), + new Projection('sk'), + ); + + expect(records.map(r => r.sk)).toEqual(['USER#2']); + }); + + it('reads whole keys with BatchGetItem', async () => { + const records = await orders().list( + caller, + new PaginatedFilter({ + conditionTree: tree({ + aggregator: 'Or', + conditions: [ + { + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c1' }, + { field: 'orderId', operator: 'Equal', value: 'o03' }, + ], + }, + { + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c2' }, + { field: 'orderId', operator: 'Equal', value: 'o25' }, + ], + }, + ], + }), + }), + new Projection('orderId', 'total'), + ); + + expect(records.map(r => r.orderId).sort()).toEqual(['o03', 'o25']); + }); + }); + + describe('aggregate', () => { + it('counts with a parallel scan', async () => { + const result = await orders().aggregate( + caller, + new Filter({}), + new Aggregation({ operation: 'Count' }), + ); + + expect(result).toEqual([{ value: 30, group: {} }]); + }); + + it('counts a filtered partition', async () => { + const result = await orders().aggregate( + caller, + new Filter({ conditionTree: leaf('customerId', 'Equal', 'c2') }), + new Aggregation({ operation: 'Count' }), + ); + + expect(result).toEqual([{ value: 10, group: {} }]); + }); + + it('sums by group and by month', async () => { + const byStatus = await orders().aggregate( + caller, + new Filter({}), + new Aggregation({ operation: 'Sum', field: 'total', groups: [{ field: 'status' }] }), + ); + const byMonth = await orders().aggregate( + caller, + new Filter({}), + new Aggregation({ + operation: 'Count', + groups: [{ field: 'createdAt', operation: 'Month' }], + }), + ); + + expect(byStatus.reduce((sum, row) => sum + (row.value as number), 0)).toBe(465); + expect(byMonth.map(row => row.group.createdAt).sort()).toEqual([ + '2024-01-01', + '2024-02-01', + '2024-03-01', + ]); + }); + }); + + describe('write', () => { + it('creates, updates (also under a map that does not exist yet) and deletes', async () => { + const key = { customerId: 'c9', orderId: 'o99' }; + await orders().create(caller, [{ ...key, total: 1, status: 'open' }]); + + await orders().update( + caller, + new Filter({ + conditionTree: tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c9' }, + { field: 'orderId', operator: 'Equal', value: 'o99' }, + ], + }), + }), + { total: 2, 'address->city': 'Lyon' }, + ); + + const [updated] = await orders().list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c9') }), + new Projection('total', 'address->city'), + ); + expect(updated).toEqual({ total: 2, 'address->city': 'Lyon' }); + + await expect(orders().create(caller, [{ ...key, total: 3 }])).rejects.toThrow( + 'already exists', + ); + + await orders().delete( + caller, + new Filter({ conditionTree: leaf('customerId', 'Equal', 'c9') }), + ); + const remaining = await orders().aggregate( + caller, + new Filter({ conditionTree: leaf('customerId', 'Equal', 'c9') }), + new Aggregation({ operation: 'Count' }), + ); + expect(remaining).toEqual([{ value: 0, group: {} }]); + }); + }); + + describe('single-table design', () => { + it('keeps each collection to its own items', async () => { + const users = await dataSource + .getCollection('users') + .list( + caller, + new PaginatedFilter({ conditionTree: leaf('pk', 'Equal', 'ACC#1') }), + new Projection('sk', 'name'), + ); + const sharedOrders = await dataSource + .getCollection('shared_orders') + .list( + caller, + new PaginatedFilter({ conditionTree: leaf('pk', 'Equal', 'ACC#1') }), + new Projection('sk', 'total'), + ); + + expect(users.map(u => u.sk)).toEqual(['USER#1', 'USER#2']); + expect(sharedOrders).toEqual([{ sk: 'ORDER#1', total: 5 }]); + }); + }); + + describe('virtual array collection', () => { + it('lists, updates and deletes the items of an array', async () => { + const lines = dataSource.getCollection('order_lines'); + const parent = ConditionTreeFactory.fromPlainObject({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c1' }, + { field: 'orderId', operator: 'Equal', value: 'o01' }, + ], + } as never); + + const before = await lines.list( + caller, + new PaginatedFilter({ conditionTree: parent }), + new Projection('index', 'sku', 'qty'), + ); + expect(before).toEqual([ + { index: 0, sku: 'A', qty: 1 }, + { index: 1, sku: 'B', qty: 2 }, + ]); + + await lines.update(caller, new Filter({ conditionTree: parent }), { qty: 7 }); + await lines.create(caller, [{ customerId: 'c1', orderId: 'o01', sku: 'C', qty: 3 }]); + await lines.delete( + caller, + new Filter({ + conditionTree: ConditionTreeFactory.intersect(parent, leaf('sku', 'Equal', 'A')), + }), + ); + + const after = await lines.list( + caller, + new PaginatedFilter({ conditionTree: parent }), + new Projection('index', 'sku', 'qty'), + ); + expect(after).toEqual([ + { index: 0, sku: 'B', qty: 7 }, + { index: 1, sku: 'C', qty: 3 }, + ]); + }); + }); + + describe('native query', () => { + it('runs a PartiQL statement with parameters', async () => { + const rows = await dataSource.executeNativeQuery( + 'dynamo', + `SELECT orderId FROM "${ORDERS}" WHERE customerId = $customer AND orderId = $order`, + { customer: 'c1', order: 'o05' }, + ); + + expect(rows).toEqual([{ orderId: 'o05' }]); + }); + }); +}); diff --git a/packages/datasource-dynamodb/test/introspection/introspector.test.ts b/packages/datasource-dynamodb/test/introspection/introspector.test.ts new file mode 100644 index 0000000000..06e61224b1 --- /dev/null +++ b/packages/datasource-dynamodb/test/introspection/introspector.test.ts @@ -0,0 +1,377 @@ +import { introspectTable, introspectTables } from '../../src/introspection/introspector'; +import { fakeClient } from '../fake-client'; + +const describeTable = (overrides: Record = {}) => ({ + Table: { + TableName: 'Orders', + KeySchema: [ + { AttributeName: 'customerId', KeyType: 'HASH' }, + { AttributeName: 'orderId', KeyType: 'RANGE' }, + ], + AttributeDefinitions: [ + { AttributeName: 'customerId', AttributeType: 'S' }, + { AttributeName: 'orderId', AttributeType: 'S' }, + { AttributeName: 'status', AttributeType: 'S' }, + { AttributeName: 'createdAt', AttributeType: 'S' }, + ], + GlobalSecondaryIndexes: [ + { + IndexName: 'by-status', + KeySchema: [ + { AttributeName: 'status', KeyType: 'HASH' }, + { AttributeName: 'createdAt', KeyType: 'RANGE' }, + ], + Projection: { ProjectionType: 'ALL' }, + }, + { + IndexName: 'keys-only', + KeySchema: [{ AttributeName: 'status', KeyType: 'HASH' }], + Projection: { ProjectionType: 'KEYS_ONLY' }, + }, + ], + LocalSecondaryIndexes: [ + { + IndexName: 'by-created', + KeySchema: [ + { AttributeName: 'customerId', KeyType: 'HASH' }, + { AttributeName: 'createdAt', KeyType: 'RANGE' }, + ], + Projection: { ProjectionType: 'ALL' }, + }, + ], + ...overrides, + }, +}); + +const items = [ + { + customerId: 'c1', + orderId: 'o1', + status: 'paid', + total: 12, + createdAt: '2024-01-15T10:00:00Z', + tags: new Set(['a']), + address: { city: 'Paris', geo: { lat: 1 } }, + note: 'x', + }, + { customerId: 'c1', orderId: 'o2', status: null, total: 3, createdAt: '2024-01-16T10:00:00Z' }, +]; + +describe('introspectTable', () => { + it('reads the key schema and the indexes from DescribeTable', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: items }), + }); + + const model = await introspectTable(client, { collectionName: 'orders', tableName: 'Orders' }); + + expect(model).toMatchObject({ + name: 'orders', + tableName: 'Orders', + partitionKey: { name: 'customerId', type: 'S' }, + sortKey: { name: 'orderId', type: 'S' }, + }); + expect(model.indexes).toEqual([ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'S' }, + sortKey: { name: 'createdAt', type: 'S' }, + projectsAllAttributes: true, + }, + { + name: 'keys-only', + type: 'GSI', + partitionKey: { name: 'status', type: 'S' }, + sortKey: undefined, + projectsAllAttributes: false, + }, + { + name: 'by-created', + type: 'LSI', + partitionKey: { name: 'customerId', type: 'S' }, + sortKey: { name: 'createdAt', type: 'S' }, + projectsAllAttributes: true, + }, + ]); + }); + + it('infers the attribute types from the sample, flattening nested maps', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: items }), + }); + + const { schema } = await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + }); + + expect(schema).toMatchObject({ + customerId: { type: 'string', nullable: false }, + orderId: { type: 'string', nullable: false }, + total: { type: 'number', nullable: false }, + createdAt: { type: 'date', nullable: false }, + tags: { type: 'array', nullable: true }, + 'address->city': { type: 'string', nullable: true }, + 'address->geo->lat': { type: 'number', nullable: true }, + note: { type: 'string', nullable: true }, + }); + expect(schema.address).toBeUndefined(); + }); + + it('marks an attribute null in the sample as nullable', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: items }), + }); + + const { schema } = await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + }); + + expect(schema.status).toMatchObject({ type: 'string', nullable: true }); + }); + + it('keeps a map as Json past the maximum depth', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: [{ customerId: 'c', orderId: 'o', a: { b: { c: 1 } } }] }), + }); + + const { schema } = await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + maxDepth: 2, + }); + + expect(schema['a->b']).toMatchObject({ type: 'object' }); + expect(schema['a->b->c']).toBeUndefined(); + }); + + it('declares the keys even when the table is empty', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: [] }), + }); + + const { schema } = await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + }); + + expect(schema).toEqual({ + customerId: { type: 'string', nullable: false }, + orderId: { type: 'string', nullable: false }, + }); + }); + + it('types a numeric key as a number', async () => { + const client = fakeClient({ + DescribeTable: () => + describeTable({ + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'N' }], + GlobalSecondaryIndexes: undefined, + LocalSecondaryIndexes: undefined, + }), + Scan: () => ({ Items: [{ id: 1 }] }), + }); + + const model = await introspectTable(client, { collectionName: 'n', tableName: 'N' }); + + expect(model.partitionKey).toEqual({ name: 'id', type: 'N' }); + expect(model.schema.id).toMatchObject({ type: 'number' }); + expect(model.sortKey).toBeUndefined(); + expect(model.indexes).toEqual([]); + }); + + it('samples with a Scan limited to the sample size', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: items }), + }); + + await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + sampleSize: 50, + }); + + expect(client.calls('Scan')[0]).toMatchObject({ TableName: 'Orders', Limit: 50 }); + }); + + it('keeps scanning until the sample is filled', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: [ + { Items: [items[0]], LastEvaluatedKey: { customerId: 'c1', orderId: 'o1' } }, + { Items: [items[1]] }, + ], + }); + + const { schema } = await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + sampleSize: 10, + }); + + expect(client.calls('Scan')[1]).toMatchObject({ + ExclusiveStartKey: { customerId: 'c1', orderId: 'o1' }, + }); + expect(schema.total).toBeDefined(); + }); + + it('restricts the sample to a single-table entity by discriminator attribute', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: items }), + }); + + const model = await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + entity: { attribute: 'entityType', value: 'ORDER' }, + }); + + expect(client.calls('Scan')[0]).toMatchObject({ + FilterExpression: '#n0 = :v0', + ExpressionAttributeNames: { '#n0': 'entityType' }, + ExpressionAttributeValues: { ':v0': 'ORDER' }, + }); + expect(model.entity).toEqual({ attribute: 'entityType', value: 'ORDER' }); + }); + + it('restricts the sample to a single-table entity by sort key prefix', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: items }), + }); + + await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + entity: { sortKeyPrefix: 'ORDER#' }, + }); + + expect(client.calls('Scan')[0]).toMatchObject({ + FilterExpression: 'begins_with(#n0, :v0)', + ExpressionAttributeNames: { '#n0': 'orderId' }, + ExpressionAttributeValues: { ':v0': 'ORDER#' }, + }); + }); + + it('refuses a sort key prefix on a table without sort key', async () => { + const client = fakeClient({ + DescribeTable: () => + describeTable({ + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }), + Scan: () => ({ Items: [] }), + }); + + await expect( + introspectTable(client, { + collectionName: 'n', + tableName: 'N', + entity: { sortKeyPrefix: 'X#' }, + }), + ).rejects.toThrow("Table 'N' has no sort key, so 'sortKeyPrefix' cannot be used"); + }); + + it('refuses a binary key', async () => { + const client = fakeClient({ + DescribeTable: () => + describeTable({ + TableName: 'N', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'B' }], + }), + Scan: () => ({ Items: [] }), + }); + + await expect(introspectTable(client, { collectionName: 'n', tableName: 'N' })).rejects.toThrow( + "Key 'id' of table 'N' is binary, which is not supported", + ); + }); +}); + +describe('introspectTables', () => { + it('introspects every table, naming collections after tables', async () => { + const client = fakeClient({ + ListTables: () => ({ TableNames: ['Orders', 'Users'] }), + DescribeTable: (input: { TableName: string }) => + input.TableName === 'Orders' + ? describeTable() + : describeTable({ + TableName: 'Users', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }), + Scan: () => ({ Items: [] }), + }); + const logger = jest.fn(); + + const models = await introspectTables(client, {}, logger); + + expect(models.map(m => m.name).sort()).toEqual(['Orders', 'Users']); + }); + + it('follows ListTables pagination', async () => { + const client = fakeClient({ + ListTables: [{ TableNames: ['A'], LastEvaluatedTableName: 'A' }, { TableNames: ['B'] }], + DescribeTable: (input: { TableName: string }) => + describeTable({ + TableName: input.TableName, + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }), + Scan: () => ({ Items: [] }), + }); + + const models = await introspectTables(client, {}, jest.fn()); + + expect(client.calls('ListTables')[1]).toMatchObject({ ExclusiveStartTableName: 'A' }); + expect(models.map(m => m.name).sort()).toEqual(['A', 'B']); + }); + + it('only introspects the tables it is asked for', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: [] }), + }); + + await introspectTables(client, { tables: ['Orders'] }, jest.fn()); + + expect(client.calls('ListTables')).toHaveLength(0); + expect(client.calls('DescribeTable')).toHaveLength(1); + }); + + it('skips a table it cannot introspect and says why', async () => { + const client = fakeClient({ + ListTables: () => ({ TableNames: ['Good', 'Bad'] }), + DescribeTable: (input: { TableName: string }) => { + if (input.TableName === 'Bad') throw new Error('AccessDeniedException'); + + return describeTable({ + TableName: 'Good', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }); + }, + Scan: () => ({ Items: [] }), + }); + const logger = jest.fn(); + + const models = await introspectTables(client, {}, logger); + + expect(models.map(m => m.name)).toEqual(['Good']); + expect(logger).toHaveBeenCalledWith( + 'Warn', + "Introspector - Skipping table 'Bad': AccessDeniedException", + ); + }); +}); diff --git a/packages/datasource-dynamodb/test/schema/manual-schema.test.ts b/packages/datasource-dynamodb/test/schema/manual-schema.test.ts new file mode 100644 index 0000000000..809ae3e145 --- /dev/null +++ b/packages/datasource-dynamodb/test/schema/manual-schema.test.ts @@ -0,0 +1,217 @@ +import type { ManualSchemaConfig } from '../../src/types'; + +import { convertManualSchema, validateManualSchema } from '../../src/schema/manual-schema'; + +const config = (overrides: Record = {}): ManualSchemaConfig => ({ + collections: [ + { + name: 'orders', + tableName: 'Orders', + partitionKey: { name: 'customerId', type: 'string' }, + sortKey: { name: 'orderId', type: 'string' }, + fields: [ + { name: 'total', type: 'number' }, + { name: 'status', type: 'string', nullable: true }, + { + name: 'address', + type: 'object', + fields: [ + { name: 'city', type: 'string' }, + { name: 'geo', type: 'object', fields: [{ name: 'lat', type: 'number' }] }, + ], + }, + { name: 'tags', type: 'array', subType: 'string' }, + ], + ...overrides, + }, + ], +}); + +describe('convertManualSchema', () => { + it('builds a table model from the definition', () => { + const [model] = convertManualSchema(config()); + + expect(model).toMatchObject({ + name: 'orders', + tableName: 'Orders', + partitionKey: { name: 'customerId', type: 'S' }, + sortKey: { name: 'orderId', type: 'S' }, + indexes: [], + }); + }); + + it('declares the keys even when they are not listed as fields', () => { + const [model] = convertManualSchema(config()); + + expect(model.schema.customerId).toEqual({ type: 'string', nullable: false }); + expect(model.schema.orderId).toEqual({ type: 'string', nullable: false }); + }); + + it('flattens nested objects into arrow-notation fields', () => { + const [model] = convertManualSchema(config()); + + expect(model.schema['address->city']).toEqual({ type: 'string', nullable: false }); + expect(model.schema['address->geo->lat']).toEqual({ type: 'number', nullable: false }); + expect(model.schema.address).toBeUndefined(); + }); + + it('keeps arrays as a single field and remembers their item schema', () => { + const [model] = convertManualSchema( + config({ + fields: [ + { + name: 'lines', + type: 'array', + subType: 'object', + fields: [{ name: 'sku', type: 'string' }], + }, + ], + }), + ); + + expect(model.schema.lines).toEqual({ + type: 'array', + nullable: false, + items: { sku: { type: 'string', nullable: false } }, + }); + }); + + it('types numeric keys as N', () => { + const [model] = convertManualSchema( + config({ partitionKey: { name: 'id', type: 'number' }, sortKey: undefined }), + ); + + expect(model.partitionKey).toEqual({ name: 'id', type: 'N' }); + expect(model.schema.id).toEqual({ type: 'number', nullable: false }); + }); + + it('carries indexes, entity, count and scan settings', () => { + const [model] = convertManualSchema( + config({ + indexes: [ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'string' }, + sortKey: { name: 'createdAt', type: 'string' }, + }, + ], + entity: { sortKeyPrefix: 'ORDER#' }, + enableCount: false, + scanPolicy: 'deny', + maxScannedItems: 50, + }), + ); + + expect(model.indexes).toEqual([ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'S' }, + sortKey: { name: 'createdAt', type: 'S' }, + projectsAllAttributes: true, + }, + ]); + expect(model).toMatchObject({ + entity: { sortKeyPrefix: 'ORDER#' }, + enableCount: false, + scanPolicy: 'deny', + maxScannedItems: 50, + }); + }); +}); + +describe('validateManualSchema', () => { + it('accepts a valid schema', () => { + expect(() => validateManualSchema(config())).not.toThrow(); + }); + + it.each([ + [{ collections: [] }, 'Manual schema must have at least one collection'], + [undefined, 'Manual schema must have a collections array'], + ])('rejects %p', (schema, message) => { + expect(() => validateManualSchema(schema as never)).toThrow(message); + }); + + it('rejects a collection without table name', () => { + expect(() => validateManualSchema(config({ tableName: undefined }))).toThrow( + "Collection 'orders' must have a tableName", + ); + }); + + it('rejects a collection without partition key', () => { + expect(() => validateManualSchema(config({ partitionKey: undefined }))).toThrow( + "Collection 'orders' must have a partitionKey", + ); + }); + + it('rejects a sort key named like the partition key', () => { + expect(() => + validateManualSchema(config({ sortKey: { name: 'customerId', type: 'string' } })), + ).toThrow("Collection 'orders' uses 'customerId' as both partition key and sort key"); + }); + + it('rejects a key type other than string or number', () => { + expect(() => + validateManualSchema(config({ partitionKey: { name: 'customerId', type: 'object' } })), + ).toThrow("Key 'customerId' of collection 'orders' must be a string or a number"); + }); + + it('rejects duplicate collection names', () => { + const duplicated = config(); + duplicated.collections.push({ ...duplicated.collections[0] }); + + expect(() => validateManualSchema(duplicated)).toThrow("Duplicate collection name: 'orders'"); + }); + + it('rejects duplicate field names at the same level', () => { + expect(() => + validateManualSchema( + config({ + fields: [ + { name: 'a', type: 'string' }, + { name: 'a', type: 'number' }, + ], + }), + ), + ).toThrow("Duplicate field name: 'a' in collection 'orders'"); + }); + + it('rejects an unknown field type', () => { + expect(() => validateManualSchema(config({ fields: [{ name: 'a', type: 'uuid' }] }))).toThrow( + "Field 'a' in collection 'orders' has invalid type 'uuid'", + ); + }); + + it('rejects an array without sub type', () => { + expect(() => validateManualSchema(config({ fields: [{ name: 'a', type: 'array' }] }))).toThrow( + "Array field 'a' in collection 'orders' must have a subType", + ); + }); + + it('rejects an object without nested fields', () => { + expect(() => validateManualSchema(config({ fields: [{ name: 'a', type: 'object' }] }))).toThrow( + "Object field 'a' in collection 'orders' must have nested field definitions", + ); + }); + + it('rejects a sort key prefix on a table without sort key', () => { + expect(() => + validateManualSchema(config({ sortKey: undefined, entity: { sortKeyPrefix: 'X#' } })), + ).toThrow("Collection 'orders' has no sortKey, so 'entity.sortKeyPrefix' cannot be used"); + }); + + it('rejects an entity naming both a discriminator and a prefix', () => { + expect(() => + validateManualSchema( + config({ entity: { attribute: 'type', value: 'O', sortKeyPrefix: 'ORDER#' } }), + ), + ).toThrow("Collection 'orders' entity must use either 'attribute'/'value' or 'sortKeyPrefix'"); + }); + + it('rejects an unknown scan policy', () => { + expect(() => validateManualSchema(config({ scanPolicy: 'sometimes' }))).toThrow( + "Collection 'orders' has an invalid scanPolicy 'sometimes'", + ); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/access-path.test.ts b/packages/datasource-dynamodb/test/utils/access-path.test.ts new file mode 100644 index 0000000000..2f386347ca --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/access-path.test.ts @@ -0,0 +1,294 @@ +import { Sort } from '@forestadmin/datasource-toolkit'; + +import { planAccessPath, planKeyLookup } from '../../src/utils/access-path'; +import { and, buildModel, buildSimpleModel, leaf, or, plain } from '../helpers'; + +const gsi = (overrides = {}) => ({ + name: 'by-status', + type: 'GSI' as const, + partitionKey: { name: 'status', type: 'S' as const }, + sortKey: { name: 'createdAt', type: 'S' as const }, + projectsAllAttributes: true, + ...overrides, +}); + +describe('planAccessPath', () => { + it('queries the table when the partition key is pinned by an equality', () => { + const plan = planAccessPath(buildModel(), leaf('customerId', 'Equal', 'c1')); + + expect(plan).toMatchObject({ + kind: 'query', + keyCondition: { partition: { attribute: 'customerId', value: 'c1' } }, + }); + expect(plan.indexName).toBeUndefined(); + expect(plan.remaining).toBeUndefined(); + }); + + it('keeps the other conditions as filter', () => { + const plan = planAccessPath( + buildModel(), + and(plain('customerId', 'Equal', 'c1'), plain('status', 'Equal', 'paid')), + ); + + expect(plan.kind).toBe('query'); + expect(plan.remaining.toPlainObject()).toEqual(plain('status', 'Equal', 'paid')); + }); + + it('scans when nothing pins a partition key', () => { + const tree = leaf('status', 'Equal', 'paid'); + + const plan = planAccessPath(buildModel(), tree); + + expect(plan.kind).toBe('scan'); + expect(plan.remaining).toBe(tree); + }); + + it('scans without a condition tree', () => { + expect(planAccessPath(buildModel(), undefined)).toMatchObject({ kind: 'scan' }); + }); + + it('does not use an equality hidden inside an OR', () => { + const plan = planAccessPath( + buildModel(), + or(plain('customerId', 'Equal', 'c1'), plain('customerId', 'Equal', 'c2')), + ); + + expect(plan.kind).toBe('scan'); + }); + + it('turns a sort key range into a key condition', () => { + const plan = planAccessPath( + buildModel(), + and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'GreaterThan', 'o5')), + ); + + expect(plan.keyCondition.sort).toEqual({ + attribute: 'orderId', + operator: 'GreaterThan', + value: 'o5', + }); + expect(plan.remaining).toBeUndefined(); + }); + + it('uses BETWEEN for an inclusive sort key interval', () => { + const plan = planAccessPath( + buildModel(), + and( + plain('customerId', 'Equal', 'c1'), + plain('orderId', 'GreaterThanOrEqual', 'o1'), + plain('orderId', 'LessThanOrEqual', 'o9'), + ), + ); + + expect(plan.keyCondition.sort).toEqual({ + attribute: 'orderId', + operator: 'Between', + value: ['o1', 'o9'], + }); + expect(plan.remaining).toBeUndefined(); + }); + + it('keeps the second bound of an exclusive interval as filter', () => { + const plan = planAccessPath( + buildModel(), + and( + plain('customerId', 'Equal', 'c1'), + plain('orderId', 'GreaterThan', 'o1'), + plain('orderId', 'LessThan', 'o9'), + ), + ); + + expect(plan.keyCondition.sort).toMatchObject({ operator: 'GreaterThan', value: 'o1' }); + expect(plan.remaining.toPlainObject()).toEqual(plain('orderId', 'LessThan', 'o9')); + }); + + it('queries a global secondary index when its partition key is pinned', () => { + const model = buildModel({ indexes: [gsi()] }); + + const plan = planAccessPath(model, leaf('status', 'Equal', 'paid')); + + expect(plan).toMatchObject({ + kind: 'query', + indexName: 'by-status', + keyCondition: { partition: { attribute: 'status', value: 'paid' } }, + }); + }); + + it('ignores an index that does not project every attribute', () => { + const model = buildModel({ indexes: [gsi({ projectsAllAttributes: false })] }); + + expect(planAccessPath(model, leaf('status', 'Equal', 'paid')).kind).toBe('scan'); + }); + + it('prefers the table when both the table and an index could serve the query', () => { + const model = buildModel({ indexes: [gsi()] }); + + const plan = planAccessPath( + model, + and(plain('customerId', 'Equal', 'c1'), plain('status', 'Equal', 'paid')), + ); + + expect(plan.indexName).toBeUndefined(); + }); + + it('reads the sort key backwards for a descending sort on it', () => { + const plan = planAccessPath( + buildModel(), + leaf('customerId', 'Equal', 'c1'), + new Sort({ field: 'orderId', ascending: false }), + ); + + expect(plan).toMatchObject({ scanIndexForward: false, sortSatisfied: true }); + }); + + it('does not satisfy a sort on a non-key attribute', () => { + const plan = planAccessPath( + buildModel(), + leaf('customerId', 'Equal', 'c1'), + new Sort({ field: 'total', ascending: true }), + ); + + expect(plan).toMatchObject({ scanIndexForward: true, sortSatisfied: false }); + }); + + it('never claims a sorted result from a scan', () => { + const plan = planAccessPath( + buildModel(), + undefined, + new Sort({ field: 'orderId', ascending: true }), + ); + + expect(plan.sortSatisfied).toBe(false); + }); + + it('restricts a single-table entity to its sort key prefix', () => { + const model = buildModel({ entity: { sortKeyPrefix: 'ORDER#' } }); + + const plan = planAccessPath(model, leaf('customerId', 'Equal', 'c1')); + + expect(plan.keyCondition.sort).toEqual({ + attribute: 'orderId', + operator: 'StartsWith', + value: 'ORDER#', + }); + }); + + it('keeps the prefix as a filter when the user already ranges on the sort key', () => { + const model = buildModel({ entity: { sortKeyPrefix: 'ORDER#' } }); + + const plan = planAccessPath( + model, + and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'GreaterThan', 'ORDER#5')), + ); + + expect(plan.keyCondition.sort).toMatchObject({ operator: 'GreaterThan' }); + expect(plan.remaining.toPlainObject()).toEqual(plain('orderId', 'StartsWith', 'ORDER#')); + }); +}); + +describe('planKeyLookup', () => { + it('resolves an equality on the partition key of a simple table', () => { + expect(planKeyLookup(buildSimpleModel(), leaf('id', 'Equal', 'u1'))).toEqual({ + keys: [{ id: 'u1' }], + remaining: undefined, + }); + }); + + it('resolves an In on the partition key of a simple table', () => { + expect(planKeyLookup(buildSimpleModel(), leaf('id', 'In', ['u1', 'u2']))).toEqual({ + keys: [{ id: 'u1' }, { id: 'u2' }], + remaining: undefined, + }); + }); + + it('resolves a composite key given as an OR of equalities (relation emulation shape)', () => { + const tree = or( + and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'Equal', 'o1')), + and(plain('customerId', 'Equal', 'c2'), plain('orderId', 'Equal', 'o2')), + ); + + expect(planKeyLookup(buildModel(), tree).keys).toEqual([ + { customerId: 'c1', orderId: 'o1' }, + { customerId: 'c2', orderId: 'o2' }, + ]); + }); + + it('resolves a composite key given as an AND of equalities', () => { + const tree = and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'Equal', 'o1')); + + expect(planKeyLookup(buildModel(), tree).keys).toEqual([{ customerId: 'c1', orderId: 'o1' }]); + }); + + it('returns what is left once the key is resolved', () => { + const tree = and(plain('id', 'Equal', 'u1'), plain('age', 'GreaterThan', 3)); + + const lookup = planKeyLookup(buildSimpleModel(), tree); + + expect(lookup.keys).toEqual([{ id: 'u1' }]); + expect(lookup.remaining.toPlainObject()).toEqual(plain('age', 'GreaterThan', 3)); + }); + + it('is not a lookup when the sort key is missing from a composite table', () => { + expect(planKeyLookup(buildModel(), leaf('customerId', 'Equal', 'c1'))).toBeNull(); + }); + + it('is not a lookup for a non-key attribute', () => { + expect(planKeyLookup(buildSimpleModel(), leaf('name', 'Equal', 'a'))).toBeNull(); + }); + + it('is not a lookup without condition', () => { + expect(planKeyLookup(buildSimpleModel(), undefined)).toBeNull(); + }); + + it('is not a lookup when a key value is null', () => { + expect(planKeyLookup(buildSimpleModel(), leaf('id', 'Equal', null))).toBeNull(); + }); +}); + +describe('key value types', () => { + const numericModel = () => + buildModel({ + partitionKey: { name: 'accountId', type: 'N' }, + sortKey: { name: 'seq', type: 'N' }, + }); + + it('reads a numeric partition key given as a numeric string', () => { + const plan = planAccessPath(numericModel(), leaf('accountId', 'Equal', '42')); + + expect(plan.keyCondition.partition).toEqual({ attribute: 'accountId', value: 42 }); + }); + + it('reads a numeric sort key bound given as a numeric string', () => { + const plan = planAccessPath( + numericModel(), + and(plain('accountId', 'Equal', 1), plain('seq', 'GreaterThan', '7')), + ); + + expect(plan.keyCondition.sort).toEqual({ + attribute: 'seq', + operator: 'GreaterThan', + value: 7, + }); + }); + + it('reads a string partition key given as a number', () => { + const plan = planAccessPath(buildModel(), leaf('customerId', 'Equal', 12)); + + expect(plan.keyCondition.partition.value).toBe('12'); + }); + + it('keeps a non numeric string as it is for a numeric key', () => { + const plan = planAccessPath(numericModel(), leaf('accountId', 'Equal', 'abc')); + + expect(plan.keyCondition.partition.value).toBe('abc'); + }); + + it('types the keys of a lookup', () => { + const lookup = planKeyLookup( + buildSimpleModel({ partitionKey: { name: 'id', type: 'N' } }), + leaf('id', 'In', ['1', 2, '3']), + ); + + expect(lookup.keys).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }]); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/aggregator.test.ts b/packages/datasource-dynamodb/test/utils/aggregator.test.ts new file mode 100644 index 0000000000..e508ab7f81 --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/aggregator.test.ts @@ -0,0 +1,196 @@ +import { Aggregation } from '@forestadmin/datasource-toolkit'; + +import StreamingAggregator from '../../src/utils/aggregator'; + +function aggregate( + aggregation: ConstructorParameters[0], + records: Record[], + options: { timezone?: string; limit?: number } = {}, +) { + const aggregator = new StreamingAggregator( + new Aggregation(aggregation), + options.timezone ?? 'UTC', + ); + records.forEach(record => aggregator.add(record)); + + return aggregator.result(options.limit); +} + +describe('StreamingAggregator', () => { + describe('without groups', () => { + it('counts every record when no field is given', () => { + expect(aggregate({ operation: 'Count' }, [{ a: 1 }, { a: null }, {}])).toEqual([ + { value: 3, group: {} }, + ]); + }); + + it('counts only the records having a value for the field', () => { + expect(aggregate({ operation: 'Count', field: 'a' }, [{ a: 1 }, { a: null }, {}])).toEqual([ + { value: 1, group: {} }, + ]); + }); + + it('sums numbers and ignores other values', () => { + expect( + aggregate({ operation: 'Sum', field: 'a' }, [{ a: 1 }, { a: 2.5 }, { a: 'x' }]), + ).toEqual([{ value: 3.5, group: {} }]); + }); + + it('averages over the records having a number', () => { + expect( + aggregate({ operation: 'Avg', field: 'a' }, [{ a: 2 }, { a: 4 }, { a: null }]), + ).toEqual([{ value: 3, group: {} }]); + }); + + it('takes the min and the max', () => { + const records = [{ a: 3 }, { a: 1 }, { a: 2 }]; + + expect(aggregate({ operation: 'Min', field: 'a' }, records)).toEqual([ + { value: 1, group: {} }, + ]); + expect(aggregate({ operation: 'Max', field: 'a' }, records)).toEqual([ + { value: 3, group: {} }, + ]); + }); + + it('returns zero for a count or a sum over nothing', () => { + expect(aggregate({ operation: 'Count' }, [])).toEqual([{ value: 0, group: {} }]); + expect(aggregate({ operation: 'Sum', field: 'a' }, [])).toEqual([{ value: 0, group: {} }]); + }); + + it('returns null for an average, min or max over nothing', () => { + expect(aggregate({ operation: 'Avg', field: 'a' }, [])).toEqual([{ value: null, group: {} }]); + expect(aggregate({ operation: 'Max', field: 'a' }, [])).toEqual([{ value: null, group: {} }]); + }); + }); + + describe('with groups', () => { + it('groups on one field and sorts by value, highest first', () => { + const result = aggregate({ operation: 'Count', groups: [{ field: 's' }] }, [ + { s: 'a' }, + { s: 'b' }, + { s: 'b' }, + ]); + + expect(result).toEqual([ + { value: 2, group: { s: 'b' } }, + { value: 1, group: { s: 'a' } }, + ]); + }); + + it('groups on several fields', () => { + const result = aggregate( + { operation: 'Sum', field: 'n', groups: [{ field: 'a' }, { field: 'b' }] }, + [ + { a: 1, b: 'x', n: 1 }, + { a: 1, b: 'x', n: 2 }, + { a: 1, b: 'y', n: 5 }, + ], + ); + + expect(result).toEqual([ + { value: 5, group: { a: 1, b: 'y' } }, + { value: 3, group: { a: 1, b: 'x' } }, + ]); + }); + + it('keeps falsy group keys', () => { + const result = aggregate({ operation: 'Count', groups: [{ field: 'k' }] }, [ + { k: 0 }, + { k: '' }, + { k: false }, + ]); + + expect(result.map(r => r.group.k).sort()).toEqual(['', 0, false].sort()); + }); + + it('groups missing values under null', () => { + const result = aggregate({ operation: 'Count', groups: [{ field: 'k' }] }, [{}, { k: null }]); + + expect(result).toEqual([{ value: 2, group: { k: null } }]); + }); + + it('truncates to the limit after sorting', () => { + const result = aggregate( + { operation: 'Count', groups: [{ field: 's' }] }, + [{ s: 'a' }, { s: 'b' }, { s: 'b' }, { s: 'c' }, { s: 'c' }, { s: 'c' }], + { limit: 2 }, + ); + + expect(result.map(r => r.group.s)).toEqual(['c', 'b']); + }); + + it('reads flattened nested fields by their arrow path', () => { + const result = aggregate({ operation: 'Count', groups: [{ field: 'address->city' }] }, [ + { 'address->city': 'Paris' }, + { 'address->city': 'Paris' }, + ]); + + expect(result).toEqual([{ value: 2, group: { 'address->city': 'Paris' } }]); + }); + + it('averages per group', () => { + const result = aggregate({ operation: 'Avg', field: 'n', groups: [{ field: 'g' }] }, [ + { g: 'a', n: 2 }, + { g: 'a', n: 4 }, + { g: 'b', n: 10 }, + ]); + + expect(result).toEqual([ + { value: 10, group: { g: 'b' } }, + { value: 3, group: { g: 'a' } }, + ]); + }); + }); + + describe('date groups', () => { + const dates = [ + { at: '2024-01-15T10:00:00Z' }, + { at: '2024-01-20T23:30:00Z' }, + { at: '2024-03-02T01:00:00Z' }, + ]; + + it.each([ + ['Year', ['2024-01-01']], + ['Quarter', ['2024-01-01']], + ['Month', ['2024-01-01', '2024-03-01']], + ['Day', ['2024-01-15', '2024-01-20', '2024-03-02']], + ['Week', ['2024-01-15', '2024-02-26']], + ])('buckets by %s', (operation, expected) => { + const result = aggregate( + { operation: 'Count', groups: [{ field: 'at', operation: operation as never }] }, + dates, + ); + + expect(result.map(r => r.group.at).sort()).toEqual([...expected].sort()); + }); + + it('buckets in the requested timezone', () => { + const result = aggregate( + { operation: 'Count', groups: [{ field: 'at', operation: 'Day' }] }, + [{ at: '2024-01-20T23:30:00Z' }], + { timezone: 'Asia/Tokyo' }, + ); + + expect(result[0].group.at).toBe('2024-01-21'); + }); + + it('accepts epoch milliseconds and seconds', () => { + const result = aggregate( + { operation: 'Count', groups: [{ field: 'at', operation: 'Day' }] }, + [{ at: Date.UTC(2024, 0, 15, 10) }, { at: Date.UTC(2024, 0, 15, 12) / 1000 }], + ); + + expect(result).toEqual([{ value: 2, group: { at: '2024-01-15' } }]); + }); + + it('groups an unparsable date under null', () => { + const result = aggregate( + { operation: 'Count', groups: [{ field: 'at', operation: 'Day' }] }, + [{ at: 'not a date' }], + ); + + expect(result).toEqual([{ value: 1, group: { at: null } }]); + }); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/expression-builder.test.ts b/packages/datasource-dynamodb/test/utils/expression-builder.test.ts new file mode 100644 index 0000000000..8d6d017ec0 --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/expression-builder.test.ts @@ -0,0 +1,292 @@ +import { ConditionTreeFactory } from '@forestadmin/datasource-toolkit'; + +import ExpressionBuilder, { splitCondition } from '../../src/utils/expression-builder'; + +const leaf = (field: string, operator: string, value?: unknown) => + ConditionTreeFactory.fromPlainObject({ field, operator, value } as never); + +describe('ExpressionBuilder', () => { + describe('condition', () => { + it('aliases attribute names and values so reserved words are safe', () => { + const builder = new ExpressionBuilder(); + + const expression = builder.condition(leaf('status', 'Equal', 'paid')); + + expect(expression).toBe('#n0 = :v0'); + expect(builder.names).toEqual({ '#n0': 'status' }); + expect(builder.values).toEqual({ ':v0': 'paid' }); + }); + + it('reuses the alias of an attribute used twice', () => { + const builder = new ExpressionBuilder(); + + builder.condition( + ConditionTreeFactory.fromPlainObject({ + aggregator: 'And', + conditions: [ + { field: 'age', operator: 'GreaterThan', value: 1 }, + { field: 'age', operator: 'LessThan', value: 9 }, + ], + }), + ); + + expect(builder.names).toEqual({ '#n0': 'age' }); + }); + + it('turns arrow notation into a nested document path', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('address->city', 'Equal', 'Paris'))).toBe('#n0.#n1 = :v0'); + expect(builder.names).toEqual({ '#n0': 'address', '#n1': 'city' }); + }); + + it('wraps and/or branches in parentheses', () => { + const builder = new ExpressionBuilder(); + + const expression = builder.condition( + ConditionTreeFactory.fromPlainObject({ + aggregator: 'Or', + conditions: [ + { field: 'a', operator: 'Equal', value: 1 }, + { + aggregator: 'And', + conditions: [ + { field: 'b', operator: 'Equal', value: 2 }, + { field: 'c', operator: 'Equal', value: 3 }, + ], + }, + ], + }), + ); + + expect(expression).toBe('(#n0 = :v0 OR (#n1 = :v1 AND #n2 = :v2))'); + }); + + it.each([ + ['NotEqual', '#n0 <> :v0'], + ['LessThan', '#n0 < :v0'], + ['LessThanOrEqual', '#n0 <= :v0'], + ['GreaterThan', '#n0 > :v0'], + ['GreaterThanOrEqual', '#n0 >= :v0'], + ['StartsWith', 'begins_with(#n0, :v0)'], + ['Contains', 'contains(#n0, :v0)'], + ['NotContains', 'NOT contains(#n0, :v0)'], + ])('translates %s', (operator, expected) => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('f', operator, 5))).toBe(expected); + expect(builder.values).toEqual({ ':v0': 5 }); + }); + + it('treats Equal null as a missing or NULL attribute', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('f', 'Equal', null))).toBe( + '(attribute_not_exists(#n0) OR attribute_type(#n0, :v0))', + ); + expect(builder.values).toEqual({ ':v0': 'NULL' }); + }); + + it('translates Present and Missing around the NULL type', () => { + const present = new ExpressionBuilder(); + const missing = new ExpressionBuilder(); + + expect(present.condition(leaf('f', 'Present'))).toBe( + '(attribute_exists(#n0) AND NOT attribute_type(#n0, :v0))', + ); + expect(missing.condition(leaf('f', 'Missing'))).toBe( + '(attribute_not_exists(#n0) OR attribute_type(#n0, :v0))', + ); + }); + + it('translates In into an IN list', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('f', 'In', ['a', 'b']))).toBe('#n0 IN (:v0, :v1)'); + expect(builder.values).toEqual({ ':v0': 'a', ':v1': 'b' }); + }); + + it('splits an In list of more than 100 values into several IN groups', () => { + const builder = new ExpressionBuilder(); + const values = Array.from({ length: 150 }, (_, i) => i); + + const expression = builder.condition(leaf('f', 'In', values)); + + expect(expression.startsWith('(#n0 IN (')).toBe(true); + expect(expression.match(/ IN \(/g)).toHaveLength(2); + expect(expression).toContain(') OR #n0 IN ('); + expect(Object.keys(builder.values)).toHaveLength(150); + }); + + it('lets In match null values too', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('f', 'In', ['a', null]))).toBe( + '(#n0 IN (:v0) OR (attribute_not_exists(#n0) OR attribute_type(#n0, :v1)))', + ); + }); + + it('turns an empty In into a contradiction', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('f', 'In', []))).toBe( + '(attribute_exists(#n0) AND attribute_not_exists(#n0))', + ); + }); + + it('negates NotIn', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('f', 'NotIn', ['a', 'b']))).toBe('NOT (#n0 IN (:v0, :v1))'); + }); + + it('requires every value for IncludesAll', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('tags', 'IncludesAll', ['a', 'b']))).toBe( + '(contains(#n0, :v0) AND contains(#n0, :v1))', + ); + }); + + it('translates Blank as missing, NULL or empty', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('f', 'Blank'))).toBe( + '(attribute_not_exists(#n0) OR attribute_type(#n0, :v0) OR #n0 = :v1)', + ); + expect(builder.values).toEqual({ ':v0': 'NULL', ':v1': '' }); + }); + + it('translates Before and After on a date as comparisons on its ISO string', () => { + const builder = new ExpressionBuilder(); + const date = new Date('2024-01-15T10:00:00.000Z'); + + expect(builder.condition(leaf('at', 'Before', date))).toBe('#n0 < :v0'); + expect(builder.condition(leaf('at', 'After', date))).toBe('#n0 > :v1'); + expect(builder.values).toEqual({ + ':v0': '2024-01-15T10:00:00.000Z', + ':v1': '2024-01-15T10:00:00.000Z', + }); + }); + + it('requires none of the values for IncludesNone', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('tags', 'IncludesNone', ['a', 'b']))).toBe( + '(NOT contains(#n0, :v0) AND NOT contains(#n0, :v1))', + ); + }); + + it('is always true for an empty IncludesAll or NotIn', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('tags', 'IncludesAll', []))).toBe( + '(attribute_exists(#n0) OR attribute_not_exists(#n0))', + ); + expect(builder.condition(leaf('tags', 'NotIn', []))).toBe( + '(attribute_exists(#n0) OR attribute_not_exists(#n0))', + ); + }); + + it('turns an In of null only into a missing check', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('f', 'In', [null]))).toBe( + '(attribute_not_exists(#n0) OR attribute_type(#n0, :v0))', + ); + }); + + it('negates a NotIn that is split in several groups as a whole', () => { + const builder = new ExpressionBuilder(); + const values = Array.from({ length: 101 }, (_, i) => i); + + expect(builder.condition(leaf('f', 'NotIn', values)).startsWith('NOT (#n0 IN (')).toBe(true); + }); + + it('throws a typed error on an operator that cannot be pushed down', () => { + const builder = new ExpressionBuilder(); + + expect(() => builder.condition(leaf('f', 'ILike', '%a%'))).toThrow( + 'Operator "ILike" cannot be evaluated by DynamoDB', + ); + }); + + it('rejects a relation field', () => { + const builder = new ExpressionBuilder(); + + expect(() => builder.condition(leaf('rel:name', 'Equal', 'a'))).toThrow( + 'Filtering on the relation field "rel:name" is not supported', + ); + }); + }); + + describe('projection', () => { + it('aliases the top-level attribute of nested fields once', () => { + const builder = new ExpressionBuilder(); + + expect(builder.projection(['pk', 'address->city', 'address->zip'])).toBe('#n0, #n1'); + expect(builder.names).toEqual({ '#n0': 'pk', '#n1': 'address' }); + }); + }); + + describe('result', () => { + it('omits empty alias maps', () => { + expect(new ExpressionBuilder().result()).toEqual({}); + }); + + it('returns names and values when used', () => { + const builder = new ExpressionBuilder(); + builder.condition(leaf('f', 'Equal', 1)); + + expect(builder.result()).toEqual({ + ExpressionAttributeNames: { '#n0': 'f' }, + ExpressionAttributeValues: { ':v0': 1 }, + }); + }); + }); +}); + +describe('splitCondition', () => { + it('returns everything as pushable when DynamoDB can evaluate it', () => { + const tree = leaf('f', 'Equal', 1); + + const { pushable, residual } = splitCondition(tree); + + expect(pushable).toBe(tree); + expect(residual).toBeUndefined(); + }); + + it('keeps unsupported leaves of an AND as residual', () => { + const tree = ConditionTreeFactory.fromPlainObject({ + aggregator: 'And', + conditions: [ + { field: 'a', operator: 'Equal', value: 1 }, + { field: 'b', operator: 'ILike', value: '%x%' }, + ], + }); + + const { pushable, residual } = splitCondition(tree); + + expect(pushable.toPlainObject()).toEqual({ field: 'a', operator: 'Equal', value: 1 }); + expect(residual.toPlainObject()).toEqual({ field: 'b', operator: 'ILike', value: '%x%' }); + }); + + it('sends a whole OR to the residual when one branch cannot be pushed down', () => { + const tree = ConditionTreeFactory.fromPlainObject({ + aggregator: 'Or', + conditions: [ + { field: 'a', operator: 'Equal', value: 1 }, + { field: 'b', operator: 'EndsWith', value: 'x' }, + ], + }); + + const { pushable, residual } = splitCondition(tree); + + expect(pushable).toBeUndefined(); + expect(residual).toBe(tree); + }); + + it('handles an absent tree', () => { + expect(splitCondition(undefined)).toEqual({ pushable: undefined, residual: undefined }); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/pagination-cache.test.ts b/packages/datasource-dynamodb/test/utils/pagination-cache.test.ts new file mode 100644 index 0000000000..76801e129e --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/pagination-cache.test.ts @@ -0,0 +1,74 @@ +import PaginationCache from '../../src/utils/pagination-cache'; + +describe('PaginationCache', () => { + it('returns nothing for an unknown query', () => { + expect(new PaginationCache().findBestToken('q', 100)).toBeUndefined(); + }); + + it('resumes from the closest stored offset not past the target', () => { + const cache = new PaginationCache(); + cache.storeToken('q', 100, { pk: 'a' }); + cache.storeToken('q', 300, { pk: 'c' }); + cache.storeToken('q', 500, { pk: 'e' }); + + expect(cache.findBestToken('q', 400)).toEqual({ offset: 300, startKey: { pk: 'c' } }); + }); + + it('never resumes past the target offset', () => { + const cache = new PaginationCache(); + cache.storeToken('q', 500, { pk: 'e' }); + + expect(cache.findBestToken('q', 400)).toBeUndefined(); + }); + + it('keeps queries apart', () => { + const cache = new PaginationCache(); + cache.storeToken('q1', 100, { pk: 'a' }); + + expect(cache.findBestToken('q2', 200)).toBeUndefined(); + }); + + it('drops entries older than the time to live', () => { + let now = 1000; + const cache = new PaginationCache({ ttlMs: 50, now: () => now }); + cache.storeToken('q', 100, { pk: 'a' }); + + now += 51; + + expect(cache.findBestToken('q', 200)).toBeUndefined(); + }); + + it('keeps at most maxEntriesPerQuery entries, dropping the oldest', () => { + const cache = new PaginationCache({ maxEntriesPerQuery: 2 }); + cache.storeToken('q', 100, { pk: 'a' }); + cache.storeToken('q', 200, { pk: 'b' }); + cache.storeToken('q', 300, { pk: 'c' }); + + expect(cache.findBestToken('q', 150)).toBeUndefined(); + expect(cache.findBestToken('q', 350)).toEqual({ offset: 300, startKey: { pk: 'c' } }); + }); + + it('forgets every query of a table when it is invalidated', () => { + const cache = new PaginationCache(); + cache.storeToken('q', 100, { pk: 'a' }, 'Orders'); + cache.storeToken('r', 100, { pk: 'a' }, 'Users'); + + cache.invalidateTable('Orders'); + + expect(cache.findBestToken('q', 200)).toBeUndefined(); + expect(cache.findBestToken('r', 200)).toBeDefined(); + }); + + it('hashes the same query to the same key and different ones apart', () => { + const cache = new PaginationCache(); + + expect(cache.hash({ a: 1, b: [2] })).toBe(cache.hash({ a: 1, b: [2] })); + expect(cache.hash({ a: 1 })).not.toBe(cache.hash({ a: 2 })); + }); + + it('exposes the interval between remembered positions', () => { + const cache = new PaginationCache({ cacheInterval: 7 }); + + expect(cache.getCacheInterval()).toBe(7); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/serializer.test.ts b/packages/datasource-dynamodb/test/utils/serializer.test.ts new file mode 100644 index 0000000000..a1acdbd2f3 --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/serializer.test.ts @@ -0,0 +1,72 @@ +import Serializer from '../../src/utils/serializer'; + +describe('Serializer', () => { + describe('flatten', () => { + it('expands nested maps declared as columns into arrow-notation fields', () => { + const record = Serializer.flatten( + { pk: 'a', address: { city: 'Paris', geo: { lat: 1 } } }, + new Set(['pk', 'address->city', 'address->geo->lat']), + ); + + expect(record).toEqual({ pk: 'a', 'address->city': 'Paris', 'address->geo->lat': 1 }); + }); + + it('keeps a map as Json when no nested column is declared under it', () => { + const record = Serializer.flatten({ pk: 'a', meta: { x: 1 } }, new Set(['pk', 'meta'])); + + expect(record).toEqual({ pk: 'a', meta: { x: 1 } }); + }); + + it('turns sets into arrays', () => { + const record = Serializer.flatten({ tags: new Set(['a', 'b']) }, new Set(['tags'])); + + expect(record).toEqual({ tags: ['a', 'b'] }); + }); + + it('turns dates into ISO strings', () => { + const record = Serializer.flatten( + { at: new Date('2024-01-15T10:00:00.000Z') }, + new Set(['at']), + ); + + expect(record).toEqual({ at: '2024-01-15T10:00:00.000Z' }); + }); + + it('turns binary values into base64 strings', () => { + const record = Serializer.flatten({ blob: Buffer.from('hi') }, new Set(['blob'])); + + expect(record).toEqual({ blob: 'aGk=' }); + }); + + it('keeps GeoJSON points untouched', () => { + const point = { type: 'Point', coordinates: [1, 2] }; + + expect(Serializer.flatten({ loc: point }, new Set(['loc']))).toEqual({ loc: point }); + }); + + it('keeps null values', () => { + expect(Serializer.flatten({ a: null }, new Set(['a']))).toEqual({ a: null }); + }); + }); + + describe('unflatten', () => { + it('rebuilds nested maps from arrow-notation fields', () => { + expect( + Serializer.unflatten({ pk: 'a', 'address->city': 'Paris', 'address->zip': '75' }), + ).toEqual({ + pk: 'a', + address: { city: 'Paris', zip: '75' }, + }); + }); + + it('leaves plain fields alone', () => { + expect(Serializer.unflatten({ a: 1, b: [1, 2] })).toEqual({ a: 1, b: [1, 2] }); + }); + }); + + describe('toDocumentPath', () => { + it('splits arrow notation into path segments', () => { + expect(Serializer.toPathSegments('a->b->c')).toEqual(['a', 'b', 'c']); + }); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/type-converter.test.ts b/packages/datasource-dynamodb/test/utils/type-converter.test.ts new file mode 100644 index 0000000000..10f2423cbe --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/type-converter.test.ts @@ -0,0 +1,111 @@ +import TypeConverter from '../../src/utils/type-converter'; + +describe('TypeConverter', () => { + describe('inferTypeFromValue', () => { + it.each([ + [true, 'boolean'], + [12, 'number'], + ['hello', 'string'], + [null, 'null'], + [undefined, 'null'], + [['a'], 'array'], + [new Set(['a']), 'array'], + [{ a: 1 }, 'object'], + [new Date(), 'date'], + ['2024-01-15T10:30:00Z', 'date'], + ['2024-01-15', 'dateonly'], + ['10:30:00', 'timeonly'], + [{ type: 'Point', coordinates: [2.35, 48.85] }, 'point'], + [new Uint8Array([1, 2]), 'binary'], + ])('infers %p as %s', (value, expected) => { + expect(TypeConverter.inferTypeFromValue(value)).toBe(expected); + }); + }); + + describe('getMostSpecificType', () => { + it('keeps the type shared by every sample', () => { + expect(TypeConverter.getMostSpecificType(['number', 'number'])).toBe('number'); + }); + + it('ignores null samples when the others agree', () => { + expect(TypeConverter.getMostSpecificType(['string', 'null'])).toBe('string'); + }); + + it('prefers date over dateonly when both are seen', () => { + expect(TypeConverter.getMostSpecificType(['date', 'dateonly'])).toBe('date'); + }); + + it('falls back to object (Json) for incompatible types', () => { + expect(TypeConverter.getMostSpecificType(['number', 'string'])).toBe('object'); + }); + + it('returns null for no sample', () => { + expect(TypeConverter.getMostSpecificType([])).toBe('null'); + }); + }); + + describe('getColumnTypeFromDataType', () => { + it.each([ + ['string', 'String'], + ['number', 'Number'], + ['boolean', 'Boolean'], + ['date', 'Date'], + ['dateonly', 'Dateonly'], + ['timeonly', 'Timeonly'], + ['point', 'Point'], + ['binary', 'String'], + ['object', 'Json'], + ['array', 'Json'], + ['null', 'Json'], + ])('maps %s to %s', (dataType, columnType) => { + expect(TypeConverter.getColumnTypeFromDataType(dataType as never)).toBe(columnType); + }); + + it('throws on an unknown data type', () => { + expect(() => TypeConverter.getColumnTypeFromDataType('weird' as never)).toThrow( + 'Unsupported data type: "weird"', + ); + }); + }); + + describe('operatorsForColumnType', () => { + it('gives strings the full text operators', () => { + const operators = TypeConverter.operatorsForColumnType('String'); + + expect([...operators]).toEqual( + expect.arrayContaining([ + 'Equal', + 'In', + 'Like', + 'ILike', + 'Contains', + 'IContains', + 'StartsWith', + 'EndsWith', + ]), + ); + }); + + it('does not give booleans range operators', () => { + expect(TypeConverter.operatorsForColumnType('Boolean').has('GreaterThan')).toBe(false); + }); + + it('gives numbers range operators', () => { + expect(TypeConverter.operatorsForColumnType('Number').has('GreaterThanOrEqual')).toBe(true); + }); + }); + + describe('isSortable', () => { + it('does not sort Json-like types', () => { + expect(TypeConverter.isSortable('object')).toBe(false); + expect(TypeConverter.isSortable('array')).toBe(false); + expect(TypeConverter.isSortable('point')).toBe(false); + }); + + it('sorts scalar types', () => { + expect(TypeConverter.isSortable('string')).toBe(true); + expect(TypeConverter.isSortable('number')).toBe(true); + expect(TypeConverter.isSortable('date')).toBe(true); + }); + }); +}); diff --git a/packages/datasource-dynamodb/test/virtual-collections.test.ts b/packages/datasource-dynamodb/test/virtual-collections.test.ts new file mode 100644 index 0000000000..ce172ae41f --- /dev/null +++ b/packages/datasource-dynamodb/test/virtual-collections.test.ts @@ -0,0 +1,102 @@ +import { createDynamoDBDataSource } from '../src'; +import { fakeClient } from './fake-client'; + +const orders = { + Table: { + TableName: 'Orders', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }, +}; + +const build = ( + items: unknown[], + virtualCollections = [ + { parentCollection: 'Orders', collectionName: 'lines', arrayField: 'lines' }, + ], +) => { + const logger = jest.fn(); + const client = fakeClient({ + DescribeTable: () => orders, + Scan: () => ({ Items: items }), + }); + + return { + logger, + client, + dataSource: createDynamoDBDataSource({ client, tables: ['Orders'], virtualCollections })( + logger, + jest.fn(), + ), + }; +}; + +describe('virtual collections', () => { + it('exposes an array as a collection whose attributes are inferred from the items', async () => { + const { dataSource } = build([ + { + id: 'o1', + lines: [ + { sku: 'A', qty: 1 }, + { sku: 'B', qty: 2, note: 'x' }, + ], + }, + { id: 'o2', lines: [{ sku: 'C', qty: 3 }] }, + ]); + + const { fields } = (await dataSource).getCollection('lines').schema; + + expect(fields).toMatchObject({ + id: { isPrimaryKey: true, columnType: 'String', isReadOnly: true }, + index: { isPrimaryKey: true, columnType: 'Number', isReadOnly: true }, + sku: { columnType: 'String', isPrimaryKey: false }, + qty: { columnType: 'Number' }, + note: { columnType: 'String' }, + }); + }); + + it('samples only the parents that have the array', async () => { + const { client, dataSource } = build([{ id: 'o1', lines: [{ sku: 'A' }] }]); + + await dataSource; + + const sample = client.calls('Scan').find(call => call.FilterExpression); + expect(sample.FilterExpression).toBe( + '(attribute_exists(#n0) AND NOT attribute_type(#n0, :v0))', + ); + expect(sample.ExpressionAttributeNames['#n0']).toBe('lines'); + }); + + it('skips a collection whose parent does not exist, and says so', async () => { + const { logger, dataSource } = build( + [], + [{ parentCollection: 'Missing', collectionName: 'lines', arrayField: 'lines' }], + ); + + const result = await dataSource; + + expect(result.collections.map(c => c.name)).toEqual(['Orders']); + expect(logger).toHaveBeenCalledWith( + 'Warn', + expect.stringContaining("Skipping virtual collection 'lines' for array 'Missing.lines'"), + ); + }); + + it('refuses to expose an array of a virtual collection', async () => { + const { logger, dataSource } = build( + [{ id: 'o1', lines: [{ sku: 'A', parts: [{ n: 1 }] }] }], + [ + { parentCollection: 'Orders', collectionName: 'lines', arrayField: 'lines' }, + { parentCollection: 'lines', collectionName: 'parts', arrayField: 'parts' }, + ], + ); + + const result = await dataSource; + + expect(result.collections.map(c => c.name).sort()).toEqual(['Orders', 'lines']); + expect(logger).toHaveBeenCalledWith( + 'Warn', + "Skipping virtual collection 'parts' for array 'lines.parts': an array of an array cannot be exposed", + ); + }); +}); diff --git a/packages/datasource-dynamodb/tsconfig.eslint.json b/packages/datasource-dynamodb/tsconfig.eslint.json new file mode 100644 index 0000000000..9bdc52705d --- /dev/null +++ b/packages/datasource-dynamodb/tsconfig.eslint.json @@ -0,0 +1,3 @@ +{ + "extends": "../../tsconfig.eslint.json" +} diff --git a/packages/datasource-dynamodb/tsconfig.json b/packages/datasource-dynamodb/tsconfig.json new file mode 100644 index 0000000000..e0d66374ae --- /dev/null +++ b/packages/datasource-dynamodb/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": ["src/**/*"] +} diff --git a/yarn.lock b/yarn.lock index 7f8fc45c4e..813168ef85 100644 --- a/yarn.lock +++ b/yarn.lock @@ -210,6 +210,22 @@ "@smithy/util-utf8" "^4.2.1" tslib "^2.6.2" +"@aws-sdk/client-dynamodb@^3.750.0": + version "3.1155.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/client-dynamodb/-/client-dynamodb-3.1155.0.tgz#46a3e27da555ff2a8669306deb6a0f3d23bdc0ea" + integrity sha512-lbssxoO5CdXhza43hEzzFa0p5P2aa9IYAa5n2l4+g1uVFIKlPybBere3mBBMOUnVwsiVQgV0n2ERg9srNT/L8g== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/credential-provider-node" "^3.972.88" + "@aws-sdk/dynamodb-codec" "^3.973.50" + "@aws-sdk/middleware-endpoint-discovery" "^3.972.34" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/fetch-http-handler" "^5.8.0" + "@smithy/node-http-handler" "^4.12.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/client-kendra@^3.1141.0": version "3.1152.0" resolved "https://registry.yarnpkg.com/@aws-sdk/client-kendra/-/client-kendra-3.1152.0.tgz#d20ff97082bb6e371215c405a0a903620d144125" @@ -332,6 +348,20 @@ bowser "^2.11.0" tslib "^2.6.2" +"@aws-sdk/core@^3.978.5": + version "3.978.5" + resolved "https://registry.yarnpkg.com/@aws-sdk/core/-/core-3.978.5.tgz#1c9442d0cf4d7acb1c51a4a35a804033f995812a" + integrity sha512-ruI0AWqcrfL20XDS6SwZIJFPOBZ6N0ADa9nLzWlBhxk1aoUjbUX3JkTR9vEmk7o35XpAHi68ahZ4T6RE2U9XyA== + dependencies: + "@aws-sdk/types" "^3.975.0" + "@aws-sdk/xml-builder" "^3.972.41" + "@aws/lambda-invoke-store" "^0.3.0" + "@smithy/core" "^3.36.1" + "@smithy/signature-v4" "^5.7.4" + "@smithy/types" "^4.19.0" + bowser "^2.11.0" + tslib "^2.6.2" + "@aws-sdk/crc64-nvme@^3.972.1": version "3.972.1" resolved "https://registry.yarnpkg.com/@aws-sdk/crc64-nvme/-/crc64-nvme-3.972.1.tgz#c32bd754b8d0240e68d8768e0332b5512360114c" @@ -384,6 +414,17 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-env@^3.972.76": + version "3.972.76" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.76.tgz#4433bf572620128375d98f1ceca8912003124734" + integrity sha512-YxmL1Ar7bV9ZkJvtv+lj+MF6le76NnZnbmgySyZuxpZaCv0/1lZ3KxKy6RIj1h9ui2hWnVjZxbmTMeO0y/NrQA== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/credential-provider-http@^3.972.13": version "3.972.13" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.13.tgz#e3874a8ef3902ebbd3d36225fdb4b9b711939625" @@ -426,6 +467,19 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-http@^3.972.78": + version "3.972.78" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.78.tgz#52ad8d402d5b216183220f787aba61ca40be8fbf" + integrity sha512-3VHknDRngoebsaqyT1iR75b5cBT2zfEtODK6yYmYltljkK1HEAQLs2FEd1IpWqAeOYdMGaG9wikngWLTufY7RA== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/fetch-http-handler" "^5.8.0" + "@smithy/node-http-handler" "^4.12.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/credential-provider-ini@^3.972.11": version "3.972.11" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.11.tgz#43a1c4c926fbb1a56b350f3afa17c8f095de27ea" @@ -484,6 +538,25 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-ini@^3.973.21": + version "3.973.21" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.21.tgz#c64cf6bd91de0e0619a1d6de09886bee4165e017" + integrity sha512-aLx9hbCX6mGAyv4ZZd8UagJ4RASHVJw6BFDSIyuKmLkqNL6H47EjBPK6qaJ4B5TDV6mc+vMa0SJFYyo4mkvVQg== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/credential-provider-env" "^3.972.76" + "@aws-sdk/credential-provider-http" "^3.972.78" + "@aws-sdk/credential-provider-login" "^3.972.83" + "@aws-sdk/credential-provider-process" "^3.972.76" + "@aws-sdk/credential-provider-sso" "^3.973.20" + "@aws-sdk/credential-provider-web-identity" "^3.972.82" + "@aws-sdk/nested-clients" "^3.997.50" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/credential-provider-imds" "^4.5.2" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/credential-provider-login@^3.972.11": version "3.972.11" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.11.tgz#33fa3b2e95e77e48be601305773622d0aee30500" @@ -522,6 +595,18 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-login@^3.972.83": + version "3.972.83" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.83.tgz#d54c74c4f8de297044ea79c679f7fc984878d44b" + integrity sha512-9bJU6FjRacF4FK/QHf8Dyxhur7B1grRj8C1Yh4IkpoFW2exQflPdCv0VrbemLW9ZctLNle+AE7TETs9KNvuLYg== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/nested-clients" "^3.997.50" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/credential-provider-node@^3.972.12": version "3.972.12" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.12.tgz#6ca15d141f400851f0b742aab98db652eee4575c" @@ -574,6 +659,23 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-node@^3.972.88": + version "3.972.88" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.88.tgz#41f751b1711a8afcc6a44a573e54f332ca41adc1" + integrity sha512-AG0KID5qhQPzrFkh5b0Cws1Amytvzo0L0Xj6hMgJXEwD6ZXAqninDhSNAWJiN5zU4YVVZnuidnqwPwYJ04IamA== + dependencies: + "@aws-sdk/credential-provider-env" "^3.972.76" + "@aws-sdk/credential-provider-http" "^3.972.78" + "@aws-sdk/credential-provider-ini" "^3.973.21" + "@aws-sdk/credential-provider-process" "^3.972.76" + "@aws-sdk/credential-provider-sso" "^3.973.20" + "@aws-sdk/credential-provider-web-identity" "^3.972.82" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/credential-provider-imds" "^4.5.2" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/credential-provider-process@^3.972.11": version "3.972.11" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.11.tgz#3434eafdc92d706eb6838c1efc5e5d04f341a731" @@ -608,6 +710,17 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-process@^3.972.76": + version "3.972.76" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.76.tgz#0155e83b93379083bbb51686fc4662b467f1acc2" + integrity sha512-pdhKFTXv1zPYhh92MTlp0jVbLWvUEV9v+rziuEkoskGGQ2iJskxHTO38nA8UAbT/N4IM3n8OZJ9rIY5IBO15/A== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/credential-provider-sso@^3.972.11": version "3.972.11" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.11.tgz#ed8184019f0e046ea07d3ca19f2a52235afc5554" @@ -648,6 +761,19 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-sso@^3.973.20": + version "3.973.20" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.20.tgz#1d5ad65a5d1e86516bc27f134140ee0b82d27e0b" + integrity sha512-DlxrAB3MnjNLx/da7b89W+3DyJQYso/q7Ns15MsST2IV2nkAYQOYTUCaffNSjgrpMP99Dm2mV2jxzcbyF94ZfA== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/nested-clients" "^3.997.50" + "@aws-sdk/token-providers" "3.1155.0" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/credential-provider-web-identity@^3.972.11": version "3.972.11" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.11.tgz#976a5499f6828b00e068a3c81b7400278e71ad03" @@ -685,6 +811,18 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-web-identity@^3.972.82": + version "3.972.82" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.82.tgz#0919b59684666ff0c5649da7bb48839c99b277c1" + integrity sha512-LUo46IqPTJ4xKBsrLuoaxiv/D2GTBCcFDMD1y3jKIqPR/p7Rw14RJFZ0iRWftqZtyLPQX9JMGnSi3mBu4y/pNA== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/nested-clients" "^3.997.50" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/credential-providers@^3.186.0": version "3.997.0" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-providers/-/credential-providers-3.997.0.tgz#1b77e660142e13e5cf483fa70d810051bcea3424" @@ -711,6 +849,24 @@ "@smithy/types" "^4.12.1" tslib "^2.6.2" +"@aws-sdk/dynamodb-codec@^3.973.50": + version "3.973.50" + resolved "https://registry.yarnpkg.com/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.973.50.tgz#0042410ed4319b06f4c9299c74f1eae0bb698fb6" + integrity sha512-1jn3q2Ik/qUjPw1su5Abu6nzeDutzYbNUYKt+AqUh+45rb+MVWhgP1QWarTuxrm4MJKcSLVHLCrvi0rY5jmJZw== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + +"@aws-sdk/endpoint-cache@^3.972.11": + version "3.972.11" + resolved "https://registry.yarnpkg.com/@aws-sdk/endpoint-cache/-/endpoint-cache-3.972.11.tgz#0d4a52d6307e240d0f404c649f9a58b773511d03" + integrity sha512-8q1ICxcDjHId3bBryuu/j+1L9y5/3uQnwzLDt5j2ElcjZSoWmFtymdJy7OjLrluSMe0Z4mq5bcH4fxBXvlEHfw== + dependencies: + mnemonist "0.38.3" + tslib "^2.6.2" + "@aws-sdk/eventstream-handler-node@^3.972.36": version "3.972.36" resolved "https://registry.yarnpkg.com/@aws-sdk/eventstream-handler-node/-/eventstream-handler-node-3.972.36.tgz#ee96a93da3ea30242259b99df858a012cd7eb057" @@ -721,6 +877,17 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/lib-dynamodb@^3.750.0": + version "3.1155.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/lib-dynamodb/-/lib-dynamodb-3.1155.0.tgz#3b802780c7ac2258b35acdba5f9e2d349ea6b481" + integrity sha512-/84eZiKfFPMPozNPMxX346C2KdziZOqrXk7lsBmcteE8HmFTphapaATgCq0785ts4TmK1J3L/65EdqU8rzYtSw== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/util-dynamodb" "^3.996.9" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/middleware-bucket-endpoint@^3.972.4": version "3.972.4" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.972.4.tgz#03f07cce35d0983c5767e4a5ac26dafeabb7050c" @@ -734,6 +901,17 @@ "@smithy/util-config-provider" "^4.2.1" tslib "^2.6.2" +"@aws-sdk/middleware-endpoint-discovery@^3.972.34": + version "3.972.34" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.34.tgz#431383cc739b586537ff166c72ba13f84470f1dc" + integrity sha512-hwPWSvttg+bEpcM9e8HdVJF/oz79xInjnou23+GdsbDv75DVDLGhn2XhIbuNghp5SqyjtngXEI+K22Vq3VvSxw== + dependencies: + "@aws-sdk/endpoint-cache" "^3.972.11" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/middleware-eventstream@^3.972.31": version "3.972.31" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-eventstream/-/middleware-eventstream-3.972.31.tgz#9a15a621dde2c880fe084613378d55d9070a740d" @@ -940,6 +1118,20 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/nested-clients@^3.997.50": + version "3.997.50" + resolved "https://registry.yarnpkg.com/@aws-sdk/nested-clients/-/nested-clients-3.997.50.tgz#fcf3263fc47040aac27990970017d737a2a21683" + integrity sha512-nKb8qu4QsnpZTq/6JIXc72mzuzfh7BK4sUQkpQznPec2z4W7Ob2xYkcM4i5e1yGTBnbJVeFv/0sf3jcz85hclA== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/signature-v4-multi-region" "^3.996.49" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/fetch-http-handler" "^5.8.0" + "@smithy/node-http-handler" "^4.12.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/region-config-resolver@^3.972.4": version "3.972.4" resolved "https://registry.yarnpkg.com/@aws-sdk/region-config-resolver/-/region-config-resolver-3.972.4.tgz#aa178d3f3cbac33e2f5ade15a160dad2dfd062c6" @@ -997,6 +1189,16 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/signature-v4-multi-region@^3.996.49": + version "3.996.49" + resolved "https://registry.yarnpkg.com/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.49.tgz#2e7d874620cc19922f613f8f1e0c4c0cb38b9598" + integrity sha512-nvuAZNRrRRVCiARRKsYlQ0VnXiEQiq7vXPH+UA+LSo+kN3Djl9eo/V2r0udkw85PnsNshmv2Em1clld5M3j67w== + dependencies: + "@aws-sdk/types" "^3.975.0" + "@smithy/signature-v4" "^5.7.4" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/token-providers@3.1129.0": version "3.1129.0" resolved "https://registry.yarnpkg.com/@aws-sdk/token-providers/-/token-providers-3.1129.0.tgz#a4a11a789d7259874ccf3733e538fc70f8a289e4" @@ -1045,6 +1247,18 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/token-providers@3.1155.0": + version "3.1155.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/token-providers/-/token-providers-3.1155.0.tgz#201abec26c2815fb60bce72583d5fc30e4b63d36" + integrity sha512-iIcQnYLhWuq9qOLQbIWFrxF3PF0svVeZvHnUS4DUnO4zUFmFHIyGyjqrntXG+e/5PUgfavV0pYfDIKHxTzN2LA== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/nested-clients" "^3.997.50" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/token-providers@3.997.0": version "3.997.0" resolved "https://registry.yarnpkg.com/@aws-sdk/token-providers/-/token-providers-3.997.0.tgz#3941e72c86c70776f60030947ae88d428e8a1591" @@ -1082,6 +1296,14 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/types@^3.975.0": + version "3.975.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.975.0.tgz#15fb614cf1877ee41c5f08edfd21818dc90edd1d" + integrity sha512-WQlhs6juByzWOzj5fwxOx0mPJaEqMcG/sDpl3Id61B3XhKE63xY+0QWcnG+sWR/sCfprmreyUo1lM7IKZmHMzw== + dependencies: + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/util-arn-parser@^3.972.2": version "3.972.2" resolved "https://registry.yarnpkg.com/@aws-sdk/util-arn-parser/-/util-arn-parser-3.972.2.tgz#ef18ba889e8ef35f083f1e962018bc0ce70acef3" @@ -1089,6 +1311,13 @@ dependencies: tslib "^2.6.2" +"@aws-sdk/util-dynamodb@^3.996.9": + version "3.996.9" + resolved "https://registry.yarnpkg.com/@aws-sdk/util-dynamodb/-/util-dynamodb-3.996.9.tgz#493c00ffec4280450492966867197bbd26d50b83" + integrity sha512-16x2tRvl7OYpZ0W/DdFJieFriD13+RvuRBDbe5sj/tCEfK86HSGd7I2s5j0ivz8p6KWGkS+5wKRO9OliJkjUOQ== + dependencies: + tslib "^2.6.2" + "@aws-sdk/util-endpoints@^3.996.1": version "3.996.1" resolved "https://registry.yarnpkg.com/@aws-sdk/util-endpoints/-/util-endpoints-3.996.1.tgz#1fd0f3800c899f80be067318795301b85412484d" @@ -2293,6 +2522,16 @@ object-hash "^3.0.0" uuid "^9.0.0" +"@forestadmin/datasource-toolkit@1.56.1": + version "1.56.1" + resolved "https://registry.yarnpkg.com/@forestadmin/datasource-toolkit/-/datasource-toolkit-1.56.1.tgz#9d4d650394ea2f79dc0e1e993ab775c6e81d79c6" + integrity sha512-ztPcbtoki4SSyKqMA6+YTaFGSo+CA0ynSQPjux9qjxD2H+OVe6nlYoMIfc/noaa9reXzuF66BJWpEoVS9MxTUg== + dependencies: + "@forestadmin/agent-toolkit" "1.2.1" + luxon "^3.2.1" + object-hash "^3.0.0" + uuid "11.1.1" + "@gar/promise-retry@^1.0.0", "@gar/promise-retry@^1.0.2": version "1.0.3" resolved "https://registry.yarnpkg.com/@gar/promise-retry/-/promise-retry-1.0.3.tgz#65e726428e794bc4453948e0a41e6de4215ce8b0" @@ -4393,6 +4632,14 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@smithy/core@^3.36.1": + version "3.36.2" + resolved "https://registry.yarnpkg.com/@smithy/core/-/core-3.36.2.tgz#dc8348dbdd9604a297e93ab791266d6511c2038c" + integrity sha512-oRw0rB9QSUiCZfEs23G5F4Af+b2rQt1lItIICWu5IejFNd+uPM8SWqiKLMpQ6bX1Q7slsLbjuoE2niErFtkVPQ== + dependencies: + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@smithy/credential-provider-imds@^4.2.10", "@smithy/credential-provider-imds@^4.2.9": version "4.2.10" resolved "https://registry.yarnpkg.com/@smithy/credential-provider-imds/-/credential-provider-imds-4.2.10.tgz#cae502b28257a110fc472a1531c19fbc4ba07037" @@ -13606,6 +13853,13 @@ mkdirp@1.0.4, mkdirp@^1.0.3, mkdirp@^1.0.4: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== +mnemonist@0.38.3: + version "0.38.3" + resolved "https://registry.yarnpkg.com/mnemonist/-/mnemonist-0.38.3.tgz#35ec79c1c1f4357cfda2fe264659c2775ccd7d9d" + integrity sha512-2K9QYubXx/NAjv4VLq1d1Ly8pWNC5L3BrixtdkyTegXWJIqY+zLNDhhX/A+ZwWt70tB1S8H4BE8FLYEFyNoOBw== + dependencies: + obliterator "^1.6.1" + mobx-react-lite@^4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/mobx-react-lite/-/mobx-react-lite-4.1.1.tgz#725d74b025235f73dc2ab815766ffe010be2cf8a" @@ -14628,6 +14882,11 @@ object.values@^1.2.1: define-properties "^1.2.1" es-object-atoms "^1.0.0" +obliterator@^1.6.1: + version "1.6.1" + resolved "https://registry.yarnpkg.com/obliterator/-/obliterator-1.6.1.tgz#dea03e8ab821f6c4d96a299e17aef6a3af994ef3" + integrity sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig== + ohash@^2.0.11: version "2.0.12" resolved "https://registry.yarnpkg.com/ohash/-/ohash-2.0.12.tgz#cb97c19888ff31e7b22214cd7924187fc97ef05c" From a148e8fe967591f77536a6674100724bfdb7e60c Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Fri, 16 Oct 2026 19:30:07 +0200 Subject: [PATCH 02/32] fix(datasource-dynamodb): depend on the workspace version of the toolkit The package pinned a toolkit version older than the workspace's, so yarn installed the published one next to it and its error classes came out untyped in the build. Refs: PRD-1670 Co-Authored-By: Claude Sonnet 5.5 --- packages/datasource-dynamodb/package.json | 2 +- yarn.lock | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/packages/datasource-dynamodb/package.json b/packages/datasource-dynamodb/package.json index 4a01cf86de..97ef334284 100644 --- a/packages/datasource-dynamodb/package.json +++ b/packages/datasource-dynamodb/package.json @@ -12,7 +12,7 @@ "directory": "packages/datasource-dynamodb" }, "dependencies": { - "@forestadmin/datasource-toolkit": "1.56.1" + "@forestadmin/datasource-toolkit": "1.56.2" }, "peerDependencies": { "@aws-sdk/client-dynamodb": "^3.750.0", diff --git a/yarn.lock b/yarn.lock index 813168ef85..4e1521e0f4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2522,16 +2522,6 @@ object-hash "^3.0.0" uuid "^9.0.0" -"@forestadmin/datasource-toolkit@1.56.1": - version "1.56.1" - resolved "https://registry.yarnpkg.com/@forestadmin/datasource-toolkit/-/datasource-toolkit-1.56.1.tgz#9d4d650394ea2f79dc0e1e993ab775c6e81d79c6" - integrity sha512-ztPcbtoki4SSyKqMA6+YTaFGSo+CA0ynSQPjux9qjxD2H+OVe6nlYoMIfc/noaa9reXzuF66BJWpEoVS9MxTUg== - dependencies: - "@forestadmin/agent-toolkit" "1.2.1" - luxon "^3.2.1" - object-hash "^3.0.0" - uuid "11.1.1" - "@gar/promise-retry@^1.0.0", "@gar/promise-retry@^1.0.2": version "1.0.3" resolved "https://registry.yarnpkg.com/@gar/promise-retry/-/promise-retry-1.0.3.tgz#65e726428e794bc4453948e0a41e6de4215ce8b0" From 8257038ff5fe3b7cc5f03cf5d8da1736bb1a6704 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Fri, 16 Oct 2026 19:36:40 +0200 Subject: [PATCH 03/32] style(datasource-dynamodb): fix the lint errors the CI reported Refs: PRD-1670 Co-Authored-By: Claude Sonnet 5.5 --- .../src/array-collection.ts | 5 ++--- packages/datasource-dynamodb/src/client.ts | 4 +++- .../datasource-dynamodb/src/collection.ts | 9 ++++---- packages/datasource-dynamodb/src/errors.ts | 1 + packages/datasource-dynamodb/src/index.ts | 2 +- .../src/utils/access-path.ts | 13 ++++++----- .../src/utils/aggregator.ts | 22 ++++++++++++++----- .../datasource-dynamodb/src/utils/executor.ts | 1 + .../src/utils/expression-builder.ts | 6 +++-- .../datasource-dynamodb/src/utils/sort.ts | 2 +- .../test/array-collection.test.ts | 3 ++- .../test/collection.aggregate.test.ts | 5 ++++- .../test/datasource.test.ts | 4 ++-- 13 files changed, 49 insertions(+), 28 deletions(-) diff --git a/packages/datasource-dynamodb/src/array-collection.ts b/packages/datasource-dynamodb/src/array-collection.ts index 80a0fa80b4..323b65d187 100644 --- a/packages/datasource-dynamodb/src/array-collection.ts +++ b/packages/datasource-dynamodb/src/array-collection.ts @@ -4,7 +4,6 @@ import type { AggregateResult, Aggregation, Caller, - ColumnSchema, ConditionTree, DataSource, Filter, @@ -26,7 +25,7 @@ import convertModelToFields from './schema/schema-converter'; import StreamingAggregator from './utils/aggregator'; import ExpressionBuilder from './utils/expression-builder'; import Serializer from './utils/serializer'; -import { compareRecords } from './utils/sort'; +import compareRecords from './utils/sort'; export const INDEX_FIELD = 'index'; @@ -123,7 +122,7 @@ export default class ArrayCollection extends BaseCollection { const skip = filter.page?.skip ?? 0; const limit = filter.page?.limit || undefined; - const columns = projection.columns; + const { columns } = projection; return records .slice(skip, limit === undefined ? undefined : skip + limit) diff --git a/packages/datasource-dynamodb/src/client.ts b/packages/datasource-dynamodb/src/client.ts index fce7d4c108..3cc1c52326 100644 --- a/packages/datasource-dynamodb/src/client.ts +++ b/packages/datasource-dynamodb/src/client.ts @@ -15,7 +15,9 @@ function isLowLevelClient(client: unknown): client is DynamoDBClient { return client instanceof DynamoDBClient || client?.constructor?.name === 'DynamoDBClient'; } -export function createDocumentClient(options: DynamoDBClientOptions): DynamoDBDocumentClient { +export default function createDocumentClient( + options: DynamoDBClientOptions, +): DynamoDBDocumentClient { if (options.client) { return isLowLevelClient(options.client) ? DynamoDBDocumentClient.from(options.client, TRANSLATE_CONFIG) diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index 704a5af50d..41ffddb3f3 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -1,6 +1,6 @@ import type { CollectionSettings, TableModel } from './types'; import type { AccessPath, KeyLookup } from './utils/access-path'; -import type { Item, ReadInput, ReadOperation, ReadBudget } from './utils/executor'; +import type { Item, ReadBudget, ReadInput, ReadOperation } from './utils/executor'; import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; import type { AggregateResult, @@ -34,7 +34,7 @@ import StreamingAggregator from './utils/aggregator'; import DynamoExecutor from './utils/executor'; import ExpressionBuilder, { splitCondition } from './utils/expression-builder'; import Serializer from './utils/serializer'; -import { compareRecords } from './utils/sort'; +import compareRecords from './utils/sort'; type ReadPlan = { lookup?: KeyLookup; access?: AccessPath; residual?: ConditionTree }; @@ -355,7 +355,7 @@ export default class DynamoDBCollection extends BaseCollection { ): Promise { const { access, residual } = plan; const cache = this.settings.paginationCache; - const maxOffset = this.settings.maxOffset; + const { maxOffset } = this.settings; if (skip > maxOffset) { throw new OffsetTooDeepError( @@ -424,7 +424,7 @@ export default class DynamoDBCollection extends BaseCollection { this.guardScan(plan, filter.conditionTree); - const columns = projection.columns; + const { columns } = projection; const fields = [ ...new Set([ ...columns, @@ -433,6 +433,7 @@ export default class DynamoDBCollection extends BaseCollection { ...this.positionAttributes(plan.access), ]), ]; + const toResult = (item: Item) => { const record = this.toRecord(item); diff --git a/packages/datasource-dynamodb/src/errors.ts b/packages/datasource-dynamodb/src/errors.ts index d241ad64bc..8dc741edea 100644 --- a/packages/datasource-dynamodb/src/errors.ts +++ b/packages/datasource-dynamodb/src/errors.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-classes-per-file */ import { BadRequestError, ForbiddenError, diff --git a/packages/datasource-dynamodb/src/index.ts b/packages/datasource-dynamodb/src/index.ts index cc0163ab63..2e2d3a65f4 100644 --- a/packages/datasource-dynamodb/src/index.ts +++ b/packages/datasource-dynamodb/src/index.ts @@ -11,7 +11,7 @@ import type { DataSourceFactory } from '@forestadmin/datasource-toolkit'; import { ValidationError } from '@forestadmin/datasource-toolkit'; -import { createDocumentClient } from './client'; +import createDocumentClient from './client'; import DynamoDBDataSource from './datasource'; import { DynamoDBDatasourceBuilder } from './introspection/builder'; import { diff --git a/packages/datasource-dynamodb/src/utils/access-path.ts b/packages/datasource-dynamodb/src/utils/access-path.ts index 89fb541bc6..f0eff21806 100644 --- a/packages/datasource-dynamodb/src/utils/access-path.ts +++ b/packages/datasource-dynamodb/src/utils/access-path.ts @@ -61,12 +61,13 @@ function withEntity(model: TableModel, tree?: ConditionTree): ConditionTree | un if (!entity) return tree; - const restriction = - entity.sortKeyPrefix !== undefined && model.sortKey - ? new ConditionTreeLeaf(model.sortKey.name, 'StartsWith', entity.sortKeyPrefix) - : entity.attribute !== undefined - ? new ConditionTreeLeaf(entity.attribute, 'Equal', entity.value) - : undefined; + let restriction: ConditionTreeLeaf | undefined; + + if (entity.sortKeyPrefix !== undefined && model.sortKey) { + restriction = new ConditionTreeLeaf(model.sortKey.name, 'StartsWith', entity.sortKeyPrefix); + } else if (entity.attribute !== undefined) { + restriction = new ConditionTreeLeaf(entity.attribute, 'Equal', entity.value); + } if (!restriction) return tree; diff --git a/packages/datasource-dynamodb/src/utils/aggregator.ts b/packages/datasource-dynamodb/src/utils/aggregator.ts index c94267629a..d7aaeb0a2b 100644 --- a/packages/datasource-dynamodb/src/utils/aggregator.ts +++ b/packages/datasource-dynamodb/src/utils/aggregator.ts @@ -1,5 +1,9 @@ -import type { AggregateResult, Aggregation, DateOperation } from '@forestadmin/datasource-toolkit'; -import type { RecordData } from '@forestadmin/datasource-toolkit'; +import type { + AggregateResult, + Aggregation, + DateOperation, + RecordData, +} from '@forestadmin/datasource-toolkit'; /** Numbers below this are epoch seconds: in milliseconds they would be before 1973. */ const EPOCH_SECONDS_CEILING = 1e11; @@ -19,11 +23,15 @@ type Accumulator = { function toDate(value: unknown): Date | null { let date: Date; - if (value instanceof Date) date = value; - else if (typeof value === 'number') { + if (value instanceof Date) { + date = value; + } else if (typeof value === 'number') { date = new Date(value < EPOCH_SECONDS_CEILING ? value * 1000 : value); - } else if (typeof value === 'string') date = new Date(value); - else return null; + } else if (typeof value === 'string') { + date = new Date(value); + } else { + return null; + } return Number.isNaN(date.getTime()) ? null : date; } @@ -66,6 +74,7 @@ function dateBucket(value: unknown, operation: DateOperation, timezone: string): )}-01`; case 'Month': return `${year}-${two(month)}-01`; + case 'Week': { const monday = new Date(Date.UTC(year, month - 1, day)); monday.setUTCDate( @@ -74,6 +83,7 @@ function dateBucket(value: unknown, operation: DateOperation, timezone: string): return monday.toISOString().slice(0, 10); } + default: return `${year}-${two(month)}-${two(day)}`; } diff --git a/packages/datasource-dynamodb/src/utils/executor.ts b/packages/datasource-dynamodb/src/utils/executor.ts index 43a1fa1cb4..d78317537d 100644 --- a/packages/datasource-dynamodb/src/utils/executor.ts +++ b/packages/datasource-dynamodb/src/utils/executor.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-classes-per-file */ import type { CollectionSettings, TableModel } from '../types'; import type { StartKey } from './pagination-cache'; import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; diff --git a/packages/datasource-dynamodb/src/utils/expression-builder.ts b/packages/datasource-dynamodb/src/utils/expression-builder.ts index 2edd9b212c..d8f98b82cd 100644 --- a/packages/datasource-dynamodb/src/utils/expression-builder.ts +++ b/packages/datasource-dynamodb/src/utils/expression-builder.ts @@ -1,7 +1,6 @@ -import type { ConditionTree, Operator } from '@forestadmin/datasource-toolkit'; +import type { ConditionTree, ConditionTreeBranch, Operator } from '@forestadmin/datasource-toolkit'; import { - ConditionTreeBranch, ConditionTreeFactory, ConditionTreeLeaf, ValidationError, @@ -210,12 +209,14 @@ export default class ExpressionBuilder { )})`; case 'In': return values.length === 0 ? this.never(path) : this.in(path, values); + case 'NotIn': { if (values.length === 0) return this.always(path); const matches = this.in(path, values); return matches.startsWith('(') ? `NOT ${matches}` : `NOT (${matches})`; } + case 'StartsWith': return `begins_with(${path}, ${this.value(value)})`; case 'Contains': @@ -232,6 +233,7 @@ export default class ExpressionBuilder { return `(${values .map(item => `NOT contains(${path}, ${this.value(item)})`) .join(' AND ')})`; + default: { const comparison = COMPARISONS[operator]; diff --git a/packages/datasource-dynamodb/src/utils/sort.ts b/packages/datasource-dynamodb/src/utils/sort.ts index 6111cc0604..53c5d3eae1 100644 --- a/packages/datasource-dynamodb/src/utils/sort.ts +++ b/packages/datasource-dynamodb/src/utils/sort.ts @@ -1,7 +1,7 @@ import type { RecordData, Sort } from '@forestadmin/datasource-toolkit'; /** Missing values come last whatever the direction, as DynamoDB has no notion of nulls order. */ -export function compareRecords(a: RecordData, b: RecordData, sort: Sort): number { +export default function compareRecords(a: RecordData, b: RecordData, sort: Sort): number { for (const { field, ascending } of sort) { const [left, right] = [a[field], b[field]]; const leftMissing = left === undefined || left === null; diff --git a/packages/datasource-dynamodb/test/array-collection.test.ts b/packages/datasource-dynamodb/test/array-collection.test.ts index 964d278386..686b3ca969 100644 --- a/packages/datasource-dynamodb/test/array-collection.test.ts +++ b/packages/datasource-dynamodb/test/array-collection.test.ts @@ -1,3 +1,5 @@ +import type { DataSource } from '@forestadmin/datasource-toolkit'; + import { Aggregation, Filter, @@ -5,7 +7,6 @@ import { Projection, Sort, } from '@forestadmin/datasource-toolkit'; -import type { DataSource } from '@forestadmin/datasource-toolkit'; import { buildCollection } from './collection-helpers'; import { and, buildModel, leaf, plain } from './helpers'; diff --git a/packages/datasource-dynamodb/test/collection.aggregate.test.ts b/packages/datasource-dynamodb/test/collection.aggregate.test.ts index 7dbb838172..9473647906 100644 --- a/packages/datasource-dynamodb/test/collection.aggregate.test.ts +++ b/packages/datasource-dynamodb/test/collection.aggregate.test.ts @@ -414,7 +414,10 @@ describe('DynamoDBCollection.aggregate', () => { aggregation({ operation: 'Count' }), ); - for (let age = 0; age <= 100; age += 1) await run(age); + for (let age = 0; age <= 100; age += 1) { + // eslint-disable-next-line no-await-in-loop + await run(age); + } client.send.mockClear(); await run(100); diff --git a/packages/datasource-dynamodb/test/datasource.test.ts b/packages/datasource-dynamodb/test/datasource.test.ts index 06b80e4df0..b9ff9ca9f4 100644 --- a/packages/datasource-dynamodb/test/datasource.test.ts +++ b/packages/datasource-dynamodb/test/datasource.test.ts @@ -1,9 +1,9 @@ import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; -import { createDocumentClient } from '../src/client'; -import { createDynamoDBDataSource } from '../src/index'; import { fakeClient } from './fake-client'; +import createDocumentClient from '../src/client'; +import { createDynamoDBDataSource } from '../src/index'; const table = (name: string, keyType = 'S') => ({ Table: { From bf716b000aa39931ebabaa43e253379bcd4d0eab Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Fri, 16 Oct 2026 19:41:51 +0200 Subject: [PATCH 04/32] feat(datasource-dynamodb): expose arrays of arrays as collections The ticket listed nested arrays among the virtual collections, and the first version refused them. A collection can now have an array collection as its parent: its records are keyed by the table key and the position in each array, and writes address the element through the whole path. An array held in a map (a->b) is exposed too. Refs: PRD-1670 Co-Authored-By: Claude Sonnet 5.5 --- packages/datasource-dynamodb/README.md | 17 +- .../src/array-collection.ts | 221 ++++++++++++------ .../src/virtual-collections.ts | 9 +- .../test/array-collection.test.ts | 120 ++++++++++ .../test/integration/dynamodb-local.test.ts | 80 ++++++- .../test/virtual-collections.test.ts | 25 +- 6 files changed, 385 insertions(+), 87 deletions(-) diff --git a/packages/datasource-dynamodb/README.md b/packages/datasource-dynamodb/README.md index 5c7917341c..5a35c024e6 100644 --- a/packages/datasource-dynamodb/README.md +++ b/packages/datasource-dynamodb/README.md @@ -142,9 +142,20 @@ createDynamoDBDataSource({ }); ``` -Each element of the array becomes a record, identified by the key of its parent and its `index` in -the array. Creating appends, updating sets the element's attributes, and deleting removes it, so the -index of the following elements shifts. +Each element of the array becomes a record, identified by the key of the table item and its `index` +in the array. Creating appends, updating sets the element's attributes, and deleting removes it, so +the index of the following elements shifts. + +`arrayField` can point inside a map (`'meta->items'`). An array of an array is a collection whose +parent is the first one, declared after it: the position in the outer array becomes a key column +named `Index`, and creating an inner element needs it. + +```ts +virtualCollections: [ + { parentCollection: 'orders', collectionName: 'order_lines', arrayField: 'lines' }, + { parentCollection: 'order_lines', collectionName: 'line_parts', arrayField: 'parts' }, +]; +``` ## Native queries diff --git a/packages/datasource-dynamodb/src/array-collection.ts b/packages/datasource-dynamodb/src/array-collection.ts index 323b65d187..e37dd07c29 100644 --- a/packages/datasource-dynamodb/src/array-collection.ts +++ b/packages/datasource-dynamodb/src/array-collection.ts @@ -29,9 +29,12 @@ import compareRecords from './utils/sort'; export const INDEX_FIELD = 'index'; +/** One array on the way from the item of the table to the records of a collection. */ +type Level = { arrayField: string; indexColumn: string }; + export type ArrayCollectionConfig = { name: string; - /** Attribute of the parent holding the array, `a->b` for a nested one. */ + /** Attribute of the parent holding the array, `a->b` for one inside a map. */ arrayField: string; /** Attributes of an item, by flattened name. */ itemSchema: Record; @@ -50,35 +53,54 @@ function conjuncts(tree?: ConditionTree): ConditionTree[] { } /** - * Exposes an array of maps stored in the items of another collection as a collection of its own. - * A record is identified by the key of its parent and its position in the array, which is why - * those positions move when an earlier item of the same array is removed. + * Exposes an array of maps stored in the items of another collection, or of another array + * collection, as a collection of its own. A record is identified by the key of the table item and + * its position in each array on the way, which is why positions move when an earlier element of + * the same array is removed. The position in the parent array is named `Index`. */ export default class ArrayCollection extends BaseCollection { - private readonly parent: DynamoDBCollection; - private readonly arrayField: string; - private readonly parentKeys: string[]; + private readonly parent: DynamoDBCollection | ArrayCollection; + private readonly root: DynamoDBCollection; + private readonly chain: Level[]; + private readonly rootKeys: string[]; private readonly itemFields: Set; - constructor(datasource: DataSource, parent: DynamoDBCollection, config: ArrayCollectionConfig) { + /** The columns identifying a record: the table keys, then one position per array. */ + readonly keyColumns: string[]; + + constructor( + datasource: DataSource, + parent: DynamoDBCollection | ArrayCollection, + config: ArrayCollectionConfig, + ) { super(config.name, datasource, parent.nativeDriver); + const nested = parent instanceof ArrayCollection; + this.parent = parent; - this.arrayField = config.arrayField; + this.root = nested ? parent.root : parent; - const { partitionKey, sortKey } = parent.model; + const { partitionKey, sortKey } = this.root.model; const keys = [partitionKey, sortKey].filter(Boolean); - this.parentKeys = keys.map(key => key.name); + this.rootKeys = keys.map(key => key.name); + + const parentLevels = nested + ? parent.chain.map((level, i, all) => + i === all.length - 1 ? { ...level, indexColumn: `${parent.name}Index` } : level, + ) + : []; + this.chain = [...parentLevels, { arrayField: config.arrayField, indexColumn: INDEX_FIELD }]; + this.keyColumns = [...this.rootKeys, ...this.chain.map(level => level.indexColumn)]; const schema: Record = { ...config.itemSchema, ...Object.fromEntries( keys.map(key => [key.name, { type: key.type === 'N' ? 'number' : 'string' }]), ), - [INDEX_FIELD]: { type: 'number' }, + ...Object.fromEntries(this.chain.map(level => [level.indexColumn, { type: 'number' }])), }; const model: TableModel = { - ...parent.model, + ...this.root.model, name: config.name, indexes: [], schema, @@ -86,7 +108,7 @@ export default class ArrayCollection extends BaseCollection { }; const fields = convertModelToFields(model); - for (const name of [...this.parentKeys, INDEX_FIELD]) { + for (const name of this.keyColumns) { Object.assign(fields[name], { isPrimaryKey: true, isReadOnly: true }); } @@ -105,7 +127,7 @@ export default class ArrayCollection extends BaseCollection { created.push(await this.append(record)); } - this.parent.invalidateCaches(); + this.root.invalidateCaches(); return created; } @@ -133,7 +155,7 @@ export default class ArrayCollection extends BaseCollection { async update(caller: Caller, filter: Filter, patch: RecordData): Promise { for (const field of Object.keys(patch)) { - if (field === INDEX_FIELD || this.parentKeys.includes(field)) { + if (this.keyColumns.includes(field)) { throw new ValidationError(`The attribute '${field}' of '${this.name}' cannot be updated`); } } @@ -142,16 +164,17 @@ export default class ArrayCollection extends BaseCollection { if (entries.length === 0) return; - await this.writePerParent(caller, filter, (builder, indexes) => { + await this.writePerTableItem(caller, filter, (builder, positions) => { const values = entries.map(([, value]) => builder.value(value)); - const array = builder.attribute(this.arrayField); return { - UpdateExpression: `SET ${indexes - .flatMap(index => + UpdateExpression: `SET ${positions + .flatMap(position => entries.map( - ([field], position) => - `${array}[${index}].${builder.attribute(field)} = ${values[position]}`, + ([field], index) => + `${this.elementPath(builder, position)}.${builder.attribute(field)} = ${ + values[index] + }`, ), ) .join(', ')}`, @@ -160,11 +183,18 @@ export default class ArrayCollection extends BaseCollection { } async delete(caller: Caller, filter: Filter): Promise { - await this.writePerParent(caller, filter, (builder, indexes) => { - const array = builder.attribute(this.arrayField); - const descending = [...indexes].sort((a, b) => b - a); + await this.writePerTableItem(caller, filter, (builder, positions) => { + const descending = [...positions].sort((a, b) => { + const different = a.findIndex((index, level) => index !== b[level]); - return { UpdateExpression: `REMOVE ${descending.map(i => `${array}[${i}]`).join(', ')}` }; + return different === -1 ? 0 : b[different] - a[different]; + }); + + return { + UpdateExpression: `REMOVE ${descending + .map(position => this.elementPath(builder, position)) + .join(', ')}`, + }; }); } @@ -183,35 +213,57 @@ export default class ArrayCollection extends BaseCollection { return aggregator.result(limit); } - /** Reads the parents, then turns each array item into a record and applies the filter to it. */ + /** The path of an element: each array of the chain followed by the position taken in it. */ + private elementPath( + builder: ExpressionBuilder, + positions: number[], + levels = this.chain.length, + ): string { + return this.chain + .slice(0, levels) + .map((level, i) => `${builder.attribute(level.arrayField)}[${positions[i]}]`) + .join('.'); + } + + /** The columns identifying the parent element, as named by the parent. */ + private parentColumns(): string[] { + return this.parent instanceof ArrayCollection ? this.parent.keyColumns : this.rootKeys; + } + + /** Reads the parents, then turns each array element into a record and applies the filter. */ private async readRecords(caller: Caller, tree?: ConditionTree): Promise { - // Conditions on the parent key narrow the parents read; the rest applies to the items. - const onParentKey = conjuncts(tree).filter( + // Conditions on the table key narrow the parents read; the rest applies to the elements. + const onTableKey = conjuncts(tree).filter( condition => - condition instanceof ConditionTreeLeaf && this.parentKeys.includes(condition.field), + condition instanceof ConditionTreeLeaf && this.rootKeys.includes(condition.field), ); + const { arrayField } = this.chain[this.chain.length - 1]; + const parentColumns = this.parentColumns(); + const ownKeyColumns = this.keyColumns.slice(0, -1); const parents = await this.parent.list( caller, new PaginatedFilterClass({ - conditionTree: onParentKey.length - ? ConditionTreeFactory.intersect(...onParentKey) + conditionTree: onTableKey.length + ? ConditionTreeFactory.intersect(...onTableKey) : undefined, }), - new Projection(...this.parentKeys, this.arrayField), + new Projection(...parentColumns, arrayField), ); const records: RecordData[] = []; for (const parent of parents) { - const items = parent[this.arrayField]; + const elements = parent[arrayField]; - if (Array.isArray(items)) { - items.forEach((item, index) => { - if (item && typeof item === 'object') { + if (Array.isArray(elements)) { + elements.forEach((element, index) => { + if (element && typeof element === 'object') { records.push({ - ...Object.fromEntries(this.parentKeys.map(key => [key, parent[key]])), + ...Object.fromEntries( + ownKeyColumns.map((name, i) => [name, parent[parentColumns[i]]]), + ), [INDEX_FIELD]: index, - ...Serializer.flatten(item as Item, this.itemFields), + ...Serializer.flatten(element as Item, this.itemFields), }); } }); @@ -222,38 +274,44 @@ export default class ArrayCollection extends BaseCollection { } private async append(record: RecordData): Promise { - const key: Item = {}; + const parentKeyColumns = this.keyColumns.slice(0, -1); - for (const name of this.parentKeys) { + for (const name of parentKeyColumns) { if (record[name] === undefined || record[name] === null) { throw new ValidationError( `The key attribute '${name}' of the parent '${this.parent.name}' is required to create ` + `a record in '${this.name}'`, ); } - - key[name] = record[name]; } + const key = Object.fromEntries(this.rootKeys.map(name => [name, record[name]])); + const parentPositions = parentKeyColumns.slice(this.rootKeys.length).map(name => record[name]); const fields = Object.fromEntries( - Object.entries(record).filter( - ([name]) => !this.parentKeys.includes(name) && name !== INDEX_FIELD, - ), + Object.entries(record).filter(([name]) => !this.keyColumns.includes(name)), ); const builder = new ExpressionBuilder(); const keyBuilder = new ExpressionBuilder('k', 'kv'); - const arrayAlias = builder.attribute(this.arrayField); + const parentPath = this.elementPath( + builder, + parentPositions as number[], + this.chain.length - 1, + ); + const { arrayField } = this.chain[this.chain.length - 1]; + const arrayPath = [parentPath, builder.attribute(arrayField)].filter(Boolean).join('.'); const emptyList = builder.value([]); const appended = builder.value([Serializer.unflatten(fields)]); - const expression = `SET ${arrayAlias} = list_append(if_not_exists(${arrayAlias}, ${emptyList}), ${appended})`; + const conditions = [`attribute_exists(${keyBuilder.attribute(this.rootKeys[0])})`]; + + if (parentPath) conditions.push(`attribute_exists(${parentPath})`); let attributes: Item; try { - attributes = await this.parent.executor.update({ + attributes = await this.root.executor.update({ Key: key, - UpdateExpression: expression, - ConditionExpression: `attribute_exists(${keyBuilder.attribute(this.parentKeys[0])})`, + UpdateExpression: `SET ${arrayPath} = list_append(if_not_exists(${arrayPath}, ${emptyList}), ${appended})`, + ConditionExpression: conditions.join(' AND '), ExpressionAttributeNames: { ...builder.names, ...keyBuilder.names }, ExpressionAttributeValues: builder.values, ReturnValues: 'UPDATED_NEW', @@ -268,48 +326,65 @@ export default class ArrayCollection extends BaseCollection { throw error; } - const array = Serializer.toPathSegments(this.arrayField).reduce( - (value, segment) => (value as Item)?.[segment], - attributes, - ) as unknown[]; + const length = this.appendedLength(attributes); + + return { + ...key, + ...Object.fromEntries( + parentKeyColumns.slice(this.rootKeys.length).map((name, i) => [name, parentPositions[i]]), + ), + [INDEX_FIELD]: length - 1, + ...fields, + }; + } + + /** UPDATED_NEW answers with only the arrays on the path, each holding just the updated element. */ + private appendedLength(attributes: Item): number { + let value: unknown = attributes; + + this.chain.forEach((level, i) => { + for (const segment of Serializer.toPathSegments(level.arrayField)) { + value = (value as Item)?.[segment]; + } - return { ...key, [INDEX_FIELD]: array.length - 1, ...fields }; + if (i < this.chain.length - 1 && Array.isArray(value)) [value] = value; + }); + + return (value as unknown[]).length; } - /** Finds the matching items, then writes once per parent whatever the items need. */ - private async writePerParent( + /** Finds the matching elements, then writes once per table item whatever they need. */ + private async writePerTableItem( caller: Caller, filter: Filter, - build: (builder: ExpressionBuilder, indexes: number[]) => { UpdateExpression: string }, + build: (builder: ExpressionBuilder, positions: number[][]) => { UpdateExpression: string }, ): Promise { const records = await this.list( caller, new PaginatedFilterClass({ conditionTree: filter.conditionTree }), - new Projection(...this.parentKeys, INDEX_FIELD), + new Projection(...this.keyColumns), ); - const perParent = new Map(); + const perItem = new Map(); + const positionColumns = this.chain.map(level => level.indexColumn); for (const record of records) { - const key = Object.fromEntries(this.parentKeys.map(name => [name, record[name]])); + const key = Object.fromEntries(this.rootKeys.map(name => [name, record[name]])); const id = JSON.stringify(key); - const entry = perParent.get(id) ?? { key, indexes: [] }; - entry.indexes.push(record[INDEX_FIELD] as number); - perParent.set(id, entry); + const entry = perItem.get(id) ?? { key, positions: [] }; + entry.positions.push(positionColumns.map(name => record[name] as number)); + perItem.set(id, entry); } - await this.parent.executor.mapWrites([...perParent.values()], async ({ key, indexes }) => { + await this.root.executor.mapWrites([...perItem.values()], async ({ key, positions }) => { const builder = new ExpressionBuilder(); const keyBuilder = new ExpressionBuilder('k', 'kv'); - const { UpdateExpression } = build( - builder, - [...indexes].sort((a, b) => a - b), - ); + const { UpdateExpression } = build(builder, positions); try { - await this.parent.executor.update({ + await this.root.executor.update({ Key: key, UpdateExpression, - ConditionExpression: `attribute_exists(${keyBuilder.attribute(this.parentKeys[0])})`, + ConditionExpression: `attribute_exists(${keyBuilder.attribute(this.rootKeys[0])})`, ExpressionAttributeNames: { ...builder.names, ...keyBuilder.names }, ...(Object.keys(builder.values).length ? { ExpressionAttributeValues: builder.values } @@ -320,6 +395,6 @@ export default class ArrayCollection extends BaseCollection { } }); - this.parent.invalidateCaches(); + this.root.invalidateCaches(); } } diff --git a/packages/datasource-dynamodb/src/virtual-collections.ts b/packages/datasource-dynamodb/src/virtual-collections.ts index 96aeabf595..9038b39302 100644 --- a/packages/datasource-dynamodb/src/virtual-collections.ts +++ b/packages/datasource-dynamodb/src/virtual-collections.ts @@ -17,11 +17,12 @@ const SAMPLING_CALLER = { timezone: 'UTC' } as Caller; /** The attributes of an item come from the schema when it declares them, else from a sample. */ async function itemSchemaOf( - parent: DynamoDBCollection, + parent: DynamoDBCollection | ArrayCollection, arrayField: string, sampleSize: number, ): Promise> { - const declared = parent.model.schema[arrayField]?.items; + const declared = + parent instanceof ArrayCollection ? undefined : parent.model.schema[arrayField]?.items; if (declared) return declared; @@ -51,10 +52,6 @@ export default async function addVirtualCollections( try { const parent = dataSource.getCollection(config.parentCollection); - if (parent instanceof ArrayCollection) { - throw new Error('an array of an array cannot be exposed'); - } - // eslint-disable-next-line no-await-in-loop const itemSchema = await itemSchemaOf(parent, config.arrayField, sampleSize); diff --git a/packages/datasource-dynamodb/test/array-collection.test.ts b/packages/datasource-dynamodb/test/array-collection.test.ts index 686b3ca969..1247e803ba 100644 --- a/packages/datasource-dynamodb/test/array-collection.test.ts +++ b/packages/datasource-dynamodb/test/array-collection.test.ts @@ -311,3 +311,123 @@ describe('ArrayCollection', () => { }); }); }); + +describe('ArrayCollection of an array', () => { + const nested = [ + { + customerId: 'c1', + orderId: 'o1', + lines: [ + { sku: 'A', parts: [{ n: 1 }, { n: 2 }] }, + { sku: 'B', parts: [{ n: 3 }] }, + ], + }, + ]; + + function buildNested(handlers: Record) { + const parent = buildCollection( + buildModel({ + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + lines: { type: 'array' }, + }, + }), + handlers, + { scanConcurrency: 1 }, + ); + const lines = new ArrayCollection({} as DataSource, parent.collection, { + name: 'order_lines', + arrayField: 'lines', + itemSchema: { sku: { type: 'string' }, parts: { type: 'array' } }, + }); + const parts = new ArrayCollection({} as DataSource, lines, { + name: 'parts', + arrayField: 'parts', + itemSchema: { n: { type: 'number' } }, + }); + + return { parts, client: parent.client }; + } + + it('is keyed by the table key and the position in each array', () => { + const { parts } = buildNested({}); + + expect( + Object.entries(parts.schema.fields) + .filter(([, field]) => field.type === 'Column' && field.isPrimaryKey) + .map(([name]) => name), + ).toEqual(['customerId', 'orderId', 'order_linesIndex', 'index']); + }); + + it('lists the elements of every inner array', async () => { + const { parts } = buildNested({ Scan: () => ({ Items: nested }) }); + + const records = await parts.list( + caller, + new PaginatedFilter({}), + new Projection('order_linesIndex', 'index', 'n'), + ); + + expect(records).toEqual([ + { order_linesIndex: 0, index: 0, n: 1 }, + { order_linesIndex: 0, index: 1, n: 2 }, + { order_linesIndex: 1, index: 0, n: 3 }, + ]); + }); + + it('sets an attribute of an inner element through the path of both arrays', async () => { + const { parts, client } = buildNested({ + Scan: () => ({ Items: nested }), + Update: () => ({}), + }); + + await parts.update(caller, new Filter({ conditionTree: leaf('n', 'Equal', 3) }), { n: 9 }); + + expect(client.calls('Update')[0]).toMatchObject({ + Key: { customerId: 'c1', orderId: 'o1' }, + UpdateExpression: 'SET #n0[1].#n1[0].#n2 = :v0', + ExpressionAttributeNames: { '#n0': 'lines', '#n1': 'parts', '#n2': 'n' }, + ExpressionAttributeValues: { ':v0': 9 }, + }); + }); + + it('removes inner elements, highest positions first', async () => { + const { parts, client } = buildNested({ + Scan: () => ({ Items: nested }), + Update: () => ({}), + }); + + await parts.delete(caller, new Filter({})); + + expect(client.calls('Update')[0].UpdateExpression).toBe( + 'REMOVE #n0[1].#n1[0], #n0[0].#n1[1], #n0[0].#n1[0]', + ); + }); + + it('appends to the inner array of an existing outer element', async () => { + const { parts, client } = buildNested({ + Update: () => ({ Attributes: { lines: [{ parts: [{ n: 3 }, { n: 4 }] }] } }), + }); + + const created = await parts.create(caller, [ + { customerId: 'c1', orderId: 'o1', order_linesIndex: 1, n: 4 }, + ]); + + expect(created).toEqual([ + { customerId: 'c1', orderId: 'o1', order_linesIndex: 1, index: 1, n: 4 }, + ]); + expect(client.calls('Update')[0]).toMatchObject({ + UpdateExpression: 'SET #n0[1].#n1 = list_append(if_not_exists(#n0[1].#n1, :v0), :v1)', + ConditionExpression: 'attribute_exists(#k0) AND attribute_exists(#n0[1])', + }); + }); + + it('requires the position in the outer array to create', async () => { + const { parts } = buildNested({}); + + await expect(parts.create(caller, [{ customerId: 'c1', orderId: 'o1', n: 4 }])).rejects.toThrow( + "The key attribute 'order_linesIndex' of the parent 'order_lines' is required to create a record in 'parts'", + ); + }); +}); diff --git a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts index 601f71407a..3d95e8390d 100644 --- a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts +++ b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts @@ -90,11 +90,12 @@ run('DynamoDB Local', () => { createdAt: `2024-0${(i % 3) + 1}-15T10:00:00Z`, tags: new Set(['a']), ...(i === 0 ? { address: { city: 'Paris' } } : {}), + ...(i === 2 ? { meta: { items: [{ x: 1 }, { x: 2 }] } } : {}), ...(i === 1 ? { lines: [ - { sku: 'A', qty: 1 }, - { sku: 'B', qty: 2 }, + { sku: 'A', qty: 1, parts: [{ n: 1 }, { n: 2 }] }, + { sku: 'B', qty: 2, parts: [{ n: 3 }] }, ], } : {}), @@ -141,6 +142,8 @@ run('DynamoDB Local', () => { }), virtualCollections: [ { parentCollection: 'orders', collectionName: 'order_lines', arrayField: 'lines' }, + { parentCollection: 'order_lines', collectionName: 'line_parts', arrayField: 'parts' }, + { parentCollection: 'orders', collectionName: 'meta_items', arrayField: 'meta->items' }, ], liveQueryConnection: 'dynamo', chartCacheTtlMs: 0, @@ -448,6 +451,79 @@ run('DynamoDB Local', () => { }); describe('virtual array collection', () => { + it('lists, updates, appends to and deletes the items of an array inside an array', async () => { + const parts = dataSource.getCollection('line_parts'); + const order = tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c1' }, + { field: 'orderId', operator: 'Equal', value: 'o01' }, + ], + }); + const read = () => + parts.list( + caller, + new PaginatedFilter({ conditionTree: order }), + new Projection('order_linesIndex', 'index', 'n'), + ); + + expect(await read()).toEqual([ + { order_linesIndex: 0, index: 0, n: 1 }, + { order_linesIndex: 0, index: 1, n: 2 }, + { order_linesIndex: 1, index: 0, n: 3 }, + ]); + + await parts.update(caller, new Filter({ conditionTree: leaf('n', 'Equal', 3) }), { n: 9 }); + const created = await parts.create(caller, [ + { customerId: 'c1', orderId: 'o01', order_linesIndex: 1, n: 4 }, + ]); + await parts.delete( + caller, + new Filter({ + conditionTree: ConditionTreeFactory.intersect(order, leaf('n', 'Equal', 1)), + }), + ); + + expect(created).toEqual([ + { customerId: 'c1', orderId: 'o01', order_linesIndex: 1, index: 1, n: 4 }, + ]); + expect(await read()).toEqual([ + { order_linesIndex: 0, index: 0, n: 2 }, + { order_linesIndex: 1, index: 0, n: 9 }, + { order_linesIndex: 1, index: 1, n: 4 }, + ]); + }); + + it('exposes an array held in a nested map', async () => { + const items = dataSource.getCollection('meta_items'); + const order = tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c1' }, + { field: 'orderId', operator: 'Equal', value: 'o02' }, + ], + }); + const read = () => + items.list( + caller, + new PaginatedFilter({ conditionTree: order }), + new Projection('index', 'x'), + ); + + await items.update(caller, new Filter({ conditionTree: leaf('x', 'Equal', 2) }), { x: 7 }); + const [created] = await items.create(caller, [{ customerId: 'c1', orderId: 'o02', x: 3 }]); + await items.delete( + caller, + new Filter({ conditionTree: ConditionTreeFactory.intersect(order, leaf('x', 'Equal', 1)) }), + ); + + expect(created.index).toBe(2); + expect(await read()).toEqual([ + { index: 0, x: 7 }, + { index: 1, x: 3 }, + ]); + }); + it('lists, updates and deletes the items of an array', async () => { const lines = dataSource.getCollection('order_lines'); const parent = ConditionTreeFactory.fromPlainObject({ diff --git a/packages/datasource-dynamodb/test/virtual-collections.test.ts b/packages/datasource-dynamodb/test/virtual-collections.test.ts index ce172ae41f..839cd99891 100644 --- a/packages/datasource-dynamodb/test/virtual-collections.test.ts +++ b/packages/datasource-dynamodb/test/virtual-collections.test.ts @@ -82,12 +82,31 @@ describe('virtual collections', () => { ); }); - it('refuses to expose an array of a virtual collection', async () => { + it('exposes an array of an array, identified by the position in each', async () => { + const { dataSource } = build( + [{ id: 'o1', lines: [{ sku: 'A', parts: [{ n: 1 }, { n: 2 }] }] }], + [ + { parentCollection: 'Orders', collectionName: 'lines', arrayField: 'lines' }, + { parentCollection: 'lines', collectionName: 'parts', arrayField: 'parts' }, + ], + ); + + const { fields } = (await dataSource).getCollection('parts').schema; + + expect( + Object.entries(fields) + .filter(([, field]) => field.type === 'Column' && field.isPrimaryKey) + .map(([name]) => name), + ).toEqual(['id', 'linesIndex', 'index']); + expect(fields.n).toMatchObject({ columnType: 'Number', isPrimaryKey: false }); + }); + + it('skips an array of an array declared before its parent, and says so', async () => { const { logger, dataSource } = build( [{ id: 'o1', lines: [{ sku: 'A', parts: [{ n: 1 }] }] }], [ - { parentCollection: 'Orders', collectionName: 'lines', arrayField: 'lines' }, { parentCollection: 'lines', collectionName: 'parts', arrayField: 'parts' }, + { parentCollection: 'Orders', collectionName: 'lines', arrayField: 'lines' }, ], ); @@ -96,7 +115,7 @@ describe('virtual collections', () => { expect(result.collections.map(c => c.name).sort()).toEqual(['Orders', 'lines']); expect(logger).toHaveBeenCalledWith( 'Warn', - "Skipping virtual collection 'parts' for array 'lines.parts': an array of an array cannot be exposed", + expect.stringContaining("Skipping virtual collection 'parts' for array 'lines.parts'"), ); }); }); From ad564d39e9cf9c917cd12c86b370921faa799860 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Fri, 16 Oct 2026 19:48:13 +0200 Subject: [PATCH 05/32] style(datasource-dynamodb): add the blank line the linter asks for Refs: PRD-1670 Co-Authored-By: Claude Sonnet 5.5 --- packages/datasource-dynamodb/test/collection.aggregate.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/datasource-dynamodb/test/collection.aggregate.test.ts b/packages/datasource-dynamodb/test/collection.aggregate.test.ts index 9473647906..720f762a0c 100644 --- a/packages/datasource-dynamodb/test/collection.aggregate.test.ts +++ b/packages/datasource-dynamodb/test/collection.aggregate.test.ts @@ -418,6 +418,7 @@ describe('DynamoDBCollection.aggregate', () => { // eslint-disable-next-line no-await-in-loop await run(age); } + client.send.mockClear(); await run(100); From 657186579e3112015565b3b394e7f6228012e9e9 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Fri, 16 Oct 2026 20:00:16 +0200 Subject: [PATCH 06/32] fix(datasource-dynamodb): address the review findings - an index that leaves out items without its sort key is only read with a condition on that key or when declared to cover every item - key lookups drop nested OR arms and duplicate keys, which BatchGetItem refuses - writes: caches are dropped after a partial failure, in-flight writes finish before the error is raised, a record cannot claim another entity, array writes stay within DynamoDB's expression size and the entity of their parent, and an update under a missing map falls back to rewriting - pagination positions are kept per timezone, Present is evaluated like the toolkit does, date-times sort and aggregate by instant, and a large client-side sort no longer overflows - native queries leave placeholders inside literals alone and bound the pages they follow - introspection accepts attributes named like Object.prototype members - nested array collections use the item schema their parent declares Refs: PRD-1670 Co-Authored-By: Claude Sonnet 5.5 --- .../src/array-collection.ts | 236 ++++++++++++++---- .../datasource-dynamodb/src/collection.ts | 36 ++- .../datasource-dynamodb/src/datasource.ts | 26 +- .../src/introspection/introspector.ts | 7 +- .../src/schema/manual-schema.ts | 6 +- packages/datasource-dynamodb/src/types.ts | 7 + .../src/utils/access-path.ts | 46 +++- .../src/utils/aggregator.ts | 11 +- .../datasource-dynamodb/src/utils/compare.ts | 19 ++ .../src/utils/concurrency.ts | 28 +++ .../datasource-dynamodb/src/utils/executor.ts | 19 +- .../src/utils/expression-builder.ts | 3 - .../datasource-dynamodb/src/utils/partiql.ts | 55 ++++ .../datasource-dynamodb/src/utils/sort.ts | 8 +- .../src/virtual-collections.ts | 4 +- .../test/array-review-findings.test.ts | 194 ++++++++++++++ .../test/integration/dynamodb-local.test.ts | 40 ++- .../test/review-findings.test.ts | 212 ++++++++++++++++ .../test/schema/manual-schema.test.ts | 1 + .../test/utils/access-path.test.ts | 59 +++++ .../test/utils/compare.test.ts | 50 ++++ .../test/utils/concurrency.test.ts | 52 ++++ .../test/utils/expression-builder.test.ts | 15 +- .../test/utils/partiql.test.ts | 46 ++++ .../test/virtual-collections.test.ts | 57 ++++- 25 files changed, 1125 insertions(+), 112 deletions(-) create mode 100644 packages/datasource-dynamodb/src/utils/compare.ts create mode 100644 packages/datasource-dynamodb/src/utils/partiql.ts create mode 100644 packages/datasource-dynamodb/test/array-review-findings.test.ts create mode 100644 packages/datasource-dynamodb/test/review-findings.test.ts create mode 100644 packages/datasource-dynamodb/test/utils/compare.test.ts create mode 100644 packages/datasource-dynamodb/test/utils/concurrency.test.ts create mode 100644 packages/datasource-dynamodb/test/utils/partiql.test.ts diff --git a/packages/datasource-dynamodb/src/array-collection.ts b/packages/datasource-dynamodb/src/array-collection.ts index e37dd07c29..5cb48d31a1 100644 --- a/packages/datasource-dynamodb/src/array-collection.ts +++ b/packages/datasource-dynamodb/src/array-collection.ts @@ -29,6 +29,9 @@ import compareRecords from './utils/sort'; export const INDEX_FIELD = 'index'; +/** Expressions are limited to 4 KB by DynamoDB, names and values apart. */ +const MAX_EXPRESSION_LENGTH = 3000; + /** One array on the way from the item of the table to the records of a collection. */ type Level = { arrayField: string; indexColumn: string }; @@ -65,6 +68,9 @@ export default class ArrayCollection extends BaseCollection { private readonly rootKeys: string[]; private readonly itemFields: Set; + /** Attributes of an element, by flattened name, as declared or sampled. */ + readonly itemSchema: Record; + /** The columns identifying a record: the table keys, then one position per array. */ readonly keyColumns: string[]; @@ -115,20 +121,23 @@ export default class ArrayCollection extends BaseCollection { this.addFields(fields); this.enableCount(); + this.itemSchema = config.itemSchema; this.itemFields = new Set(Object.keys(config.itemSchema)); } async create(caller: Caller, data: RecordData[]): Promise { const created: RecordData[] = []; - // Appended one after the other: each append reads the length the previous one left. - for (const record of data) { - // eslint-disable-next-line no-await-in-loop - created.push(await this.append(record)); + try { + // Appended one after the other: each append reads the length the previous one left. + for (const record of data) { + // eslint-disable-next-line no-await-in-loop + created.push(await this.append(record)); + } + } finally { + this.root.invalidateCaches(); } - this.root.invalidateCaches(); - return created; } @@ -164,37 +173,28 @@ export default class ArrayCollection extends BaseCollection { if (entries.length === 0) return; - await this.writePerTableItem(caller, filter, (builder, positions) => { - const values = entries.map(([, value]) => builder.value(value)); - - return { - UpdateExpression: `SET ${positions - .flatMap(position => - entries.map( - ([field], index) => - `${this.elementPath(builder, position)}.${builder.attribute(field)} = ${ - values[index] - }`, - ), - ) - .join(', ')}`, - }; + await this.writePerTableItem(caller, filter, { + keyword: 'SET', + clauses: (builder, position, state) => { + state.values ??= entries.map(([, value]) => builder.value(value)); + + return entries.map( + ([field], index) => + `${this.elementPath(builder, position)}.${builder.attribute(field)} = ${ + state.values[index] + }`, + ); + }, + fallback: (key, positions) => this.replaceElements(key, positions, entries), }); } async delete(caller: Caller, filter: Filter): Promise { - await this.writePerTableItem(caller, filter, (builder, positions) => { - const descending = [...positions].sort((a, b) => { - const different = a.findIndex((index, level) => index !== b[level]); - - return different === -1 ? 0 : b[different] - a[different]; - }); - - return { - UpdateExpression: `REMOVE ${descending - .map(position => this.elementPath(builder, position)) - .join(', ')}`, - }; + await this.writePerTableItem(caller, filter, { + keyword: 'REMOVE', + // Highest positions first, so that removing one does not shift those still to remove. + descending: true, + clauses: (builder, position) => [this.elementPath(builder, position)], }); } @@ -259,11 +259,12 @@ export default class ArrayCollection extends BaseCollection { elements.forEach((element, index) => { if (element && typeof element === 'object') { records.push({ + // The identity comes last: an element's own `index` must not take its place. + ...Serializer.flatten(element as Item, this.itemFields), ...Object.fromEntries( ownKeyColumns.map((name, i) => [name, parent[parentColumns[i]]]), ), [INDEX_FIELD]: index, - ...Serializer.flatten(element as Item, this.itemFields), }); } }); @@ -305,6 +306,10 @@ export default class ArrayCollection extends BaseCollection { if (parentPath) conditions.push(`attribute_exists(${parentPath})`); + const entity = this.entityCondition(builder); + + if (entity) conditions.push(entity); + let attributes: Item; try { @@ -353,11 +358,115 @@ export default class ArrayCollection extends BaseCollection { return (value as unknown[]).length; } + /** Restricts a write to the items of the entity the parent collection stands for. */ + private entityCondition(builder: ExpressionBuilder): string | undefined { + const { entity, sortKey } = this.root.model; + + if (entity?.sortKeyPrefix !== undefined && sortKey) { + return `begins_with(${builder.attribute(sortKey.name)}, ${builder.value( + entity.sortKeyPrefix, + )})`; + } + + if (entity?.attribute !== undefined) { + return `${builder.attribute(entity.attribute)} = ${builder.value(entity.value)}`; + } + + return undefined; + } + + private async writeExpression( + key: Item, + expression: string, + builder: ExpressionBuilder, + ): Promise { + const keyBuilder = new ExpressionBuilder('k', 'kv'); + const conditions = [`attribute_exists(${keyBuilder.attribute(this.rootKeys[0])})`]; + const entity = this.entityCondition(builder); + + if (entity) conditions.push(entity); + + try { + await this.root.executor.update({ + Key: key, + UpdateExpression: expression, + ConditionExpression: conditions.join(' AND '), + ExpressionAttributeNames: { ...builder.names, ...keyBuilder.names }, + ...(Object.keys(builder.values).length + ? { ExpressionAttributeValues: builder.values } + : {}), + }); + } catch (error) { + if (error.name !== 'ConditionalCheckFailedException') throw error; + } + } + + /** Sets attributes of elements one by one in the item itself, for when DynamoDB cannot. */ + private async replaceElements( + key: Item, + positions: number[][], + entries: Array<[string, unknown]>, + ): Promise { + const item = await this.root.executor.get(key); + + if (!item) return; + + const patch = Serializer.unflatten(Object.fromEntries(entries)); + + for (const position of positions) { + let container: Item = item; + + this.chain.forEach((level, depth) => { + const segments = Serializer.toPathSegments(level.arrayField); + const holder = segments + .slice(0, -1) + .reduce((value, segment) => value[segment] as Item, container); + const array = holder[segments[segments.length - 1]] as Item[]; + + if (depth === this.chain.length - 1) { + array[position[depth]] = Serializer.deepMerge(array[position[depth]], patch); + } else { + container = array[position[depth]]; + } + }); + } + + const keyBuilder = new ExpressionBuilder('k', 'kv'); + const builder = new ExpressionBuilder(); + const conditions = [`attribute_exists(${keyBuilder.attribute(this.rootKeys[0])})`]; + const entity = this.entityCondition(builder); + + if (entity) conditions.push(entity); + + try { + await this.root.executor.put({ + Item: item, + ConditionExpression: conditions.join(' AND '), + ExpressionAttributeNames: { ...builder.names, ...keyBuilder.names }, + ...(Object.keys(builder.values).length + ? { ExpressionAttributeValues: builder.values } + : {}), + }); + } catch (error) { + if (error.name !== 'ConditionalCheckFailedException') throw error; + } + } + /** Finds the matching elements, then writes once per table item whatever they need. */ private async writePerTableItem( caller: Caller, filter: Filter, - build: (builder: ExpressionBuilder, positions: number[][]) => { UpdateExpression: string }, + write: { + keyword: 'SET' | 'REMOVE'; + descending?: boolean; + clauses: ( + builder: ExpressionBuilder, + position: number[], + state: { values?: string[] }, + ) => string[]; + /** What to do when DynamoDB refuses the path, for instance under a map that is missing. */ + fallback?: (key: Item, positions: number[][]) => Promise; + }, ): Promise { const records = await this.list( caller, @@ -375,26 +484,45 @@ export default class ArrayCollection extends BaseCollection { perItem.set(id, entry); } - await this.root.executor.mapWrites([...perItem.values()], async ({ key, positions }) => { - const builder = new ExpressionBuilder(); - const keyBuilder = new ExpressionBuilder('k', 'kv'); - const { UpdateExpression } = build(builder, positions); - - try { - await this.root.executor.update({ - Key: key, - UpdateExpression, - ConditionExpression: `attribute_exists(${keyBuilder.attribute(this.rootKeys[0])})`, - ExpressionAttributeNames: { ...builder.names, ...keyBuilder.names }, - ...(Object.keys(builder.values).length - ? { ExpressionAttributeValues: builder.values } - : {}), + try { + await this.root.executor.mapWrites([...perItem.values()], async ({ key, positions }) => { + const ordered = [...positions].sort((a, b) => { + const different = a.findIndex((index, depth) => index !== b[depth]); + + return different === -1 ? 0 : (a[different] - b[different]) * (write.descending ? -1 : 1); }); - } catch (error) { - if (error.name !== 'ConditionalCheckFailedException') throw error; - } - }); + let builder = new ExpressionBuilder(); + let state: { values?: string[] } = {}; + let clauses: string[] = []; - this.root.invalidateCaches(); + const flush = async () => { + if (clauses.length) { + await this.writeExpression(key, `${write.keyword} ${clauses.join(', ')}`, builder); + } + + builder = new ExpressionBuilder(); + state = {}; + clauses = []; + }; + + try { + for (const position of ordered) { + clauses.push(...write.clauses(builder, position, state)); + + // An expression is limited to 4 KB: past a safe size, what is left goes in the next. + // eslint-disable-next-line no-await-in-loop + if (clauses.join(', ').length > MAX_EXPRESSION_LENGTH) await flush(); + } + + await flush(); + } catch (error) { + if (!write.fallback || !/document path/i.test(error.message)) throw error; + + await write.fallback(key, ordered); + } + }); + } finally { + this.root.invalidateCaches(); + } } } diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index 41ffddb3f3..ab646708d2 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -144,12 +144,12 @@ export default class DynamoDBCollection extends BaseCollection { async create(caller: Caller, data: RecordData[]): Promise { try { - const created = await this.executor.mapWrites(data, record => this.createOne(record)); - this.invalidateCaches(); - - return created; + return await this.executor.mapWrites(data, record => this.createOne(record)); } catch (error) { throw this.wrap(error, 'create'); + } finally { + // Also after a failure: the writes that went through must not stay hidden by a cache. + this.invalidateCaches(); } } @@ -175,15 +175,23 @@ export default class DynamoDBCollection extends BaseCollection { `The key attribute '${field}' of '${this.name}' cannot be updated`, ); } + + if (field === this.model.entity?.attribute) { + throw new ValidationError( + `The attribute '${field}' tells which entity a record of '${this.name}' belongs to, ` + + `so it cannot be updated`, + ); + } } if (entries.length === 0) return; const keys = await this.resolveKeys(caller, filter); await this.executor.mapWrites(keys, key => this.updateOne(key, entries)); - this.invalidateCaches(); } catch (error) { throw this.wrap(error, 'update'); + } finally { + this.invalidateCaches(); } } @@ -194,9 +202,10 @@ export default class DynamoDBCollection extends BaseCollection { if (keys.length === 0) return; await this.executor.batchDelete(keys); - this.invalidateCaches(); } catch (error) { throw this.wrap(error, 'delete'); + } finally { + this.invalidateCaches(); } } @@ -368,6 +377,7 @@ export default class DynamoDBCollection extends BaseCollection { const hash = cache.hash({ base, residual: residual?.toPlainObject(), + timezone, table: this.model.tableName, }); const cached = limit === undefined ? undefined : cache.findBestToken(hash, skip); @@ -446,12 +456,12 @@ export default class DynamoDBCollection extends BaseCollection { return (await this.readPage(plan, fields, skip, limit, caller.timezone)).map(toResult); } - const items = await this.readAll(plan, fields, caller.timezone); + let items = await this.readAll(plan, fields, caller.timezone); if (sort && !plan.access?.sortSatisfied) { const rows = items.map(item => ({ item, record: this.toRecord(item) })); rows.sort((a, b) => compareRecords(a.record, b.record, sort)); - items.splice(0, items.length, ...rows.map(row => row.item)); + items = rows.map(row => row.item); } return items.slice(skip, limit === undefined ? undefined : skip + limit).map(toResult); @@ -504,7 +514,15 @@ export default class DynamoDBCollection extends BaseCollection { } const item = Serializer.unflatten(created); - if (entity?.attribute !== undefined) item[entity.attribute] ??= entity.value; + if (entity?.attribute !== undefined) { + if (item[entity.attribute] !== undefined && item[entity.attribute] !== entity.value) { + throw new ValidationError( + `The attribute '${entity.attribute}' of '${this.name}' must be '${entity.value}'`, + ); + } + + item[entity.attribute] = entity.value; + } const keyBuilder = new ExpressionBuilder('k', 'kv'); diff --git a/packages/datasource-dynamodb/src/datasource.ts b/packages/datasource-dynamodb/src/datasource.ts index f4bc9a5259..50b2822546 100644 --- a/packages/datasource-dynamodb/src/datasource.ts +++ b/packages/datasource-dynamodb/src/datasource.ts @@ -8,8 +8,13 @@ import { BaseDataSource, ValidationError } from '@forestadmin/datasource-toolkit import DynamoDBCollection from './collection'; import { ScanLimitExceededError } from './errors'; +import parameterize from './utils/partiql'; -const PLACEHOLDER = /\$(\w+)/g; +/** + * PartiQL does not say how many items a statement evaluated, only how many it returned, so the + * read budget bounds the rows returned and the pages (up to 1 MB each) fetched. + */ +const MAX_NATIVE_QUERY_PAGES = 1000; const READ_ONLY_STATEMENT = /^\s*select\b/i; type NativeQueryConnection = { client: DynamoDBDocumentClient }; @@ -54,19 +59,11 @@ export default class DynamoDBDataSource extends BaseDataSource< throw new ValidationError('Only SELECT statements can be run as native queries'); } - const parameters: unknown[] = []; - const statement = query.replace(PLACEHOLDER, (_match, name: string) => { - if (!(name in contextVariables)) { - throw new ValidationError(`No value was given for the placeholder $${name}`); - } - - parameters.push(contextVariables[name]); - - return '?'; - }); + const { statement, parameters } = parameterize(query, contextVariables); const rows: Record[] = []; let nextToken: string; + let pages = 0; do { // eslint-disable-next-line no-await-in-loop @@ -80,6 +77,13 @@ export default class DynamoDBDataSource extends BaseDataSource< rows.push(...((response.Items ?? []) as Record[])); nextToken = response.NextToken; + pages += 1; + + if (nextToken && pages >= MAX_NATIVE_QUERY_PAGES) { + throw new ScanLimitExceededError( + `The native query still has results after ${pages} pages. Narrow it with a WHERE clause.`, + ); + } if (rows.length > this.settings.maxScannedItems) { throw new ScanLimitExceededError( diff --git a/packages/datasource-dynamodb/src/introspection/introspector.ts b/packages/datasource-dynamodb/src/introspection/introspector.ts index 35d7738882..9ee6e4aaa6 100644 --- a/packages/datasource-dynamodb/src/introspection/introspector.ts +++ b/packages/datasource-dynamodb/src/introspection/introspector.ts @@ -137,8 +137,9 @@ export function inferSchema( keys: KeyAttribute[], maxDepth: number, ): Record { - const types: Record = {}; - const presence: Record = {}; + // Attribute names are data: `constructor` or `__proto__` must not reach Object.prototype. + const types: Record = Object.create(null); + const presence: Record = Object.create(null); for (const item of items) { const present = new Set(); @@ -148,7 +149,7 @@ export function inferSchema( }); } - const schema: Record = {}; + const schema: Record = Object.create(null); for (const [path, observed] of Object.entries(types)) { schema[path] = { diff --git a/packages/datasource-dynamodb/src/schema/manual-schema.ts b/packages/datasource-dynamodb/src/schema/manual-schema.ts index eadc11d887..23459542c5 100644 --- a/packages/datasource-dynamodb/src/schema/manual-schema.ts +++ b/packages/datasource-dynamodb/src/schema/manual-schema.ts @@ -112,7 +112,10 @@ function validateCollection(collection: CollectionDefinition): void { if (entity) { const hasDiscriminator = entity.attribute !== undefined || entity.value !== undefined; - if (hasDiscriminator === (entity.sortKeyPrefix !== undefined)) { + const incompleteDiscriminator = + hasDiscriminator && (entity.attribute === undefined || entity.value === undefined); + + if (hasDiscriminator === (entity.sortKeyPrefix !== undefined) || incompleteDiscriminator) { throw new ValidationError( `Collection '${name}' entity must use either 'attribute'/'value' or 'sortKeyPrefix'`, ); @@ -198,6 +201,7 @@ function toModel(collection: CollectionDefinition): TableModel { partitionKey: toKey(index.partitionKey), sortKey: index.sortKey ? toKey(index.sortKey) : undefined, projectsAllAttributes: index.projectsAllAttributes ?? true, + coversAllItems: index.coversAllItems ?? false, })); return { diff --git a/packages/datasource-dynamodb/src/types.ts b/packages/datasource-dynamodb/src/types.ts index a609ecbfd9..929e8e6aa9 100644 --- a/packages/datasource-dynamodb/src/types.ts +++ b/packages/datasource-dynamodb/src/types.ts @@ -36,6 +36,11 @@ export type IndexModel = { partitionKey: KeyAttribute; sortKey?: KeyAttribute; projectsAllAttributes: boolean; + /** + * Whether every item of the table is in the index. An index leaves out the items that lack its + * sort key, so it is only read without a condition on that key when this is true. + */ + coversAllItems?: boolean; }; /** @@ -83,6 +88,8 @@ export type IndexDefinition = { sortKey?: { name: string; type: 'string' | 'number' }; /** Whether the index projects every attribute, which a query on it needs. Default: true. */ projectsAllAttributes?: boolean; + /** Every item has the index keys, so the index may be read without a sort key condition. */ + coversAllItems?: boolean; }; export type CollectionDefinition = { diff --git a/packages/datasource-dynamodb/src/utils/access-path.ts b/packages/datasource-dynamodb/src/utils/access-path.ts index f0eff21806..039b832b87 100644 --- a/packages/datasource-dynamodb/src/utils/access-path.ts +++ b/packages/datasource-dynamodb/src/utils/access-path.ts @@ -37,7 +37,13 @@ export type AccessPath = { export type KeyLookup = { keys: Array>; remaining?: ConditionTree }; -type Candidate = { indexName?: string; partitionKey: KeyAttribute; sortKey?: KeyAttribute }; +type Candidate = { + indexName?: string; + partitionKey: KeyAttribute; + sortKey?: KeyAttribute; + /** Whether an item missing the sort key can still be in the index: it cannot, unless told so. */ + coversAllItems: boolean; +}; const RANGE_OPERATORS = ['LessThan', 'LessThanOrEqual', 'GreaterThan', 'GreaterThanOrEqual']; @@ -55,6 +61,20 @@ function coerceKeyValue(attribute: KeyAttribute, value: T): T | string | numb return value; } +/** BatchGetItem refuses the same key twice. */ +function uniqueKeys(keys: Array>): Array> { + const seen = new Set(); + + return keys.filter(key => { + const id = JSON.stringify(key); + + if (seen.has(id)) return false; + seen.add(id); + + return true; + }); +} + /** The restriction a single-table entity puts on the items of its table. */ function withEntity(model: TableModel, tree?: ConditionTree): ConditionTree | undefined { const { entity } = model; @@ -159,13 +179,14 @@ export function planAccessPath( const leaves = leavesOf(parts); const candidates: Candidate[] = [ - { partitionKey: model.partitionKey, sortKey: model.sortKey }, + { partitionKey: model.partitionKey, sortKey: model.sortKey, coversAllItems: true }, ...model.indexes .filter(index => index.projectsAllAttributes) .map(index => ({ indexName: index.name, partitionKey: index.partitionKey, sortKey: index.sortKey, + coversAllItems: !!index.coversAllItems, })), ]; @@ -181,6 +202,13 @@ export function planAccessPath( if (partition) { const picked = candidate.sortKey ? pickSortCondition(candidate.sortKey, parts) : undefined; + + // An index leaves out the items that lack its sort key: without a condition on that key, it + // could return fewer items than the table holds. + if (candidate.indexName && candidate.sortKey && !picked && !candidate.coversAllItems) { + continue; + } + const sortClause = sort?.length === 1 ? sort[0] : undefined; const sortSatisfied = !!sortClause && sortClause.field === candidate.sortKey?.name; const consumed = new Set([partition, ...(picked?.consumed ?? [])]); @@ -225,7 +253,12 @@ function keysFromOr(model: TableModel, part: ConditionTree): KeyLookup['keys'] | const keys: KeyLookup['keys'] = []; for (const condition of part.conditions) { - const equalities = leavesOf(conjuncts(condition)); + const armParts = conjuncts(condition); + + // An arm with a nested branch carries a filter the keys alone would drop. + if (armParts.some(armPart => !(armPart instanceof ConditionTreeLeaf))) return null; + + const equalities = leavesOf(armParts); const key: Record = {}; for (const attribute of [model.partitionKey, model.sortKey].filter(Boolean)) { @@ -242,7 +275,7 @@ function keysFromOr(model: TableModel, part: ConditionTree): KeyLookup['keys'] | keys.push(key); } - return keys; + return uniqueKeys(keys); } /** @@ -287,7 +320,10 @@ export function planKeyLookup(model: TableModel, conditionTree?: ConditionTree): ); const consumed = new Set(resolved.map(({ found }) => found.leaf)); - return { keys, remaining: rebuild(parts.filter(part => !consumed.has(part))) }; + return { + keys: uniqueKeys(keys), + remaining: rebuild(parts.filter(part => !consumed.has(part))), + }; } for (const part of parts) { diff --git a/packages/datasource-dynamodb/src/utils/aggregator.ts b/packages/datasource-dynamodb/src/utils/aggregator.ts index d7aaeb0a2b..126b67e245 100644 --- a/packages/datasource-dynamodb/src/utils/aggregator.ts +++ b/packages/datasource-dynamodb/src/utils/aggregator.ts @@ -5,6 +5,8 @@ import type { RecordData, } from '@forestadmin/datasource-toolkit'; +import compareValues from './compare'; + /** Numbers below this are epoch seconds: in milliseconds they would be before 1973. */ const EPOCH_SECONDS_CEILING = 1e11; const MONTHS_PER_QUARTER = 3; @@ -123,8 +125,13 @@ export default class StreamingAggregator { if (value === undefined || value === null) return; accumulator.values += 1; - if (accumulator.min === undefined || value < accumulator.min) accumulator.min = value; - if (accumulator.max === undefined || value > accumulator.max) accumulator.max = value; + if (accumulator.min === undefined || compareValues(value, accumulator.min) < 0) { + accumulator.min = value; + } + + if (accumulator.max === undefined || compareValues(value, accumulator.max) > 0) { + accumulator.max = value; + } if (typeof value === 'number' && !Number.isNaN(value)) { accumulator.numbers += 1; diff --git a/packages/datasource-dynamodb/src/utils/compare.ts b/packages/datasource-dynamodb/src/utils/compare.ts new file mode 100644 index 0000000000..e28645fcb4 --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/compare.ts @@ -0,0 +1,19 @@ +const ISO_DATE_TIME = /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:?\d{2})?$/; + +/** Orders two values, date-times by the instant they stand for rather than by their text. */ +export default function compareValues(left: unknown, right: unknown): number { + if ( + typeof left === 'string' && + typeof right === 'string' && + ISO_DATE_TIME.test(left) && + ISO_DATE_TIME.test(right) + ) { + const difference = Date.parse(left) - Date.parse(right); + + if (!Number.isNaN(difference)) return Math.sign(difference); + } + + if (left === right) return 0; + + return (left as number) < (right as number) ? -1 : 1; +} diff --git a/packages/datasource-dynamodb/src/utils/concurrency.ts b/packages/datasource-dynamodb/src/utils/concurrency.ts index 93eda46fd9..e049912c58 100644 --- a/packages/datasource-dynamodb/src/utils/concurrency.ts +++ b/packages/datasource-dynamodb/src/utils/concurrency.ts @@ -21,6 +21,34 @@ export async function mapWithConcurrency( return results; } +/** + * Like mapWithConcurrency, for writes: once one fails nothing new is started, and the call only + * rejects after the writes already in flight have finished, so what follows sees their effect. + */ +export async function mapUntilSettled( + items: T[], + limit: number, + handler: (item: T, index: number) => Promise, +): Promise { + let failure: { error: unknown } | undefined; + + const results = await mapWithConcurrency(items, limit, async (item, index) => { + if (failure) return undefined; + + try { + return await handler(item, index); + } catch (error) { + failure ??= { error }; + + return undefined; + } + }); + + if (failure) throw failure.error; + + return results; +} + export function chunk(items: T[], size: number): T[][] { const chunks: T[][] = []; diff --git a/packages/datasource-dynamodb/src/utils/executor.ts b/packages/datasource-dynamodb/src/utils/executor.ts index d78317537d..fcf729a1e4 100644 --- a/packages/datasource-dynamodb/src/utils/executor.ts +++ b/packages/datasource-dynamodb/src/utils/executor.ts @@ -16,7 +16,7 @@ import { } from '@aws-sdk/lib-dynamodb'; import { DynamoDBOperationError, ScanLimitExceededError } from '../errors'; -import { chunk, mapWithConcurrency, sleep } from './concurrency'; +import { chunk, mapUntilSettled, sleep } from './concurrency'; export type Item = Record; @@ -132,8 +132,9 @@ export default class DynamoExecutor { projection: Pick = {}, ): Promise { const items: Item[] = []; + const distinct = [...new Map(keys.map(key => [JSON.stringify(key), key])).values()]; - for (const keysChunk of chunk(keys, BATCH_GET_SIZE)) { + for (const keysChunk of chunk(distinct, BATCH_GET_SIZE)) { let pending: Item[] = keysChunk; for (let attempt = 0; pending.length > 0; attempt += 1) { @@ -150,6 +151,7 @@ export default class DynamoExecutor { }), ); + this.logBatchCapacity('batchGet', response.ConsumedCapacity); items.push(...((response.Responses?.[this.model.tableName] ?? []) as Item[])); pending = (response.UnprocessedKeys?.[this.model.tableName]?.Keys ?? []) as Item[]; } @@ -159,7 +161,9 @@ export default class DynamoExecutor { } async batchDelete(keys: Item[]): Promise { - await mapWithConcurrency(chunk(keys, BATCH_WRITE_SIZE), WRITE_CONCURRENCY, async keysChunk => { + const distinct = [...new Map(keys.map(key => [JSON.stringify(key), key])).values()]; + + await mapUntilSettled(chunk(distinct, BATCH_WRITE_SIZE), WRITE_CONCURRENCY, async keysChunk => { let pending = keysChunk.map(Key => ({ DeleteRequest: { Key } })); for (let attempt = 0; pending.length > 0; attempt += 1) { @@ -176,13 +180,14 @@ export default class DynamoExecutor { }), ); + this.logBatchCapacity('batchDelete', response.ConsumedCapacity); pending = (response.UnprocessedItems?.[this.model.tableName] ?? []) as typeof pending; } }); } async mapWrites(items: T[], handler: (item: T) => Promise): Promise { - return mapWithConcurrency(items, WRITE_CONCURRENCY, handler); + return mapUntilSettled(items, WRITE_CONCURRENCY, handler); } async put(input: Item): Promise { @@ -242,6 +247,12 @@ export default class DynamoExecutor { return this.settings.logConsumedCapacity ? { ReturnConsumedCapacity: 'TOTAL' } : {}; } + private logBatchCapacity(operation: string, consumed?: Array<{ CapacityUnits?: number }>): void { + const units = consumed?.reduce((sum, entry) => sum + (entry.CapacityUnits ?? 0), 0); + + if (units !== undefined) this.logCapacity(operation, units); + } + private logCapacity(operation: string, units?: number): void { if (this.settings.logConsumedCapacity && units !== undefined) { this.logger('Info', `[DynamoDB Capacity] ${this.model.name}.${operation}: ${units} units`); diff --git a/packages/datasource-dynamodb/src/utils/expression-builder.ts b/packages/datasource-dynamodb/src/utils/expression-builder.ts index d8f98b82cd..86cc95346c 100644 --- a/packages/datasource-dynamodb/src/utils/expression-builder.ts +++ b/packages/datasource-dynamodb/src/utils/expression-builder.ts @@ -27,7 +27,6 @@ export const PUSHABLE_OPERATORS = new Set([ 'Equal', 'In', 'NotIn', - 'Present', 'Missing', 'Blank', 'StartsWith', @@ -199,8 +198,6 @@ export default class ExpressionBuilder { return value === null || value === undefined ? this.missing(path) : `${path} = ${this.value(value)}`; - case 'Present': - return `(attribute_exists(${path}) AND NOT ${this.isNull(path)})`; case 'Missing': return this.missing(path); case 'Blank': diff --git a/packages/datasource-dynamodb/src/utils/partiql.ts b/packages/datasource-dynamodb/src/utils/partiql.ts new file mode 100644 index 0000000000..22204fee9d --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/partiql.ts @@ -0,0 +1,55 @@ +import { ValidationError } from '@forestadmin/datasource-toolkit'; + +const PLACEHOLDER_START = /[A-Za-z0-9_]/; + +/** + * Replaces `$name` placeholders by `?` and lists their values in order. What sits inside a string + * literal ('…') or a quoted identifier ("…") is text, so it is left alone. + */ +export default function parameterize( + query: string, + variables: Record, +): { statement: string; parameters: unknown[] } { + const parameters: unknown[] = []; + let statement = ''; + let quote: string | undefined; + + for (let i = 0; i < query.length; i += 1) { + const char = query[i]; + + if (quote) { + statement += char; + + if (char === quote) { + // A doubled quote is an escaped one, not the end of the literal. + if (query[i + 1] === quote) { + i += 1; + statement += quote; + } else { + quote = undefined; + } + } + } else if (char === "'" || char === '"') { + quote = char; + statement += char; + } else if (char === '$' && PLACEHOLDER_START.test(query[i + 1] ?? '')) { + let end = i + 1; + + while (end < query.length && PLACEHOLDER_START.test(query[end])) end += 1; + + const name = query.slice(i + 1, end); + + if (!(name in variables)) { + throw new ValidationError(`No value was given for the placeholder $${name}`); + } + + parameters.push(variables[name]); + statement += '?'; + i = end - 1; + } else { + statement += char; + } + } + + return { statement, parameters }; +} diff --git a/packages/datasource-dynamodb/src/utils/sort.ts b/packages/datasource-dynamodb/src/utils/sort.ts index 53c5d3eae1..0fd8582d73 100644 --- a/packages/datasource-dynamodb/src/utils/sort.ts +++ b/packages/datasource-dynamodb/src/utils/sort.ts @@ -1,5 +1,7 @@ import type { RecordData, Sort } from '@forestadmin/datasource-toolkit'; +import compareValues from './compare'; + /** Missing values come last whatever the direction, as DynamoDB has no notion of nulls order. */ export default function compareRecords(a: RecordData, b: RecordData, sort: Sort): number { for (const { field, ascending } of sort) { @@ -9,10 +11,10 @@ export default function compareRecords(a: RecordData, b: RecordData, sort: Sort) if (leftMissing !== rightMissing) return leftMissing ? 1 : -1; - if (!leftMissing && left !== right) { - const order = (left as number) < (right as number) ? -1 : 1; + if (!leftMissing) { + const order = compareValues(left, right); - return ascending ? order : -order; + if (order !== 0) return ascending ? order : -order; } } diff --git a/packages/datasource-dynamodb/src/virtual-collections.ts b/packages/datasource-dynamodb/src/virtual-collections.ts index 9038b39302..f840003916 100644 --- a/packages/datasource-dynamodb/src/virtual-collections.ts +++ b/packages/datasource-dynamodb/src/virtual-collections.ts @@ -22,7 +22,9 @@ async function itemSchemaOf( sampleSize: number, ): Promise> { const declared = - parent instanceof ArrayCollection ? undefined : parent.model.schema[arrayField]?.items; + parent instanceof ArrayCollection + ? parent.itemSchema[arrayField]?.items + : parent.model.schema[arrayField]?.items; if (declared) return declared; diff --git a/packages/datasource-dynamodb/test/array-review-findings.test.ts b/packages/datasource-dynamodb/test/array-review-findings.test.ts new file mode 100644 index 0000000000..d38c529bd8 --- /dev/null +++ b/packages/datasource-dynamodb/test/array-review-findings.test.ts @@ -0,0 +1,194 @@ +import type { DataSource } from '@forestadmin/datasource-toolkit'; + +import { Filter, PaginatedFilter, Projection } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { buildModel, leaf } from './helpers'; +import ArrayCollection from '../src/array-collection'; +import { createDynamoDBDataSource } from '../src'; +import { fakeClient } from './fake-client'; + +const caller = { timezone: 'UTC' } as never; + +function build(handlers: Record, model = buildModel()) { + const parent = buildCollection( + { + ...model, + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + lines: { type: 'array' }, + }, + }, + handlers as never, + { scanConcurrency: 1, chartCacheTtlMs: 60000 }, + ); + const collection = new ArrayCollection({} as DataSource, parent.collection, { + name: 'order_lines', + arrayField: 'lines', + itemSchema: { sku: { type: 'string' }, qty: { type: 'number' }, index: { type: 'number' } }, + }); + + return { collection, client: parent.client }; +} + +const order = (lines: unknown[]) => ({ customerId: 'c1', orderId: 'o1', lines }); + +describe('ArrayCollection review findings', () => { + it('keeps the identity of a record when the element has an attribute of the same name', async () => { + const { collection } = build({ + Scan: () => ({ Items: [order([{ sku: 'A', index: 99, customerId: 'other' }])] }), + }); + + const [record] = await collection.list( + caller, + new PaginatedFilter({}), + new Projection('customerId', 'index', 'sku'), + ); + + expect(record).toEqual({ customerId: 'c1', index: 0, sku: 'A' }); + }); + + it('splits the removal of many elements into expressions DynamoDB accepts', async () => { + const lines = Array.from({ length: 1000 }, (_, i) => ({ sku: `S${i}` })); + const { collection, client } = build({ + Scan: () => ({ Items: [order(lines)] }), + Update: () => ({}), + }); + + await collection.delete(caller, new Filter({})); + + const expressions = client.calls('Update').map(call => call.UpdateExpression as string); + const removed = expressions.flatMap(e => [...e.matchAll(/\[(\d+)\]/g)].map(m => Number(m[1]))); + + expect(expressions.length).toBeGreaterThan(1); + expect(Math.max(...expressions.map(e => e.length))).toBeLessThan(4096); + expect(removed).toEqual(Array.from({ length: 1000 }, (_, i) => 999 - i)); + }); + + it('splits the update of many elements too', async () => { + const lines = Array.from({ length: 1000 }, (_, i) => ({ sku: `S${i}` })); + const { collection, client } = build({ + Scan: () => ({ Items: [order(lines)] }), + Update: () => ({}), + }); + + await collection.update(caller, new Filter({}), { qty: 1 }); + + const lengths = client.calls('Update').map(call => (call.UpdateExpression as string).length); + + expect(lengths.length).toBeGreaterThan(1); + expect(Math.max(...lengths)).toBeLessThan(4096); + }); + + it('sets an attribute under a map the element does not have yet, by rewriting the item', async () => { + const { collection, client } = build({ + Scan: () => ({ Items: [order([{ sku: 'A' }])] }), + Update: () => { + throw Object.assign( + new Error('The document path provided in the update expression is invalid for update'), + { name: 'ValidationException' }, + ); + }, + Get: () => ({ Item: order([{ sku: 'A' }]) }), + Put: () => ({}), + }); + + await collection.update(caller, new Filter({}), { 'details->note': 'hi' }); + + expect(client.calls('Put')[0].Item).toEqual(order([{ sku: 'A', details: { note: 'hi' } }])); + }); + + it('only touches items of the entity the parent stands for', async () => { + const { collection, client } = build( + { + Scan: () => ({ Items: [order([{ sku: 'A' }])] }), + Update: () => ({}), + }, + buildModel({ entity: { attribute: 'entityType', value: 'ORDER' } }), + ); + + await collection.update(caller, new Filter({}), { qty: 2 }); + await collection + .create(caller, [{ customerId: 'c1', orderId: 'o1', sku: 'B' }]) + .catch(() => {}); + + for (const call of client.calls('Update')) { + expect(call.ConditionExpression).toMatch(/^attribute_exists\(#k0\) AND .*#n\d = :v\d$/); + expect(Object.values(call.ExpressionAttributeValues)).toContain('ORDER'); + } + }); + + it('forgets the cached charts when an append fails halfway', async () => { + const { collection, client } = build({ + Update: [ + { Attributes: { lines: [{ sku: 'A' }] } }, + Object.assign(new Error('throttled'), { name: 'ThrottlingException' }), + ], + Scan: () => ({ Items: [order([{ sku: 'A' }])] }), + }); + const count = () => + collection.aggregate(caller, new Filter({}), { operation: 'Count' } as never); + + await count().catch(() => undefined); + await expect( + collection.create(caller, [ + { customerId: 'c1', orderId: 'o1', sku: 'A' }, + { customerId: 'c1', orderId: 'o1', sku: 'B' }, + ]), + ).rejects.toThrow('throttled'); + + expect(client.calls('Update')).toHaveLength(2); + }); +}); + +describe('native queries', () => { + const orders = { + Table: { + TableName: 'Orders', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }, + }; + + it('does not take a placeholder inside a string literal for a parameter', async () => { + const client = fakeClient({ + DescribeTable: () => orders, + Scan: () => ({ Items: [] }), + ExecuteStatement: () => ({ Items: [] }), + }); + const dataSource = await createDynamoDBDataSource({ + client, + tables: ['Orders'], + liveQueryConnection: 'dynamo', + })(jest.fn(), jest.fn()); + + await dataSource.executeNativeQuery( + 'dynamo', + `SELECT * FROM "Orders" WHERE name = '$name' AND id = $id`, + { id: 'a' }, + ); + + expect(client.calls('ExecuteStatement')[0]).toEqual({ + Statement: `SELECT * FROM "Orders" WHERE name = '$name' AND id = ?`, + Parameters: ['a'], + }); + }); + + it('stops following a statement that never ends', async () => { + const client = fakeClient({ + DescribeTable: () => orders, + Scan: () => ({ Items: [] }), + ExecuteStatement: () => ({ Items: [], NextToken: 'again' }), + }); + const dataSource = await createDynamoDBDataSource({ + client, + tables: ['Orders'], + liveQueryConnection: 'dynamo', + })(jest.fn(), jest.fn()); + + await expect( + dataSource.executeNativeQuery('dynamo', 'SELECT * FROM "Orders" WHERE a = 1', {}), + ).rejects.toThrow('still has results after 1000 pages'); + }); +}); diff --git a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts index 3d95e8390d..ad92014aba 100644 --- a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts +++ b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts @@ -1,7 +1,12 @@ import type { DataSource } from '@forestadmin/datasource-toolkit'; import { CreateTableCommand, DeleteTableCommand, DynamoDBClient } from '@aws-sdk/client-dynamodb'; -import { BatchWriteCommand, DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import { + BatchWriteCommand, + DeleteCommand, + DynamoDBDocumentClient, + PutCommand, +} from '@aws-sdk/lib-dynamodb'; import { Aggregation, ConditionTreeFactory, @@ -204,12 +209,20 @@ run('DynamoDB Local', () => { expect(records.map(r => r.orderId)).toEqual(['o12', 'o11', 'o10']); }); - it('queries a global secondary index', async () => { + it('queries a global secondary index when a condition on its sort key is given', async () => { logger.mockClear(); const records = await orders().list( caller, - new PaginatedFilter({ conditionTree: leaf('status', 'Equal', 'paid') }), + new PaginatedFilter({ + conditionTree: tree({ + aggregator: 'And', + conditions: [ + { field: 'status', operator: 'Equal', value: 'paid' }, + { field: 'createdAt', operator: 'GreaterThan', value: '2000' }, + ], + }), + }), new Projection('orderId'), ); @@ -217,6 +230,27 @@ run('DynamoDB Local', () => { expect(logger).not.toHaveBeenCalledWith('Warn', expect.stringContaining('Scan on table')); }); + it('does not let an index hide the items that lack its sort key', async () => { + await documents.send( + new PutCommand({ + TableName: ORDERS, + Item: { customerId: 'c3', orderId: 'o90', status: 'paid', total: 1 }, + }), + ); + + const records = await orders().list( + caller, + new PaginatedFilter({ conditionTree: leaf('status', 'Equal', 'paid') }), + new Projection('orderId'), + ); + await documents.send( + new DeleteCommand({ TableName: ORDERS, Key: { customerId: 'c3', orderId: 'o90' } }), + ); + + expect(records.map(r => r.orderId)).toContain('o90'); + expect(records).toHaveLength(11); + }); + it('scans, with a warning, for a filter no key serves', async () => { const records = await orders().list( caller, diff --git a/packages/datasource-dynamodb/test/review-findings.test.ts b/packages/datasource-dynamodb/test/review-findings.test.ts new file mode 100644 index 0000000000..c12186c054 --- /dev/null +++ b/packages/datasource-dynamodb/test/review-findings.test.ts @@ -0,0 +1,212 @@ +import { + Filter, + PaginatedFilter, + Projection, + Sort, + Aggregation, +} from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { fakeClient } from './fake-client'; +import { buildModel, buildSimpleModel, leaf } from './helpers'; +import { introspectTable } from '../src/introspection/introspector'; +import { validateManualSchema } from '../src/schema/manual-schema'; + +const caller = { timezone: 'UTC' } as never; + +describe('pagination across timezones', () => { + it('does not resume a page from a position remembered for another timezone', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Scan: [ + { + Items: [ + { id: 'a', name: 'ab' }, + { id: 'b', name: 'ac' }, + ], + LastEvaluatedKey: { id: 'b' }, + }, + { Items: [{ id: 'c', name: 'ad' }] }, + ], + }); + const page = (timezone: string, skip: number) => + collection.list( + { timezone } as never, + new PaginatedFilter({ + conditionTree: leaf('name', 'ILike', 'a%'), + page: { skip, limit: 1 } as never, + }), + new Projection('id'), + ); + + await page('Europe/Paris', 0); + client.send.mockClear(); + await page('Asia/Tokyo', 1); + + expect(client.calls('Scan')[0].ExclusiveStartKey).toBeUndefined(); + }); +}); + +describe('caches after a failed write', () => { + it('forgets the cached charts when part of a bulk write failed', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { + Scan: () => ({ Items: [{ age: 1 }] }), + Put: [{}, Object.assign(new Error('throttled'), { name: 'ThrottlingException' })], + }, + { chartCacheTtlMs: 60000, scanConcurrency: 1 }, + ); + const sum = new Aggregation({ operation: 'Sum', field: 'age' }); + + await collection.aggregate(caller, new Filter({}), sum); + await expect(collection.create(caller, [{ id: 'a' }, { id: 'b' }])).rejects.toThrow( + 'throttled', + ); + await collection.aggregate(caller, new Filter({}), sum); + + expect(client.calls('Scan')).toHaveLength(2); + }); +}); + +describe('sorting a large result', () => { + it('sorts more records than a function call can take as arguments', async () => { + const items = Array.from({ length: 150000 }, (_, i) => ({ id: `u${i}`, age: i % 1000 })); + const { collection } = buildCollection(buildSimpleModel(), { Scan: () => ({ Items: items }) }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + sort: new Sort({ field: 'age', ascending: false }), + page: { skip: 0, limit: 2 } as never, + }), + new Projection('age'), + ); + + expect(records).toEqual([{ age: 999 }, { age: 999 }]); + }); +}); + +describe('single-table discriminator', () => { + const entityModel = () => + buildModel({ + entity: { attribute: 'entityType', value: 'ORDER' }, + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + entityType: { type: 'string' }, + }, + }); + + it('refuses to create a record that claims another entity', async () => { + const { collection, client } = buildCollection(entityModel(), { Put: () => ({}) }); + + await expect( + collection.create(caller, [{ customerId: 'c', orderId: 'o', entityType: 'USER' }]), + ).rejects.toThrow("The attribute 'entityType' of 'orders' must be 'ORDER'"); + expect(client.calls('Put')).toHaveLength(0); + }); + + it('accepts the discriminator of its own entity', async () => { + const { collection, client } = buildCollection(entityModel(), { Put: () => ({}) }); + + await collection.create(caller, [{ customerId: 'c', orderId: 'o', entityType: 'ORDER' }]); + + expect(client.calls('Put')[0].Item.entityType).toBe('ORDER'); + }); + + it('refuses to move a record to another entity', async () => { + const { collection } = buildCollection(entityModel(), {}); + + await expect( + collection.update(caller, new Filter({ conditionTree: leaf('customerId', 'Equal', 'c') }), { + entityType: 'USER', + }), + ).rejects.toThrow( + "The attribute 'entityType' tells which entity a record of 'orders' belongs to", + ); + }); +}); + +describe('batch requests', () => { + it('asks for each key once', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + BatchGet: () => ({ Responses: { Users: [{ id: 'a' }] } }), + }); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('id', 'In', ['a', 'a']) }), + new Projection('id'), + ); + + expect(client.calls('BatchGet')[0].RequestItems.Users.Keys).toEqual([{ id: 'a' }]); + }); + + it('logs the capacity consumed by batch reads and writes', async () => { + const { collection, logger } = buildCollection( + buildSimpleModel(), + { + BatchGet: () => ({ + Responses: { Users: [{ id: 'a' }] }, + ConsumedCapacity: [{ CapacityUnits: 1.5 }, { CapacityUnits: 0.5 }], + }), + BatchWrite: () => ({ ConsumedCapacity: [{ CapacityUnits: 3 }] }), + }, + { logConsumedCapacity: true }, + ); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('id', 'Equal', 'a') }), + new Projection('id'), + ); + await collection.delete(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'a') })); + + expect(logger).toHaveBeenCalledWith('Info', '[DynamoDB Capacity] users.batchGet: 2 units'); + expect(logger).toHaveBeenCalledWith('Info', '[DynamoDB Capacity] users.batchDelete: 3 units'); + }); +}); + +describe('introspection of unusual attribute names', () => { + it.each(['constructor', 'toString', '__proto__'])('infers an attribute named %s', async name => { + const item = Object.fromEntries([['id', 'a']]); + Object.defineProperty(item, name, { value: 'x', enumerable: true, configurable: true }); + const client = fakeClient({ + DescribeTable: () => ({ + Table: { + TableName: 'T', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }, + }), + Scan: () => ({ Items: [item] }), + }); + + const { schema } = await introspectTable(client, { collectionName: 't', tableName: 'T' }); + + expect(Object.keys(schema).sort()).toEqual(['id', name].sort()); + }); +}); + +describe('manual schema entity', () => { + const withEntity = (entity: Record) => ({ + collections: [ + { + name: 'c', + tableName: 'T', + partitionKey: { name: 'pk', type: 'string' as const }, + fields: [], + entity, + }, + ], + }); + + it.each([{ value: 'ORDER' }, { attribute: 'type' }])( + 'rejects a discriminator with only one of attribute and value: %p', + entity => { + expect(() => validateManualSchema(withEntity(entity))).toThrow( + "Collection 'c' entity must use either 'attribute'/'value' or 'sortKeyPrefix'", + ); + }, + ); +}); diff --git a/packages/datasource-dynamodb/test/schema/manual-schema.test.ts b/packages/datasource-dynamodb/test/schema/manual-schema.test.ts index 809ae3e145..c1fa7d27c7 100644 --- a/packages/datasource-dynamodb/test/schema/manual-schema.test.ts +++ b/packages/datasource-dynamodb/test/schema/manual-schema.test.ts @@ -110,6 +110,7 @@ describe('convertManualSchema', () => { partitionKey: { name: 'status', type: 'S' }, sortKey: { name: 'createdAt', type: 'S' }, projectsAllAttributes: true, + coversAllItems: false, }, ]); expect(model).toMatchObject({ diff --git a/packages/datasource-dynamodb/test/utils/access-path.test.ts b/packages/datasource-dynamodb/test/utils/access-path.test.ts index 2f386347ca..0b0e935b04 100644 --- a/packages/datasource-dynamodb/test/utils/access-path.test.ts +++ b/packages/datasource-dynamodb/test/utils/access-path.test.ts @@ -9,6 +9,7 @@ const gsi = (overrides = {}) => ({ partitionKey: { name: 'status', type: 'S' as const }, sortKey: { name: 'createdAt', type: 'S' as const }, projectsAllAttributes: true, + coversAllItems: true, ...overrides, }); @@ -292,3 +293,61 @@ describe('key value types', () => { expect(lookup.keys).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }]); }); }); + +describe('secondary indexes that leave items out', () => { + const sparse = () => buildModel({ indexes: [gsi({ coversAllItems: false })] }); + + it('does not read an index by its partition key alone, as items lacking its sort key are not in it', () => { + expect(planAccessPath(sparse(), leaf('status', 'Equal', 'paid')).kind).toBe('scan'); + }); + + it('reads it once a condition on its sort key means the item must be in it', () => { + const plan = planAccessPath( + sparse(), + and(plain('status', 'Equal', 'paid'), plain('createdAt', 'GreaterThan', '2024')), + ); + + expect(plan).toMatchObject({ kind: 'query', indexName: 'by-status' }); + }); + + it('reads an index without sort key by its partition key', () => { + const model = buildModel({ + indexes: [gsi({ sortKey: undefined, coversAllItems: false })], + }); + + expect(planAccessPath(model, leaf('status', 'Equal', 'paid'))).toMatchObject({ + kind: 'query', + indexName: 'by-status', + }); + }); +}); + +describe('planKeyLookup soundness', () => { + it('is not a lookup when an arm of the OR carries a nested filter', () => { + const tree = or( + and(plain('id', 'Equal', 'a'), { + aggregator: 'Or', + conditions: [plain('age', 'Equal', 1), plain('age', 'Equal', 2)], + }), + plain('id', 'Equal', 'b'), + ); + + expect(planKeyLookup(buildSimpleModel(), tree)).toBeNull(); + }); + + it('lists each key once for a repeated value', () => { + expect(planKeyLookup(buildSimpleModel(), leaf('id', 'In', ['a', 'a', 'b'])).keys).toEqual([ + { id: 'a' }, + { id: 'b' }, + ]); + }); + + it('lists each key once for overlapping OR arms', () => { + const tree = or( + and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'Equal', 'o1')), + and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'Equal', 'o1')), + ); + + expect(planKeyLookup(buildModel(), tree).keys).toEqual([{ customerId: 'c1', orderId: 'o1' }]); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/compare.test.ts b/packages/datasource-dynamodb/test/utils/compare.test.ts new file mode 100644 index 0000000000..5e9675a7e2 --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/compare.test.ts @@ -0,0 +1,50 @@ +import { Aggregation } from '@forestadmin/datasource-toolkit'; + +import compareValues from '../../src/utils/compare'; +import StreamingAggregator from '../../src/utils/aggregator'; +import compareRecords from '../../src/utils/sort'; +import { Sort } from '@forestadmin/datasource-toolkit'; + +describe('compareValues', () => { + it('orders date-times by the instant, not by their text', () => { + expect(compareValues('2024-01-01T00:30:00+02:00', '2023-12-31T23:00:00Z')).toBe(-1); + }); + + it('treats the same instant written two ways as equal', () => { + expect(compareValues('2024-01-01T02:00:00+02:00', '2024-01-01T00:00:00Z')).toBe(0); + }); + + it('orders other strings and numbers as usual', () => { + expect(compareValues('a', 'b')).toBe(-1); + expect(compareValues(2, 1)).toBe(1); + expect(compareValues(1, 1)).toBe(0); + }); +}); + +describe('sorting and aggregating dates with offsets', () => { + const earlier = '2024-01-01T00:30:00+02:00'; + const later = '2023-12-31T23:00:00Z'; + + it('sorts ascending by instant', () => { + const sort = new Sort({ field: 'at', ascending: true }); + const records = [{ at: later }, { at: earlier }]; + + records.sort((a, b) => compareRecords(a, b, sort)); + + expect(records.map(r => r.at)).toEqual([earlier, later]); + }); + + it.each([ + ['Min', earlier], + ['Max', later], + ])('takes the %s instant while keeping its text', (operation, expected) => { + const aggregator = new StreamingAggregator( + new Aggregation({ operation: operation as 'Min' | 'Max', field: 'at' }), + 'UTC', + ); + aggregator.add({ at: later }); + aggregator.add({ at: earlier }); + + expect(aggregator.result()).toEqual([{ value: expected, group: {} }]); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/concurrency.test.ts b/packages/datasource-dynamodb/test/utils/concurrency.test.ts new file mode 100644 index 0000000000..4a0d33491a --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/concurrency.test.ts @@ -0,0 +1,52 @@ +import { mapUntilSettled, mapWithConcurrency } from '../../src/utils/concurrency'; + +describe('mapWithConcurrency', () => { + it('keeps the order of the results', async () => { + const result = await mapWithConcurrency([3, 1, 2], 2, async n => n * 2); + + expect(result).toEqual([6, 2, 4]); + }); + + it('runs at most the given number of calls at once', async () => { + let running = 0; + let peak = 0; + + await mapWithConcurrency([1, 2, 3, 4, 5], 2, async () => { + running += 1; + peak = Math.max(peak, running); + await new Promise(resolve => { + setImmediate(resolve); + }); + running -= 1; + }); + + expect(peak).toBe(2); + }); +}); + +describe('mapUntilSettled', () => { + it('rejects only once the writes in flight have finished, and starts no new one', async () => { + const finished: number[] = []; + const started: number[] = []; + + await expect( + mapUntilSettled([1, 2, 3, 4, 5, 6], 2, async n => { + started.push(n); + + if (n === 1) throw new Error('boom'); + + await new Promise(resolve => { + setTimeout(resolve, 10); + }); + finished.push(n); + }), + ).rejects.toThrow('boom'); + + expect(finished).toContain(2); + expect(started).not.toContain(6); + }); + + it('returns the results when nothing fails', async () => { + expect(await mapUntilSettled([1, 2], 2, async n => n + 1)).toEqual([2, 3]); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/expression-builder.test.ts b/packages/datasource-dynamodb/test/utils/expression-builder.test.ts index 8d6d017ec0..4166466414 100644 --- a/packages/datasource-dynamodb/test/utils/expression-builder.test.ts +++ b/packages/datasource-dynamodb/test/utils/expression-builder.test.ts @@ -87,13 +87,9 @@ describe('ExpressionBuilder', () => { expect(builder.values).toEqual({ ':v0': 'NULL' }); }); - it('translates Present and Missing around the NULL type', () => { - const present = new ExpressionBuilder(); + it('translates Missing as a missing or NULL attribute', () => { const missing = new ExpressionBuilder(); - expect(present.condition(leaf('f', 'Present'))).toBe( - '(attribute_exists(#n0) AND NOT attribute_type(#n0, :v0))', - ); expect(missing.condition(leaf('f', 'Missing'))).toBe( '(attribute_not_exists(#n0) OR attribute_type(#n0, :v0))', ); @@ -247,6 +243,15 @@ describe('ExpressionBuilder', () => { }); describe('splitCondition', () => { + it('leaves Present to the items, since the toolkit also excludes empty strings from it', () => { + const tree = leaf('f', 'Present'); + + const { pushable, residual } = splitCondition(tree); + + expect(pushable).toBeUndefined(); + expect(residual).toBe(tree); + }); + it('returns everything as pushable when DynamoDB can evaluate it', () => { const tree = leaf('f', 'Equal', 1); diff --git a/packages/datasource-dynamodb/test/utils/partiql.test.ts b/packages/datasource-dynamodb/test/utils/partiql.test.ts new file mode 100644 index 0000000000..6548dcdc6a --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/partiql.test.ts @@ -0,0 +1,46 @@ +import parameterize from '../../src/utils/partiql'; + +describe('parameterize', () => { + it('replaces placeholders by positional parameters, in order, repeats included', () => { + expect( + parameterize('SELECT * FROM "T" WHERE a = $x AND b = $y AND c = $x', { x: 1, y: 'z' }), + ).toEqual({ + statement: 'SELECT * FROM "T" WHERE a = ? AND b = ? AND c = ?', + parameters: [1, 'z', 1], + }); + }); + + it('leaves a placeholder inside a string literal alone', () => { + expect(parameterize('SELECT * FROM "T" WHERE name = \'$name\'', {})).toEqual({ + statement: 'SELECT * FROM "T" WHERE name = \'$name\'', + parameters: [], + }); + }); + + it('leaves a placeholder inside a quoted identifier alone', () => { + expect(parameterize('SELECT "$cost" FROM "T" WHERE a = $a', { a: 1 })).toEqual({ + statement: 'SELECT "$cost" FROM "T" WHERE a = ?', + parameters: [1], + }); + }); + + it('keeps reading a literal past an escaped quote', () => { + expect(parameterize("SELECT * FROM \"T\" WHERE a = 'it''s $x' AND b = $y", { y: 2 })).toEqual({ + statement: "SELECT * FROM \"T\" WHERE a = 'it''s $x' AND b = ?", + parameters: [2], + }); + }); + + it('refuses a placeholder without value', () => { + expect(() => parameterize('SELECT * FROM "T" WHERE a = $missing', {})).toThrow( + 'No value was given for the placeholder $missing', + ); + }); + + it('keeps a lone dollar sign', () => { + expect(parameterize('SELECT * FROM "T" WHERE a = $ AND b = $1', { 1: 'x' })).toEqual({ + statement: 'SELECT * FROM "T" WHERE a = $ AND b = ?', + parameters: ['x'], + }); + }); +}); diff --git a/packages/datasource-dynamodb/test/virtual-collections.test.ts b/packages/datasource-dynamodb/test/virtual-collections.test.ts index 839cd99891..82c4e053ac 100644 --- a/packages/datasource-dynamodb/test/virtual-collections.test.ts +++ b/packages/datasource-dynamodb/test/virtual-collections.test.ts @@ -55,16 +55,13 @@ describe('virtual collections', () => { }); }); - it('samples only the parents that have the array', async () => { - const { client, dataSource } = build([{ id: 'o1', lines: [{ sku: 'A' }] }]); + it('samples the parents that have the array, applying the filter on the items read', async () => { + const { client, dataSource } = build([{ id: 'o1', lines: [{ sku: 'A' }] }, { id: 'o2' }]); - await dataSource; + const result = await dataSource; - const sample = client.calls('Scan').find(call => call.FilterExpression); - expect(sample.FilterExpression).toBe( - '(attribute_exists(#n0) AND NOT attribute_type(#n0, :v0))', - ); - expect(sample.ExpressionAttributeNames['#n0']).toBe('lines'); + expect(client.calls('Scan')[0].FilterExpression).toBeUndefined(); + expect(result.getCollection('lines').schema.fields.sku).toBeDefined(); }); it('skips a collection whose parent does not exist, and says so', async () => { @@ -118,4 +115,48 @@ describe('virtual collections', () => { expect.stringContaining("Skipping virtual collection 'parts' for array 'lines.parts'"), ); }); + + it('uses the item schema a parent array collection declares when the sample is empty', async () => { + const client = fakeClient({ + DescribeTable: () => orders, + Scan: () => ({ Items: [] }), + }); + + const dataSource = await createDynamoDBDataSource({ + client, + disableIntrospection: true, + schema: { + collections: [ + { + name: 'Orders', + tableName: 'Orders', + partitionKey: { name: 'id', type: 'string' }, + fields: [ + { + name: 'lines', + type: 'array', + subType: 'object', + fields: [ + { + name: 'parts', + type: 'array', + subType: 'object', + fields: [{ name: 'n', type: 'number' }], + }, + ], + }, + ], + }, + ], + }, + virtualCollections: [ + { parentCollection: 'Orders', collectionName: 'lines', arrayField: 'lines' }, + { parentCollection: 'lines', collectionName: 'parts', arrayField: 'parts' }, + ], + })(jest.fn(), jest.fn()); + + expect(dataSource.getCollection('parts').schema.fields.n).toMatchObject({ + columnType: 'Number', + }); + }); }); From 39b409bfdbc2f9c9a6accd00ca52cad2459ed3da Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Fri, 16 Oct 2026 20:00:46 +0200 Subject: [PATCH 07/32] docs(datasource-dynamodb): document indexes, native query limits and date comparison Refs: PRD-1670 Co-Authored-By: Claude Sonnet 5.5 --- packages/datasource-dynamodb/README.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/datasource-dynamodb/README.md b/packages/datasource-dynamodb/README.md index 5a35c024e6..5fdaabc662 100644 --- a/packages/datasource-dynamodb/README.md +++ b/packages/datasource-dynamodb/README.md @@ -26,15 +26,15 @@ DynamoDB reads fast by key and scans otherwise, and it cannot sort, count or agg server. The datasource uses the key whenever a filter allows it and does the rest on the items it reads. What that means for each feature: -| Feature | How it works | -| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Filters | Pushed down when DynamoDB can evaluate them. `Like`, `ILike`, `IContains`, `EndsWith`, date intervals and the like are applied on the returned items, so they read more than they return. | -| Access path | A `Query` on the table, a global or local secondary index when an equality pins its partition key (with the sort key range when there is one), a `BatchGetItem` when the filter names whole keys, a `Scan` otherwise. | -| Sort | Free on the sort key of the table or index being read. On any other attribute, the matching items are read, then sorted in memory. | -| Pagination | There is no offset: the datasource remembers the key where earlier pages stopped and resumes from the closest one. `maxOffset` (100 000) caps how deep a page can be. | -| Count and charts | Computed by reading the items (a counting `Scan`/`Query` for a plain count), in parallel segments for a table. Results are kept in memory for `chartCacheTtlMs` (30 s). | -| Search | Evaluated on the items: it scans. Case-insensitive, accents kept. | -| Relations | None in DynamoDB. Declare them with `addManyToOneRelation` and friends: a lookup by key is one `BatchGetItem`, a relation backed by a secondary index is a `Query`. | +| Feature | How it works | +| ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Filters | Pushed down when DynamoDB can evaluate them. `Like`, `ILike`, `IContains`, `EndsWith`, date intervals and the like are applied on the returned items, so they read more than they return. | +| Access path | A `Query` on the table, a global or local secondary index when an equality pins its partition key (with the sort key range when there is one), a `BatchGetItem` when the filter names whole keys, a `Scan` otherwise. An index leaves out the items that lack its sort key, so one that has a sort key is only read when the filter has a condition on that key, or when the index is declared with `coversAllItems: true` in a manual schema. | +| Sort | Free on the sort key of the table or index being read. On any other attribute, the matching items are read, then sorted in memory. | +| Pagination | There is no offset: the datasource remembers the key where earlier pages stopped and resumes from the closest one. `maxOffset` (100 000) caps how deep a page can be. | +| Count and charts | Computed by reading the items (a counting `Scan`/`Query` for a plain count), in parallel segments for a table. Results are kept in memory for `chartCacheTtlMs` (30 s). | +| Search | Evaluated on the items: it scans. Case-insensitive, accents kept. | +| Relations | None in DynamoDB. Declare them with `addManyToOneRelation` and friends: a lookup by key is one `BatchGetItem`, a relation backed by a secondary index is a `Query`. | ## Controlling cost: `scanPolicy` @@ -47,7 +47,8 @@ A request that no key or index serves reads the whole table. `scanPolicy` decide | `'deny'` | Refuses with an explicit error. | `maxScannedItems` (1 000 000) stops any single request that reads more items than that, with an error -rather than a partial result. Both can be set on the datasource or on a collection. +rather than a partial result. PartiQL native queries do not report how many items they evaluated, +so for them it bounds the rows returned, and the pages fetched are capped at 1000. Both can be set on the datasource or on a collection. ## Choosing the tables and naming collections @@ -103,6 +104,9 @@ createDynamoDBDataSource({ }); ``` +Dates are compared by their instant when sorted or aggregated here, but DynamoDB compares them as +text when it filters: store them in one offset (UTC) if you filter on ranges of them. + Types: `string`, `number`, `boolean`, `date` and `dateonly` (ISO 8601 strings), `timeonly`, `object`, `array`, `point` (GeoJSON), `binary` (base64). Keys are strings or numbers. From 27fae4d98a9f6a85950c7bbad61efcd1abb64839 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Fri, 16 Oct 2026 20:10:09 +0200 Subject: [PATCH 08/32] perf(datasource-dynamodb): query each partition of an In on the key, and warn less Running a real agent over DynamoDB showed that a filter through a relation sends an In on the partition key, which was scanned: each value is now read with a Query of its own. The same run showed a scan warning on every first page of an unfiltered collection, which reads one page: it is no longer warned about or denied, and the same warning is repeated at most once a minute. Refs: PRD-1670 Co-Authored-By: Claude Sonnet 5.5 --- .../datasource-dynamodb/src/collection.ts | 100 ++++++++++---- .../src/utils/access-path.ts | 43 ++++-- .../test/fan-out-and-warnings.test.ts | 124 ++++++++++++++++++ .../test/integration/dynamodb-local.test.ts | 27 ++++ .../test/utils/access-path.test.ts | 64 +++++++++ 5 files changed, 326 insertions(+), 32 deletions(-) create mode 100644 packages/datasource-dynamodb/test/fan-out-and-warnings.test.ts diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index ab646708d2..4028d7fed5 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -33,9 +33,13 @@ import { planAccessPath, planKeyLookup } from './utils/access-path'; import StreamingAggregator from './utils/aggregator'; import DynamoExecutor from './utils/executor'; import ExpressionBuilder, { splitCondition } from './utils/expression-builder'; +import { mapWithConcurrency } from './utils/concurrency'; import Serializer from './utils/serializer'; import compareRecords from './utils/sort'; +/** What a parallel read splits into: the segments of a scan, or the partitions of a filter. */ +type ReadUnit = { segment?: { index: number; total: number }; partition?: string | number }; + type ReadPlan = { lookup?: KeyLookup; access?: AccessPath; residual?: ConditionTree }; function intersect(...trees: ConditionTree[]): ConditionTree | undefined { @@ -47,6 +51,10 @@ function intersect(...trees: ConditionTree[]): ConditionTree | undefined { /** Values a single expression may carry before it is evaluated on the returned items instead. */ const MAX_PUSHED_LIST_VALUES = 200; const LIST_OPERATORS = ['In', 'NotIn', 'IncludesAll', 'IncludesNone']; +/** Items below which a page read without any filter is too cheap to call a scan. */ +const MAX_CHEAP_PAGE_ITEMS = 1000; +/** A scan warning is repeated at most this often. */ +const WARN_INTERVAL_MS = 60000; /** Charts remembered per collection, the oldest dropped first. */ const MAX_CACHED_CHARTS = 100; @@ -68,9 +76,10 @@ function describeFilter(tree?: ConditionTree): string { function keyConditionExpression( builder: ExpressionBuilder, { partition, sort }: AccessPath['keyCondition'], + partitionValue: string | number, ): string { const expressions = [ - `${builder.attribute(partition.attribute)} = ${builder.value(partition.value)}`, + `${builder.attribute(partition.attribute)} = ${builder.value(partitionValue)}`, ]; if (sort) { @@ -103,6 +112,7 @@ export default class DynamoDBCollection extends BaseCollection { private readonly logger: Logger; private readonly fieldNames: Set; private readonly matchContext: Collection; + private readonly warnedAt = new Map(); private readonly chartCache = new Map(); constructor( @@ -287,23 +297,32 @@ export default class DynamoDBCollection extends BaseCollection { ); } - this.logger( - 'Warn', - `Scan on table '${this.model.tableName}' for collection '${this.name}': ${reason}`, - ); + const message = `Scan on table '${this.model.tableName}' for collection '${this.name}': ${reason}`; + const now = this.settings.now(); + + // The same scan is asked for again each time a page or a chart is displayed: say it once. + if (now - (this.warnedAt.get(message) ?? -Infinity) < WARN_INTERVAL_MS) return; + + this.warnedAt.set(message, now); + this.logger('Warn', message); } private buildReadInput( access: AccessPath, fields: string[], extra: Partial = {}, + partitionValue?: string | number, ): ReadInput { const builder = new ExpressionBuilder(); const input: ReadInput = {}; if (access.kind === 'query') { input.IndexName = access.indexName; - input.KeyConditionExpression = keyConditionExpression(builder, access.keyCondition); + input.KeyConditionExpression = keyConditionExpression( + builder, + access.keyCondition, + partitionValue ?? access.keyCondition.partition.value, + ); if (!access.scanIndexForward) input.ScanIndexForward = false; } @@ -326,7 +345,7 @@ export default class DynamoDBCollection extends BaseCollection { plan: ReadPlan, fields: string[], budget: ReadBudget, - segment?: { index: number; total: number }, + unit?: ReadUnit, ): AsyncGenerator { if (plan.lookup) { yield await this.executor.batchGet(plan.lookup.keys, this.projectionParams(fields) as never); @@ -334,10 +353,24 @@ export default class DynamoDBCollection extends BaseCollection { return; } + const partitions = plan.access.keyCondition?.partition.values; + + // A filter naming several partitions is one Query per partition, read one after the other. + if (partitions && !unit) { + for (const partition of partitions) { + yield* this.readBatches(plan, fields, budget, { partition }); + } + + return; + } + const operation: ReadOperation = plan.access.kind; - const input = this.buildReadInput(plan.access, fields, { - ...(segment ? { Segment: segment.index, TotalSegments: segment.total } : {}), - }); + const input = this.buildReadInput( + plan.access, + fields, + unit?.segment ? { Segment: unit.segment.index, TotalSegments: unit.segment.total } : {}, + unit?.partition, + ); for await (const page of this.executor.readPages(operation, input, budget)) yield page.items; } @@ -432,7 +465,15 @@ export default class DynamoDBCollection extends BaseCollection { const limit = filter.page?.limit || undefined; const plan = this.plan(filter.conditionTree, sort); - this.guardScan(plan, filter.conditionTree); + // Reading the first page of a collection without any filter costs one page, not a table. + const isCheapPage = + !filter.conditionTree && + !plan.residual && + (!sort || plan.access?.sortSatisfied) && + limit !== undefined && + skip + limit <= MAX_CHEAP_PAGE_ITEMS; + + if (!isCheapPage) this.guardScan(plan, filter.conditionTree); const { columns } = projection; const fields = [ @@ -452,7 +493,9 @@ export default class DynamoDBCollection extends BaseCollection { ); }; - if (!plan.lookup && (!sort || plan.access.sortSatisfied)) { + const singlePartition = !plan.access?.keyCondition?.partition.values; + + if (!plan.lookup && singlePartition && (!sort || plan.access.sortSatisfied)) { return (await this.readPage(plan, fields, skip, limit, caller.timezone)).map(toResult); } @@ -640,7 +683,7 @@ export default class DynamoDBCollection extends BaseCollection { read: (segment: T) => Promise, ): Promise { try { - await Promise.all(segments.map(read)); + await mapWithConcurrency(segments, Math.max(this.settings.scanConcurrency, 1), read); } catch (error) { budget.abort(); @@ -672,19 +715,30 @@ export default class DynamoDBCollection extends BaseCollection { const budget = this.executor.newBudget(); const segments = plan.access?.kind === 'scan' ? Math.max(this.settings.scanConcurrency, 1) : 1; - const segmentsOf = (): Array<{ index: number; total: number } | undefined> => - segments > 1 - ? Array.from({ length: segments }, (_, index) => ({ index, total: segments })) + const partitions = plan.access?.keyCondition?.partition.values; + const segmentsOf = (): Array => { + if (partitions) return partitions.map(partition => ({ partition })); + + return segments > 1 + ? Array.from({ length: segments }, (_, index) => ({ segment: { index, total: segments } })) : [undefined]; + }; if (isCount && ungrouped && !plan.lookup && !plan.residual) { let total = 0; - await this.inSegments(budget, segmentsOf(), async segment => { - const input = this.buildReadInput(plan.access, [], { - Select: 'COUNT', - ...(segment ? { Segment: segment.index, TotalSegments: segment.total } : {}), - }); + await this.inSegments(budget, segmentsOf(), async unit => { + const input = this.buildReadInput( + plan.access, + [], + { + Select: 'COUNT', + ...(unit?.segment + ? { Segment: unit.segment.index, TotalSegments: unit.segment.total } + : {}), + }, + unit?.partition, + ); for await (const page of this.executor.readPages(plan.access.kind, input, budget)) { total += page.count; @@ -702,8 +756,8 @@ export default class DynamoDBCollection extends BaseCollection { ]; const aggregator = new StreamingAggregator(aggregation, timezone); - await this.inSegments(budget, segmentsOf(), async segment => { - for await (const batch of this.readBatches(plan, fields, budget, segment)) { + await this.inSegments(budget, segmentsOf(), async unit => { + for await (const batch of this.readBatches(plan, fields, budget, unit)) { for (const item of batch) { if (!plan.residual || this.matches(plan.residual, item, timezone)) { aggregator.add(this.toRecord(item)); diff --git a/packages/datasource-dynamodb/src/utils/access-path.ts b/packages/datasource-dynamodb/src/utils/access-path.ts index 039b832b87..acdd0d2d30 100644 --- a/packages/datasource-dynamodb/src/utils/access-path.ts +++ b/packages/datasource-dynamodb/src/utils/access-path.ts @@ -25,7 +25,8 @@ export type AccessPath = { /** Key attributes of the table or index read: a Query filter cannot reference them. */ keyAttributes?: string[]; keyCondition?: { - partition: { attribute: string; value: string | number }; + /** `values` lists every partition to read, when the filter names several: one Query each. */ + partition: { attribute: string; value: string | number; values?: Array }; sort?: SortKeyCondition; }; /** What the key condition leaves for a filter expression. */ @@ -45,6 +46,9 @@ type Candidate = { coversAllItems: boolean; }; +/** Partitions a single filter may fan out to: each one is a request of its own. */ +const MAX_PARTITIONS = 50; + const RANGE_OPERATORS = ['LessThan', 'LessThanOrEqual', 'GreaterThan', 'GreaterThanOrEqual']; const isKeyValue = (value: unknown): value is string | number => @@ -193,12 +197,28 @@ export function planAccessPath( let best: { path: AccessPath; score: number } | undefined; for (const candidate of candidates) { - const partition = leaves.find( - leaf => - leaf.field === candidate.partitionKey.name && - leaf.operator === 'Equal' && - isKeyValue(leaf.value), - ); + const onPartitionKey = leaves.filter(leaf => leaf.field === candidate.partitionKey.name); + const equal = onPartitionKey.find(leaf => leaf.operator === 'Equal' && isKeyValue(leaf.value)); + const partition = + equal ?? + onPartitionKey.find( + leaf => + leaf.operator === 'In' && + Array.isArray(leaf.value) && + leaf.value.length > 0 && + leaf.value.length <= MAX_PARTITIONS && + leaf.value.every(isKeyValue), + ); + const partitionValues = partition + ? [ + ...new Set( + (partition.operator === 'Equal' + ? [partition.value] + : (partition.value as unknown[]) + ).map(value => coerceKeyValue(candidate.partitionKey, value) as string | number), + ), + ] + : []; if (partition) { const picked = candidate.sortKey ? pickSortCondition(candidate.sortKey, parts) : undefined; @@ -210,7 +230,11 @@ export function planAccessPath( } const sortClause = sort?.length === 1 ? sort[0] : undefined; - const sortSatisfied = !!sortClause && sortClause.field === candidate.sortKey?.name; + // Several partitions are read one after the other: their items are not in one key order. + const sortSatisfied = + !!sortClause && + sortClause.field === candidate.sortKey?.name && + partitionValues.length === 1; const consumed = new Set([partition, ...(picked?.consumed ?? [])]); const score = (picked ? 2 : 0) + (sortSatisfied ? 1 : 0); @@ -224,7 +248,8 @@ export function planAccessPath( keyCondition: { partition: { attribute: candidate.partitionKey.name, - value: coerceKeyValue(candidate.partitionKey, partition.value) as string | number, + value: partitionValues[0], + ...(partitionValues.length > 1 ? { values: partitionValues } : {}), }, sort: picked?.condition, }, diff --git a/packages/datasource-dynamodb/test/fan-out-and-warnings.test.ts b/packages/datasource-dynamodb/test/fan-out-and-warnings.test.ts new file mode 100644 index 0000000000..baaf2d4800 --- /dev/null +++ b/packages/datasource-dynamodb/test/fan-out-and-warnings.test.ts @@ -0,0 +1,124 @@ +import { Aggregation, Filter, PaginatedFilter, Projection } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { buildModel, buildSimpleModel, leaf } from './helpers'; + +const caller = { timezone: 'UTC' } as never; +const order = (customerId: string, n: number) => ({ customerId, orderId: `o${n}`, total: n }); + +describe('an In on the partition key', () => { + it('reads each partition with a Query and merges the items', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: (input: any) => + input.ExpressionAttributeValues[':v0'] === 'c1' + ? { Items: [order('c1', 1), order('c1', 2)] } + : { Items: [order('c2', 3)] }, + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: leaf('customerId', 'In', ['c1', 'c2']), + page: { skip: 1, limit: 2 } as never, + }), + new Projection('orderId'), + ); + + expect(records).toEqual([{ orderId: 'o2' }, { orderId: 'o3' }]); + expect(client.calls('Query').map(call => call.ExpressionAttributeValues[':v0'])).toEqual([ + 'c1', + 'c2', + ]); + expect(client.calls('Scan')).toHaveLength(0); + }); + + it('counts the partitions without reading items', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: (input: any) => ({ Count: input.ExpressionAttributeValues[':v0'] === 'c1' ? 4 : 3 }), + }); + + const result = await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('customerId', 'In', ['c1', 'c2']) }), + new Aggregation({ operation: 'Count' }), + ); + + expect(result).toEqual([{ value: 7, group: {} }]); + expect(client.calls('Query').every(call => call.Select === 'COUNT')).toBe(true); + }); + + it('aggregates the items of every partition', async () => { + const { collection } = buildCollection(buildModel(), { + Query: (input: any) => ({ + Items: + input.ExpressionAttributeValues[':v0'] === 'c1' ? [order('c1', 1)] : [order('c2', 5)], + }), + }); + + const result = await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('customerId', 'In', ['c1', 'c2']) }), + new Aggregation({ operation: 'Sum', field: 'total' }), + ); + + expect(result).toEqual([{ value: 6, group: {} }]); + }); +}); + +describe('scan warnings', () => { + const scanning = () => ({ Scan: () => ({ Items: [] }) }); + + it('does not warn for the first page of a collection read without a filter', async () => { + const { collection, logger } = buildCollection(buildSimpleModel(), scanning()); + + await collection.list( + caller, + new PaginatedFilter({ page: { skip: 0, limit: 15 } as never }), + new Projection('id'), + ); + + expect(logger).not.toHaveBeenCalledWith('Warn', expect.anything()); + }); + + it('still refuses that page under a deny policy only when it is deep', async () => { + const { collection } = buildCollection(buildSimpleModel({ scanPolicy: 'deny' }), scanning()); + + await expect( + collection.list( + caller, + new PaginatedFilter({ page: { skip: 0, limit: 15 } as never }), + new Projection('id'), + ), + ).resolves.toEqual([]); + await expect( + collection.list( + caller, + new PaginatedFilter({ page: { skip: 5000, limit: 15 } as never }), + new Projection('id'), + ), + ).rejects.toThrow("refused by scanPolicy 'deny'"); + }); + + it('warns once a minute for the same scan', async () => { + let now = 0; + const { collection, logger } = buildCollection(buildSimpleModel(), scanning(), { + now: () => now, + }); + const list = () => + collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('name', 'Equal', 'a') }), + new Projection('id'), + ); + const warnings = () => logger.mock.calls.filter(([level]) => level === 'Warn').length; + + await list(); + now = 59000; + await list(); + expect(warnings()).toBe(1); + + now = 61000; + await list(); + expect(warnings()).toBe(2); + }); +}); diff --git a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts index ad92014aba..fb9e740452 100644 --- a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts +++ b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts @@ -343,6 +343,33 @@ run('DynamoDB Local', () => { expect(records.map(r => r.sk)).toEqual(['USER#2']); }); + it('queries each partition of an In on the partition key', async () => { + logger.mockClear(); + + const records = await orders().list( + caller, + new PaginatedFilter({ + conditionTree: leaf('customerId', 'In', ['c1', 'c2']), + page: { skip: 18, limit: 4 } as never, + }), + new Projection('customerId', 'orderId'), + ); + const count = await orders().aggregate( + caller, + new Filter({ conditionTree: leaf('customerId', 'In', ['c1', 'c2']) }), + new Aggregation({ operation: 'Count' }), + ); + + expect(records.map(r => `${r.customerId}${r.orderId}`)).toEqual([ + 'c1o18', + 'c1o19', + 'c2o20', + 'c2o21', + ]); + expect(count).toEqual([{ value: 30, group: {} }]); + expect(logger).not.toHaveBeenCalledWith('Warn', expect.stringContaining('Scan on table')); + }); + it('reads whole keys with BatchGetItem', async () => { const records = await orders().list( caller, diff --git a/packages/datasource-dynamodb/test/utils/access-path.test.ts b/packages/datasource-dynamodb/test/utils/access-path.test.ts index 0b0e935b04..bb88498bda 100644 --- a/packages/datasource-dynamodb/test/utils/access-path.test.ts +++ b/packages/datasource-dynamodb/test/utils/access-path.test.ts @@ -351,3 +351,67 @@ describe('planKeyLookup soundness', () => { expect(planKeyLookup(buildModel(), tree).keys).toEqual([{ customerId: 'c1', orderId: 'o1' }]); }); }); + +describe('a filter naming several partitions', () => { + it('queries each partition of an In on the partition key', () => { + const plan = planAccessPath(buildModel(), leaf('customerId', 'In', ['c1', 'c2', 'c1'])); + + expect(plan.kind).toBe('query'); + expect(plan.keyCondition.partition).toEqual({ + attribute: 'customerId', + value: 'c1', + values: ['c1', 'c2'], + }); + expect(plan.remaining).toBeUndefined(); + }); + + it('reads a single value In as one partition', () => { + const plan = planAccessPath(buildModel(), leaf('customerId', 'In', ['c1'])); + + expect(plan.keyCondition.partition).toEqual({ attribute: 'customerId', value: 'c1' }); + }); + + it('prefers an equality to an In on the same key', () => { + const plan = planAccessPath( + buildModel(), + and(plain('customerId', 'In', ['c1', 'c2']), plain('customerId', 'Equal', 'c2')), + ); + + expect(plan.keyCondition.partition.values).toBeUndefined(); + expect(plan.keyCondition.partition.value).toBe('c2'); + }); + + it('keeps the sort key range for every partition', () => { + const plan = planAccessPath( + buildModel(), + and(plain('customerId', 'In', ['c1', 'c2']), plain('orderId', 'GreaterThan', 'o5')), + ); + + expect(plan.keyCondition.sort).toMatchObject({ operator: 'GreaterThan', value: 'o5' }); + }); + + it('does not claim a sort order across partitions', () => { + const plan = planAccessPath( + buildModel(), + leaf('customerId', 'In', ['c1', 'c2']), + new Sort({ field: 'orderId', ascending: true }), + ); + + expect(plan.sortSatisfied).toBe(false); + }); + + it('scans instead of sending dozens of queries for a long list', () => { + const values = Array.from({ length: 51 }, (_, i) => `c${i}`); + + expect(planAccessPath(buildModel(), leaf('customerId', 'In', values)).kind).toBe('scan'); + }); + + it('types the values like the key', () => { + const plan = planAccessPath( + buildModel({ partitionKey: { name: 'accountId', type: 'N' } }), + leaf('accountId', 'In', ['1', 2]), + ); + + expect(plan.keyCondition.partition.values).toEqual([1, 2]); + }); +}); From d233523d4522593b4090648734152a8c9a631d16 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Fri, 16 Oct 2026 20:17:23 +0200 Subject: [PATCH 09/32] style(datasource-dynamodb): fix the lint errors the CI reported Refs: PRD-1670 Co-Authored-By: Claude Sonnet 5.5 --- packages/datasource-dynamodb/src/collection.ts | 4 +++- .../datasource-dynamodb/src/utils/access-path.ts | 14 +++++++------- .../datasource-dynamodb/src/utils/aggregator.ts | 1 + .../test/array-review-findings.test.ts | 6 +++--- .../test/review-findings.test.ts | 2 +- .../datasource-dynamodb/test/utils/compare.test.ts | 5 ++--- 6 files changed, 17 insertions(+), 15 deletions(-) diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index 4028d7fed5..e8a1ae2c30 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -31,9 +31,9 @@ import { DynamoDBOperationError, OffsetTooDeepError, ScanDeniedError } from './e import convertModelToFields from './schema/schema-converter'; import { planAccessPath, planKeyLookup } from './utils/access-path'; import StreamingAggregator from './utils/aggregator'; +import { mapWithConcurrency } from './utils/concurrency'; import DynamoExecutor from './utils/executor'; import ExpressionBuilder, { splitCondition } from './utils/expression-builder'; -import { mapWithConcurrency } from './utils/concurrency'; import Serializer from './utils/serializer'; import compareRecords from './utils/sort'; @@ -557,6 +557,7 @@ export default class DynamoDBCollection extends BaseCollection { } const item = Serializer.unflatten(created); + if (entity?.attribute !== undefined) { if (item[entity.attribute] !== undefined && item[entity.attribute] !== entity.value) { throw new ValidationError( @@ -716,6 +717,7 @@ export default class DynamoDBCollection extends BaseCollection { const budget = this.executor.newBudget(); const segments = plan.access?.kind === 'scan' ? Math.max(this.settings.scanConcurrency, 1) : 1; const partitions = plan.access?.keyCondition?.partition.values; + const segmentsOf = (): Array => { if (partitions) return partitions.map(partition => ({ partition })); diff --git a/packages/datasource-dynamodb/src/utils/access-path.ts b/packages/datasource-dynamodb/src/utils/access-path.ts index acdd0d2d30..7b6a33a6be 100644 --- a/packages/datasource-dynamodb/src/utils/access-path.ts +++ b/packages/datasource-dynamodb/src/utils/access-path.ts @@ -220,15 +220,15 @@ export function planAccessPath( ] : []; - if (partition) { - const picked = candidate.sortKey ? pickSortCondition(candidate.sortKey, parts) : undefined; + const picked = + partition && candidate.sortKey ? pickSortCondition(candidate.sortKey, parts) : undefined; - // An index leaves out the items that lack its sort key: without a condition on that key, it - // could return fewer items than the table holds. - if (candidate.indexName && candidate.sortKey && !picked && !candidate.coversAllItems) { - continue; - } + // An index leaves out the items that lack its sort key: without a condition on that key, it + // could return fewer items than the table holds. + const mayMissItems = + !!candidate.indexName && !!candidate.sortKey && !picked && !candidate.coversAllItems; + if (partition && !mayMissItems) { const sortClause = sort?.length === 1 ? sort[0] : undefined; // Several partitions are read one after the other: their items are not in one key order. const sortSatisfied = diff --git a/packages/datasource-dynamodb/src/utils/aggregator.ts b/packages/datasource-dynamodb/src/utils/aggregator.ts index 126b67e245..50cd97a4bd 100644 --- a/packages/datasource-dynamodb/src/utils/aggregator.ts +++ b/packages/datasource-dynamodb/src/utils/aggregator.ts @@ -125,6 +125,7 @@ export default class StreamingAggregator { if (value === undefined || value === null) return; accumulator.values += 1; + if (accumulator.min === undefined || compareValues(value, accumulator.min) < 0) { accumulator.min = value; } diff --git a/packages/datasource-dynamodb/test/array-review-findings.test.ts b/packages/datasource-dynamodb/test/array-review-findings.test.ts index d38c529bd8..a16971f0f4 100644 --- a/packages/datasource-dynamodb/test/array-review-findings.test.ts +++ b/packages/datasource-dynamodb/test/array-review-findings.test.ts @@ -3,10 +3,10 @@ import type { DataSource } from '@forestadmin/datasource-toolkit'; import { Filter, PaginatedFilter, Projection } from '@forestadmin/datasource-toolkit'; import { buildCollection } from './collection-helpers'; -import { buildModel, leaf } from './helpers'; -import ArrayCollection from '../src/array-collection'; -import { createDynamoDBDataSource } from '../src'; import { fakeClient } from './fake-client'; +import { buildModel } from './helpers'; +import { createDynamoDBDataSource } from '../src'; +import ArrayCollection from '../src/array-collection'; const caller = { timezone: 'UTC' } as never; diff --git a/packages/datasource-dynamodb/test/review-findings.test.ts b/packages/datasource-dynamodb/test/review-findings.test.ts index c12186c054..48f9e6f24e 100644 --- a/packages/datasource-dynamodb/test/review-findings.test.ts +++ b/packages/datasource-dynamodb/test/review-findings.test.ts @@ -1,9 +1,9 @@ import { + Aggregation, Filter, PaginatedFilter, Projection, Sort, - Aggregation, } from '@forestadmin/datasource-toolkit'; import { buildCollection } from './collection-helpers'; diff --git a/packages/datasource-dynamodb/test/utils/compare.test.ts b/packages/datasource-dynamodb/test/utils/compare.test.ts index 5e9675a7e2..c22f662d61 100644 --- a/packages/datasource-dynamodb/test/utils/compare.test.ts +++ b/packages/datasource-dynamodb/test/utils/compare.test.ts @@ -1,9 +1,8 @@ -import { Aggregation } from '@forestadmin/datasource-toolkit'; +import { Aggregation, Sort } from '@forestadmin/datasource-toolkit'; -import compareValues from '../../src/utils/compare'; import StreamingAggregator from '../../src/utils/aggregator'; +import compareValues from '../../src/utils/compare'; import compareRecords from '../../src/utils/sort'; -import { Sort } from '@forestadmin/datasource-toolkit'; describe('compareValues', () => { it('orders date-times by the instant, not by their text', () => { From cbc84289da9d8c069c3d74ef79f6a48548c07341 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Fri, 16 Oct 2026 20:19:12 +0200 Subject: [PATCH 10/32] fix(datasource-dynamodb): address the second round of review findings - a map seen empty in some items is no longer both a column and its nested columns - a field of a manual schema cannot change the type of a key - a key lookup never builds the product of long lists, and lists each value once - date-only strings are compared by instant too - Before and After are evaluated on the items, since DynamoDB compares dates as text Refs: PRD-1670 Co-Authored-By: Claude Sonnet 5.5 --- .../src/introspection/introspector.ts | 8 ++++++- .../src/schema/manual-schema.ts | 3 ++- .../src/utils/access-path.ts | 18 +++++++++++--- .../datasource-dynamodb/src/utils/compare.ts | 2 +- .../src/utils/expression-builder.ts | 2 -- .../test/introspection/introspector.test.ts | 21 ++++++++++++++++ .../test/schema/manual-schema.test.ts | 14 +++++++++++ .../test/utils/access-path.test.ts | 23 ++++++++++++++++++ .../test/utils/compare.test.ts | 5 ++++ .../test/utils/expression-builder.test.ts | 24 +++++++++---------- 10 files changed, 100 insertions(+), 20 deletions(-) diff --git a/packages/datasource-dynamodb/src/introspection/introspector.ts b/packages/datasource-dynamodb/src/introspection/introspector.ts index 9ee6e4aaa6..dbe63b3307 100644 --- a/packages/datasource-dynamodb/src/introspection/introspector.ts +++ b/packages/datasource-dynamodb/src/introspection/introspector.ts @@ -151,7 +151,13 @@ export function inferSchema( const schema: Record = Object.create(null); - for (const [path, observed] of Object.entries(types)) { + // A map seen empty or null in some items and populated in others was recorded as a value and + // as nested attributes: only the nested ones are columns, the map itself is expanded. + const paths = Object.keys(types); + const hasNested = (path: string) => + paths.some(other => other.startsWith(`${path}${PATH_SEPARATOR}`)); + + for (const [path, observed] of Object.entries(types).filter(([path]) => !hasNested(path))) { schema[path] = { type: TypeConverter.getMostSpecificType(observed), nullable: observed.includes('null') || presence[path] < items.length, diff --git a/packages/datasource-dynamodb/src/schema/manual-schema.ts b/packages/datasource-dynamodb/src/schema/manual-schema.ts index 23459542c5..a75d34567b 100644 --- a/packages/datasource-dynamodb/src/schema/manual-schema.ts +++ b/packages/datasource-dynamodb/src/schema/manual-schema.ts @@ -210,7 +210,8 @@ function toModel(collection: CollectionDefinition): TableModel { partitionKey, sortKey, indexes, - schema: { ...keys, ...flattenFields(collection.fields) }, + // The configured key types are the table's: a field of the same name cannot change them. + schema: { ...flattenFields(collection.fields), ...keys }, entity: collection.entity, enableCount: collection.enableCount, scanPolicy: collection.scanPolicy, diff --git a/packages/datasource-dynamodb/src/utils/access-path.ts b/packages/datasource-dynamodb/src/utils/access-path.ts index 7b6a33a6be..f8ef924bb0 100644 --- a/packages/datasource-dynamodb/src/utils/access-path.ts +++ b/packages/datasource-dynamodb/src/utils/access-path.ts @@ -46,6 +46,9 @@ type Candidate = { coversAllItems: boolean; }; +/** Keys a lookup may name: a longer list is read some other way than one BatchGetItem. */ +const MAX_LOOKUP_KEYS = 10000; + /** Partitions a single filter may fan out to: each one is a request of its own. */ const MAX_PARTITIONS = 50; @@ -329,15 +332,24 @@ export function planKeyLookup(model: TableModel, conditionTree?: ConditionTree): return { leaf, - values: ( - (leaf.operator === 'Equal' ? [leaf.value] : leaf.value) as Array - ).map(value => coerceKeyValue(attribute, value)), + values: [ + ...new Set( + ((leaf.operator === 'Equal' ? [leaf.value] : leaf.value) as Array).map( + value => coerceKeyValue(attribute, value), + ), + ), + ], }; }; const resolved = keyAttributes.map(attribute => ({ attribute, found: valuesOf(attribute) })); if (resolved.every(({ found }) => found)) { + // The keys are the product of the lists: a huge one is never built. + const product = resolved.reduce((count, { found }) => count * found.values.length, 1); + + if (product > MAX_LOOKUP_KEYS) return null; + const keys = resolved.reduce>>( (combined, { attribute, found }) => combined.flatMap(key => found.values.map(value => ({ ...key, [attribute.name]: value }))), diff --git a/packages/datasource-dynamodb/src/utils/compare.ts b/packages/datasource-dynamodb/src/utils/compare.ts index e28645fcb4..b795ddf9c4 100644 --- a/packages/datasource-dynamodb/src/utils/compare.ts +++ b/packages/datasource-dynamodb/src/utils/compare.ts @@ -1,4 +1,4 @@ -const ISO_DATE_TIME = /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:?\d{2})?$/; +const ISO_DATE_TIME = /^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:?\d{2})?)?$/; /** Orders two values, date-times by the instant they stand for rather than by their text. */ export default function compareValues(left: unknown, right: unknown): number { diff --git a/packages/datasource-dynamodb/src/utils/expression-builder.ts b/packages/datasource-dynamodb/src/utils/expression-builder.ts index 86cc95346c..4638e7b2c6 100644 --- a/packages/datasource-dynamodb/src/utils/expression-builder.ts +++ b/packages/datasource-dynamodb/src/utils/expression-builder.ts @@ -18,8 +18,6 @@ const COMPARISONS: Partial> = { LessThanOrEqual: '<=', GreaterThan: '>', GreaterThanOrEqual: '>=', - Before: '<', - After: '>', }; /** What DynamoDB evaluates itself; any other operator is applied on the items it returns. */ diff --git a/packages/datasource-dynamodb/test/introspection/introspector.test.ts b/packages/datasource-dynamodb/test/introspection/introspector.test.ts index 06e61224b1..be7c553c30 100644 --- a/packages/datasource-dynamodb/test/introspection/introspector.test.ts +++ b/packages/datasource-dynamodb/test/introspection/introspector.test.ts @@ -151,6 +151,27 @@ describe('introspectTable', () => { expect(schema['a->b->c']).toBeUndefined(); }); + it('exposes only the nested columns of a map seen empty in some items', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ + Items: [ + { customerId: 'c', orderId: 'o1', address: null }, + { customerId: 'c', orderId: 'o2', address: {} }, + { customerId: 'c', orderId: 'o3', address: { city: 'Paris' } }, + ], + }), + }); + + const { schema } = await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + }); + + expect(schema.address).toBeUndefined(); + expect(schema['address->city']).toMatchObject({ type: 'string', nullable: true }); + }); + it('declares the keys even when the table is empty', async () => { const client = fakeClient({ DescribeTable: () => describeTable(), diff --git a/packages/datasource-dynamodb/test/schema/manual-schema.test.ts b/packages/datasource-dynamodb/test/schema/manual-schema.test.ts index c1fa7d27c7..ca03b878d2 100644 --- a/packages/datasource-dynamodb/test/schema/manual-schema.test.ts +++ b/packages/datasource-dynamodb/test/schema/manual-schema.test.ts @@ -122,6 +122,20 @@ describe('convertManualSchema', () => { }); }); +describe('keys and fields of the same name', () => { + it('keeps the type configured for the key', () => { + const [model] = convertManualSchema( + config({ + partitionKey: { name: 'id', type: 'number' }, + sortKey: undefined, + fields: [{ name: 'id', type: 'string' }], + }), + ); + + expect(model.schema.id).toEqual({ type: 'number', nullable: false }); + }); +}); + describe('validateManualSchema', () => { it('accepts a valid schema', () => { expect(() => validateManualSchema(config())).not.toThrow(); diff --git a/packages/datasource-dynamodb/test/utils/access-path.test.ts b/packages/datasource-dynamodb/test/utils/access-path.test.ts index bb88498bda..f9a06667e1 100644 --- a/packages/datasource-dynamodb/test/utils/access-path.test.ts +++ b/packages/datasource-dynamodb/test/utils/access-path.test.ts @@ -415,3 +415,26 @@ describe('a filter naming several partitions', () => { expect(plan.keyCondition.partition.values).toEqual([1, 2]); }); }); + +describe('planKeyLookup bounds', () => { + const composite = () => buildModel(); + + it('lists each value once before building the keys', () => { + const tree = and( + plain('customerId', 'In', ['c1', 'c1', 'c2']), + plain('orderId', 'In', ['o1', 'o1']), + ); + + expect(planKeyLookup(composite(), tree).keys).toEqual([ + { customerId: 'c1', orderId: 'o1' }, + { customerId: 'c2', orderId: 'o1' }, + ]); + }); + + it('does not build the product of two long lists, which would be millions of keys', () => { + const many = (prefix: string) => Array.from({ length: 10000 }, (_, i) => `${prefix}${i}`); + const tree = and(plain('customerId', 'In', many('c')), plain('orderId', 'In', many('o'))); + + expect(planKeyLookup(composite(), tree)).toBeNull(); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/compare.test.ts b/packages/datasource-dynamodb/test/utils/compare.test.ts index c22f662d61..7add257407 100644 --- a/packages/datasource-dynamodb/test/utils/compare.test.ts +++ b/packages/datasource-dynamodb/test/utils/compare.test.ts @@ -9,6 +9,11 @@ describe('compareValues', () => { expect(compareValues('2024-01-01T00:30:00+02:00', '2023-12-31T23:00:00Z')).toBe(-1); }); + it('compares a date without time as the instant it starts at, in UTC', () => { + expect(compareValues('2024-01-01', '2024-01-01T00:30:00+02:00')).toBe(1); + expect(compareValues('2024-01-01', '2024-01-01T00:00:00Z')).toBe(0); + }); + it('treats the same instant written two ways as equal', () => { expect(compareValues('2024-01-01T02:00:00+02:00', '2024-01-01T00:00:00Z')).toBe(0); }); diff --git a/packages/datasource-dynamodb/test/utils/expression-builder.test.ts b/packages/datasource-dynamodb/test/utils/expression-builder.test.ts index 4166466414..f0e104c234 100644 --- a/packages/datasource-dynamodb/test/utils/expression-builder.test.ts +++ b/packages/datasource-dynamodb/test/utils/expression-builder.test.ts @@ -153,18 +153,6 @@ describe('ExpressionBuilder', () => { expect(builder.values).toEqual({ ':v0': 'NULL', ':v1': '' }); }); - it('translates Before and After on a date as comparisons on its ISO string', () => { - const builder = new ExpressionBuilder(); - const date = new Date('2024-01-15T10:00:00.000Z'); - - expect(builder.condition(leaf('at', 'Before', date))).toBe('#n0 < :v0'); - expect(builder.condition(leaf('at', 'After', date))).toBe('#n0 > :v1'); - expect(builder.values).toEqual({ - ':v0': '2024-01-15T10:00:00.000Z', - ':v1': '2024-01-15T10:00:00.000Z', - }); - }); - it('requires none of the values for IncludesNone', () => { const builder = new ExpressionBuilder(); @@ -243,6 +231,18 @@ describe('ExpressionBuilder', () => { }); describe('splitCondition', () => { + it.each(['Before', 'After'])( + 'leaves %s to the items, since DynamoDB compares dates as text whatever their offset', + operator => { + const tree = leaf('at', operator, '2024-01-01T00:00:00Z'); + + const { pushable, residual } = splitCondition(tree); + + expect(pushable).toBeUndefined(); + expect(residual).toBe(tree); + }, + ); + it('leaves Present to the items, since the toolkit also excludes empty strings from it', () => { const tree = leaf('f', 'Present'); From d890826bd51aa41f3344ff332fe7050ca179d814 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Fri, 16 Oct 2026 20:29:17 +0200 Subject: [PATCH 11/32] fix(datasource-dynamodb): address the third round of review findings - chart caches are shared by every collection of a table and hold copies - array collections refuse key columns with the same name, create and update under a map the item lacks by rewriting it, and replace what a patch names - attributes named with the path separator are refused, not silently merged - a create returns the discriminator it stored, a first page of an entity collection no longer bypasses scanPolicy, and a reversed range is not a BETWEEN Refs: PRD-1670 Co-Authored-By: Claude Sonnet 5.5 --- packages/datasource-dynamodb/README.md | 5 +- .../src/array-collection.ts | 96 ++++++- .../datasource-dynamodb/src/collection.ts | 37 ++- .../src/introspection/introspector.ts | 8 +- .../src/schema/manual-schema.ts | 7 + packages/datasource-dynamodb/src/settings.ts | 1 + packages/datasource-dynamodb/src/types.ts | 4 +- .../src/utils/access-path.ts | 2 +- .../src/utils/serializer.ts | 12 + .../test/second-round-findings.test.ts | 255 ++++++++++++++++++ 10 files changed, 404 insertions(+), 23 deletions(-) create mode 100644 packages/datasource-dynamodb/test/second-round-findings.test.ts diff --git a/packages/datasource-dynamodb/README.md b/packages/datasource-dynamodb/README.md index 5fdaabc662..d9004c1aee 100644 --- a/packages/datasource-dynamodb/README.md +++ b/packages/datasource-dynamodb/README.md @@ -104,8 +104,9 @@ createDynamoDBDataSource({ }); ``` -Dates are compared by their instant when sorted or aggregated here, but DynamoDB compares them as -text when it filters: store them in one offset (UTC) if you filter on ranges of them. +Dates are compared by their instant when sorted or aggregated here, but DynamoDB orders and compares +them as text on a key and when it filters: store them in one offset (UTC) if a date is a sort key or +you filter on ranges of it. Types: `string`, `number`, `boolean`, `date` and `dateonly` (ISO 8601 strings), `timeonly`, `object`, `array`, `point` (GeoJSON), `binary` (base64). Keys are strings or numbers. diff --git a/packages/datasource-dynamodb/src/array-collection.ts b/packages/datasource-dynamodb/src/array-collection.ts index 5cb48d31a1..5072aee96d 100644 --- a/packages/datasource-dynamodb/src/array-collection.ts +++ b/packages/datasource-dynamodb/src/array-collection.ts @@ -98,6 +98,11 @@ export default class ArrayCollection extends BaseCollection { this.chain = [...parentLevels, { arrayField: config.arrayField, indexColumn: INDEX_FIELD }]; this.keyColumns = [...this.rootKeys, ...this.chain.map(level => level.indexColumn)]; + // A key named like a position column would be overwritten by it, and so address another item. + if (new Set(this.keyColumns).size !== this.keyColumns.length) { + throw new ValidationError(`The key columns of '${config.name}' must have distinct names`); + } + const schema: Record = { ...config.itemSchema, ...Object.fromEntries( @@ -322,10 +327,22 @@ export default class ArrayCollection extends BaseCollection { ReturnValues: 'UPDATED_NEW', }); } catch (error) { - if (error.name === 'ConditionalCheckFailedException') { - throw new ValidationError( - `The parent record of '${this.name}' does not exist in '${this.parent.name}'`, - ); + if (error.name === 'ConditionalCheckFailedException') throw this.missingParent(); + + // The array sits under a map the item does not have yet, which SET cannot create. + if (error.name === 'ValidationException' && /document path/i.test(error.message)) { + const index = await this.appendByRewriting(key, parentPositions as number[], fields); + + return { + ...key, + ...Object.fromEntries( + parentKeyColumns + .slice(this.rootKeys.length) + .map((name, i) => [name, parentPositions[i]]), + ), + [INDEX_FIELD]: index, + ...fields, + }; } throw error; @@ -343,6 +360,70 @@ export default class ArrayCollection extends BaseCollection { }; } + private missingParent(): ValidationError { + return new ValidationError( + `The parent record of '${this.name}' does not exist in '${this.parent.name}'`, + ); + } + + /** Appends in the item itself, creating the maps on the way, for when DynamoDB cannot. */ + private async appendByRewriting( + key: Item, + parentPositions: number[], + fields: Record, + ): Promise { + const item = await this.root.executor.get(key); + + if (!item) throw this.missingParent(); + + let container: Item = item; + let index = -1; + + this.chain.forEach((level, depth) => { + const segments = Serializer.toPathSegments(level.arrayField); + const holder = segments.slice(0, -1).reduce((value, segment) => { + if (!value[segment]) value[segment] = {}; + + return value[segment] as Item; + }, container); + const last = segments[segments.length - 1]; + + if (depth === this.chain.length - 1) { + holder[last] ??= []; + (holder[last] as unknown[]).push(Serializer.unflatten(fields)); + index = (holder[last] as unknown[]).length - 1; + } else { + container = (holder[last] as Item[] | undefined)?.[parentPositions[depth]]; + + if (!container) throw this.missingParent(); + } + }); + + const keyBuilder = new ExpressionBuilder('k', 'kv'); + const builder = new ExpressionBuilder(); + const conditions = [`attribute_exists(${keyBuilder.attribute(this.rootKeys[0])})`]; + const entity = this.entityCondition(builder); + + if (entity) conditions.push(entity); + + try { + await this.root.executor.put({ + Item: item, + ConditionExpression: conditions.join(' AND '), + ExpressionAttributeNames: { ...builder.names, ...keyBuilder.names }, + ...(Object.keys(builder.values).length + ? { ExpressionAttributeValues: builder.values } + : {}), + }); + } catch (error) { + if (error.name === 'ConditionalCheckFailedException') throw this.missingParent(); + + throw error; + } + + return index; + } + /** UPDATED_NEW answers with only the arrays on the path, each holding just the updated element. */ private appendedLength(attributes: Item): number { let value: unknown = attributes; @@ -411,8 +492,6 @@ export default class ArrayCollection extends BaseCollection { if (!item) return; - const patch = Serializer.unflatten(Object.fromEntries(entries)); - for (const position of positions) { let container: Item = item; @@ -424,7 +503,10 @@ export default class ArrayCollection extends BaseCollection { const array = holder[segments[segments.length - 1]] as Item[]; if (depth === this.chain.length - 1) { - array[position[depth]] = Serializer.deepMerge(array[position[depth]], patch); + // Each attribute of the patch replaces what it names, as SET does, and nothing else. + for (const [field, value] of entries) { + Serializer.setPath(array[position[depth]], Serializer.toPathSegments(field), value); + } } else { container = array[position[depth]]; } diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index e8a1ae2c30..c0f6da7503 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -63,6 +63,9 @@ const hasOversizedList = (leaf: ConditionTreeLeaf): boolean => Array.isArray(leaf.value) && leaf.value.length > MAX_PUSHED_LIST_VALUES; +const copyOf = (rows: AggregateResult[]): AggregateResult[] => + rows.map(row => ({ ...row, group: { ...row.group } })); + const isEmptyKey = (value: unknown): boolean => value === undefined || value === null || value === ''; @@ -113,7 +116,6 @@ export default class DynamoDBCollection extends BaseCollection { private readonly fieldNames: Set; private readonly matchContext: Collection; private readonly warnedAt = new Map(); - private readonly chartCache = new Map(); constructor( datasource: DataSource, @@ -149,7 +151,7 @@ export default class DynamoDBCollection extends BaseCollection { invalidateCaches(): void { this.settings.paginationCache.invalidateTable(this.model.tableName); - this.chartCache.clear(); + this.settings.chartCaches.get(this.model.tableName)?.clear(); } async create(caller: Caller, data: RecordData[]): Promise { @@ -469,6 +471,7 @@ export default class DynamoDBCollection extends BaseCollection { const isCheapPage = !filter.conditionTree && !plan.residual && + !plan.access?.remaining && (!sort || plan.access?.sortSatisfied) && limit !== undefined && skip + limit <= MAX_CHEAP_PAGE_ITEMS; @@ -566,6 +569,7 @@ export default class DynamoDBCollection extends BaseCollection { } item[entity.attribute] = entity.value; + created[entity.attribute] = entity.value; } const keyBuilder = new ExpressionBuilder('k', 'kv'); @@ -648,6 +652,7 @@ export default class DynamoDBCollection extends BaseCollection { const { timezone } = caller; const { paginationCache: cache, chartCacheTtlMs, now } = this.settings; const cacheKey = cache.hash({ + collection: this.name, filter: filter.conditionTree?.toPlainObject(), aggregation: { field: aggregation.field, @@ -657,26 +662,36 @@ export default class DynamoDBCollection extends BaseCollection { limit, timezone, }); - const cached = this.chartCache.get(cacheKey); + const charts = this.chartsOfTable(); + const cached = charts.get(cacheKey); - if (cached && now() - cached.at < chartCacheTtlMs) { - return cached.value.map(row => ({ ...row, group: { ...row.group } })); - } + if (cached && now() - cached.at < chartCacheTtlMs) return copyOf(cached.value); const value = await this.computeAggregation(filter, aggregation, limit, timezone); if (chartCacheTtlMs > 0) { - this.chartCache.delete(cacheKey); - this.chartCache.set(cacheKey, { at: now(), value }); + // A copy: the caller may rename the groups it is given, which must not reach the cache. + charts.delete(cacheKey); + charts.set(cacheKey, { at: now(), value: copyOf(value) }); - if (this.chartCache.size > MAX_CACHED_CHARTS) { - this.chartCache.delete(this.chartCache.keys().next().value); - } + if (charts.size > MAX_CACHED_CHARTS) charts.delete(charts.keys().next().value); } return value; } + private chartsOfTable(): Map { + const { chartCaches } = this.settings; + let charts = chartCaches.get(this.model.tableName); + + if (!charts) { + charts = new Map(); + chartCaches.set(this.model.tableName, charts); + } + + return charts; + } + /** Reads the segments of a parallel scan at once; a failure stops the others from reading on. */ private async inSegments( budget: ReadBudget, diff --git a/packages/datasource-dynamodb/src/introspection/introspector.ts b/packages/datasource-dynamodb/src/introspection/introspector.ts index dbe63b3307..8389accb18 100644 --- a/packages/datasource-dynamodb/src/introspection/introspector.ts +++ b/packages/datasource-dynamodb/src/introspection/introspector.ts @@ -121,6 +121,12 @@ function collectTypes( if (entries.length === 0) record('object'); for (const [key, nested] of entries) { + if (key.includes(PATH_SEPARATOR)) { + throw new ValidationError( + `Attribute '${key}' contains '${PATH_SEPARATOR}', which names nested attributes`, + ); + } + collectTypes( nested, path ? `${path}${PATH_SEPARATOR}${key}` : key, @@ -157,7 +163,7 @@ export function inferSchema( const hasNested = (path: string) => paths.some(other => other.startsWith(`${path}${PATH_SEPARATOR}`)); - for (const [path, observed] of Object.entries(types).filter(([path]) => !hasNested(path))) { + for (const [path, observed] of Object.entries(types).filter(([name]) => !hasNested(name))) { schema[path] = { type: TypeConverter.getMostSpecificType(observed), nullable: observed.includes('null') || presence[path] < items.length, diff --git a/packages/datasource-dynamodb/src/schema/manual-schema.ts b/packages/datasource-dynamodb/src/schema/manual-schema.ts index a75d34567b..8504fc05b5 100644 --- a/packages/datasource-dynamodb/src/schema/manual-schema.ts +++ b/packages/datasource-dynamodb/src/schema/manual-schema.ts @@ -39,6 +39,13 @@ function validateFields(fields: FieldDefinition[], collectionName: string, path throw new ValidationError(`A field of collection '${collectionName}' has no name`); } + if (field.name.includes(PATH_SEPARATOR)) { + throw new ValidationError( + `Field '${fullPath}' in collection '${collectionName}' contains '${PATH_SEPARATOR}', ` + + `which names nested attributes`, + ); + } + if (names.has(field.name)) { throw new ValidationError( `Duplicate field name: '${fullPath}' in collection '${collectionName}'`, diff --git a/packages/datasource-dynamodb/src/settings.ts b/packages/datasource-dynamodb/src/settings.ts index 59293698b6..5e5e9bb5c0 100644 --- a/packages/datasource-dynamodb/src/settings.ts +++ b/packages/datasource-dynamodb/src/settings.ts @@ -21,5 +21,6 @@ export function resolveSettings(settings: Partial = {}): Col ...settings, now, paginationCache: settings.paginationCache ?? new PaginationCache({ now }), + chartCaches: settings.chartCaches ?? new Map(), }; } diff --git a/packages/datasource-dynamodb/src/types.ts b/packages/datasource-dynamodb/src/types.ts index 929e8e6aa9..e7b0324061 100644 --- a/packages/datasource-dynamodb/src/types.ts +++ b/packages/datasource-dynamodb/src/types.ts @@ -2,7 +2,7 @@ import type PaginationCache from './utils/pagination-cache'; import type { PaginationCacheOptions } from './utils/pagination-cache'; import type { DynamoDBClientConfig } from '@aws-sdk/client-dynamodb'; import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; -import type { FieldSchema } from '@forestadmin/datasource-toolkit'; +import type { AggregateResult, FieldSchema } from '@forestadmin/datasource-toolkit'; export type ScanPolicy = 'allow' | 'warn' | 'deny'; @@ -129,6 +129,8 @@ export type CollectionSettings = { logConsumedCapacity: boolean; logQueries: boolean; paginationCache: PaginationCache; + /** Charts by table then by request: every collection of a table drops them on a write. */ + chartCaches: Map>; now: () => number; }; diff --git a/packages/datasource-dynamodb/src/utils/access-path.ts b/packages/datasource-dynamodb/src/utils/access-path.ts index f8ef924bb0..584658c51c 100644 --- a/packages/datasource-dynamodb/src/utils/access-path.ts +++ b/packages/datasource-dynamodb/src/utils/access-path.ts @@ -142,7 +142,7 @@ function pickSortCondition( const [lower] = find(['GreaterThanOrEqual']); const [upper] = find(['LessThanOrEqual']); - if (lower && upper) { + if (lower && upper && !(valueOf(lower) > valueOf(upper))) { return { condition: { attribute, operator: 'Between', value: [valueOf(lower), valueOf(upper)] }, consumed: [lower, upper], diff --git a/packages/datasource-dynamodb/src/utils/serializer.ts b/packages/datasource-dynamodb/src/utils/serializer.ts index 3b1f2db4af..3088de390e 100644 --- a/packages/datasource-dynamodb/src/utils/serializer.ts +++ b/packages/datasource-dynamodb/src/utils/serializer.ts @@ -94,6 +94,18 @@ export default class Serializer { return item; } + /** Sets the value at a path, creating the maps on the way, and replaces what was there. */ + static setPath(target: Record, segments: string[], value: unknown): void { + let holder = target; + + for (const segment of segments.slice(0, -1)) { + if (!isPlainObject(holder[segment])) holder[segment] = {}; + holder = holder[segment] as Record; + } + + holder[segments[segments.length - 1]] = value instanceof Date ? value.toISOString() : value; + } + static toPathSegments(field: string): string[] { return field.split(PATH_SEPARATOR); } diff --git a/packages/datasource-dynamodb/test/second-round-findings.test.ts b/packages/datasource-dynamodb/test/second-round-findings.test.ts new file mode 100644 index 0000000000..c8110fe28e --- /dev/null +++ b/packages/datasource-dynamodb/test/second-round-findings.test.ts @@ -0,0 +1,255 @@ +import type { DataSource } from '@forestadmin/datasource-toolkit'; + +import { Aggregation, Filter, PaginatedFilter, Projection } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { fakeClient } from './fake-client'; +import { and, buildModel, buildSimpleModel, plain } from './helpers'; +import ArrayCollection from '../src/array-collection'; +import DynamoDBCollection from '../src/collection'; +import { introspectTables } from '../src/introspection/introspector'; +import { validateManualSchema } from '../src/schema/manual-schema'; +import { resolveSettings } from '../src/settings'; +import { planAccessPath } from '../src/utils/access-path'; + +const caller = { timezone: 'UTC' } as never; +const sum = new Aggregation({ operation: 'Sum', field: 'age' }); + +describe('chart caches', () => { + it('drops the charts of every collection of a table when one of them writes', async () => { + const client = fakeClient({ Scan: () => ({ Items: [{ age: 1 }] }), Put: () => ({}) }); + const settings = resolveSettings({ chartCacheTtlMs: 60000, scanConcurrency: 1 }); + const build = (name: string, entity?: { sortKeyPrefix: string }) => + new DynamoDBCollection( + {} as DataSource, + buildModel({ + name, + entity, + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + age: { type: 'number' }, + }, + }), + client, + jest.fn(), + settings, + ); + const whole = build('whole'); + const orders = build('orders', { sortKeyPrefix: 'ORDER#' }); + + await whole.aggregate(caller, new Filter({}), sum); + await orders.create(caller, [{ customerId: 'c', orderId: 'ORDER#1' }]); + await whole.aggregate(caller, new Filter({}), sum); + + expect(client.calls('Scan')).toHaveLength(2); + }); + + it('is not poisoned by a caller that renames the groups it is given', async () => { + const { collection } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Items: [{ age: 1, name: 'a' }] }) }, + { chartCacheTtlMs: 60000, scanConcurrency: 1 }, + ); + const byName = new Aggregation({ operation: 'Sum', field: 'age', groups: [{ field: 'name' }] }); + + const first = await collection.aggregate(caller, new Filter({}), byName); + first[0].group['relation:id'] = first[0].group.name; + delete first[0].group.name; + const second = await collection.aggregate(caller, new Filter({}), byName); + + expect(second).toEqual([{ value: 1, group: { name: 'a' } }]); + }); +}); + +describe('array collections', () => { + const parentWith = (handlers: Record, model = buildModel()) => { + const parent = buildCollection( + { + ...model, + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + lines: { type: 'array' }, + details: { type: 'object' }, + }, + }, + handlers, + { scanConcurrency: 1 }, + ); + + return { + client: parent.client, + parent: parent.collection, + lines: (arrayField = 'lines') => + new ArrayCollection({} as DataSource, parent.collection, { + name: 'order_lines', + arrayField, + itemSchema: { sku: { type: 'string' }, metadata: { type: 'object' } }, + }), + }; + }; + + it('refuses a table key named like the position column', () => { + const { lines } = parentWith({}, buildModel({ sortKey: { name: 'index', type: 'S' } })); + + expect(() => lines()).toThrow("The key columns of 'order_lines' must have distinct names"); + }); + + it('replaces an object attribute as a whole when it rewrites the item, as SET would', async () => { + const order = { + customerId: 'c1', + orderId: 'o1', + lines: [{ sku: 'A', metadata: { a: 1, b: 2 } }], + }; + const { lines, client } = parentWith({ + Scan: () => ({ Items: [order] }), + Update: () => { + throw Object.assign(new Error('The document path provided is invalid'), { + name: 'ValidationException', + }); + }, + Get: () => ({ Item: JSON.parse(JSON.stringify(order)) }), + Put: () => ({}), + }); + + await lines().update(caller, new Filter({}), { metadata: { a: 3 } }); + + expect(client.calls('Put')[0].Item.lines[0].metadata).toEqual({ a: 3 }); + }); + + it('creates the first element of an array held in a map the item does not have', async () => { + const { lines, client } = parentWith({ + Update: () => { + throw Object.assign(new Error('The document path provided is invalid'), { + name: 'ValidationException', + }); + }, + Get: () => ({ Item: { customerId: 'c1', orderId: 'o1' } }), + Put: () => ({}), + }); + + const [created] = await lines('details->lines').create(caller, [ + { customerId: 'c1', orderId: 'o1', sku: 'A' }, + ]); + + expect(created).toMatchObject({ index: 0, sku: 'A' }); + expect(client.calls('Put')[0].Item).toEqual({ + customerId: 'c1', + orderId: 'o1', + details: { lines: [{ sku: 'A' }] }, + }); + }); +}); + +describe('attribute names holding the path separator', () => { + it('skips a table with such an attribute, and says why', async () => { + const client = fakeClient({ + DescribeTable: () => ({ + Table: { + TableName: 'T', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }, + }), + Scan: () => ({ Items: [{ id: 'a', 'a->b': 1 }] }), + }); + const logger = jest.fn(); + + const models = await introspectTables(client, { tables: ['T'] }, logger); + + expect(models).toEqual([]); + expect(logger).toHaveBeenCalledWith( + 'Warn', + "Introspector - Skipping table 'T': Attribute 'a->b' contains '->', which names nested attributes", + ); + }); + + it('rejects a manual field named so', () => { + expect(() => + validateManualSchema({ + collections: [ + { + name: 'c', + tableName: 'T', + partitionKey: { name: 'pk', type: 'string' }, + fields: [{ name: 'a->b', type: 'string' }], + }, + ], + }), + ).toThrow("Field 'a->b' in collection 'c' contains '->'"); + }); +}); + +describe('single-table creation and reads', () => { + const entityModel = () => + buildModel({ + entity: { attribute: 'entityType', value: 'ORDER' }, + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + entityType: { type: 'string' }, + }, + }); + + it('returns the discriminator the record was stored with', async () => { + const { collection } = buildCollection(entityModel(), { Put: () => ({}) }); + + const [created] = await collection.create(caller, [{ customerId: 'c', orderId: 'o' }]); + + expect(created.entityType).toBe('ORDER'); + }); + + it('does not let a first page of an entity collection bypass a deny policy', async () => { + const { collection } = buildCollection( + { ...entityModel(), scanPolicy: 'deny' }, + { Scan: () => ({ Items: [] }) }, + ); + + await expect( + collection.list( + caller, + new PaginatedFilter({ page: { skip: 0, limit: 15 } as never }), + new Projection('orderId'), + ), + ).rejects.toThrow("refused by scanPolicy 'deny'"); + }); +}); + +describe('a reversed range on the sort key', () => { + it('is not sent as a BETWEEN, which DynamoDB rejects', () => { + const plan = planAccessPath( + buildModel(), + and( + plain('customerId', 'Equal', 'c1'), + plain('orderId', 'GreaterThanOrEqual', 'o9'), + plain('orderId', 'LessThanOrEqual', 'o1'), + ), + ); + + expect(plan.keyCondition.sort?.operator).not.toBe('Between'); + expect(plan.remaining.toPlainObject()).toEqual(plain('orderId', 'LessThanOrEqual', 'o1')); + }); + + it('matches nothing when read', async () => { + const { collection } = buildCollection(buildModel(), { + Query: () => ({ + Items: [{ customerId: 'c1', orderId: 'o9' }], + }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: and( + plain('customerId', 'Equal', 'c1'), + plain('orderId', 'GreaterThanOrEqual', 'o9'), + plain('orderId', 'LessThanOrEqual', 'o1'), + ), + }), + new Projection('orderId'), + ); + + expect(records).toEqual([]); + }); +}); From bd1d9a81772a36ee604b06e9c0472b7161fcec07 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Fri, 16 Oct 2026 20:36:05 +0200 Subject: [PATCH 12/32] fix(datasource-dynamodb): keep records from writing to __proto__ and read strongly before rewriting Refs: PRD-1670 Co-Authored-By: Claude Sonnet 5.5 --- .../datasource-dynamodb/src/utils/executor.ts | 8 +++- .../src/utils/serializer.ts | 22 +++++++++- .../test/second-round-findings.test.ts | 41 ++++++++++++++++++- 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/packages/datasource-dynamodb/src/utils/executor.ts b/packages/datasource-dynamodb/src/utils/executor.ts index fcf729a1e4..1a91b060e0 100644 --- a/packages/datasource-dynamodb/src/utils/executor.ts +++ b/packages/datasource-dynamodb/src/utils/executor.ts @@ -214,9 +214,15 @@ export default class DynamoExecutor { return response.Attributes; } + /** Strongly consistent, for the reads a write is built from: a stale copy would undo writes. */ async get(key: Item): Promise { const response = await this.client.send( - new GetCommand({ TableName: this.model.tableName, Key: key, ...this.capacityRequest() }), + new GetCommand({ + TableName: this.model.tableName, + Key: key, + ConsistentRead: true, + ...this.capacityRequest(), + }), ); this.logCapacity('get', response.ConsumedCapacity?.CapacityUnits); diff --git a/packages/datasource-dynamodb/src/utils/serializer.ts b/packages/datasource-dynamodb/src/utils/serializer.ts index 3088de390e..bc19e93690 100644 --- a/packages/datasource-dynamodb/src/utils/serializer.ts +++ b/packages/datasource-dynamodb/src/utils/serializer.ts @@ -1,5 +1,7 @@ import type { RecordData } from '@forestadmin/datasource-toolkit'; +import { ValidationError } from '@forestadmin/datasource-toolkit'; + import TypeConverter from './type-converter'; export const PATH_SEPARATOR = '->'; @@ -39,6 +41,18 @@ function isPlainObject(value: unknown): value is Record { ); } +/** Writing to `__proto__` would change what every object of the process inherits. */ +function assertWritable(segments: string[]): void { + if (segments.includes('__proto__')) { + throw new ValidationError("An attribute path cannot contain '__proto__'"); + } +} + +/** Own properties only: what an object inherits is not a map to write into. */ +function hasOwnMap(target: Record, key: string): boolean { + return Object.prototype.hasOwnProperty.call(target, key) && isPlainObject(target[key]); +} + export default class Serializer { /** Makes a value out of what the DocumentClient reads: no set, no date object, no buffer. */ static normalize(value: unknown): unknown { @@ -83,8 +97,10 @@ export default class Serializer { const parts = Serializer.toPathSegments(field); let target = item; + assertWritable(parts); + for (const part of parts.slice(0, -1)) { - if (!isPlainObject(target[part])) target[part] = {}; + if (!hasOwnMap(target, part)) target[part] = {}; target = target[part] as Record; } @@ -98,8 +114,10 @@ export default class Serializer { static setPath(target: Record, segments: string[], value: unknown): void { let holder = target; + assertWritable(segments); + for (const segment of segments.slice(0, -1)) { - if (!isPlainObject(holder[segment])) holder[segment] = {}; + if (!hasOwnMap(holder, segment)) holder[segment] = {}; holder = holder[segment] as Record; } diff --git a/packages/datasource-dynamodb/test/second-round-findings.test.ts b/packages/datasource-dynamodb/test/second-round-findings.test.ts index c8110fe28e..5f31582c8d 100644 --- a/packages/datasource-dynamodb/test/second-round-findings.test.ts +++ b/packages/datasource-dynamodb/test/second-round-findings.test.ts @@ -4,13 +4,14 @@ import { Aggregation, Filter, PaginatedFilter, Projection } from '@forestadmin/d import { buildCollection } from './collection-helpers'; import { fakeClient } from './fake-client'; -import { and, buildModel, buildSimpleModel, plain } from './helpers'; +import { and, buildModel, buildSimpleModel, leaf, plain } from './helpers'; import ArrayCollection from '../src/array-collection'; import DynamoDBCollection from '../src/collection'; import { introspectTables } from '../src/introspection/introspector'; import { validateManualSchema } from '../src/schema/manual-schema'; import { resolveSettings } from '../src/settings'; import { planAccessPath } from '../src/utils/access-path'; +import Serializer from '../src/utils/serializer'; const caller = { timezone: 'UTC' } as never; const sum = new Aggregation({ operation: 'Sum', field: 'age' }); @@ -253,3 +254,41 @@ describe('a reversed range on the sort key', () => { expect(records).toEqual([]); }); }); + +describe('writing records', () => { + it('refuses a path through __proto__ instead of changing every object', () => { + expect(() => Serializer.unflatten({ '__proto__->polluted': 'yes' })).toThrow( + "An attribute path cannot contain '__proto__'", + ); + expect(() => Serializer.setPath({}, ['__proto__', 'polluted'], 'yes')).toThrow( + "An attribute path cannot contain '__proto__'", + ); + expect(({} as Record).polluted).toBeUndefined(); + }); + + it('does not take an inherited property for a map to write into', () => { + const item = Serializer.unflatten({ 'constructor->name': 'x' }); + + expect(Object.keys(item)).toEqual(['constructor']); + expect((item.constructor as unknown as Record).name).toBe('x'); + expect(Object.prototype.constructor.name).toBe('Object'); + }); + + it('reads the item strongly consistent before rewriting it', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Update: () => { + throw Object.assign(new Error('The document path provided is invalid'), { + name: 'ValidationException', + }); + }, + Get: () => ({ Item: { id: 'u1' } }), + Put: () => ({}), + }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + 'address->city': 'Lyon', + }); + + expect(client.calls('Get')[0].ConsistentRead).toBe(true); + }); +}); From 331e9ecc789471ce71db7e61c4f25ac1f6442d18 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Fri, 16 Oct 2026 20:43:10 +0200 Subject: [PATCH 13/32] fix(datasource-dynamodb): group a date without time on its own day Refs: PRD-1670 Co-Authored-By: Claude Sonnet 5.5 --- .../src/utils/aggregator.ts | 24 ++++++++++++++++--- .../test/utils/aggregator.test.ts | 10 ++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/packages/datasource-dynamodb/src/utils/aggregator.ts b/packages/datasource-dynamodb/src/utils/aggregator.ts index 50cd97a4bd..2b78cec03b 100644 --- a/packages/datasource-dynamodb/src/utils/aggregator.ts +++ b/packages/datasource-dynamodb/src/utils/aggregator.ts @@ -60,12 +60,30 @@ function localDay(date: Date, timezone: string): { year: number; month: number; const two = (n: number) => String(n).padStart(2, '0'); -function dateBucket(value: unknown, operation: DateOperation, timezone: string): string | null { +const DATE_ONLY = /^(\d{4})-(\d{2})-(\d{2})$/; + +/** A date without time is a calendar day, which no timezone moves; an instant is read in one. */ +function calendarDay( + value: unknown, + timezone: string, +): { year: number; month: number; day: number } | null { + const dateOnly = typeof value === 'string' ? DATE_ONLY.exec(value) : null; + + if (dateOnly) { + return { year: Number(dateOnly[1]), month: Number(dateOnly[2]), day: Number(dateOnly[3]) }; + } + const date = toDate(value); - if (!date) return null; + return date ? localDay(date, timezone) : null; +} + +function dateBucket(value: unknown, operation: DateOperation, timezone: string): string | null { + const calendar = calendarDay(value, timezone); + + if (!calendar) return null; - const { year, month, day } = localDay(date, timezone); + const { year, month, day } = calendar; switch (operation) { case 'Year': diff --git a/packages/datasource-dynamodb/test/utils/aggregator.test.ts b/packages/datasource-dynamodb/test/utils/aggregator.test.ts index e508ab7f81..d8843010c7 100644 --- a/packages/datasource-dynamodb/test/utils/aggregator.test.ts +++ b/packages/datasource-dynamodb/test/utils/aggregator.test.ts @@ -184,6 +184,16 @@ describe('StreamingAggregator', () => { expect(result).toEqual([{ value: 2, group: { at: '2024-01-15' } }]); }); + it('keeps a date without time on its own day whatever the timezone', () => { + const result = aggregate( + { operation: 'Count', groups: [{ field: 'at', operation: 'Month' }] }, + [{ at: '2024-01-01' }], + { timezone: 'America/Los_Angeles' }, + ); + + expect(result).toEqual([{ value: 1, group: { at: '2024-01-01' } }]); + }); + it('groups an unparsable date under null', () => { const result = aggregate( { operation: 'Count', groups: [{ field: 'at', operation: 'Day' }] }, From f095cf4f06b68986d2e77bffd9e51ef4714d49ff Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Fri, 16 Oct 2026 20:50:50 +0200 Subject: [PATCH 14/32] fix(datasource-dynamodb): rank aggregate groups by instant for dates Refs: PRD-1670 Co-Authored-By: Claude Sonnet 5.5 --- .../datasource-dynamodb/src/utils/aggregator.ts | 3 ++- .../datasource-dynamodb/test/utils/compare.test.ts | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/datasource-dynamodb/src/utils/aggregator.ts b/packages/datasource-dynamodb/src/utils/aggregator.ts index 2b78cec03b..2bf9b854d9 100644 --- a/packages/datasource-dynamodb/src/utils/aggregator.ts +++ b/packages/datasource-dynamodb/src/utils/aggregator.ts @@ -175,7 +175,8 @@ export default class StreamingAggregator { if (a.value === null) return 1; if (b.value === null) return -1; - return a.value < b.value ? 1 : -1; + // Highest first, dates by the instant they stand for. + return -compareValues(a.value, b.value); }); return limit && rows.length > limit ? rows.slice(0, limit) : rows; diff --git a/packages/datasource-dynamodb/test/utils/compare.test.ts b/packages/datasource-dynamodb/test/utils/compare.test.ts index 7add257407..aa474830de 100644 --- a/packages/datasource-dynamodb/test/utils/compare.test.ts +++ b/packages/datasource-dynamodb/test/utils/compare.test.ts @@ -51,4 +51,17 @@ describe('sorting and aggregating dates with offsets', () => { expect(aggregator.result()).toEqual([{ value: expected, group: {} }]); }); + + it('ranks groups by the instant of their Max, so a limit keeps the latest', () => { + const aggregator = new StreamingAggregator( + new Aggregation({ operation: 'Max', field: 'at', groups: [{ field: 'g' }] }), + 'UTC', + ); + aggregator.add({ g: 'earlier', at: '2024-01-01T00:30:00+02:00' }); + aggregator.add({ g: 'later', at: '2023-12-31T23:00:00Z' }); + + expect(aggregator.result(1)).toEqual([ + { value: '2023-12-31T23:00:00Z', group: { g: 'later' } }, + ]); + }); }); From 73594e6859ec9df50580a7e0acec300d4ce43e28 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Sun, 18 Oct 2026 21:01:16 +0200 Subject: [PATCH 15/32] perf(datasource-dynamodb): read only the table items a page of array elements needs A real run over 100 000 items showed the first page of an array collection reading the whole table. Without a sort, parents are now read in batches of 100 until the page is filled. Refs: PRD-1670 Co-Authored-By: Claude Sonnet 5.5 --- .../src/array-collection.ts | 86 ++++++++++++------- .../test/array-review-findings.test.ts | 52 ++++++++++- 2 files changed, 104 insertions(+), 34 deletions(-) diff --git a/packages/datasource-dynamodb/src/array-collection.ts b/packages/datasource-dynamodb/src/array-collection.ts index 5072aee96d..e0cd8b730d 100644 --- a/packages/datasource-dynamodb/src/array-collection.ts +++ b/packages/datasource-dynamodb/src/array-collection.ts @@ -16,6 +16,7 @@ import { ConditionTreeBranch, ConditionTreeFactory, ConditionTreeLeaf, + Page, PaginatedFilter as PaginatedFilterClass, Projection, ValidationError, @@ -29,6 +30,9 @@ import compareRecords from './utils/sort'; export const INDEX_FIELD = 'index'; +/** Table items read at once when only the first elements of the array are needed. */ +const PARENTS_PER_READ = 100; + /** Expressions are limited to 4 KB by DynamoDB, names and values apart. */ const MAX_EXPRESSION_LENGTH = 3000; @@ -151,15 +155,17 @@ export default class ArrayCollection extends BaseCollection { filter: PaginatedFilter, projection: Projection, ): Promise { - const records = await this.readRecords(caller, filter.conditionTree); const sort = filter.sort?.length ? filter.sort : undefined; - - if (sort) records.sort((a, b) => compareRecords(a, b, sort)); - const skip = filter.page?.skip ?? 0; const limit = filter.page?.limit || undefined; + // Without a sort the first records are the first elements: parents are read only as far as + // the page needs, instead of the whole table. + const wanted = !sort && limit !== undefined ? skip + limit : undefined; + const records = await this.readRecords(caller, filter.conditionTree, wanted); const { columns } = projection; + if (sort) records.sort((a, b) => compareRecords(a, b, sort)); + return records .slice(skip, limit === undefined ? undefined : skip + limit) .map(record => @@ -236,7 +242,11 @@ export default class ArrayCollection extends BaseCollection { } /** Reads the parents, then turns each array element into a record and applies the filter. */ - private async readRecords(caller: Caller, tree?: ConditionTree): Promise { + private async readRecords( + caller: Caller, + tree?: ConditionTree, + wanted?: number, + ): Promise { // Conditions on the table key narrow the parents read; the rest applies to the elements. const onTableKey = conjuncts(tree).filter( condition => @@ -245,38 +255,48 @@ export default class ArrayCollection extends BaseCollection { const { arrayField } = this.chain[this.chain.length - 1]; const parentColumns = this.parentColumns(); const ownKeyColumns = this.keyColumns.slice(0, -1); - const parents = await this.parent.list( - caller, - new PaginatedFilterClass({ - conditionTree: onTableKey.length - ? ConditionTreeFactory.intersect(...onTableKey) - : undefined, - }), - new Projection(...parentColumns, arrayField), - ); - + const parentFilter = onTableKey.length + ? ConditionTreeFactory.intersect(...onTableKey) + : undefined; + const parentProjection = new Projection(...parentColumns, arrayField); const records: RecordData[] = []; - for (const parent of parents) { - const elements = parent[arrayField]; - - if (Array.isArray(elements)) { - elements.forEach((element, index) => { - if (element && typeof element === 'object') { - records.push({ - // The identity comes last: an element's own `index` must not take its place. - ...Serializer.flatten(element as Item, this.itemFields), - ...Object.fromEntries( - ownKeyColumns.map((name, i) => [name, parent[parentColumns[i]]]), - ), - [INDEX_FIELD]: index, - }); - } - }); + for (let offset = 0; ; offset += PARENTS_PER_READ) { + // eslint-disable-next-line no-await-in-loop + const parents = await this.parent.list( + caller, + new PaginatedFilterClass({ + conditionTree: parentFilter, + ...(wanted === undefined ? {} : { page: new Page(offset, PARENTS_PER_READ) }), + }), + parentProjection, + ); + + for (const parent of parents) { + const elements = parent[arrayField]; + + if (Array.isArray(elements)) { + elements.forEach((element, index) => { + if (element && typeof element === 'object') { + const record = { + // The identity comes last: an element's own `index` must not take its place. + ...Serializer.flatten(element as Item, this.itemFields), + ...Object.fromEntries( + ownKeyColumns.map((name, i) => [name, parent[parentColumns[i]]]), + ), + [INDEX_FIELD]: index, + }; + + if (!tree || tree.match(record, this, caller.timezone)) records.push(record); + } + }); + } } - } - return tree ? records.filter(record => tree.match(record, this, caller.timezone)) : records; + if (wanted === undefined || records.length >= wanted || parents.length < PARENTS_PER_READ) { + return records; + } + } } private async append(record: RecordData): Promise { diff --git a/packages/datasource-dynamodb/test/array-review-findings.test.ts b/packages/datasource-dynamodb/test/array-review-findings.test.ts index a16971f0f4..b87098f6fa 100644 --- a/packages/datasource-dynamodb/test/array-review-findings.test.ts +++ b/packages/datasource-dynamodb/test/array-review-findings.test.ts @@ -4,7 +4,7 @@ import { Filter, PaginatedFilter, Projection } from '@forestadmin/datasource-too import { buildCollection } from './collection-helpers'; import { fakeClient } from './fake-client'; -import { buildModel } from './helpers'; +import { buildModel, leaf } from './helpers'; import { createDynamoDBDataSource } from '../src'; import ArrayCollection from '../src/array-collection'; @@ -192,3 +192,53 @@ describe('native queries', () => { ).rejects.toThrow('still has results after 1000 pages'); }); }); + +describe('listing the elements of an array page by page', () => { + const parents = (count: number, sku: (i: number) => string) => + Array.from({ length: count }, (_, i) => ({ + customerId: 'c1', + orderId: `o${String(i).padStart(4, '0')}`, + lines: [{ sku: sku(i) }], + })); + + it('reads only as many table items as the page needs', async () => { + const { collection, client } = build({ + Scan: () => ({ Items: parents(250, () => 'A') }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ page: { skip: 0, limit: 3 } as never }), + new Projection('sku'), + ); + + expect(records).toHaveLength(3); + expect(client.calls('Scan')).toHaveLength(1); + expect(client.calls('Scan')[0].Limit).toBe(100); + }); + + it('reads the next table items when the first ones do not hold enough matches', async () => { + const all = parents(200, i => (i >= 150 ? 'MATCH' : 'A')); + const { collection, client } = build({ + Scan: (input: any) => { + const start = input.ExclusiveStartKey + ? all.findIndex(item => item.orderId === input.ExclusiveStartKey.orderId) + 1 + : 0; + + return { Items: all.slice(start, start + 100) }; + }, + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: leaf('sku', 'Equal', 'MATCH'), + page: { skip: 0, limit: 5 } as never, + }), + new Projection('sku', 'orderId'), + ); + + expect(records.map(r => r.orderId)).toEqual(['o0150', 'o0151', 'o0152', 'o0153', 'o0154']); + expect(client.calls('Scan')).toHaveLength(2); + }); +}); From 3cf2232079264e80f844ae5c510da17ed4ae36bb Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Sun, 18 Oct 2026 21:49:46 +0200 Subject: [PATCH 16/32] fix(datasource-dynamodb): generate string sort keys and accept unchanged keys on update Forest greys the primary key fields of the creation form, so a string sort key left empty now gets a UUID (behind the entity prefix, if any), like the partition key. Setting a many-to-one relation writes its foreign key, which can be a key of the table: writing the value a key already has is a no-op, changing it is still refused. Co-Authored-By: Claude Sonnet 5.5 --- packages/datasource-dynamodb/README.md | 4 +- .../datasource-dynamodb/src/collection.ts | 36 ++++++++---- .../test/collection.write.test.ts | 55 ++++++++++++++++++- 3 files changed, 83 insertions(+), 12 deletions(-) diff --git a/packages/datasource-dynamodb/README.md b/packages/datasource-dynamodb/README.md index d9004c1aee..c9de34da1a 100644 --- a/packages/datasource-dynamodb/README.md +++ b/packages/datasource-dynamodb/README.md @@ -114,7 +114,9 @@ Types: `string`, `number`, `boolean`, `date` and `dateonly` (ISO 8601 strings), ## Composite keys The partition key and the sort key are both primary key columns of the collection, so Forest -identifies a record by the two. A record created without a string partition key gets a UUID. +identifies a record by the two. A record created without a string partition or sort key gets a UUID (the sort key keeps its +entity prefix, when there is one), since Forest greys the key fields of the creation form. A +numeric key has to be supplied. Keys cannot be updated. ## Single-table design diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index c0f6da7503..a2aaa48022 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -182,12 +182,6 @@ export default class DynamoDBCollection extends BaseCollection { const entries = Object.entries(patch).filter(([, value]) => value !== undefined); for (const [field] of entries) { - if (this.keyNames().includes(field)) { - throw new ValidationError( - `The key attribute '${field}' of '${this.name}' cannot be updated`, - ); - } - if (field === this.model.entity?.attribute) { throw new ValidationError( `The attribute '${field}' tells which entity a record of '${this.name}' belongs to, ` + @@ -199,7 +193,25 @@ export default class DynamoDBCollection extends BaseCollection { if (entries.length === 0) return; const keys = await this.resolveKeys(caller, filter); - await this.executor.mapWrites(keys, key => this.updateOne(key, entries)); + const keyNames = this.keyNames(); + + // Forest sets a many-to-one relation by writing its foreign key, which is a key of the + // record when the relation follows the table's own keys: writing the value it has is harmless. + const changedKey = entries.find( + ([field, value]) => keyNames.includes(field) && keys.some(key => key[field] !== value), + ); + + if (changedKey) { + throw new ValidationError( + `The key attribute '${changedKey[0]}' of '${this.name}' cannot be updated`, + ); + } + + const writable = entries.filter(([field]) => !keyNames.includes(field)); + + if (writable.length === 0) return; + + await this.executor.mapWrites(keys, key => this.updateOne(key, writable)); } catch (error) { throw this.wrap(error, 'update'); } finally { @@ -545,9 +557,13 @@ export default class DynamoDBCollection extends BaseCollection { } if (sortKey && isEmptyKey(created[sortKey.name])) { - throw new ValidationError( - `The sort key '${sortKey.name}' is required to create a record in '${this.name}'`, - ); + if (sortKey.type !== 'S') { + throw new ValidationError( + `The sort key '${sortKey.name}' is required to create a record in '${this.name}'`, + ); + } + + created[sortKey.name] = `${entity?.sortKeyPrefix ?? ''}${randomUUID()}`; } if ( diff --git a/packages/datasource-dynamodb/test/collection.write.test.ts b/packages/datasource-dynamodb/test/collection.write.test.ts index a97acf7a4e..1fa93fa8ac 100644 --- a/packages/datasource-dynamodb/test/collection.write.test.ts +++ b/packages/datasource-dynamodb/test/collection.write.test.ts @@ -37,9 +37,37 @@ describe('DynamoDBCollection.create', () => { expect(client.calls('Put')[0].Item.id).toBe(created.id); }); - it('refuses a record without a sort key', async () => { + it('generates a sort key for a string key left empty', async () => { const { collection, client } = buildCollection(buildModel(), { Put: () => ({}) }); + const [created] = await collection.create(caller, [{ customerId: 'c1' }]); + + expect(created.orderId).toMatch(/^[0-9a-f-]{36}$/); + expect(client.calls('Put')[0].Item).toMatchObject({ + customerId: 'c1', + orderId: created.orderId, + }); + }); + + it('prefixes a generated sort key with the entity prefix', async () => { + const { collection } = buildCollection(buildModel({ entity: { sortKeyPrefix: 'ORDER#' } }), { + Put: () => ({}), + }); + + const [created] = await collection.create(caller, [{ customerId: 'c1', orderId: '' }]); + + expect(created.orderId).toMatch(/^ORDER#[0-9a-f-]{36}$/); + }); + + it('refuses a numeric sort key left empty', async () => { + const { collection, client } = buildCollection( + buildModel({ + sortKey: { name: 'orderId', type: 'N' }, + schema: { orderId: { type: 'number' } }, + }), + { Put: () => ({}) }, + ); + await expect(collection.create(caller, [{ customerId: 'c1' }])).rejects.toThrow( "The sort key 'orderId' is required to create a record in 'orders'", ); @@ -203,6 +231,31 @@ describe('DynamoDBCollection.update', () => { ).rejects.toThrow("The key attribute 'id' of 'users' cannot be updated"); }); + it('accepts a key written with the value it already has', async () => { + const { collection, client } = buildCollection(buildModel(), { Update: () => ({}) }); + + await collection.update( + caller, + new Filter({ + conditionTree: and(leaf('customerId', 'Equal', 'c1'), leaf('orderId', 'Equal', 'o1')), + }), + { customerId: 'c1', status: 'paid' }, + ); + + expect(client.calls('Update')).toHaveLength(1); + expect(client.calls('Update')[0].UpdateExpression).not.toContain('customerId'); + }); + + it('writes nothing when the only change is a key it already has', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { Update: () => ({}) }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + id: 'u1', + }); + + expect(client.calls('Update')).toHaveLength(0); + }); + it('merges client-side when the parent map of a nested path does not exist yet', async () => { const { collection, client } = buildCollection(buildSimpleModel(), { Update: () => { From 221634ef0890c524c913aee57b869f597a966cb5 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Sun, 18 Oct 2026 21:51:55 +0200 Subject: [PATCH 17/32] fix(datasource-dynamodb): explain why a key cannot be changed on update Co-Authored-By: Claude Sonnet 5.5 --- packages/datasource-dynamodb/src/collection.ts | 3 ++- packages/datasource-dynamodb/test/collection.write.test.ts | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index a2aaa48022..273fdb2715 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -203,7 +203,8 @@ export default class DynamoDBCollection extends BaseCollection { if (changedKey) { throw new ValidationError( - `The key attribute '${changedKey[0]}' of '${this.name}' cannot be updated`, + `The key attribute '${changedKey[0]}' of '${this.name}' cannot be changed on an existing ` + + `record, since DynamoDB keys are immutable: delete the record and create a new one instead`, ); } diff --git a/packages/datasource-dynamodb/test/collection.write.test.ts b/packages/datasource-dynamodb/test/collection.write.test.ts index 1fa93fa8ac..97e518048c 100644 --- a/packages/datasource-dynamodb/test/collection.write.test.ts +++ b/packages/datasource-dynamodb/test/collection.write.test.ts @@ -228,7 +228,10 @@ describe('DynamoDBCollection.update', () => { collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { id: 'u2', }), - ).rejects.toThrow("The key attribute 'id' of 'users' cannot be updated"); + ).rejects.toThrow( + "The key attribute 'id' of 'users' cannot be changed on an existing record, " + + 'since DynamoDB keys are immutable: delete the record and create a new one instead', + ); }); it('accepts a key written with the value it already has', async () => { From 26e131d9324841343da3908e93370aadb66b1504 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Sun, 18 Oct 2026 21:59:30 +0200 Subject: [PATCH 18/32] test(datasource-dynamodb): cover the manual schema against DynamoDB Local Co-Authored-By: Claude Sonnet 5.5 --- .../test/integration/dynamodb-local.test.ts | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts index fb9e740452..49671dfade 100644 --- a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts +++ b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts @@ -626,6 +626,171 @@ run('DynamoDB Local', () => { }); }); + describe('manual schema', () => { + const MANUAL = `manual-${suffix}`; + let manual: DataSource; + let manualLogger: jest.Mock; + + beforeAll(async () => { + await client.send( + new CreateTableCommand({ + TableName: MANUAL, + BillingMode: 'PAY_PER_REQUEST', + KeySchema: [ + { AttributeName: 'customerId', KeyType: 'HASH' }, + { AttributeName: 'orderId', KeyType: 'RANGE' }, + ], + AttributeDefinitions: [ + { AttributeName: 'customerId', AttributeType: 'S' }, + { AttributeName: 'orderId', AttributeType: 'S' }, + { AttributeName: 'status', AttributeType: 'S' }, + { AttributeName: 'createdAt', AttributeType: 'S' }, + ], + GlobalSecondaryIndexes: [ + { + IndexName: 'by-status', + KeySchema: [ + { AttributeName: 'status', KeyType: 'HASH' }, + { AttributeName: 'createdAt', KeyType: 'RANGE' }, + ], + Projection: { ProjectionType: 'ALL' }, + }, + ], + }), + ); + + await documents.send( + new BatchWriteCommand({ + RequestItems: { + [MANUAL]: Array.from({ length: 6 }, (_, i) => ({ + PutRequest: { + Item: { + customerId: 'c1', + orderId: `o${i}`, + status: i % 2 ? 'paid' : 'open', + total: i + 1, + createdAt: `2024-01-0${i + 1}T00:00:00Z`, + shipping: { city: i % 2 ? 'Paris' : 'Austin' }, + }, + }, + })), + }, + }), + ); + + manualLogger = jest.fn(); + manual = await createDynamoDBDataSource({ + client: documents, + disableIntrospection: true, + schema: { + collections: [ + { + name: 'manual_orders', + tableName: MANUAL, + partitionKey: { name: 'customerId', type: 'string' }, + sortKey: { name: 'orderId', type: 'string' }, + indexes: [ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'string' }, + sortKey: { name: 'createdAt', type: 'string' }, + }, + ], + fields: [ + { name: 'total', type: 'number' }, + { name: 'status', type: 'string' }, + { name: 'createdAt', type: 'date' }, + { name: 'shipping', type: 'object', fields: [{ name: 'city', type: 'string' }] }, + ], + }, + ], + }, + chartCacheTtlMs: 0, + })(manualLogger, jest.fn()); + }); + + afterAll(async () => { + await client.send(new DeleteTableCommand({ TableName: MANUAL })).catch(() => undefined); + }); + + it('declares the keys, the typed attributes and the flattened nested ones', () => { + expect(manual.getCollection('manual_orders').schema.fields).toMatchObject({ + customerId: { isPrimaryKey: true, columnType: 'String' }, + orderId: { isPrimaryKey: true, columnType: 'String' }, + total: { columnType: 'Number' }, + status: { columnType: 'String' }, + createdAt: { columnType: 'Date' }, + 'shipping->city': { columnType: 'String' }, + }); + }); + + it('exposes only the declared collections', () => { + expect(manual.collections.map(c => c.name)).toEqual(['manual_orders']); + }); + + it('reads the declared index and filters on a nested attribute', async () => { + manualLogger.mockClear(); + + const records = await manual.getCollection('manual_orders').list( + caller, + new PaginatedFilter({ + conditionTree: tree({ + aggregator: 'And', + conditions: [ + { field: 'status', operator: 'Equal', value: 'paid' }, + { field: 'createdAt', operator: 'GreaterThan', value: '2000-01-01T00:00:00Z' }, + { field: 'shipping->city', operator: 'Equal', value: 'Paris' }, + ], + }), + }), + new Projection('orderId', 'shipping->city'), + ); + + expect(records.map(r => r.orderId).sort()).toEqual(['o1', 'o3', 'o5']); + expect(manualLogger).not.toHaveBeenCalledWith( + 'Warn', + expect.stringContaining('Scan on table'), + ); + }); + + it('creates, updates and deletes through the declared schema', async () => { + const collection = manual.getCollection('manual_orders'); + const key = tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c9' }, + { field: 'orderId', operator: 'Equal', value: 'o1' }, + ], + }); + + await collection.create(caller, [ + { customerId: 'c9', orderId: 'o1', total: 5, 'shipping->city': 'Rome' }, + ]); + await collection.update(caller, new Filter({ conditionTree: key }), { + 'shipping->city': 'Milan', + }); + + expect( + await collection.list( + caller, + new PaginatedFilter({ conditionTree: key }), + new Projection('total', 'shipping->city'), + ), + ).toEqual([{ total: 5, 'shipping->city': 'Milan' }]); + + await collection.delete(caller, new Filter({ conditionTree: key })); + + expect( + await collection.list( + caller, + new PaginatedFilter({ conditionTree: key }), + new Projection('total'), + ), + ).toEqual([]); + }); + }); + describe('native query', () => { it('runs a PartiQL statement with parameters', async () => { const rows = await dataSource.executeNativeQuery( From 7f68052557012a142b10c6e6044a5f61c3a2548d Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Sun, 18 Oct 2026 22:15:31 +0200 Subject: [PATCH 19/32] fix(datasource-dynamodb): replace maps like SET does in the rewrite fallback and share the parent read budget A map written through the client-side rewrite of an update was merged with the old one, keeping keys the SET would have dropped: each entry now replaces what is at its path. The pages of parents an array collection reads were each given a budget of their own, so a selective read could go past maxScannedItems: the parents read now count together. Co-Authored-By: Claude Sonnet 5.5 --- .../src/array-collection.ts | 13 +++++ .../datasource-dynamodb/src/collection.ts | 7 ++- .../src/utils/serializer.ts | 17 ------- .../test/array-collection.test.ts | 49 ++++++++++++++++++- .../test/collection.write.test.ts | 24 +++++++++ 5 files changed, 90 insertions(+), 20 deletions(-) diff --git a/packages/datasource-dynamodb/src/array-collection.ts b/packages/datasource-dynamodb/src/array-collection.ts index e0cd8b730d..2d0a996769 100644 --- a/packages/datasource-dynamodb/src/array-collection.ts +++ b/packages/datasource-dynamodb/src/array-collection.ts @@ -22,6 +22,7 @@ import { ValidationError, } from '@forestadmin/datasource-toolkit'; +import { ScanLimitExceededError } from './errors'; import convertModelToFields from './schema/schema-converter'; import StreamingAggregator from './utils/aggregator'; import ExpressionBuilder from './utils/expression-builder'; @@ -260,6 +261,7 @@ export default class ArrayCollection extends BaseCollection { : undefined; const parentProjection = new Projection(...parentColumns, arrayField); const records: RecordData[] = []; + let parentsRead = 0; for (let offset = 0; ; offset += PARENTS_PER_READ) { // eslint-disable-next-line no-await-in-loop @@ -296,6 +298,17 @@ export default class ArrayCollection extends BaseCollection { if (wanted === undefined || records.length >= wanted || parents.length < PARENTS_PER_READ) { return records; } + + // Each page of parents is read with a budget of its own: this one is shared by all of them. + parentsRead += parents.length; + + if (parentsRead > this.root.executor.maxScannedItems) { + throw new ScanLimitExceededError( + `Reading the parents of '${this.name}' stopped after ${parentsRead} items of table ` + + `'${this.root.model.tableName}': maxScannedItems is ${this.root.executor.maxScannedItems}. ` + + `Narrow the results with filters.`, + ); + } } } diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index 273fdb2715..b11a1e8e8a 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -645,9 +645,14 @@ export default class DynamoDBCollection extends BaseCollection { const keyBuilder = new ExpressionBuilder('k', 'kv'); + // Each entry replaces what is at its path, as a SET does: a map is not merged with the old one. + for (const [field, value] of entries) { + Serializer.setPath(existing, Serializer.toPathSegments(field), value); + } + try { await this.executor.put({ - Item: Serializer.deepMerge(existing, Serializer.unflatten(Object.fromEntries(entries))), + Item: existing, ConditionExpression: `attribute_exists(${keyBuilder.attribute( this.model.partitionKey.name, )})`, diff --git a/packages/datasource-dynamodb/src/utils/serializer.ts b/packages/datasource-dynamodb/src/utils/serializer.ts index bc19e93690..5968fa5dfe 100644 --- a/packages/datasource-dynamodb/src/utils/serializer.ts +++ b/packages/datasource-dynamodb/src/utils/serializer.ts @@ -127,21 +127,4 @@ export default class Serializer { static toPathSegments(field: string): string[] { return field.split(PATH_SEPARATOR); } - - /** Merges `patch` into `item`, recursing into maps only. */ - static deepMerge( - item: Record, - patch: Record, - ): Record { - const merged = { ...item }; - - for (const [key, value] of Object.entries(patch)) { - merged[key] = - isPlainObject(value) && isPlainObject(merged[key]) - ? Serializer.deepMerge(merged[key] as Record, value) - : value; - } - - return merged; - } } diff --git a/packages/datasource-dynamodb/test/array-collection.test.ts b/packages/datasource-dynamodb/test/array-collection.test.ts index 1247e803ba..eea7ffc598 100644 --- a/packages/datasource-dynamodb/test/array-collection.test.ts +++ b/packages/datasource-dynamodb/test/array-collection.test.ts @@ -3,6 +3,7 @@ import type { DataSource } from '@forestadmin/datasource-toolkit'; import { Aggregation, Filter, + Page, PaginatedFilter, Projection, Sort, @@ -27,7 +28,7 @@ const orders = [ { customerId: 'c3', orderId: 'o3' }, ]; -function build(handlers: Record) { +function build(handlers: Record, settings = {}) { const parent = buildCollection( buildModel({ schema: { @@ -37,7 +38,7 @@ function build(handlers: Record) { }, }), handlers, - { scanConcurrency: 1 }, + { scanConcurrency: 1, ...settings }, ); const collection = new ArrayCollection({} as DataSource, parent.collection, { name: 'order_lines', @@ -430,4 +431,48 @@ describe('ArrayCollection of an array', () => { "The key attribute 'order_linesIndex' of the parent 'order_lines' is required to create a record in 'parts'", ); }); + + describe('read budget', () => { + it('counts the parents of every page against maxScannedItems', async () => { + const parents = Array.from({ length: 1000 }, (_, i) => ({ + customerId: `c${i}`, + orderId: `o${i}`, + lines: [{ sku: 'A', qty: 1 }], + })); + const { collection } = build( + { + Scan: (input: Record) => { + const start = input.ExclusiveStartKey + ? Number(input.ExclusiveStartKey.orderId.slice(1)) + 1 + : 0; + const items = parents.slice(start, start + (input.Limit ?? parents.length)); + + return { + Items: items, + ...(start + items.length < parents.length + ? { + LastEvaluatedKey: { + customerId: items[items.length - 1].customerId, + orderId: items[items.length - 1].orderId, + }, + } + : {}), + }; + }, + }, + { maxScannedItems: 300 }, + ); + + await expect( + collection.list( + caller, + new PaginatedFilter({ + conditionTree: leaf('sku', 'Equal', 'none'), + page: new Page(0, 15), + }), + new Projection('sku'), + ), + ).rejects.toThrow(/maxScannedItems is 300/); + }); + }); }); diff --git a/packages/datasource-dynamodb/test/collection.write.test.ts b/packages/datasource-dynamodb/test/collection.write.test.ts index 97e518048c..8a9865f948 100644 --- a/packages/datasource-dynamodb/test/collection.write.test.ts +++ b/packages/datasource-dynamodb/test/collection.write.test.ts @@ -282,6 +282,30 @@ describe('DynamoDBCollection.update', () => { }); }); + it('replaces a map the way SET does when it falls back to a client-side rewrite', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Update: () => { + throw Object.assign( + new Error('The document path provided in the update expression is invalid for update'), + { name: 'ValidationException' }, + ); + }, + Get: () => ({ Item: { id: 'u1', config: { keep: 1, remove: 2 } } }), + Put: () => ({}), + }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + config: { keep: 3 }, + 'address->city': 'Lyon', + }); + + expect(client.calls('Put')[0].Item).toEqual({ + id: 'u1', + config: { keep: 3 }, + address: { city: 'Lyon' }, + }); + }); + it('does not create an item that does not exist', async () => { const { collection } = buildCollection(buildSimpleModel(), { Update: () => { From 8aeed6ef8cef21761e424f69a81a30ba0d4a73ae Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Thu, 22 Oct 2026 11:03:20 +0200 Subject: [PATCH 20/32] fix(datasource-dynamodb): align types with the toolkit and enforce limits on every read timeonly is published as Time and GeoJSON points as Json, the types whose values the toolkit validates. Binary attributes are filtered on the returned items, since DynamoDB would compare them to text. BatchGetItem lookups count against maxScannedItems and a skip beyond maxOffset is refused whichever way the page is read. Co-Authored-By: Claude Sonnet 5.5 --- packages/datasource-dynamodb/README.md | 3 +- .../datasource-dynamodb/src/collection.ts | 27 +++-- .../src/utils/type-converter.ts | 3 +- .../test/third-round-findings.test.ts | 100 ++++++++++++++++++ .../test/utils/type-converter.test.ts | 4 +- 5 files changed, 122 insertions(+), 15 deletions(-) create mode 100644 packages/datasource-dynamodb/test/third-round-findings.test.ts diff --git a/packages/datasource-dynamodb/README.md b/packages/datasource-dynamodb/README.md index c9de34da1a..d2a7cc7aa0 100644 --- a/packages/datasource-dynamodb/README.md +++ b/packages/datasource-dynamodb/README.md @@ -109,7 +109,8 @@ them as text on a key and when it filters: store them in one offset (UTC) if a d you filter on ranges of it. Types: `string`, `number`, `boolean`, `date` and `dateonly` (ISO 8601 strings), `timeonly`, `object`, -`array`, `point` (GeoJSON), `binary` (base64). Keys are strings or numbers. +`array`, `point` (GeoJSON, exposed as Json), `binary` (base64, filtered on the returned items). Keys +are strings or numbers. `timeonly` is exposed as Forest's `Time`. ## Composite keys diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index b11a1e8e8a..aafd7731c7 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -276,7 +276,8 @@ export default class DynamoDBCollection extends BaseCollection { // ---- reads ------------------------------------------------------------------------------ private plan(tree?: ConditionTree, sort?: Sort): ReadPlan { - const { pushable, residual } = splitCondition(tree); + // DynamoDB compares a binary attribute to a binary, never to the base64 text Forest sends. + const { pushable, residual } = splitCondition(tree, leaf => !this.isBinary(leaf.field)); const lookup = planKeyLookup(this.model, pushable); if (lookup) return { lookup, residual: intersect(lookup.remaining, residual) }; @@ -296,6 +297,10 @@ export default class DynamoDBCollection extends BaseCollection { return { access: { ...access, remaining: filter }, residual: intersect(moved, residual) }; } + private isBinary(field: string): boolean { + return this.model.schema[field]?.type === 'binary'; + } + private guardScan(plan: ReadPlan, tree?: ConditionTree): void { if (plan.access?.kind !== 'scan') return; @@ -363,6 +368,8 @@ export default class DynamoDBCollection extends BaseCollection { unit?: ReadUnit, ): AsyncGenerator { if (plan.lookup) { + budget.charge(plan.lookup.keys.length, 'query'); + yield await this.executor.batchGet(plan.lookup.keys, this.projectionParams(fields) as never); return; @@ -412,15 +419,6 @@ export default class DynamoDBCollection extends BaseCollection { ): Promise { const { access, residual } = plan; const cache = this.settings.paginationCache; - const { maxOffset } = this.settings; - - if (skip > maxOffset) { - throw new OffsetTooDeepError( - `Offset ${skip} exceeds the maximum allowed offset of ${maxOffset}. ` + - `Narrow the results with filters.`, - ); - } - const base = this.buildReadInput(access, fields); const hash = cache.hash({ base, @@ -478,6 +476,15 @@ export default class DynamoDBCollection extends BaseCollection { const sort = filter.sort?.length ? filter.sort : undefined; const skip = filter.page?.skip ?? 0; const limit = filter.page?.limit || undefined; + const { maxOffset } = this.settings; + + if (skip > maxOffset) { + throw new OffsetTooDeepError( + `Offset ${skip} exceeds the maximum allowed offset of ${maxOffset}. ` + + `Narrow the results with filters.`, + ); + } + const plan = this.plan(filter.conditionTree, sort); // Reading the first page of a collection without any filter costs one page, not a table. diff --git a/packages/datasource-dynamodb/src/utils/type-converter.ts b/packages/datasource-dynamodb/src/utils/type-converter.ts index 40b3646c4c..fec0fd0330 100644 --- a/packages/datasource-dynamodb/src/utils/type-converter.ts +++ b/packages/datasource-dynamodb/src/utils/type-converter.ts @@ -34,14 +34,13 @@ export default class TypeConverter { case 'dateonly': return 'Dateonly'; case 'timeonly': - return 'Timeonly'; + return 'Time'; case 'number': return 'Number'; case 'string': case 'binary': return 'String'; case 'point': - return 'Point'; case 'object': case 'array': case 'null': diff --git a/packages/datasource-dynamodb/test/third-round-findings.test.ts b/packages/datasource-dynamodb/test/third-round-findings.test.ts new file mode 100644 index 0000000000..f4e27ec31d --- /dev/null +++ b/packages/datasource-dynamodb/test/third-round-findings.test.ts @@ -0,0 +1,100 @@ +import { PaginatedFilter, Projection, Sort } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { buildSimpleModel, leaf } from './helpers'; + +const caller = { timezone: 'UTC' } as never; + +describe('maxScannedItems on key lookups', () => { + it('counts the keys read by BatchGetItem', async () => { + const ids = Array.from({ length: 250 }, (_, i) => `u${i}`); + const { collection } = buildCollection( + buildSimpleModel(), + { + BatchGet: (input: any) => ({ + Responses: { + Users: (input.RequestItems.Users.Keys as Array<{ id: string }>).map(key => ({ + id: key.id, + })), + }, + }), + }, + { maxScannedItems: 50 }, + ); + + await expect( + collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('id', 'In', ids) }), + new Projection('id'), + ), + ).rejects.toThrow(/maxScannedItems is 50/); + }); + + it('reads the keys of a lookup that fits the limit', async () => { + const { collection } = buildCollection( + buildSimpleModel(), + { BatchGet: () => ({ Responses: { Users: [{ id: 'u1' }, { id: 'u2' }] } }) }, + { maxScannedItems: 50 }, + ); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('id', 'In', ['u1', 'u2']) }), + new Projection('id'), + ); + + expect(records).toEqual([{ id: 'u1' }, { id: 'u2' }]); + }); +}); + +describe('maxOffset on reads that sort in memory', () => { + it('refuses a skip beyond the limit, as the paged read does', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Items: [{ id: 'u1', name: 'a' }] }) }, + { maxOffset: 100, scanConcurrency: 1 }, + ); + + await expect( + collection.list( + caller, + new PaginatedFilter({ + sort: new Sort({ field: 'name', ascending: true }), + page: { skip: 200, limit: 15 } as never, + }), + new Projection('id'), + ), + ).rejects.toThrow('Offset 200 exceeds the maximum allowed offset of 100'); + expect(client.calls('Scan')).toHaveLength(0); + }); +}); + +describe('binary attributes', () => { + it('filters them on the returned items, not in DynamoDB', async () => { + const model = buildSimpleModel({ + schema: { id: { type: 'string' }, blob: { type: 'binary' } }, + }); + const { collection, client } = buildCollection( + model, + { + Scan: () => ({ + Items: [ + { id: 'u1', blob: Uint8Array.from([1, 2, 3]) }, + { id: 'u2', blob: Uint8Array.from([9]) }, + ], + }), + }, + { scanConcurrency: 1 }, + ); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('blob', 'Equal', 'AQID') }), + new Projection('id', 'blob'), + ); + + expect(records).toEqual([{ id: 'u1', blob: 'AQID' }]); + expect(client.calls('Scan')[0].FilterExpression).toBeUndefined(); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/type-converter.test.ts b/packages/datasource-dynamodb/test/utils/type-converter.test.ts index 10f2423cbe..1c9f73c07c 100644 --- a/packages/datasource-dynamodb/test/utils/type-converter.test.ts +++ b/packages/datasource-dynamodb/test/utils/type-converter.test.ts @@ -51,8 +51,8 @@ describe('TypeConverter', () => { ['boolean', 'Boolean'], ['date', 'Date'], ['dateonly', 'Dateonly'], - ['timeonly', 'Timeonly'], - ['point', 'Point'], + ['timeonly', 'Time'], + ['point', 'Json'], ['binary', 'String'], ['object', 'Json'], ['array', 'Json'], From b1a529bc3389613d3adc4bae19bbf6a7db4b67c4 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Thu, 22 Oct 2026 11:11:07 +0200 Subject: [PATCH 21/32] fix(datasource-dynamodb): store binary attributes as bytes Forest sends a binary attribute as base64 text: create and update decode it, so the item holds a DynamoDB B attribute again instead of the text. Co-Authored-By: Claude Sonnet 5.5 --- .../datasource-dynamodb/src/collection.ts | 17 ++++++++- .../test/integration/dynamodb-local.test.ts | 20 ++++++++++ .../test/third-round-findings.test.ts | 37 ++++++++++++++++++- 3 files changed, 71 insertions(+), 3 deletions(-) diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index aafd7731c7..0f7a27e59e 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -179,7 +179,9 @@ export default class DynamoDBCollection extends BaseCollection { async update(caller: Caller, filter: Filter, patch: RecordData): Promise { try { - const entries = Object.entries(patch).filter(([, value]) => value !== undefined); + const entries: Array<[string, unknown]> = Object.entries(patch) + .filter(([, value]) => value !== undefined) + .map(([field, value]) => [field, this.toStored(field, value)]); for (const [field] of entries) { if (field === this.model.entity?.attribute) { @@ -297,6 +299,13 @@ export default class DynamoDBCollection extends BaseCollection { return { access: { ...access, remaining: filter }, residual: intersect(moved, residual) }; } + /** Forest sends a binary attribute as base64 text, which DynamoDB must store as bytes. */ + private toStored(field: string, value: unknown): unknown { + return this.isBinary(field) && typeof value === 'string' + ? Uint8Array.from(Buffer.from(value, 'base64')) + : value; + } + private isBinary(field: string): boolean { return this.model.schema[field]?.type === 'binary'; } @@ -583,7 +592,11 @@ export default class DynamoDBCollection extends BaseCollection { ); } - const item = Serializer.unflatten(created); + const item = Serializer.unflatten( + Object.fromEntries( + Object.entries(created).map(([field, value]) => [field, this.toStored(field, value)]), + ), + ); if (entity?.attribute !== undefined) { if (item[entity.attribute] !== undefined && item[entity.attribute] !== entity.value) { diff --git a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts index 49671dfade..1540ddd9ba 100644 --- a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts +++ b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts @@ -5,6 +5,7 @@ import { BatchWriteCommand, DeleteCommand, DynamoDBDocumentClient, + GetCommand, PutCommand, } from '@aws-sdk/lib-dynamodb'; import { @@ -701,6 +702,7 @@ run('DynamoDB Local', () => { { name: 'total', type: 'number' }, { name: 'status', type: 'string' }, { name: 'createdAt', type: 'date' }, + { name: 'blob', type: 'binary', nullable: true }, { name: 'shipping', type: 'object', fields: [{ name: 'city', type: 'string' }] }, ], }, @@ -754,6 +756,24 @@ run('DynamoDB Local', () => { ); }); + it('stores a binary attribute as bytes and reads it back as base64', async () => { + const collection = manual.getCollection('manual_orders'); + + await collection.create(caller, [{ customerId: 'c8', orderId: 'o1', blob: 'AQID' }]); + + const stored = await documents.send( + new GetCommand({ TableName: MANUAL, Key: { customerId: 'c8', orderId: 'o1' } }), + ); + expect(Array.from(stored.Item.blob as Uint8Array)).toEqual([1, 2, 3]); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('blob', 'Equal', 'AQID') }), + new Projection('orderId', 'blob'), + ); + expect(records).toEqual([{ orderId: 'o1', blob: 'AQID' }]); + }); + it('creates, updates and deletes through the declared schema', async () => { const collection = manual.getCollection('manual_orders'); const key = tree({ diff --git a/packages/datasource-dynamodb/test/third-round-findings.test.ts b/packages/datasource-dynamodb/test/third-round-findings.test.ts index f4e27ec31d..2545da12fe 100644 --- a/packages/datasource-dynamodb/test/third-round-findings.test.ts +++ b/packages/datasource-dynamodb/test/third-round-findings.test.ts @@ -1,4 +1,4 @@ -import { PaginatedFilter, Projection, Sort } from '@forestadmin/datasource-toolkit'; +import { Filter, PaginatedFilter, Projection, Sort } from '@forestadmin/datasource-toolkit'; import { buildCollection } from './collection-helpers'; import { buildSimpleModel, leaf } from './helpers'; @@ -98,3 +98,38 @@ describe('binary attributes', () => { expect(client.calls('Scan')[0].FilterExpression).toBeUndefined(); }); }); + +describe('writing binary attributes', () => { + const model = () => + buildSimpleModel({ schema: { id: { type: 'string' }, blob: { type: 'binary' } } }); + + it('stores the bytes of the base64 text on create and returns the text', async () => { + const { collection, client } = buildCollection(model(), { Put: () => ({}) }); + + const [created] = await collection.create(caller, [{ id: 'u1', blob: 'AQID' }]); + + expect(Array.from(client.calls('Put')[0].Item.blob as Uint8Array)).toEqual([1, 2, 3]); + expect(created.blob).toBe('AQID'); + }); + + it('stores the bytes of the base64 text on update', async () => { + const { collection, client } = buildCollection(model(), { Update: () => ({}) }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + blob: 'AQID', + }); + + const [value] = Object.values(client.calls('Update')[0].ExpressionAttributeValues); + expect(Array.from(value as Uint8Array)).toEqual([1, 2, 3]); + }); + + it('keeps a null binary as null', async () => { + const { collection, client } = buildCollection(model(), { Update: () => ({}) }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + blob: null, + }); + + expect(Object.values(client.calls('Update')[0].ExpressionAttributeValues)).toContain(null); + }); +}); From 76b430723eec1f61c194a17e35ecb14330c8c9f1 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Thu, 22 Oct 2026 11:19:17 +0200 Subject: [PATCH 22/32] fix(datasource-dynamodb): stamp a nested entity discriminator under its path Co-Authored-By: Claude Sonnet 5.5 --- .../datasource-dynamodb/src/collection.ts | 15 +++++------ .../test/third-round-findings.test.ts | 26 ++++++++++++++++++- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index 0f7a27e59e..51e36ea3c2 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -592,23 +592,22 @@ export default class DynamoDBCollection extends BaseCollection { ); } - const item = Serializer.unflatten( - Object.fromEntries( - Object.entries(created).map(([field, value]) => [field, this.toStored(field, value)]), - ), - ); - if (entity?.attribute !== undefined) { - if (item[entity.attribute] !== undefined && item[entity.attribute] !== entity.value) { + if (created[entity.attribute] !== undefined && created[entity.attribute] !== entity.value) { throw new ValidationError( `The attribute '${entity.attribute}' of '${this.name}' must be '${entity.value}'`, ); } - item[entity.attribute] = entity.value; created[entity.attribute] = entity.value; } + const item = Serializer.unflatten( + Object.fromEntries( + Object.entries(created).map(([field, value]) => [field, this.toStored(field, value)]), + ), + ); + const keyBuilder = new ExpressionBuilder('k', 'kv'); try { diff --git a/packages/datasource-dynamodb/test/third-round-findings.test.ts b/packages/datasource-dynamodb/test/third-round-findings.test.ts index 2545da12fe..7a4d994396 100644 --- a/packages/datasource-dynamodb/test/third-round-findings.test.ts +++ b/packages/datasource-dynamodb/test/third-round-findings.test.ts @@ -1,7 +1,7 @@ import { Filter, PaginatedFilter, Projection, Sort } from '@forestadmin/datasource-toolkit'; import { buildCollection } from './collection-helpers'; -import { buildSimpleModel, leaf } from './helpers'; +import { buildModel, buildSimpleModel, leaf } from './helpers'; const caller = { timezone: 'UTC' } as never; @@ -133,3 +133,27 @@ describe('writing binary attributes', () => { expect(Object.values(client.calls('Update')[0].ExpressionAttributeValues)).toContain(null); }); }); + +describe('a nested entity discriminator', () => { + const model = () => buildModel({ entity: { attribute: 'meta->type', value: 'ORDER' } }); + + it('is stamped under its path, not under a literal key', async () => { + const { collection, client } = buildCollection(model(), { Put: () => ({}) }); + + const [created] = await collection.create(caller, [{ customerId: 'c1', orderId: 'o1' }]); + const { Item } = client.calls('Put')[0]; + + expect(Item.meta).toEqual({ type: 'ORDER' }); + expect(Object.keys(Item)).not.toContain('meta->type'); + expect(created['meta->type']).toBe('ORDER'); + }); + + it('refuses another value supplied for it', async () => { + const { collection, client } = buildCollection(model(), { Put: () => ({}) }); + + await expect( + collection.create(caller, [{ customerId: 'c1', orderId: 'o1', 'meta->type': 'INVOICE' }]), + ).rejects.toThrow("The attribute 'meta->type' of 'orders' must be 'ORDER'"); + expect(client.calls('Put')).toHaveLength(0); + }); +}); From 9522c7dd3811761eef1d4d40ac777f7592195267 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Thu, 22 Oct 2026 11:23:59 +0200 Subject: [PATCH 23/32] fix(datasource-dynamodb): read a nested entity discriminator left out of the schema Co-Authored-By: Claude Sonnet 5.5 --- .../datasource-dynamodb/src/collection.ts | 8 +++-- .../test/third-round-findings.test.ts | 30 ++++++++++++++++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index 51e36ea3c2..a4e9bddd32 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -134,9 +134,13 @@ export default class DynamoDBCollection extends BaseCollection { this.addFields(convertModelToFields(model, logger)); if (model.enableCount ?? true) this.enableCount(); - this.fieldNames = new Set(Object.keys(this.schema.fields)); - const entityAttribute = model.entity?.attribute; + + // The discriminator may be left out of the schema, and is still read to tell the entities apart. + this.fieldNames = new Set([ + ...Object.keys(this.schema.fields), + ...(entityAttribute !== undefined ? [entityAttribute] : []), + ]); this.matchContext = { schema: { fields: { diff --git a/packages/datasource-dynamodb/test/third-round-findings.test.ts b/packages/datasource-dynamodb/test/third-round-findings.test.ts index 7a4d994396..675c97e90e 100644 --- a/packages/datasource-dynamodb/test/third-round-findings.test.ts +++ b/packages/datasource-dynamodb/test/third-round-findings.test.ts @@ -1,7 +1,7 @@ import { Filter, PaginatedFilter, Projection, Sort } from '@forestadmin/datasource-toolkit'; import { buildCollection } from './collection-helpers'; -import { buildModel, buildSimpleModel, leaf } from './helpers'; +import { and, buildModel, buildSimpleModel, leaf } from './helpers'; const caller = { timezone: 'UTC' } as never; @@ -157,3 +157,31 @@ describe('a nested entity discriminator', () => { expect(client.calls('Put')).toHaveLength(0); }); }); + +describe('a nested entity discriminator left out of the schema', () => { + it('still matches the items of the entity when they are read by key', async () => { + const { collection } = buildCollection( + buildModel({ entity: { attribute: 'meta->type', value: 'ORDER' } }), + { + BatchGet: () => ({ + Responses: { + Orders: [ + { customerId: 'c1', orderId: 'o1', meta: { type: 'ORDER' } }, + { customerId: 'c1', orderId: 'o2', meta: { type: 'INVOICE' } }, + ], + }, + }), + }, + ); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: and(leaf('customerId', 'Equal', 'c1'), leaf('orderId', 'In', ['o1', 'o2'])), + }), + new Projection('orderId'), + ); + + expect(records).toEqual([{ orderId: 'o1' }]); + }); +}); From 8c16d2c79b4e458c597e5a7fbcb1f521500f601a Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Thu, 22 Oct 2026 11:46:19 +0200 Subject: [PATCH 24/32] fix(datasource-dynamodb): address the validator review A number beyond 2^53 is read as text, string and number sets are written back as sets, a manual schema collection may omit its fields, virtualArrayCollections is read as an alias, an unknown scanPolicy is refused, and the SELECT-only rule of native queries is tested. Co-Authored-By: Claude Sonnet 5.5 --- packages/datasource-dynamodb/README.md | 9 +- .../datasource-dynamodb/src/collection.ts | 12 +- packages/datasource-dynamodb/src/index.ts | 11 +- .../src/introspection/introspector.ts | 22 +- .../src/schema/manual-schema.ts | 10 +- packages/datasource-dynamodb/src/settings.ts | 10 + packages/datasource-dynamodb/src/types.ts | 4 + .../src/utils/serializer.ts | 3 +- .../test/integration/dynamodb-local.test.ts | 25 +++ .../test/validator-findings.test.ts | 189 ++++++++++++++++++ 10 files changed, 283 insertions(+), 12 deletions(-) create mode 100644 packages/datasource-dynamodb/test/validator-findings.test.ts diff --git a/packages/datasource-dynamodb/README.md b/packages/datasource-dynamodb/README.md index d2a7cc7aa0..807d0c01bc 100644 --- a/packages/datasource-dynamodb/README.md +++ b/packages/datasource-dynamodb/README.md @@ -46,6 +46,8 @@ A request that no key or index serves reads the whole table. `scanPolicy` decide | `'allow'` | Scans silently. | | `'deny'` | Refuses with an explicit error. | +Any other value is refused at startup. + `maxScannedItems` (1 000 000) stops any single request that reads more items than that, with an error rather than a partial result. PartiQL native queries do not report how many items they evaluated, so for them it bounds the rows returned, and the pages fetched are capped at 1000. Both can be set on the datasource or on a collection. @@ -110,7 +112,9 @@ you filter on ranges of it. Types: `string`, `number`, `boolean`, `date` and `dateonly` (ISO 8601 strings), `timeonly`, `object`, `array`, `point` (GeoJSON, exposed as Json), `binary` (base64, filtered on the returned items). Keys -are strings or numbers. `timeonly` is exposed as Forest's `Time`. +are strings or numbers. `timeonly` is exposed as Forest's `Time`. A number beyond 2^53 is read as +text, to keep every digit. A string or number set is read as an array and written back as a set when +the introspection saw it as one, or when the manual schema declares `set: true` on the array. ## Composite keys @@ -142,6 +146,9 @@ A sort key prefix is also used as the sort key condition of queries, and checked ## Arrays as collections +`virtualArrayCollections`, the name the Cosmos DB datasource uses, is read as an alias of +`virtualCollections`. + ```ts createDynamoDBDataSource({ virtualCollections: [ diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index a4e9bddd32..22b8ae72a2 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -303,11 +303,15 @@ export default class DynamoDBCollection extends BaseCollection { return { access: { ...access, remaining: filter }, residual: intersect(moved, residual) }; } - /** Forest sends a binary attribute as base64 text, which DynamoDB must store as bytes. */ + /** Gives back the DynamoDB type Forest cannot send: bytes for base64 text, a set for an array. */ private toStored(field: string, value: unknown): unknown { - return this.isBinary(field) && typeof value === 'string' - ? Uint8Array.from(Buffer.from(value, 'base64')) - : value; + if (this.isBinary(field) && typeof value === 'string') { + return Uint8Array.from(Buffer.from(value, 'base64')); + } + + if (this.model.schema[field]?.set && Array.isArray(value)) return new Set(value); + + return value; } private isBinary(field: string): boolean { diff --git a/packages/datasource-dynamodb/src/index.ts b/packages/datasource-dynamodb/src/index.ts index 2e2d3a65f4..73313f919b 100644 --- a/packages/datasource-dynamodb/src/index.ts +++ b/packages/datasource-dynamodb/src/index.ts @@ -58,6 +58,8 @@ export type DynamoDBDataSourceOptions = DynamoDBClientOptions & introspectionConfig?: IntrospectionConfig; /** Exposes arrays of maps as collections of their own. */ virtualCollections?: VirtualCollectionConfig[]; + /** Another name for `virtualCollections`, the one the Cosmos DB datasource uses. */ + virtualArrayCollections?: VirtualCollectionConfig[]; /** Name of the connection that runs PartiQL native queries. Without it, none is offered. */ liveQueryConnection?: string; }; @@ -136,10 +138,15 @@ export function createDynamoDBDataSource( liveQueryConnection: options.liveQueryConnection, }); - if (options.virtualCollections?.length) { + const virtualCollections = [ + ...(options.virtualCollections ?? []), + ...(options.virtualArrayCollections ?? []), + ]; + + if (virtualCollections.length) { await addVirtualCollections( dataSource, - options.virtualCollections, + virtualCollections, options.introspectionConfig?.sampleSize, logger, ); diff --git a/packages/datasource-dynamodb/src/introspection/introspector.ts b/packages/datasource-dynamodb/src/introspection/introspector.ts index 8389accb18..e095753c73 100644 --- a/packages/datasource-dynamodb/src/introspection/introspector.ts +++ b/packages/datasource-dynamodb/src/introspection/introspector.ts @@ -95,6 +95,7 @@ function collectTypes( maxDepth: number, types: Record, presentInItem: Set, + arrayKinds: Record, ): void { const record = (type: DynamoDataType) => { types[path] ??= []; @@ -105,7 +106,14 @@ function collectTypes( const inferred = TypeConverter.inferTypeFromValue(value); if (inferred !== 'object') { - if (path) record(inferred); + if (path) { + record(inferred); + + if (inferred === 'array') { + arrayKinds[path] ??= { sets: 0, lists: 0 }; + arrayKinds[path][value instanceof Set ? 'sets' : 'lists'] += 1; + } + } return; } @@ -134,6 +142,7 @@ function collectTypes( maxDepth, types, presentInItem, + arrayKinds, ); } } @@ -146,10 +155,11 @@ export function inferSchema( // Attribute names are data: `constructor` or `__proto__` must not reach Object.prototype. const types: Record = Object.create(null); const presence: Record = Object.create(null); + const arrayKinds: Record = Object.create(null); for (const item of items) { const present = new Set(); - collectTypes(item, '', 0, maxDepth, types, present); + collectTypes(item, '', 0, maxDepth, types, present, arrayKinds); present.forEach(path => { presence[path] = (presence[path] ?? 0) + 1; }); @@ -164,9 +174,15 @@ export function inferSchema( paths.some(other => other.startsWith(`${path}${PATH_SEPARATOR}`)); for (const [path, observed] of Object.entries(types).filter(([name]) => !hasNested(name))) { + const type = TypeConverter.getMostSpecificType(observed); + schema[path] = { - type: TypeConverter.getMostSpecificType(observed), + type, nullable: observed.includes('null') || presence[path] < items.length, + // Only an attribute that was always a set is written back as one. + ...(type === 'array' && arrayKinds[path]?.sets > 0 && arrayKinds[path].lists === 0 + ? { set: true } + : {}), }; } diff --git a/packages/datasource-dynamodb/src/schema/manual-schema.ts b/packages/datasource-dynamodb/src/schema/manual-schema.ts index 8504fc05b5..b99bfc0eaa 100644 --- a/packages/datasource-dynamodb/src/schema/manual-schema.ts +++ b/packages/datasource-dynamodb/src/schema/manual-schema.ts @@ -61,6 +61,13 @@ function validateFields(fields: FieldDefinition[], collectionName: string, path ); } + if (field.set && !(field.type === 'array' && ['string', 'number'].includes(field.subType))) { + throw new ValidationError( + `Field '${fullPath}' in collection '${collectionName}' cannot be a set: only an array ` + + `of strings or numbers can`, + ); + } + if (field.type === 'array') { if (!field.subType) { throw new ValidationError( @@ -178,6 +185,7 @@ function flattenFields(fields: FieldDefinition[], prefix = ''): Record = {}): CollectionSettings { const now = settings.now ?? (() => Date.now()); + if (settings.scanPolicy !== undefined && !SCAN_POLICIES.includes(settings.scanPolicy)) { + throw new ValidationError( + `Invalid scanPolicy '${settings.scanPolicy}': use allow, warn or deny`, + ); + } + return { scanPolicy: 'warn', maxScannedItems: DEFAULT_MAX_SCANNED_ITEMS, diff --git a/packages/datasource-dynamodb/src/types.ts b/packages/datasource-dynamodb/src/types.ts index e7b0324061..446274636b 100644 --- a/packages/datasource-dynamodb/src/types.ts +++ b/packages/datasource-dynamodb/src/types.ts @@ -26,6 +26,8 @@ export type DynamoDataType = export type AttributeSchema = { type: DynamoDataType; nullable?: boolean; + /** An array stored as a string or number set, which must be written back as one. */ + set?: boolean; /** Schema of the items of an array of maps, used to expose it as a virtual collection. */ items?: Record; }; @@ -77,6 +79,8 @@ export type FieldDefinition = { nullable?: boolean; /** For arrays: the type of the items. */ subType?: DynamoDataType; + /** For arrays of strings or numbers: the attribute is a DynamoDB set, not a list. */ + set?: boolean; /** For objects, and arrays of objects: the nested attributes. */ fields?: FieldDefinition[]; }; diff --git a/packages/datasource-dynamodb/src/utils/serializer.ts b/packages/datasource-dynamodb/src/utils/serializer.ts index 5968fa5dfe..07ca4a708b 100644 --- a/packages/datasource-dynamodb/src/utils/serializer.ts +++ b/packages/datasource-dynamodb/src/utils/serializer.ts @@ -54,8 +54,9 @@ function hasOwnMap(target: Record, key: string): boolean { } export default class Serializer { - /** Makes a value out of what the DocumentClient reads: no set, no date object, no buffer. */ + /** Makes a value out of what the DocumentClient reads: no set, date object, buffer or bigint. */ static normalize(value: unknown): unknown { + if (typeof value === 'bigint') return value.toString(); if (value instanceof Set) return [...value].map(item => Serializer.normalize(item)); if (value instanceof Date) return value.toISOString(); if (value instanceof Uint8Array) return Buffer.from(value).toString('base64'); diff --git a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts index 1540ddd9ba..9156f0201f 100644 --- a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts +++ b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts @@ -703,6 +703,7 @@ run('DynamoDB Local', () => { { name: 'status', type: 'string' }, { name: 'createdAt', type: 'date' }, { name: 'blob', type: 'binary', nullable: true }, + { name: 'labels', type: 'array', subType: 'string', set: true, nullable: true }, { name: 'shipping', type: 'object', fields: [{ name: 'city', type: 'string' }] }, ], }, @@ -774,6 +775,30 @@ run('DynamoDB Local', () => { expect(records).toEqual([{ orderId: 'o1', blob: 'AQID' }]); }); + it('writes a declared set back as a DynamoDB set', async () => { + const collection = manual.getCollection('manual_orders'); + const key = { customerId: 'c7', orderId: 'o1' }; + + await collection.create(caller, [{ ...key, labels: ['a', 'b'] }]); + await collection.update( + caller, + new Filter({ + conditionTree: tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c7' }, + { field: 'orderId', operator: 'Equal', value: 'o1' }, + ], + }), + }), + { labels: ['b', 'c'] }, + ); + + const stored = await documents.send(new GetCommand({ TableName: MANUAL, Key: key })); + expect(stored.Item.labels).toBeInstanceOf(Set); + expect([...stored.Item.labels].sort()).toEqual(['b', 'c']); + }); + it('creates, updates and deletes through the declared schema', async () => { const collection = manual.getCollection('manual_orders'); const key = tree({ diff --git a/packages/datasource-dynamodb/test/validator-findings.test.ts b/packages/datasource-dynamodb/test/validator-findings.test.ts new file mode 100644 index 0000000000..2ea1e796de --- /dev/null +++ b/packages/datasource-dynamodb/test/validator-findings.test.ts @@ -0,0 +1,189 @@ +import { Filter, PaginatedFilter, Projection } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { fakeClient } from './fake-client'; +import { buildSimpleModel, leaf } from './helpers'; +import { createDynamoDBDataSource } from '../src'; +import { inferSchema } from '../src/introspection/introspector'; +import { convertManualSchema, validateManualSchema } from '../src/schema/manual-schema'; +import { resolveSettings } from '../src/settings'; +import Serializer from '../src/utils/serializer'; + +const caller = { timezone: 'UTC' } as never; + +describe('a number beyond the safe integer range', () => { + it('is read as text so that the response can be serialized', () => { + expect(Serializer.normalize(10n ** 20n)).toBe('100000000000000000000'); + }); + + it('does not make a list fail', async () => { + const { collection } = buildCollection( + buildSimpleModel({ schema: { id: { type: 'string' }, big: { type: 'number' } } }), + { Scan: () => ({ Items: [{ id: 'u1', big: 10n ** 20n }] }) }, + { scanConcurrency: 1 }, + ); + + const records = await collection.list( + caller, + new PaginatedFilter({}), + new Projection('id', 'big'), + ); + + expect(JSON.stringify(records)).toBe('[{"id":"u1","big":"100000000000000000000"}]'); + }); +}); + +describe('string and number sets', () => { + const model = () => + buildSimpleModel({ + schema: { id: { type: 'string' }, tags: { type: 'array', set: true } }, + }); + + it('are marked by the introspection when every sampled value is a Set', () => { + const schema = inferSchema( + [ + { id: 'a', tags: new Set(['x']), list: ['x'] }, + { id: 'b', tags: new Set(['y']), list: new Set(['z']) }, + ], + [{ name: 'id', type: 'S' }], + 5, + ); + + expect(schema.tags.set).toBe(true); + expect(schema.list.set).toBeUndefined(); + }); + + it('can be declared in a manual schema', () => { + const [table] = convertManualSchema({ + collections: [ + { + name: 'a', + tableName: 'A', + partitionKey: { name: 'id', type: 'string' }, + fields: [{ name: 'tags', type: 'array', subType: 'string', set: true }], + }, + ], + }); + + expect(table.schema.tags).toMatchObject({ type: 'array', set: true }); + }); + + it('are refused on a field that is not an array of strings or numbers', () => { + expect(() => + validateManualSchema({ + collections: [ + { + name: 'a', + tableName: 'A', + partitionKey: { name: 'id', type: 'string' }, + fields: [{ name: 'tags', type: 'string', set: true }], + }, + ], + }), + ).toThrow("Field 'tags' in collection 'a' cannot be a set"); + }); + + it('are written back as sets on create', async () => { + const { collection, client } = buildCollection(model(), { Put: () => ({}) }); + + await collection.create(caller, [{ id: 'u1', tags: ['a', 'b'] }]); + + const { tags } = client.calls('Put')[0].Item; + expect(tags).toBeInstanceOf(Set); + expect([...tags]).toEqual(['a', 'b']); + }); + + it('are written back as sets on update', async () => { + const { collection, client } = buildCollection(model(), { Update: () => ({}) }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + tags: ['a'], + }); + + const [value] = Object.values(client.calls('Update')[0].ExpressionAttributeValues); + expect(value).toBeInstanceOf(Set); + }); +}); + +describe('a manual schema collection without fields', () => { + it('exposes its keys instead of failing', () => { + const [table] = convertManualSchema({ + collections: [{ name: 'a', tableName: 'A', partitionKey: { name: 'id', type: 'string' } }], + } as never); + + expect(Object.keys(table.schema)).toEqual(['id']); + }); +}); + +describe('the option naming the virtual collections', () => { + it('also reads the name the ticket gives it', async () => { + const dataSource = await createDynamoDBDataSource({ + client: fakeClient({}), + disableIntrospection: true, + virtualArrayCollections: [ + { parentCollection: 'a', collectionName: 'a_items', arrayField: 'items' }, + ], + schema: { + collections: [ + { + name: 'a', + tableName: 'A', + partitionKey: { name: 'id', type: 'string' }, + fields: [ + { + name: 'items', + type: 'array', + subType: 'object', + fields: [{ name: 'k', type: 'string' }], + }, + ], + }, + ], + }, + })(jest.fn(), jest.fn()); + + expect(dataSource.collections.map(c => c.name).sort()).toEqual(['a', 'a_items']); + }); +}); + +describe('the scanPolicy option', () => { + it('refuses a value that is not allow, warn or deny', () => { + expect(() => resolveSettings({ scanPolicy: 'Deny' as never })).toThrow( + "Invalid scanPolicy 'Deny': use allow, warn or deny", + ); + }); + + it.each(['allow', 'warn', 'deny'] as const)('accepts %s', scanPolicy => { + expect(resolveSettings({ scanPolicy }).scanPolicy).toBe(scanPolicy); + }); +}); + +describe('native queries', () => { + it.each([ + "INSERT INTO \"Users\" VALUE {'id': 'x'}", + "UPDATE \"Users\" SET name = 'x' WHERE id = 'a'", + 'DELETE FROM "Users" WHERE id = \'a\'', + ])('refuse the statement %s', async statement => { + const client = fakeClient({ ExecuteStatement: () => ({ Items: [] }) }); + const dataSource = await createDynamoDBDataSource({ + client, + disableIntrospection: true, + schema: { + collections: [ + { + name: 'u', + tableName: 'Users', + partitionKey: { name: 'id', type: 'string' }, + fields: [], + }, + ], + }, + liveQueryConnection: 'dynamo', + })(jest.fn(), jest.fn()); + + await expect(dataSource.executeNativeQuery('dynamo', statement, {})).rejects.toThrow( + 'Only SELECT statements can be run as native queries', + ); + expect(client.calls('ExecuteStatement')).toHaveLength(0); + }); +}); From a5c4ee83d79f04646e19ad29555366f14ee9431b Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Thu, 22 Oct 2026 11:52:17 +0200 Subject: [PATCH 25/32] fix(datasource-dynamodb): remove a set emptied in Forest instead of writing an empty set Co-Authored-By: Claude Sonnet 5.5 --- .../datasource-dynamodb/src/collection.ts | 35 +++++++++--- .../src/utils/serializer.ts | 14 +++++ .../test/integration/dynamodb-local.test.ts | 17 ++++++ .../test/validator-findings.test.ts | 56 +++++++++++++++++++ 4 files changed, 114 insertions(+), 8 deletions(-) diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index 22b8ae72a2..65488d4a4d 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -108,6 +108,9 @@ function keyConditionExpression( return expressions.join(' AND '); } +/** What a set emptied in Forest turns into: DynamoDB holds no empty set, so the attribute goes. */ +const CLEARED = Symbol('cleared'); + export default class DynamoDBCollection extends BaseCollection { readonly model: TableModel; readonly executor: DynamoExecutor; @@ -309,7 +312,9 @@ export default class DynamoDBCollection extends BaseCollection { return Uint8Array.from(Buffer.from(value, 'base64')); } - if (this.model.schema[field]?.set && Array.isArray(value)) return new Set(value); + if (this.model.schema[field]?.set && Array.isArray(value)) { + return value.length ? new Set(value) : CLEARED; + } return value; } @@ -612,7 +617,9 @@ export default class DynamoDBCollection extends BaseCollection { const item = Serializer.unflatten( Object.fromEntries( - Object.entries(created).map(([field, value]) => [field, this.toStored(field, value)]), + Object.entries(created) + .map(([field, value]) => [field, this.toStored(field, value)]) + .filter(([, value]) => value !== CLEARED), ), ); @@ -638,18 +645,29 @@ export default class DynamoDBCollection extends BaseCollection { private async updateOne(key: Item, entries: Array<[string, unknown]>): Promise { const builder = new ExpressionBuilder(); const keyBuilder = new ExpressionBuilder('k', 'kv'); - const sets = entries.map( - ([field, value]) => `${builder.attribute(field)} = ${builder.value(value)}`, - ); + const sets = entries + .filter(([, value]) => value !== CLEARED) + .map(([field, value]) => `${builder.attribute(field)} = ${builder.value(value)}`); + const removes = entries + .filter(([, value]) => value === CLEARED) + .map(([field]) => builder.attribute(field)); const condition = `attribute_exists(${keyBuilder.attribute(this.model.partitionKey.name)})`; + const updateExpression = [ + sets.length ? `SET ${sets.join(', ')}` : '', + removes.length ? `REMOVE ${removes.join(', ')}` : '', + ] + .filter(Boolean) + .join(' '); try { await this.executor.update({ Key: key, - UpdateExpression: `SET ${sets.join(', ')}`, + UpdateExpression: updateExpression, ConditionExpression: condition, ExpressionAttributeNames: { ...builder.names, ...keyBuilder.names }, - ExpressionAttributeValues: builder.values, + ...(Object.keys(builder.values).length + ? { ExpressionAttributeValues: builder.values } + : {}), }); } catch (error) { if (error.name === 'ConditionalCheckFailedException') return; @@ -674,7 +692,8 @@ export default class DynamoDBCollection extends BaseCollection { // Each entry replaces what is at its path, as a SET does: a map is not merged with the old one. for (const [field, value] of entries) { - Serializer.setPath(existing, Serializer.toPathSegments(field), value); + if (value === CLEARED) Serializer.removePath(existing, Serializer.toPathSegments(field)); + else Serializer.setPath(existing, Serializer.toPathSegments(field), value); } try { diff --git a/packages/datasource-dynamodb/src/utils/serializer.ts b/packages/datasource-dynamodb/src/utils/serializer.ts index 07ca4a708b..b6afa25dff 100644 --- a/packages/datasource-dynamodb/src/utils/serializer.ts +++ b/packages/datasource-dynamodb/src/utils/serializer.ts @@ -125,6 +125,20 @@ export default class Serializer { holder[segments[segments.length - 1]] = value instanceof Date ? value.toISOString() : value; } + /** Removes the value at a path, as REMOVE does, and leaves the maps on the way as they are. */ + static removePath(target: Record, segments: string[]): void { + let holder = target; + + assertWritable(segments); + + for (const segment of segments.slice(0, -1)) { + if (!hasOwnMap(holder, segment)) return; + holder = holder[segment] as Record; + } + + delete holder[segments[segments.length - 1]]; + } + static toPathSegments(field: string): string[] { return field.split(PATH_SEPARATOR); } diff --git a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts index 9156f0201f..6c80a2ec27 100644 --- a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts +++ b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts @@ -797,6 +797,23 @@ run('DynamoDB Local', () => { const stored = await documents.send(new GetCommand({ TableName: MANUAL, Key: key })); expect(stored.Item.labels).toBeInstanceOf(Set); expect([...stored.Item.labels].sort()).toEqual(['b', 'c']); + + await collection.update( + caller, + new Filter({ + conditionTree: tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c7' }, + { field: 'orderId', operator: 'Equal', value: 'o1' }, + ], + }), + }), + { labels: [] }, + ); + + const cleared = await documents.send(new GetCommand({ TableName: MANUAL, Key: key })); + expect(cleared.Item).toEqual(key); }); it('creates, updates and deletes through the declared schema', async () => { diff --git a/packages/datasource-dynamodb/test/validator-findings.test.ts b/packages/datasource-dynamodb/test/validator-findings.test.ts index 2ea1e796de..2743b63154 100644 --- a/packages/datasource-dynamodb/test/validator-findings.test.ts +++ b/packages/datasource-dynamodb/test/validator-findings.test.ts @@ -105,6 +105,62 @@ describe('string and number sets', () => { }); }); +describe('clearing a set', () => { + const model = () => + buildSimpleModel({ + schema: { + id: { type: 'string' }, + name: { type: 'string', nullable: true }, + tags: { type: 'array', set: true, nullable: true }, + }, + }); + const byId = () => new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }); + + it('leaves the attribute out on create, since DynamoDB holds no empty set', async () => { + const { collection, client } = buildCollection(model(), { Put: () => ({}) }); + + const [created] = await collection.create(caller, [{ id: 'u1', tags: [] }]); + + expect(Object.keys(client.calls('Put')[0].Item)).toEqual(['id']); + expect(created.tags).toEqual([]); + }); + + it('removes the attribute on update', async () => { + const { collection, client } = buildCollection(model(), { Update: () => ({}) }); + + await collection.update(caller, byId(), { tags: [] }); + + const call = client.calls('Update')[0]; + expect(call.UpdateExpression).toBe('REMOVE #n0'); + expect(call.ExpressionAttributeNames).toMatchObject({ '#n0': 'tags' }); + expect(call.ExpressionAttributeValues).toBeUndefined(); + }); + + it('removes it next to the other attributes of the same update', async () => { + const { collection, client } = buildCollection(model(), { Update: () => ({}) }); + + await collection.update(caller, byId(), { name: 'x', tags: [] }); + + expect(client.calls('Update')[0].UpdateExpression).toBe('SET #n0 = :v0 REMOVE #n1'); + }); + + it('removes it in the client-side rewrite too', async () => { + const { collection, client } = buildCollection(model(), { + Update: () => { + throw Object.assign(new Error('The document path provided is invalid for update'), { + name: 'ValidationException', + }); + }, + Get: () => ({ Item: { id: 'u1', tags: new Set(['a']), name: 'n' } }), + Put: () => ({}), + }); + + await collection.update(caller, byId(), { name: 'x', tags: [] }); + + expect(client.calls('Put')[0].Item).toEqual({ id: 'u1', name: 'x' }); + }); +}); + describe('a manual schema collection without fields', () => { it('exposes its keys instead of failing', () => { const [table] = convertManualSchema({ From 170c5d3d68cad1df5ce9a626b2517301b390ffa9 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Thu, 22 Oct 2026 11:58:30 +0200 Subject: [PATCH 26/32] fix(datasource-dynamodb): keep a chart computed during a write out of the cache Co-Authored-By: Claude Sonnet 5.5 --- .../datasource-dynamodb/src/collection.ts | 3 +- .../test/validator-findings.test.ts | 37 ++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index 65488d4a4d..06d5f2ae05 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -158,7 +158,8 @@ export default class DynamoDBCollection extends BaseCollection { invalidateCaches(): void { this.settings.paginationCache.invalidateTable(this.model.tableName); - this.settings.chartCaches.get(this.model.tableName)?.clear(); + // Dropped, not cleared: a chart still being computed holds the old map and writes into it. + this.settings.chartCaches.delete(this.model.tableName); } async create(caller: Caller, data: RecordData[]): Promise { diff --git a/packages/datasource-dynamodb/test/validator-findings.test.ts b/packages/datasource-dynamodb/test/validator-findings.test.ts index 2743b63154..ec260e4b5c 100644 --- a/packages/datasource-dynamodb/test/validator-findings.test.ts +++ b/packages/datasource-dynamodb/test/validator-findings.test.ts @@ -1,4 +1,4 @@ -import { Filter, PaginatedFilter, Projection } from '@forestadmin/datasource-toolkit'; +import { Aggregation, Filter, PaginatedFilter, Projection } from '@forestadmin/datasource-toolkit'; import { buildCollection } from './collection-helpers'; import { fakeClient } from './fake-client'; @@ -243,3 +243,38 @@ describe('native queries', () => { expect(client.calls('ExecuteStatement')).toHaveLength(0); }); }); + +describe('a chart computed while a write happens', () => { + it('does not put its stale result back in the cache', async () => { + let release: (value: unknown) => void; + const firstScan = new Promise(resolve => { + release = resolve; + }); + let scans = 0; + const { collection } = buildCollection( + buildSimpleModel(), + { + Scan: () => { + scans += 1; + + return scans === 1 ? firstScan : { Items: [{ age: 2 }] }; + }, + Put: () => ({}), + }, + { chartCacheTtlMs: 60000, scanConcurrency: 1 }, + ); + const sum = new Aggregation({ operation: 'Sum', field: 'age' }); + + const started = collection.aggregate(caller, new Filter({}), sum); + await new Promise(resolve => { + setTimeout(resolve, 0); + }); + await collection.create(caller, [{ id: 'u9', age: 5 }]); + release({ Items: [{ age: 1 }] }); + await started; + + const after = await collection.aggregate(caller, new Filter({}), sum); + + expect(after).toEqual([{ value: 2, group: {} }]); + }); +}); From cfbd556e9e4b232f2b2e367ea52f592936046888 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Thu, 22 Oct 2026 14:56:08 +0200 Subject: [PATCH 27/32] test(datasource-dynamodb): check the raw item after writes through virtual collections Co-Authored-By: Claude Sonnet 5.5 --- .../test/integration/dynamodb-local.test.ts | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts index 6c80a2ec27..53ede9345d 100644 --- a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts +++ b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts @@ -853,6 +853,161 @@ run('DynamoDB Local', () => { }); }); + describe('writes through virtual collections of a composite key table', () => { + const VC = `vc-${suffix}`; + let vc: DataSource; + + const original = () => ({ + 'c1|o1': { + customerId: 'c1', + orderId: 'o1', + status: 'open', + note: 'keep', + lines: [ + { sku: 'A', qty: 1, parts: [{ n: 1 }, { n: 2 }] }, + { sku: 'B', qty: 2, parts: [{ n: 3 }] }, + ], + }, + 'c1|o2': { + customerId: 'c1', + orderId: 'o2', + lines: [{ sku: 'A', qty: 10, parts: [{ n: 1 }] }], + }, + 'c2|o1': { customerId: 'c2', orderId: 'o1', lines: [{ sku: 'A', qty: 100, parts: [] }] }, + }); + const keyOf = (id: string) => { + const [customerId, orderId] = id.split('|'); + + return { customerId, orderId }; + }; + + const raw = async (id: string) => + (await documents.send(new GetCommand({ TableName: VC, Key: keyOf(id) }))).Item; + + const at = (id: string, index: number, extra: Record = {}) => + tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: keyOf(id).customerId }, + { field: 'orderId', operator: 'Equal', value: keyOf(id).orderId }, + { field: 'index', operator: 'Equal', value: index }, + ...Object.entries(extra).map(([field, value]) => ({ + field, + operator: 'Equal', + value, + })), + ], + }); + + beforeAll(async () => { + await client.send( + new CreateTableCommand({ + TableName: VC, + BillingMode: 'PAY_PER_REQUEST', + KeySchema: [ + { AttributeName: 'customerId', KeyType: 'HASH' }, + { AttributeName: 'orderId', KeyType: 'RANGE' }, + ], + AttributeDefinitions: [ + { AttributeName: 'customerId', AttributeType: 'S' }, + { AttributeName: 'orderId', AttributeType: 'S' }, + ], + }), + ); + vc = await createDynamoDBDataSource({ + client: documents, + builder: configurator => + configurator.addCollectionFromTable({ name: 'vc_orders', tableName: VC }), + virtualCollections: [ + { parentCollection: 'vc_orders', collectionName: 'order_lines', arrayField: 'lines' }, + { parentCollection: 'order_lines', collectionName: 'line_parts', arrayField: 'parts' }, + ], + chartCacheTtlMs: 0, + })(jest.fn(), jest.fn()); + }); + + beforeEach(async () => { + await documents.send( + new BatchWriteCommand({ + RequestItems: { + [VC]: Object.values(original()).map(Item => ({ PutRequest: { Item } })), + }, + }), + ); + }); + + afterAll(async () => { + await client.send(new DeleteTableCommand({ TableName: VC })).catch(() => undefined); + }); + + it('edits the line of one order and leaves every other item and attribute alone', async () => { + await vc + .getCollection('order_lines') + .update(caller, new Filter({ conditionTree: at('c1|o1', 1) }), { qty: 20 }); + + const expected = original(); + expected['c1|o1'].lines[1].qty = 20; + + expect(await raw('c1|o1')).toEqual(expected['c1|o1']); + expect(await raw('c1|o2')).toEqual(expected['c1|o2']); + expect(await raw('c2|o1')).toEqual(expected['c2|o1']); + }); + + it('tells apart two items that share the sort key, or the partition key', async () => { + await vc + .getCollection('order_lines') + .update(caller, new Filter({ conditionTree: at('c2|o1', 0) }), { sku: 'Z' }); + + const expected = original(); + expected['c2|o1'].lines[0].sku = 'Z'; + + expect(await raw('c2|o1')).toEqual(expected['c2|o1']); + expect(await raw('c1|o1')).toEqual(expected['c1|o1']); + expect(await raw('c1|o2')).toEqual(expected['c1|o2']); + }); + + it('edits the element of an array inside an array', async () => { + await vc + .getCollection('line_parts') + .update(caller, new Filter({ conditionTree: at('c1|o1', 1, { order_linesIndex: 0 }) }), { + n: 99, + }); + + const expected = original(); + expected['c1|o1'].lines[0].parts[1].n = 99; + + expect(await raw('c1|o1')).toEqual(expected['c1|o1']); + expect(await raw('c1|o2')).toEqual(expected['c1|o2']); + }); + + it('deletes the line it names and keeps the order and its other lines', async () => { + await vc + .getCollection('order_lines') + .delete(caller, new Filter({ conditionTree: at('c1|o1', 0) })); + + const expected = original(); + expected['c1|o1'].lines.splice(0, 1); + + expect(await raw('c1|o1')).toEqual(expected['c1|o1']); + expect(await raw('c1|o2')).toEqual(expected['c1|o2']); + expect(await raw('c2|o1')).toEqual(expected['c2|o1']); + }); + + it('appends a line to the order its keys name', async () => { + const [created] = await vc + .getCollection('order_lines') + .create(caller, [{ customerId: 'c1', orderId: 'o2', sku: 'N', qty: 5 }]); + + const expected = original(); + expected['c1|o2'].lines.push({ sku: 'N', qty: 5 } as never); + + expect(created).toMatchObject({ customerId: 'c1', orderId: 'o2', index: 1 }); + expect(await raw('c1|o2')).toEqual(expected['c1|o2']); + expect(await raw('c1|o1')).toEqual(expected['c1|o1']); + expect(await raw('c2|o1')).toEqual(expected['c2|o1']); + }); + }); + describe('native query', () => { it('runs a PartiQL statement with parameters', async () => { const rows = await dataSource.executeNativeQuery( From 519b61f17c057f1b5f792ee95d871364e5c57491 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Thu, 22 Oct 2026 14:56:25 +0200 Subject: [PATCH 28/32] style(datasource-dynamodb): pad the statements of the virtual collection tests Co-Authored-By: Claude Sonnet 5.5 --- .../datasource-dynamodb/test/integration/dynamodb-local.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts index 53ede9345d..78d84e54e3 100644 --- a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts +++ b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts @@ -875,6 +875,7 @@ run('DynamoDB Local', () => { }, 'c2|o1': { customerId: 'c2', orderId: 'o1', lines: [{ sku: 'A', qty: 100, parts: [] }] }, }); + const keyOf = (id: string) => { const [customerId, orderId] = id.split('|'); From 7960ec9912b352ff5da80fb4171676e4657c8d73 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Tue, 27 Oct 2026 18:04:18 +0100 Subject: [PATCH 29/32] feat(datasource-dynamodb): add introspect, an additive builder and the @@@ separator The AWS SDK clients are dependencies instead of peer dependencies. Nested maps are columns named with @@@, which Forest shows as a path. The builder adjusts the tables it names and the other tables are still introspected. The introspection option and the introspect function replace the schema and disableIntrospection options, as in the other datasources. An attribute an item does not have is read as null, so Present and Blank tell it apart from a value. Co-Authored-By: Claude Sonnet 5.5 --- packages/datasource-dynamodb/README.md | 133 ++++++---- packages/datasource-dynamodb/package.json | 10 +- .../src/array-collection.ts | 11 +- .../datasource-dynamodb/src/collection.ts | 11 +- packages/datasource-dynamodb/src/index.ts | 95 +++++--- .../src/introspection/introspector.ts | 7 +- .../src/schema/manual-schema.ts | 122 ++++++++-- packages/datasource-dynamodb/src/types.ts | 19 +- .../src/utils/expression-builder.ts | 2 +- .../src/utils/serializer.ts | 6 +- .../test/array-review-findings.test.ts | 2 +- .../test/collection.list.test.ts | 10 +- .../test/collection.schema.test.ts | 4 +- .../test/collection.write.test.ts | 10 +- .../test/datasource.test.ts | 56 ++--- .../test/integration/dynamodb-local.test.ts | 137 +++++++---- .../test/introspection-api.test.ts | 228 ++++++++++++++++++ .../test/introspection/introspector.test.ts | 10 +- .../test/schema/manual-schema.test.ts | 30 ++- .../test/second-round-findings.test.ts | 16 +- .../test/third-round-findings.test.ts | 12 +- .../test/utils/aggregator.test.ts | 8 +- .../test/utils/expression-builder.test.ts | 4 +- .../test/utils/serializer.test.ts | 8 +- .../test/validator-findings.test.ts | 139 ++++++++--- .../test/virtual-collections.test.ts | 64 ++--- 26 files changed, 844 insertions(+), 310 deletions(-) create mode 100644 packages/datasource-dynamodb/test/introspection-api.test.ts diff --git a/packages/datasource-dynamodb/README.md b/packages/datasource-dynamodb/README.md index 807d0c01bc..7115e7ee1d 100644 --- a/packages/datasource-dynamodb/README.md +++ b/packages/datasource-dynamodb/README.md @@ -1,10 +1,10 @@ # @forestadmin/datasource-dynamodb -Forest Admin datasource for Amazon DynamoDB: introspection or manual schema, composite keys, +Forest Admin datasource for Amazon DynamoDB: introspection (live, saved or written by hand), composite keys, relations, search, aggregations and charts, single-table design. ```bash -yarn add @forestadmin/datasource-dynamodb @aws-sdk/client-dynamodb @aws-sdk/lib-dynamodb +yarn add @forestadmin/datasource-dynamodb ``` ```ts @@ -26,15 +26,15 @@ DynamoDB reads fast by key and scans otherwise, and it cannot sort, count or agg server. The datasource uses the key whenever a filter allows it and does the rest on the items it reads. What that means for each feature: -| Feature | How it works | -| ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Filters | Pushed down when DynamoDB can evaluate them. `Like`, `ILike`, `IContains`, `EndsWith`, date intervals and the like are applied on the returned items, so they read more than they return. | -| Access path | A `Query` on the table, a global or local secondary index when an equality pins its partition key (with the sort key range when there is one), a `BatchGetItem` when the filter names whole keys, a `Scan` otherwise. An index leaves out the items that lack its sort key, so one that has a sort key is only read when the filter has a condition on that key, or when the index is declared with `coversAllItems: true` in a manual schema. | -| Sort | Free on the sort key of the table or index being read. On any other attribute, the matching items are read, then sorted in memory. | -| Pagination | There is no offset: the datasource remembers the key where earlier pages stopped and resumes from the closest one. `maxOffset` (100 000) caps how deep a page can be. | -| Count and charts | Computed by reading the items (a counting `Scan`/`Query` for a plain count), in parallel segments for a table. Results are kept in memory for `chartCacheTtlMs` (30 s). | -| Search | Evaluated on the items: it scans. Case-insensitive, accents kept. | -| Relations | None in DynamoDB. Declare them with `addManyToOneRelation` and friends: a lookup by key is one `BatchGetItem`, a relation backed by a secondary index is a `Query`. | +| Feature | How it works | +| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Filters | Pushed down when DynamoDB can evaluate them. `Like`, `ILike`, `IContains`, `EndsWith`, date intervals and the like are applied on the returned items, so they read more than they return. | +| Access path | A `Query` on the table, a global or local secondary index when an equality pins its partition key (with the sort key range when there is one), a `BatchGetItem` when the filter names whole keys, a `Scan` otherwise. An index leaves out the items that lack its sort key, so one that has a sort key is only read when the filter has a condition on that key, or when the index is declared with `coversAllItems: true` in the introspection. | +| Sort | Free on the sort key of the table or index being read. On any other attribute, the matching items are read, then sorted in memory. | +| Pagination | There is no offset: the datasource remembers the key where earlier pages stopped and resumes from the closest one. `maxOffset` (100 000) caps how deep a page can be. | +| Count and charts | Computed by reading the items (a counting `Scan`/`Query` for a plain count), in parallel segments for a table. Results are kept in memory for `chartCacheTtlMs` (30 s). | +| Search | Evaluated on the items: it scans. Case-insensitive, accents kept. | +| Relations | None in DynamoDB. Declare them with `addManyToOneRelation` and friends: a lookup by key is one `BatchGetItem`, a relation backed by a secondary index is a `Query`. | ## Controlling cost: `scanPolicy` @@ -52,60 +52,101 @@ Any other value is refused at startup. rather than a partial result. PartiQL native queries do not report how many items they evaluated, so for them it bounds the rows returned, and the pages fetched are capped at 1000. Both can be set on the datasource or on a collection. -## Choosing the tables and naming collections +## Choosing the tables and adjusting collections + +`tables: ['Customers', 'Orders']` only introspects the listed tables, and names each collection +after its table. + +The builder adjusts collections one table at a time: to name them, to tune them, or to share a table +between several collections. The tables it names are not introspected a second time, and the other +tables are still introspected as usual: ```ts createDynamoDBDataSource({ region: 'eu-west-1', builder: configurator => configurator - .addCollectionFromTable({ name: 'customers', tableName: 'Customers', sampleSize: 200 }) - .addCollectionFromTable({ name: 'orders', tableName: 'Orders', scanPolicy: 'deny' }), + .addCollectionFromTable({ + name: 'users', + tableName: 'App', + entity: { sortKeyPrefix: 'USER#' }, + }) + .addCollectionFromTable({ + name: 'orders', + tableName: 'App', + entity: { sortKeyPrefix: 'ORDER#' }, + }), }); ``` -`tables: ['Customers', 'Orders']` keeps the default naming (collections are named after tables) and -only introspects the listed tables. +Here `App` becomes two collections and every other table keeps its own. Add `tables: []` to expose +the builder collections only. ## Introspection `DescribeTable` gives the keys and the indexes. The other attributes are typed from a sample of `sampleSize` items (default 100): DynamoDB does not type them. A scan has no order, so the sample is -not the latest items. Nested maps become `address->city` columns up to `maxDepth` (5); an attribute +not the latest items. Nested maps become `address@@@city` columns up to `maxDepth` (5); an attribute that is absent or null in part of the sample is nullable; mixed types become Json. -## Manual schema +## Saving the introspection, or writing it by hand + +`introspect` samples the tables and describes them as collection definitions. Save the result and +give it back as the `introspection` of the datasource: the agent then starts without calling +`DescribeTable` or sampling the tables. ```ts -createDynamoDBDataSource({ - region: 'eu-west-1', - disableIntrospection: true, - schema: { - collections: [ - { - name: 'orders', - tableName: 'Orders', - partitionKey: { name: 'customerId', type: 'string' }, - sortKey: { name: 'orderId', type: 'string' }, - indexes: [ - { - name: 'by-status', - type: 'GSI', - partitionKey: { name: 'status', type: 'string' }, - sortKey: { name: 'createdAt', type: 'string' }, - }, - ], - fields: [ - { name: 'total', type: 'number' }, - { name: 'status', type: 'string', nullable: true }, - { name: 'address', type: 'object', fields: [{ name: 'city', type: 'string' }] }, - ], - }, - ], +import { createDynamoDBDataSource, introspect } from '@forestadmin/datasource-dynamodb'; + +// Once, from a script +const introspection = await introspect({ region: 'eu-west-1' }); +await fs.promises.writeFile('introspection.json', JSON.stringify(introspection, null, 2)); + +// At startup +createDynamoDBDataSource( + { region: 'eu-west-1' }, + { introspection: JSON.parse(await fs.promises.readFile('introspection.json', 'utf8')) }, +); +``` + +The file is plain JSON, so it can be edited: add what the sampling cannot know (a nullable +attribute, an index that `coversAllItems`, a set), or write the whole of it by hand. An +`introspection` cannot be combined with a `builder`. + +```ts +createDynamoDBDataSource( + { region: 'eu-west-1' }, + { + introspection: { + version: 1, + collections: [ + { + name: 'orders', + tableName: 'Orders', + partitionKey: { name: 'customerId', type: 'string' }, + sortKey: { name: 'orderId', type: 'string' }, + indexes: [ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'string' }, + sortKey: { name: 'createdAt', type: 'string' }, + }, + ], + fields: [ + { name: 'total', type: 'number' }, + { name: 'status', type: 'string', nullable: true }, + { name: 'address', type: 'object', fields: [{ name: 'city', type: 'string' }] }, + ], + }, + ], + }, }, -}); +); ``` +Nested maps are columns named with `@@@` (`address@@@city`), which Forest shows as a path. + Dates are compared by their instant when sorted or aggregated here, but DynamoDB orders and compares them as text on a key and when it filters: store them in one offset (UTC) if a date is a sort key or you filter on ranges of it. @@ -114,7 +155,7 @@ Types: `string`, `number`, `boolean`, `date` and `dateonly` (ISO 8601 strings), `array`, `point` (GeoJSON, exposed as Json), `binary` (base64, filtered on the returned items). Keys are strings or numbers. `timeonly` is exposed as Forest's `Time`. A number beyond 2^53 is read as text, to keep every digit. A string or number set is read as an array and written back as a set when -the introspection saw it as one, or when the manual schema declares `set: true` on the array. +the introspection saw it as one, or when the introspection declares `set: true` on the array. ## Composite keys @@ -161,7 +202,7 @@ Each element of the array becomes a record, identified by the key of the table i in the array. Creating appends, updating sets the element's attributes, and deleting removes it, so the index of the following elements shifts. -`arrayField` can point inside a map (`'meta->items'`). An array of an array is a collection whose +`arrayField` can point inside a map (`'meta@@@items'`). An array of an array is a collection whose parent is the first one, declared after it: the position in the outer array becomes a key column named `Index`, and creating an inner element needs it. diff --git a/packages/datasource-dynamodb/package.json b/packages/datasource-dynamodb/package.json index 97ef334284..72736dcfc6 100644 --- a/packages/datasource-dynamodb/package.json +++ b/packages/datasource-dynamodb/package.json @@ -12,15 +12,9 @@ "directory": "packages/datasource-dynamodb" }, "dependencies": { - "@forestadmin/datasource-toolkit": "1.56.2" - }, - "peerDependencies": { "@aws-sdk/client-dynamodb": "^3.750.0", - "@aws-sdk/lib-dynamodb": "^3.750.0" - }, - "devDependencies": { - "@aws-sdk/client-dynamodb": "^3.750.0", - "@aws-sdk/lib-dynamodb": "^3.750.0" + "@aws-sdk/lib-dynamodb": "^3.750.0", + "@forestadmin/datasource-toolkit": "1.56.2" }, "files": [ "dist/**/*.js", diff --git a/packages/datasource-dynamodb/src/array-collection.ts b/packages/datasource-dynamodb/src/array-collection.ts index 2d0a996769..0af9d82bda 100644 --- a/packages/datasource-dynamodb/src/array-collection.ts +++ b/packages/datasource-dynamodb/src/array-collection.ts @@ -42,7 +42,7 @@ type Level = { arrayField: string; indexColumn: string }; export type ArrayCollectionConfig = { name: string; - /** Attribute of the parent holding the array, `a->b` for one inside a map. */ + /** Attribute of the parent holding the array, `a@@@b` for one inside a map. */ arrayField: string; /** Attributes of an item, by flattened name. */ itemSchema: Record; @@ -237,6 +237,13 @@ export default class ArrayCollection extends BaseCollection { .join('.'); } + /** Reads an attribute the element does not have as a null, as the collection of a table does. */ + private itemColumns(record: RecordData): RecordData { + for (const column of this.itemFields) record[column] ??= null; + + return record; + } + /** The columns identifying the parent element, as named by the parent. */ private parentColumns(): string[] { return this.parent instanceof ArrayCollection ? this.parent.keyColumns : this.rootKeys; @@ -282,7 +289,7 @@ export default class ArrayCollection extends BaseCollection { if (element && typeof element === 'object') { const record = { // The identity comes last: an element's own `index` must not take its place. - ...Serializer.flatten(element as Item, this.itemFields), + ...this.itemColumns(Serializer.flatten(element as Item, this.itemFields)), ...Object.fromEntries( ownKeyColumns.map((name, i) => [name, parent[parentColumns[i]]]), ), diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index 06d5f2ae05..ac2172f7d5 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -117,6 +117,7 @@ export default class DynamoDBCollection extends BaseCollection { readonly settings: CollectionSettings; private readonly logger: Logger; private readonly fieldNames: Set; + private readonly columnNames: string[]; private readonly matchContext: Collection; private readonly warnedAt = new Map(); @@ -137,6 +138,8 @@ export default class DynamoDBCollection extends BaseCollection { this.addFields(convertModelToFields(model, logger)); if (model.enableCount ?? true) this.enableCount(); + this.columnNames = Object.keys(this.schema.fields); + const entityAttribute = model.entity?.attribute; // The discriminator may be left out of the schema, and is still read to tell the entities apart. @@ -276,7 +279,13 @@ export default class DynamoDBCollection extends BaseCollection { } private toRecord(item: Item): RecordData { - return Serializer.flatten(item, this.fieldNames); + const record = Serializer.flatten(item, this.fieldNames); + + // An item has no attribute it never got, where Forest reads a null, and filters on Present + // or Blank tell the two apart. + for (const column of this.columnNames) record[column] ??= null; + + return record; } private matches(tree: ConditionTree, item: Item, timezone: string): boolean { diff --git a/packages/datasource-dynamodb/src/index.ts b/packages/datasource-dynamodb/src/index.ts index 73313f919b..3e157fd2db 100644 --- a/packages/datasource-dynamodb/src/index.ts +++ b/packages/datasource-dynamodb/src/index.ts @@ -2,12 +2,12 @@ import type { ConfigurationOptions } from './introspection/builder'; import type { DynamoDBClientOptions, DynamoDBSettingsOptions, + Introspection, IntrospectionConfig, - ManualSchemaConfig, TableModel, VirtualCollectionConfig, } from './types'; -import type { DataSourceFactory } from '@forestadmin/datasource-toolkit'; +import type { DataSourceFactory, Logger } from '@forestadmin/datasource-toolkit'; import { ValidationError } from '@forestadmin/datasource-toolkit'; @@ -19,8 +19,9 @@ import { DEFAULT_SAMPLE_SIZE, introspectTables, } from './introspection/introspector'; -import { convertManualSchema } from './schema/manual-schema'; +import { convertManualSchema, toDefinition } from './schema/manual-schema'; import { resolveSettings } from './settings'; +import { INTROSPECTION_VERSION } from './types'; import PaginationCache from './utils/pagination-cache'; import addVirtualCollections from './virtual-collections'; @@ -40,6 +41,7 @@ export type { EntityDefinition, FieldDefinition, IndexDefinition, + Introspection, IntrospectionConfig, ManualSchemaConfig, ScanPolicy, @@ -48,13 +50,16 @@ export type { export type DynamoDBDataSourceOptions = DynamoDBClientOptions & DynamoDBSettingsOptions & { - /** Tables to introspect, instead of every table of the account and region. */ + /** + * Tables to introspect, instead of every table of the account and region. An empty list + * introspects none, which leaves the builder collections alone. + */ tables?: string[]; - /** Adds collections one table at a time, to name them or to share a table between several. */ + /** + * Adjusts the collections one table at a time: to name them, tune them or share a table + * between several. The tables it names are not introspected a second time. + */ builder?: ConfigurationOptions; - /** Declares the collections instead of discovering them. Needs `disableIntrospection`. */ - schema?: ManualSchemaConfig; - disableIntrospection?: boolean; introspectionConfig?: IntrospectionConfig; /** Exposes arrays of maps as collections of their own. */ virtualCollections?: VirtualCollectionConfig[]; @@ -67,45 +72,74 @@ export type DynamoDBDataSourceOptions = DynamoDBClientOptions & const withoutUndefined = (value: T): T => Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined)) as T; -async function loadModels( +async function liveModels( options: DynamoDBDataSourceOptions, client: ReturnType, - logger: Parameters[0], + logger: Logger, ): Promise { const sampleSize = options.introspectionConfig?.sampleSize ?? DEFAULT_SAMPLE_SIZE; const maxDepth = options.introspectionConfig?.maxDepth ?? DEFAULT_MAX_DEPTH; - if (options.disableIntrospection) { - if (!options.schema) { - throw new ValidationError( - 'When disableIntrospection is true, you must provide a schema with collection definitions', - ); - } + const fromBuilder = options.builder + ? await ( + options.builder( + new DynamoDBDatasourceBuilder(client, { sampleSize, maxDepth }), + ) as DynamoDBDatasourceBuilder + ).createCollectionsFromConfiguration() + : []; - logger('Info', 'Using the manual schema (introspection disabled)'); + const models = await introspectTables( + client, + { + tables: options.tables, + exclude: fromBuilder.map(model => model.tableName), + sampleSize, + maxDepth, + }, + logger, + ); - return convertManualSchema(options.schema); - } + return [...fromBuilder, ...models]; +} + +async function loadModels( + options: DynamoDBDataSourceOptions, + introspection: Introspection | undefined, + client: ReturnType, + logger: Logger, +): Promise { + if (introspection) { + if (options.builder) { + throw new ValidationError('An introspection and a builder cannot be used together'); + } - if (options.builder) { - const builder = options.builder( - new DynamoDBDatasourceBuilder(client, { sampleSize, maxDepth }), - ) as DynamoDBDatasourceBuilder; + logger('Info', 'Using the given introspection'); - return builder.createCollectionsFromConfiguration(); + return convertManualSchema(introspection); } - const models = await introspectTables( - client, - { tables: options.tables, sampleSize, maxDepth }, - logger, - ); + const models = await liveModels(options, client, logger); if (models.length === 0) logger('Warn', 'No collections were introspected.'); return models; } +/** + * Reads the tables and describes them as collection definitions. Save the result, for instance in + * a file, and give it back as the `introspection` of `createDynamoDBDataSource` to start without + * sampling the tables. + */ +export async function introspect( + options: DynamoDBDataSourceOptions = {}, + logger: Logger = () => {}, +): Promise { + const client = createDocumentClient(options); + const models = await liveModels(options, client, logger); + + return { version: INTROSPECTION_VERSION, collections: models.map(toDefinition) }; +} + /** * Creates a datasource over DynamoDB tables. * @@ -116,6 +150,7 @@ async function loadModels( */ export function createDynamoDBDataSource( options: DynamoDBDataSourceOptions = {}, + dataSourceOptions: { introspection?: Introspection } = {}, ): DataSourceFactory { return async logger => { const client = createDocumentClient(options); @@ -133,7 +168,7 @@ export function createDynamoDBDataSource( }), ); - const models = await loadModels(options, client, logger); + const models = await loadModels(options, dataSourceOptions.introspection, client, logger); const dataSource = new DynamoDBDataSource(client, models, logger, settings, { liveQueryConnection: options.liveQueryConnection, }); diff --git a/packages/datasource-dynamodb/src/introspection/introspector.ts b/packages/datasource-dynamodb/src/introspection/introspector.ts index e095753c73..25c8dd69e5 100644 --- a/packages/datasource-dynamodb/src/introspection/introspector.ts +++ b/packages/datasource-dynamodb/src/introspection/introspector.ts @@ -291,10 +291,13 @@ async function listTables(client: DynamoDBDocumentClient): Promise { /** Introspects the given tables, or every table of the account and region. */ export async function introspectTables( client: DynamoDBDocumentClient, - options: { tables?: string[]; sampleSize?: number; maxDepth?: number }, + options: { tables?: string[]; exclude?: string[]; sampleSize?: number; maxDepth?: number }, logger: Logger, ): Promise { - const tableNames = options.tables ?? (await listTables(client)); + const excluded = new Set(options.exclude); + const tableNames = (options.tables ?? (await listTables(client))).filter( + name => !excluded.has(name), + ); const models = await mapWithConcurrency( tableNames, diff --git a/packages/datasource-dynamodb/src/schema/manual-schema.ts b/packages/datasource-dynamodb/src/schema/manual-schema.ts index b99bfc0eaa..d612a37f70 100644 --- a/packages/datasource-dynamodb/src/schema/manual-schema.ts +++ b/packages/datasource-dynamodb/src/schema/manual-schema.ts @@ -11,6 +11,7 @@ import type { import { ValidationError } from '@forestadmin/datasource-toolkit'; +import { INTROSPECTION_VERSION } from '../types'; import { PATH_SEPARATOR } from '../utils/serializer'; const FIELD_TYPES: DynamoDataType[] = [ @@ -61,32 +62,20 @@ function validateFields(fields: FieldDefinition[], collectionName: string, path ); } - if (field.set && !(field.type === 'array' && ['string', 'number'].includes(field.subType))) { + const isSetCandidate = + field.type === 'array' && (!field.subType || ['string', 'number'].includes(field.subType)); + + if (field.set && !isSetCandidate) { throw new ValidationError( `Field '${fullPath}' in collection '${collectionName}' cannot be a set: only an array ` + `of strings or numbers can`, ); } - if (field.type === 'array') { - if (!field.subType) { - throw new ValidationError( - `Array field '${fullPath}' in collection '${collectionName}' must have a subType`, - ); - } - - if (field.subType === 'object' && !field.fields?.length) { - throw new ValidationError( - `Array field '${fullPath}' in collection '${collectionName}' of objects must have ` + - `nested field definitions`, - ); - } - } - - if (field.type === 'object' && !field.fields?.length) { + if (field.type === 'array' && field.subType === 'object' && !field.fields?.length) { throw new ValidationError( - `Object field '${fullPath}' in collection '${collectionName}' must have nested field ` + - `definitions`, + `Array field '${fullPath}' in collection '${collectionName}' of objects must have ` + + `nested field definitions`, ); } @@ -152,12 +141,19 @@ function validateCollection(collection: CollectionDefinition): void { } export function validateManualSchema(schema: ManualSchemaConfig): void { + if (schema?.version !== undefined && schema.version !== INTROSPECTION_VERSION) { + throw new ValidationError( + `The introspection is in version ${schema.version}, this datasource reads version ` + + `${INTROSPECTION_VERSION}`, + ); + } + if (!Array.isArray(schema?.collections)) { - throw new ValidationError('Manual schema must have a collections array'); + throw new ValidationError('The introspection must have a collections array'); } if (schema.collections.length === 0) { - throw new ValidationError('Manual schema must have at least one collection'); + throw new ValidationError('The introspection must have at least one collection'); } const names = new Set(); @@ -179,7 +175,7 @@ function flattenFields(fields: FieldDefinition[], prefix = ''): Record + key.type === 'N' ? 'number' : 'string'; + +/** Nests the attributes of a table, flattened by name, back into field definitions. */ +function nestFields( + attributes: Record, + keys: Set, +): FieldDefinition[] { + const fields: FieldDefinition[] = []; + const objects = new Map(); + + for (const [path, attribute] of Object.entries(attributes)) { + const segments = path.split(PATH_SEPARATOR); + + if (!keys.has(path)) { + let siblings = fields; + + segments.slice(0, -1).forEach((segment, depth) => { + const objectPath = segments.slice(0, depth + 1).join(PATH_SEPARATOR); + let object = objects.get(objectPath); + + if (!object) { + object = { name: segment, type: 'object', fields: [] }; + objects.set(objectPath, object); + siblings.push(object); + } + + siblings = object.fields; + }); + + const field: FieldDefinition = { + name: segments[segments.length - 1], + type: attribute.type, + nullable: attribute.nullable ?? false, + ...(attribute.set ? { set: true } : {}), + }; + + if (attribute.items) { + field.subType = 'object'; + field.fields = nestFields(attribute.items, new Set()); + } + + siblings.push(field); + } + } + + return fields; +} + +/** The collection definition a table model comes from: what `introspect` returns. */ +export function toDefinition(model: TableModel): CollectionDefinition { + const keys = new Set([model.partitionKey.name, model.sortKey?.name].filter(Boolean)); + + return { + name: model.name, + tableName: model.tableName, + partitionKey: { name: model.partitionKey.name, type: keyType(model.partitionKey) }, + ...(model.sortKey + ? { sortKey: { name: model.sortKey.name, type: keyType(model.sortKey) } } + : {}), + ...(model.indexes.length + ? { + indexes: model.indexes.map(index => ({ + name: index.name, + type: index.type, + partitionKey: { name: index.partitionKey.name, type: keyType(index.partitionKey) }, + ...(index.sortKey + ? { sortKey: { name: index.sortKey.name, type: keyType(index.sortKey) } } + : {}), + projectsAllAttributes: index.projectsAllAttributes, + coversAllItems: index.coversAllItems ?? false, + })), + } + : {}), + fields: nestFields(model.schema, keys), + ...(model.entity ? { entity: model.entity } : {}), + ...(model.enableCount !== undefined ? { enableCount: model.enableCount } : {}), + ...(model.scanPolicy ? { scanPolicy: model.scanPolicy } : {}), + ...(model.maxScannedItems !== undefined ? { maxScannedItems: model.maxScannedItems } : {}), + }; +} diff --git a/packages/datasource-dynamodb/src/types.ts b/packages/datasource-dynamodb/src/types.ts index 446274636b..df3ba588b4 100644 --- a/packages/datasource-dynamodb/src/types.ts +++ b/packages/datasource-dynamodb/src/types.ts @@ -64,7 +64,7 @@ export type TableModel = { partitionKey: KeyAttribute; sortKey?: KeyAttribute; indexes: IndexModel[]; - /** Attributes by flattened name: nested maps are `a->b`. */ + /** Attributes by flattened name: nested maps are `a@@@b`. */ schema: Record; entity?: EntityDefinition; enableCount?: boolean; @@ -110,14 +110,27 @@ export type CollectionDefinition = { overrideTypeConverter?: OverrideTypeConverter; }; -export type ManualSchemaConfig = { collections: CollectionDefinition[] }; +/** The version of the introspection this datasource reads and writes. */ +export const INTROSPECTION_VERSION = 1; + +/** + * What the datasource knows of the tables: the output of `introspect` and the input of the + * `introspection` option, which skips the live introspection. Written by hand, it is the manual schema. + */ +export type Introspection = { + version?: typeof INTROSPECTION_VERSION; + collections: CollectionDefinition[]; +}; + +/** @deprecated The manual schema is an introspection. */ +export type ManualSchemaConfig = Introspection; export type VirtualCollectionConfig = { /** Name of the collection holding the array. */ parentCollection: string; /** Name of the new collection. */ collectionName: string; - /** Attribute holding the array, `a->b` for a nested one. */ + /** Attribute holding the array, `a@@@b` for a nested one. */ arrayField: string; }; diff --git a/packages/datasource-dynamodb/src/utils/expression-builder.ts b/packages/datasource-dynamodb/src/utils/expression-builder.ts index 4638e7b2c6..b48e79dd77 100644 --- a/packages/datasource-dynamodb/src/utils/expression-builder.ts +++ b/packages/datasource-dynamodb/src/utils/expression-builder.ts @@ -94,7 +94,7 @@ export default class ExpressionBuilder { constructor(private readonly namePrefix = 'n', private readonly valuePrefix = 'v') {} - /** Alias of an attribute path: `a->b` becomes `#n0.#n1`. */ + /** Alias of an attribute path: `a@@@b` becomes `#n0.#n1`. */ attribute(field: string): string { if (field.includes(':')) { throw new ValidationError(`Filtering on the relation field "${field}" is not supported`); diff --git a/packages/datasource-dynamodb/src/utils/serializer.ts b/packages/datasource-dynamodb/src/utils/serializer.ts index b6afa25dff..4fe0c85a3c 100644 --- a/packages/datasource-dynamodb/src/utils/serializer.ts +++ b/packages/datasource-dynamodb/src/utils/serializer.ts @@ -4,7 +4,7 @@ import { ValidationError } from '@forestadmin/datasource-toolkit'; import TypeConverter from './type-converter'; -export const PATH_SEPARATOR = '->'; +export const PATH_SEPARATOR = '@@@'; const nestedPrefixesCache = new WeakMap, Set>(); @@ -71,7 +71,7 @@ export default class Serializer { return value; } - /** Turns an item into a record: maps that have declared columns become `a->b` fields. */ + /** Turns an item into a record: maps that have declared columns become `a@@@b` fields. */ static flatten(item: Record, fields: Set): RecordData { const prefixes = nestedPrefixes(fields); const record: RecordData = {}; @@ -90,7 +90,7 @@ export default class Serializer { return record; } - /** Turns `a->b` fields back into nested maps, ready to be written. */ + /** Turns `a@@@b` fields back into nested maps, ready to be written. */ static unflatten(record: RecordData): Record { const item: Record = {}; diff --git a/packages/datasource-dynamodb/test/array-review-findings.test.ts b/packages/datasource-dynamodb/test/array-review-findings.test.ts index b87098f6fa..df3f90f203 100644 --- a/packages/datasource-dynamodb/test/array-review-findings.test.ts +++ b/packages/datasource-dynamodb/test/array-review-findings.test.ts @@ -94,7 +94,7 @@ describe('ArrayCollection review findings', () => { Put: () => ({}), }); - await collection.update(caller, new Filter({}), { 'details->note': 'hi' }); + await collection.update(caller, new Filter({}), { 'details@@@note': 'hi' }); expect(client.calls('Put')[0].Item).toEqual(order([{ sku: 'A', details: { note: 'hi' } }])); }); diff --git a/packages/datasource-dynamodb/test/collection.list.test.ts b/packages/datasource-dynamodb/test/collection.list.test.ts index cdbda404b1..f8973b5c4d 100644 --- a/packages/datasource-dynamodb/test/collection.list.test.ts +++ b/packages/datasource-dynamodb/test/collection.list.test.ts @@ -467,7 +467,7 @@ describe('DynamoDBCollection.list', () => { schema: { id: { type: 'string' }, tags: { type: 'array' }, - 'address->city': { type: 'string' }, + 'address@@@city': { type: 'string' }, }, }), { @@ -480,21 +480,21 @@ describe('DynamoDBCollection.list', () => { const records = await collection.list( caller, new PaginatedFilter({}), - new Projection('id', 'tags', 'address->city'), + new Projection('id', 'tags', 'address@@@city'), ); - expect(records).toEqual([{ id: 'u1', tags: ['a'], 'address->city': 'Paris' }]); + expect(records).toEqual([{ id: 'u1', tags: ['a'], 'address@@@city': 'Paris' }]); }); it('fetches the top-level attribute of a nested projection, with the key', async () => { const { collection, client } = buildCollection( buildSimpleModel({ - schema: { id: { type: 'string' }, 'address->city': { type: 'string' } }, + schema: { id: { type: 'string' }, 'address@@@city': { type: 'string' } }, }), { Scan: () => ({ Items: [] }) }, ); - await collection.list(caller, new PaginatedFilter({}), new Projection('address->city')); + await collection.list(caller, new PaginatedFilter({}), new Projection('address@@@city')); expect(client.calls('Scan')[0]).toMatchObject({ ProjectionExpression: '#n0, #n1', diff --git a/packages/datasource-dynamodb/test/collection.schema.test.ts b/packages/datasource-dynamodb/test/collection.schema.test.ts index 5d1fefd13f..17290ad665 100644 --- a/packages/datasource-dynamodb/test/collection.schema.test.ts +++ b/packages/datasource-dynamodb/test/collection.schema.test.ts @@ -42,12 +42,12 @@ describe('DynamoDBCollection schema', () => { it('exposes nested attributes as arrow-notation columns', () => { const { collection } = buildCollection( buildSimpleModel({ - schema: { id: { type: 'string' }, 'address->city': { type: 'string' } }, + schema: { id: { type: 'string' }, 'address@@@city': { type: 'string' } }, }), {}, ); - expect(collection.schema.fields['address->city']).toMatchObject({ columnType: 'String' }); + expect(collection.schema.fields['address@@@city']).toMatchObject({ columnType: 'String' }); }); it('does not allow sorting on Json columns', () => { diff --git a/packages/datasource-dynamodb/test/collection.write.test.ts b/packages/datasource-dynamodb/test/collection.write.test.ts index 8a9865f948..30dba29082 100644 --- a/packages/datasource-dynamodb/test/collection.write.test.ts +++ b/packages/datasource-dynamodb/test/collection.write.test.ts @@ -14,11 +14,11 @@ describe('DynamoDBCollection.create', () => { const { collection, client } = buildCollection(buildModel(), { Put: () => ({}) }); const created = await collection.create(caller, [ - { customerId: 'c1', orderId: 'o1', total: 5, 'address->city': 'Paris' }, + { customerId: 'c1', orderId: 'o1', total: 5, 'address@@@city': 'Paris' }, ]); expect(created).toEqual([ - { customerId: 'c1', orderId: 'o1', total: 5, 'address->city': 'Paris' }, + { customerId: 'c1', orderId: 'o1', total: 5, 'address@@@city': 'Paris' }, ]); expect(client.calls('Put')[0]).toEqual({ TableName: 'Orders', @@ -176,7 +176,7 @@ describe('DynamoDBCollection.update', () => { const { collection, client } = buildCollection(buildSimpleModel(), { Update: () => ({}) }); await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { - 'address->city': 'Lyon', + 'address@@@city': 'Lyon', }); expect(client.calls('Update')[0]).toMatchObject({ @@ -272,7 +272,7 @@ describe('DynamoDBCollection.update', () => { }); await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { - 'address->city': 'Lyon', + 'address@@@city': 'Lyon', }); expect(client.calls('Put')[0]).toMatchObject({ @@ -296,7 +296,7 @@ describe('DynamoDBCollection.update', () => { await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { config: { keep: 3 }, - 'address->city': 'Lyon', + 'address@@@city': 'Lyon', }); expect(client.calls('Put')[0].Item).toEqual({ diff --git a/packages/datasource-dynamodb/test/datasource.test.ts b/packages/datasource-dynamodb/test/datasource.test.ts index b9ff9ca9f4..6d4cb5d0df 100644 --- a/packages/datasource-dynamodb/test/datasource.test.ts +++ b/packages/datasource-dynamodb/test/datasource.test.ts @@ -52,6 +52,7 @@ describe('createDynamoDBDataSource', () => { const dataSource = await createDynamoDBDataSource({ client, + tables: [], builder: configurator => configurator.addCollectionFromTable({ name: 'customers', @@ -67,23 +68,26 @@ describe('createDynamoDBDataSource', () => { expect(client.calls('ListTables')).toHaveLength(0); }); - it('builds collections from a manual schema without calling AWS', async () => { + it('builds collections from a given introspection without calling AWS', async () => { const client = fakeClient({}); - const dataSource = await createDynamoDBDataSource({ - client, - disableIntrospection: true, - schema: { - collections: [ - { - name: 'users', - tableName: 'Users', - partitionKey: { name: 'id', type: 'string' }, - fields: [{ name: 'name', type: 'string' }], - }, - ], + const dataSource = await createDynamoDBDataSource( + { + client, }, - })(jest.fn(), jest.fn()); + { + introspection: { + collections: [ + { + name: 'users', + tableName: 'Users', + partitionKey: { name: 'id', type: 'string' }, + fields: [{ name: 'name', type: 'string' }], + }, + ], + }, + }, + )(jest.fn(), jest.fn()); expect(dataSource.getCollection('users').schema.fields.name).toMatchObject({ columnType: 'String', @@ -91,26 +95,16 @@ describe('createDynamoDBDataSource', () => { expect(client.send).not.toHaveBeenCalled(); }); - it('requires a schema when introspection is disabled', async () => { - const factory = createDynamoDBDataSource({ - client: fakeClient({}), - disableIntrospection: true, - }); - - await expect(factory(jest.fn(), jest.fn())).rejects.toThrow( - 'When disableIntrospection is true, you must provide a schema', + it('rejects an invalid introspection', async () => { + const factory = createDynamoDBDataSource( + { + client: fakeClient({}), + }, + { introspection: { collections: [] } }, ); - }); - - it('rejects an invalid manual schema', async () => { - const factory = createDynamoDBDataSource({ - client: fakeClient({}), - disableIntrospection: true, - schema: { collections: [] }, - }); await expect(factory(jest.fn(), jest.fn())).rejects.toThrow( - 'Manual schema must have at least one collection', + 'The introspection must have at least one collection', ); }); diff --git a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts index 78d84e54e3..507603ce56 100644 --- a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts +++ b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts @@ -17,7 +17,7 @@ import { Sort, } from '@forestadmin/datasource-toolkit'; -import { createDynamoDBDataSource } from '../../src'; +import { createDynamoDBDataSource, introspect } from '../../src'; /** * Runs against DynamoDB Local (`docker compose up -d`, then DYNAMODB_ENDPOINT=http://localhost:8000): @@ -149,7 +149,7 @@ run('DynamoDB Local', () => { virtualCollections: [ { parentCollection: 'orders', collectionName: 'order_lines', arrayField: 'lines' }, { parentCollection: 'order_lines', collectionName: 'line_parts', arrayField: 'parts' }, - { parentCollection: 'orders', collectionName: 'meta_items', arrayField: 'meta->items' }, + { parentCollection: 'orders', collectionName: 'meta_items', arrayField: 'meta@@@items' }, ], liveQueryConnection: 'dynamo', chartCacheTtlMs: 0, @@ -463,15 +463,15 @@ run('DynamoDB Local', () => { ], }), }), - { total: 2, 'address->city': 'Lyon' }, + { total: 2, 'address@@@city': 'Lyon' }, ); const [updated] = await orders().list( caller, new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c9') }), - new Projection('total', 'address->city'), + new Projection('total', 'address@@@city'), ); - expect(updated).toEqual({ total: 2, 'address->city': 'Lyon' }); + expect(updated).toEqual({ total: 2, 'address@@@city': 'Lyon' }); await expect(orders().create(caller, [{ ...key, total: 3 }])).rejects.toThrow( 'already exists', @@ -680,37 +680,41 @@ run('DynamoDB Local', () => { ); manualLogger = jest.fn(); - manual = await createDynamoDBDataSource({ - client: documents, - disableIntrospection: true, - schema: { - collections: [ - { - name: 'manual_orders', - tableName: MANUAL, - partitionKey: { name: 'customerId', type: 'string' }, - sortKey: { name: 'orderId', type: 'string' }, - indexes: [ - { - name: 'by-status', - type: 'GSI', - partitionKey: { name: 'status', type: 'string' }, - sortKey: { name: 'createdAt', type: 'string' }, - }, - ], - fields: [ - { name: 'total', type: 'number' }, - { name: 'status', type: 'string' }, - { name: 'createdAt', type: 'date' }, - { name: 'blob', type: 'binary', nullable: true }, - { name: 'labels', type: 'array', subType: 'string', set: true, nullable: true }, - { name: 'shipping', type: 'object', fields: [{ name: 'city', type: 'string' }] }, - ], - }, - ], + manual = await createDynamoDBDataSource( + { + client: documents, + + chartCacheTtlMs: 0, }, - chartCacheTtlMs: 0, - })(manualLogger, jest.fn()); + { + introspection: { + collections: [ + { + name: 'manual_orders', + tableName: MANUAL, + partitionKey: { name: 'customerId', type: 'string' }, + sortKey: { name: 'orderId', type: 'string' }, + indexes: [ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'string' }, + sortKey: { name: 'createdAt', type: 'string' }, + }, + ], + fields: [ + { name: 'total', type: 'number' }, + { name: 'status', type: 'string' }, + { name: 'createdAt', type: 'date' }, + { name: 'blob', type: 'binary', nullable: true }, + { name: 'labels', type: 'array', subType: 'string', set: true, nullable: true }, + { name: 'shipping', type: 'object', fields: [{ name: 'city', type: 'string' }] }, + ], + }, + ], + }, + }, + )(manualLogger, jest.fn()); }); afterAll(async () => { @@ -724,7 +728,7 @@ run('DynamoDB Local', () => { total: { columnType: 'Number' }, status: { columnType: 'String' }, createdAt: { columnType: 'Date' }, - 'shipping->city': { columnType: 'String' }, + 'shipping@@@city': { columnType: 'String' }, }); }); @@ -743,11 +747,11 @@ run('DynamoDB Local', () => { conditions: [ { field: 'status', operator: 'Equal', value: 'paid' }, { field: 'createdAt', operator: 'GreaterThan', value: '2000-01-01T00:00:00Z' }, - { field: 'shipping->city', operator: 'Equal', value: 'Paris' }, + { field: 'shipping@@@city', operator: 'Equal', value: 'Paris' }, ], }), }), - new Projection('orderId', 'shipping->city'), + new Projection('orderId', 'shipping@@@city'), ); expect(records.map(r => r.orderId).sort()).toEqual(['o1', 'o3', 'o5']); @@ -827,19 +831,19 @@ run('DynamoDB Local', () => { }); await collection.create(caller, [ - { customerId: 'c9', orderId: 'o1', total: 5, 'shipping->city': 'Rome' }, + { customerId: 'c9', orderId: 'o1', total: 5, 'shipping@@@city': 'Rome' }, ]); await collection.update(caller, new Filter({ conditionTree: key }), { - 'shipping->city': 'Milan', + 'shipping@@@city': 'Milan', }); expect( await collection.list( caller, new PaginatedFilter({ conditionTree: key }), - new Projection('total', 'shipping->city'), + new Projection('total', 'shipping@@@city'), ), - ).toEqual([{ total: 5, 'shipping->city': 'Milan' }]); + ).toEqual([{ total: 5, 'shipping@@@city': 'Milan' }]); await collection.delete(caller, new Filter({ conditionTree: key })); @@ -1009,6 +1013,55 @@ run('DynamoDB Local', () => { }); }); + describe('introspect', () => { + it('describes a table so that a datasource starts from it without sampling', async () => { + const introspection = await introspect({ client: documents, tables: [ORDERS] }); + + expect(JSON.parse(JSON.stringify(introspection))).toEqual(introspection); + expect(introspection.collections[0]).toMatchObject({ + tableName: ORDERS, + partitionKey: { name: 'customerId', type: 'string' }, + sortKey: { name: 'orderId', type: 'string' }, + indexes: [{ name: 'by-status', type: 'GSI' }], + }); + + const offline = await createDynamoDBDataSource({ client: documents }, { introspection })( + jest.fn(), + jest.fn(), + ); + const records = await offline.getCollection(ORDERS).list( + caller, + new PaginatedFilter({ + conditionTree: tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c1' }, + { field: 'orderId', operator: 'Equal', value: 'o05' }, + ], + }), + }), + new Projection('customerId', 'orderId'), + ); + + expect(records).toEqual([{ customerId: 'c1', orderId: 'o05' }]); + }); + + it('keeps introspecting the other tables when the builder adjusts one', async () => { + const adjusted = await createDynamoDBDataSource({ + client: documents, + tables: [ORDERS], + builder: configurator => + configurator.addCollectionFromTable({ + name: 'shared_users', + tableName: SHARED, + entity: { sortKeyPrefix: 'USER#' }, + }), + })(jest.fn(), jest.fn()); + + expect(adjusted.collections.map(c => c.name).sort()).toEqual([ORDERS, 'shared_users'].sort()); + }); + }); + describe('native query', () => { it('runs a PartiQL statement with parameters', async () => { const rows = await dataSource.executeNativeQuery( diff --git a/packages/datasource-dynamodb/test/introspection-api.test.ts b/packages/datasource-dynamodb/test/introspection-api.test.ts new file mode 100644 index 0000000000..2556260a7f --- /dev/null +++ b/packages/datasource-dynamodb/test/introspection-api.test.ts @@ -0,0 +1,228 @@ +import { fakeClient } from './fake-client'; +import { createDynamoDBDataSource, introspect } from '../src'; +import { convertManualSchema } from '../src/schema/manual-schema'; + +const ordersTable = { + Table: { + TableName: 'Orders', + KeySchema: [ + { AttributeName: 'customerId', KeyType: 'HASH' }, + { AttributeName: 'orderId', KeyType: 'RANGE' }, + ], + AttributeDefinitions: [ + { AttributeName: 'customerId', AttributeType: 'S' }, + { AttributeName: 'orderId', AttributeType: 'S' }, + { AttributeName: 'status', AttributeType: 'S' }, + { AttributeName: 'createdAt', AttributeType: 'S' }, + ], + GlobalSecondaryIndexes: [ + { + IndexName: 'by-status', + KeySchema: [ + { AttributeName: 'status', KeyType: 'HASH' }, + { AttributeName: 'createdAt', KeyType: 'RANGE' }, + ], + Projection: { ProjectionType: 'ALL' }, + }, + ], + }, +}; + +const orders = [ + { + customerId: 'c1', + orderId: 'o1', + status: 'paid', + total: 12, + createdAt: '2024-01-15T10:00:00Z', + tags: new Set(['a']), + address: { city: 'Paris', geo: { lat: 1 } }, + lines: [{ sku: 'A' }], + }, + { customerId: 'c1', orderId: 'o2', status: null, total: 3, createdAt: '2024-01-16T10:00:00Z' }, +]; + +const awsClient = () => + fakeClient({ + ListTables: () => ({ TableNames: ['Orders'] }), + DescribeTable: () => ordersTable, + Scan: () => ({ Items: orders }), + }); + +const appTable = (name: string) => ({ + Table: { + TableName: name, + KeySchema: [ + { AttributeName: 'pk', KeyType: 'HASH' }, + { AttributeName: 'sk', KeyType: 'RANGE' }, + ], + AttributeDefinitions: [ + { AttributeName: 'pk', AttributeType: 'S' }, + { AttributeName: 'sk', AttributeType: 'S' }, + ], + }, +}); + +describe('introspect', () => { + it('describes each table as a collection definition', async () => { + const introspection = await introspect({ client: awsClient() }); + + expect(introspection.version).toBe(1); + expect(introspection.collections).toHaveLength(1); + expect(introspection.collections[0]).toMatchObject({ + name: 'Orders', + tableName: 'Orders', + partitionKey: { name: 'customerId', type: 'string' }, + sortKey: { name: 'orderId', type: 'string' }, + indexes: [ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'string' }, + sortKey: { name: 'createdAt', type: 'string' }, + }, + ], + }); + expect(introspection.collections[0].fields).toEqual( + expect.arrayContaining([ + { name: 'total', type: 'number', nullable: false }, + { name: 'tags', type: 'array', set: true, nullable: true }, + { + name: 'address', + type: 'object', + fields: [ + { name: 'city', type: 'string', nullable: true }, + { + name: 'geo', + type: 'object', + fields: [{ name: 'lat', type: 'number', nullable: true }], + }, + ], + }, + ]), + ); + expect(introspection.collections[0].fields.map(field => field.name)).not.toContain('orderId'); + }); + + it('survives a trip through JSON', async () => { + const introspection = await introspect({ client: awsClient() }); + + expect(JSON.parse(JSON.stringify(introspection))).toEqual(introspection); + }); + + it('builds the same collections as the live introspection, without calling AWS', async () => { + const live = await createDynamoDBDataSource({ client: awsClient() })(jest.fn(), jest.fn()); + const introspection = await introspect({ client: awsClient() }); + const offline = fakeClient({}); + + const dataSource = await createDynamoDBDataSource({ client: offline }, { introspection })( + jest.fn(), + jest.fn(), + ); + + expect(dataSource.getCollection('Orders').schema).toEqual(live.getCollection('Orders').schema); + expect(offline.send).not.toHaveBeenCalled(); + }); + + it('refuses an introspection of another version', async () => { + const factory = createDynamoDBDataSource( + { client: fakeClient({}) }, + { introspection: { version: 2, collections: [] } as never }, + ); + + await expect(factory(jest.fn(), jest.fn())).rejects.toThrow( + 'The introspection is in version 2, this datasource reads version 1', + ); + }); + + it('refuses an introspection with a builder', async () => { + const factory = createDynamoDBDataSource( + { client: fakeClient({}), builder: configurator => configurator }, + { introspection: { version: 1, collections: [] } }, + ); + + await expect(factory(jest.fn(), jest.fn())).rejects.toThrow( + 'An introspection and a builder cannot be used together', + ); + }); +}); + +describe('the builder next to the introspection', () => { + const client = () => + fakeClient({ + ListTables: () => ({ TableNames: ['App', 'Other'] }), + DescribeTable: (input: { TableName: string }) => appTable(input.TableName), + Scan: () => ({ Items: [] }), + }); + const twoEntities = (configurator: any) => + configurator + .addCollectionFromTable({ + name: 'users', + tableName: 'App', + entity: { sortKeyPrefix: 'USER#' }, + }) + .addCollectionFromTable({ + name: 'orders', + tableName: 'App', + entity: { sortKeyPrefix: 'ORDER#' }, + }); + + it('keeps introspecting the tables it does not name', async () => { + const dataSource = await createDynamoDBDataSource({ + client: client(), + builder: twoEntities, + })(jest.fn(), jest.fn()); + + expect(dataSource.collections.map(c => c.name)).toEqual(['Other', 'orders', 'users']); + }); + + it('introspects the table the builder names only once', async () => { + const aws = client(); + + await createDynamoDBDataSource({ client: aws, builder: twoEntities })(jest.fn(), jest.fn()); + + const described = aws.calls('DescribeTable').map(input => input.TableName); + expect(described.filter(name => name === 'App')).toHaveLength(2); + expect(described.filter(name => name === 'Other')).toHaveLength(1); + }); + + it('exposes only the builder collections when no table is listed', async () => { + const aws = client(); + + const dataSource = await createDynamoDBDataSource({ + client: aws, + tables: [], + builder: twoEntities, + })(jest.fn(), jest.fn()); + + expect(dataSource.collections.map(c => c.name)).toEqual(['orders', 'users']); + expect(aws.calls('ListTables')).toHaveLength(0); + }); + + it('introspects the listed tables next to the builder collections', async () => { + const dataSource = await createDynamoDBDataSource({ + client: client(), + tables: ['Other'], + builder: twoEntities, + })(jest.fn(), jest.fn()); + + expect(dataSource.collections.map(c => c.name)).toEqual(['Other', 'orders', 'users']); + }); +}); + +describe('an object field without nested fields', () => { + it('is a column of its own', () => { + const [table] = convertManualSchema({ + collections: [ + { + name: 'a', + tableName: 'A', + partitionKey: { name: 'id', type: 'string' }, + fields: [{ name: 'meta', type: 'object' }], + }, + ], + }); + + expect(table.schema.meta).toMatchObject({ type: 'object' }); + }); +}); diff --git a/packages/datasource-dynamodb/test/introspection/introspector.test.ts b/packages/datasource-dynamodb/test/introspection/introspector.test.ts index be7c553c30..72c6a4c754 100644 --- a/packages/datasource-dynamodb/test/introspection/introspector.test.ts +++ b/packages/datasource-dynamodb/test/introspection/introspector.test.ts @@ -114,8 +114,8 @@ describe('introspectTable', () => { total: { type: 'number', nullable: false }, createdAt: { type: 'date', nullable: false }, tags: { type: 'array', nullable: true }, - 'address->city': { type: 'string', nullable: true }, - 'address->geo->lat': { type: 'number', nullable: true }, + 'address@@@city': { type: 'string', nullable: true }, + 'address@@@geo@@@lat': { type: 'number', nullable: true }, note: { type: 'string', nullable: true }, }); expect(schema.address).toBeUndefined(); @@ -147,8 +147,8 @@ describe('introspectTable', () => { maxDepth: 2, }); - expect(schema['a->b']).toMatchObject({ type: 'object' }); - expect(schema['a->b->c']).toBeUndefined(); + expect(schema['a@@@b']).toMatchObject({ type: 'object' }); + expect(schema['a@@@b@@@c']).toBeUndefined(); }); it('exposes only the nested columns of a map seen empty in some items', async () => { @@ -169,7 +169,7 @@ describe('introspectTable', () => { }); expect(schema.address).toBeUndefined(); - expect(schema['address->city']).toMatchObject({ type: 'string', nullable: true }); + expect(schema['address@@@city']).toMatchObject({ type: 'string', nullable: true }); }); it('declares the keys even when the table is empty', async () => { diff --git a/packages/datasource-dynamodb/test/schema/manual-schema.test.ts b/packages/datasource-dynamodb/test/schema/manual-schema.test.ts index ca03b878d2..1653bf6a6c 100644 --- a/packages/datasource-dynamodb/test/schema/manual-schema.test.ts +++ b/packages/datasource-dynamodb/test/schema/manual-schema.test.ts @@ -50,8 +50,8 @@ describe('convertManualSchema', () => { it('flattens nested objects into arrow-notation fields', () => { const [model] = convertManualSchema(config()); - expect(model.schema['address->city']).toEqual({ type: 'string', nullable: false }); - expect(model.schema['address->geo->lat']).toEqual({ type: 'number', nullable: false }); + expect(model.schema['address@@@city']).toEqual({ type: 'string', nullable: false }); + expect(model.schema['address@@@geo@@@lat']).toEqual({ type: 'number', nullable: false }); expect(model.schema.address).toBeUndefined(); }); @@ -142,8 +142,8 @@ describe('validateManualSchema', () => { }); it.each([ - [{ collections: [] }, 'Manual schema must have at least one collection'], - [undefined, 'Manual schema must have a collections array'], + [{ collections: [] }, 'The introspection must have at least one collection'], + [undefined, 'The introspection must have a collections array'], ])('rejects %p', (schema, message) => { expect(() => validateManualSchema(schema as never)).toThrow(message); }); @@ -198,15 +198,23 @@ describe('validateManualSchema', () => { ); }); - it('rejects an array without sub type', () => { - expect(() => validateManualSchema(config({ fields: [{ name: 'a', type: 'array' }] }))).toThrow( - "Array field 'a' in collection 'orders' must have a subType", - ); + it('accepts an array without sub type, as the introspection writes it', () => { + expect(() => + validateManualSchema(config({ fields: [{ name: 'a', type: 'array' }] })), + ).not.toThrow(); + }); + + it('accepts an object without nested fields, which is a column of its own', () => { + expect(() => + validateManualSchema(config({ fields: [{ name: 'a', type: 'object' }] })), + ).not.toThrow(); }); - it('rejects an object without nested fields', () => { - expect(() => validateManualSchema(config({ fields: [{ name: 'a', type: 'object' }] }))).toThrow( - "Object field 'a' in collection 'orders' must have nested field definitions", + it('rejects an array of objects without nested fields', () => { + expect(() => + validateManualSchema(config({ fields: [{ name: 'a', type: 'array', subType: 'object' }] })), + ).toThrow( + "Array field 'a' in collection 'orders' of objects must have nested field definitions", ); }); diff --git a/packages/datasource-dynamodb/test/second-round-findings.test.ts b/packages/datasource-dynamodb/test/second-round-findings.test.ts index 5f31582c8d..0eaffa2af0 100644 --- a/packages/datasource-dynamodb/test/second-round-findings.test.ts +++ b/packages/datasource-dynamodb/test/second-round-findings.test.ts @@ -130,7 +130,7 @@ describe('array collections', () => { Put: () => ({}), }); - const [created] = await lines('details->lines').create(caller, [ + const [created] = await lines('details@@@lines').create(caller, [ { customerId: 'c1', orderId: 'o1', sku: 'A' }, ]); @@ -153,7 +153,7 @@ describe('attribute names holding the path separator', () => { AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], }, }), - Scan: () => ({ Items: [{ id: 'a', 'a->b': 1 }] }), + Scan: () => ({ Items: [{ id: 'a', 'a@@@b': 1 }] }), }); const logger = jest.fn(); @@ -162,7 +162,7 @@ describe('attribute names holding the path separator', () => { expect(models).toEqual([]); expect(logger).toHaveBeenCalledWith( 'Warn', - "Introspector - Skipping table 'T': Attribute 'a->b' contains '->', which names nested attributes", + "Introspector - Skipping table 'T': Attribute 'a@@@b' contains '@@@', which names nested attributes", ); }); @@ -174,11 +174,11 @@ describe('attribute names holding the path separator', () => { name: 'c', tableName: 'T', partitionKey: { name: 'pk', type: 'string' }, - fields: [{ name: 'a->b', type: 'string' }], + fields: [{ name: 'a@@@b', type: 'string' }], }, ], }), - ).toThrow("Field 'a->b' in collection 'c' contains '->'"); + ).toThrow("Field 'a@@@b' in collection 'c' contains '@@@'"); }); }); @@ -257,7 +257,7 @@ describe('a reversed range on the sort key', () => { describe('writing records', () => { it('refuses a path through __proto__ instead of changing every object', () => { - expect(() => Serializer.unflatten({ '__proto__->polluted': 'yes' })).toThrow( + expect(() => Serializer.unflatten({ '__proto__@@@polluted': 'yes' })).toThrow( "An attribute path cannot contain '__proto__'", ); expect(() => Serializer.setPath({}, ['__proto__', 'polluted'], 'yes')).toThrow( @@ -267,7 +267,7 @@ describe('writing records', () => { }); it('does not take an inherited property for a map to write into', () => { - const item = Serializer.unflatten({ 'constructor->name': 'x' }); + const item = Serializer.unflatten({ 'constructor@@@name': 'x' }); expect(Object.keys(item)).toEqual(['constructor']); expect((item.constructor as unknown as Record).name).toBe('x'); @@ -286,7 +286,7 @@ describe('writing records', () => { }); await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { - 'address->city': 'Lyon', + 'address@@@city': 'Lyon', }); expect(client.calls('Get')[0].ConsistentRead).toBe(true); diff --git a/packages/datasource-dynamodb/test/third-round-findings.test.ts b/packages/datasource-dynamodb/test/third-round-findings.test.ts index 675c97e90e..d602b3e621 100644 --- a/packages/datasource-dynamodb/test/third-round-findings.test.ts +++ b/packages/datasource-dynamodb/test/third-round-findings.test.ts @@ -135,7 +135,7 @@ describe('writing binary attributes', () => { }); describe('a nested entity discriminator', () => { - const model = () => buildModel({ entity: { attribute: 'meta->type', value: 'ORDER' } }); + const model = () => buildModel({ entity: { attribute: 'meta@@@type', value: 'ORDER' } }); it('is stamped under its path, not under a literal key', async () => { const { collection, client } = buildCollection(model(), { Put: () => ({}) }); @@ -144,16 +144,16 @@ describe('a nested entity discriminator', () => { const { Item } = client.calls('Put')[0]; expect(Item.meta).toEqual({ type: 'ORDER' }); - expect(Object.keys(Item)).not.toContain('meta->type'); - expect(created['meta->type']).toBe('ORDER'); + expect(Object.keys(Item)).not.toContain('meta@@@type'); + expect(created['meta@@@type']).toBe('ORDER'); }); it('refuses another value supplied for it', async () => { const { collection, client } = buildCollection(model(), { Put: () => ({}) }); await expect( - collection.create(caller, [{ customerId: 'c1', orderId: 'o1', 'meta->type': 'INVOICE' }]), - ).rejects.toThrow("The attribute 'meta->type' of 'orders' must be 'ORDER'"); + collection.create(caller, [{ customerId: 'c1', orderId: 'o1', 'meta@@@type': 'INVOICE' }]), + ).rejects.toThrow("The attribute 'meta@@@type' of 'orders' must be 'ORDER'"); expect(client.calls('Put')).toHaveLength(0); }); }); @@ -161,7 +161,7 @@ describe('a nested entity discriminator', () => { describe('a nested entity discriminator left out of the schema', () => { it('still matches the items of the entity when they are read by key', async () => { const { collection } = buildCollection( - buildModel({ entity: { attribute: 'meta->type', value: 'ORDER' } }), + buildModel({ entity: { attribute: 'meta@@@type', value: 'ORDER' } }), { BatchGet: () => ({ Responses: { diff --git a/packages/datasource-dynamodb/test/utils/aggregator.test.ts b/packages/datasource-dynamodb/test/utils/aggregator.test.ts index d8843010c7..c41e9f9868 100644 --- a/packages/datasource-dynamodb/test/utils/aggregator.test.ts +++ b/packages/datasource-dynamodb/test/utils/aggregator.test.ts @@ -121,12 +121,12 @@ describe('StreamingAggregator', () => { }); it('reads flattened nested fields by their arrow path', () => { - const result = aggregate({ operation: 'Count', groups: [{ field: 'address->city' }] }, [ - { 'address->city': 'Paris' }, - { 'address->city': 'Paris' }, + const result = aggregate({ operation: 'Count', groups: [{ field: 'address@@@city' }] }, [ + { 'address@@@city': 'Paris' }, + { 'address@@@city': 'Paris' }, ]); - expect(result).toEqual([{ value: 2, group: { 'address->city': 'Paris' } }]); + expect(result).toEqual([{ value: 2, group: { 'address@@@city': 'Paris' } }]); }); it('averages per group', () => { diff --git a/packages/datasource-dynamodb/test/utils/expression-builder.test.ts b/packages/datasource-dynamodb/test/utils/expression-builder.test.ts index f0e104c234..5d376ac8f2 100644 --- a/packages/datasource-dynamodb/test/utils/expression-builder.test.ts +++ b/packages/datasource-dynamodb/test/utils/expression-builder.test.ts @@ -36,7 +36,7 @@ describe('ExpressionBuilder', () => { it('turns arrow notation into a nested document path', () => { const builder = new ExpressionBuilder(); - expect(builder.condition(leaf('address->city', 'Equal', 'Paris'))).toBe('#n0.#n1 = :v0'); + expect(builder.condition(leaf('address@@@city', 'Equal', 'Paris'))).toBe('#n0.#n1 = :v0'); expect(builder.names).toEqual({ '#n0': 'address', '#n1': 'city' }); }); @@ -208,7 +208,7 @@ describe('ExpressionBuilder', () => { it('aliases the top-level attribute of nested fields once', () => { const builder = new ExpressionBuilder(); - expect(builder.projection(['pk', 'address->city', 'address->zip'])).toBe('#n0, #n1'); + expect(builder.projection(['pk', 'address@@@city', 'address@@@zip'])).toBe('#n0, #n1'); expect(builder.names).toEqual({ '#n0': 'pk', '#n1': 'address' }); }); }); diff --git a/packages/datasource-dynamodb/test/utils/serializer.test.ts b/packages/datasource-dynamodb/test/utils/serializer.test.ts index a1acdbd2f3..a795574df1 100644 --- a/packages/datasource-dynamodb/test/utils/serializer.test.ts +++ b/packages/datasource-dynamodb/test/utils/serializer.test.ts @@ -5,10 +5,10 @@ describe('Serializer', () => { it('expands nested maps declared as columns into arrow-notation fields', () => { const record = Serializer.flatten( { pk: 'a', address: { city: 'Paris', geo: { lat: 1 } } }, - new Set(['pk', 'address->city', 'address->geo->lat']), + new Set(['pk', 'address@@@city', 'address@@@geo@@@lat']), ); - expect(record).toEqual({ pk: 'a', 'address->city': 'Paris', 'address->geo->lat': 1 }); + expect(record).toEqual({ pk: 'a', 'address@@@city': 'Paris', 'address@@@geo@@@lat': 1 }); }); it('keeps a map as Json when no nested column is declared under it', () => { @@ -52,7 +52,7 @@ describe('Serializer', () => { describe('unflatten', () => { it('rebuilds nested maps from arrow-notation fields', () => { expect( - Serializer.unflatten({ pk: 'a', 'address->city': 'Paris', 'address->zip': '75' }), + Serializer.unflatten({ pk: 'a', 'address@@@city': 'Paris', 'address@@@zip': '75' }), ).toEqual({ pk: 'a', address: { city: 'Paris', zip: '75' }, @@ -66,7 +66,7 @@ describe('Serializer', () => { describe('toDocumentPath', () => { it('splits arrow notation into path segments', () => { - expect(Serializer.toPathSegments('a->b->c')).toEqual(['a', 'b', 'c']); + expect(Serializer.toPathSegments('a@@@b@@@c')).toEqual(['a', 'b', 'c']); }); }); }); diff --git a/packages/datasource-dynamodb/test/validator-findings.test.ts b/packages/datasource-dynamodb/test/validator-findings.test.ts index ec260e4b5c..7096f09c90 100644 --- a/packages/datasource-dynamodb/test/validator-findings.test.ts +++ b/packages/datasource-dynamodb/test/validator-findings.test.ts @@ -173,30 +173,34 @@ describe('a manual schema collection without fields', () => { describe('the option naming the virtual collections', () => { it('also reads the name the ticket gives it', async () => { - const dataSource = await createDynamoDBDataSource({ - client: fakeClient({}), - disableIntrospection: true, - virtualArrayCollections: [ - { parentCollection: 'a', collectionName: 'a_items', arrayField: 'items' }, - ], - schema: { - collections: [ - { - name: 'a', - tableName: 'A', - partitionKey: { name: 'id', type: 'string' }, - fields: [ - { - name: 'items', - type: 'array', - subType: 'object', - fields: [{ name: 'k', type: 'string' }], - }, - ], - }, + const dataSource = await createDynamoDBDataSource( + { + client: fakeClient({}), + + virtualArrayCollections: [ + { parentCollection: 'a', collectionName: 'a_items', arrayField: 'items' }, ], }, - })(jest.fn(), jest.fn()); + { + introspection: { + collections: [ + { + name: 'a', + tableName: 'A', + partitionKey: { name: 'id', type: 'string' }, + fields: [ + { + name: 'items', + type: 'array', + subType: 'object', + fields: [{ name: 'k', type: 'string' }], + }, + ], + }, + ], + }, + }, + )(jest.fn(), jest.fn()); expect(dataSource.collections.map(c => c.name).sort()).toEqual(['a', 'a_items']); }); @@ -221,21 +225,25 @@ describe('native queries', () => { 'DELETE FROM "Users" WHERE id = \'a\'', ])('refuse the statement %s', async statement => { const client = fakeClient({ ExecuteStatement: () => ({ Items: [] }) }); - const dataSource = await createDynamoDBDataSource({ - client, - disableIntrospection: true, - schema: { - collections: [ - { - name: 'u', - tableName: 'Users', - partitionKey: { name: 'id', type: 'string' }, - fields: [], - }, - ], + const dataSource = await createDynamoDBDataSource( + { + client, + + liveQueryConnection: 'dynamo', }, - liveQueryConnection: 'dynamo', - })(jest.fn(), jest.fn()); + { + introspection: { + collections: [ + { + name: 'u', + tableName: 'Users', + partitionKey: { name: 'id', type: 'string' }, + fields: [], + }, + ], + }, + }, + )(jest.fn(), jest.fn()); await expect(dataSource.executeNativeQuery('dynamo', statement, {})).rejects.toThrow( 'Only SELECT statements can be run as native queries', @@ -278,3 +286,62 @@ describe('a chart computed while a write happens', () => { expect(after).toEqual([{ value: 2, group: {} }]); }); }); + +describe('an attribute an item does not have', () => { + const model = () => + buildSimpleModel({ + schema: { + id: { type: 'string' }, + name: { type: 'string', nullable: true }, + 'address@@@city': { type: 'string', nullable: true }, + }, + }); + const items = [ + { id: 'a', name: 'x', address: { city: 'Paris' } }, + { id: 'b' }, + { id: 'c', name: null, address: {} }, + ]; + + const idsFor = async (field: string, operator: string) => { + const { collection } = buildCollection( + model(), + { Scan: () => ({ Items: items }) }, + { scanConcurrency: 1 }, + ); + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf(field, operator as never) }), + new Projection('id'), + ); + + return records.map(record => record.id).sort(); + }; + + it('is not Present, like a null one', async () => { + expect(await idsFor('name', 'Present')).toEqual(['a']); + }); + + it('is not Present when it sits in a map the item lacks', async () => { + expect(await idsFor('address@@@city', 'Present')).toEqual(['a']); + }); + + it('is read as null', async () => { + const { collection } = buildCollection( + model(), + { Scan: () => ({ Items: items }) }, + { scanConcurrency: 1 }, + ); + + const records = await collection.list( + caller, + new PaginatedFilter({ sort: undefined }), + new Projection('id', 'name', 'address@@@city'), + ); + + expect(records.find(record => record.id === 'b')).toEqual({ + id: 'b', + name: null, + 'address@@@city': null, + }); + }); +}); diff --git a/packages/datasource-dynamodb/test/virtual-collections.test.ts b/packages/datasource-dynamodb/test/virtual-collections.test.ts index 82c4e053ac..fc7d95dcb1 100644 --- a/packages/datasource-dynamodb/test/virtual-collections.test.ts +++ b/packages/datasource-dynamodb/test/virtual-collections.test.ts @@ -122,38 +122,42 @@ describe('virtual collections', () => { Scan: () => ({ Items: [] }), }); - const dataSource = await createDynamoDBDataSource({ - client, - disableIntrospection: true, - schema: { - collections: [ - { - name: 'Orders', - tableName: 'Orders', - partitionKey: { name: 'id', type: 'string' }, - fields: [ - { - name: 'lines', - type: 'array', - subType: 'object', - fields: [ - { - name: 'parts', - type: 'array', - subType: 'object', - fields: [{ name: 'n', type: 'number' }], - }, - ], - }, - ], - }, + const dataSource = await createDynamoDBDataSource( + { + client, + + virtualCollections: [ + { parentCollection: 'Orders', collectionName: 'lines', arrayField: 'lines' }, + { parentCollection: 'lines', collectionName: 'parts', arrayField: 'parts' }, ], }, - virtualCollections: [ - { parentCollection: 'Orders', collectionName: 'lines', arrayField: 'lines' }, - { parentCollection: 'lines', collectionName: 'parts', arrayField: 'parts' }, - ], - })(jest.fn(), jest.fn()); + { + introspection: { + collections: [ + { + name: 'Orders', + tableName: 'Orders', + partitionKey: { name: 'id', type: 'string' }, + fields: [ + { + name: 'lines', + type: 'array', + subType: 'object', + fields: [ + { + name: 'parts', + type: 'array', + subType: 'object', + fields: [{ name: 'n', type: 'number' }], + }, + ], + }, + ], + }, + ], + }, + }, + )(jest.fn(), jest.fn()); expect(dataSource.getCollection('parts').schema.fields.n).toMatchObject({ columnType: 'Number', From 7a5016538eae63f78d430f5ca4e3adc1dd27a4bc Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Tue, 27 Oct 2026 18:13:55 +0100 Subject: [PATCH 30/32] feat(datasource-dynamodb): offer only the filters and sorts DynamoDB evaluates itself A column offers the operators DynamoDB evaluates and only the sort keys sort. The elements of an array collection only filter on the keys of the item and their position. inMemoryProcessing brings the rest back. Present is evaluated by DynamoDB as the negation of Blank. Co-Authored-By: Claude Sonnet 5.5 --- packages/datasource-dynamodb/README.md | 32 +-- .../src/array-collection.ts | 12 ++ .../datasource-dynamodb/src/collection.ts | 28 ++- packages/datasource-dynamodb/src/index.ts | 1 + .../src/schema/schema-converter.ts | 35 ++- packages/datasource-dynamodb/src/settings.ts | 1 + packages/datasource-dynamodb/src/types.ts | 7 + .../src/utils/expression-builder.ts | 11 +- .../test/capabilities.test.ts | 203 ++++++++++++++++++ .../test/collection.list.test.ts | 16 +- .../test/integration/dynamodb-local.test.ts | 54 +++++ .../test/review-findings.test.ts | 6 +- .../test/third-round-findings.test.ts | 2 +- .../test/utils/expression-builder.test.ts | 6 +- .../test/validator-findings.test.ts | 23 -- 15 files changed, 382 insertions(+), 55 deletions(-) create mode 100644 packages/datasource-dynamodb/test/capabilities.test.ts diff --git a/packages/datasource-dynamodb/README.md b/packages/datasource-dynamodb/README.md index 7115e7ee1d..ef75df4eff 100644 --- a/packages/datasource-dynamodb/README.md +++ b/packages/datasource-dynamodb/README.md @@ -23,18 +23,26 @@ Credentials come from the usual AWS chain (environment, profile, role). Pass `cr ## Why DynamoDB is not like the other datasources DynamoDB reads fast by key and scans otherwise, and it cannot sort, count or aggregate on the -server. The datasource uses the key whenever a filter allows it and does the rest on the items it -reads. What that means for each feature: - -| Feature | How it works | -| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Filters | Pushed down when DynamoDB can evaluate them. `Like`, `ILike`, `IContains`, `EndsWith`, date intervals and the like are applied on the returned items, so they read more than they return. | -| Access path | A `Query` on the table, a global or local secondary index when an equality pins its partition key (with the sort key range when there is one), a `BatchGetItem` when the filter names whole keys, a `Scan` otherwise. An index leaves out the items that lack its sort key, so one that has a sort key is only read when the filter has a condition on that key, or when the index is declared with `coversAllItems: true` in the introspection. | -| Sort | Free on the sort key of the table or index being read. On any other attribute, the matching items are read, then sorted in memory. | -| Pagination | There is no offset: the datasource remembers the key where earlier pages stopped and resumes from the closest one. `maxOffset` (100 000) caps how deep a page can be. | -| Count and charts | Computed by reading the items (a counting `Scan`/`Query` for a plain count), in parallel segments for a table. Results are kept in memory for `chartCacheTtlMs` (30 s). | -| Search | Evaluated on the items: it scans. Case-insensitive, accents kept. | -| Relations | None in DynamoDB. Declare them with `addManyToOneRelation` and friends: a lookup by key is one `BatchGetItem`, a relation backed by a secondary index is a `Query`. | +server. The datasource declares to Forest only what DynamoDB does itself, so that nothing in the +interface silently reads a whole table to answer in memory. What that means for each feature: + +| Feature | How it works | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Filters | A column offers the operators DynamoDB evaluates (`Equal`, `In`, comparisons, `StartsWith`, `Contains`, `Blank`, `Present`, …). `IContains`, `Like`, `EndsWith` and the relative date operators are not offered. A filter on a key uses it; a filter on another attribute is a `FilterExpression`, which still reads the table. | +| Access path | A `Query` on the table, a global or local secondary index when an equality pins its partition key (with the sort key range when there is one), a `BatchGetItem` when the filter names whole keys, a `Scan` otherwise. An index leaves out the items that lack its sort key, so one that has a sort key is only read when the filter has a condition on that key, or when the index is declared with `coversAllItems: true` in the introspection. | +| Sort | Only the sort key of the table or of an index sorts, as DynamoDB orders by it. A sort on another attribute is not offered, and a request for it is refused. | +| Pagination | There is no offset: the datasource remembers the key where earlier pages stopped and resumes from the closest one. `maxOffset` (100 000) caps how deep a page can be. | +| Count and charts | Computed by reading the items (a counting `Scan`/`Query` for a plain count), in parallel segments for a table. Results are kept in memory for `chartCacheTtlMs` (30 s). | +| Search | Forest searches with `Contains` on the string columns, which is case-sensitive: it scans. | +| Relations | None in DynamoDB. Declare them with `addManyToOneRelation` and friends: a lookup by key is one `BatchGetItem`, a relation backed by a secondary index is a `Query`. | +| Array collections | The attributes of an element live inside the item, so only the keys of the item and the position of the element filter. Other columns neither filter nor sort. | + +### `inMemoryProcessing` + +`inMemoryProcessing: true` brings the rest back: every operator Forest knows (`IContains`, `Like`, +relative dates, …), a sort on any attribute, and filters on the attributes of array elements. The +datasource then reads the matching items and does the work itself, bounded by `maxScannedItems` and +governed by `scanPolicy`. It is off by default, because it reads more than it returns. ## Controlling cost: `scanPolicy` diff --git a/packages/datasource-dynamodb/src/array-collection.ts b/packages/datasource-dynamodb/src/array-collection.ts index 0af9d82bda..4e7f1d5404 100644 --- a/packages/datasource-dynamodb/src/array-collection.ts +++ b/packages/datasource-dynamodb/src/array-collection.ts @@ -123,6 +123,18 @@ export default class ArrayCollection extends BaseCollection { overrideTypeConverter: undefined, }; const fields = convertModelToFields(model); + const inMemory = this.root.settings.inMemoryProcessing; + + // The attributes of an element live inside the item: a filter on one reads every parent. Only + // the keys of the item and the position of the element narrow what is read. + for (const [name, field] of Object.entries(fields)) { + if (!inMemory && field.type === 'Column') { + const isKey = this.keyColumns.includes(name); + + field.filterOperators = new Set(isKey ? ['Equal', 'In'] : []); + field.isSortable = false; + } + } for (const name of this.keyColumns) { Object.assign(fields[name], { isPrimaryKey: true, isReadOnly: true }); diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index ac2172f7d5..18d1038986 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -135,7 +135,7 @@ export default class DynamoDBCollection extends BaseCollection { this.logger = logger; this.executor = new DynamoExecutor(client, model, settings, logger); - this.addFields(convertModelToFields(model, logger)); + this.addFields(convertModelToFields(model, logger, settings.inMemoryProcessing)); if (model.enableCount ?? true) this.enableCount(); this.columnNames = Object.keys(this.schema.fields); @@ -329,6 +329,29 @@ export default class DynamoDBCollection extends BaseCollection { return value; } + /** Forest hides the sort of a column that is not ordered by a key: a request for it is refused. */ + private assertSortable(sort?: Sort): void { + if (this.settings.inMemoryProcessing || !sort) return; + + const keys = new Set( + [ + this.model.partitionKey, + this.model.sortKey, + ...this.model.indexes.flatMap(index => [index.partitionKey, index.sortKey]), + ] + .filter(Boolean) + .map(key => key.name), + ); + const unordered = sort.find(clause => !keys.has(clause.field)); + + if (unordered) { + throw new ValidationError( + `Sorting on '${unordered.field}' reads every item and sorts them in memory: DynamoDB ` + + `sorts on keys only. Set inMemoryProcessing to allow it.`, + ); + } + } + private isBinary(field: string): boolean { return this.model.schema[field]?.type === 'binary'; } @@ -506,6 +529,9 @@ export default class DynamoDBCollection extends BaseCollection { projection: Projection, ): Promise { const sort = filter.sort?.length ? filter.sort : undefined; + + this.assertSortable(sort); + const skip = filter.page?.skip ?? 0; const limit = filter.page?.limit || undefined; const { maxOffset } = this.settings; diff --git a/packages/datasource-dynamodb/src/index.ts b/packages/datasource-dynamodb/src/index.ts index 3e157fd2db..5f29af0439 100644 --- a/packages/datasource-dynamodb/src/index.ts +++ b/packages/datasource-dynamodb/src/index.ts @@ -164,6 +164,7 @@ export function createDynamoDBDataSource( maxOffset: options.maxOffset, logConsumedCapacity: options.logConsumedCapacity, logQueries: options.logQueries, + inMemoryProcessing: options.inMemoryProcessing, paginationCache: new PaginationCache(options.paginationCache), }), ); diff --git a/packages/datasource-dynamodb/src/schema/schema-converter.ts b/packages/datasource-dynamodb/src/schema/schema-converter.ts index 1be125a02b..4c9c8d7bc1 100644 --- a/packages/datasource-dynamodb/src/schema/schema-converter.ts +++ b/packages/datasource-dynamodb/src/schema/schema-converter.ts @@ -1,26 +1,41 @@ import type { AttributeSchema, TableModel } from '../types'; import type { ColumnSchema, FieldSchema, Logger } from '@forestadmin/datasource-toolkit'; +import { PUSHABLE_OPERATORS } from '../utils/expression-builder'; import TypeConverter from '../utils/type-converter'; -function convertAttribute(attribute: AttributeSchema, isPrimaryKey: boolean): ColumnSchema { +function convertAttribute( + attribute: AttributeSchema, + isPrimaryKey: boolean, + isOrdered: boolean, + inMemoryProcessing: boolean, +): ColumnSchema { const columnType = TypeConverter.getColumnTypeFromDataType(attribute.type); + const operators = TypeConverter.operatorsForColumnType(columnType); return { type: 'Column', columnType, - filterOperators: TypeConverter.operatorsForColumnType(columnType), + filterOperators: inMemoryProcessing + ? operators + : new Set([...operators].filter(operator => PUSHABLE_OPERATORS.has(operator))), isPrimaryKey, isReadOnly: false, - isSortable: TypeConverter.isSortable(attribute.type), + // DynamoDB orders by the sort key of the table or of an index and by nothing else. + isSortable: TypeConverter.isSortable(attribute.type) && (inMemoryProcessing || isOrdered), validation: [], }; } -/** Turns the attributes of a table into the columns of its collection, keys being the primary key. */ +/** + * Turns the attributes of a table into the columns of its collection, keys being the primary key. + * Without `inMemoryProcessing`, a column only offers the filters DynamoDB evaluates itself, and + * only the sort keys sort. + */ export default function convertModelToFields( model: TableModel, logger?: Logger, + inMemoryProcessing = true, ): Record { const keys = [model.partitionKey, model.sortKey].filter(Boolean); const attributes: Record = { ...model.schema }; @@ -29,12 +44,22 @@ export default function convertModelToFields( attributes[key.name] ??= { type: key.type === 'N' ? 'number' : 'string', nullable: false }; } + const sortKeys = new Set( + [model.sortKey, ...model.indexes.map(index => index.sortKey)] + .filter(Boolean) + .map(key => key.name), + ); const fields: Record = {}; for (const [name, attribute] of Object.entries(attributes)) { try { const isPrimaryKey = keys.some(key => key.name === name); - const generated = convertAttribute(attribute, isPrimaryKey); + const generated = convertAttribute( + attribute, + isPrimaryKey, + sortKeys.has(name), + inMemoryProcessing, + ); const overridden = model.overrideTypeConverter?.({ fieldName: name, attribute, diff --git a/packages/datasource-dynamodb/src/settings.ts b/packages/datasource-dynamodb/src/settings.ts index dff60d5d69..5c68f36d27 100644 --- a/packages/datasource-dynamodb/src/settings.ts +++ b/packages/datasource-dynamodb/src/settings.ts @@ -28,6 +28,7 @@ export function resolveSettings(settings: Partial = {}): Col writeRetryBaseDelayMs: 50, logConsumedCapacity: false, logQueries: false, + inMemoryProcessing: false, ...settings, now, paginationCache: settings.paginationCache ?? new PaginationCache({ now }), diff --git a/packages/datasource-dynamodb/src/types.ts b/packages/datasource-dynamodb/src/types.ts index df3ba588b4..be3ca9fe8d 100644 --- a/packages/datasource-dynamodb/src/types.ts +++ b/packages/datasource-dynamodb/src/types.ts @@ -145,6 +145,7 @@ export type CollectionSettings = { writeRetryBaseDelayMs: number; logConsumedCapacity: boolean; logQueries: boolean; + inMemoryProcessing: boolean; paginationCache: PaginationCache; /** Charts by table then by request: every collection of a table drops them on a write. */ chartCaches: Map>; @@ -178,6 +179,12 @@ export type DynamoDBSettingsOptions = { logConsumedCapacity?: boolean; /** Log the expressions sent to DynamoDB. */ logQueries?: boolean; + /** + * Lets Forest filter and sort on what DynamoDB does not evaluate itself (`IContains`, `Like`, + * relative dates, a sort on any attribute): the datasource reads the items and does it. Off by + * default: those columns are then neither filterable that way nor sortable. + */ + inMemoryProcessing?: boolean; paginationCache?: PaginationCacheOptions; }; diff --git a/packages/datasource-dynamodb/src/utils/expression-builder.ts b/packages/datasource-dynamodb/src/utils/expression-builder.ts index b48e79dd77..8277bd2e26 100644 --- a/packages/datasource-dynamodb/src/utils/expression-builder.ts +++ b/packages/datasource-dynamodb/src/utils/expression-builder.ts @@ -27,6 +27,7 @@ export const PUSHABLE_OPERATORS = new Set([ 'NotIn', 'Missing', 'Blank', + 'Present', 'StartsWith', 'Contains', 'NotContains', @@ -186,6 +187,10 @@ export default class ExpressionBuilder { return hasNull ? `(${matches} OR ${this.missing(path)})` : matches; } + private blank(path: string): string { + return `(attribute_not_exists(${path}) OR ${this.isNull(path)} OR ${path} = ${this.value('')})`; + } + private leaf(leaf: ConditionTreeLeaf): string { const path = this.attribute(leaf.field); const { operator, value } = leaf; @@ -199,9 +204,9 @@ export default class ExpressionBuilder { case 'Missing': return this.missing(path); case 'Blank': - return `(attribute_not_exists(${path}) OR ${this.isNull(path)} OR ${path} = ${this.value( - '', - )})`; + return this.blank(path); + case 'Present': + return `NOT ${this.blank(path)}`; case 'In': return values.length === 0 ? this.never(path) : this.in(path, values); diff --git a/packages/datasource-dynamodb/test/capabilities.test.ts b/packages/datasource-dynamodb/test/capabilities.test.ts new file mode 100644 index 0000000000..3910e64fe9 --- /dev/null +++ b/packages/datasource-dynamodb/test/capabilities.test.ts @@ -0,0 +1,203 @@ +import type { DataSource } from '@forestadmin/datasource-toolkit'; + +import { PaginatedFilter, Projection, Sort } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { buildModel, leaf } from './helpers'; +import ArrayCollection from '../src/array-collection'; +import { resolveSettings } from '../src/settings'; + +const caller = { timezone: 'UTC' } as never; + +const model = () => + buildModel({ + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + status: { type: 'string', nullable: true }, + total: { type: 'number' }, + createdAt: { type: 'date' }, + lines: { type: 'array' }, + }, + indexes: [ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'S' }, + sortKey: { name: 'createdAt', type: 'S' }, + projectsAllAttributes: true, + }, + ], + }); + +const fieldsOf = (settings = {}) => + buildCollection(model(), {}, settings).collection.schema.fields as Record< + string, + { filterOperators: Set; isSortable: boolean } + >; + +describe('what Forest may ask of a table collection', () => { + const IN_MEMORY_ONLY = ['IContains', 'NotIContains', 'EndsWith', 'Like', 'ILike', 'Match']; + + it('keeps the filters DynamoDB evaluates itself on every attribute', () => { + const { status, total, orderId } = fieldsOf(); + + expect([...status.filterOperators]).toEqual( + expect.arrayContaining([ + 'Equal', + 'NotEqual', + 'In', + 'StartsWith', + 'Contains', + 'Blank', + 'Present', + ]), + ); + expect([...total.filterOperators]).toEqual( + expect.arrayContaining(['Equal', 'LessThan', 'GreaterThanOrEqual', 'In']), + ); + expect([...orderId.filterOperators]).toEqual(expect.arrayContaining(['Equal', 'In'])); + }); + + it.each(IN_MEMORY_ONLY)( + 'does not offer %s, which would read every item to filter it', + operator => { + const { status } = fieldsOf(); + + expect(status.filterOperators.has(operator)).toBe(false); + }, + ); + + it('does not offer the relative date operators', () => { + const { createdAt } = fieldsOf(); + + expect( + [...createdAt.filterOperators].filter(op => + /Ago|Today|Yesterday|Previous|Future|Past|Before|After/.test(op), + ), + ).toEqual([]); + }); + + it('sorts on the sort key of the table and of an index only', () => { + const fields = fieldsOf(); + + expect(fields.orderId.isSortable).toBe(true); + expect(fields.createdAt.isSortable).toBe(true); + expect(fields.customerId.isSortable).toBe(false); + expect(fields.status.isSortable).toBe(false); + expect(fields.total.isSortable).toBe(false); + }); + + it('gives back every operator and every sort when the items may be processed in memory', () => { + const fields = fieldsOf({ inMemoryProcessing: true }); + + expect(fields.status.filterOperators.has('IContains')).toBe(true); + expect(fields.status.filterOperators.has('Like')).toBe(true); + expect(fields.total.isSortable).toBe(true); + expect(fields.customerId.isSortable).toBe(true); + }); + + it('is off by default', () => { + expect(resolveSettings().inMemoryProcessing).toBe(false); + }); +}); + +describe('Present', () => { + it('is evaluated by DynamoDB, as the negation of Blank', async () => { + const { collection, client } = buildCollection( + model(), + { Scan: () => ({ Items: [] }) }, + { scanConcurrency: 1 }, + ); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('status', 'Present') }), + new Projection('orderId'), + ); + + expect(client.calls('Scan')[0].FilterExpression).toMatch( + /^NOT \(attribute_not_exists\(#n0\) OR attribute_type\(#n0, :v0\) OR #n0 = :v1\)$/, + ); + }); +}); + +describe('what Forest may ask of a collection of array elements', () => { + const build = (settings = {}) => { + const parent = buildCollection(model(), {}, settings); + + return new ArrayCollection({} as DataSource, parent.collection, { + name: 'order_lines', + arrayField: 'lines', + itemSchema: { sku: { type: 'string' }, qty: { type: 'number' } }, + }).schema.fields as Record; isSortable: boolean }>; + }; + + it('filters on the keys of the table item and the position of the element', () => { + const fields = build(); + + for (const key of ['customerId', 'orderId', 'index']) { + expect([...fields[key].filterOperators].sort()).toEqual(['Equal', 'In']); + expect(fields[key].isSortable).toBe(false); + } + }); + + it('does not filter or sort on the attributes of the elements, which live inside the item', () => { + const fields = build(); + + expect(fields.sku.filterOperators.size).toBe(0); + expect(fields.sku.isSortable).toBe(false); + expect(fields.qty.filterOperators.size).toBe(0); + }); + + it('gives them back when the items may be processed in memory', () => { + const fields = build({ inMemoryProcessing: true }); + + expect(fields.sku.filterOperators.has('Equal')).toBe(true); + expect(fields.qty.isSortable).toBe(true); + }); +}); + +describe('a sort Forest should not ask for', () => { + const list = (sortField: string, settings = {}) => { + const { collection, client } = buildCollection( + model(), + { Scan: () => ({ Items: [{ customerId: 'c', orderId: 'o', status: 'b', total: 1 }] }) }, + { scanConcurrency: 1, ...settings }, + ); + + return { + client, + result: collection.list( + caller, + new PaginatedFilter({ sort: new Sort({ field: sortField, ascending: true }) }), + new Projection('orderId'), + ), + }; + }; + + it('is refused on an attribute that is not a key, which would read every item', async () => { + const { client, result } = list('total'); + + await expect(result).rejects.toThrow( + "Sorting on 'total' reads every item and sorts them in memory: DynamoDB sorts on keys only. " + + 'Set inMemoryProcessing to allow it.', + ); + expect(client.calls('Scan')).toHaveLength(0); + }); + + it.each(['orderId', 'createdAt', 'customerId', 'status'])( + 'is allowed on %s when the items may be processed in memory', + async field => { + const { result } = list(field, { inMemoryProcessing: true }); + + await expect(result).resolves.toHaveLength(1); + }, + ); + + it.each(['orderId', 'customerId'])('is allowed on the key %s', async field => { + const { result } = list(field); + + await expect(result).resolves.toHaveLength(1); + }); +}); diff --git a/packages/datasource-dynamodb/test/collection.list.test.ts b/packages/datasource-dynamodb/test/collection.list.test.ts index f8973b5c4d..bba1865f71 100644 --- a/packages/datasource-dynamodb/test/collection.list.test.ts +++ b/packages/datasource-dynamodb/test/collection.list.test.ts @@ -329,9 +329,11 @@ describe('DynamoDBCollection.list', () => { describe('sort', () => { it('sorts on a non-key attribute after reading the matching items', async () => { - const { collection } = buildCollection(buildModel(), { - Query: () => ({ Items: [order(3), order(1), order(2)] }), - }); + const { collection } = buildCollection( + buildModel(), + { Query: () => ({ Items: [order(3), order(1), order(2)] }) }, + { inMemoryProcessing: true }, + ); const records = await collection.list( caller, @@ -346,9 +348,11 @@ describe('DynamoDBCollection.list', () => { }); it('sorts null and missing values last when ascending', async () => { - const { collection } = buildCollection(buildSimpleModel(), { - Scan: () => ({ Items: [{ id: 'a' }, { id: 'b', age: 2 }, { id: 'c', age: 1 }] }), - }); + const { collection } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Items: [{ id: 'a' }, { id: 'b', age: 2 }, { id: 'c', age: 1 }] }) }, + { inMemoryProcessing: true }, + ); const records = await collection.list( caller, diff --git a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts index 507603ce56..27439aebdb 100644 --- a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts +++ b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts @@ -152,6 +152,7 @@ run('DynamoDB Local', () => { { parentCollection: 'orders', collectionName: 'meta_items', arrayField: 'meta@@@items' }, ], liveQueryConnection: 'dynamo', + inMemoryProcessing: true, chartCacheTtlMs: 0, scanConcurrency: 2, })(logger, jest.fn()); @@ -1062,6 +1063,59 @@ run('DynamoDB Local', () => { }); }); + describe('Present and Blank', () => { + const PB = `pb-${suffix}`; + + beforeAll(async () => { + await client.send( + new CreateTableCommand({ + TableName: PB, + BillingMode: 'PAY_PER_REQUEST', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }), + ); + + for (const Item of [ + { id: 'a', name: 'x' }, + { id: 'b' }, + { id: 'c', name: null }, + { id: 'd', name: '' }, + ]) { + // eslint-disable-next-line no-await-in-loop + await documents.send(new PutCommand({ TableName: PB, Item })); + } + }); + + afterAll(async () => { + await client.send(new DeleteTableCommand({ TableName: PB })).catch(() => undefined); + }); + + const idsFor = async (operator: string) => { + const source = await createDynamoDBDataSource({ client: documents, tables: [PB] })( + jest.fn(), + jest.fn(), + ); + const records = await source + .getCollection(PB) + .list( + caller, + new PaginatedFilter({ conditionTree: leaf('name', operator) }), + new Projection('id'), + ); + + return records.map(record => record.id).sort(); + }; + + it('keeps the items that hold a value, as DynamoDB evaluates it', async () => { + expect(await idsFor('Present')).toEqual(['a']); + }); + + it('keeps the items without a value, a null or an empty string', async () => { + expect(await idsFor('Blank')).toEqual(['b', 'c', 'd']); + }); + }); + describe('native query', () => { it('runs a PartiQL statement with parameters', async () => { const rows = await dataSource.executeNativeQuery( diff --git a/packages/datasource-dynamodb/test/review-findings.test.ts b/packages/datasource-dynamodb/test/review-findings.test.ts index 48f9e6f24e..28f778d5ed 100644 --- a/packages/datasource-dynamodb/test/review-findings.test.ts +++ b/packages/datasource-dynamodb/test/review-findings.test.ts @@ -71,7 +71,11 @@ describe('caches after a failed write', () => { describe('sorting a large result', () => { it('sorts more records than a function call can take as arguments', async () => { const items = Array.from({ length: 150000 }, (_, i) => ({ id: `u${i}`, age: i % 1000 })); - const { collection } = buildCollection(buildSimpleModel(), { Scan: () => ({ Items: items }) }); + const { collection } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Items: items }) }, + { inMemoryProcessing: true }, + ); const records = await collection.list( caller, diff --git a/packages/datasource-dynamodb/test/third-round-findings.test.ts b/packages/datasource-dynamodb/test/third-round-findings.test.ts index d602b3e621..418ffdd28e 100644 --- a/packages/datasource-dynamodb/test/third-round-findings.test.ts +++ b/packages/datasource-dynamodb/test/third-round-findings.test.ts @@ -53,7 +53,7 @@ describe('maxOffset on reads that sort in memory', () => { const { collection, client } = buildCollection( buildSimpleModel(), { Scan: () => ({ Items: [{ id: 'u1', name: 'a' }] }) }, - { maxOffset: 100, scanConcurrency: 1 }, + { maxOffset: 100, scanConcurrency: 1, inMemoryProcessing: true }, ); await expect( diff --git a/packages/datasource-dynamodb/test/utils/expression-builder.test.ts b/packages/datasource-dynamodb/test/utils/expression-builder.test.ts index 5d376ac8f2..2c6dac627b 100644 --- a/packages/datasource-dynamodb/test/utils/expression-builder.test.ts +++ b/packages/datasource-dynamodb/test/utils/expression-builder.test.ts @@ -243,13 +243,13 @@ describe('splitCondition', () => { }, ); - it('leaves Present to the items, since the toolkit also excludes empty strings from it', () => { + it('pushes Present as the negation of Blank, which also excludes empty strings', () => { const tree = leaf('f', 'Present'); const { pushable, residual } = splitCondition(tree); - expect(pushable).toBeUndefined(); - expect(residual).toBe(tree); + expect(pushable).toBe(tree); + expect(residual).toBeUndefined(); }); it('returns everything as pushable when DynamoDB can evaluate it', () => { diff --git a/packages/datasource-dynamodb/test/validator-findings.test.ts b/packages/datasource-dynamodb/test/validator-findings.test.ts index 7096f09c90..ef64c44eae 100644 --- a/packages/datasource-dynamodb/test/validator-findings.test.ts +++ b/packages/datasource-dynamodb/test/validator-findings.test.ts @@ -302,29 +302,6 @@ describe('an attribute an item does not have', () => { { id: 'c', name: null, address: {} }, ]; - const idsFor = async (field: string, operator: string) => { - const { collection } = buildCollection( - model(), - { Scan: () => ({ Items: items }) }, - { scanConcurrency: 1 }, - ); - const records = await collection.list( - caller, - new PaginatedFilter({ conditionTree: leaf(field, operator as never) }), - new Projection('id'), - ); - - return records.map(record => record.id).sort(); - }; - - it('is not Present, like a null one', async () => { - expect(await idsFor('name', 'Present')).toEqual(['a']); - }); - - it('is not Present when it sits in a map the item lacks', async () => { - expect(await idsFor('address@@@city', 'Present')).toEqual(['a']); - }); - it('is read as null', async () => { const { collection } = buildCollection( model(), From 37c3c3259dfb2312ac652b1d01a5e01f5b7cb332 Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Tue, 27 Oct 2026 18:17:45 +0100 Subject: [PATCH 31/32] fix(datasource-dynamodb): store the binary and set attributes of array elements as such Co-Authored-By: Claude Sonnet 5.5 --- .../src/array-collection.ts | 78 ++++++++---- .../datasource-dynamodb/src/collection.ts | 5 +- .../src/utils/serializer.ts | 3 + .../test/validator-findings.test.ts | 113 +++++++++++++++++- 4 files changed, 173 insertions(+), 26 deletions(-) diff --git a/packages/datasource-dynamodb/src/array-collection.ts b/packages/datasource-dynamodb/src/array-collection.ts index 4e7f1d5404..c2c8c6fe26 100644 --- a/packages/datasource-dynamodb/src/array-collection.ts +++ b/packages/datasource-dynamodb/src/array-collection.ts @@ -26,7 +26,7 @@ import { ScanLimitExceededError } from './errors'; import convertModelToFields from './schema/schema-converter'; import StreamingAggregator from './utils/aggregator'; import ExpressionBuilder from './utils/expression-builder'; -import Serializer from './utils/serializer'; +import Serializer, { CLEARED } from './utils/serializer'; import compareRecords from './utils/sort'; export const INDEX_FIELD = 'index'; @@ -193,24 +193,39 @@ export default class ArrayCollection extends BaseCollection { } } - const entries = Object.entries(patch).filter(([, value]) => value !== undefined); - - if (entries.length === 0) return; + const entries: Array<[string, unknown]> = Object.entries(patch) + .filter(([, value]) => value !== undefined) + .map(([field, value]) => [field, this.toStored(field, value)]); + const assigned = entries.filter(([, value]) => value !== CLEARED); + const cleared = entries.filter(([, value]) => value === CLEARED); + + if (assigned.length) { + await this.writePerTableItem(caller, filter, { + keyword: 'SET', + clauses: (builder, position, state) => { + state.values ??= assigned.map(([, value]) => builder.value(value)); + + return assigned.map( + ([field], index) => + `${this.elementPath(builder, position)}.${builder.attribute(field)} = ${ + state.values[index] + }`, + ); + }, + fallback: (key, positions) => this.replaceElements(key, positions, assigned), + }); + } - await this.writePerTableItem(caller, filter, { - keyword: 'SET', - clauses: (builder, position, state) => { - state.values ??= entries.map(([, value]) => builder.value(value)); - - return entries.map( - ([field], index) => - `${this.elementPath(builder, position)}.${builder.attribute(field)} = ${ - state.values[index] - }`, - ); - }, - fallback: (key, positions) => this.replaceElements(key, positions, entries), - }); + if (cleared.length) { + await this.writePerTableItem(caller, filter, { + keyword: 'REMOVE', + clauses: (builder, position) => + cleared.map( + ([field]) => `${this.elementPath(builder, position)}.${builder.attribute(field)}`, + ), + fallback: (key, positions) => this.replaceElements(key, positions, cleared), + }); + } } async delete(caller: Caller, filter: Filter): Promise { @@ -249,6 +264,19 @@ export default class ArrayCollection extends BaseCollection { .join('.'); } + /** Gives back the DynamoDB type Forest cannot send: bytes for base64 text, a set for an array. */ + private toStored(field: string, value: unknown): unknown { + const attribute = this.itemSchema[field]; + + if (attribute?.type === 'binary' && typeof value === 'string') { + return Uint8Array.from(Buffer.from(value, 'base64')); + } + + if (attribute?.set && Array.isArray(value)) return value.length ? new Set(value) : CLEARED; + + return value; + } + /** Reads an attribute the element does not have as a null, as the collection of a table does. */ private itemColumns(record: RecordData): RecordData { for (const column of this.itemFields) record[column] ??= null; @@ -348,6 +376,11 @@ export default class ArrayCollection extends BaseCollection { const fields = Object.fromEntries( Object.entries(record).filter(([name]) => !this.keyColumns.includes(name)), ); + const stored = Object.fromEntries( + Object.entries(fields) + .map(([name, value]) => [name, this.toStored(name, value)]) + .filter(([, value]) => value !== CLEARED), + ); const builder = new ExpressionBuilder(); const keyBuilder = new ExpressionBuilder('k', 'kv'); const parentPath = this.elementPath( @@ -358,7 +391,7 @@ export default class ArrayCollection extends BaseCollection { const { arrayField } = this.chain[this.chain.length - 1]; const arrayPath = [parentPath, builder.attribute(arrayField)].filter(Boolean).join('.'); const emptyList = builder.value([]); - const appended = builder.value([Serializer.unflatten(fields)]); + const appended = builder.value([Serializer.unflatten(stored)]); const conditions = [`attribute_exists(${keyBuilder.attribute(this.rootKeys[0])})`]; if (parentPath) conditions.push(`attribute_exists(${parentPath})`); @@ -383,7 +416,7 @@ export default class ArrayCollection extends BaseCollection { // The array sits under a map the item does not have yet, which SET cannot create. if (error.name === 'ValidationException' && /document path/i.test(error.message)) { - const index = await this.appendByRewriting(key, parentPositions as number[], fields); + const index = await this.appendByRewriting(key, parentPositions as number[], stored); return { ...key, @@ -557,7 +590,10 @@ export default class ArrayCollection extends BaseCollection { if (depth === this.chain.length - 1) { // Each attribute of the patch replaces what it names, as SET does, and nothing else. for (const [field, value] of entries) { - Serializer.setPath(array[position[depth]], Serializer.toPathSegments(field), value); + const path = Serializer.toPathSegments(field); + + if (value === CLEARED) Serializer.removePath(array[position[depth]], path); + else Serializer.setPath(array[position[depth]], path, value); } } else { container = array[position[depth]]; diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts index 18d1038986..a31ce54ad9 100644 --- a/packages/datasource-dynamodb/src/collection.ts +++ b/packages/datasource-dynamodb/src/collection.ts @@ -34,7 +34,7 @@ import StreamingAggregator from './utils/aggregator'; import { mapWithConcurrency } from './utils/concurrency'; import DynamoExecutor from './utils/executor'; import ExpressionBuilder, { splitCondition } from './utils/expression-builder'; -import Serializer from './utils/serializer'; +import Serializer, { CLEARED } from './utils/serializer'; import compareRecords from './utils/sort'; /** What a parallel read splits into: the segments of a scan, or the partitions of a filter. */ @@ -108,9 +108,6 @@ function keyConditionExpression( return expressions.join(' AND '); } -/** What a set emptied in Forest turns into: DynamoDB holds no empty set, so the attribute goes. */ -const CLEARED = Symbol('cleared'); - export default class DynamoDBCollection extends BaseCollection { readonly model: TableModel; readonly executor: DynamoExecutor; diff --git a/packages/datasource-dynamodb/src/utils/serializer.ts b/packages/datasource-dynamodb/src/utils/serializer.ts index 4fe0c85a3c..43f1fd30f1 100644 --- a/packages/datasource-dynamodb/src/utils/serializer.ts +++ b/packages/datasource-dynamodb/src/utils/serializer.ts @@ -6,6 +6,9 @@ import TypeConverter from './type-converter'; export const PATH_SEPARATOR = '@@@'; +/** What a set emptied in Forest turns into: DynamoDB holds no empty set, so the attribute goes. */ +export const CLEARED = Symbol('cleared'); + const nestedPrefixesCache = new WeakMap, Set>(); /** Paths that have columns declared under them, hence are expanded rather than kept as Json. */ diff --git a/packages/datasource-dynamodb/test/validator-findings.test.ts b/packages/datasource-dynamodb/test/validator-findings.test.ts index ef64c44eae..d7e387dde9 100644 --- a/packages/datasource-dynamodb/test/validator-findings.test.ts +++ b/packages/datasource-dynamodb/test/validator-findings.test.ts @@ -1,9 +1,12 @@ +import type { DataSource } from '@forestadmin/datasource-toolkit'; + import { Aggregation, Filter, PaginatedFilter, Projection } from '@forestadmin/datasource-toolkit'; import { buildCollection } from './collection-helpers'; import { fakeClient } from './fake-client'; -import { buildSimpleModel, leaf } from './helpers'; +import { buildModel, buildSimpleModel, leaf } from './helpers'; import { createDynamoDBDataSource } from '../src'; +import ArrayCollection from '../src/array-collection'; import { inferSchema } from '../src/introspection/introspector'; import { convertManualSchema, validateManualSchema } from '../src/schema/manual-schema'; import { resolveSettings } from '../src/settings'; @@ -322,3 +325,111 @@ describe('an attribute an item does not have', () => { }); }); }); + +describe('the binary and set attributes of an array element', () => { + const build = (handlers: Record) => { + const parent = buildCollection( + buildModel({ + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + lines: { type: 'array' }, + }, + }), + handlers, + { scanConcurrency: 1 }, + ); + const collection = new ArrayCollection({} as DataSource, parent.collection, { + name: 'order_lines', + arrayField: 'lines', + itemSchema: { + sku: { type: 'string' }, + blob: { type: 'binary', nullable: true }, + labels: { type: 'array', set: true, nullable: true }, + }, + }); + + return { collection, client: parent.client }; + }; + + const order = { customerId: 'c1', orderId: 'o1', lines: [{ sku: 'A' }] }; + const first = new Filter({ + conditionTree: leaf('customerId', 'Equal', 'c1'), + }); + const valuesOf = (call: Record) => Object.values(call.ExpressionAttributeValues); + + it('stores bytes and a set when an element is appended', async () => { + const { collection, client } = build({ + Update: () => ({ Attributes: { lines: [{ sku: 'A' }, { sku: 'N' }] } }), + }); + + await collection.create(caller, [ + { customerId: 'c1', orderId: 'o1', sku: 'N', blob: 'AQID', labels: ['a', 'b'] }, + ]); + + const appended = valuesOf(client.calls('Update')[0]).find( + value => Array.isArray(value) && value.length, + ) as any[]; + const [element] = appended.filter(value => value.sku === 'N'); + expect(Array.from(element.blob)).toEqual([1, 2, 3]); + expect(element.labels).toBeInstanceOf(Set); + }); + + it('leaves an emptied set out of an appended element', async () => { + const { collection, client } = build({ + Update: () => ({ Attributes: { lines: [{ sku: 'N' }] } }), + }); + + await collection.create(caller, [{ customerId: 'c1', orderId: 'o1', sku: 'N', labels: [] }]); + + const [element] = valuesOf(client.calls('Update')[0]).find( + value => Array.isArray(value) && value.length, + ) as any[]; + expect(Object.keys(element)).toEqual(['sku']); + }); + + it('stores bytes and a set when an element is updated', async () => { + const { collection, client } = build({ + Query: () => ({ Items: [order] }), + Update: () => ({}), + }); + + await collection.update(caller, first, { blob: 'AQID', labels: ['x'] }); + + const [bytes, set] = valuesOf(client.calls('Update')[0]); + expect(Array.from(bytes as Uint8Array)).toEqual([1, 2, 3]); + expect(set).toBeInstanceOf(Set); + }); + + it('removes the attribute of an element whose set is emptied', async () => { + const { collection, client } = build({ + Query: () => ({ Items: [order] }), + Update: () => ({}), + }); + + await collection.update(caller, first, { labels: [] }); + + const call = client.calls('Update')[0]; + expect(call.UpdateExpression).toMatch(/^REMOVE #n0\[0\]\.#n1$|^REMOVE #n0\[0\]\.#n1 *$/); + expect(call.ExpressionAttributeValues).toBeUndefined(); + }); + + it('stores bytes and a set in the client-side rewrite too', async () => { + const { collection, client } = build({ + Query: () => ({ Items: [order] }), + Update: () => { + throw Object.assign(new Error('The document path provided is invalid for update'), { + name: 'ValidationException', + }); + }, + Get: () => ({ Item: { ...order, lines: [{ sku: 'A', labels: new Set(['old']) }] } }), + Put: () => ({}), + }); + + await collection.update(caller, first, { blob: 'AQID', labels: [] }); + + const [assigned, cleared] = client.calls('Put').map(call => call.Item.lines[0]); + expect(Array.from(assigned.blob)).toEqual([1, 2, 3]); + expect(cleared).not.toHaveProperty('labels'); + }); +}); From c6eb1132a28aa46082e86ce1fa1d746989f9f1ec Mon Sep 17 00:00:00 2001 From: Pierre Merlet Date: Tue, 27 Oct 2026 18:29:52 +0100 Subject: [PATCH 32/32] fix(datasource-dynamodb): update and clear the attributes of an array element in one statement Co-Authored-By: Claude Sonnet 5.5 --- .../src/array-collection.ts | 70 +++++++++---------- .../test/validator-findings.test.ts | 29 ++++++-- 2 files changed, 60 insertions(+), 39 deletions(-) diff --git a/packages/datasource-dynamodb/src/array-collection.ts b/packages/datasource-dynamodb/src/array-collection.ts index c2c8c6fe26..b7dada826c 100644 --- a/packages/datasource-dynamodb/src/array-collection.ts +++ b/packages/datasource-dynamodb/src/array-collection.ts @@ -199,41 +199,30 @@ export default class ArrayCollection extends BaseCollection { const assigned = entries.filter(([, value]) => value !== CLEARED); const cleared = entries.filter(([, value]) => value === CLEARED); - if (assigned.length) { - await this.writePerTableItem(caller, filter, { - keyword: 'SET', - clauses: (builder, position, state) => { - state.values ??= assigned.map(([, value]) => builder.value(value)); - - return assigned.map( - ([field], index) => - `${this.elementPath(builder, position)}.${builder.attribute(field)} = ${ - state.values[index] - }`, - ); - }, - fallback: (key, positions) => this.replaceElements(key, positions, assigned), - }); - } + // One statement for the whole patch: a second one would look for the record again by the very + // attribute the first changed. + await this.writePerTableItem(caller, filter, { + clauses: (builder, position, state) => { + state.values ??= assigned.map(([, value]) => builder.value(value)); - if (cleared.length) { - await this.writePerTableItem(caller, filter, { - keyword: 'REMOVE', - clauses: (builder, position) => - cleared.map( - ([field]) => `${this.elementPath(builder, position)}.${builder.attribute(field)}`, + const path = this.elementPath(builder, position); + + return { + set: assigned.map( + ([field], index) => `${path}.${builder.attribute(field)} = ${state.values[index]}`, ), - fallback: (key, positions) => this.replaceElements(key, positions, cleared), - }); - } + remove: cleared.map(([field]) => `${path}.${builder.attribute(field)}`), + }; + }, + fallback: (key, positions) => this.replaceElements(key, positions, entries), + }); } async delete(caller: Caller, filter: Filter): Promise { await this.writePerTableItem(caller, filter, { - keyword: 'REMOVE', // Highest positions first, so that removing one does not shift those still to remove. descending: true, - clauses: (builder, position) => [this.elementPath(builder, position)], + clauses: (builder, position) => ({ set: [], remove: [this.elementPath(builder, position)] }), }); } @@ -627,13 +616,12 @@ export default class ArrayCollection extends BaseCollection { caller: Caller, filter: Filter, write: { - keyword: 'SET' | 'REMOVE'; descending?: boolean; clauses: ( builder: ExpressionBuilder, position: number[], state: { values?: string[] }, - ) => string[]; + ) => { set: string[]; remove: string[] }; /** What to do when DynamoDB refuses the path, for instance under a map that is missing. */ fallback?: (key: Item, positions: number[][]) => Promise; }, @@ -663,25 +651,37 @@ export default class ArrayCollection extends BaseCollection { }); let builder = new ExpressionBuilder(); let state: { values?: string[] } = {}; - let clauses: string[] = []; + let sets: string[] = []; + let removes: string[] = []; const flush = async () => { - if (clauses.length) { - await this.writeExpression(key, `${write.keyword} ${clauses.join(', ')}`, builder); + if (sets.length || removes.length) { + const expression = [ + sets.length ? `SET ${sets.join(', ')}` : '', + removes.length ? `REMOVE ${removes.join(', ')}` : '', + ] + .filter(Boolean) + .join(' '); + + await this.writeExpression(key, expression, builder); } builder = new ExpressionBuilder(); state = {}; - clauses = []; + sets = []; + removes = []; }; try { for (const position of ordered) { - clauses.push(...write.clauses(builder, position, state)); + const clauses = write.clauses(builder, position, state); + + sets.push(...clauses.set); + removes.push(...clauses.remove); // An expression is limited to 4 KB: past a safe size, what is left goes in the next. // eslint-disable-next-line no-await-in-loop - if (clauses.join(', ').length > MAX_EXPRESSION_LENGTH) await flush(); + if ([...sets, ...removes].join(', ').length > MAX_EXPRESSION_LENGTH) await flush(); } await flush(); diff --git a/packages/datasource-dynamodb/test/validator-findings.test.ts b/packages/datasource-dynamodb/test/validator-findings.test.ts index d7e387dde9..d99921274f 100644 --- a/packages/datasource-dynamodb/test/validator-findings.test.ts +++ b/packages/datasource-dynamodb/test/validator-findings.test.ts @@ -4,7 +4,7 @@ import { Aggregation, Filter, PaginatedFilter, Projection } from '@forestadmin/d import { buildCollection } from './collection-helpers'; import { fakeClient } from './fake-client'; -import { buildModel, buildSimpleModel, leaf } from './helpers'; +import { and, buildModel, buildSimpleModel, leaf } from './helpers'; import { createDynamoDBDataSource } from '../src'; import ArrayCollection from '../src/array-collection'; import { inferSchema } from '../src/introspection/introspector'; @@ -414,6 +414,26 @@ describe('the binary and set attributes of an array element', () => { expect(call.ExpressionAttributeValues).toBeUndefined(); }); + it('sets and removes in one statement, so the second does not look for a record the first changed', async () => { + const { collection, client } = build({ + Query: () => ({ Items: [{ ...order, lines: [{ sku: 'A', labels: new Set(['old']) }] }] }), + Update: () => ({}), + }); + + await collection.update( + caller, + new Filter({ + conditionTree: and(leaf('customerId', 'Equal', 'c1'), leaf('sku', 'Equal', 'A')), + }), + { sku: 'B', labels: [] }, + ); + + expect(client.calls('Update')).toHaveLength(1); + expect(client.calls('Update')[0].UpdateExpression).toMatch( + /^SET #n0\[0\]\.#n1 = :v0 REMOVE #n0\[0\]\.#n2$/, + ); + }); + it('stores bytes and a set in the client-side rewrite too', async () => { const { collection, client } = build({ Query: () => ({ Items: [order] }), @@ -428,8 +448,9 @@ describe('the binary and set attributes of an array element', () => { await collection.update(caller, first, { blob: 'AQID', labels: [] }); - const [assigned, cleared] = client.calls('Put').map(call => call.Item.lines[0]); - expect(Array.from(assigned.blob)).toEqual([1, 2, 3]); - expect(cleared).not.toHaveProperty('labels'); + expect(client.calls('Put')).toHaveLength(1); + const [line] = client.calls('Put')[0].Item.lines; + expect(Array.from(line.blob)).toEqual([1, 2, 3]); + expect(line).not.toHaveProperty('labels'); }); });