diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7d3bf6a6b7..77e2591ff0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -72,6 +72,7 @@ jobs: - datasource-customizer - datasource-demo-fintech - datasource-dummy + - datasource-dynamodb - datasource-mongoose - datasource-replica - datasource-mongo @@ -90,7 +91,7 @@ jobs: with: node-version-file: ".nvmrc" - name: Start docker containers - if: ${{ matrix.package == 'datasource-mongo' || matrix.package == 'datasource-mongoose' || matrix.package == 'datasource-sql' || matrix.package == 'datasource-sequelize' }} + if: ${{ matrix.package == 'datasource-mongo' || matrix.package == 'datasource-mongoose' || matrix.package == 'datasource-sql' || matrix.package == 'datasource-sequelize' || matrix.package == 'datasource-dynamodb' }} run: docker compose -f ./packages/${{ matrix.package }}/docker-compose.yml up -d; sleep 5 - name: Restore dependencies from cache uses: actions/cache/restore@v4 @@ -112,6 +113,8 @@ jobs: # broadly: most packages keep genuine unit tests under `test/integration*/`, and a wide # pattern would silently stop running hundreds of them. - name: Test code (excluding integration suites with their own job) + env: + DYNAMODB_ENDPOINT: ${{ matrix.package == 'datasource-dynamodb' && 'http://localhost:8000' || '' }} run: cd packages/${{ matrix.package }} && yarn test --coverage --testPathIgnorePatterns='llm.integration|bedrock.integration|embedded-bff.e2e' && cd - - name: Upload coverage uses: actions/upload-artifact@v4 diff --git a/CLAUDE.md b/CLAUDE.md index 91a18bc97b..398139d614 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,6 +34,7 @@ yarn test:coverage # Run tests with coverage - `datasource-sequelize` - Sequelize datasource - `datasource-mongo` - MongoDB datasource - `datasource-mongoose` - Mongoose datasource +- `datasource-dynamodb` - DynamoDB datasource **Other:** - `mcp-server` - MCP (Model Context Protocol) server for AI integration diff --git a/packages/datasource-dynamodb/LICENSE b/packages/datasource-dynamodb/LICENSE new file mode 100644 index 0000000000..acb12af3ef --- /dev/null +++ b/packages/datasource-dynamodb/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2021-2026 Forest Admin + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/datasource-dynamodb/README.md b/packages/datasource-dynamodb/README.md new file mode 100644 index 0000000000..ef75df4eff --- /dev/null +++ b/packages/datasource-dynamodb/README.md @@ -0,0 +1,247 @@ +# @forestadmin/datasource-dynamodb + +Forest Admin datasource for Amazon DynamoDB: introspection (live, saved or written by hand), composite keys, +relations, search, aggregations and charts, single-table design. + +```bash +yarn add @forestadmin/datasource-dynamodb +``` + +```ts +import { createAgent } from '@forestadmin/agent'; +import { createDynamoDBDataSource } from '@forestadmin/datasource-dynamodb'; + +createAgent({ authSecret, envSecret, isProduction }) + // Every table of the account and region, introspected + .addDataSource(createDynamoDBDataSource({ region: 'eu-west-1' })); +``` + +Credentials come from the usual AWS chain (environment, profile, role). Pass `credentials`, +`endpoint` (DynamoDB Local) or `clientConfig` to override them, or `client` to reuse a +`DynamoDBClient` or `DynamoDBDocumentClient` you already have. + +## Why DynamoDB is not like the other datasources + +DynamoDB reads fast by key and scans otherwise, and it cannot sort, count or aggregate on the +server. The datasource declares to Forest only what DynamoDB does itself, so that nothing in the +interface silently reads a whole table to answer in memory. What that means for each feature: + +| Feature | How it works | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Filters | A column offers the operators DynamoDB evaluates (`Equal`, `In`, comparisons, `StartsWith`, `Contains`, `Blank`, `Present`, …). `IContains`, `Like`, `EndsWith` and the relative date operators are not offered. A filter on a key uses it; a filter on another attribute is a `FilterExpression`, which still reads the table. | +| Access path | A `Query` on the table, a global or local secondary index when an equality pins its partition key (with the sort key range when there is one), a `BatchGetItem` when the filter names whole keys, a `Scan` otherwise. An index leaves out the items that lack its sort key, so one that has a sort key is only read when the filter has a condition on that key, or when the index is declared with `coversAllItems: true` in the introspection. | +| Sort | Only the sort key of the table or of an index sorts, as DynamoDB orders by it. A sort on another attribute is not offered, and a request for it is refused. | +| Pagination | There is no offset: the datasource remembers the key where earlier pages stopped and resumes from the closest one. `maxOffset` (100 000) caps how deep a page can be. | +| Count and charts | Computed by reading the items (a counting `Scan`/`Query` for a plain count), in parallel segments for a table. Results are kept in memory for `chartCacheTtlMs` (30 s). | +| Search | Forest searches with `Contains` on the string columns, which is case-sensitive: it scans. | +| Relations | None in DynamoDB. Declare them with `addManyToOneRelation` and friends: a lookup by key is one `BatchGetItem`, a relation backed by a secondary index is a `Query`. | +| Array collections | The attributes of an element live inside the item, so only the keys of the item and the position of the element filter. Other columns neither filter nor sort. | + +### `inMemoryProcessing` + +`inMemoryProcessing: true` brings the rest back: every operator Forest knows (`IContains`, `Like`, +relative dates, …), a sort on any attribute, and filters on the attributes of array elements. The +datasource then reads the matching items and does the work itself, bounded by `maxScannedItems` and +governed by `scanPolicy`. It is off by default, because it reads more than it returns. + +## Controlling cost: `scanPolicy` + +A request that no key or index serves reads the whole table. `scanPolicy` decides what happens: + +| Value | Behaviour | +| ------------------ | ------------------------------------------------------------------------------------ | +| `'warn'` (default) | Scans and logs a warning naming the table and the filtered fields, never the values. | +| `'allow'` | Scans silently. | +| `'deny'` | Refuses with an explicit error. | + +Any other value is refused at startup. + +`maxScannedItems` (1 000 000) stops any single request that reads more items than that, with an error +rather than a partial result. PartiQL native queries do not report how many items they evaluated, +so for them it bounds the rows returned, and the pages fetched are capped at 1000. Both can be set on the datasource or on a collection. + +## Choosing the tables and adjusting collections + +`tables: ['Customers', 'Orders']` only introspects the listed tables, and names each collection +after its table. + +The builder adjusts collections one table at a time: to name them, to tune them, or to share a table +between several collections. The tables it names are not introspected a second time, and the other +tables are still introspected as usual: + +```ts +createDynamoDBDataSource({ + region: 'eu-west-1', + builder: configurator => + configurator + .addCollectionFromTable({ + name: 'users', + tableName: 'App', + entity: { sortKeyPrefix: 'USER#' }, + }) + .addCollectionFromTable({ + name: 'orders', + tableName: 'App', + entity: { sortKeyPrefix: 'ORDER#' }, + }), +}); +``` + +Here `App` becomes two collections and every other table keeps its own. Add `tables: []` to expose +the builder collections only. + +## Introspection + +`DescribeTable` gives the keys and the indexes. The other attributes are typed from a sample of +`sampleSize` items (default 100): DynamoDB does not type them. A scan has no order, so the sample is +not the latest items. Nested maps become `address@@@city` columns up to `maxDepth` (5); an attribute +that is absent or null in part of the sample is nullable; mixed types become Json. + +## Saving the introspection, or writing it by hand + +`introspect` samples the tables and describes them as collection definitions. Save the result and +give it back as the `introspection` of the datasource: the agent then starts without calling +`DescribeTable` or sampling the tables. + +```ts +import { createDynamoDBDataSource, introspect } from '@forestadmin/datasource-dynamodb'; + +// Once, from a script +const introspection = await introspect({ region: 'eu-west-1' }); +await fs.promises.writeFile('introspection.json', JSON.stringify(introspection, null, 2)); + +// At startup +createDynamoDBDataSource( + { region: 'eu-west-1' }, + { introspection: JSON.parse(await fs.promises.readFile('introspection.json', 'utf8')) }, +); +``` + +The file is plain JSON, so it can be edited: add what the sampling cannot know (a nullable +attribute, an index that `coversAllItems`, a set), or write the whole of it by hand. An +`introspection` cannot be combined with a `builder`. + +```ts +createDynamoDBDataSource( + { region: 'eu-west-1' }, + { + introspection: { + version: 1, + collections: [ + { + name: 'orders', + tableName: 'Orders', + partitionKey: { name: 'customerId', type: 'string' }, + sortKey: { name: 'orderId', type: 'string' }, + indexes: [ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'string' }, + sortKey: { name: 'createdAt', type: 'string' }, + }, + ], + fields: [ + { name: 'total', type: 'number' }, + { name: 'status', type: 'string', nullable: true }, + { name: 'address', type: 'object', fields: [{ name: 'city', type: 'string' }] }, + ], + }, + ], + }, + }, +); +``` + +Nested maps are columns named with `@@@` (`address@@@city`), which Forest shows as a path. + +Dates are compared by their instant when sorted or aggregated here, but DynamoDB orders and compares +them as text on a key and when it filters: store them in one offset (UTC) if a date is a sort key or +you filter on ranges of it. + +Types: `string`, `number`, `boolean`, `date` and `dateonly` (ISO 8601 strings), `timeonly`, `object`, +`array`, `point` (GeoJSON, exposed as Json), `binary` (base64, filtered on the returned items). Keys +are strings or numbers. `timeonly` is exposed as Forest's `Time`. A number beyond 2^53 is read as +text, to keep every digit. A string or number set is read as an array and written back as a set when +the introspection saw it as one, or when the introspection declares `set: true` on the array. + +## Composite keys + +The partition key and the sort key are both primary key columns of the collection, so Forest +identifies a record by the two. A record created without a string partition or sort key gets a UUID (the sort key keeps its +entity prefix, when there is one), since Forest greys the key fields of the creation form. A +numeric key has to be supplied. +Keys cannot be updated. + +## Single-table design + +Several collections can share a table, each restricted to its items: + +```ts +configurator + .addCollectionFromTable({ + name: 'users', + tableName: 'App', + entity: { sortKeyPrefix: 'USER#' }, // or { attribute: 'entityType', value: 'USER' } + }) + .addCollectionFromTable({ + name: 'orders', + tableName: 'App', + entity: { sortKeyPrefix: 'ORDER#' }, + }); +``` + +A sort key prefix is also used as the sort key condition of queries, and checked on create. + +## Arrays as collections + +`virtualArrayCollections`, the name the Cosmos DB datasource uses, is read as an alias of +`virtualCollections`. + +```ts +createDynamoDBDataSource({ + virtualCollections: [ + { parentCollection: 'orders', collectionName: 'order_lines', arrayField: 'lines' }, + ], +}); +``` + +Each element of the array becomes a record, identified by the key of the table item and its `index` +in the array. Creating appends, updating sets the element's attributes, and deleting removes it, so +the index of the following elements shifts. + +`arrayField` can point inside a map (`'meta@@@items'`). An array of an array is a collection whose +parent is the first one, declared after it: the position in the outer array becomes a key column +named `Index`, and creating an inner element needs it. + +```ts +virtualCollections: [ + { parentCollection: 'orders', collectionName: 'order_lines', arrayField: 'lines' }, + { parentCollection: 'order_lines', collectionName: 'line_parts', arrayField: 'parts' }, +]; +``` + +## Native queries + +`liveQueryConnection: 'dynamo'` offers PartiQL `SELECT` statements as native queries, with `$name` +placeholders: `SELECT * FROM "Orders" WHERE customerId = $customer`. PartiQL cannot aggregate, so +charts are built with the collection's aggregations instead. + +## Observability + +`logQueries` logs the expressions sent to DynamoDB, `logConsumedCapacity` the capacity units each +request consumed. Throttled requests are retried by the AWS SDK (`adaptive`, 10 attempts). + +## Not supported yet + +Collections backed by a secondary index, optimistic updates, binary keys, exact counts without +reading the items. + +## Testing + +```bash +docker compose up -d +DYNAMODB_ENDPOINT=http://localhost:8000 yarn test +``` + +Without `DYNAMODB_ENDPOINT` the integration suite against DynamoDB Local is skipped. diff --git a/packages/datasource-dynamodb/docker-compose.yml b/packages/datasource-dynamodb/docker-compose.yml new file mode 100644 index 0000000000..873d974da8 --- /dev/null +++ b/packages/datasource-dynamodb/docker-compose.yml @@ -0,0 +1,8 @@ +version: '3.1' +services: + dynamodb: + image: amazon/dynamodb-local + container_name: forest_datasource_dynamodb_test + command: -jar DynamoDBLocal.jar -inMemory -sharedDb + ports: + - 8000:8000 diff --git a/packages/datasource-dynamodb/jest.config.ts b/packages/datasource-dynamodb/jest.config.ts new file mode 100644 index 0000000000..d622773e8a --- /dev/null +++ b/packages/datasource-dynamodb/jest.config.ts @@ -0,0 +1,8 @@ +/* eslint-disable import/no-relative-packages */ +import jestConfig from '../../jest.config'; + +export default { + ...jestConfig, + collectCoverageFrom: ['/src/**/*.ts'], + testMatch: ['/test/**/*.test.ts'], +}; diff --git a/packages/datasource-dynamodb/package.json b/packages/datasource-dynamodb/package.json new file mode 100644 index 0000000000..72736dcfc6 --- /dev/null +++ b/packages/datasource-dynamodb/package.json @@ -0,0 +1,30 @@ +{ + "name": "@forestadmin/datasource-dynamodb", + "version": "0.0.0", + "main": "dist/index.js", + "license": "Apache-2.0", + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/ForestAdmin/agent-nodejs.git", + "directory": "packages/datasource-dynamodb" + }, + "dependencies": { + "@aws-sdk/client-dynamodb": "^3.750.0", + "@aws-sdk/lib-dynamodb": "^3.750.0", + "@forestadmin/datasource-toolkit": "1.56.2" + }, + "files": [ + "dist/**/*.js", + "dist/**/*.d.ts" + ], + "scripts": { + "build": "tsc", + "build:watch": "tsc --watch", + "clean": "rm -rf coverage dist", + "lint": "eslint src test", + "test": "jest" + } +} diff --git a/packages/datasource-dynamodb/src/array-collection.ts b/packages/datasource-dynamodb/src/array-collection.ts new file mode 100644 index 0000000000..b7dada826c --- /dev/null +++ b/packages/datasource-dynamodb/src/array-collection.ts @@ -0,0 +1,698 @@ +import type DynamoDBCollection from './collection'; +import type { AttributeSchema, TableModel } from './types'; +import type { + AggregateResult, + Aggregation, + Caller, + ConditionTree, + DataSource, + Filter, + PaginatedFilter, + RecordData, +} from '@forestadmin/datasource-toolkit'; + +import { + BaseCollection, + ConditionTreeBranch, + ConditionTreeFactory, + ConditionTreeLeaf, + Page, + PaginatedFilter as PaginatedFilterClass, + Projection, + ValidationError, +} from '@forestadmin/datasource-toolkit'; + +import { ScanLimitExceededError } from './errors'; +import convertModelToFields from './schema/schema-converter'; +import StreamingAggregator from './utils/aggregator'; +import ExpressionBuilder from './utils/expression-builder'; +import Serializer, { CLEARED } from './utils/serializer'; +import compareRecords from './utils/sort'; + +export const INDEX_FIELD = 'index'; + +/** Table items read at once when only the first elements of the array are needed. */ +const PARENTS_PER_READ = 100; + +/** Expressions are limited to 4 KB by DynamoDB, names and values apart. */ +const MAX_EXPRESSION_LENGTH = 3000; + +/** One array on the way from the item of the table to the records of a collection. */ +type Level = { arrayField: string; indexColumn: string }; + +export type ArrayCollectionConfig = { + name: string; + /** Attribute of the parent holding the array, `a@@@b` for one inside a map. */ + arrayField: string; + /** Attributes of an item, by flattened name. */ + itemSchema: Record; +}; + +type Item = Record; + +function conjuncts(tree?: ConditionTree): ConditionTree[] { + if (!tree) return []; + + if (tree instanceof ConditionTreeBranch && tree.aggregator === 'And') { + return tree.conditions.flatMap(condition => conjuncts(condition)); + } + + return [tree]; +} + +/** + * Exposes an array of maps stored in the items of another collection, or of another array + * collection, as a collection of its own. A record is identified by the key of the table item and + * its position in each array on the way, which is why positions move when an earlier element of + * the same array is removed. The position in the parent array is named `Index`. + */ +export default class ArrayCollection extends BaseCollection { + private readonly parent: DynamoDBCollection | ArrayCollection; + private readonly root: DynamoDBCollection; + private readonly chain: Level[]; + private readonly rootKeys: string[]; + private readonly itemFields: Set; + + /** Attributes of an element, by flattened name, as declared or sampled. */ + readonly itemSchema: Record; + + /** The columns identifying a record: the table keys, then one position per array. */ + readonly keyColumns: string[]; + + constructor( + datasource: DataSource, + parent: DynamoDBCollection | ArrayCollection, + config: ArrayCollectionConfig, + ) { + super(config.name, datasource, parent.nativeDriver); + + const nested = parent instanceof ArrayCollection; + + this.parent = parent; + this.root = nested ? parent.root : parent; + + const { partitionKey, sortKey } = this.root.model; + const keys = [partitionKey, sortKey].filter(Boolean); + this.rootKeys = keys.map(key => key.name); + + const parentLevels = nested + ? parent.chain.map((level, i, all) => + i === all.length - 1 ? { ...level, indexColumn: `${parent.name}Index` } : level, + ) + : []; + this.chain = [...parentLevels, { arrayField: config.arrayField, indexColumn: INDEX_FIELD }]; + this.keyColumns = [...this.rootKeys, ...this.chain.map(level => level.indexColumn)]; + + // A key named like a position column would be overwritten by it, and so address another item. + if (new Set(this.keyColumns).size !== this.keyColumns.length) { + throw new ValidationError(`The key columns of '${config.name}' must have distinct names`); + } + + const schema: Record = { + ...config.itemSchema, + ...Object.fromEntries( + keys.map(key => [key.name, { type: key.type === 'N' ? 'number' : 'string' }]), + ), + ...Object.fromEntries(this.chain.map(level => [level.indexColumn, { type: 'number' }])), + }; + const model: TableModel = { + ...this.root.model, + name: config.name, + indexes: [], + schema, + overrideTypeConverter: undefined, + }; + const fields = convertModelToFields(model); + const inMemory = this.root.settings.inMemoryProcessing; + + // The attributes of an element live inside the item: a filter on one reads every parent. Only + // the keys of the item and the position of the element narrow what is read. + for (const [name, field] of Object.entries(fields)) { + if (!inMemory && field.type === 'Column') { + const isKey = this.keyColumns.includes(name); + + field.filterOperators = new Set(isKey ? ['Equal', 'In'] : []); + field.isSortable = false; + } + } + + for (const name of this.keyColumns) { + Object.assign(fields[name], { isPrimaryKey: true, isReadOnly: true }); + } + + this.addFields(fields); + this.enableCount(); + + this.itemSchema = config.itemSchema; + this.itemFields = new Set(Object.keys(config.itemSchema)); + } + + async create(caller: Caller, data: RecordData[]): Promise { + const created: RecordData[] = []; + + try { + // Appended one after the other: each append reads the length the previous one left. + for (const record of data) { + // eslint-disable-next-line no-await-in-loop + created.push(await this.append(record)); + } + } finally { + this.root.invalidateCaches(); + } + + return created; + } + + async list( + caller: Caller, + filter: PaginatedFilter, + projection: Projection, + ): Promise { + const sort = filter.sort?.length ? filter.sort : undefined; + const skip = filter.page?.skip ?? 0; + const limit = filter.page?.limit || undefined; + // Without a sort the first records are the first elements: parents are read only as far as + // the page needs, instead of the whole table. + const wanted = !sort && limit !== undefined ? skip + limit : undefined; + const records = await this.readRecords(caller, filter.conditionTree, wanted); + const { columns } = projection; + + if (sort) records.sort((a, b) => compareRecords(a, b, sort)); + + return records + .slice(skip, limit === undefined ? undefined : skip + limit) + .map(record => + Object.fromEntries(columns.filter(field => field in record).map(f => [f, record[f]])), + ); + } + + async update(caller: Caller, filter: Filter, patch: RecordData): Promise { + for (const field of Object.keys(patch)) { + if (this.keyColumns.includes(field)) { + throw new ValidationError(`The attribute '${field}' of '${this.name}' cannot be updated`); + } + } + + const entries: Array<[string, unknown]> = Object.entries(patch) + .filter(([, value]) => value !== undefined) + .map(([field, value]) => [field, this.toStored(field, value)]); + const assigned = entries.filter(([, value]) => value !== CLEARED); + const cleared = entries.filter(([, value]) => value === CLEARED); + + // One statement for the whole patch: a second one would look for the record again by the very + // attribute the first changed. + await this.writePerTableItem(caller, filter, { + clauses: (builder, position, state) => { + state.values ??= assigned.map(([, value]) => builder.value(value)); + + const path = this.elementPath(builder, position); + + return { + set: assigned.map( + ([field], index) => `${path}.${builder.attribute(field)} = ${state.values[index]}`, + ), + remove: cleared.map(([field]) => `${path}.${builder.attribute(field)}`), + }; + }, + fallback: (key, positions) => this.replaceElements(key, positions, entries), + }); + } + + async delete(caller: Caller, filter: Filter): Promise { + await this.writePerTableItem(caller, filter, { + // Highest positions first, so that removing one does not shift those still to remove. + descending: true, + clauses: (builder, position) => ({ set: [], remove: [this.elementPath(builder, position)] }), + }); + } + + async aggregate( + caller: Caller, + filter: Filter, + aggregation: Aggregation, + limit?: number, + ): Promise { + const aggregator = new StreamingAggregator(aggregation, caller.timezone); + + for (const record of await this.readRecords(caller, filter.conditionTree)) { + aggregator.add(record); + } + + return aggregator.result(limit); + } + + /** The path of an element: each array of the chain followed by the position taken in it. */ + private elementPath( + builder: ExpressionBuilder, + positions: number[], + levels = this.chain.length, + ): string { + return this.chain + .slice(0, levels) + .map((level, i) => `${builder.attribute(level.arrayField)}[${positions[i]}]`) + .join('.'); + } + + /** Gives back the DynamoDB type Forest cannot send: bytes for base64 text, a set for an array. */ + private toStored(field: string, value: unknown): unknown { + const attribute = this.itemSchema[field]; + + if (attribute?.type === 'binary' && typeof value === 'string') { + return Uint8Array.from(Buffer.from(value, 'base64')); + } + + if (attribute?.set && Array.isArray(value)) return value.length ? new Set(value) : CLEARED; + + return value; + } + + /** Reads an attribute the element does not have as a null, as the collection of a table does. */ + private itemColumns(record: RecordData): RecordData { + for (const column of this.itemFields) record[column] ??= null; + + return record; + } + + /** The columns identifying the parent element, as named by the parent. */ + private parentColumns(): string[] { + return this.parent instanceof ArrayCollection ? this.parent.keyColumns : this.rootKeys; + } + + /** Reads the parents, then turns each array element into a record and applies the filter. */ + private async readRecords( + caller: Caller, + tree?: ConditionTree, + wanted?: number, + ): Promise { + // Conditions on the table key narrow the parents read; the rest applies to the elements. + const onTableKey = conjuncts(tree).filter( + condition => + condition instanceof ConditionTreeLeaf && this.rootKeys.includes(condition.field), + ); + const { arrayField } = this.chain[this.chain.length - 1]; + const parentColumns = this.parentColumns(); + const ownKeyColumns = this.keyColumns.slice(0, -1); + const parentFilter = onTableKey.length + ? ConditionTreeFactory.intersect(...onTableKey) + : undefined; + const parentProjection = new Projection(...parentColumns, arrayField); + const records: RecordData[] = []; + let parentsRead = 0; + + for (let offset = 0; ; offset += PARENTS_PER_READ) { + // eslint-disable-next-line no-await-in-loop + const parents = await this.parent.list( + caller, + new PaginatedFilterClass({ + conditionTree: parentFilter, + ...(wanted === undefined ? {} : { page: new Page(offset, PARENTS_PER_READ) }), + }), + parentProjection, + ); + + for (const parent of parents) { + const elements = parent[arrayField]; + + if (Array.isArray(elements)) { + elements.forEach((element, index) => { + if (element && typeof element === 'object') { + const record = { + // The identity comes last: an element's own `index` must not take its place. + ...this.itemColumns(Serializer.flatten(element as Item, this.itemFields)), + ...Object.fromEntries( + ownKeyColumns.map((name, i) => [name, parent[parentColumns[i]]]), + ), + [INDEX_FIELD]: index, + }; + + if (!tree || tree.match(record, this, caller.timezone)) records.push(record); + } + }); + } + } + + if (wanted === undefined || records.length >= wanted || parents.length < PARENTS_PER_READ) { + return records; + } + + // Each page of parents is read with a budget of its own: this one is shared by all of them. + parentsRead += parents.length; + + if (parentsRead > this.root.executor.maxScannedItems) { + throw new ScanLimitExceededError( + `Reading the parents of '${this.name}' stopped after ${parentsRead} items of table ` + + `'${this.root.model.tableName}': maxScannedItems is ${this.root.executor.maxScannedItems}. ` + + `Narrow the results with filters.`, + ); + } + } + } + + private async append(record: RecordData): Promise { + const parentKeyColumns = this.keyColumns.slice(0, -1); + + for (const name of parentKeyColumns) { + if (record[name] === undefined || record[name] === null) { + throw new ValidationError( + `The key attribute '${name}' of the parent '${this.parent.name}' is required to create ` + + `a record in '${this.name}'`, + ); + } + } + + const key = Object.fromEntries(this.rootKeys.map(name => [name, record[name]])); + const parentPositions = parentKeyColumns.slice(this.rootKeys.length).map(name => record[name]); + const fields = Object.fromEntries( + Object.entries(record).filter(([name]) => !this.keyColumns.includes(name)), + ); + const stored = Object.fromEntries( + Object.entries(fields) + .map(([name, value]) => [name, this.toStored(name, value)]) + .filter(([, value]) => value !== CLEARED), + ); + const builder = new ExpressionBuilder(); + const keyBuilder = new ExpressionBuilder('k', 'kv'); + const parentPath = this.elementPath( + builder, + parentPositions as number[], + this.chain.length - 1, + ); + const { arrayField } = this.chain[this.chain.length - 1]; + const arrayPath = [parentPath, builder.attribute(arrayField)].filter(Boolean).join('.'); + const emptyList = builder.value([]); + const appended = builder.value([Serializer.unflatten(stored)]); + const conditions = [`attribute_exists(${keyBuilder.attribute(this.rootKeys[0])})`]; + + if (parentPath) conditions.push(`attribute_exists(${parentPath})`); + + const entity = this.entityCondition(builder); + + if (entity) conditions.push(entity); + + let attributes: Item; + + try { + attributes = await this.root.executor.update({ + Key: key, + UpdateExpression: `SET ${arrayPath} = list_append(if_not_exists(${arrayPath}, ${emptyList}), ${appended})`, + ConditionExpression: conditions.join(' AND '), + ExpressionAttributeNames: { ...builder.names, ...keyBuilder.names }, + ExpressionAttributeValues: builder.values, + ReturnValues: 'UPDATED_NEW', + }); + } catch (error) { + if (error.name === 'ConditionalCheckFailedException') throw this.missingParent(); + + // The array sits under a map the item does not have yet, which SET cannot create. + if (error.name === 'ValidationException' && /document path/i.test(error.message)) { + const index = await this.appendByRewriting(key, parentPositions as number[], stored); + + return { + ...key, + ...Object.fromEntries( + parentKeyColumns + .slice(this.rootKeys.length) + .map((name, i) => [name, parentPositions[i]]), + ), + [INDEX_FIELD]: index, + ...fields, + }; + } + + throw error; + } + + const length = this.appendedLength(attributes); + + return { + ...key, + ...Object.fromEntries( + parentKeyColumns.slice(this.rootKeys.length).map((name, i) => [name, parentPositions[i]]), + ), + [INDEX_FIELD]: length - 1, + ...fields, + }; + } + + private missingParent(): ValidationError { + return new ValidationError( + `The parent record of '${this.name}' does not exist in '${this.parent.name}'`, + ); + } + + /** Appends in the item itself, creating the maps on the way, for when DynamoDB cannot. */ + private async appendByRewriting( + key: Item, + parentPositions: number[], + fields: Record, + ): Promise { + const item = await this.root.executor.get(key); + + if (!item) throw this.missingParent(); + + let container: Item = item; + let index = -1; + + this.chain.forEach((level, depth) => { + const segments = Serializer.toPathSegments(level.arrayField); + const holder = segments.slice(0, -1).reduce((value, segment) => { + if (!value[segment]) value[segment] = {}; + + return value[segment] as Item; + }, container); + const last = segments[segments.length - 1]; + + if (depth === this.chain.length - 1) { + holder[last] ??= []; + (holder[last] as unknown[]).push(Serializer.unflatten(fields)); + index = (holder[last] as unknown[]).length - 1; + } else { + container = (holder[last] as Item[] | undefined)?.[parentPositions[depth]]; + + if (!container) throw this.missingParent(); + } + }); + + const keyBuilder = new ExpressionBuilder('k', 'kv'); + const builder = new ExpressionBuilder(); + const conditions = [`attribute_exists(${keyBuilder.attribute(this.rootKeys[0])})`]; + const entity = this.entityCondition(builder); + + if (entity) conditions.push(entity); + + try { + await this.root.executor.put({ + Item: item, + ConditionExpression: conditions.join(' AND '), + ExpressionAttributeNames: { ...builder.names, ...keyBuilder.names }, + ...(Object.keys(builder.values).length + ? { ExpressionAttributeValues: builder.values } + : {}), + }); + } catch (error) { + if (error.name === 'ConditionalCheckFailedException') throw this.missingParent(); + + throw error; + } + + return index; + } + + /** UPDATED_NEW answers with only the arrays on the path, each holding just the updated element. */ + private appendedLength(attributes: Item): number { + let value: unknown = attributes; + + this.chain.forEach((level, i) => { + for (const segment of Serializer.toPathSegments(level.arrayField)) { + value = (value as Item)?.[segment]; + } + + if (i < this.chain.length - 1 && Array.isArray(value)) [value] = value; + }); + + return (value as unknown[]).length; + } + + /** Restricts a write to the items of the entity the parent collection stands for. */ + private entityCondition(builder: ExpressionBuilder): string | undefined { + const { entity, sortKey } = this.root.model; + + if (entity?.sortKeyPrefix !== undefined && sortKey) { + return `begins_with(${builder.attribute(sortKey.name)}, ${builder.value( + entity.sortKeyPrefix, + )})`; + } + + if (entity?.attribute !== undefined) { + return `${builder.attribute(entity.attribute)} = ${builder.value(entity.value)}`; + } + + return undefined; + } + + private async writeExpression( + key: Item, + expression: string, + builder: ExpressionBuilder, + ): Promise { + const keyBuilder = new ExpressionBuilder('k', 'kv'); + const conditions = [`attribute_exists(${keyBuilder.attribute(this.rootKeys[0])})`]; + const entity = this.entityCondition(builder); + + if (entity) conditions.push(entity); + + try { + await this.root.executor.update({ + Key: key, + UpdateExpression: expression, + ConditionExpression: conditions.join(' AND '), + ExpressionAttributeNames: { ...builder.names, ...keyBuilder.names }, + ...(Object.keys(builder.values).length + ? { ExpressionAttributeValues: builder.values } + : {}), + }); + } catch (error) { + if (error.name !== 'ConditionalCheckFailedException') throw error; + } + } + + /** Sets attributes of elements one by one in the item itself, for when DynamoDB cannot. */ + private async replaceElements( + key: Item, + positions: number[][], + entries: Array<[string, unknown]>, + ): Promise { + const item = await this.root.executor.get(key); + + if (!item) return; + + for (const position of positions) { + let container: Item = item; + + this.chain.forEach((level, depth) => { + const segments = Serializer.toPathSegments(level.arrayField); + const holder = segments + .slice(0, -1) + .reduce((value, segment) => value[segment] as Item, container); + const array = holder[segments[segments.length - 1]] as Item[]; + + if (depth === this.chain.length - 1) { + // Each attribute of the patch replaces what it names, as SET does, and nothing else. + for (const [field, value] of entries) { + const path = Serializer.toPathSegments(field); + + if (value === CLEARED) Serializer.removePath(array[position[depth]], path); + else Serializer.setPath(array[position[depth]], path, value); + } + } else { + container = array[position[depth]]; + } + }); + } + + const keyBuilder = new ExpressionBuilder('k', 'kv'); + const builder = new ExpressionBuilder(); + const conditions = [`attribute_exists(${keyBuilder.attribute(this.rootKeys[0])})`]; + const entity = this.entityCondition(builder); + + if (entity) conditions.push(entity); + + try { + await this.root.executor.put({ + Item: item, + ConditionExpression: conditions.join(' AND '), + ExpressionAttributeNames: { ...builder.names, ...keyBuilder.names }, + ...(Object.keys(builder.values).length + ? { ExpressionAttributeValues: builder.values } + : {}), + }); + } catch (error) { + if (error.name !== 'ConditionalCheckFailedException') throw error; + } + } + + /** Finds the matching elements, then writes once per table item whatever they need. */ + private async writePerTableItem( + caller: Caller, + filter: Filter, + write: { + descending?: boolean; + clauses: ( + builder: ExpressionBuilder, + position: number[], + state: { values?: string[] }, + ) => { set: string[]; remove: string[] }; + /** What to do when DynamoDB refuses the path, for instance under a map that is missing. */ + fallback?: (key: Item, positions: number[][]) => Promise; + }, + ): Promise { + const records = await this.list( + caller, + new PaginatedFilterClass({ conditionTree: filter.conditionTree }), + new Projection(...this.keyColumns), + ); + const perItem = new Map(); + const positionColumns = this.chain.map(level => level.indexColumn); + + for (const record of records) { + const key = Object.fromEntries(this.rootKeys.map(name => [name, record[name]])); + const id = JSON.stringify(key); + const entry = perItem.get(id) ?? { key, positions: [] }; + entry.positions.push(positionColumns.map(name => record[name] as number)); + perItem.set(id, entry); + } + + try { + await this.root.executor.mapWrites([...perItem.values()], async ({ key, positions }) => { + const ordered = [...positions].sort((a, b) => { + const different = a.findIndex((index, depth) => index !== b[depth]); + + return different === -1 ? 0 : (a[different] - b[different]) * (write.descending ? -1 : 1); + }); + let builder = new ExpressionBuilder(); + let state: { values?: string[] } = {}; + let sets: string[] = []; + let removes: string[] = []; + + const flush = async () => { + if (sets.length || removes.length) { + const expression = [ + sets.length ? `SET ${sets.join(', ')}` : '', + removes.length ? `REMOVE ${removes.join(', ')}` : '', + ] + .filter(Boolean) + .join(' '); + + await this.writeExpression(key, expression, builder); + } + + builder = new ExpressionBuilder(); + state = {}; + sets = []; + removes = []; + }; + + try { + for (const position of ordered) { + const clauses = write.clauses(builder, position, state); + + sets.push(...clauses.set); + removes.push(...clauses.remove); + + // An expression is limited to 4 KB: past a safe size, what is left goes in the next. + // eslint-disable-next-line no-await-in-loop + if ([...sets, ...removes].join(', ').length > MAX_EXPRESSION_LENGTH) await flush(); + } + + await flush(); + } catch (error) { + if (!write.fallback || !/document path/i.test(error.message)) throw error; + + await write.fallback(key, ordered); + } + }); + } finally { + this.root.invalidateCaches(); + } + } +} diff --git a/packages/datasource-dynamodb/src/client.ts b/packages/datasource-dynamodb/src/client.ts new file mode 100644 index 0000000000..3cc1c52326 --- /dev/null +++ b/packages/datasource-dynamodb/src/client.ts @@ -0,0 +1,38 @@ +import type { DynamoDBClientOptions } from './types'; + +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; + +/** The AWS SDK default is 3 attempts: throttling on a busy table is worth more patience. */ +const DEFAULT_MAX_ATTEMPTS = 10; + +const TRANSLATE_CONFIG = { + marshallOptions: { removeUndefinedValues: true }, + unmarshallOptions: { wrapNumbers: false }, +}; + +function isLowLevelClient(client: unknown): client is DynamoDBClient { + return client instanceof DynamoDBClient || client?.constructor?.name === 'DynamoDBClient'; +} + +export default function createDocumentClient( + options: DynamoDBClientOptions, +): DynamoDBDocumentClient { + if (options.client) { + return isLowLevelClient(options.client) + ? DynamoDBDocumentClient.from(options.client, TRANSLATE_CONFIG) + : (options.client as DynamoDBDocumentClient); + } + + return DynamoDBDocumentClient.from( + new DynamoDBClient({ + maxAttempts: DEFAULT_MAX_ATTEMPTS, + retryMode: 'adaptive', + region: options.region, + endpoint: options.endpoint, + credentials: options.credentials, + ...options.clientConfig, + }), + TRANSLATE_CONFIG, + ); +} diff --git a/packages/datasource-dynamodb/src/collection.ts b/packages/datasource-dynamodb/src/collection.ts new file mode 100644 index 0000000000..a31ce54ad9 --- /dev/null +++ b/packages/datasource-dynamodb/src/collection.ts @@ -0,0 +1,889 @@ +import type { CollectionSettings, TableModel } from './types'; +import type { AccessPath, KeyLookup } from './utils/access-path'; +import type { Item, ReadBudget, ReadInput, ReadOperation } from './utils/executor'; +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import type { + AggregateResult, + Aggregation, + Caller, + Collection, + ConditionTree, + ConditionTreeLeaf, + DataSource, + Filter, + Logger, + PaginatedFilter, + Projection, + RecordData, + Sort, +} from '@forestadmin/datasource-toolkit'; + +import { + BaseCollection, + ConditionTreeFactory, + PaginatedFilter as PaginatedFilterClass, + Projection as ProjectionClass, + ValidationError, +} from '@forestadmin/datasource-toolkit'; +import { randomUUID } from 'crypto'; + +import { DynamoDBOperationError, OffsetTooDeepError, ScanDeniedError } from './errors'; +import convertModelToFields from './schema/schema-converter'; +import { planAccessPath, planKeyLookup } from './utils/access-path'; +import StreamingAggregator from './utils/aggregator'; +import { mapWithConcurrency } from './utils/concurrency'; +import DynamoExecutor from './utils/executor'; +import ExpressionBuilder, { splitCondition } from './utils/expression-builder'; +import Serializer, { CLEARED } from './utils/serializer'; +import compareRecords from './utils/sort'; + +/** What a parallel read splits into: the segments of a scan, or the partitions of a filter. */ +type ReadUnit = { segment?: { index: number; total: number }; partition?: string | number }; + +type ReadPlan = { lookup?: KeyLookup; access?: AccessPath; residual?: ConditionTree }; + +function intersect(...trees: ConditionTree[]): ConditionTree | undefined { + const present = trees.filter(Boolean); + + return present.length ? ConditionTreeFactory.intersect(...present) : undefined; +} + +/** Values a single expression may carry before it is evaluated on the returned items instead. */ +const MAX_PUSHED_LIST_VALUES = 200; +const LIST_OPERATORS = ['In', 'NotIn', 'IncludesAll', 'IncludesNone']; +/** Items below which a page read without any filter is too cheap to call a scan. */ +const MAX_CHEAP_PAGE_ITEMS = 1000; +/** A scan warning is repeated at most this often. */ +const WARN_INTERVAL_MS = 60000; +/** Charts remembered per collection, the oldest dropped first. */ +const MAX_CACHED_CHARTS = 100; + +const hasOversizedList = (leaf: ConditionTreeLeaf): boolean => + LIST_OPERATORS.includes(leaf.operator) && + Array.isArray(leaf.value) && + leaf.value.length > MAX_PUSHED_LIST_VALUES; + +const copyOf = (rows: AggregateResult[]): AggregateResult[] => + rows.map(row => ({ ...row, group: { ...row.group } })); + +const isEmptyKey = (value: unknown): boolean => + value === undefined || value === null || value === ''; + +function describeFilter(tree?: ConditionTree): string { + const parts = new Set(); + tree?.forEachLeaf(leaf => parts.add(`${leaf.field} ${leaf.operator}`)); + + return [...parts].join(', '); +} + +function keyConditionExpression( + builder: ExpressionBuilder, + { partition, sort }: AccessPath['keyCondition'], + partitionValue: string | number, +): string { + const expressions = [ + `${builder.attribute(partition.attribute)} = ${builder.value(partitionValue)}`, + ]; + + if (sort) { + const attribute = builder.attribute(sort.attribute); + const comparisons = { + Equal: '=', + LessThan: '<', + LessThanOrEqual: '<=', + GreaterThan: '>', + GreaterThanOrEqual: '>=', + }; + + if (sort.operator === 'StartsWith') { + expressions.push(`begins_with(${attribute}, ${builder.value(sort.value)})`); + } else if (sort.operator === 'Between') { + const [lower, upper] = sort.value as unknown[]; + expressions.push(`${attribute} BETWEEN ${builder.value(lower)} AND ${builder.value(upper)}`); + } else { + expressions.push(`${attribute} ${comparisons[sort.operator]} ${builder.value(sort.value)}`); + } + } + + return expressions.join(' AND '); +} + +export default class DynamoDBCollection extends BaseCollection { + readonly model: TableModel; + readonly executor: DynamoExecutor; + readonly settings: CollectionSettings; + private readonly logger: Logger; + private readonly fieldNames: Set; + private readonly columnNames: string[]; + private readonly matchContext: Collection; + private readonly warnedAt = new Map(); + + constructor( + datasource: DataSource, + model: TableModel, + client: DynamoDBDocumentClient, + logger: Logger, + settings: CollectionSettings, + ) { + super(model.name, datasource, client); + + this.model = model; + this.settings = settings; + this.logger = logger; + this.executor = new DynamoExecutor(client, model, settings, logger); + + this.addFields(convertModelToFields(model, logger, settings.inMemoryProcessing)); + if (model.enableCount ?? true) this.enableCount(); + + this.columnNames = Object.keys(this.schema.fields); + + const entityAttribute = model.entity?.attribute; + + // The discriminator may be left out of the schema, and is still read to tell the entities apart. + this.fieldNames = new Set([ + ...Object.keys(this.schema.fields), + ...(entityAttribute !== undefined ? [entityAttribute] : []), + ]); + this.matchContext = { + schema: { + fields: { + ...this.schema.fields, + ...(entityAttribute && !this.schema.fields[entityAttribute] + ? { [entityAttribute]: { type: 'Column', columnType: 'String' } } + : {}), + }, + }, + } as unknown as Collection; + } + + invalidateCaches(): void { + this.settings.paginationCache.invalidateTable(this.model.tableName); + // Dropped, not cleared: a chart still being computed holds the old map and writes into it. + this.settings.chartCaches.delete(this.model.tableName); + } + + async create(caller: Caller, data: RecordData[]): Promise { + try { + return await this.executor.mapWrites(data, record => this.createOne(record)); + } catch (error) { + throw this.wrap(error, 'create'); + } finally { + // Also after a failure: the writes that went through must not stay hidden by a cache. + this.invalidateCaches(); + } + } + + async list( + caller: Caller, + filter: PaginatedFilter, + projection: Projection, + ): Promise { + try { + return await this.listRecords(caller, filter, projection); + } catch (error) { + throw this.wrap(error, 'list'); + } + } + + async update(caller: Caller, filter: Filter, patch: RecordData): Promise { + try { + const entries: Array<[string, unknown]> = Object.entries(patch) + .filter(([, value]) => value !== undefined) + .map(([field, value]) => [field, this.toStored(field, value)]); + + for (const [field] of entries) { + if (field === this.model.entity?.attribute) { + throw new ValidationError( + `The attribute '${field}' tells which entity a record of '${this.name}' belongs to, ` + + `so it cannot be updated`, + ); + } + } + + if (entries.length === 0) return; + + const keys = await this.resolveKeys(caller, filter); + const keyNames = this.keyNames(); + + // Forest sets a many-to-one relation by writing its foreign key, which is a key of the + // record when the relation follows the table's own keys: writing the value it has is harmless. + const changedKey = entries.find( + ([field, value]) => keyNames.includes(field) && keys.some(key => key[field] !== value), + ); + + if (changedKey) { + throw new ValidationError( + `The key attribute '${changedKey[0]}' of '${this.name}' cannot be changed on an existing ` + + `record, since DynamoDB keys are immutable: delete the record and create a new one instead`, + ); + } + + const writable = entries.filter(([field]) => !keyNames.includes(field)); + + if (writable.length === 0) return; + + await this.executor.mapWrites(keys, key => this.updateOne(key, writable)); + } catch (error) { + throw this.wrap(error, 'update'); + } finally { + this.invalidateCaches(); + } + } + + async delete(caller: Caller, filter: Filter): Promise { + try { + const keys = await this.resolveKeys(caller, filter); + + if (keys.length === 0) return; + + await this.executor.batchDelete(keys); + } catch (error) { + throw this.wrap(error, 'delete'); + } finally { + this.invalidateCaches(); + } + } + + async aggregate( + caller: Caller, + filter: Filter, + aggregation: Aggregation, + limit?: number, + ): Promise { + try { + return await this.aggregateRecords(caller, filter, aggregation, limit); + } catch (error) { + throw this.wrap(error, 'aggregate'); + } + } + + private wrap(error: Error & { isBusinessError?: boolean }, verb: string): Error { + if (error?.isBusinessError && !(error instanceof DynamoDBOperationError)) return error; + + return new DynamoDBOperationError( + `Failed to ${verb} records of '${this.name}': ${error.message}`, + error, + ); + } + + private keyNames(): string[] { + return [this.model.partitionKey, this.model.sortKey].filter(Boolean).map(key => key.name); + } + + /** The attributes a position in the table or index read is made of. */ + private positionAttributes(access?: AccessPath): string[] { + return [...new Set([...this.keyNames(), ...(access?.keyAttributes ?? [])])]; + } + + private toRecord(item: Item): RecordData { + const record = Serializer.flatten(item, this.fieldNames); + + // An item has no attribute it never got, where Forest reads a null, and filters on Present + // or Blank tell the two apart. + for (const column of this.columnNames) record[column] ??= null; + + return record; + } + + private matches(tree: ConditionTree, item: Item, timezone: string): boolean { + return tree.match(this.toRecord(item), this.matchContext, timezone); + } + + // ---- reads ------------------------------------------------------------------------------ + + private plan(tree?: ConditionTree, sort?: Sort): ReadPlan { + // DynamoDB compares a binary attribute to a binary, never to the base64 text Forest sends. + const { pushable, residual } = splitCondition(tree, leaf => !this.isBinary(leaf.field)); + const lookup = planKeyLookup(this.model, pushable); + + if (lookup) return { lookup, residual: intersect(lookup.remaining, residual) }; + + const access = planAccessPath(this.model, pushable, sort); + + if (!access.remaining) return { access, residual }; + + // A Query filter cannot reference the key attributes it reads by, and an expression has a size + // limit that a long list of values would exceed: both are applied on the returned items. + const keys = new Set(access.kind === 'query' ? access.keyAttributes : []); + const { pushable: filter, residual: moved } = splitCondition( + access.remaining, + leaf => !keys.has(leaf.field) && !hasOversizedList(leaf), + ); + + return { access: { ...access, remaining: filter }, residual: intersect(moved, residual) }; + } + + /** Gives back the DynamoDB type Forest cannot send: bytes for base64 text, a set for an array. */ + private toStored(field: string, value: unknown): unknown { + if (this.isBinary(field) && typeof value === 'string') { + return Uint8Array.from(Buffer.from(value, 'base64')); + } + + if (this.model.schema[field]?.set && Array.isArray(value)) { + return value.length ? new Set(value) : CLEARED; + } + + return value; + } + + /** Forest hides the sort of a column that is not ordered by a key: a request for it is refused. */ + private assertSortable(sort?: Sort): void { + if (this.settings.inMemoryProcessing || !sort) return; + + const keys = new Set( + [ + this.model.partitionKey, + this.model.sortKey, + ...this.model.indexes.flatMap(index => [index.partitionKey, index.sortKey]), + ] + .filter(Boolean) + .map(key => key.name), + ); + const unordered = sort.find(clause => !keys.has(clause.field)); + + if (unordered) { + throw new ValidationError( + `Sorting on '${unordered.field}' reads every item and sorts them in memory: DynamoDB ` + + `sorts on keys only. Set inMemoryProcessing to allow it.`, + ); + } + } + + private isBinary(field: string): boolean { + return this.model.schema[field]?.type === 'binary'; + } + + private guardScan(plan: ReadPlan, tree?: ConditionTree): void { + if (plan.access?.kind !== 'scan') return; + + const policy = this.model.scanPolicy ?? this.settings.scanPolicy; + + if (policy === 'allow') return; + + const filter = describeFilter(tree); + const reason = `no key or index serves the filter${filter ? ` (${filter})` : ''}`; + + if (policy === 'deny') { + throw new ScanDeniedError( + `Scan on table '${this.model.tableName}' refused by scanPolicy 'deny': ${reason}`, + ); + } + + const message = `Scan on table '${this.model.tableName}' for collection '${this.name}': ${reason}`; + const now = this.settings.now(); + + // The same scan is asked for again each time a page or a chart is displayed: say it once. + if (now - (this.warnedAt.get(message) ?? -Infinity) < WARN_INTERVAL_MS) return; + + this.warnedAt.set(message, now); + this.logger('Warn', message); + } + + private buildReadInput( + access: AccessPath, + fields: string[], + extra: Partial = {}, + partitionValue?: string | number, + ): ReadInput { + const builder = new ExpressionBuilder(); + const input: ReadInput = {}; + + if (access.kind === 'query') { + input.IndexName = access.indexName; + input.KeyConditionExpression = keyConditionExpression( + builder, + access.keyCondition, + partitionValue ?? access.keyCondition.partition.value, + ); + if (!access.scanIndexForward) input.ScanIndexForward = false; + } + + if (access.remaining) input.FilterExpression = builder.condition(access.remaining); + if (fields.length) input.ProjectionExpression = builder.projection(fields); + + return { ...input, ...builder.result(), ...extra } as ReadInput; + } + + private projectionParams(fields: string[]): Partial { + if (!fields.length) return {}; + + const builder = new ExpressionBuilder(); + + return { ProjectionExpression: builder.projection(fields), ...builder.result() } as never; + } + + /** Reads the items DynamoDB selects, as the batches it returns them in. */ + private async *readBatches( + plan: ReadPlan, + fields: string[], + budget: ReadBudget, + unit?: ReadUnit, + ): AsyncGenerator { + if (plan.lookup) { + budget.charge(plan.lookup.keys.length, 'query'); + + yield await this.executor.batchGet(plan.lookup.keys, this.projectionParams(fields) as never); + + return; + } + + const partitions = plan.access.keyCondition?.partition.values; + + // A filter naming several partitions is one Query per partition, read one after the other. + if (partitions && !unit) { + for (const partition of partitions) { + yield* this.readBatches(plan, fields, budget, { partition }); + } + + return; + } + + const operation: ReadOperation = plan.access.kind; + const input = this.buildReadInput( + plan.access, + fields, + unit?.segment ? { Segment: unit.segment.index, TotalSegments: unit.segment.total } : {}, + unit?.partition, + ); + + for await (const page of this.executor.readPages(operation, input, budget)) yield page.items; + } + + private async readAll(plan: ReadPlan, fields: string[], timezone: string): Promise { + const items: Item[] = []; + + for await (const batch of this.readBatches(plan, fields, this.executor.newBudget())) { + for (const item of batch) { + if (!plan.residual || this.matches(plan.residual, item, timezone)) items.push(item); + } + } + + return items; + } + + /** Resumes from the closest position already seen, since a page cannot be reached by offset. */ + private async readPage( + plan: ReadPlan, + fields: string[], + skip: number, + limit: number | undefined, + timezone: string, + ): Promise { + const { access, residual } = plan; + const cache = this.settings.paginationCache; + const base = this.buildReadInput(access, fields); + const hash = cache.hash({ + base, + residual: residual?.toPlainObject(), + timezone, + table: this.model.tableName, + }); + const cached = limit === undefined ? undefined : cache.findBestToken(hash, skip); + let matched = cached?.offset ?? 0; + let lastStored = matched; + const unfiltered = !base.FilterExpression && !residual; + const input: ReadInput = { + ...base, + ...(cached ? { ExclusiveStartKey: cached.startKey } : {}), + ...(unfiltered && limit !== undefined ? { Limit: skip + limit - matched } : {}), + }; + const collected: Item[] = []; + + for await (const page of this.executor.readPages(access.kind, input)) { + for (const item of page.items) { + if (!residual || this.matches(residual, item, timezone)) { + if (matched >= skip) collected.push(item); + matched += 1; + + if (limit !== undefined && collected.length >= limit) { + cache.storeToken(hash, matched, this.positionOf(item, access), this.model.tableName); + + return collected; + } + } + } + + if (page.lastKey && matched - lastStored >= cache.getCacheInterval()) { + cache.storeToken(hash, matched, page.lastKey, this.model.tableName); + lastStored = matched; + } + } + + return collected; + } + + private positionOf(item: Item, access: AccessPath): Record { + return Object.fromEntries( + this.positionAttributes(access) + .filter(attribute => item[attribute] !== undefined) + .map(attribute => [attribute, item[attribute]]), + ); + } + + private async listRecords( + caller: Caller, + filter: PaginatedFilter, + projection: Projection, + ): Promise { + const sort = filter.sort?.length ? filter.sort : undefined; + + this.assertSortable(sort); + + const skip = filter.page?.skip ?? 0; + const limit = filter.page?.limit || undefined; + const { maxOffset } = this.settings; + + if (skip > maxOffset) { + throw new OffsetTooDeepError( + `Offset ${skip} exceeds the maximum allowed offset of ${maxOffset}. ` + + `Narrow the results with filters.`, + ); + } + + const plan = this.plan(filter.conditionTree, sort); + + // Reading the first page of a collection without any filter costs one page, not a table. + const isCheapPage = + !filter.conditionTree && + !plan.residual && + !plan.access?.remaining && + (!sort || plan.access?.sortSatisfied) && + limit !== undefined && + skip + limit <= MAX_CHEAP_PAGE_ITEMS; + + if (!isCheapPage) this.guardScan(plan, filter.conditionTree); + + const { columns } = projection; + const fields = [ + ...new Set([ + ...columns, + ...(plan.residual?.projection ?? []).filter(field => !field.includes(':')), + ...(sort ?? []).map(clause => clause.field), + ...this.positionAttributes(plan.access), + ]), + ]; + + const toResult = (item: Item) => { + const record = this.toRecord(item); + + return Object.fromEntries( + columns.filter(field => field in record).map(field => [field, record[field]]), + ); + }; + + const singlePartition = !plan.access?.keyCondition?.partition.values; + + if (!plan.lookup && singlePartition && (!sort || plan.access.sortSatisfied)) { + return (await this.readPage(plan, fields, skip, limit, caller.timezone)).map(toResult); + } + + let items = await this.readAll(plan, fields, caller.timezone); + + if (sort && !plan.access?.sortSatisfied) { + const rows = items.map(item => ({ item, record: this.toRecord(item) })); + rows.sort((a, b) => compareRecords(a.record, b.record, sort)); + items = rows.map(row => row.item); + } + + return items.slice(skip, limit === undefined ? undefined : skip + limit).map(toResult); + } + + private async resolveKeys(caller: Caller, filter: Filter): Promise { + const plan = this.plan(filter.conditionTree); + + if (plan.lookup && !plan.residual) return plan.lookup.keys; + + const keyNames = this.keyNames(); + const records = await this.listRecords( + caller, + new PaginatedFilterClass({ conditionTree: filter.conditionTree }), + new ProjectionClass(...keyNames), + ); + + return records.map(record => Object.fromEntries(keyNames.map(name => [name, record[name]]))); + } + + // ---- writes ----------------------------------------------------------------------------- + + private async createOne(record: RecordData): Promise { + const { partitionKey, sortKey, entity } = this.model; + const created: RecordData = { ...record }; + + if (isEmptyKey(created[partitionKey.name])) { + if (partitionKey.type !== 'S') { + throw new ValidationError( + `The partition key '${partitionKey.name}' is required to create a record in '${this.name}'`, + ); + } + + created[partitionKey.name] = randomUUID(); + } + + if (sortKey && isEmptyKey(created[sortKey.name])) { + if (sortKey.type !== 'S') { + throw new ValidationError( + `The sort key '${sortKey.name}' is required to create a record in '${this.name}'`, + ); + } + + created[sortKey.name] = `${entity?.sortKeyPrefix ?? ''}${randomUUID()}`; + } + + if ( + entity?.sortKeyPrefix !== undefined && + !String(created[sortKey.name]).startsWith(entity.sortKeyPrefix) + ) { + throw new ValidationError( + `The sort key '${sortKey.name}' of '${this.name}' must start with '${entity.sortKeyPrefix}'`, + ); + } + + if (entity?.attribute !== undefined) { + if (created[entity.attribute] !== undefined && created[entity.attribute] !== entity.value) { + throw new ValidationError( + `The attribute '${entity.attribute}' of '${this.name}' must be '${entity.value}'`, + ); + } + + created[entity.attribute] = entity.value; + } + + const item = Serializer.unflatten( + Object.fromEntries( + Object.entries(created) + .map(([field, value]) => [field, this.toStored(field, value)]) + .filter(([, value]) => value !== CLEARED), + ), + ); + + const keyBuilder = new ExpressionBuilder('k', 'kv'); + + try { + await this.executor.put({ + Item: item, + ConditionExpression: `attribute_not_exists(${keyBuilder.attribute(partitionKey.name)})`, + ...keyBuilder.result(), + }); + } catch (error) { + if (error.name === 'ConditionalCheckFailedException') { + throw new ValidationError(`A record with the same key already exists in '${this.name}'`); + } + + throw error; + } + + return created; + } + + private async updateOne(key: Item, entries: Array<[string, unknown]>): Promise { + const builder = new ExpressionBuilder(); + const keyBuilder = new ExpressionBuilder('k', 'kv'); + const sets = entries + .filter(([, value]) => value !== CLEARED) + .map(([field, value]) => `${builder.attribute(field)} = ${builder.value(value)}`); + const removes = entries + .filter(([, value]) => value === CLEARED) + .map(([field]) => builder.attribute(field)); + const condition = `attribute_exists(${keyBuilder.attribute(this.model.partitionKey.name)})`; + const updateExpression = [ + sets.length ? `SET ${sets.join(', ')}` : '', + removes.length ? `REMOVE ${removes.join(', ')}` : '', + ] + .filter(Boolean) + .join(' '); + + try { + await this.executor.update({ + Key: key, + UpdateExpression: updateExpression, + ConditionExpression: condition, + ExpressionAttributeNames: { ...builder.names, ...keyBuilder.names }, + ...(Object.keys(builder.values).length + ? { ExpressionAttributeValues: builder.values } + : {}), + }); + } catch (error) { + if (error.name === 'ConditionalCheckFailedException') return; + + // A nested path can only be set under a map that already exists: merge client-side instead. + if (error.name === 'ValidationException' && /document path/i.test(error.message)) { + await this.replaceOne(key, entries); + + return; + } + + throw error; + } + } + + private async replaceOne(key: Item, entries: Array<[string, unknown]>): Promise { + const existing = await this.executor.get(key); + + if (!existing) return; + + const keyBuilder = new ExpressionBuilder('k', 'kv'); + + // Each entry replaces what is at its path, as a SET does: a map is not merged with the old one. + for (const [field, value] of entries) { + if (value === CLEARED) Serializer.removePath(existing, Serializer.toPathSegments(field)); + else Serializer.setPath(existing, Serializer.toPathSegments(field), value); + } + + try { + await this.executor.put({ + Item: existing, + ConditionExpression: `attribute_exists(${keyBuilder.attribute( + this.model.partitionKey.name, + )})`, + ...keyBuilder.result(), + }); + } catch (error) { + if (error.name !== 'ConditionalCheckFailedException') throw error; + } + } + + // ---- aggregation ------------------------------------------------------------------------ + + private async aggregateRecords( + caller: Caller, + filter: Filter, + aggregation: Aggregation, + limit?: number, + ): Promise { + const { timezone } = caller; + const { paginationCache: cache, chartCacheTtlMs, now } = this.settings; + const cacheKey = cache.hash({ + collection: this.name, + filter: filter.conditionTree?.toPlainObject(), + aggregation: { + field: aggregation.field, + operation: aggregation.operation, + groups: aggregation.groups, + }, + limit, + timezone, + }); + const charts = this.chartsOfTable(); + const cached = charts.get(cacheKey); + + if (cached && now() - cached.at < chartCacheTtlMs) return copyOf(cached.value); + + const value = await this.computeAggregation(filter, aggregation, limit, timezone); + + if (chartCacheTtlMs > 0) { + // A copy: the caller may rename the groups it is given, which must not reach the cache. + charts.delete(cacheKey); + charts.set(cacheKey, { at: now(), value: copyOf(value) }); + + if (charts.size > MAX_CACHED_CHARTS) charts.delete(charts.keys().next().value); + } + + return value; + } + + private chartsOfTable(): Map { + const { chartCaches } = this.settings; + let charts = chartCaches.get(this.model.tableName); + + if (!charts) { + charts = new Map(); + chartCaches.set(this.model.tableName, charts); + } + + return charts; + } + + /** Reads the segments of a parallel scan at once; a failure stops the others from reading on. */ + private async inSegments( + budget: ReadBudget, + segments: T[], + read: (segment: T) => Promise, + ): Promise { + try { + await mapWithConcurrency(segments, Math.max(this.settings.scanConcurrency, 1), read); + } catch (error) { + budget.abort(); + + throw error; + } + } + + private async computeAggregation( + filter: Filter, + aggregation: Aggregation, + limit: number | undefined, + timezone: string, + ): Promise { + const plan = this.plan(filter.conditionTree); + const isCount = aggregation.operation === 'Count' && !aggregation.field; + const ungrouped = !aggregation.groups?.length; + + if ( + isCount && + ungrouped && + this.settings.approximateCount && + !filter.conditionTree && + !this.model.entity + ) { + return [{ value: await this.executor.describeItemCount(), group: {} }]; + } + + this.guardScan(plan, filter.conditionTree); + + const budget = this.executor.newBudget(); + const segments = plan.access?.kind === 'scan' ? Math.max(this.settings.scanConcurrency, 1) : 1; + const partitions = plan.access?.keyCondition?.partition.values; + + const segmentsOf = (): Array => { + if (partitions) return partitions.map(partition => ({ partition })); + + return segments > 1 + ? Array.from({ length: segments }, (_, index) => ({ segment: { index, total: segments } })) + : [undefined]; + }; + + if (isCount && ungrouped && !plan.lookup && !plan.residual) { + let total = 0; + + await this.inSegments(budget, segmentsOf(), async unit => { + const input = this.buildReadInput( + plan.access, + [], + { + Select: 'COUNT', + ...(unit?.segment + ? { Segment: unit.segment.index, TotalSegments: unit.segment.total } + : {}), + }, + unit?.partition, + ); + + for await (const page of this.executor.readPages(plan.access.kind, input, budget)) { + total += page.count; + } + }); + + return [{ value: total, group: {} }]; + } + + const fields = [ + ...new Set([ + ...aggregation.projection.filter(field => !field.includes(':')), + ...(plan.residual?.projection ?? []).filter(field => !field.includes(':')), + ]), + ]; + const aggregator = new StreamingAggregator(aggregation, timezone); + + await this.inSegments(budget, segmentsOf(), async unit => { + for await (const batch of this.readBatches(plan, fields, budget, unit)) { + for (const item of batch) { + if (!plan.residual || this.matches(plan.residual, item, timezone)) { + aggregator.add(this.toRecord(item)); + } + } + } + }); + + return aggregator.result(limit); + } +} diff --git a/packages/datasource-dynamodb/src/datasource.ts b/packages/datasource-dynamodb/src/datasource.ts new file mode 100644 index 0000000000..50b2822546 --- /dev/null +++ b/packages/datasource-dynamodb/src/datasource.ts @@ -0,0 +1,98 @@ +import type ArrayCollection from './array-collection'; +import type { CollectionSettings, TableModel } from './types'; +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import type { Logger } from '@forestadmin/datasource-toolkit'; + +import { ExecuteStatementCommand } from '@aws-sdk/lib-dynamodb'; +import { BaseDataSource, ValidationError } from '@forestadmin/datasource-toolkit'; + +import DynamoDBCollection from './collection'; +import { ScanLimitExceededError } from './errors'; +import parameterize from './utils/partiql'; + +/** + * PartiQL does not say how many items a statement evaluated, only how many it returned, so the + * read budget bounds the rows returned and the pages (up to 1 MB each) fetched. + */ +const MAX_NATIVE_QUERY_PAGES = 1000; +const READ_ONLY_STATEMENT = /^\s*select\b/i; + +type NativeQueryConnection = { client: DynamoDBDocumentClient }; + +export default class DynamoDBDataSource extends BaseDataSource< + DynamoDBCollection | ArrayCollection +> { + constructor( + private readonly client: DynamoDBDocumentClient, + models: TableModel[], + logger: Logger, + private readonly settings: CollectionSettings, + options?: { liveQueryConnection?: string }, + ) { + super(); + + [...models] + // keeps the schema from being reordered between two starts + .sort((a, b) => (a.name > b.name ? 1 : -1)) + .forEach(model => + this.addCollection(new DynamoDBCollection(this, model, client, logger, settings)), + ); + + if (options?.liveQueryConnection) { + this.addNativeQueryConnection(options.liveQueryConnection, { client }); + } + + logger('Info', 'DynamoDBDataSource - Built'); + } + + /** Runs a PartiQL SELECT: `$name` placeholders become positional parameters. */ + override async executeNativeQuery( + connectionName: string, + query: string, + contextVariables: Record = {}, + ): Promise[]> { + const connection = this.nativeQueryConnections[connectionName] as NativeQueryConnection; + + if (!connection) throw new ValidationError(`Unknown connection name '${connectionName}'`); + + if (!READ_ONLY_STATEMENT.test(query)) { + throw new ValidationError('Only SELECT statements can be run as native queries'); + } + + const { statement, parameters } = parameterize(query, contextVariables); + + const rows: Record[] = []; + let nextToken: string; + let pages = 0; + + do { + // eslint-disable-next-line no-await-in-loop + const response = await connection.client.send( + new ExecuteStatementCommand({ + Statement: statement, + ...(parameters.length ? { Parameters: parameters as never } : {}), + ...(nextToken ? { NextToken: nextToken } : {}), + }), + ); + + rows.push(...((response.Items ?? []) as Record[])); + nextToken = response.NextToken; + pages += 1; + + if (nextToken && pages >= MAX_NATIVE_QUERY_PAGES) { + throw new ScanLimitExceededError( + `The native query still has results after ${pages} pages. Narrow it with a WHERE clause.`, + ); + } + + if (rows.length > this.settings.maxScannedItems) { + throw new ScanLimitExceededError( + `The native query returned more than ${this.settings.maxScannedItems} items, which is ` + + `maxScannedItems. Narrow the results with a WHERE clause.`, + ); + } + } while (nextToken); + + return rows; + } +} diff --git a/packages/datasource-dynamodb/src/errors.ts b/packages/datasource-dynamodb/src/errors.ts new file mode 100644 index 0000000000..8dc741edea --- /dev/null +++ b/packages/datasource-dynamodb/src/errors.ts @@ -0,0 +1,20 @@ +/* eslint-disable max-classes-per-file */ +import { + BadRequestError, + ForbiddenError, + InternalServerError, + UnprocessableError, +} from '@forestadmin/datasource-toolkit'; + +export class ScanDeniedError extends ForbiddenError {} + +export class ScanLimitExceededError extends UnprocessableError {} + +export class OffsetTooDeepError extends BadRequestError {} + +export class DynamoDBOperationError extends InternalServerError { + constructor(message: string, cause: unknown) { + super(message); + Object.assign(this, { cause }); + } +} diff --git a/packages/datasource-dynamodb/src/index.ts b/packages/datasource-dynamodb/src/index.ts new file mode 100644 index 0000000000..5f29af0439 --- /dev/null +++ b/packages/datasource-dynamodb/src/index.ts @@ -0,0 +1,193 @@ +import type { ConfigurationOptions } from './introspection/builder'; +import type { + DynamoDBClientOptions, + DynamoDBSettingsOptions, + Introspection, + IntrospectionConfig, + TableModel, + VirtualCollectionConfig, +} from './types'; +import type { DataSourceFactory, Logger } from '@forestadmin/datasource-toolkit'; + +import { ValidationError } from '@forestadmin/datasource-toolkit'; + +import createDocumentClient from './client'; +import DynamoDBDataSource from './datasource'; +import { DynamoDBDatasourceBuilder } from './introspection/builder'; +import { + DEFAULT_MAX_DEPTH, + DEFAULT_SAMPLE_SIZE, + introspectTables, +} from './introspection/introspector'; +import { convertManualSchema, toDefinition } from './schema/manual-schema'; +import { resolveSettings } from './settings'; +import { INTROSPECTION_VERSION } from './types'; +import PaginationCache from './utils/pagination-cache'; +import addVirtualCollections from './virtual-collections'; + +export { default as DynamoDBCollection } from './collection'; +export { default as ArrayCollection } from './array-collection'; +export { default as DynamoDBDataSource } from './datasource'; +export { ScanDeniedError, ScanLimitExceededError, OffsetTooDeepError } from './errors'; +export type { + ConfigurationOptions, + DynamoDBDatasourceOptionsBuilder, + TableCollectionOptions, +} from './introspection/builder'; +export type { + CollectionDefinition, + DynamoDBClientOptions, + DynamoDBSettingsOptions, + EntityDefinition, + FieldDefinition, + IndexDefinition, + Introspection, + IntrospectionConfig, + ManualSchemaConfig, + ScanPolicy, + VirtualCollectionConfig, +} from './types'; + +export type DynamoDBDataSourceOptions = DynamoDBClientOptions & + DynamoDBSettingsOptions & { + /** + * Tables to introspect, instead of every table of the account and region. An empty list + * introspects none, which leaves the builder collections alone. + */ + tables?: string[]; + /** + * Adjusts the collections one table at a time: to name them, tune them or share a table + * between several. The tables it names are not introspected a second time. + */ + builder?: ConfigurationOptions; + introspectionConfig?: IntrospectionConfig; + /** Exposes arrays of maps as collections of their own. */ + virtualCollections?: VirtualCollectionConfig[]; + /** Another name for `virtualCollections`, the one the Cosmos DB datasource uses. */ + virtualArrayCollections?: VirtualCollectionConfig[]; + /** Name of the connection that runs PartiQL native queries. Without it, none is offered. */ + liveQueryConnection?: string; + }; + +const withoutUndefined = (value: T): T => + Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined)) as T; + +async function liveModels( + options: DynamoDBDataSourceOptions, + client: ReturnType, + logger: Logger, +): Promise { + const sampleSize = options.introspectionConfig?.sampleSize ?? DEFAULT_SAMPLE_SIZE; + const maxDepth = options.introspectionConfig?.maxDepth ?? DEFAULT_MAX_DEPTH; + + const fromBuilder = options.builder + ? await ( + options.builder( + new DynamoDBDatasourceBuilder(client, { sampleSize, maxDepth }), + ) as DynamoDBDatasourceBuilder + ).createCollectionsFromConfiguration() + : []; + + const models = await introspectTables( + client, + { + tables: options.tables, + exclude: fromBuilder.map(model => model.tableName), + sampleSize, + maxDepth, + }, + logger, + ); + + return [...fromBuilder, ...models]; +} + +async function loadModels( + options: DynamoDBDataSourceOptions, + introspection: Introspection | undefined, + client: ReturnType, + logger: Logger, +): Promise { + if (introspection) { + if (options.builder) { + throw new ValidationError('An introspection and a builder cannot be used together'); + } + + logger('Info', 'Using the given introspection'); + + return convertManualSchema(introspection); + } + + const models = await liveModels(options, client, logger); + + if (models.length === 0) logger('Warn', 'No collections were introspected.'); + + return models; +} + +/** + * Reads the tables and describes them as collection definitions. Save the result, for instance in + * a file, and give it back as the `introspection` of `createDynamoDBDataSource` to start without + * sampling the tables. + */ +export async function introspect( + options: DynamoDBDataSourceOptions = {}, + logger: Logger = () => {}, +): Promise { + const client = createDocumentClient(options); + const models = await liveModels(options, client, logger); + + return { version: INTROSPECTION_VERSION, collections: models.map(toDefinition) }; +} + +/** + * Creates a datasource over DynamoDB tables. + * + * @example + * agent.addDataSource( + * createDynamoDBDataSource({ region: 'eu-west-1', tables: ['Orders'], scanPolicy: 'warn' }), + * ); + */ +export function createDynamoDBDataSource( + options: DynamoDBDataSourceOptions = {}, + dataSourceOptions: { introspection?: Introspection } = {}, +): DataSourceFactory { + return async logger => { + const client = createDocumentClient(options); + const settings = resolveSettings( + withoutUndefined({ + scanPolicy: options.scanPolicy, + maxScannedItems: options.maxScannedItems, + scanConcurrency: options.scanConcurrency, + approximateCount: options.approximateCount, + chartCacheTtlMs: options.chartCacheTtlMs, + maxOffset: options.maxOffset, + logConsumedCapacity: options.logConsumedCapacity, + logQueries: options.logQueries, + inMemoryProcessing: options.inMemoryProcessing, + paginationCache: new PaginationCache(options.paginationCache), + }), + ); + + const models = await loadModels(options, dataSourceOptions.introspection, client, logger); + const dataSource = new DynamoDBDataSource(client, models, logger, settings, { + liveQueryConnection: options.liveQueryConnection, + }); + + const virtualCollections = [ + ...(options.virtualCollections ?? []), + ...(options.virtualArrayCollections ?? []), + ]; + + if (virtualCollections.length) { + await addVirtualCollections( + dataSource, + virtualCollections, + options.introspectionConfig?.sampleSize, + logger, + ); + } + + return dataSource; + }; +} diff --git a/packages/datasource-dynamodb/src/introspection/builder.ts b/packages/datasource-dynamodb/src/introspection/builder.ts new file mode 100644 index 0000000000..9d84010b7f --- /dev/null +++ b/packages/datasource-dynamodb/src/introspection/builder.ts @@ -0,0 +1,61 @@ +import type { + EntityDefinition, + IntrospectionConfig, + OverrideTypeConverter, + ScanPolicy, + TableModel, +} from '../types'; +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; + +import { introspectTable } from './introspector'; + +export type TableCollectionOptions = { + /** Name of the collection in Forest Admin. */ + name: string; + tableName: string; + sampleSize?: number; + maxDepth?: number; + /** Keeps the collection to a part of a table shared with other collections. */ + entity?: EntityDefinition; + /** Allows the pagination to show the number of pages, at the cost of counting. Default: true. */ + enableCount?: boolean; + scanPolicy?: ScanPolicy; + maxScannedItems?: number; + overrideTypeConverter?: OverrideTypeConverter; +}; + +export interface DynamoDBDatasourceOptionsBuilder { + addCollectionFromTable(options: TableCollectionOptions): this; +} + +export type ConfigurationOptions = ( + configurator: DynamoDBDatasourceOptionsBuilder, +) => DynamoDBDatasourceOptionsBuilder; + +/** Adds collections one table at a time, to name them or to share a table between several. */ +export class DynamoDBDatasourceBuilder implements DynamoDBDatasourceOptionsBuilder { + private readonly pending: Array> = []; + + constructor( + private readonly client: DynamoDBDocumentClient, + private readonly defaults: IntrospectionConfig = {}, + ) {} + + addCollectionFromTable(options: TableCollectionOptions): this { + const { name, ...rest } = options; + + this.pending.push( + introspectTable(this.client, { + ...this.defaults, + ...rest, + collectionName: name, + }), + ); + + return this; + } + + createCollectionsFromConfiguration(): Promise { + return Promise.all(this.pending); + } +} diff --git a/packages/datasource-dynamodb/src/introspection/introspector.ts b/packages/datasource-dynamodb/src/introspection/introspector.ts new file mode 100644 index 0000000000..25c8dd69e5 --- /dev/null +++ b/packages/datasource-dynamodb/src/introspection/introspector.ts @@ -0,0 +1,322 @@ +import type { + AttributeSchema, + DynamoDataType, + EntityDefinition, + IndexModel, + KeyAttribute, + OverrideTypeConverter, + ScanPolicy, + TableModel, +} from '../types'; +import type { TableDescription } from '@aws-sdk/client-dynamodb'; +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import type { Logger } from '@forestadmin/datasource-toolkit'; + +import { DescribeTableCommand, ListTablesCommand } from '@aws-sdk/client-dynamodb'; +import { ScanCommand } from '@aws-sdk/lib-dynamodb'; +import { ValidationError } from '@forestadmin/datasource-toolkit'; + +import { mapWithConcurrency } from '../utils/concurrency'; +import ExpressionBuilder from '../utils/expression-builder'; +import { PATH_SEPARATOR } from '../utils/serializer'; +import TypeConverter from '../utils/type-converter'; + +export const DEFAULT_SAMPLE_SIZE = 100; +export const DEFAULT_MAX_DEPTH = 5; +const INTROSPECTION_CONCURRENCY = 5; +/** A sample restricted to an entity can need several scans, as the limit counts unfiltered items. */ +const MAX_SAMPLE_SCANS = 20; +const KEY_TYPE_FOR_ATTRIBUTE = { S: 'S', N: 'N' } as const; + +export type TableIntrospectionOptions = { + collectionName: string; + tableName: string; + sampleSize?: number; + maxDepth?: number; + entity?: EntityDefinition; + enableCount?: boolean; + scanPolicy?: ScanPolicy; + maxScannedItems?: number; + overrideTypeConverter?: OverrideTypeConverter; +}; + +type Item = Record; + +function toKeyAttribute(table: TableDescription, name: string): KeyAttribute { + const attributeType = table.AttributeDefinitions?.find( + definition => definition.AttributeName === name, + )?.AttributeType; + + if (attributeType === 'B') { + throw new ValidationError( + `Key '${name}' of table '${table.TableName}' is binary, which is not supported`, + ); + } + + return { name, type: KEY_TYPE_FOR_ATTRIBUTE[attributeType as 'S' | 'N'] ?? 'S' }; +} + +function keysOf( + table: TableDescription, + keySchema: TableDescription['KeySchema'], +): { partitionKey: KeyAttribute; sortKey?: KeyAttribute } { + const partition = keySchema.find(key => key.KeyType === 'HASH'); + const sort = keySchema.find(key => key.KeyType === 'RANGE'); + + return { + partitionKey: toKeyAttribute(table, partition.AttributeName), + sortKey: sort ? toKeyAttribute(table, sort.AttributeName) : undefined, + }; +} + +function indexesOf(table: TableDescription): IndexModel[] { + const describe = ( + indexes: TableDescription['GlobalSecondaryIndexes'], + type: IndexModel['type'], + ): IndexModel[] => + (indexes ?? []).map(index => ({ + name: index.IndexName, + type, + ...keysOf(table, index.KeySchema), + projectsAllAttributes: index.Projection?.ProjectionType === 'ALL', + })); + + return [ + ...describe(table.GlobalSecondaryIndexes, 'GSI'), + ...describe(table.LocalSecondaryIndexes, 'LSI'), + ]; +} + +/** Records the type of every attribute of an item, expanding nested maps up to `maxDepth`. */ +function collectTypes( + value: unknown, + path: string, + depth: number, + maxDepth: number, + types: Record, + presentInItem: Set, + arrayKinds: Record, +): void { + const record = (type: DynamoDataType) => { + types[path] ??= []; + types[path].push(type); + presentInItem.add(path); + }; + + const inferred = TypeConverter.inferTypeFromValue(value); + + if (inferred !== 'object') { + if (path) { + record(inferred); + + if (inferred === 'array') { + arrayKinds[path] ??= { sets: 0, lists: 0 }; + arrayKinds[path][value instanceof Set ? 'sets' : 'lists'] += 1; + } + } + + return; + } + + if (depth >= maxDepth) { + record('object'); + + return; + } + + const entries = Object.entries(value as Item); + + if (entries.length === 0) record('object'); + + for (const [key, nested] of entries) { + if (key.includes(PATH_SEPARATOR)) { + throw new ValidationError( + `Attribute '${key}' contains '${PATH_SEPARATOR}', which names nested attributes`, + ); + } + + collectTypes( + nested, + path ? `${path}${PATH_SEPARATOR}${key}` : key, + depth + 1, + maxDepth, + types, + presentInItem, + arrayKinds, + ); + } +} + +export function inferSchema( + items: Item[], + keys: KeyAttribute[], + maxDepth: number, +): Record { + // Attribute names are data: `constructor` or `__proto__` must not reach Object.prototype. + const types: Record = Object.create(null); + const presence: Record = Object.create(null); + const arrayKinds: Record = Object.create(null); + + for (const item of items) { + const present = new Set(); + collectTypes(item, '', 0, maxDepth, types, present, arrayKinds); + present.forEach(path => { + presence[path] = (presence[path] ?? 0) + 1; + }); + } + + const schema: Record = Object.create(null); + + // A map seen empty or null in some items and populated in others was recorded as a value and + // as nested attributes: only the nested ones are columns, the map itself is expanded. + const paths = Object.keys(types); + const hasNested = (path: string) => + paths.some(other => other.startsWith(`${path}${PATH_SEPARATOR}`)); + + for (const [path, observed] of Object.entries(types).filter(([name]) => !hasNested(name))) { + const type = TypeConverter.getMostSpecificType(observed); + + schema[path] = { + type, + nullable: observed.includes('null') || presence[path] < items.length, + // Only an attribute that was always a set is written back as one. + ...(type === 'array' && arrayKinds[path]?.sets > 0 && arrayKinds[path].lists === 0 + ? { set: true } + : {}), + }; + } + + for (const key of keys) { + schema[key.name] = { type: key.type === 'N' ? 'number' : 'string', nullable: false }; + } + + return schema; +} + +async function sampleItems( + client: DynamoDBDocumentClient, + table: TableDescription, + sortKey: KeyAttribute | undefined, + options: TableIntrospectionOptions, +): Promise { + const sampleSize = options.sampleSize ?? DEFAULT_SAMPLE_SIZE; + const { entity } = options; + const builder = new ExpressionBuilder(); + let filter: string; + + if (entity?.sortKeyPrefix !== undefined) { + filter = `begins_with(${builder.attribute(sortKey.name)}, ${builder.value( + entity.sortKeyPrefix, + )})`; + } else if (entity?.attribute !== undefined) { + filter = `${builder.attribute(entity.attribute)} = ${builder.value(entity.value)}`; + } + + const items: Item[] = []; + let startKey: Item; + + for (let scans = 0; scans < MAX_SAMPLE_SCANS && items.length < sampleSize; scans += 1) { + // eslint-disable-next-line no-await-in-loop + const response = await client.send( + new ScanCommand({ + TableName: table.TableName, + Limit: sampleSize, + ...(filter ? { FilterExpression: filter, ...builder.result() } : {}), + ...(startKey ? { ExclusiveStartKey: startKey } : {}), + }), + ); + + items.push(...((response.Items ?? []) as Item[])); + startKey = response.LastEvaluatedKey; + + if (!startKey) break; + } + + return items.slice(0, sampleSize); +} + +/** Reads the key schema and indexes of a table, and infers its attributes from a sample. */ +export async function introspectTable( + client: DynamoDBDocumentClient, + options: TableIntrospectionOptions, +): Promise { + const { Table: table } = await client.send( + new DescribeTableCommand({ TableName: options.tableName }), + ); + const { partitionKey, sortKey } = keysOf(table, table.KeySchema); + + if (options.entity?.sortKeyPrefix !== undefined && !sortKey) { + throw new ValidationError( + `Table '${options.tableName}' has no sort key, so 'sortKeyPrefix' cannot be used`, + ); + } + + const items = await sampleItems(client, table, sortKey, options); + + return { + name: options.collectionName, + tableName: options.tableName, + partitionKey, + sortKey, + indexes: indexesOf(table), + schema: inferSchema( + items, + [partitionKey, sortKey].filter(Boolean), + options.maxDepth ?? DEFAULT_MAX_DEPTH, + ), + entity: options.entity, + enableCount: options.enableCount, + scanPolicy: options.scanPolicy, + maxScannedItems: options.maxScannedItems, + overrideTypeConverter: options.overrideTypeConverter, + }; +} + +async function listTables(client: DynamoDBDocumentClient): Promise { + const names: string[] = []; + let startName: string; + + do { + // eslint-disable-next-line no-await-in-loop + const response = await client.send( + new ListTablesCommand(startName ? { ExclusiveStartTableName: startName } : {}), + ); + names.push(...(response.TableNames ?? [])); + startName = response.LastEvaluatedTableName; + } while (startName); + + return names; +} + +/** Introspects the given tables, or every table of the account and region. */ +export async function introspectTables( + client: DynamoDBDocumentClient, + options: { tables?: string[]; exclude?: string[]; sampleSize?: number; maxDepth?: number }, + logger: Logger, +): Promise { + const excluded = new Set(options.exclude); + const tableNames = (options.tables ?? (await listTables(client))).filter( + name => !excluded.has(name), + ); + + const models = await mapWithConcurrency( + tableNames, + INTROSPECTION_CONCURRENCY, + async tableName => { + try { + return await introspectTable(client, { + collectionName: tableName, + tableName, + sampleSize: options.sampleSize, + maxDepth: options.maxDepth, + }); + } catch (error) { + logger('Warn', `Introspector - Skipping table '${tableName}': ${error.message}`); + + return null; + } + }, + ); + + return models.filter(Boolean); +} diff --git a/packages/datasource-dynamodb/src/schema/manual-schema.ts b/packages/datasource-dynamodb/src/schema/manual-schema.ts new file mode 100644 index 0000000000..d612a37f70 --- /dev/null +++ b/packages/datasource-dynamodb/src/schema/manual-schema.ts @@ -0,0 +1,320 @@ +import type { + AttributeSchema, + CollectionDefinition, + DynamoDataType, + FieldDefinition, + IndexModel, + KeyAttribute, + ManualSchemaConfig, + TableModel, +} from '../types'; + +import { ValidationError } from '@forestadmin/datasource-toolkit'; + +import { INTROSPECTION_VERSION } from '../types'; +import { PATH_SEPARATOR } from '../utils/serializer'; + +const FIELD_TYPES: DynamoDataType[] = [ + 'string', + 'number', + 'boolean', + 'date', + 'dateonly', + 'timeonly', + 'array', + 'object', + 'null', + 'point', + 'binary', +]; +const SCAN_POLICIES = ['allow', 'warn', 'deny']; +const KEY_TYPES = ['string', 'number']; + +function validateFields(fields: FieldDefinition[], collectionName: string, path = ''): void { + const names = new Set(); + + for (const field of fields) { + const fullPath = path ? `${path}${PATH_SEPARATOR}${field.name}` : field.name; + + if (!field.name) { + throw new ValidationError(`A field of collection '${collectionName}' has no name`); + } + + if (field.name.includes(PATH_SEPARATOR)) { + throw new ValidationError( + `Field '${fullPath}' in collection '${collectionName}' contains '${PATH_SEPARATOR}', ` + + `which names nested attributes`, + ); + } + + if (names.has(field.name)) { + throw new ValidationError( + `Duplicate field name: '${fullPath}' in collection '${collectionName}'`, + ); + } + + names.add(field.name); + + if (!FIELD_TYPES.includes(field.type)) { + throw new ValidationError( + `Field '${fullPath}' in collection '${collectionName}' has invalid type '${field.type}'. ` + + `Valid types: ${FIELD_TYPES.join(', ')}`, + ); + } + + const isSetCandidate = + field.type === 'array' && (!field.subType || ['string', 'number'].includes(field.subType)); + + if (field.set && !isSetCandidate) { + throw new ValidationError( + `Field '${fullPath}' in collection '${collectionName}' cannot be a set: only an array ` + + `of strings or numbers can`, + ); + } + + if (field.type === 'array' && field.subType === 'object' && !field.fields?.length) { + throw new ValidationError( + `Array field '${fullPath}' in collection '${collectionName}' of objects must have ` + + `nested field definitions`, + ); + } + + if (field.fields) validateFields(field.fields, collectionName, fullPath); + } +} + +function validateCollection(collection: CollectionDefinition): void { + const { name } = collection; + + if (!name) throw new ValidationError('Collection must have a name'); + + if (!collection.tableName) { + throw new ValidationError(`Collection '${name}' must have a tableName`); + } + + if (!collection.partitionKey?.name) { + throw new ValidationError(`Collection '${name}' must have a partitionKey`); + } + + for (const key of [collection.partitionKey, collection.sortKey].filter(Boolean)) { + if (!KEY_TYPES.includes(key.type)) { + throw new ValidationError( + `Key '${key.name}' of collection '${name}' must be a string or a number`, + ); + } + } + + if (collection.sortKey?.name === collection.partitionKey.name) { + throw new ValidationError( + `Collection '${name}' uses '${collection.partitionKey.name}' as both partition key and sort key`, + ); + } + + const { entity } = collection; + + if (entity) { + const hasDiscriminator = entity.attribute !== undefined || entity.value !== undefined; + + const incompleteDiscriminator = + hasDiscriminator && (entity.attribute === undefined || entity.value === undefined); + + if (hasDiscriminator === (entity.sortKeyPrefix !== undefined) || incompleteDiscriminator) { + throw new ValidationError( + `Collection '${name}' entity must use either 'attribute'/'value' or 'sortKeyPrefix'`, + ); + } + + if (entity.sortKeyPrefix !== undefined && !collection.sortKey) { + throw new ValidationError( + `Collection '${name}' has no sortKey, so 'entity.sortKeyPrefix' cannot be used`, + ); + } + } + + if (collection.scanPolicy !== undefined && !SCAN_POLICIES.includes(collection.scanPolicy)) { + throw new ValidationError( + `Collection '${name}' has an invalid scanPolicy '${collection.scanPolicy}'`, + ); + } + + validateFields(collection.fields ?? [], name); +} + +export function validateManualSchema(schema: ManualSchemaConfig): void { + if (schema?.version !== undefined && schema.version !== INTROSPECTION_VERSION) { + throw new ValidationError( + `The introspection is in version ${schema.version}, this datasource reads version ` + + `${INTROSPECTION_VERSION}`, + ); + } + + if (!Array.isArray(schema?.collections)) { + throw new ValidationError('The introspection must have a collections array'); + } + + if (schema.collections.length === 0) { + throw new ValidationError('The introspection must have at least one collection'); + } + + const names = new Set(); + + for (const collection of schema.collections) { + validateCollection(collection); + + if (names.has(collection.name)) { + throw new ValidationError(`Duplicate collection name: '${collection.name}'`); + } + + names.add(collection.name); + } +} + +function flattenFields(fields: FieldDefinition[], prefix = ''): Record { + const attributes: Record = {}; + + for (const field of fields) { + const path = prefix ? `${prefix}${PATH_SEPARATOR}${field.name}` : field.name; + + if (field.type === 'object' && field.fields?.length) { + Object.assign(attributes, flattenFields(field.fields, path)); + } else { + attributes[path] = { + type: field.type, + nullable: !!field.nullable, + ...(field.type === 'array' && field.set ? { set: true } : {}), + ...(field.type === 'array' && field.subType === 'object' + ? { items: flattenFields(field.fields) } + : {}), + }; + } + } + + return attributes; +} + +const toKey = (key: { name: string; type: 'string' | 'number' }): KeyAttribute => ({ + name: key.name, + type: key.type === 'number' ? 'N' : 'S', +}); + +function toModel(collection: CollectionDefinition): TableModel { + const partitionKey = toKey(collection.partitionKey); + const sortKey = collection.sortKey ? toKey(collection.sortKey) : undefined; + const keys: Record = {}; + + for (const key of [partitionKey, sortKey].filter(Boolean)) { + keys[key.name] = { type: key.type === 'N' ? 'number' : 'string', nullable: false }; + } + + const indexes: IndexModel[] = (collection.indexes ?? []).map(index => ({ + name: index.name, + type: index.type, + partitionKey: toKey(index.partitionKey), + sortKey: index.sortKey ? toKey(index.sortKey) : undefined, + projectsAllAttributes: index.projectsAllAttributes ?? true, + coversAllItems: index.coversAllItems ?? false, + })); + + return { + name: collection.name, + tableName: collection.tableName, + partitionKey, + sortKey, + indexes, + // The configured key types are the table's: a field of the same name cannot change them. + schema: { ...flattenFields(collection.fields ?? []), ...keys }, + entity: collection.entity, + enableCount: collection.enableCount, + scanPolicy: collection.scanPolicy, + maxScannedItems: collection.maxScannedItems, + overrideTypeConverter: collection.overrideTypeConverter, + }; +} + +export function convertManualSchema(schema: ManualSchemaConfig): TableModel[] { + validateManualSchema(schema); + + return schema.collections.map(toModel); +} + +const keyType = (key: KeyAttribute): 'string' | 'number' => + key.type === 'N' ? 'number' : 'string'; + +/** Nests the attributes of a table, flattened by name, back into field definitions. */ +function nestFields( + attributes: Record, + keys: Set, +): FieldDefinition[] { + const fields: FieldDefinition[] = []; + const objects = new Map(); + + for (const [path, attribute] of Object.entries(attributes)) { + const segments = path.split(PATH_SEPARATOR); + + if (!keys.has(path)) { + let siblings = fields; + + segments.slice(0, -1).forEach((segment, depth) => { + const objectPath = segments.slice(0, depth + 1).join(PATH_SEPARATOR); + let object = objects.get(objectPath); + + if (!object) { + object = { name: segment, type: 'object', fields: [] }; + objects.set(objectPath, object); + siblings.push(object); + } + + siblings = object.fields; + }); + + const field: FieldDefinition = { + name: segments[segments.length - 1], + type: attribute.type, + nullable: attribute.nullable ?? false, + ...(attribute.set ? { set: true } : {}), + }; + + if (attribute.items) { + field.subType = 'object'; + field.fields = nestFields(attribute.items, new Set()); + } + + siblings.push(field); + } + } + + return fields; +} + +/** The collection definition a table model comes from: what `introspect` returns. */ +export function toDefinition(model: TableModel): CollectionDefinition { + const keys = new Set([model.partitionKey.name, model.sortKey?.name].filter(Boolean)); + + return { + name: model.name, + tableName: model.tableName, + partitionKey: { name: model.partitionKey.name, type: keyType(model.partitionKey) }, + ...(model.sortKey + ? { sortKey: { name: model.sortKey.name, type: keyType(model.sortKey) } } + : {}), + ...(model.indexes.length + ? { + indexes: model.indexes.map(index => ({ + name: index.name, + type: index.type, + partitionKey: { name: index.partitionKey.name, type: keyType(index.partitionKey) }, + ...(index.sortKey + ? { sortKey: { name: index.sortKey.name, type: keyType(index.sortKey) } } + : {}), + projectsAllAttributes: index.projectsAllAttributes, + coversAllItems: index.coversAllItems ?? false, + })), + } + : {}), + fields: nestFields(model.schema, keys), + ...(model.entity ? { entity: model.entity } : {}), + ...(model.enableCount !== undefined ? { enableCount: model.enableCount } : {}), + ...(model.scanPolicy ? { scanPolicy: model.scanPolicy } : {}), + ...(model.maxScannedItems !== undefined ? { maxScannedItems: model.maxScannedItems } : {}), + }; +} diff --git a/packages/datasource-dynamodb/src/schema/schema-converter.ts b/packages/datasource-dynamodb/src/schema/schema-converter.ts new file mode 100644 index 0000000000..4c9c8d7bc1 --- /dev/null +++ b/packages/datasource-dynamodb/src/schema/schema-converter.ts @@ -0,0 +1,76 @@ +import type { AttributeSchema, TableModel } from '../types'; +import type { ColumnSchema, FieldSchema, Logger } from '@forestadmin/datasource-toolkit'; + +import { PUSHABLE_OPERATORS } from '../utils/expression-builder'; +import TypeConverter from '../utils/type-converter'; + +function convertAttribute( + attribute: AttributeSchema, + isPrimaryKey: boolean, + isOrdered: boolean, + inMemoryProcessing: boolean, +): ColumnSchema { + const columnType = TypeConverter.getColumnTypeFromDataType(attribute.type); + const operators = TypeConverter.operatorsForColumnType(columnType); + + return { + type: 'Column', + columnType, + filterOperators: inMemoryProcessing + ? operators + : new Set([...operators].filter(operator => PUSHABLE_OPERATORS.has(operator))), + isPrimaryKey, + isReadOnly: false, + // DynamoDB orders by the sort key of the table or of an index and by nothing else. + isSortable: TypeConverter.isSortable(attribute.type) && (inMemoryProcessing || isOrdered), + validation: [], + }; +} + +/** + * Turns the attributes of a table into the columns of its collection, keys being the primary key. + * Without `inMemoryProcessing`, a column only offers the filters DynamoDB evaluates itself, and + * only the sort keys sort. + */ +export default function convertModelToFields( + model: TableModel, + logger?: Logger, + inMemoryProcessing = true, +): Record { + const keys = [model.partitionKey, model.sortKey].filter(Boolean); + const attributes: Record = { ...model.schema }; + + for (const key of keys) { + attributes[key.name] ??= { type: key.type === 'N' ? 'number' : 'string', nullable: false }; + } + + const sortKeys = new Set( + [model.sortKey, ...model.indexes.map(index => index.sortKey)] + .filter(Boolean) + .map(key => key.name), + ); + const fields: Record = {}; + + for (const [name, attribute] of Object.entries(attributes)) { + try { + const isPrimaryKey = keys.some(key => key.name === name); + const generated = convertAttribute( + attribute, + isPrimaryKey, + sortKeys.has(name), + inMemoryProcessing, + ); + const overridden = model.overrideTypeConverter?.({ + fieldName: name, + attribute, + generatedFieldSchema: generated, + }); + + fields[name] = overridden || generated; + } catch (error) { + logger?.('Warn', `Skipping column '${model.name}.${name}' (${error.message})`); + } + } + + return fields; +} diff --git a/packages/datasource-dynamodb/src/settings.ts b/packages/datasource-dynamodb/src/settings.ts new file mode 100644 index 0000000000..5c68f36d27 --- /dev/null +++ b/packages/datasource-dynamodb/src/settings.ts @@ -0,0 +1,37 @@ +import type { CollectionSettings } from './types'; + +import { ValidationError } from '@forestadmin/datasource-toolkit'; + +import PaginationCache from './utils/pagination-cache'; + +export const DEFAULT_MAX_SCANNED_ITEMS = 1000000; + +const SCAN_POLICIES = ['allow', 'warn', 'deny']; + +export function resolveSettings(settings: Partial = {}): CollectionSettings { + const now = settings.now ?? (() => Date.now()); + + if (settings.scanPolicy !== undefined && !SCAN_POLICIES.includes(settings.scanPolicy)) { + throw new ValidationError( + `Invalid scanPolicy '${settings.scanPolicy}': use allow, warn or deny`, + ); + } + + return { + scanPolicy: 'warn', + maxScannedItems: DEFAULT_MAX_SCANNED_ITEMS, + scanConcurrency: 4, + approximateCount: false, + chartCacheTtlMs: 30000, + maxOffset: 100000, + maxWriteRetries: 5, + writeRetryBaseDelayMs: 50, + logConsumedCapacity: false, + logQueries: false, + inMemoryProcessing: false, + ...settings, + now, + paginationCache: settings.paginationCache ?? new PaginationCache({ now }), + chartCaches: settings.chartCaches ?? new Map(), + }; +} diff --git a/packages/datasource-dynamodb/src/types.ts b/packages/datasource-dynamodb/src/types.ts new file mode 100644 index 0000000000..be3ca9fe8d --- /dev/null +++ b/packages/datasource-dynamodb/src/types.ts @@ -0,0 +1,196 @@ +import type PaginationCache from './utils/pagination-cache'; +import type { PaginationCacheOptions } from './utils/pagination-cache'; +import type { DynamoDBClientConfig } from '@aws-sdk/client-dynamodb'; +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import type { AggregateResult, FieldSchema } from '@forestadmin/datasource-toolkit'; + +export type ScanPolicy = 'allow' | 'warn' | 'deny'; + +export type KeyType = 'S' | 'N'; + +export type KeyAttribute = { name: string; type: KeyType }; + +export type DynamoDataType = + | 'string' + | 'number' + | 'boolean' + | 'date' + | 'dateonly' + | 'timeonly' + | 'object' + | 'array' + | 'null' + | 'binary' + | 'point'; + +export type AttributeSchema = { + type: DynamoDataType; + nullable?: boolean; + /** An array stored as a string or number set, which must be written back as one. */ + set?: boolean; + /** Schema of the items of an array of maps, used to expose it as a virtual collection. */ + items?: Record; +}; + +export type IndexModel = { + name: string; + type: 'GSI' | 'LSI'; + partitionKey: KeyAttribute; + sortKey?: KeyAttribute; + projectsAllAttributes: boolean; + /** + * Whether every item of the table is in the index. An index leaves out the items that lack its + * sort key, so it is only read without a condition on that key when this is true. + */ + coversAllItems?: boolean; +}; + +/** + * Tells which items of a table belong to a collection, so that several collections can share one + * table (single-table design): either a discriminator attribute or a prefix of the sort key. + */ +export type EntityDefinition = { attribute?: string; value?: string; sortKeyPrefix?: string }; + +export type OverrideTypeConverter = (field: { + fieldName: string; + attribute: AttributeSchema; + generatedFieldSchema: FieldSchema; +}) => void | FieldSchema; + +export type TableModel = { + /** Name of the collection in Forest Admin. */ + name: string; + tableName: string; + partitionKey: KeyAttribute; + sortKey?: KeyAttribute; + indexes: IndexModel[]; + /** Attributes by flattened name: nested maps are `a@@@b`. */ + schema: Record; + entity?: EntityDefinition; + enableCount?: boolean; + scanPolicy?: ScanPolicy; + maxScannedItems?: number; + overrideTypeConverter?: OverrideTypeConverter; +}; + +export type FieldDefinition = { + name: string; + type: DynamoDataType; + nullable?: boolean; + /** For arrays: the type of the items. */ + subType?: DynamoDataType; + /** For arrays of strings or numbers: the attribute is a DynamoDB set, not a list. */ + set?: boolean; + /** For objects, and arrays of objects: the nested attributes. */ + fields?: FieldDefinition[]; +}; + +export type IndexDefinition = { + name: string; + type: 'GSI' | 'LSI'; + partitionKey: { name: string; type: 'string' | 'number' }; + sortKey?: { name: string; type: 'string' | 'number' }; + /** Whether the index projects every attribute, which a query on it needs. Default: true. */ + projectsAllAttributes?: boolean; + /** Every item has the index keys, so the index may be read without a sort key condition. */ + coversAllItems?: boolean; +}; + +export type CollectionDefinition = { + name: string; + tableName: string; + partitionKey: { name: string; type: 'string' | 'number' }; + sortKey?: { name: string; type: 'string' | 'number' }; + fields: FieldDefinition[]; + indexes?: IndexDefinition[]; + entity?: EntityDefinition; + enableCount?: boolean; + scanPolicy?: ScanPolicy; + maxScannedItems?: number; + overrideTypeConverter?: OverrideTypeConverter; +}; + +/** The version of the introspection this datasource reads and writes. */ +export const INTROSPECTION_VERSION = 1; + +/** + * What the datasource knows of the tables: the output of `introspect` and the input of the + * `introspection` option, which skips the live introspection. Written by hand, it is the manual schema. + */ +export type Introspection = { + version?: typeof INTROSPECTION_VERSION; + collections: CollectionDefinition[]; +}; + +/** @deprecated The manual schema is an introspection. */ +export type ManualSchemaConfig = Introspection; + +export type VirtualCollectionConfig = { + /** Name of the collection holding the array. */ + parentCollection: string; + /** Name of the new collection. */ + collectionName: string; + /** Attribute holding the array, `a@@@b` for a nested one. */ + arrayField: string; +}; + +export type CollectionSettings = { + scanPolicy: ScanPolicy; + maxScannedItems: number; + scanConcurrency: number; + approximateCount: boolean; + chartCacheTtlMs: number; + maxOffset: number; + maxWriteRetries: number; + writeRetryBaseDelayMs: number; + logConsumedCapacity: boolean; + logQueries: boolean; + inMemoryProcessing: boolean; + paginationCache: PaginationCache; + /** Charts by table then by request: every collection of a table drops them on a write. */ + chartCaches: Map>; + now: () => number; +}; + +export type DynamoDBClientOptions = { + /** An existing client, low-level or document client. When set, the options below are ignored. */ + client?: DynamoDBDocumentClient | { send: (command: never) => Promise }; + region?: string; + endpoint?: string; + credentials?: DynamoDBClientConfig['credentials']; + /** Anything else the AWS SDK client accepts. */ + clientConfig?: DynamoDBClientConfig; +}; + +export type DynamoDBSettingsOptions = { + /** What happens when a request has to scan the table. Default: 'warn'. */ + scanPolicy?: ScanPolicy; + /** Items a single request may read before it is stopped. Default: 1 000 000. */ + maxScannedItems?: number; + /** Segments of the parallel scans run for charts. Default: 4. */ + scanConcurrency?: number; + /** Answer an unfiltered count from the table description (refreshed about every 6 hours). */ + approximateCount?: boolean; + /** How long a chart is served from memory, in milliseconds. 0 disables it. Default: 30 000. */ + chartCacheTtlMs?: number; + /** Deepest page that can be requested. Default: 100 000. */ + maxOffset?: number; + /** Log the capacity units each request consumed. */ + logConsumedCapacity?: boolean; + /** Log the expressions sent to DynamoDB. */ + logQueries?: boolean; + /** + * Lets Forest filter and sort on what DynamoDB does not evaluate itself (`IContains`, `Like`, + * relative dates, a sort on any attribute): the datasource reads the items and does it. Off by + * default: those columns are then neither filterable that way nor sortable. + */ + inMemoryProcessing?: boolean; + paginationCache?: PaginationCacheOptions; +}; + +export type IntrospectionConfig = { + /** Items read per table to infer its attributes. Default: 100. */ + sampleSize?: number; + /** Deepest nested map exposed as separate attributes. Default: 5. */ + maxDepth?: number; +}; diff --git a/packages/datasource-dynamodb/src/utils/access-path.ts b/packages/datasource-dynamodb/src/utils/access-path.ts new file mode 100644 index 0000000000..584658c51c --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/access-path.ts @@ -0,0 +1,373 @@ +import type { KeyAttribute, TableModel } from '../types'; +import type { ConditionTree, Sort } from '@forestadmin/datasource-toolkit'; + +import { + ConditionTreeBranch, + ConditionTreeFactory, + ConditionTreeLeaf, +} from '@forestadmin/datasource-toolkit'; + +export type SortKeyOperator = + | 'Equal' + | 'LessThan' + | 'LessThanOrEqual' + | 'GreaterThan' + | 'GreaterThanOrEqual' + | 'StartsWith' + | 'Between'; + +export type SortKeyCondition = { attribute: string; operator: SortKeyOperator; value: unknown }; + +export type AccessPath = { + kind: 'query' | 'scan'; + /** Undefined for the table itself. */ + indexName?: string; + /** Key attributes of the table or index read: a Query filter cannot reference them. */ + keyAttributes?: string[]; + keyCondition?: { + /** `values` lists every partition to read, when the filter names several: one Query each. */ + partition: { attribute: string; value: string | number; values?: Array }; + sort?: SortKeyCondition; + }; + /** What the key condition leaves for a filter expression. */ + remaining?: ConditionTree; + scanIndexForward: boolean; + /** Whether reading in key order already gives the requested sort. */ + sortSatisfied: boolean; +}; + +export type KeyLookup = { keys: Array>; remaining?: ConditionTree }; + +type Candidate = { + indexName?: string; + partitionKey: KeyAttribute; + sortKey?: KeyAttribute; + /** Whether an item missing the sort key can still be in the index: it cannot, unless told so. */ + coversAllItems: boolean; +}; + +/** Keys a lookup may name: a longer list is read some other way than one BatchGetItem. */ +const MAX_LOOKUP_KEYS = 10000; + +/** Partitions a single filter may fan out to: each one is a request of its own. */ +const MAX_PARTITIONS = 50; + +const RANGE_OPERATORS = ['LessThan', 'LessThanOrEqual', 'GreaterThan', 'GreaterThanOrEqual']; + +const isKeyValue = (value: unknown): value is string | number => + typeof value === 'string' || typeof value === 'number'; + +/** DynamoDB tells 1 from "1": a foreign key stored as the other type would find nothing. */ +function coerceKeyValue(attribute: KeyAttribute, value: T): T | string | number { + if (attribute.type === 'N' && typeof value === 'string' && value.trim() !== '') { + return Number.isNaN(Number(value)) ? value : Number(value); + } + + if (attribute.type === 'S' && typeof value === 'number') return String(value); + + return value; +} + +/** BatchGetItem refuses the same key twice. */ +function uniqueKeys(keys: Array>): Array> { + const seen = new Set(); + + return keys.filter(key => { + const id = JSON.stringify(key); + + if (seen.has(id)) return false; + seen.add(id); + + return true; + }); +} + +/** The restriction a single-table entity puts on the items of its table. */ +function withEntity(model: TableModel, tree?: ConditionTree): ConditionTree | undefined { + const { entity } = model; + + if (!entity) return tree; + + let restriction: ConditionTreeLeaf | undefined; + + if (entity.sortKeyPrefix !== undefined && model.sortKey) { + restriction = new ConditionTreeLeaf(model.sortKey.name, 'StartsWith', entity.sortKeyPrefix); + } else if (entity.attribute !== undefined) { + restriction = new ConditionTreeLeaf(entity.attribute, 'Equal', entity.value); + } + + if (!restriction) return tree; + + return tree ? ConditionTreeFactory.intersect(tree, restriction) : restriction; +} + +/** The conditions that must all hold, as the root of an AND or alone. */ +function conjuncts(tree?: ConditionTree): ConditionTree[] { + if (!tree) return []; + + if (tree instanceof ConditionTreeBranch && tree.aggregator === 'And') { + return tree.conditions.flatMap(condition => conjuncts(condition)); + } + + return [tree]; +} + +const leavesOf = (parts: ConditionTree[]): ConditionTreeLeaf[] => + parts.filter((part): part is ConditionTreeLeaf => part instanceof ConditionTreeLeaf); + +function rebuild(parts: ConditionTree[]): ConditionTree | undefined { + return parts.length ? ConditionTreeFactory.intersect(...parts) : undefined; +} + +function pickSortCondition( + sortKey: KeyAttribute, + parts: ConditionTree[], +): { condition: SortKeyCondition; consumed: ConditionTree[] } | undefined { + const attribute = sortKey.name; + const onSortKey = leavesOf(parts).filter( + leaf => leaf.field === attribute && isKeyValue(leaf.value), + ); + const find = (operators: string[]) => onSortKey.filter(leaf => operators.includes(leaf.operator)); + const valueOf = (leaf: ConditionTreeLeaf) => coerceKeyValue(sortKey, leaf.value); + + const [equal] = find(['Equal']); + + if (equal) { + return { + condition: { attribute, operator: 'Equal', value: valueOf(equal) }, + consumed: [equal], + }; + } + + const [lower] = find(['GreaterThanOrEqual']); + const [upper] = find(['LessThanOrEqual']); + + if (lower && upper && !(valueOf(lower) > valueOf(upper))) { + return { + condition: { attribute, operator: 'Between', value: [valueOf(lower), valueOf(upper)] }, + consumed: [lower, upper], + }; + } + + const [range] = find(RANGE_OPERATORS); + + if (range) { + return { + condition: { attribute, operator: range.operator as SortKeyOperator, value: valueOf(range) }, + consumed: [range], + }; + } + + const [prefix] = find(['StartsWith']).sort( + (a, b) => String(b.value).length - String(a.value).length, + ); + + if (prefix) { + return { + condition: { attribute, operator: 'StartsWith', value: valueOf(prefix) }, + consumed: [prefix], + }; + } + + return undefined; +} + +/** + * Picks the cheapest way to read the items a condition tree selects: a Query on the table or on + * an index whenever an equality pins a partition key, a Scan otherwise. + */ +export function planAccessPath( + model: TableModel, + conditionTree?: ConditionTree, + sort?: Sort, +): AccessPath { + const tree = withEntity(model, conditionTree); + const parts = conjuncts(tree); + const leaves = leavesOf(parts); + + const candidates: Candidate[] = [ + { partitionKey: model.partitionKey, sortKey: model.sortKey, coversAllItems: true }, + ...model.indexes + .filter(index => index.projectsAllAttributes) + .map(index => ({ + indexName: index.name, + partitionKey: index.partitionKey, + sortKey: index.sortKey, + coversAllItems: !!index.coversAllItems, + })), + ]; + + let best: { path: AccessPath; score: number } | undefined; + + for (const candidate of candidates) { + const onPartitionKey = leaves.filter(leaf => leaf.field === candidate.partitionKey.name); + const equal = onPartitionKey.find(leaf => leaf.operator === 'Equal' && isKeyValue(leaf.value)); + const partition = + equal ?? + onPartitionKey.find( + leaf => + leaf.operator === 'In' && + Array.isArray(leaf.value) && + leaf.value.length > 0 && + leaf.value.length <= MAX_PARTITIONS && + leaf.value.every(isKeyValue), + ); + const partitionValues = partition + ? [ + ...new Set( + (partition.operator === 'Equal' + ? [partition.value] + : (partition.value as unknown[]) + ).map(value => coerceKeyValue(candidate.partitionKey, value) as string | number), + ), + ] + : []; + + const picked = + partition && candidate.sortKey ? pickSortCondition(candidate.sortKey, parts) : undefined; + + // An index leaves out the items that lack its sort key: without a condition on that key, it + // could return fewer items than the table holds. + const mayMissItems = + !!candidate.indexName && !!candidate.sortKey && !picked && !candidate.coversAllItems; + + if (partition && !mayMissItems) { + const sortClause = sort?.length === 1 ? sort[0] : undefined; + // Several partitions are read one after the other: their items are not in one key order. + const sortSatisfied = + !!sortClause && + sortClause.field === candidate.sortKey?.name && + partitionValues.length === 1; + const consumed = new Set([partition, ...(picked?.consumed ?? [])]); + const score = (picked ? 2 : 0) + (sortSatisfied ? 1 : 0); + + if (!best || score > best.score) { + best = { + score, + path: { + kind: 'query', + indexName: candidate.indexName, + keyAttributes: [candidate.partitionKey.name, candidate.sortKey?.name].filter(Boolean), + keyCondition: { + partition: { + attribute: candidate.partitionKey.name, + value: partitionValues[0], + ...(partitionValues.length > 1 ? { values: partitionValues } : {}), + }, + sort: picked?.condition, + }, + remaining: rebuild(parts.filter(part => !consumed.has(part))), + scanIndexForward: sortSatisfied ? sortClause.ascending : true, + sortSatisfied, + }, + }; + } + } + } + + return ( + best?.path ?? { + kind: 'scan', + remaining: tree, + scanIndexForward: true, + sortSatisfied: false, + } + ); +} + +function keysFromOr(model: TableModel, part: ConditionTree): KeyLookup['keys'] | null { + if (!(part instanceof ConditionTreeBranch) || part.aggregator !== 'Or') return null; + + const keys: KeyLookup['keys'] = []; + + for (const condition of part.conditions) { + const armParts = conjuncts(condition); + + // An arm with a nested branch carries a filter the keys alone would drop. + if (armParts.some(armPart => !(armPart instanceof ConditionTreeLeaf))) return null; + + const equalities = leavesOf(armParts); + const key: Record = {}; + + for (const attribute of [model.partitionKey, model.sortKey].filter(Boolean)) { + const equal = equalities.find( + leaf => + leaf.field === attribute.name && leaf.operator === 'Equal' && isKeyValue(leaf.value), + ); + + if (!equal) return null; + key[attribute.name] = coerceKeyValue(attribute, equal.value); + } + + if (equalities.length !== Object.keys(key).length) return null; + keys.push(key); + } + + return uniqueKeys(keys); +} + +/** + * Detects the filters that name whole keys, which BatchGetItem reads without scanning or + * querying: a single key, a list of keys, or the OR of key equalities the relation emulation sends. + */ +export function planKeyLookup(model: TableModel, conditionTree?: ConditionTree): KeyLookup | null { + const tree = withEntity(model, conditionTree); + + if (!tree) return null; + + const parts = conjuncts(tree); + const keyAttributes = [model.partitionKey, model.sortKey].filter(Boolean); + + const valuesOf = (attribute: KeyAttribute) => { + const leaf = leavesOf(parts).find( + candidate => + candidate.field === attribute.name && + (candidate.operator === 'Equal' || candidate.operator === 'In') && + (candidate.operator === 'Equal' + ? isKeyValue(candidate.value) + : Array.isArray(candidate.value) && candidate.value.every(isKeyValue)), + ); + + if (!leaf) return undefined; + + return { + leaf, + values: [ + ...new Set( + ((leaf.operator === 'Equal' ? [leaf.value] : leaf.value) as Array).map( + value => coerceKeyValue(attribute, value), + ), + ), + ], + }; + }; + + const resolved = keyAttributes.map(attribute => ({ attribute, found: valuesOf(attribute) })); + + if (resolved.every(({ found }) => found)) { + // The keys are the product of the lists: a huge one is never built. + const product = resolved.reduce((count, { found }) => count * found.values.length, 1); + + if (product > MAX_LOOKUP_KEYS) return null; + + const keys = resolved.reduce>>( + (combined, { attribute, found }) => + combined.flatMap(key => found.values.map(value => ({ ...key, [attribute.name]: value }))), + [{}], + ); + const consumed = new Set(resolved.map(({ found }) => found.leaf)); + + return { + keys: uniqueKeys(keys), + remaining: rebuild(parts.filter(part => !consumed.has(part))), + }; + } + + for (const part of parts) { + const keys = keysFromOr(model, part); + + if (keys) return { keys, remaining: rebuild(parts.filter(other => other !== part)) }; + } + + return null; +} diff --git a/packages/datasource-dynamodb/src/utils/aggregator.ts b/packages/datasource-dynamodb/src/utils/aggregator.ts new file mode 100644 index 0000000000..2bf9b854d9 --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/aggregator.ts @@ -0,0 +1,211 @@ +import type { + AggregateResult, + Aggregation, + DateOperation, + RecordData, +} from '@forestadmin/datasource-toolkit'; + +import compareValues from './compare'; + +/** Numbers below this are epoch seconds: in milliseconds they would be before 1973. */ +const EPOCH_SECONDS_CEILING = 1e11; +const MONTHS_PER_QUARTER = 3; +const DAYS_PER_WEEK = 7; + +type Accumulator = { + group: RecordData; + records: number; + values: number; + numbers: number; + sum: number; + min: unknown; + max: unknown; +}; + +function toDate(value: unknown): Date | null { + let date: Date; + + if (value instanceof Date) { + date = value; + } else if (typeof value === 'number') { + date = new Date(value < EPOCH_SECONDS_CEILING ? value * 1000 : value); + } else if (typeof value === 'string') { + date = new Date(value); + } else { + return null; + } + + return Number.isNaN(date.getTime()) ? null : date; +} + +const formatters = new Map(); + +function localDay(date: Date, timezone: string): { year: number; month: number; day: number } { + let formatter = formatters.get(timezone); + + if (!formatter) { + formatter = new Intl.DateTimeFormat('en-CA', { + timeZone: timezone, + year: 'numeric', + month: '2-digit', + day: '2-digit', + }); + formatters.set(timezone, formatter); + } + + const parts = Object.fromEntries(formatter.formatToParts(date).map(p => [p.type, p.value])); + + return { year: Number(parts.year), month: Number(parts.month), day: Number(parts.day) }; +} + +const two = (n: number) => String(n).padStart(2, '0'); + +const DATE_ONLY = /^(\d{4})-(\d{2})-(\d{2})$/; + +/** A date without time is a calendar day, which no timezone moves; an instant is read in one. */ +function calendarDay( + value: unknown, + timezone: string, +): { year: number; month: number; day: number } | null { + const dateOnly = typeof value === 'string' ? DATE_ONLY.exec(value) : null; + + if (dateOnly) { + return { year: Number(dateOnly[1]), month: Number(dateOnly[2]), day: Number(dateOnly[3]) }; + } + + const date = toDate(value); + + return date ? localDay(date, timezone) : null; +} + +function dateBucket(value: unknown, operation: DateOperation, timezone: string): string | null { + const calendar = calendarDay(value, timezone); + + if (!calendar) return null; + + const { year, month, day } = calendar; + + switch (operation) { + case 'Year': + return `${year}-01-01`; + case 'Quarter': + return `${year}-${two( + Math.floor((month - 1) / MONTHS_PER_QUARTER) * MONTHS_PER_QUARTER + 1, + )}-01`; + case 'Month': + return `${year}-${two(month)}-01`; + + case 'Week': { + const monday = new Date(Date.UTC(year, month - 1, day)); + monday.setUTCDate( + monday.getUTCDate() - ((monday.getUTCDay() + DAYS_PER_WEEK - 1) % DAYS_PER_WEEK), + ); + + return monday.toISOString().slice(0, 10); + } + + default: + return `${year}-${two(month)}-${two(day)}`; + } +} + +/** + * DynamoDB cannot aggregate, so records are reduced here as they are read, group by group, + * without ever holding the whole table. + */ +export default class StreamingAggregator { + private readonly accumulators = new Map(); + + constructor(private readonly aggregation: Aggregation, private readonly timezone: string) {} + + add(record: RecordData): void { + const group: RecordData = {}; + + for (const { field, operation } of this.aggregation.groups ?? []) { + const value = record[field] ?? null; + group[field] = operation ? dateBucket(value, operation, this.timezone) : value; + } + + const key = JSON.stringify(Object.values(group)); + let accumulator = this.accumulators.get(key); + + if (!accumulator) { + accumulator = this.emptyAccumulator(group); + this.accumulators.set(key, accumulator); + } + + accumulator.records += 1; + + if (!this.aggregation.field) return; + + const value = record[this.aggregation.field]; + + if (value === undefined || value === null) return; + + accumulator.values += 1; + + if (accumulator.min === undefined || compareValues(value, accumulator.min) < 0) { + accumulator.min = value; + } + + if (accumulator.max === undefined || compareValues(value, accumulator.max) > 0) { + accumulator.max = value; + } + + if (typeof value === 'number' && !Number.isNaN(value)) { + accumulator.numbers += 1; + accumulator.sum += value; + } + } + + result(limit?: number): AggregateResult[] { + const accumulators = [...this.accumulators.values()]; + + if (accumulators.length === 0 && !this.aggregation.groups?.length) { + accumulators.push(this.emptyAccumulator({})); + } + + const rows = accumulators.map(accumulator => ({ + group: accumulator.group, + value: this.valueOf(accumulator), + })); + + rows.sort((a, b) => { + if (a.value === b.value) return 0; + if (a.value === null) return 1; + if (b.value === null) return -1; + + // Highest first, dates by the instant they stand for. + return -compareValues(a.value, b.value); + }); + + return limit && rows.length > limit ? rows.slice(0, limit) : rows; + } + + private emptyAccumulator(group: RecordData): Accumulator { + return { + group, + records: 0, + values: 0, + numbers: 0, + sum: 0, + min: undefined, + max: undefined, + }; + } + + private valueOf(accumulator: Accumulator): unknown { + switch (this.aggregation.operation) { + case 'Count': + return this.aggregation.field ? accumulator.values : accumulator.records; + case 'Sum': + return accumulator.sum; + case 'Avg': + return accumulator.numbers ? accumulator.sum / accumulator.numbers : null; + case 'Min': + return accumulator.min ?? null; + default: + return accumulator.max ?? null; + } + } +} diff --git a/packages/datasource-dynamodb/src/utils/compare.ts b/packages/datasource-dynamodb/src/utils/compare.ts new file mode 100644 index 0000000000..b795ddf9c4 --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/compare.ts @@ -0,0 +1,19 @@ +const ISO_DATE_TIME = /^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:?\d{2})?)?$/; + +/** Orders two values, date-times by the instant they stand for rather than by their text. */ +export default function compareValues(left: unknown, right: unknown): number { + if ( + typeof left === 'string' && + typeof right === 'string' && + ISO_DATE_TIME.test(left) && + ISO_DATE_TIME.test(right) + ) { + const difference = Date.parse(left) - Date.parse(right); + + if (!Number.isNaN(difference)) return Math.sign(difference); + } + + if (left === right) return 0; + + return (left as number) < (right as number) ? -1 : 1; +} diff --git a/packages/datasource-dynamodb/src/utils/concurrency.ts b/packages/datasource-dynamodb/src/utils/concurrency.ts new file mode 100644 index 0000000000..e049912c58 --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/concurrency.ts @@ -0,0 +1,63 @@ +/** Maps with at most `limit` calls in flight, keeping the order of the results. */ +export async function mapWithConcurrency( + items: T[], + limit: number, + handler: (item: T, index: number) => Promise, +): Promise { + const results = new Array(items.length); + let next = 0; + + const worker = async () => { + while (next < items.length) { + const index = next; + next += 1; + // eslint-disable-next-line no-await-in-loop + results[index] = await handler(items[index], index); + } + }; + + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)); + + return results; +} + +/** + * Like mapWithConcurrency, for writes: once one fails nothing new is started, and the call only + * rejects after the writes already in flight have finished, so what follows sees their effect. + */ +export async function mapUntilSettled( + items: T[], + limit: number, + handler: (item: T, index: number) => Promise, +): Promise { + let failure: { error: unknown } | undefined; + + const results = await mapWithConcurrency(items, limit, async (item, index) => { + if (failure) return undefined; + + try { + return await handler(item, index); + } catch (error) { + failure ??= { error }; + + return undefined; + } + }); + + if (failure) throw failure.error; + + return results; +} + +export function chunk(items: T[], size: number): T[][] { + const chunks: T[][] = []; + + for (let i = 0; i < items.length; i += size) chunks.push(items.slice(i, i + size)); + + return chunks; +} + +export const sleep = (ms: number): Promise => + new Promise(resolve => { + setTimeout(resolve, ms); + }); diff --git a/packages/datasource-dynamodb/src/utils/executor.ts b/packages/datasource-dynamodb/src/utils/executor.ts new file mode 100644 index 0000000000..1a91b060e0 --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/executor.ts @@ -0,0 +1,278 @@ +/* eslint-disable max-classes-per-file */ +import type { CollectionSettings, TableModel } from '../types'; +import type { StartKey } from './pagination-cache'; +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import type { Logger } from '@forestadmin/datasource-toolkit'; + +import { DescribeTableCommand } from '@aws-sdk/client-dynamodb'; +import { + BatchGetCommand, + BatchWriteCommand, + GetCommand, + PutCommand, + QueryCommand, + ScanCommand, + UpdateCommand, +} from '@aws-sdk/lib-dynamodb'; + +import { DynamoDBOperationError, ScanLimitExceededError } from '../errors'; +import { chunk, mapUntilSettled, sleep } from './concurrency'; + +export type Item = Record; + +export type ReadOperation = 'query' | 'scan'; + +export type ReadInput = { + IndexName?: string; + KeyConditionExpression?: string; + FilterExpression?: string; + ProjectionExpression?: string; + ExpressionAttributeNames?: Record; + ExpressionAttributeValues?: Record; + ScanIndexForward?: boolean; + Limit?: number; + Select?: 'COUNT'; + Segment?: number; + TotalSegments?: number; + ExclusiveStartKey?: StartKey; +}; + +export type Page = { items: Item[]; count: number; lastKey?: StartKey }; + +const BATCH_GET_SIZE = 100; +const BATCH_WRITE_SIZE = 25; +const WRITE_CONCURRENCY = 10; + +/** Counts the items a request reads, across the segments of a parallel scan, and caps them. */ +export class ReadBudget { + private scanned = 0; + private aborted = false; + + constructor(private readonly tableName: string, private readonly max: number) {} + + /** Stops the other segments of a parallel read once one of them has failed. */ + abort(): void { + this.aborted = true; + } + + get isAborted(): boolean { + return this.aborted; + } + + charge(items: number, operation: ReadOperation): void { + this.scanned += items; + + if (this.scanned > this.max) { + throw new ScanLimitExceededError( + `${operation === 'scan' ? 'Scan' : 'Query'} on table '${this.tableName}' stopped after ` + + `${this.scanned} items: maxScannedItems is ${this.max}. Narrow the results with filters.`, + ); + } + } +} + +/** Sends the requests of one collection: paging, batching, retries, and what gets logged. */ +export default class DynamoExecutor { + constructor( + private readonly client: DynamoDBDocumentClient, + private readonly model: TableModel, + private readonly settings: CollectionSettings, + private readonly logger: Logger, + ) {} + + get maxScannedItems(): number { + return this.model.maxScannedItems ?? this.settings.maxScannedItems; + } + + newBudget(): ReadBudget { + return new ReadBudget(this.model.tableName, this.maxScannedItems); + } + + async *readPages( + operation: ReadOperation, + input: ReadInput, + budget: ReadBudget = this.newBudget(), + ): AsyncGenerator { + let startKey = input.ExclusiveStartKey; + + this.logQuery(operation, input); + + do { + if (budget.isAborted) return; + + const request = { + TableName: this.model.tableName, + ...input, + ...this.capacityRequest(), + ...(startKey ? { ExclusiveStartKey: startKey } : {}), + }; + // eslint-disable-next-line no-await-in-loop + const response = await this.client.send( + operation === 'query' ? new QueryCommand(request) : new ScanCommand(request), + ); + + this.logCapacity(operation, response.ConsumedCapacity?.CapacityUnits); + budget.charge( + response.ScannedCount ?? response.Count ?? response.Items?.length ?? 0, + operation, + ); + startKey = response.LastEvaluatedKey; + + yield { + items: (response.Items ?? []) as Item[], + count: response.Count ?? response.Items?.length ?? 0, + lastKey: startKey, + }; + } while (startKey); + } + + /** Reads items by key, 100 per request, asking again for the ones DynamoDB did not process. */ + async batchGet( + keys: Item[], + projection: Pick = {}, + ): Promise { + const items: Item[] = []; + const distinct = [...new Map(keys.map(key => [JSON.stringify(key), key])).values()]; + + for (const keysChunk of chunk(distinct, BATCH_GET_SIZE)) { + let pending: Item[] = keysChunk; + + for (let attempt = 0; pending.length > 0; attempt += 1) { + if (attempt > 0) { + // eslint-disable-next-line no-await-in-loop + await this.backoff(attempt, pending.length); + } + + // eslint-disable-next-line no-await-in-loop + const response = await this.client.send( + new BatchGetCommand({ + RequestItems: { [this.model.tableName]: { Keys: pending, ...projection } }, + ...this.capacityRequest(), + }), + ); + + this.logBatchCapacity('batchGet', response.ConsumedCapacity); + items.push(...((response.Responses?.[this.model.tableName] ?? []) as Item[])); + pending = (response.UnprocessedKeys?.[this.model.tableName]?.Keys ?? []) as Item[]; + } + } + + return items; + } + + async batchDelete(keys: Item[]): Promise { + const distinct = [...new Map(keys.map(key => [JSON.stringify(key), key])).values()]; + + await mapUntilSettled(chunk(distinct, BATCH_WRITE_SIZE), WRITE_CONCURRENCY, async keysChunk => { + let pending = keysChunk.map(Key => ({ DeleteRequest: { Key } })); + + for (let attempt = 0; pending.length > 0; attempt += 1) { + if (attempt > 0) { + // eslint-disable-next-line no-await-in-loop + await this.backoff(attempt, pending.length); + } + + // eslint-disable-next-line no-await-in-loop + const response = await this.client.send( + new BatchWriteCommand({ + RequestItems: { [this.model.tableName]: pending }, + ...this.capacityRequest(), + }), + ); + + this.logBatchCapacity('batchDelete', response.ConsumedCapacity); + pending = (response.UnprocessedItems?.[this.model.tableName] ?? []) as typeof pending; + } + }); + } + + async mapWrites(items: T[], handler: (item: T) => Promise): Promise { + return mapUntilSettled(items, WRITE_CONCURRENCY, handler); + } + + async put(input: Item): Promise { + const response = await this.client.send( + new PutCommand({ + TableName: this.model.tableName, + ...input, + ...this.capacityRequest(), + } as never), + ); + this.logCapacity('put', response.ConsumedCapacity?.CapacityUnits); + } + + async update(input: Item): Promise { + const response = await this.client.send( + new UpdateCommand({ + TableName: this.model.tableName, + ...input, + ...this.capacityRequest(), + } as never), + ); + this.logCapacity('update', response.ConsumedCapacity?.CapacityUnits); + + return response.Attributes; + } + + /** Strongly consistent, for the reads a write is built from: a stale copy would undo writes. */ + async get(key: Item): Promise { + const response = await this.client.send( + new GetCommand({ + TableName: this.model.tableName, + Key: key, + ConsistentRead: true, + ...this.capacityRequest(), + }), + ); + this.logCapacity('get', response.ConsumedCapacity?.CapacityUnits); + + return response.Item; + } + + /** The item count DynamoDB keeps for the table, refreshed about every six hours. */ + async describeItemCount(): Promise { + const { Table: table } = await this.client.send( + new DescribeTableCommand({ TableName: this.model.tableName }), + ); + + return table.ItemCount ?? 0; + } + + private async backoff(attempt: number, unprocessed: number): Promise { + if (attempt > this.settings.maxWriteRetries) { + throw new DynamoDBOperationError( + `${unprocessed} item(s) are still unprocessed after ${this.settings.maxWriteRetries} retries`, + undefined, + ); + } + + await sleep(this.settings.writeRetryBaseDelayMs * 2 ** (attempt - 1)); + } + + private capacityRequest(): { ReturnConsumedCapacity?: 'TOTAL' } { + return this.settings.logConsumedCapacity ? { ReturnConsumedCapacity: 'TOTAL' } : {}; + } + + private logBatchCapacity(operation: string, consumed?: Array<{ CapacityUnits?: number }>): void { + const units = consumed?.reduce((sum, entry) => sum + (entry.CapacityUnits ?? 0), 0); + + if (units !== undefined) this.logCapacity(operation, units); + } + + private logCapacity(operation: string, units?: number): void { + if (this.settings.logConsumedCapacity && units !== undefined) { + this.logger('Info', `[DynamoDB Capacity] ${this.model.name}.${operation}: ${units} units`); + } + } + + private logQuery(operation: ReadOperation, input: ReadInput): void { + if (!this.settings.logQueries) return; + + this.logger( + 'Info', + `[DynamoDB Query] ${this.model.name}.${operation}: ` + + `KeyCondition=${input.KeyConditionExpression ?? '-'} | ` + + `Filter=${input.FilterExpression ?? '-'} | Index=${input.IndexName ?? 'table'}`, + ); + } +} diff --git a/packages/datasource-dynamodb/src/utils/expression-builder.ts b/packages/datasource-dynamodb/src/utils/expression-builder.ts new file mode 100644 index 0000000000..8277bd2e26 --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/expression-builder.ts @@ -0,0 +1,248 @@ +import type { ConditionTree, ConditionTreeBranch, Operator } from '@forestadmin/datasource-toolkit'; + +import { + ConditionTreeFactory, + ConditionTreeLeaf, + ValidationError, +} from '@forestadmin/datasource-toolkit'; + +import Serializer from './serializer'; + +/** DynamoDB accepts at most 100 operands in an IN. */ +const MAX_IN_OPERANDS = 100; +const NULL_TYPE = 'NULL'; + +const COMPARISONS: Partial> = { + NotEqual: '<>', + LessThan: '<', + LessThanOrEqual: '<=', + GreaterThan: '>', + GreaterThanOrEqual: '>=', +}; + +/** What DynamoDB evaluates itself; any other operator is applied on the items it returns. */ +export const PUSHABLE_OPERATORS = new Set([ + 'Equal', + 'In', + 'NotIn', + 'Missing', + 'Blank', + 'Present', + 'StartsWith', + 'Contains', + 'NotContains', + 'IncludesAll', + 'IncludesNone', + ...(Object.keys(COMPARISONS) as Operator[]), +]); + +type LeafFilter = (leaf: ConditionTreeLeaf) => boolean; + +function isPushable(leaf: ConditionTreeLeaf, accept: LeafFilter): boolean { + return PUSHABLE_OPERATORS.has(leaf.operator) && !leaf.field.includes(':') && accept(leaf); +} + +function intersect(trees: ConditionTree[]): ConditionTree | undefined { + return trees.length ? ConditionTreeFactory.intersect(...trees) : undefined; +} + +/** + * Splits a condition tree between what DynamoDB can evaluate and what must be applied on the + * returned items. An OR can only be pushed whole: half of it evaluated server-side would drop + * items the other half still matches. + */ +export function splitCondition( + tree?: ConditionTree, + accept: LeafFilter = () => true, +): { + pushable?: ConditionTree; + residual?: ConditionTree; +} { + if (!tree) return { pushable: undefined, residual: undefined }; + + if (tree instanceof ConditionTreeLeaf) { + return isPushable(tree, accept) ? { pushable: tree } : { residual: tree }; + } + + const branch = tree as ConditionTreeBranch; + + if (branch.aggregator === 'Or') { + return branch.conditions.every( + condition => splitCondition(condition, accept).residual === undefined, + ) + ? { pushable: tree } + : { residual: tree }; + } + + const parts = branch.conditions.map(condition => splitCondition(condition, accept)); + const pushable = parts.map(part => part.pushable).filter(Boolean); + const residual = parts.map(part => part.residual).filter(Boolean); + + if (residual.length === 0) return { pushable: tree }; + if (pushable.length === 0) return { residual: tree }; + + return { pushable: intersect(pushable), residual: intersect(residual) }; +} + +/** + * Builds expressions that reference attributes and values by alias, which is both what + * DynamoDB's reserved words require and what keeps user values out of the expression text. + */ +export default class ExpressionBuilder { + readonly names: Record = {}; + readonly values: Record = {}; + private readonly nameAliases = new Map(); + + constructor(private readonly namePrefix = 'n', private readonly valuePrefix = 'v') {} + + /** Alias of an attribute path: `a@@@b` becomes `#n0.#n1`. */ + attribute(field: string): string { + if (field.includes(':')) { + throw new ValidationError(`Filtering on the relation field "${field}" is not supported`); + } + + return Serializer.toPathSegments(field) + .map(segment => this.name(segment)) + .join('.'); + } + + name(attributeName: string): string { + let alias = this.nameAliases.get(attributeName); + + if (!alias) { + alias = `#${this.namePrefix}${this.nameAliases.size}`; + this.nameAliases.set(attributeName, alias); + this.names[alias] = attributeName; + } + + return alias; + } + + value(value: unknown): string { + const alias = `:${this.valuePrefix}${Object.keys(this.values).length}`; + this.values[alias] = value instanceof Date ? value.toISOString() : value; + + return alias; + } + + /** The top-level attributes a projection needs. */ + projection(fields: string[]): string { + return [...new Set(fields.map(field => Serializer.toPathSegments(field)[0]))] + .map(attribute => this.name(attribute)) + .join(', '); + } + + result(): { + ExpressionAttributeNames?: Record; + ExpressionAttributeValues?: Record; + } { + return { + ...(Object.keys(this.names).length ? { ExpressionAttributeNames: { ...this.names } } : {}), + ...(Object.keys(this.values).length ? { ExpressionAttributeValues: { ...this.values } } : {}), + }; + } + + condition(tree: ConditionTree): string { + if (tree instanceof ConditionTreeLeaf) return this.leaf(tree); + + const { aggregator, conditions } = tree as ConditionTreeBranch; + const expressions = conditions.map(condition => this.condition(condition)); + + if (expressions.length === 1) return expressions[0]; + + return `(${expressions.join(aggregator === 'And' ? ' AND ' : ' OR ')})`; + } + + private isNull(path: string): string { + return `attribute_type(${path}, ${this.value(NULL_TYPE)})`; + } + + private missing(path: string): string { + return `(attribute_not_exists(${path}) OR ${this.isNull(path)})`; + } + + private never(path: string): string { + return `(attribute_exists(${path}) AND attribute_not_exists(${path}))`; + } + + private always(path: string): string { + return `(attribute_exists(${path}) OR attribute_not_exists(${path}))`; + } + + private in(path: string, values: unknown[]): string { + const nonNull = values.filter(value => value !== null && value !== undefined); + const hasNull = nonNull.length !== values.length; + + if (nonNull.length === 0) return hasNull ? this.missing(path) : this.never(path); + + const groups: string[] = []; + + for (let i = 0; i < nonNull.length; i += MAX_IN_OPERANDS) { + const operands = nonNull.slice(i, i + MAX_IN_OPERANDS).map(value => this.value(value)); + groups.push(`${path} IN (${operands.join(', ')})`); + } + + const matches = groups.length === 1 ? groups[0] : `(${groups.join(' OR ')})`; + + return hasNull ? `(${matches} OR ${this.missing(path)})` : matches; + } + + private blank(path: string): string { + return `(attribute_not_exists(${path}) OR ${this.isNull(path)} OR ${path} = ${this.value('')})`; + } + + private leaf(leaf: ConditionTreeLeaf): string { + const path = this.attribute(leaf.field); + const { operator, value } = leaf; + const values = Array.isArray(value) ? value : [value]; + + switch (operator) { + case 'Equal': + return value === null || value === undefined + ? this.missing(path) + : `${path} = ${this.value(value)}`; + case 'Missing': + return this.missing(path); + case 'Blank': + return this.blank(path); + case 'Present': + return `NOT ${this.blank(path)}`; + case 'In': + return values.length === 0 ? this.never(path) : this.in(path, values); + + case 'NotIn': { + if (values.length === 0) return this.always(path); + const matches = this.in(path, values); + + return matches.startsWith('(') ? `NOT ${matches}` : `NOT (${matches})`; + } + + case 'StartsWith': + return `begins_with(${path}, ${this.value(value)})`; + case 'Contains': + return `contains(${path}, ${this.value(value)})`; + case 'NotContains': + return `NOT contains(${path}, ${this.value(value)})`; + case 'IncludesAll': + if (values.length === 0) return this.always(path); + + return `(${values.map(item => `contains(${path}, ${this.value(item)})`).join(' AND ')})`; + case 'IncludesNone': + if (values.length === 0) return this.always(path); + + return `(${values + .map(item => `NOT contains(${path}, ${this.value(item)})`) + .join(' AND ')})`; + + default: { + const comparison = COMPARISONS[operator]; + + if (!comparison) { + throw new ValidationError(`Operator "${operator}" cannot be evaluated by DynamoDB`); + } + + return `${path} ${comparison} ${this.value(value)}`; + } + } + } +} diff --git a/packages/datasource-dynamodb/src/utils/pagination-cache.ts b/packages/datasource-dynamodb/src/utils/pagination-cache.ts new file mode 100644 index 0000000000..1ce66df51b --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/pagination-cache.ts @@ -0,0 +1,97 @@ +import { createHash } from 'crypto'; + +export type StartKey = Record; + +type Entry = { offset: number; startKey: StartKey; createdAt: number }; + +export type PaginationCacheOptions = { + /** Positions remembered per query. Default: 100. */ + maxEntriesPerQuery?: number; + /** Queries remembered. Default: 1000. */ + maxQueries?: number; + /** How long a position stays usable, in milliseconds. Default: 5 minutes. */ + ttlMs?: number; + /** Records between two positions remembered while reading. Default: 1000. */ + cacheInterval?: number; + now?: () => number; +}; + +const DEFAULTS = { + maxEntriesPerQuery: 100, + maxQueries: 1000, + ttlMs: 5 * 60 * 1000, + cacheInterval: 1000, +}; + +function stableStringify(value: unknown): string { + return JSON.stringify(value, (_key, nested) => { + if (nested && typeof nested === 'object' && !Array.isArray(nested)) { + return Object.fromEntries(Object.entries(nested).sort(([a], [b]) => (a < b ? -1 : 1))); + } + + return nested; + }); +} + +/** + * DynamoDB has no OFFSET: a page is reached by resuming from the key where an earlier read of + * the same query stopped. This remembers those keys, by the number of matching records before them. + */ +export default class PaginationCache { + private readonly queries = new Map(); + private readonly options: Required; + + constructor(options: PaginationCacheOptions = {}) { + this.options = { now: () => Date.now(), ...DEFAULTS, ...options }; + } + + hash(query: unknown): string { + return createHash('sha1').update(stableStringify(query)).digest('hex'); + } + + getCacheInterval(): number { + return this.options.cacheInterval; + } + + storeToken(queryHash: string, offset: number, startKey: StartKey, table?: string): void { + const query = this.queries.get(queryHash) ?? { table, entries: [] }; + this.queries.delete(queryHash); + this.queries.set(queryHash, query); + + query.entries = query.entries.filter(entry => entry.offset !== offset); + query.entries.push({ offset, startKey, createdAt: this.options.now() }); + + if (query.entries.length > this.options.maxEntriesPerQuery) query.entries.shift(); + + if (this.queries.size > this.options.maxQueries) { + this.queries.delete(this.queries.keys().next().value); + } + } + + findBestToken( + queryHash: string, + offset: number, + ): { offset: number; startKey: StartKey } | undefined { + const query = this.queries.get(queryHash); + + if (!query) return undefined; + + const now = this.options.now(); + query.entries = query.entries.filter(entry => now - entry.createdAt <= this.options.ttlMs); + + const best = query.entries + .filter(entry => entry.offset <= offset) + .reduce( + (closest, entry) => (!closest || entry.offset > closest.offset ? entry : closest), + undefined, + ); + + return best && { offset: best.offset, startKey: best.startKey }; + } + + invalidateTable(table: string): void { + for (const [hash, query] of this.queries) { + if (query.table === table) this.queries.delete(hash); + } + } +} diff --git a/packages/datasource-dynamodb/src/utils/partiql.ts b/packages/datasource-dynamodb/src/utils/partiql.ts new file mode 100644 index 0000000000..22204fee9d --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/partiql.ts @@ -0,0 +1,55 @@ +import { ValidationError } from '@forestadmin/datasource-toolkit'; + +const PLACEHOLDER_START = /[A-Za-z0-9_]/; + +/** + * Replaces `$name` placeholders by `?` and lists their values in order. What sits inside a string + * literal ('…') or a quoted identifier ("…") is text, so it is left alone. + */ +export default function parameterize( + query: string, + variables: Record, +): { statement: string; parameters: unknown[] } { + const parameters: unknown[] = []; + let statement = ''; + let quote: string | undefined; + + for (let i = 0; i < query.length; i += 1) { + const char = query[i]; + + if (quote) { + statement += char; + + if (char === quote) { + // A doubled quote is an escaped one, not the end of the literal. + if (query[i + 1] === quote) { + i += 1; + statement += quote; + } else { + quote = undefined; + } + } + } else if (char === "'" || char === '"') { + quote = char; + statement += char; + } else if (char === '$' && PLACEHOLDER_START.test(query[i + 1] ?? '')) { + let end = i + 1; + + while (end < query.length && PLACEHOLDER_START.test(query[end])) end += 1; + + const name = query.slice(i + 1, end); + + if (!(name in variables)) { + throw new ValidationError(`No value was given for the placeholder $${name}`); + } + + parameters.push(variables[name]); + statement += '?'; + i = end - 1; + } else { + statement += char; + } + } + + return { statement, parameters }; +} diff --git a/packages/datasource-dynamodb/src/utils/serializer.ts b/packages/datasource-dynamodb/src/utils/serializer.ts new file mode 100644 index 0000000000..43f1fd30f1 --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/serializer.ts @@ -0,0 +1,148 @@ +import type { RecordData } from '@forestadmin/datasource-toolkit'; + +import { ValidationError } from '@forestadmin/datasource-toolkit'; + +import TypeConverter from './type-converter'; + +export const PATH_SEPARATOR = '@@@'; + +/** What a set emptied in Forest turns into: DynamoDB holds no empty set, so the attribute goes. */ +export const CLEARED = Symbol('cleared'); + +const nestedPrefixesCache = new WeakMap, Set>(); + +/** Paths that have columns declared under them, hence are expanded rather than kept as Json. */ +function nestedPrefixes(fields: Set): Set { + let prefixes = nestedPrefixesCache.get(fields); + + if (!prefixes) { + prefixes = new Set(); + + for (const field of fields) { + const parts = field.split(PATH_SEPARATOR); + + for (let i = 1; i < parts.length; i += 1) { + prefixes.add(parts.slice(0, i).join(PATH_SEPARATOR)); + } + } + + nestedPrefixesCache.set(fields, prefixes); + } + + return prefixes; +} + +function isPlainObject(value: unknown): value is Record { + return ( + typeof value === 'object' && + value !== null && + !Array.isArray(value) && + !(value instanceof Date) && + !(value instanceof Set) && + !(value instanceof Uint8Array) && + !TypeConverter.isGeoPoint(value) + ); +} + +/** Writing to `__proto__` would change what every object of the process inherits. */ +function assertWritable(segments: string[]): void { + if (segments.includes('__proto__')) { + throw new ValidationError("An attribute path cannot contain '__proto__'"); + } +} + +/** Own properties only: what an object inherits is not a map to write into. */ +function hasOwnMap(target: Record, key: string): boolean { + return Object.prototype.hasOwnProperty.call(target, key) && isPlainObject(target[key]); +} + +export default class Serializer { + /** Makes a value out of what the DocumentClient reads: no set, date object, buffer or bigint. */ + static normalize(value: unknown): unknown { + if (typeof value === 'bigint') return value.toString(); + if (value instanceof Set) return [...value].map(item => Serializer.normalize(item)); + if (value instanceof Date) return value.toISOString(); + if (value instanceof Uint8Array) return Buffer.from(value).toString('base64'); + if (Array.isArray(value)) return value.map(item => Serializer.normalize(item)); + + if (isPlainObject(value)) { + return Object.fromEntries( + Object.entries(value).map(([key, nested]) => [key, Serializer.normalize(nested)]), + ); + } + + return value; + } + + /** Turns an item into a record: maps that have declared columns become `a@@@b` fields. */ + static flatten(item: Record, fields: Set): RecordData { + const prefixes = nestedPrefixes(fields); + const record: RecordData = {}; + + const visit = (value: Record, prefix: string) => { + for (const [key, nested] of Object.entries(value)) { + const path = prefix ? `${prefix}${PATH_SEPARATOR}${key}` : key; + + if (isPlainObject(nested) && prefixes.has(path)) visit(nested, path); + else record[path] = Serializer.normalize(nested); + } + }; + + visit(item, ''); + + return record; + } + + /** Turns `a@@@b` fields back into nested maps, ready to be written. */ + static unflatten(record: RecordData): Record { + const item: Record = {}; + + for (const [field, value] of Object.entries(record)) { + const parts = Serializer.toPathSegments(field); + let target = item; + + assertWritable(parts); + + for (const part of parts.slice(0, -1)) { + if (!hasOwnMap(target, part)) target[part] = {}; + target = target[part] as Record; + } + + target[parts[parts.length - 1]] = value instanceof Date ? value.toISOString() : value; + } + + return item; + } + + /** Sets the value at a path, creating the maps on the way, and replaces what was there. */ + static setPath(target: Record, segments: string[], value: unknown): void { + let holder = target; + + assertWritable(segments); + + for (const segment of segments.slice(0, -1)) { + if (!hasOwnMap(holder, segment)) holder[segment] = {}; + holder = holder[segment] as Record; + } + + holder[segments[segments.length - 1]] = value instanceof Date ? value.toISOString() : value; + } + + /** Removes the value at a path, as REMOVE does, and leaves the maps on the way as they are. */ + static removePath(target: Record, segments: string[]): void { + let holder = target; + + assertWritable(segments); + + for (const segment of segments.slice(0, -1)) { + if (!hasOwnMap(holder, segment)) return; + holder = holder[segment] as Record; + } + + delete holder[segments[segments.length - 1]]; + } + + static toPathSegments(field: string): string[] { + return field.split(PATH_SEPARATOR); + } +} diff --git a/packages/datasource-dynamodb/src/utils/sort.ts b/packages/datasource-dynamodb/src/utils/sort.ts new file mode 100644 index 0000000000..0fd8582d73 --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/sort.ts @@ -0,0 +1,22 @@ +import type { RecordData, Sort } from '@forestadmin/datasource-toolkit'; + +import compareValues from './compare'; + +/** Missing values come last whatever the direction, as DynamoDB has no notion of nulls order. */ +export default function compareRecords(a: RecordData, b: RecordData, sort: Sort): number { + for (const { field, ascending } of sort) { + const [left, right] = [a[field], b[field]]; + const leftMissing = left === undefined || left === null; + const rightMissing = right === undefined || right === null; + + if (leftMissing !== rightMissing) return leftMissing ? 1 : -1; + + if (!leftMissing) { + const order = compareValues(left, right); + + if (order !== 0) return ascending ? order : -order; + } + } + + return 0; +} diff --git a/packages/datasource-dynamodb/src/utils/type-converter.ts b/packages/datasource-dynamodb/src/utils/type-converter.ts new file mode 100644 index 0000000000..fec0fd0330 --- /dev/null +++ b/packages/datasource-dynamodb/src/utils/type-converter.ts @@ -0,0 +1,105 @@ +import type { DynamoDataType } from '../types'; +import type { ColumnType, Operator, PrimitiveTypes } from '@forestadmin/datasource-toolkit'; + +import { allowedOperatorsForColumnType } from '@forestadmin/datasource-toolkit'; + +const ISO_DATE_TIME = /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:?\d{2})?$/; +const DATE_ONLY = /^\d{4}-\d{2}-\d{2}$/; +const TIME_ONLY = /^([01]\d|2[0-3]):([0-5]\d):([0-5]\d)(\.\d+)?$/; +const NON_SORTABLE_TYPES: DynamoDataType[] = ['object', 'array', 'point', 'null']; + +function isGeoPoint(value: unknown): boolean { + const point = value as { type?: unknown; coordinates?: unknown }; + + return ( + typeof value === 'object' && + point.type === 'Point' && + Array.isArray(point.coordinates) && + point.coordinates.length >= 2 && + point.coordinates.every(coordinate => typeof coordinate === 'number') + ); +} + +function isValidDate(value: string): boolean { + return !Number.isNaN(new Date(value).getTime()); +} + +export default class TypeConverter { + static getColumnTypeFromDataType(dataType: DynamoDataType): PrimitiveTypes { + switch (dataType) { + case 'boolean': + return 'Boolean'; + case 'date': + return 'Date'; + case 'dateonly': + return 'Dateonly'; + case 'timeonly': + return 'Time'; + case 'number': + return 'Number'; + case 'string': + case 'binary': + return 'String'; + case 'point': + case 'object': + case 'array': + case 'null': + return 'Json'; + default: + throw new Error(`Unsupported data type: "${dataType}"`); + } + } + + /** DynamoDB only types its keys, so the other attributes are typed from their values. */ + static inferTypeFromValue(value: unknown): DynamoDataType { + if (value === null || value === undefined) return 'null'; + if (typeof value === 'boolean') return 'boolean'; + if (typeof value === 'number') return 'number'; + + if (typeof value === 'string') { + if (TIME_ONLY.test(value)) return 'timeonly'; + if (DATE_ONLY.test(value) && isValidDate(value)) return 'dateonly'; + if (ISO_DATE_TIME.test(value) && isValidDate(value)) return 'date'; + + return 'string'; + } + + if (Array.isArray(value) || value instanceof Set) return 'array'; + if (value instanceof Date) return 'date'; + if (value instanceof Uint8Array) return 'binary'; + if (typeof value === 'object') return isGeoPoint(value) ? 'point' : 'object'; + + return 'string'; + } + + static isGeoPoint(value: unknown): boolean { + return isGeoPoint(value); + } + + static getMostSpecificType(types: DynamoDataType[]): DynamoDataType { + const unique = [...new Set(types)]; + const nonNull = unique.filter(type => type !== 'null'); + + if (unique.length === 0) return 'null'; + if (nonNull.length === 0) return 'null'; + if (nonNull.length === 1) return nonNull[0]; + + if (nonNull.every(type => type === 'date' || type === 'dateonly')) return 'date'; + + return 'object'; + } + + static isSortable(dataType: DynamoDataType): boolean { + return !NON_SORTABLE_TYPES.includes(dataType); + } + + /** + * Every operator Forest allows for the type: the ones DynamoDB cannot evaluate are applied + * on the items it returns. + */ + static operatorsForColumnType(columnType: ColumnType): Set { + return new Set( + typeof columnType === 'string' ? allowedOperatorsForColumnType[columnType] : ['Present'], + ); + } +} diff --git a/packages/datasource-dynamodb/src/virtual-collections.ts b/packages/datasource-dynamodb/src/virtual-collections.ts new file mode 100644 index 0000000000..f840003916 --- /dev/null +++ b/packages/datasource-dynamodb/src/virtual-collections.ts @@ -0,0 +1,75 @@ +import type DynamoDBCollection from './collection'; +import type DynamoDBDataSource from './datasource'; +import type { AttributeSchema, VirtualCollectionConfig } from './types'; +import type { Caller, Logger } from '@forestadmin/datasource-toolkit'; + +import { + ConditionTreeLeaf, + Page, + PaginatedFilter, + Projection, +} from '@forestadmin/datasource-toolkit'; + +import ArrayCollection from './array-collection'; +import { DEFAULT_MAX_DEPTH, DEFAULT_SAMPLE_SIZE, inferSchema } from './introspection/introspector'; + +const SAMPLING_CALLER = { timezone: 'UTC' } as Caller; + +/** The attributes of an item come from the schema when it declares them, else from a sample. */ +async function itemSchemaOf( + parent: DynamoDBCollection | ArrayCollection, + arrayField: string, + sampleSize: number, +): Promise> { + const declared = + parent instanceof ArrayCollection + ? parent.itemSchema[arrayField]?.items + : parent.model.schema[arrayField]?.items; + + if (declared) return declared; + + const parents = await parent.list( + SAMPLING_CALLER, + new PaginatedFilter({ + conditionTree: new ConditionTreeLeaf(arrayField, 'Present'), + page: new Page(0, sampleSize), + }), + new Projection(arrayField), + ); + const items = parents + .flatMap(record => (Array.isArray(record[arrayField]) ? (record[arrayField] as unknown[]) : [])) + .filter((item): item is Record => !!item && typeof item === 'object'); + + return inferSchema(items, [], DEFAULT_MAX_DEPTH); +} + +/** Adds one collection per configured array, each skipped with a warning if it cannot be built. */ +export default async function addVirtualCollections( + dataSource: DynamoDBDataSource, + configs: VirtualCollectionConfig[], + sampleSize: number = DEFAULT_SAMPLE_SIZE, + logger?: Logger, +): Promise { + for (const config of configs) { + try { + const parent = dataSource.getCollection(config.parentCollection); + + // eslint-disable-next-line no-await-in-loop + const itemSchema = await itemSchemaOf(parent, config.arrayField, sampleSize); + + dataSource.addCollection( + new ArrayCollection(dataSource, parent, { + name: config.collectionName, + arrayField: config.arrayField, + itemSchema, + }), + ); + } catch (error) { + logger?.( + 'Warn', + `Skipping virtual collection '${config.collectionName}' for array ` + + `'${config.parentCollection}.${config.arrayField}': ${error.message}`, + ); + } + } +} diff --git a/packages/datasource-dynamodb/test/array-collection.test.ts b/packages/datasource-dynamodb/test/array-collection.test.ts new file mode 100644 index 0000000000..eea7ffc598 --- /dev/null +++ b/packages/datasource-dynamodb/test/array-collection.test.ts @@ -0,0 +1,478 @@ +import type { DataSource } from '@forestadmin/datasource-toolkit'; + +import { + Aggregation, + Filter, + Page, + PaginatedFilter, + Projection, + Sort, +} from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { and, buildModel, leaf, plain } from './helpers'; +import ArrayCollection from '../src/array-collection'; + +const caller = { timezone: 'UTC' } as never; + +const orders = [ + { + customerId: 'c1', + orderId: 'o1', + lines: [ + { sku: 'A', qty: 1 }, + { sku: 'B', qty: 5 }, + ], + }, + { customerId: 'c2', orderId: 'o2', lines: [{ sku: 'C', qty: 3 }] }, + { customerId: 'c3', orderId: 'o3' }, +]; + +function build(handlers: Record, settings = {}) { + const parent = buildCollection( + buildModel({ + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + lines: { type: 'array' }, + }, + }), + handlers, + { scanConcurrency: 1, ...settings }, + ); + const collection = new ArrayCollection({} as DataSource, parent.collection, { + name: 'order_lines', + arrayField: 'lines', + itemSchema: { sku: { type: 'string' }, qty: { type: 'number' } }, + }); + + return { collection, client: parent.client }; +} + +describe('ArrayCollection', () => { + describe('schema', () => { + it('is keyed by the parent key and the position in the array', () => { + const { collection } = build({}); + + expect(collection.name).toBe('order_lines'); + expect( + Object.entries(collection.schema.fields) + .filter(([, field]) => field.type === 'Column' && field.isPrimaryKey) + .map(([name]) => name), + ).toEqual(['customerId', 'orderId', 'index']); + expect(collection.schema.fields.index).toMatchObject({ + columnType: 'Number', + isReadOnly: true, + }); + expect(collection.schema.fields.customerId).toMatchObject({ isReadOnly: true }); + }); + + it('exposes the item attributes as columns', () => { + const { collection } = build({}); + + expect(collection.schema.fields.sku).toMatchObject({ columnType: 'String' }); + expect(collection.schema.fields.qty).toMatchObject({ columnType: 'Number' }); + }); + }); + + describe('list', () => { + it('turns each array item of each parent into a record', async () => { + const { collection } = build({ Scan: () => ({ Items: orders }) }); + + const records = await collection.list( + caller, + new PaginatedFilter({}), + new Projection('customerId', 'orderId', 'index', 'sku', 'qty'), + ); + + expect(records).toEqual([ + { customerId: 'c1', orderId: 'o1', index: 0, sku: 'A', qty: 1 }, + { customerId: 'c1', orderId: 'o1', index: 1, sku: 'B', qty: 5 }, + { customerId: 'c2', orderId: 'o2', index: 0, sku: 'C', qty: 3 }, + ]); + }); + + it('reads the parents through their key when the filter pins it', async () => { + const { collection, client } = build({ Query: () => ({ Items: [orders[0]] }) }); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + new Projection('index', 'sku'), + ); + + expect(records).toHaveLength(2); + expect(client.calls('Query')[0]).toMatchObject({ KeyConditionExpression: '#n0 = :v0' }); + expect(client.calls('Scan')).toHaveLength(0); + }); + + it('evaluates the conditions on the items themselves', async () => { + const { collection } = build({ Scan: () => ({ Items: orders }) }); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('qty', 'GreaterThan', 2) }), + new Projection('sku'), + ); + + expect(records).toEqual([{ sku: 'B' }, { sku: 'C' }]); + }); + + it('sorts and paginates the records', async () => { + const { collection } = build({ Scan: () => ({ Items: orders }) }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + sort: new Sort({ field: 'qty', ascending: false }), + page: { limit: 1, skip: 1 } as never, + }), + new Projection('sku'), + ); + + expect(records).toEqual([{ sku: 'C' }]); + }); + + it('selects the item by its position', async () => { + const { collection } = build({ BatchGet: () => ({ Responses: { Orders: [orders[0]] } }) }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: and( + plain('customerId', 'Equal', 'c1'), + plain('orderId', 'Equal', 'o1'), + plain('index', 'Equal', 1), + ), + }), + new Projection('sku'), + ); + + expect(records).toEqual([{ sku: 'B' }]); + }); + }); + + describe('create', () => { + it('appends the item to the array of its parent and returns its position', async () => { + const { collection, client } = build({ + Update: () => ({ + Attributes: { + lines: [ + { sku: 'A', qty: 1 }, + { sku: 'N', qty: 2 }, + ], + }, + }), + }); + + const created = await collection.create(caller, [ + { customerId: 'c1', orderId: 'o1', sku: 'N', qty: 2 }, + ]); + + expect(created).toEqual([{ customerId: 'c1', orderId: 'o1', index: 1, sku: 'N', qty: 2 }]); + expect(client.calls('Update')[0]).toEqual({ + TableName: 'Orders', + Key: { customerId: 'c1', orderId: 'o1' }, + UpdateExpression: 'SET #n0 = list_append(if_not_exists(#n0, :v0), :v1)', + ConditionExpression: 'attribute_exists(#k0)', + ExpressionAttributeNames: { '#n0': 'lines', '#k0': 'customerId' }, + ExpressionAttributeValues: { ':v0': [], ':v1': [{ sku: 'N', qty: 2 }] }, + ReturnValues: 'UPDATED_NEW', + }); + }); + + it('requires the whole key of the parent', async () => { + const { collection } = build({}); + + await expect(collection.create(caller, [{ customerId: 'c1', sku: 'N' }])).rejects.toThrow( + "The key attribute 'orderId' of the parent 'orders' is required to create a record in 'order_lines'", + ); + }); + }); + + describe('update', () => { + it('sets the attributes of the item at its position', async () => { + const { collection, client } = build({ + BatchGet: () => ({ Responses: { Orders: [orders[0]] } }), + Update: () => ({}), + }); + + await collection.update( + caller, + new Filter({ + conditionTree: and( + plain('customerId', 'Equal', 'c1'), + plain('orderId', 'Equal', 'o1'), + plain('index', 'Equal', 1), + ), + }), + { qty: 9 }, + ); + + expect(client.calls('Update')).toEqual([ + { + TableName: 'Orders', + Key: { customerId: 'c1', orderId: 'o1' }, + UpdateExpression: 'SET #n0[1].#n1 = :v0', + ConditionExpression: 'attribute_exists(#k0)', + ExpressionAttributeNames: { '#n0': 'lines', '#n1': 'qty', '#k0': 'customerId' }, + ExpressionAttributeValues: { ':v0': 9 }, + }, + ]); + }); + + it('groups the items of the same parent in one update', async () => { + const { collection, client } = build({ + Query: () => ({ Items: [orders[0]] }), + Update: () => ({}), + }); + + await collection.update( + caller, + new Filter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + { qty: 0 }, + ); + + expect(client.calls('Update')).toHaveLength(1); + expect(client.calls('Update')[0].UpdateExpression).toBe( + 'SET #n0[0].#n1 = :v0, #n0[1].#n1 = :v0', + ); + }); + + it('refuses to change the parent key or the position', async () => { + const { collection } = build({}); + + await expect(collection.update(caller, new Filter({}), { customerId: 'x' })).rejects.toThrow( + "The attribute 'customerId' of 'order_lines' cannot be updated", + ); + await expect(collection.update(caller, new Filter({}), { index: 3 })).rejects.toThrow( + "The attribute 'index' of 'order_lines' cannot be updated", + ); + }); + }); + + describe('delete', () => { + it('removes the matching items from their parent, highest position first', async () => { + const { collection, client } = build({ + Query: () => ({ Items: [orders[0]] }), + Update: () => ({}), + }); + + await collection.delete( + caller, + new Filter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + ); + + expect(client.calls('Update')).toEqual([ + { + TableName: 'Orders', + Key: { customerId: 'c1', orderId: 'o1' }, + UpdateExpression: 'REMOVE #n0[1], #n0[0]', + ConditionExpression: 'attribute_exists(#k0)', + ExpressionAttributeNames: { '#n0': 'lines', '#k0': 'customerId' }, + }, + ]); + }); + + it('does nothing when no item matches', async () => { + const { collection, client } = build({ Scan: () => ({ Items: [orders[2]] }) }); + + await collection.delete(caller, new Filter({})); + + expect(client.calls('Update')).toHaveLength(0); + }); + }); + + describe('aggregate', () => { + it('aggregates the items', async () => { + const { collection } = build({ Scan: () => ({ Items: orders }) }); + + const result = await collection.aggregate( + caller, + new Filter({}), + new Aggregation({ operation: 'Sum', field: 'qty' }), + ); + + expect(result).toEqual([{ value: 9, group: {} }]); + }); + + it('groups the items by an attribute of their parent', async () => { + const { collection } = build({ Scan: () => ({ Items: orders }) }); + + const result = await collection.aggregate( + caller, + new Filter({}), + new Aggregation({ operation: 'Count', groups: [{ field: 'customerId' }] }), + ); + + expect(result).toEqual([ + { value: 2, group: { customerId: 'c1' } }, + { value: 1, group: { customerId: 'c2' } }, + ]); + }); + }); +}); + +describe('ArrayCollection of an array', () => { + const nested = [ + { + customerId: 'c1', + orderId: 'o1', + lines: [ + { sku: 'A', parts: [{ n: 1 }, { n: 2 }] }, + { sku: 'B', parts: [{ n: 3 }] }, + ], + }, + ]; + + function buildNested(handlers: Record) { + const parent = buildCollection( + buildModel({ + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + lines: { type: 'array' }, + }, + }), + handlers, + { scanConcurrency: 1 }, + ); + const lines = new ArrayCollection({} as DataSource, parent.collection, { + name: 'order_lines', + arrayField: 'lines', + itemSchema: { sku: { type: 'string' }, parts: { type: 'array' } }, + }); + const parts = new ArrayCollection({} as DataSource, lines, { + name: 'parts', + arrayField: 'parts', + itemSchema: { n: { type: 'number' } }, + }); + + return { parts, client: parent.client }; + } + + it('is keyed by the table key and the position in each array', () => { + const { parts } = buildNested({}); + + expect( + Object.entries(parts.schema.fields) + .filter(([, field]) => field.type === 'Column' && field.isPrimaryKey) + .map(([name]) => name), + ).toEqual(['customerId', 'orderId', 'order_linesIndex', 'index']); + }); + + it('lists the elements of every inner array', async () => { + const { parts } = buildNested({ Scan: () => ({ Items: nested }) }); + + const records = await parts.list( + caller, + new PaginatedFilter({}), + new Projection('order_linesIndex', 'index', 'n'), + ); + + expect(records).toEqual([ + { order_linesIndex: 0, index: 0, n: 1 }, + { order_linesIndex: 0, index: 1, n: 2 }, + { order_linesIndex: 1, index: 0, n: 3 }, + ]); + }); + + it('sets an attribute of an inner element through the path of both arrays', async () => { + const { parts, client } = buildNested({ + Scan: () => ({ Items: nested }), + Update: () => ({}), + }); + + await parts.update(caller, new Filter({ conditionTree: leaf('n', 'Equal', 3) }), { n: 9 }); + + expect(client.calls('Update')[0]).toMatchObject({ + Key: { customerId: 'c1', orderId: 'o1' }, + UpdateExpression: 'SET #n0[1].#n1[0].#n2 = :v0', + ExpressionAttributeNames: { '#n0': 'lines', '#n1': 'parts', '#n2': 'n' }, + ExpressionAttributeValues: { ':v0': 9 }, + }); + }); + + it('removes inner elements, highest positions first', async () => { + const { parts, client } = buildNested({ + Scan: () => ({ Items: nested }), + Update: () => ({}), + }); + + await parts.delete(caller, new Filter({})); + + expect(client.calls('Update')[0].UpdateExpression).toBe( + 'REMOVE #n0[1].#n1[0], #n0[0].#n1[1], #n0[0].#n1[0]', + ); + }); + + it('appends to the inner array of an existing outer element', async () => { + const { parts, client } = buildNested({ + Update: () => ({ Attributes: { lines: [{ parts: [{ n: 3 }, { n: 4 }] }] } }), + }); + + const created = await parts.create(caller, [ + { customerId: 'c1', orderId: 'o1', order_linesIndex: 1, n: 4 }, + ]); + + expect(created).toEqual([ + { customerId: 'c1', orderId: 'o1', order_linesIndex: 1, index: 1, n: 4 }, + ]); + expect(client.calls('Update')[0]).toMatchObject({ + UpdateExpression: 'SET #n0[1].#n1 = list_append(if_not_exists(#n0[1].#n1, :v0), :v1)', + ConditionExpression: 'attribute_exists(#k0) AND attribute_exists(#n0[1])', + }); + }); + + it('requires the position in the outer array to create', async () => { + const { parts } = buildNested({}); + + await expect(parts.create(caller, [{ customerId: 'c1', orderId: 'o1', n: 4 }])).rejects.toThrow( + "The key attribute 'order_linesIndex' of the parent 'order_lines' is required to create a record in 'parts'", + ); + }); + + describe('read budget', () => { + it('counts the parents of every page against maxScannedItems', async () => { + const parents = Array.from({ length: 1000 }, (_, i) => ({ + customerId: `c${i}`, + orderId: `o${i}`, + lines: [{ sku: 'A', qty: 1 }], + })); + const { collection } = build( + { + Scan: (input: Record) => { + const start = input.ExclusiveStartKey + ? Number(input.ExclusiveStartKey.orderId.slice(1)) + 1 + : 0; + const items = parents.slice(start, start + (input.Limit ?? parents.length)); + + return { + Items: items, + ...(start + items.length < parents.length + ? { + LastEvaluatedKey: { + customerId: items[items.length - 1].customerId, + orderId: items[items.length - 1].orderId, + }, + } + : {}), + }; + }, + }, + { maxScannedItems: 300 }, + ); + + await expect( + collection.list( + caller, + new PaginatedFilter({ + conditionTree: leaf('sku', 'Equal', 'none'), + page: new Page(0, 15), + }), + new Projection('sku'), + ), + ).rejects.toThrow(/maxScannedItems is 300/); + }); + }); +}); diff --git a/packages/datasource-dynamodb/test/array-review-findings.test.ts b/packages/datasource-dynamodb/test/array-review-findings.test.ts new file mode 100644 index 0000000000..df3f90f203 --- /dev/null +++ b/packages/datasource-dynamodb/test/array-review-findings.test.ts @@ -0,0 +1,244 @@ +import type { DataSource } from '@forestadmin/datasource-toolkit'; + +import { Filter, PaginatedFilter, Projection } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { fakeClient } from './fake-client'; +import { buildModel, leaf } from './helpers'; +import { createDynamoDBDataSource } from '../src'; +import ArrayCollection from '../src/array-collection'; + +const caller = { timezone: 'UTC' } as never; + +function build(handlers: Record, model = buildModel()) { + const parent = buildCollection( + { + ...model, + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + lines: { type: 'array' }, + }, + }, + handlers as never, + { scanConcurrency: 1, chartCacheTtlMs: 60000 }, + ); + const collection = new ArrayCollection({} as DataSource, parent.collection, { + name: 'order_lines', + arrayField: 'lines', + itemSchema: { sku: { type: 'string' }, qty: { type: 'number' }, index: { type: 'number' } }, + }); + + return { collection, client: parent.client }; +} + +const order = (lines: unknown[]) => ({ customerId: 'c1', orderId: 'o1', lines }); + +describe('ArrayCollection review findings', () => { + it('keeps the identity of a record when the element has an attribute of the same name', async () => { + const { collection } = build({ + Scan: () => ({ Items: [order([{ sku: 'A', index: 99, customerId: 'other' }])] }), + }); + + const [record] = await collection.list( + caller, + new PaginatedFilter({}), + new Projection('customerId', 'index', 'sku'), + ); + + expect(record).toEqual({ customerId: 'c1', index: 0, sku: 'A' }); + }); + + it('splits the removal of many elements into expressions DynamoDB accepts', async () => { + const lines = Array.from({ length: 1000 }, (_, i) => ({ sku: `S${i}` })); + const { collection, client } = build({ + Scan: () => ({ Items: [order(lines)] }), + Update: () => ({}), + }); + + await collection.delete(caller, new Filter({})); + + const expressions = client.calls('Update').map(call => call.UpdateExpression as string); + const removed = expressions.flatMap(e => [...e.matchAll(/\[(\d+)\]/g)].map(m => Number(m[1]))); + + expect(expressions.length).toBeGreaterThan(1); + expect(Math.max(...expressions.map(e => e.length))).toBeLessThan(4096); + expect(removed).toEqual(Array.from({ length: 1000 }, (_, i) => 999 - i)); + }); + + it('splits the update of many elements too', async () => { + const lines = Array.from({ length: 1000 }, (_, i) => ({ sku: `S${i}` })); + const { collection, client } = build({ + Scan: () => ({ Items: [order(lines)] }), + Update: () => ({}), + }); + + await collection.update(caller, new Filter({}), { qty: 1 }); + + const lengths = client.calls('Update').map(call => (call.UpdateExpression as string).length); + + expect(lengths.length).toBeGreaterThan(1); + expect(Math.max(...lengths)).toBeLessThan(4096); + }); + + it('sets an attribute under a map the element does not have yet, by rewriting the item', async () => { + const { collection, client } = build({ + Scan: () => ({ Items: [order([{ sku: 'A' }])] }), + Update: () => { + throw Object.assign( + new Error('The document path provided in the update expression is invalid for update'), + { name: 'ValidationException' }, + ); + }, + Get: () => ({ Item: order([{ sku: 'A' }]) }), + Put: () => ({}), + }); + + await collection.update(caller, new Filter({}), { 'details@@@note': 'hi' }); + + expect(client.calls('Put')[0].Item).toEqual(order([{ sku: 'A', details: { note: 'hi' } }])); + }); + + it('only touches items of the entity the parent stands for', async () => { + const { collection, client } = build( + { + Scan: () => ({ Items: [order([{ sku: 'A' }])] }), + Update: () => ({}), + }, + buildModel({ entity: { attribute: 'entityType', value: 'ORDER' } }), + ); + + await collection.update(caller, new Filter({}), { qty: 2 }); + await collection + .create(caller, [{ customerId: 'c1', orderId: 'o1', sku: 'B' }]) + .catch(() => {}); + + for (const call of client.calls('Update')) { + expect(call.ConditionExpression).toMatch(/^attribute_exists\(#k0\) AND .*#n\d = :v\d$/); + expect(Object.values(call.ExpressionAttributeValues)).toContain('ORDER'); + } + }); + + it('forgets the cached charts when an append fails halfway', async () => { + const { collection, client } = build({ + Update: [ + { Attributes: { lines: [{ sku: 'A' }] } }, + Object.assign(new Error('throttled'), { name: 'ThrottlingException' }), + ], + Scan: () => ({ Items: [order([{ sku: 'A' }])] }), + }); + const count = () => + collection.aggregate(caller, new Filter({}), { operation: 'Count' } as never); + + await count().catch(() => undefined); + await expect( + collection.create(caller, [ + { customerId: 'c1', orderId: 'o1', sku: 'A' }, + { customerId: 'c1', orderId: 'o1', sku: 'B' }, + ]), + ).rejects.toThrow('throttled'); + + expect(client.calls('Update')).toHaveLength(2); + }); +}); + +describe('native queries', () => { + const orders = { + Table: { + TableName: 'Orders', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }, + }; + + it('does not take a placeholder inside a string literal for a parameter', async () => { + const client = fakeClient({ + DescribeTable: () => orders, + Scan: () => ({ Items: [] }), + ExecuteStatement: () => ({ Items: [] }), + }); + const dataSource = await createDynamoDBDataSource({ + client, + tables: ['Orders'], + liveQueryConnection: 'dynamo', + })(jest.fn(), jest.fn()); + + await dataSource.executeNativeQuery( + 'dynamo', + `SELECT * FROM "Orders" WHERE name = '$name' AND id = $id`, + { id: 'a' }, + ); + + expect(client.calls('ExecuteStatement')[0]).toEqual({ + Statement: `SELECT * FROM "Orders" WHERE name = '$name' AND id = ?`, + Parameters: ['a'], + }); + }); + + it('stops following a statement that never ends', async () => { + const client = fakeClient({ + DescribeTable: () => orders, + Scan: () => ({ Items: [] }), + ExecuteStatement: () => ({ Items: [], NextToken: 'again' }), + }); + const dataSource = await createDynamoDBDataSource({ + client, + tables: ['Orders'], + liveQueryConnection: 'dynamo', + })(jest.fn(), jest.fn()); + + await expect( + dataSource.executeNativeQuery('dynamo', 'SELECT * FROM "Orders" WHERE a = 1', {}), + ).rejects.toThrow('still has results after 1000 pages'); + }); +}); + +describe('listing the elements of an array page by page', () => { + const parents = (count: number, sku: (i: number) => string) => + Array.from({ length: count }, (_, i) => ({ + customerId: 'c1', + orderId: `o${String(i).padStart(4, '0')}`, + lines: [{ sku: sku(i) }], + })); + + it('reads only as many table items as the page needs', async () => { + const { collection, client } = build({ + Scan: () => ({ Items: parents(250, () => 'A') }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ page: { skip: 0, limit: 3 } as never }), + new Projection('sku'), + ); + + expect(records).toHaveLength(3); + expect(client.calls('Scan')).toHaveLength(1); + expect(client.calls('Scan')[0].Limit).toBe(100); + }); + + it('reads the next table items when the first ones do not hold enough matches', async () => { + const all = parents(200, i => (i >= 150 ? 'MATCH' : 'A')); + const { collection, client } = build({ + Scan: (input: any) => { + const start = input.ExclusiveStartKey + ? all.findIndex(item => item.orderId === input.ExclusiveStartKey.orderId) + 1 + : 0; + + return { Items: all.slice(start, start + 100) }; + }, + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: leaf('sku', 'Equal', 'MATCH'), + page: { skip: 0, limit: 5 } as never, + }), + new Projection('sku', 'orderId'), + ); + + expect(records.map(r => r.orderId)).toEqual(['o0150', 'o0151', 'o0152', 'o0153', 'o0154']); + expect(client.calls('Scan')).toHaveLength(2); + }); +}); diff --git a/packages/datasource-dynamodb/test/capabilities.test.ts b/packages/datasource-dynamodb/test/capabilities.test.ts new file mode 100644 index 0000000000..3910e64fe9 --- /dev/null +++ b/packages/datasource-dynamodb/test/capabilities.test.ts @@ -0,0 +1,203 @@ +import type { DataSource } from '@forestadmin/datasource-toolkit'; + +import { PaginatedFilter, Projection, Sort } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { buildModel, leaf } from './helpers'; +import ArrayCollection from '../src/array-collection'; +import { resolveSettings } from '../src/settings'; + +const caller = { timezone: 'UTC' } as never; + +const model = () => + buildModel({ + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + status: { type: 'string', nullable: true }, + total: { type: 'number' }, + createdAt: { type: 'date' }, + lines: { type: 'array' }, + }, + indexes: [ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'S' }, + sortKey: { name: 'createdAt', type: 'S' }, + projectsAllAttributes: true, + }, + ], + }); + +const fieldsOf = (settings = {}) => + buildCollection(model(), {}, settings).collection.schema.fields as Record< + string, + { filterOperators: Set; isSortable: boolean } + >; + +describe('what Forest may ask of a table collection', () => { + const IN_MEMORY_ONLY = ['IContains', 'NotIContains', 'EndsWith', 'Like', 'ILike', 'Match']; + + it('keeps the filters DynamoDB evaluates itself on every attribute', () => { + const { status, total, orderId } = fieldsOf(); + + expect([...status.filterOperators]).toEqual( + expect.arrayContaining([ + 'Equal', + 'NotEqual', + 'In', + 'StartsWith', + 'Contains', + 'Blank', + 'Present', + ]), + ); + expect([...total.filterOperators]).toEqual( + expect.arrayContaining(['Equal', 'LessThan', 'GreaterThanOrEqual', 'In']), + ); + expect([...orderId.filterOperators]).toEqual(expect.arrayContaining(['Equal', 'In'])); + }); + + it.each(IN_MEMORY_ONLY)( + 'does not offer %s, which would read every item to filter it', + operator => { + const { status } = fieldsOf(); + + expect(status.filterOperators.has(operator)).toBe(false); + }, + ); + + it('does not offer the relative date operators', () => { + const { createdAt } = fieldsOf(); + + expect( + [...createdAt.filterOperators].filter(op => + /Ago|Today|Yesterday|Previous|Future|Past|Before|After/.test(op), + ), + ).toEqual([]); + }); + + it('sorts on the sort key of the table and of an index only', () => { + const fields = fieldsOf(); + + expect(fields.orderId.isSortable).toBe(true); + expect(fields.createdAt.isSortable).toBe(true); + expect(fields.customerId.isSortable).toBe(false); + expect(fields.status.isSortable).toBe(false); + expect(fields.total.isSortable).toBe(false); + }); + + it('gives back every operator and every sort when the items may be processed in memory', () => { + const fields = fieldsOf({ inMemoryProcessing: true }); + + expect(fields.status.filterOperators.has('IContains')).toBe(true); + expect(fields.status.filterOperators.has('Like')).toBe(true); + expect(fields.total.isSortable).toBe(true); + expect(fields.customerId.isSortable).toBe(true); + }); + + it('is off by default', () => { + expect(resolveSettings().inMemoryProcessing).toBe(false); + }); +}); + +describe('Present', () => { + it('is evaluated by DynamoDB, as the negation of Blank', async () => { + const { collection, client } = buildCollection( + model(), + { Scan: () => ({ Items: [] }) }, + { scanConcurrency: 1 }, + ); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('status', 'Present') }), + new Projection('orderId'), + ); + + expect(client.calls('Scan')[0].FilterExpression).toMatch( + /^NOT \(attribute_not_exists\(#n0\) OR attribute_type\(#n0, :v0\) OR #n0 = :v1\)$/, + ); + }); +}); + +describe('what Forest may ask of a collection of array elements', () => { + const build = (settings = {}) => { + const parent = buildCollection(model(), {}, settings); + + return new ArrayCollection({} as DataSource, parent.collection, { + name: 'order_lines', + arrayField: 'lines', + itemSchema: { sku: { type: 'string' }, qty: { type: 'number' } }, + }).schema.fields as Record; isSortable: boolean }>; + }; + + it('filters on the keys of the table item and the position of the element', () => { + const fields = build(); + + for (const key of ['customerId', 'orderId', 'index']) { + expect([...fields[key].filterOperators].sort()).toEqual(['Equal', 'In']); + expect(fields[key].isSortable).toBe(false); + } + }); + + it('does not filter or sort on the attributes of the elements, which live inside the item', () => { + const fields = build(); + + expect(fields.sku.filterOperators.size).toBe(0); + expect(fields.sku.isSortable).toBe(false); + expect(fields.qty.filterOperators.size).toBe(0); + }); + + it('gives them back when the items may be processed in memory', () => { + const fields = build({ inMemoryProcessing: true }); + + expect(fields.sku.filterOperators.has('Equal')).toBe(true); + expect(fields.qty.isSortable).toBe(true); + }); +}); + +describe('a sort Forest should not ask for', () => { + const list = (sortField: string, settings = {}) => { + const { collection, client } = buildCollection( + model(), + { Scan: () => ({ Items: [{ customerId: 'c', orderId: 'o', status: 'b', total: 1 }] }) }, + { scanConcurrency: 1, ...settings }, + ); + + return { + client, + result: collection.list( + caller, + new PaginatedFilter({ sort: new Sort({ field: sortField, ascending: true }) }), + new Projection('orderId'), + ), + }; + }; + + it('is refused on an attribute that is not a key, which would read every item', async () => { + const { client, result } = list('total'); + + await expect(result).rejects.toThrow( + "Sorting on 'total' reads every item and sorts them in memory: DynamoDB sorts on keys only. " + + 'Set inMemoryProcessing to allow it.', + ); + expect(client.calls('Scan')).toHaveLength(0); + }); + + it.each(['orderId', 'createdAt', 'customerId', 'status'])( + 'is allowed on %s when the items may be processed in memory', + async field => { + const { result } = list(field, { inMemoryProcessing: true }); + + await expect(result).resolves.toHaveLength(1); + }, + ); + + it.each(['orderId', 'customerId'])('is allowed on the key %s', async field => { + const { result } = list(field); + + await expect(result).resolves.toHaveLength(1); + }); +}); diff --git a/packages/datasource-dynamodb/test/collection-helpers.ts b/packages/datasource-dynamodb/test/collection-helpers.ts new file mode 100644 index 0000000000..3bf7813d33 --- /dev/null +++ b/packages/datasource-dynamodb/test/collection-helpers.ts @@ -0,0 +1,26 @@ +import type { CollectionSettings, TableModel } from '../src/types'; +import type { DataSource } from '@forestadmin/datasource-toolkit'; + +import { fakeClient } from './fake-client'; +import DynamoDBCollection from '../src/collection'; +import { resolveSettings } from '../src/settings'; + +export function buildCollection( + model: TableModel, + handlers: Record) => unknown) | unknown[]>, + settings: Partial = {}, +) { + const client = fakeClient(handlers); + const logger = jest.fn(); + const collection = new DynamoDBCollection( + {} as DataSource, + model, + client, + logger, + resolveSettings(settings), + ); + + return { collection, client, logger }; +} + +export const caller = { timezone: 'UTC' } as never; diff --git a/packages/datasource-dynamodb/test/collection.aggregate.test.ts b/packages/datasource-dynamodb/test/collection.aggregate.test.ts new file mode 100644 index 0000000000..720f762a0c --- /dev/null +++ b/packages/datasource-dynamodb/test/collection.aggregate.test.ts @@ -0,0 +1,466 @@ +import { Aggregation, Filter } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { and, buildModel, buildSimpleModel, leaf, plain } from './helpers'; + +const caller = { timezone: 'UTC' } as never; +const aggregation = (plainAggregation: ConstructorParameters[0]) => + new Aggregation(plainAggregation); + +describe('DynamoDBCollection.aggregate', () => { + describe('count', () => { + it('counts a query with Select COUNT, reading no item', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: [ + { Count: 40, LastEvaluatedKey: { customerId: 'c1', orderId: 'o40' } }, + { Count: 2 }, + ], + }); + + const result = await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + aggregation({ operation: 'Count' }), + ); + + expect(result).toEqual([{ value: 42, group: {} }]); + expect(client.calls('Query')[0]).toMatchObject({ Select: 'COUNT' }); + expect(client.calls('Query')[1]).toMatchObject({ + ExclusiveStartKey: { customerId: 'c1', orderId: 'o40' }, + }); + }); + + it('counts a table with a parallel Scan and sums the segments', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: (input: any) => ({ Count: input.Segment + 1 }) }, + { scanConcurrency: 3 }, + ); + + const result = await collection.aggregate( + caller, + new Filter({}), + aggregation({ operation: 'Count' }), + ); + + expect(result).toEqual([{ value: 1 + 2 + 3, group: {} }]); + expect(client.calls('Scan').map(c => [c.Segment, c.TotalSegments])).toEqual([ + [0, 3], + [1, 3], + [2, 3], + ]); + expect(client.calls('Scan')[0]).toMatchObject({ Select: 'COUNT' }); + }); + + it('does not use segments for a single-segment scan', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Count: 7 }) }, + { scanConcurrency: 1 }, + ); + + await collection.aggregate(caller, new Filter({}), aggregation({ operation: 'Count' })); + + expect(client.calls('Scan')[0].Segment).toBeUndefined(); + expect(client.calls('Scan')[0].TotalSegments).toBeUndefined(); + }); + + it('pushes the filter into the counting request', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Count: 1 }) }, + { scanConcurrency: 1 }, + ); + + await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('age', 'GreaterThan', 3) }), + aggregation({ operation: 'Count' }), + ); + + expect(client.calls('Scan')[0]).toMatchObject({ + FilterExpression: '#n0 > :v0', + Select: 'COUNT', + }); + }); + + it('reads the items when part of the filter cannot be pushed down', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { + Scan: () => ({ + Items: [ + { id: 'a', name: 'xc' }, + { id: 'b', name: 'xd' }, + ], + }), + }, + { scanConcurrency: 1 }, + ); + + const result = await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('name', 'EndsWith', 'c') }), + aggregation({ operation: 'Count' }), + ); + + expect(result).toEqual([{ value: 1, group: {} }]); + expect(client.calls('Scan')[0].Select).toBeUndefined(); + }); + + it('answers an unfiltered count from the table description when approximate counts are on', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { DescribeTable: () => ({ Table: { ItemCount: 1234 } }) }, + { approximateCount: true }, + ); + + const result = await collection.aggregate( + caller, + new Filter({}), + aggregation({ operation: 'Count' }), + ); + + expect(result).toEqual([{ value: 1234, group: {} }]); + expect(client.calls('Scan')).toHaveLength(0); + }); + + it('still counts exactly a filtered request when approximate counts are on', async () => { + const { collection } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Count: 3 }) }, + { approximateCount: true, scanConcurrency: 1 }, + ); + + const result = await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('age', 'Equal', 3) }), + aggregation({ operation: 'Count' }), + ); + + expect(result).toEqual([{ value: 3, group: {} }]); + }); + }); + + describe('reducing operations', () => { + it('reads only the attributes it needs and reduces them', async () => { + const { collection, client } = buildCollection( + buildModel(), + { + Query: () => ({ + Items: [ + { total: 10, status: 'paid' }, + { total: 5, status: 'paid' }, + { total: 1, status: 'open' }, + ], + }), + }, + { scanConcurrency: 1 }, + ); + + const result = await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + aggregation({ operation: 'Sum', field: 'total', groups: [{ field: 'status' }] }), + ); + + expect(result).toEqual([ + { value: 15, group: { status: 'paid' } }, + { value: 1, group: { status: 'open' } }, + ]); + expect(client.calls('Query')[0]).toMatchObject({ + ProjectionExpression: '#n1, #n2', + ExpressionAttributeNames: { '#n0': 'customerId', '#n1': 'total', '#n2': 'status' }, + }); + }); + + it('groups on several fields', async () => { + const { collection } = buildCollection( + buildSimpleModel(), + { + Scan: () => ({ + Items: [ + { a: 1, b: 'x' }, + { a: 1, b: 'x' }, + { a: 2, b: 'x' }, + ], + }), + }, + { scanConcurrency: 1 }, + ); + + const result = await collection.aggregate( + caller, + new Filter({}), + aggregation({ operation: 'Count', groups: [{ field: 'a' }, { field: 'b' }] }), + ); + + expect(result).toEqual([ + { value: 2, group: { a: 1, b: 'x' } }, + { value: 1, group: { a: 2, b: 'x' } }, + ]); + }); + + it('groups by a date part in the caller timezone', async () => { + const { collection } = buildCollection( + buildModel(), + { Scan: () => ({ Items: [{ createdAt: '2024-01-20T23:30:00Z' }] }) }, + { scanConcurrency: 1 }, + ); + + const result = await collection.aggregate( + { timezone: 'Asia/Tokyo' } as never, + new Filter({}), + aggregation({ operation: 'Count', groups: [{ field: 'createdAt', operation: 'Day' }] }), + ); + + expect(result).toEqual([{ value: 1, group: { createdAt: '2024-01-21' } }]); + }); + + it('applies the limit to the groups', async () => { + const { collection } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Items: [{ a: 1 }, { a: 2 }, { a: 2 }, { a: 3 }, { a: 3 }, { a: 3 }] }) }, + { scanConcurrency: 1 }, + ); + + const result = await collection.aggregate( + caller, + new Filter({}), + aggregation({ operation: 'Count', groups: [{ field: 'a' }] }), + 2, + ); + + expect(result.map(r => r.group.a)).toEqual([3, 2]); + }); + + it('merges the segments of a parallel scan', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { + Scan: (input: any) => ({ + Items: [{ age: input.Segment + 1 }], + }), + }, + { scanConcurrency: 2 }, + ); + + const result = await collection.aggregate( + caller, + new Filter({}), + aggregation({ operation: 'Sum', field: 'age' }), + ); + + expect(result).toEqual([{ value: 3, group: {} }]); + expect(client.calls('Scan')).toHaveLength(2); + }); + + it('follows LastEvaluatedKey inside each segment', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { + Scan: [{ Items: [{ age: 1 }], LastEvaluatedKey: { id: 'a' } }, { Items: [{ age: 2 }] }], + }, + { scanConcurrency: 1 }, + ); + + const result = await collection.aggregate( + caller, + new Filter({}), + aggregation({ operation: 'Sum', field: 'age' }), + ); + + expect(result).toEqual([{ value: 3, group: {} }]); + expect(client.calls('Scan')[1]).toMatchObject({ ExclusiveStartKey: { id: 'a' } }); + }); + + it('stops reading beyond maxScannedItems', async () => { + const { collection } = buildCollection( + buildSimpleModel({ maxScannedItems: 2 }), + { + Scan: () => ({ + Items: [{ age: 1 }, { age: 1 }], + ScannedCount: 2, + LastEvaluatedKey: { id: 'a' }, + }), + }, + { scanConcurrency: 1 }, + ); + + await expect( + collection.aggregate( + caller, + new Filter({}), + aggregation({ operation: 'Sum', field: 'age' }), + ), + ).rejects.toThrow("Scan on table 'Users' stopped after 4 items: maxScannedItems is 2"); + }); + + it('stops the other segments from reading on when one fails', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { + Scan: (input: any) => { + if (input.Segment === 0) throw new Error('boom'); + + return { Items: [{ age: 1 }], LastEvaluatedKey: { id: 'a' } }; + }, + }, + { scanConcurrency: 2 }, + ); + + await expect( + collection.aggregate( + caller, + new Filter({}), + aggregation({ operation: 'Sum', field: 'age' }), + ), + ).rejects.toThrow("Failed to aggregate records of 'users': boom"); + await new Promise(resolve => { + setImmediate(resolve); + }); + + expect(client.calls('Scan').filter(call => call.Segment === 1).length).toBeLessThanOrEqual(2); + }); + + it('refuses to scan under a deny policy', async () => { + const { collection } = buildCollection(buildSimpleModel({ scanPolicy: 'deny' }), {}); + + await expect( + collection.aggregate(caller, new Filter({}), aggregation({ operation: 'Count' })), + ).rejects.toThrow("Scan on table 'Users' refused by scanPolicy 'deny'"); + }); + + it('wraps a failure with the collection name', async () => { + const { collection } = buildCollection(buildSimpleModel(), { + Scan: () => { + throw new Error('boom'); + }, + }); + + await expect( + collection.aggregate(caller, new Filter({}), aggregation({ operation: 'Count' })), + ).rejects.toThrow("Failed to aggregate records of 'users': boom"); + }); + }); + + describe('result cache', () => { + it('serves the same chart from memory within the time to live', async () => { + let now = 1000; + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Count: 5 }) }, + { chartCacheTtlMs: 1000, scanConcurrency: 1, now: () => now }, + ); + const run = () => + collection.aggregate(caller, new Filter({}), aggregation({ operation: 'Count' })); + + await run(); + now += 999; + const second = await run(); + + expect(second).toEqual([{ value: 5, group: {} }]); + expect(client.calls('Scan')).toHaveLength(1); + }); + + it('reads again once the time to live has passed', async () => { + let now = 1000; + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Count: 5 }) }, + { chartCacheTtlMs: 1000, scanConcurrency: 1, now: () => now }, + ); + const run = () => + collection.aggregate(caller, new Filter({}), aggregation({ operation: 'Count' })); + + await run(); + now += 1001; + await run(); + + expect(client.calls('Scan')).toHaveLength(2); + }); + + it('does not mix charts of different filters', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Count: 5 }) }, + { chartCacheTtlMs: 1000, scanConcurrency: 1 }, + ); + + await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('age', 'Equal', 1) }), + aggregation({ operation: 'Count' }), + ); + await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('age', 'Equal', 2) }), + aggregation({ operation: 'Count' }), + ); + + expect(client.calls('Scan')).toHaveLength(2); + }); + + it('remembers at most 100 charts, dropping the oldest', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Count: 5 }) }, + { chartCacheTtlMs: 60000, scanConcurrency: 1 }, + ); + const run = (age: number) => + collection.aggregate( + caller, + new Filter({ conditionTree: leaf('age', 'Equal', age) }), + aggregation({ operation: 'Count' }), + ); + + for (let age = 0; age <= 100; age += 1) { + // eslint-disable-next-line no-await-in-loop + await run(age); + } + + client.send.mockClear(); + + await run(100); + await run(0); + + expect(client.calls('Scan')).toHaveLength(1); + }); + + it('is off when the time to live is zero', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Count: 5 }) }, + { chartCacheTtlMs: 0, scanConcurrency: 1 }, + ); + const run = () => + collection.aggregate(caller, new Filter({}), aggregation({ operation: 'Count' })); + + await run(); + await run(); + + expect(client.calls('Scan')).toHaveLength(2); + }); + }); + + it('uses the access path of the filter for the aggregation', async () => { + const { collection, client } = buildCollection( + buildModel(), + { Query: () => ({ Count: 2 }) }, + { scanConcurrency: 1 }, + ); + + await collection.aggregate( + caller, + new Filter({ + conditionTree: and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'StartsWith', 'o')), + }), + aggregation({ operation: 'Count' }), + ); + + expect(client.calls('Query')[0]).toMatchObject({ + KeyConditionExpression: '#n0 = :v0 AND begins_with(#n1, :v1)', + Select: 'COUNT', + }); + }); +}); diff --git a/packages/datasource-dynamodb/test/collection.list.test.ts b/packages/datasource-dynamodb/test/collection.list.test.ts new file mode 100644 index 0000000000..bba1865f71 --- /dev/null +++ b/packages/datasource-dynamodb/test/collection.list.test.ts @@ -0,0 +1,608 @@ +import { PaginatedFilter, Projection, Sort } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { and, buildModel, buildSimpleModel, leaf, or, plain } from './helpers'; + +const caller = { timezone: 'UTC' } as never; +const order = (n: number, extra = {}) => ({ + customerId: 'c1', + orderId: `o${n}`, + total: n, + ...extra, +}); + +describe('DynamoDBCollection.list', () => { + describe('access path', () => { + it('queries the table with a key condition when the partition key is pinned', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: () => ({ Items: [order(1)] }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + new Projection('customerId', 'orderId', 'total'), + ); + + expect(records).toEqual([{ customerId: 'c1', orderId: 'o1', total: 1 }]); + expect(client.calls('Query')[0]).toMatchObject({ + TableName: 'Orders', + KeyConditionExpression: '#n0 = :v0', + ProjectionExpression: '#n0, #n1, #n2', + ExpressionAttributeNames: { '#n0': 'customerId', '#n1': 'orderId', '#n2': 'total' }, + ExpressionAttributeValues: { ':v0': 'c1' }, + }); + expect(client.calls('Query')[0].FilterExpression).toBeUndefined(); + }); + + it('puts the remaining conditions in a filter expression', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: () => ({ Items: [] }), + }); + + await collection.list( + caller, + new PaginatedFilter({ + conditionTree: and(plain('customerId', 'Equal', 'c1'), plain('status', 'Equal', 'paid')), + }), + new Projection('orderId'), + ); + + expect(client.calls('Query')[0]).toMatchObject({ + KeyConditionExpression: '#n0 = :v0', + FilterExpression: '#n1 = :v1', + ExpressionAttributeValues: { ':v0': 'c1', ':v1': 'paid' }, + }); + }); + + it('queries a secondary index when its partition key is pinned', async () => { + const { collection, client } = buildCollection( + buildModel({ + indexes: [ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'S' }, + projectsAllAttributes: true, + }, + ], + }), + { Query: () => ({ Items: [] }) }, + ); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('status', 'Equal', 'paid') }), + new Projection('orderId'), + ); + + expect(client.calls('Query')[0]).toMatchObject({ IndexName: 'by-status' }); + }); + + it('scans when no key is pinned', async () => { + const { collection, client } = buildCollection(buildModel(), { + Scan: () => ({ Items: [order(1)] }), + }); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('status', 'Equal', 'paid') }), + new Projection('orderId'), + ); + + expect(client.calls('Scan')[0]).toMatchObject({ + TableName: 'Orders', + FilterExpression: '#n0 = :v0', + }); + }); + + it('reads the sort key backwards for a descending sort', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: () => ({ Items: [] }), + }); + + await collection.list( + caller, + new PaginatedFilter({ + conditionTree: leaf('customerId', 'Equal', 'c1'), + sort: new Sort({ field: 'orderId', ascending: false }), + }), + new Projection('orderId'), + ); + + expect(client.calls('Query')[0].ScanIndexForward).toBe(false); + }); + + it('looks keys up with BatchGetItem when the filter pins whole keys', async () => { + const { collection, client } = buildCollection(buildModel(), { + BatchGet: () => ({ + Responses: { Orders: [order(1), order(2)] }, + }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: or( + and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'Equal', 'o1')), + and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'Equal', 'o2')), + ), + }), + new Projection('customerId', 'orderId', 'total'), + ); + + expect(records).toHaveLength(2); + expect(client.calls('BatchGet')[0].RequestItems.Orders.Keys).toEqual([ + { customerId: 'c1', orderId: 'o1' }, + { customerId: 'c1', orderId: 'o2' }, + ]); + expect(client.calls('Query')).toHaveLength(0); + expect(client.calls('Scan')).toHaveLength(0); + }); + + it('splits a BatchGetItem in chunks of 100 keys and retries unprocessed keys', async () => { + const ids = Array.from({ length: 120 }, (_, i) => `u${i}`); + let call = 0; + const { collection, client } = buildCollection(buildSimpleModel(), { + BatchGet: (input: any) => { + call += 1; + const keys = input.RequestItems.Users.Keys as Array<{ id: string }>; + + if (call === 1) { + return { + Responses: { Users: keys.slice(0, 99).map(k => ({ id: k.id })) }, + UnprocessedKeys: { Users: { Keys: [keys[99]] } }, + }; + } + + return { Responses: { Users: keys.map(k => ({ id: k.id })) } }; + }, + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('id', 'In', ids) }), + new Projection('id'), + ); + + expect(records).toHaveLength(120); + expect(client.calls('BatchGet').map(c => c.RequestItems.Users.Keys.length)).toEqual([ + 100, 1, 20, + ]); + }); + + it('applies the remaining filter to a key lookup', async () => { + const { collection } = buildCollection(buildSimpleModel(), { + BatchGet: () => ({ + Responses: { + Users: [ + { id: 'u1', age: 10 }, + { id: 'u2', age: 50 }, + ], + }, + }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: and(plain('id', 'In', ['u1', 'u2']), plain('age', 'GreaterThan', 20)), + }), + new Projection('id', 'age'), + ); + + expect(records).toEqual([{ id: 'u2', age: 50 }]); + }); + }); + + describe('scan policy', () => { + it('warns by default when it has to scan, without leaking filter values', async () => { + const { collection, logger } = buildCollection(buildModel(), { + Scan: () => ({ Items: [] }), + }); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('status', 'Equal', 'secret-value') }), + new Projection('orderId'), + ); + + expect(logger).toHaveBeenCalledWith( + 'Warn', + "Scan on table 'Orders' for collection 'orders': no key or index serves the filter (status Equal)", + ); + expect(JSON.stringify(logger.mock.calls)).not.toContain('secret-value'); + }); + + it('does not warn when the scan policy is allow', async () => { + const { collection, logger } = buildCollection(buildModel({ scanPolicy: 'allow' }), { + Scan: () => ({ Items: [] }), + }); + + await collection.list(caller, new PaginatedFilter({}), new Projection('orderId')); + + expect(logger).not.toHaveBeenCalledWith('Warn', expect.stringContaining('Scan on table')); + }); + + it('refuses to scan when the scan policy is deny', async () => { + const { collection, client } = buildCollection(buildModel({ scanPolicy: 'deny' }), { + Scan: () => ({ Items: [] }), + }); + + await expect( + collection.list(caller, new PaginatedFilter({}), new Projection('orderId')), + ).rejects.toThrow( + "Scan on table 'Orders' refused by scanPolicy 'deny': no key or index serves the filter", + ); + expect(client.calls('Scan')).toHaveLength(0); + }); + + it('lets the collection override the datasource policy', async () => { + const { collection } = buildCollection( + buildModel({ scanPolicy: 'allow' }), + { Scan: () => ({ Items: [] }) }, + { scanPolicy: 'deny' }, + ); + + await expect( + collection.list(caller, new PaginatedFilter({}), new Projection('orderId')), + ).resolves.toEqual([]); + }); + + it('still queries under a deny policy', async () => { + const { collection } = buildCollection(buildModel({ scanPolicy: 'deny' }), { + Query: () => ({ Items: [order(1)] }), + }); + + await expect( + collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + new Projection('orderId'), + ), + ).resolves.toHaveLength(1); + }); + + it('stops a scan that reads more items than maxScannedItems', async () => { + const { collection } = buildCollection(buildModel({ maxScannedItems: 3 }), { + Scan: () => ({ + Items: [order(1), order(2)], + ScannedCount: 2, + LastEvaluatedKey: { customerId: 'c1', orderId: 'o2' }, + }), + }); + + await expect( + collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('status', 'Equal', 'x') }), + new Projection('orderId'), + ), + ).rejects.toThrow("Scan on table 'Orders' stopped after 4 items: maxScannedItems is 3"); + }); + }); + + describe('client-side evaluation', () => { + it('evaluates operators DynamoDB lacks on the returned items', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Scan: () => ({ + Items: [ + { id: 'u1', name: 'Émile' }, + { id: 'u2', name: 'Bob' }, + ], + }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('name', 'ILike', '%émi%') }), + new Projection('id'), + ); + + expect(records).toEqual([{ id: 'u1' }]); + expect(client.calls('Scan')[0].FilterExpression).toBeUndefined(); + expect(client.calls('Scan')[0].ProjectionExpression).toBe('#n0, #n1'); + }); + + it('pushes the supported part of the filter and evaluates the rest on the items', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Scan: () => ({ + Items: [ + { id: 'u1', name: 'abc', age: 30 }, + { id: 'u2', name: 'xbz', age: 30 }, + ], + }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: and(plain('age', 'Equal', 30), plain('name', 'EndsWith', 'c')), + }), + new Projection('id'), + ); + + expect(records).toEqual([{ id: 'u1' }]); + expect(client.calls('Scan')[0].FilterExpression).toBe('#n0 = :v0'); + }); + }); + + describe('sort', () => { + it('sorts on a non-key attribute after reading the matching items', async () => { + const { collection } = buildCollection( + buildModel(), + { Query: () => ({ Items: [order(3), order(1), order(2)] }) }, + { inMemoryProcessing: true }, + ); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: leaf('customerId', 'Equal', 'c1'), + sort: new Sort({ field: 'total', ascending: false }), + }), + new Projection('orderId', 'total'), + ); + + expect(records.map(r => r.total)).toEqual([3, 2, 1]); + }); + + it('sorts null and missing values last when ascending', async () => { + const { collection } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Items: [{ id: 'a' }, { id: 'b', age: 2 }, { id: 'c', age: 1 }] }) }, + { inMemoryProcessing: true }, + ); + + const records = await collection.list( + caller, + new PaginatedFilter({ sort: new Sort({ field: 'age', ascending: true }) }), + new Projection('id', 'age'), + ); + + expect(records.map(r => r.id)).toEqual(['c', 'b', 'a']); + }); + }); + + describe('pagination', () => { + it('limits the items read when nothing filters them', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: () => ({ Items: [order(1), order(2)] }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: leaf('customerId', 'Equal', 'c1'), + page: { limit: 2, skip: 0 } as never, + }), + new Projection('orderId'), + ); + + expect(records).toHaveLength(2); + expect(client.calls('Query')[0].Limit).toBe(2); + }); + + it('skips records and follows LastEvaluatedKey until the page is full', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: [ + { Items: [order(1), order(2)], LastEvaluatedKey: { customerId: 'c1', orderId: 'o2' } }, + { Items: [order(3), order(4)], LastEvaluatedKey: { customerId: 'c1', orderId: 'o4' } }, + { Items: [order(5)] }, + ], + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: and(plain('customerId', 'Equal', 'c1'), plain('status', 'Equal', 'paid')), + page: { limit: 2, skip: 3 } as never, + }), + new Projection('orderId'), + ); + + expect(records.map(r => r.orderId)).toEqual(['o4', 'o5']); + expect(client.calls('Query')[1]).toMatchObject({ + ExclusiveStartKey: { customerId: 'c1', orderId: 'o2' }, + }); + }); + + it('resumes from a cached key when jumping to a deeper page', async () => { + const first = buildCollection(buildModel(), { + Query: [ + { Items: [order(1), order(2)], LastEvaluatedKey: { customerId: 'c1', orderId: 'o2' } }, + { Items: [order(3), order(4)] }, + ], + }); + const filter = (skip: number) => + new PaginatedFilter({ + conditionTree: and(plain('customerId', 'Equal', 'c1'), plain('status', 'Equal', 'paid')), + page: { limit: 2, skip } as never, + }); + + await first.collection.list(caller, filter(0), new Projection('orderId')); + first.client.send.mockClear(); + await first.collection.list(caller, filter(2), new Projection('orderId')); + + expect(first.client.calls('Query')[0]).toMatchObject({ + ExclusiveStartKey: { customerId: 'c1', orderId: 'o2' }, + }); + }); + + it('refuses an offset deeper than the maximum', async () => { + const { collection } = buildCollection(buildModel(), {}, { maxOffset: 10 }); + + await expect( + collection.list( + caller, + new PaginatedFilter({ + conditionTree: leaf('customerId', 'Equal', 'c1'), + page: { limit: 5, skip: 11 } as never, + }), + new Projection('orderId'), + ), + ).rejects.toThrow( + 'Offset 11 exceeds the maximum allowed offset of 10. Narrow the results with filters.', + ); + }); + + it('returns every matching record when no page is requested', async () => { + const { collection } = buildCollection(buildModel(), { + Query: [ + { Items: [order(1)], LastEvaluatedKey: { customerId: 'c1', orderId: 'o1' } }, + { Items: [order(2)] }, + ], + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + new Projection('orderId'), + ); + + expect(records).toHaveLength(2); + }); + }); + + describe('records', () => { + it('serialises sets, nested maps and dates', async () => { + const { collection } = buildCollection( + buildSimpleModel({ + schema: { + id: { type: 'string' }, + tags: { type: 'array' }, + 'address@@@city': { type: 'string' }, + }, + }), + { + Scan: () => ({ + Items: [{ id: 'u1', tags: new Set(['a']), address: { city: 'Paris', zip: '75' } }], + }), + }, + ); + + const records = await collection.list( + caller, + new PaginatedFilter({}), + new Projection('id', 'tags', 'address@@@city'), + ); + + expect(records).toEqual([{ id: 'u1', tags: ['a'], 'address@@@city': 'Paris' }]); + }); + + it('fetches the top-level attribute of a nested projection, with the key', async () => { + const { collection, client } = buildCollection( + buildSimpleModel({ + schema: { id: { type: 'string' }, 'address@@@city': { type: 'string' } }, + }), + { Scan: () => ({ Items: [] }) }, + ); + + await collection.list(caller, new PaginatedFilter({}), new Projection('address@@@city')); + + expect(client.calls('Scan')[0]).toMatchObject({ + ProjectionExpression: '#n0, #n1', + ExpressionAttributeNames: { '#n0': 'address', '#n1': 'id' }, + }); + }); + }); + + describe('single-table entity', () => { + it('restricts reads to the entity discriminator', async () => { + const { collection, client } = buildCollection( + buildModel({ entity: { attribute: 'entityType', value: 'ORDER' } }), + { Query: () => ({ Items: [] }) }, + ); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + new Projection('orderId'), + ); + + expect(client.calls('Query')[0]).toMatchObject({ + FilterExpression: '#n1 = :v1', + ExpressionAttributeNames: expect.objectContaining({ '#n1': 'entityType' }), + ExpressionAttributeValues: { ':v0': 'c1', ':v1': 'ORDER' }, + }); + }); + }); + + describe('observability', () => { + it('logs the generated expressions when asked', async () => { + const { collection, logger } = buildCollection( + buildModel(), + { Query: () => ({ Items: [] }) }, + { logQueries: true }, + ); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + new Projection('orderId'), + ); + + expect(logger).toHaveBeenCalledWith( + 'Info', + '[DynamoDB Query] orders.query: KeyCondition=#n0 = :v0 | Filter=- | Index=table', + ); + }); + + it('logs the consumed capacity when asked', async () => { + const { collection, logger, client } = buildCollection( + buildModel(), + { Query: () => ({ Items: [], ConsumedCapacity: { CapacityUnits: 1.5 } }) }, + { logConsumedCapacity: true }, + ); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + new Projection('orderId'), + ); + + expect(client.calls('Query')[0].ReturnConsumedCapacity).toBe('TOTAL'); + expect(logger).toHaveBeenCalledWith('Info', '[DynamoDB Capacity] orders.query: 1.5 units'); + }); + }); +}); + +describe('DynamoDBCollection.list with long lists of values', () => { + const values = Array.from({ length: 250 }, (_, i) => `v${i}`); + + it('evaluates a long In list on the returned items instead of overflowing the expression', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Scan: () => ({ + Items: [ + { id: 'a', name: 'v7' }, + { id: 'b', name: 'other' }, + ], + }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('name', 'In', values) }), + new Projection('id'), + ); + + expect(records).toEqual([{ id: 'a' }]); + expect(client.calls('Scan')[0].FilterExpression).toBeUndefined(); + }); + + it('still reads a long list of keys with BatchGetItem', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + BatchGet: () => ({ Responses: { Users: [] } }), + }); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('id', 'In', values) }), + new Projection('id'), + ); + + expect(client.calls('BatchGet').map(call => call.RequestItems.Users.Keys.length)).toEqual([ + 100, 100, 50, + ]); + }); +}); diff --git a/packages/datasource-dynamodb/test/collection.schema.test.ts b/packages/datasource-dynamodb/test/collection.schema.test.ts new file mode 100644 index 0000000000..17290ad665 --- /dev/null +++ b/packages/datasource-dynamodb/test/collection.schema.test.ts @@ -0,0 +1,108 @@ +import { buildCollection } from './collection-helpers'; +import { buildModel, buildSimpleModel } from './helpers'; + +describe('DynamoDBCollection schema', () => { + it('marks the partition key and the sort key as the composite primary key', () => { + const { collection } = buildCollection(buildModel(), {}); + + expect(collection.schema.fields.customerId).toMatchObject({ + type: 'Column', + columnType: 'String', + isPrimaryKey: true, + }); + expect(collection.schema.fields.orderId).toMatchObject({ isPrimaryKey: true }); + expect(collection.schema.fields.status).toMatchObject({ isPrimaryKey: false }); + }); + + it('marks a lone partition key as the primary key', () => { + const { collection } = buildCollection(buildSimpleModel(), {}); + + expect(collection.schema.fields.id).toMatchObject({ isPrimaryKey: true }); + }); + + it('types a numeric key as a Number', () => { + const { collection } = buildCollection( + buildSimpleModel({ + partitionKey: { name: 'id', type: 'N' }, + schema: { id: { type: 'number' } }, + }), + {}, + ); + + expect(collection.schema.fields.id).toMatchObject({ columnType: 'Number', isPrimaryKey: true }); + }); + + it('maps inferred types to column types', () => { + const { collection } = buildCollection(buildModel(), {}); + + expect(collection.schema.fields.total).toMatchObject({ columnType: 'Number' }); + expect(collection.schema.fields.createdAt).toMatchObject({ columnType: 'Date' }); + }); + + it('exposes nested attributes as arrow-notation columns', () => { + const { collection } = buildCollection( + buildSimpleModel({ + schema: { id: { type: 'string' }, 'address@@@city': { type: 'string' } }, + }), + {}, + ); + + expect(collection.schema.fields['address@@@city']).toMatchObject({ columnType: 'String' }); + }); + + it('does not allow sorting on Json columns', () => { + const { collection } = buildCollection( + buildSimpleModel({ schema: { id: { type: 'string' }, meta: { type: 'object' } } }), + {}, + ); + + expect(collection.schema.fields.meta).toMatchObject({ columnType: 'Json', isSortable: false }); + }); + + it('is countable by default and not searchable natively', () => { + const { collection } = buildCollection(buildModel(), {}); + + expect(collection.schema.countable).toBe(true); + expect(collection.schema.searchable).toBe(false); + }); + + it('can disable the count', () => { + const { collection } = buildCollection(buildModel({ enableCount: false }), {}); + + expect(collection.schema.countable).toBe(false); + }); + + it('lets the user override a generated column', () => { + const { collection } = buildCollection( + buildModel({ + overrideTypeConverter: ({ fieldName, generatedFieldSchema }) => + fieldName === 'status' + ? ({ + ...generatedFieldSchema, + columnType: 'Enum', + enumValues: ['paid', 'open'], + } as never) + : undefined, + }), + {}, + ); + + expect(collection.schema.fields.status).toMatchObject({ + columnType: 'Enum', + enumValues: ['paid', 'open'], + }); + }); + + it('skips a column it cannot type and logs it', () => { + const { collection, logger } = buildCollection( + buildSimpleModel({ schema: { id: { type: 'string' }, weird: { type: 'nope' as never } } }), + {}, + ); + + expect(collection.schema.fields.weird).toBeUndefined(); + expect(logger).toHaveBeenCalledWith( + 'Warn', + expect.stringContaining("Skipping column 'users.weird'"), + ); + }); +}); diff --git a/packages/datasource-dynamodb/test/collection.write.test.ts b/packages/datasource-dynamodb/test/collection.write.test.ts new file mode 100644 index 0000000000..30dba29082 --- /dev/null +++ b/packages/datasource-dynamodb/test/collection.write.test.ts @@ -0,0 +1,423 @@ +import { Aggregation, Filter } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { and, buildModel, buildSimpleModel, leaf, plain } from './helpers'; + +const caller = { timezone: 'UTC' } as never; +const conditional = () => + Object.assign(new Error('The conditional request failed'), { + name: 'ConditionalCheckFailedException', + }); + +describe('DynamoDBCollection.create', () => { + it('puts each record unless it would overwrite an existing item', async () => { + const { collection, client } = buildCollection(buildModel(), { Put: () => ({}) }); + + const created = await collection.create(caller, [ + { customerId: 'c1', orderId: 'o1', total: 5, 'address@@@city': 'Paris' }, + ]); + + expect(created).toEqual([ + { customerId: 'c1', orderId: 'o1', total: 5, 'address@@@city': 'Paris' }, + ]); + expect(client.calls('Put')[0]).toEqual({ + TableName: 'Orders', + Item: { customerId: 'c1', orderId: 'o1', total: 5, address: { city: 'Paris' } }, + ConditionExpression: 'attribute_not_exists(#k0)', + ExpressionAttributeNames: { '#k0': 'customerId' }, + }); + }); + + it('generates a partition key for a string key left empty', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { Put: () => ({}) }); + + const [created] = await collection.create(caller, [{ name: 'Bob' }]); + + expect(created.id).toMatch(/^[0-9a-f-]{36}$/); + expect(client.calls('Put')[0].Item.id).toBe(created.id); + }); + + it('generates a sort key for a string key left empty', async () => { + const { collection, client } = buildCollection(buildModel(), { Put: () => ({}) }); + + const [created] = await collection.create(caller, [{ customerId: 'c1' }]); + + expect(created.orderId).toMatch(/^[0-9a-f-]{36}$/); + expect(client.calls('Put')[0].Item).toMatchObject({ + customerId: 'c1', + orderId: created.orderId, + }); + }); + + it('prefixes a generated sort key with the entity prefix', async () => { + const { collection } = buildCollection(buildModel({ entity: { sortKeyPrefix: 'ORDER#' } }), { + Put: () => ({}), + }); + + const [created] = await collection.create(caller, [{ customerId: 'c1', orderId: '' }]); + + expect(created.orderId).toMatch(/^ORDER#[0-9a-f-]{36}$/); + }); + + it('refuses a numeric sort key left empty', async () => { + const { collection, client } = buildCollection( + buildModel({ + sortKey: { name: 'orderId', type: 'N' }, + schema: { orderId: { type: 'number' } }, + }), + { Put: () => ({}) }, + ); + + await expect(collection.create(caller, [{ customerId: 'c1' }])).rejects.toThrow( + "The sort key 'orderId' is required to create a record in 'orders'", + ); + expect(client.calls('Put')).toHaveLength(0); + }); + + it('treats an empty string key as missing', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { Put: () => ({}) }); + + const [created] = await collection.create(caller, [{ id: '', name: 'Bob' }]); + + expect(created.id).toMatch(/^[0-9a-f-]{36}$/); + expect(client.calls('Put')[0].Item.id).toBe(created.id); + }); + + it('refuses a numeric partition key left empty', async () => { + const { collection } = buildCollection( + buildSimpleModel({ + partitionKey: { name: 'id', type: 'N' }, + schema: { id: { type: 'number' } }, + }), + { Put: () => ({}) }, + ); + + await expect(collection.create(caller, [{}])).rejects.toThrow( + "The partition key 'id' is required to create a record in 'users'", + ); + }); + + it('reports an existing item as a validation error', async () => { + const { collection } = buildCollection(buildModel(), { + Put: () => { + throw conditional(); + }, + }); + + await expect(collection.create(caller, [{ customerId: 'c1', orderId: 'o1' }])).rejects.toThrow( + "A record with the same key already exists in 'orders'", + ); + }); + + it('stamps the entity discriminator', async () => { + const { collection, client } = buildCollection( + buildModel({ entity: { attribute: 'entityType', value: 'ORDER' } }), + { Put: () => ({}) }, + ); + + await collection.create(caller, [{ customerId: 'c1', orderId: 'o1' }]); + + expect(client.calls('Put')[0].Item.entityType).toBe('ORDER'); + }); + + it('refuses a sort key outside the entity prefix', async () => { + const { collection } = buildCollection(buildModel({ entity: { sortKeyPrefix: 'ORDER#' } }), { + Put: () => ({}), + }); + + await expect( + collection.create(caller, [{ customerId: 'c1', orderId: 'INVOICE#1' }]), + ).rejects.toThrow("The sort key 'orderId' of 'orders' must start with 'ORDER#'"); + }); + + it('forgets the cached pages and charts of the table', async () => { + const { collection, client } = buildCollection( + buildModel(), + { Put: () => ({}), Scan: () => ({ Items: [{ total: 1 }] }) }, + { chartCacheTtlMs: 60000, scanConcurrency: 4 }, + ); + const aggregation = new Aggregation({ operation: 'Sum', field: 'total' }); + + await collection.aggregate(caller, new Filter({}), aggregation); + await collection.create(caller, [{ customerId: 'c1', orderId: 'o1' }]); + await collection.aggregate(caller, new Filter({}), aggregation); + + expect(client.calls('Scan')).toHaveLength(8); + }); +}); + +describe('DynamoDBCollection.update', () => { + it('updates by key without reading when the filter pins the key', async () => { + const { collection, client } = buildCollection(buildModel(), { Update: () => ({}) }); + + await collection.update( + caller, + new Filter({ + conditionTree: and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'Equal', 'o1')), + }), + { total: 9, status: 'paid' }, + ); + + expect(client.calls('Update')).toEqual([ + { + TableName: 'Orders', + Key: { customerId: 'c1', orderId: 'o1' }, + UpdateExpression: 'SET #n0 = :v0, #n1 = :v1', + ConditionExpression: 'attribute_exists(#k0)', + ExpressionAttributeNames: { '#n0': 'total', '#n1': 'status', '#k0': 'customerId' }, + ExpressionAttributeValues: { ':v0': 9, ':v1': 'paid' }, + }, + ]); + expect(client.calls('Scan')).toHaveLength(0); + expect(client.calls('Query')).toHaveLength(0); + }); + + it('sets a nested attribute through its document path', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { Update: () => ({}) }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + 'address@@@city': 'Lyon', + }); + + expect(client.calls('Update')[0]).toMatchObject({ + UpdateExpression: 'SET #n0.#n1 = :v0', + ExpressionAttributeNames: expect.objectContaining({ '#n0': 'address', '#n1': 'city' }), + }); + }); + + it('stores a null as an explicit NULL', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { Update: () => ({}) }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + name: null, + }); + + expect(client.calls('Update')[0].ExpressionAttributeValues).toEqual({ ':v0': null }); + }); + + it('resolves the keys of the matching items when the filter does not pin them', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Scan: () => ({ Items: [{ id: 'u1' }, { id: 'u2' }] }), + Update: () => ({}), + }); + + await collection.update(caller, new Filter({ conditionTree: leaf('age', 'Equal', 5) }), { + name: 'x', + }); + + expect(client.calls('Scan')[0]).toMatchObject({ ProjectionExpression: '#n1' }); + expect(client.calls('Update').map(c => c.Key)).toEqual([{ id: 'u1' }, { id: 'u2' }]); + }); + + it('does nothing when no item matches', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Scan: () => ({ Items: [] }), + }); + + await collection.update(caller, new Filter({ conditionTree: leaf('age', 'Equal', 5) }), { + name: 'x', + }); + + expect(client.calls('Update')).toHaveLength(0); + }); + + it('refuses to change a key', async () => { + const { collection } = buildCollection(buildSimpleModel(), {}); + + await expect( + collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + id: 'u2', + }), + ).rejects.toThrow( + "The key attribute 'id' of 'users' cannot be changed on an existing record, " + + 'since DynamoDB keys are immutable: delete the record and create a new one instead', + ); + }); + + it('accepts a key written with the value it already has', async () => { + const { collection, client } = buildCollection(buildModel(), { Update: () => ({}) }); + + await collection.update( + caller, + new Filter({ + conditionTree: and(leaf('customerId', 'Equal', 'c1'), leaf('orderId', 'Equal', 'o1')), + }), + { customerId: 'c1', status: 'paid' }, + ); + + expect(client.calls('Update')).toHaveLength(1); + expect(client.calls('Update')[0].UpdateExpression).not.toContain('customerId'); + }); + + it('writes nothing when the only change is a key it already has', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { Update: () => ({}) }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + id: 'u1', + }); + + expect(client.calls('Update')).toHaveLength(0); + }); + + it('merges client-side when the parent map of a nested path does not exist yet', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Update: () => { + throw Object.assign( + new Error('The document path provided in the update expression is invalid for update'), + { name: 'ValidationException' }, + ); + }, + Get: () => ({ Item: { id: 'u1', name: 'a' } }), + Put: () => ({}), + }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + 'address@@@city': 'Lyon', + }); + + expect(client.calls('Put')[0]).toMatchObject({ + TableName: 'Users', + Item: { id: 'u1', name: 'a', address: { city: 'Lyon' } }, + ConditionExpression: 'attribute_exists(#k0)', + }); + }); + + it('replaces a map the way SET does when it falls back to a client-side rewrite', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Update: () => { + throw Object.assign( + new Error('The document path provided in the update expression is invalid for update'), + { name: 'ValidationException' }, + ); + }, + Get: () => ({ Item: { id: 'u1', config: { keep: 1, remove: 2 } } }), + Put: () => ({}), + }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + config: { keep: 3 }, + 'address@@@city': 'Lyon', + }); + + expect(client.calls('Put')[0].Item).toEqual({ + id: 'u1', + config: { keep: 3 }, + address: { city: 'Lyon' }, + }); + }); + + it('does not create an item that does not exist', async () => { + const { collection } = buildCollection(buildSimpleModel(), { + Update: () => { + throw conditional(); + }, + }); + + await expect( + collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'ghost') }), { + name: 'x', + }), + ).resolves.toBeUndefined(); + }); + + it('wraps a failure with the collection name and keeps the cause', async () => { + const boom = new Error('boom'); + const { collection } = buildCollection(buildSimpleModel(), { + Update: () => { + throw boom; + }, + }); + + const error = await collection + .update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { name: 'x' }) + .catch(e => e); + + expect(error.message).toBe("Failed to update records of 'users': boom"); + expect(error.cause).toBe(boom); + }); +}); + +describe('DynamoDBCollection.delete', () => { + it('deletes by key without reading when the filter pins the keys', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { BatchWrite: () => ({}) }); + + await collection.delete(caller, new Filter({ conditionTree: leaf('id', 'In', ['u1', 'u2']) })); + + expect(client.calls('BatchWrite')).toEqual([ + { + RequestItems: { + Users: [ + { DeleteRequest: { Key: { id: 'u1' } } }, + { DeleteRequest: { Key: { id: 'u2' } } }, + ], + }, + }, + ]); + expect(client.calls('Scan')).toHaveLength(0); + }); + + it('resolves the keys of the matching items', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: () => ({ Items: [{ customerId: 'c1', orderId: 'o1' }] }), + BatchWrite: () => ({}), + }); + + await collection.delete( + caller, + new Filter({ + conditionTree: and(plain('customerId', 'Equal', 'c1'), plain('status', 'Equal', 'x')), + }), + ); + + expect(client.calls('BatchWrite')[0].RequestItems.Orders).toEqual([ + { DeleteRequest: { Key: { customerId: 'c1', orderId: 'o1' } } }, + ]); + }); + + it('writes in batches of 25 and retries unprocessed items', async () => { + const ids = Array.from({ length: 30 }, (_, i) => `u${i}`); + let first = true; + const { collection, client } = buildCollection(buildSimpleModel(), { + BatchWrite: (input: any) => { + if (first) { + first = false; + + return { UnprocessedItems: { Users: [input.RequestItems.Users[0]] } }; + } + + return {}; + }, + }); + + await collection.delete(caller, new Filter({ conditionTree: leaf('id', 'In', ids) })); + + const sizes = client.calls('BatchWrite').map(c => c.RequestItems.Users.length); + expect(sizes.sort((a, b) => a - b)).toEqual([1, 5, 25]); + }); + + it('gives up when items stay unprocessed', async () => { + const { collection } = buildCollection( + buildSimpleModel(), + { + BatchWrite: (input: any) => ({ UnprocessedItems: { Users: input.RequestItems.Users } }), + }, + { maxWriteRetries: 2 }, + ); + + await expect( + collection.delete(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') })), + ).rejects.toThrow( + "Failed to delete records of 'users': 1 item(s) are still unprocessed after 2 retries", + ); + }); + + it('does nothing when no item matches', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Scan: () => ({ Items: [] }), + }); + + await collection.delete(caller, new Filter({ conditionTree: leaf('age', 'Equal', 1) })); + + expect(client.calls('BatchWrite')).toHaveLength(0); + }); +}); diff --git a/packages/datasource-dynamodb/test/datasource.test.ts b/packages/datasource-dynamodb/test/datasource.test.ts new file mode 100644 index 0000000000..6d4cb5d0df --- /dev/null +++ b/packages/datasource-dynamodb/test/datasource.test.ts @@ -0,0 +1,242 @@ +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; + +import { fakeClient } from './fake-client'; +import createDocumentClient from '../src/client'; +import { createDynamoDBDataSource } from '../src/index'; + +const table = (name: string, keyType = 'S') => ({ + Table: { + TableName: name, + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: keyType }], + }, +}); + +const handlers = () => ({ + ListTables: () => ({ TableNames: ['Users', 'Orders'] }), + DescribeTable: (input: { TableName: string }) => table(input.TableName), + Scan: () => ({ Items: [{ id: 'a', name: 'Bob' }] }), +}); + +describe('createDynamoDBDataSource', () => { + it('introspects every table into a collection, sorted by name', async () => { + const dataSource = await createDynamoDBDataSource({ client: fakeClient(handlers()) })( + jest.fn(), + jest.fn(), + ); + + expect(dataSource.collections.map(c => c.name)).toEqual(['Orders', 'Users']); + expect(dataSource.getCollection('Users').schema.fields.name).toMatchObject({ + columnType: 'String', + }); + }); + + it('only introspects the listed tables', async () => { + const client = fakeClient(handlers()); + + const dataSource = await createDynamoDBDataSource({ client, tables: ['Users'] })( + jest.fn(), + jest.fn(), + ); + + expect(dataSource.collections.map(c => c.name)).toEqual(['Users']); + expect(client.calls('ListTables')).toHaveLength(0); + }); + + it('builds collections from the builder, with a custom name and an entity', async () => { + const client = fakeClient({ + DescribeTable: (input: { TableName: string }) => table(input.TableName), + Scan: () => ({ Items: [] }), + }); + + const dataSource = await createDynamoDBDataSource({ + client, + tables: [], + builder: configurator => + configurator.addCollectionFromTable({ + name: 'customers', + tableName: 'AppTable', + entity: { attribute: 'entityType', value: 'CUSTOMER' }, + sampleSize: 20, + scanPolicy: 'deny', + }), + })(jest.fn(), jest.fn()); + + expect(dataSource.collections.map(c => c.name)).toEqual(['customers']); + expect(client.calls('Scan')[0]).toMatchObject({ TableName: 'AppTable', Limit: 20 }); + expect(client.calls('ListTables')).toHaveLength(0); + }); + + it('builds collections from a given introspection without calling AWS', async () => { + const client = fakeClient({}); + + const dataSource = await createDynamoDBDataSource( + { + client, + }, + { + introspection: { + collections: [ + { + name: 'users', + tableName: 'Users', + partitionKey: { name: 'id', type: 'string' }, + fields: [{ name: 'name', type: 'string' }], + }, + ], + }, + }, + )(jest.fn(), jest.fn()); + + expect(dataSource.getCollection('users').schema.fields.name).toMatchObject({ + columnType: 'String', + }); + expect(client.send).not.toHaveBeenCalled(); + }); + + it('rejects an invalid introspection', async () => { + const factory = createDynamoDBDataSource( + { + client: fakeClient({}), + }, + { introspection: { collections: [] } }, + ); + + await expect(factory(jest.fn(), jest.fn())).rejects.toThrow( + 'The introspection must have at least one collection', + ); + }); + + it('warns when nothing could be introspected', async () => { + const logger = jest.fn(); + + await createDynamoDBDataSource({ + client: fakeClient({ ListTables: () => ({ TableNames: [] }) }), + })(logger, jest.fn()); + + expect(logger).toHaveBeenCalledWith('Warn', 'No collections were introspected.'); + }); + + it('applies the datasource settings to its collections', async () => { + const client = fakeClient({ + ...handlers(), + Scan: () => ({ Items: [] }), + }); + + const dataSource = await createDynamoDBDataSource({ client, scanPolicy: 'deny' })( + jest.fn(), + jest.fn(), + ); + + await expect( + dataSource + .getCollection('Users') + .list({ timezone: 'UTC' } as never, { page: undefined } as never, ['id'] as never), + ).rejects.toThrow("refused by scanPolicy 'deny'"); + }); + + describe('native queries (PartiQL)', () => { + const build = (extra: Record = {}) => + createDynamoDBDataSource({ + client: fakeClient({ + ...handlers(), + ExecuteStatement: [{ Items: [{ id: 'a' }], NextToken: 'next' }, { Items: [{ id: 'b' }] }], + ...extra, + }), + tables: ['Users'], + liveQueryConnection: 'dynamo', + }); + + it('runs a statement with positional parameters and follows NextToken', async () => { + const client = fakeClient({ + ...handlers(), + ExecuteStatement: [{ Items: [{ id: 'a' }], NextToken: 'next' }, { Items: [{ id: 'b' }] }], + }); + const dataSource = await createDynamoDBDataSource({ + client, + tables: ['Users'], + liveQueryConnection: 'dynamo', + })(jest.fn(), jest.fn()); + + const rows = await dataSource.executeNativeQuery( + 'dynamo', + 'SELECT * FROM "Users" WHERE id = $id AND age > $age AND id <> $id', + { id: 'a', age: 3 }, + ); + + expect(rows).toEqual([{ id: 'a' }, { id: 'b' }]); + expect(client.calls('ExecuteStatement')).toEqual([ + { + Statement: 'SELECT * FROM "Users" WHERE id = ? AND age > ? AND id <> ?', + Parameters: ['a', 3, 'a'], + }, + { + Statement: 'SELECT * FROM "Users" WHERE id = ? AND age > ? AND id <> ?', + Parameters: ['a', 3, 'a'], + NextToken: 'next', + }, + ]); + }); + + it('refuses a placeholder without value', async () => { + const dataSource = await build()(jest.fn(), jest.fn()); + + await expect( + dataSource.executeNativeQuery('dynamo', 'SELECT * FROM "Users" WHERE id = $id', {}), + ).rejects.toThrow('No value was given for the placeholder $id'); + }); + + it('refuses an unknown connection', async () => { + const dataSource = await build()(jest.fn(), jest.fn()); + + await expect(dataSource.executeNativeQuery('nope', 'SELECT 1', {})).rejects.toThrow( + "Unknown connection name 'nope'", + ); + }); + + it('is not available unless a connection name is configured', async () => { + const dataSource = await createDynamoDBDataSource({ client: fakeClient(handlers()) })( + jest.fn(), + jest.fn(), + ); + + expect(Object.keys(dataSource.nativeQueryConnections)).toEqual([]); + }); + }); +}); + +describe('createDocumentClient', () => { + it('builds a client from region, endpoint and credentials', async () => { + const client = createDocumentClient({ + region: 'eu-west-1', + endpoint: 'http://localhost:8000', + credentials: { accessKeyId: 'k', secretAccessKey: 's' }, + }); + + expect(client).toBeInstanceOf(DynamoDBDocumentClient); + expect(await client.config.region()).toBe('eu-west-1'); + expect(await client.config.maxAttempts()).toBe(10); + }); + + it('lets the user tune the underlying client', async () => { + const client = createDocumentClient({ + region: 'eu-west-1', + clientConfig: { maxAttempts: 3 }, + }); + + expect(await client.config.maxAttempts()).toBe(3); + }); + + it('wraps an existing low-level client', () => { + const client = createDocumentClient({ client: new DynamoDBClient({ region: 'eu-west-1' }) }); + + expect(client).toBeInstanceOf(DynamoDBDocumentClient); + }); + + it('uses an existing document client as is', () => { + const existing = DynamoDBDocumentClient.from(new DynamoDBClient({ region: 'eu-west-1' })); + + expect(createDocumentClient({ client: existing })).toBe(existing); + }); +}); diff --git a/packages/datasource-dynamodb/test/fake-client.ts b/packages/datasource-dynamodb/test/fake-client.ts new file mode 100644 index 0000000000..7c7a709a50 --- /dev/null +++ b/packages/datasource-dynamodb/test/fake-client.ts @@ -0,0 +1,42 @@ +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; + +export type Handler = (input: Record, name: string) => unknown; + +export type FakeClient = DynamoDBDocumentClient & { + send: jest.Mock; + calls: (name?: string) => Array>; +}; + +/** A DocumentClient whose `send` is routed by command name to the given handlers. */ +export function fakeClient(handlers: Record): FakeClient { + const queues: Record = {}; + + const send = jest.fn(async (command: { constructor: { name: string }; input: object }) => { + const name = command.constructor.name.replace(/Command$/, ''); + const handler = handlers[name]; + + if (handler === undefined) throw new Error(`Unexpected command ${name}`); + + if (Array.isArray(handler)) { + queues[name] ??= [...handler]; + if (queues[name].length === 0) throw new Error(`No more response queued for ${name}`); + + const next = queues[name].length > 1 ? queues[name].shift() : queues[name][0]; + + return next instanceof Error ? Promise.reject(next) : next; + } + + return handler(command.input as Record, name); + }); + + const client = { send } as unknown as FakeClient; + client.calls = (name?: string) => + send.mock.calls + .map(([command]) => command) + .filter(command => !name || command.constructor.name === `${name}Command`) + .map(command => command.input); + + return client; +} + +export const silentLogger = jest.fn(); diff --git a/packages/datasource-dynamodb/test/fan-out-and-warnings.test.ts b/packages/datasource-dynamodb/test/fan-out-and-warnings.test.ts new file mode 100644 index 0000000000..baaf2d4800 --- /dev/null +++ b/packages/datasource-dynamodb/test/fan-out-and-warnings.test.ts @@ -0,0 +1,124 @@ +import { Aggregation, Filter, PaginatedFilter, Projection } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { buildModel, buildSimpleModel, leaf } from './helpers'; + +const caller = { timezone: 'UTC' } as never; +const order = (customerId: string, n: number) => ({ customerId, orderId: `o${n}`, total: n }); + +describe('an In on the partition key', () => { + it('reads each partition with a Query and merges the items', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: (input: any) => + input.ExpressionAttributeValues[':v0'] === 'c1' + ? { Items: [order('c1', 1), order('c1', 2)] } + : { Items: [order('c2', 3)] }, + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: leaf('customerId', 'In', ['c1', 'c2']), + page: { skip: 1, limit: 2 } as never, + }), + new Projection('orderId'), + ); + + expect(records).toEqual([{ orderId: 'o2' }, { orderId: 'o3' }]); + expect(client.calls('Query').map(call => call.ExpressionAttributeValues[':v0'])).toEqual([ + 'c1', + 'c2', + ]); + expect(client.calls('Scan')).toHaveLength(0); + }); + + it('counts the partitions without reading items', async () => { + const { collection, client } = buildCollection(buildModel(), { + Query: (input: any) => ({ Count: input.ExpressionAttributeValues[':v0'] === 'c1' ? 4 : 3 }), + }); + + const result = await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('customerId', 'In', ['c1', 'c2']) }), + new Aggregation({ operation: 'Count' }), + ); + + expect(result).toEqual([{ value: 7, group: {} }]); + expect(client.calls('Query').every(call => call.Select === 'COUNT')).toBe(true); + }); + + it('aggregates the items of every partition', async () => { + const { collection } = buildCollection(buildModel(), { + Query: (input: any) => ({ + Items: + input.ExpressionAttributeValues[':v0'] === 'c1' ? [order('c1', 1)] : [order('c2', 5)], + }), + }); + + const result = await collection.aggregate( + caller, + new Filter({ conditionTree: leaf('customerId', 'In', ['c1', 'c2']) }), + new Aggregation({ operation: 'Sum', field: 'total' }), + ); + + expect(result).toEqual([{ value: 6, group: {} }]); + }); +}); + +describe('scan warnings', () => { + const scanning = () => ({ Scan: () => ({ Items: [] }) }); + + it('does not warn for the first page of a collection read without a filter', async () => { + const { collection, logger } = buildCollection(buildSimpleModel(), scanning()); + + await collection.list( + caller, + new PaginatedFilter({ page: { skip: 0, limit: 15 } as never }), + new Projection('id'), + ); + + expect(logger).not.toHaveBeenCalledWith('Warn', expect.anything()); + }); + + it('still refuses that page under a deny policy only when it is deep', async () => { + const { collection } = buildCollection(buildSimpleModel({ scanPolicy: 'deny' }), scanning()); + + await expect( + collection.list( + caller, + new PaginatedFilter({ page: { skip: 0, limit: 15 } as never }), + new Projection('id'), + ), + ).resolves.toEqual([]); + await expect( + collection.list( + caller, + new PaginatedFilter({ page: { skip: 5000, limit: 15 } as never }), + new Projection('id'), + ), + ).rejects.toThrow("refused by scanPolicy 'deny'"); + }); + + it('warns once a minute for the same scan', async () => { + let now = 0; + const { collection, logger } = buildCollection(buildSimpleModel(), scanning(), { + now: () => now, + }); + const list = () => + collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('name', 'Equal', 'a') }), + new Projection('id'), + ); + const warnings = () => logger.mock.calls.filter(([level]) => level === 'Warn').length; + + await list(); + now = 59000; + await list(); + expect(warnings()).toBe(1); + + now = 61000; + await list(); + expect(warnings()).toBe(2); + }); +}); diff --git a/packages/datasource-dynamodb/test/helpers.ts b/packages/datasource-dynamodb/test/helpers.ts new file mode 100644 index 0000000000..9d84f144cc --- /dev/null +++ b/packages/datasource-dynamodb/test/helpers.ts @@ -0,0 +1,47 @@ +import type { TableModel } from '../src/types'; + +import { ConditionTreeFactory } from '@forestadmin/datasource-toolkit'; + +export const leaf = (field: string, operator: string, value?: unknown) => + ConditionTreeFactory.fromPlainObject({ field, operator, value } as never); + +export const and = (...conditions: unknown[]) => + ConditionTreeFactory.fromPlainObject({ aggregator: 'And', conditions } as never); + +export const or = (...conditions: unknown[]) => + ConditionTreeFactory.fromPlainObject({ aggregator: 'Or', conditions } as never); + +export const plain = (field: string, operator: string, value?: unknown) => ({ + field, + operator, + value, +}); + +export function buildModel(overrides: Partial = {}): TableModel { + return { + name: 'orders', + tableName: 'Orders', + partitionKey: { name: 'customerId', type: 'S' }, + sortKey: { name: 'orderId', type: 'S' }, + indexes: [], + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + status: { type: 'string', nullable: true }, + total: { type: 'number' }, + createdAt: { type: 'date' }, + }, + ...overrides, + }; +} + +export function buildSimpleModel(overrides: Partial = {}): TableModel { + return buildModel({ + name: 'users', + tableName: 'Users', + partitionKey: { name: 'id', type: 'S' }, + sortKey: undefined, + schema: { id: { type: 'string' }, name: { type: 'string' }, age: { type: 'number' } }, + ...overrides, + }); +} diff --git a/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts new file mode 100644 index 0000000000..27439aebdb --- /dev/null +++ b/packages/datasource-dynamodb/test/integration/dynamodb-local.test.ts @@ -0,0 +1,1130 @@ +import type { DataSource } from '@forestadmin/datasource-toolkit'; + +import { CreateTableCommand, DeleteTableCommand, DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { + BatchWriteCommand, + DeleteCommand, + DynamoDBDocumentClient, + GetCommand, + PutCommand, +} from '@aws-sdk/lib-dynamodb'; +import { + Aggregation, + ConditionTreeFactory, + Filter, + PaginatedFilter, + Projection, + Sort, +} from '@forestadmin/datasource-toolkit'; + +import { createDynamoDBDataSource, introspect } from '../../src'; + +/** + * Runs against DynamoDB Local (`docker compose up -d`, then DYNAMODB_ENDPOINT=http://localhost:8000): + * the unit tests use a fake client, which cannot tell whether DynamoDB accepts the expressions. + */ +const endpoint = process.env.DYNAMODB_ENDPOINT; +const run = endpoint ? describe : describe.skip; + +const caller = { timezone: 'UTC' } as never; +const suffix = `${Date.now()}`; +const ORDERS = `orders-${suffix}`; +const SHARED = `shared-${suffix}`; +const credentials = { accessKeyId: 'local', secretAccessKey: 'local' }; + +const tree = (plainTree: unknown) => ConditionTreeFactory.fromPlainObject(plainTree as never); +const leaf = (field: string, operator: string, value?: unknown) => tree({ field, operator, value }); + +run('DynamoDB Local', () => { + let client: DynamoDBClient; + let documents: DynamoDBDocumentClient; + let dataSource: DataSource; + let logger: jest.Mock; + + const orders = () => dataSource.getCollection('orders'); + + beforeAll(async () => { + client = new DynamoDBClient({ region: 'local', endpoint, credentials }); + documents = DynamoDBDocumentClient.from(client); + + await client.send( + new CreateTableCommand({ + TableName: ORDERS, + BillingMode: 'PAY_PER_REQUEST', + KeySchema: [ + { AttributeName: 'customerId', KeyType: 'HASH' }, + { AttributeName: 'orderId', KeyType: 'RANGE' }, + ], + AttributeDefinitions: [ + { AttributeName: 'customerId', AttributeType: 'S' }, + { AttributeName: 'orderId', AttributeType: 'S' }, + { AttributeName: 'status', AttributeType: 'S' }, + { AttributeName: 'createdAt', AttributeType: 'S' }, + ], + GlobalSecondaryIndexes: [ + { + IndexName: 'by-status', + KeySchema: [ + { AttributeName: 'status', KeyType: 'HASH' }, + { AttributeName: 'createdAt', KeyType: 'RANGE' }, + ], + Projection: { ProjectionType: 'ALL' }, + }, + ], + }), + ); + await client.send( + new CreateTableCommand({ + TableName: SHARED, + BillingMode: 'PAY_PER_REQUEST', + KeySchema: [ + { AttributeName: 'pk', KeyType: 'HASH' }, + { AttributeName: 'sk', KeyType: 'RANGE' }, + ], + AttributeDefinitions: [ + { AttributeName: 'pk', AttributeType: 'S' }, + { AttributeName: 'sk', AttributeType: 'S' }, + ], + }), + ); + + const items = Array.from({ length: 30 }, (_, i) => ({ + customerId: i < 20 ? 'c1' : 'c2', + orderId: `o${String(i).padStart(2, '0')}`, + status: i % 3 === 0 ? 'paid' : 'open', + total: i + 1, + createdAt: `2024-0${(i % 3) + 1}-15T10:00:00Z`, + tags: new Set(['a']), + ...(i === 0 ? { address: { city: 'Paris' } } : {}), + ...(i === 2 ? { meta: { items: [{ x: 1 }, { x: 2 }] } } : {}), + ...(i === 1 + ? { + lines: [ + { sku: 'A', qty: 1, parts: [{ n: 1 }, { n: 2 }] }, + { sku: 'B', qty: 2, parts: [{ n: 3 }] }, + ], + } + : {}), + })); + + for (let i = 0; i < items.length; i += 25) { + // eslint-disable-next-line no-await-in-loop + await documents.send( + new BatchWriteCommand({ + RequestItems: { + [ORDERS]: items.slice(i, i + 25).map(Item => ({ PutRequest: { Item } })), + }, + }), + ); + } + + await documents.send( + new BatchWriteCommand({ + RequestItems: { + [SHARED]: [ + { PutRequest: { Item: { pk: 'ACC#1', sk: 'USER#1', name: 'Ann' } } }, + { PutRequest: { Item: { pk: 'ACC#1', sk: 'USER#2', name: 'Bob' } } }, + { PutRequest: { Item: { pk: 'ACC#1', sk: 'ORDER#1', total: 5 } } }, + ], + }, + }), + ); + + logger = jest.fn(); + dataSource = await createDynamoDBDataSource({ + client: documents, + builder: configurator => + configurator + .addCollectionFromTable({ name: 'orders', tableName: ORDERS }) + .addCollectionFromTable({ + name: 'users', + tableName: SHARED, + entity: { sortKeyPrefix: 'USER#' }, + }) + .addCollectionFromTable({ + name: 'shared_orders', + tableName: SHARED, + entity: { sortKeyPrefix: 'ORDER#' }, + }), + virtualCollections: [ + { parentCollection: 'orders', collectionName: 'order_lines', arrayField: 'lines' }, + { parentCollection: 'order_lines', collectionName: 'line_parts', arrayField: 'parts' }, + { parentCollection: 'orders', collectionName: 'meta_items', arrayField: 'meta@@@items' }, + ], + liveQueryConnection: 'dynamo', + inMemoryProcessing: true, + chartCacheTtlMs: 0, + scanConcurrency: 2, + })(logger, jest.fn()); + }); + + afterAll(async () => { + for (const TableName of [ORDERS, SHARED]) { + // eslint-disable-next-line no-await-in-loop + await client.send(new DeleteTableCommand({ TableName })).catch(() => undefined); + } + }); + + describe('introspection', () => { + it('exposes the key as the primary key and infers the attributes', () => { + expect(orders().schema.fields).toMatchObject({ + customerId: { isPrimaryKey: true, columnType: 'String' }, + orderId: { isPrimaryKey: true }, + total: { columnType: 'Number' }, + createdAt: { columnType: 'Date' }, + status: { columnType: 'String' }, + }); + }); + }); + + describe('list', () => { + it('queries a partition in sort key order', async () => { + const records = await orders().list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c1') }), + new Projection('orderId'), + ); + + expect(records).toHaveLength(20); + expect(records[0].orderId).toBe('o00'); + expect(records[19].orderId).toBe('o19'); + }); + + it('reads backwards and pages through a sort key range', async () => { + const records = await orders().list( + caller, + new PaginatedFilter({ + conditionTree: tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c1' }, + { field: 'orderId', operator: 'GreaterThan', value: 'o02' }, + { field: 'orderId', operator: 'LessThan', value: 'o15' }, + ], + }), + sort: new Sort({ field: 'orderId', ascending: false }), + page: { skip: 2, limit: 3 } as never, + }), + new Projection('orderId'), + ); + + expect(records.map(r => r.orderId)).toEqual(['o12', 'o11', 'o10']); + }); + + it('queries a global secondary index when a condition on its sort key is given', async () => { + logger.mockClear(); + + const records = await orders().list( + caller, + new PaginatedFilter({ + conditionTree: tree({ + aggregator: 'And', + conditions: [ + { field: 'status', operator: 'Equal', value: 'paid' }, + { field: 'createdAt', operator: 'GreaterThan', value: '2000' }, + ], + }), + }), + new Projection('orderId'), + ); + + expect(records).toHaveLength(10); + expect(logger).not.toHaveBeenCalledWith('Warn', expect.stringContaining('Scan on table')); + }); + + it('does not let an index hide the items that lack its sort key', async () => { + await documents.send( + new PutCommand({ + TableName: ORDERS, + Item: { customerId: 'c3', orderId: 'o90', status: 'paid', total: 1 }, + }), + ); + + const records = await orders().list( + caller, + new PaginatedFilter({ conditionTree: leaf('status', 'Equal', 'paid') }), + new Projection('orderId'), + ); + await documents.send( + new DeleteCommand({ TableName: ORDERS, Key: { customerId: 'c3', orderId: 'o90' } }), + ); + + expect(records.map(r => r.orderId)).toContain('o90'); + expect(records).toHaveLength(11); + }); + + it('scans, with a warning, for a filter no key serves', async () => { + const records = await orders().list( + caller, + new PaginatedFilter({ conditionTree: leaf('total', 'GreaterThan', 28) }), + new Projection('orderId', 'total'), + ); + + expect(records.map(r => r.total).sort()).toEqual([29, 30]); + expect(logger).toHaveBeenCalledWith('Warn', expect.stringContaining('Scan on table')); + }); + + it('evaluates operators DynamoDB lacks on the returned items', async () => { + const records = await orders().list( + caller, + new PaginatedFilter({ conditionTree: leaf('orderId', 'ILike', '%O1_') }), + new Projection('orderId'), + ); + + expect(records.map(r => r.orderId).sort()).toEqual( + Array.from({ length: 10 }, (_, i) => `o1${i}`), + ); + }); + + it('sorts on a non-key attribute', async () => { + const records = await orders().list( + caller, + new PaginatedFilter({ + sort: new Sort({ field: 'total', ascending: false }), + page: { skip: 0, limit: 2 } as never, + }), + new Projection('total'), + ); + + expect(records.map(r => r.total)).toEqual([30, 29]); + }); + + it('reads a global secondary index in its sort order', async () => { + const records = await orders().list( + caller, + new PaginatedFilter({ + conditionTree: leaf('status', 'Equal', 'open'), + sort: new Sort({ field: 'createdAt', ascending: false }), + page: { skip: 0, limit: 3 } as never, + }), + new Projection('createdAt'), + ); + + expect(records.map(r => r.createdAt)).toEqual([ + '2024-03-15T10:00:00Z', + '2024-03-15T10:00:00Z', + '2024-03-15T10:00:00Z', + ]); + }); + + it('pages through a scan, resuming from the keys it remembers', async () => { + const page = async (skip: number) => + ( + await orders().list( + caller, + new PaginatedFilter({ + conditionTree: leaf('total', 'GreaterThan', 4), + page: { skip, limit: 7 } as never, + }), + new Projection('orderId'), + ) + ).map(r => r.orderId); + + const pages = [await page(0), await page(7), await page(14), await page(21), await page(7)]; + + expect(pages.slice(0, 4).flat()).toHaveLength(26); + expect(new Set(pages.slice(0, 4).flat()).size).toBe(26); + expect(pages[4]).toEqual(pages[1]); + }); + + it('keeps a sort key bound out of the filter expression, where DynamoDB forbids it', async () => { + const records = await dataSource.getCollection('users').list( + caller, + new PaginatedFilter({ + conditionTree: tree({ + aggregator: 'And', + conditions: [ + { field: 'pk', operator: 'Equal', value: 'ACC#1' }, + { field: 'sk', operator: 'GreaterThan', value: 'USER#1' }, + ], + }), + }), + new Projection('sk'), + ); + + expect(records.map(r => r.sk)).toEqual(['USER#2']); + }); + + it('queries each partition of an In on the partition key', async () => { + logger.mockClear(); + + const records = await orders().list( + caller, + new PaginatedFilter({ + conditionTree: leaf('customerId', 'In', ['c1', 'c2']), + page: { skip: 18, limit: 4 } as never, + }), + new Projection('customerId', 'orderId'), + ); + const count = await orders().aggregate( + caller, + new Filter({ conditionTree: leaf('customerId', 'In', ['c1', 'c2']) }), + new Aggregation({ operation: 'Count' }), + ); + + expect(records.map(r => `${r.customerId}${r.orderId}`)).toEqual([ + 'c1o18', + 'c1o19', + 'c2o20', + 'c2o21', + ]); + expect(count).toEqual([{ value: 30, group: {} }]); + expect(logger).not.toHaveBeenCalledWith('Warn', expect.stringContaining('Scan on table')); + }); + + it('reads whole keys with BatchGetItem', async () => { + const records = await orders().list( + caller, + new PaginatedFilter({ + conditionTree: tree({ + aggregator: 'Or', + conditions: [ + { + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c1' }, + { field: 'orderId', operator: 'Equal', value: 'o03' }, + ], + }, + { + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c2' }, + { field: 'orderId', operator: 'Equal', value: 'o25' }, + ], + }, + ], + }), + }), + new Projection('orderId', 'total'), + ); + + expect(records.map(r => r.orderId).sort()).toEqual(['o03', 'o25']); + }); + }); + + describe('aggregate', () => { + it('counts with a parallel scan', async () => { + const result = await orders().aggregate( + caller, + new Filter({}), + new Aggregation({ operation: 'Count' }), + ); + + expect(result).toEqual([{ value: 30, group: {} }]); + }); + + it('counts a filtered partition', async () => { + const result = await orders().aggregate( + caller, + new Filter({ conditionTree: leaf('customerId', 'Equal', 'c2') }), + new Aggregation({ operation: 'Count' }), + ); + + expect(result).toEqual([{ value: 10, group: {} }]); + }); + + it('sums by group and by month', async () => { + const byStatus = await orders().aggregate( + caller, + new Filter({}), + new Aggregation({ operation: 'Sum', field: 'total', groups: [{ field: 'status' }] }), + ); + const byMonth = await orders().aggregate( + caller, + new Filter({}), + new Aggregation({ + operation: 'Count', + groups: [{ field: 'createdAt', operation: 'Month' }], + }), + ); + + expect(byStatus.reduce((sum, row) => sum + (row.value as number), 0)).toBe(465); + expect(byMonth.map(row => row.group.createdAt).sort()).toEqual([ + '2024-01-01', + '2024-02-01', + '2024-03-01', + ]); + }); + }); + + describe('write', () => { + it('creates, updates (also under a map that does not exist yet) and deletes', async () => { + const key = { customerId: 'c9', orderId: 'o99' }; + await orders().create(caller, [{ ...key, total: 1, status: 'open' }]); + + await orders().update( + caller, + new Filter({ + conditionTree: tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c9' }, + { field: 'orderId', operator: 'Equal', value: 'o99' }, + ], + }), + }), + { total: 2, 'address@@@city': 'Lyon' }, + ); + + const [updated] = await orders().list( + caller, + new PaginatedFilter({ conditionTree: leaf('customerId', 'Equal', 'c9') }), + new Projection('total', 'address@@@city'), + ); + expect(updated).toEqual({ total: 2, 'address@@@city': 'Lyon' }); + + await expect(orders().create(caller, [{ ...key, total: 3 }])).rejects.toThrow( + 'already exists', + ); + + await orders().delete( + caller, + new Filter({ conditionTree: leaf('customerId', 'Equal', 'c9') }), + ); + const remaining = await orders().aggregate( + caller, + new Filter({ conditionTree: leaf('customerId', 'Equal', 'c9') }), + new Aggregation({ operation: 'Count' }), + ); + expect(remaining).toEqual([{ value: 0, group: {} }]); + }); + }); + + describe('single-table design', () => { + it('keeps each collection to its own items', async () => { + const users = await dataSource + .getCollection('users') + .list( + caller, + new PaginatedFilter({ conditionTree: leaf('pk', 'Equal', 'ACC#1') }), + new Projection('sk', 'name'), + ); + const sharedOrders = await dataSource + .getCollection('shared_orders') + .list( + caller, + new PaginatedFilter({ conditionTree: leaf('pk', 'Equal', 'ACC#1') }), + new Projection('sk', 'total'), + ); + + expect(users.map(u => u.sk)).toEqual(['USER#1', 'USER#2']); + expect(sharedOrders).toEqual([{ sk: 'ORDER#1', total: 5 }]); + }); + }); + + describe('virtual array collection', () => { + it('lists, updates, appends to and deletes the items of an array inside an array', async () => { + const parts = dataSource.getCollection('line_parts'); + const order = tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c1' }, + { field: 'orderId', operator: 'Equal', value: 'o01' }, + ], + }); + const read = () => + parts.list( + caller, + new PaginatedFilter({ conditionTree: order }), + new Projection('order_linesIndex', 'index', 'n'), + ); + + expect(await read()).toEqual([ + { order_linesIndex: 0, index: 0, n: 1 }, + { order_linesIndex: 0, index: 1, n: 2 }, + { order_linesIndex: 1, index: 0, n: 3 }, + ]); + + await parts.update(caller, new Filter({ conditionTree: leaf('n', 'Equal', 3) }), { n: 9 }); + const created = await parts.create(caller, [ + { customerId: 'c1', orderId: 'o01', order_linesIndex: 1, n: 4 }, + ]); + await parts.delete( + caller, + new Filter({ + conditionTree: ConditionTreeFactory.intersect(order, leaf('n', 'Equal', 1)), + }), + ); + + expect(created).toEqual([ + { customerId: 'c1', orderId: 'o01', order_linesIndex: 1, index: 1, n: 4 }, + ]); + expect(await read()).toEqual([ + { order_linesIndex: 0, index: 0, n: 2 }, + { order_linesIndex: 1, index: 0, n: 9 }, + { order_linesIndex: 1, index: 1, n: 4 }, + ]); + }); + + it('exposes an array held in a nested map', async () => { + const items = dataSource.getCollection('meta_items'); + const order = tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c1' }, + { field: 'orderId', operator: 'Equal', value: 'o02' }, + ], + }); + const read = () => + items.list( + caller, + new PaginatedFilter({ conditionTree: order }), + new Projection('index', 'x'), + ); + + await items.update(caller, new Filter({ conditionTree: leaf('x', 'Equal', 2) }), { x: 7 }); + const [created] = await items.create(caller, [{ customerId: 'c1', orderId: 'o02', x: 3 }]); + await items.delete( + caller, + new Filter({ conditionTree: ConditionTreeFactory.intersect(order, leaf('x', 'Equal', 1)) }), + ); + + expect(created.index).toBe(2); + expect(await read()).toEqual([ + { index: 0, x: 7 }, + { index: 1, x: 3 }, + ]); + }); + + it('lists, updates and deletes the items of an array', async () => { + const lines = dataSource.getCollection('order_lines'); + const parent = ConditionTreeFactory.fromPlainObject({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c1' }, + { field: 'orderId', operator: 'Equal', value: 'o01' }, + ], + } as never); + + const before = await lines.list( + caller, + new PaginatedFilter({ conditionTree: parent }), + new Projection('index', 'sku', 'qty'), + ); + expect(before).toEqual([ + { index: 0, sku: 'A', qty: 1 }, + { index: 1, sku: 'B', qty: 2 }, + ]); + + await lines.update(caller, new Filter({ conditionTree: parent }), { qty: 7 }); + await lines.create(caller, [{ customerId: 'c1', orderId: 'o01', sku: 'C', qty: 3 }]); + await lines.delete( + caller, + new Filter({ + conditionTree: ConditionTreeFactory.intersect(parent, leaf('sku', 'Equal', 'A')), + }), + ); + + const after = await lines.list( + caller, + new PaginatedFilter({ conditionTree: parent }), + new Projection('index', 'sku', 'qty'), + ); + expect(after).toEqual([ + { index: 0, sku: 'B', qty: 7 }, + { index: 1, sku: 'C', qty: 3 }, + ]); + }); + }); + + describe('manual schema', () => { + const MANUAL = `manual-${suffix}`; + let manual: DataSource; + let manualLogger: jest.Mock; + + beforeAll(async () => { + await client.send( + new CreateTableCommand({ + TableName: MANUAL, + BillingMode: 'PAY_PER_REQUEST', + KeySchema: [ + { AttributeName: 'customerId', KeyType: 'HASH' }, + { AttributeName: 'orderId', KeyType: 'RANGE' }, + ], + AttributeDefinitions: [ + { AttributeName: 'customerId', AttributeType: 'S' }, + { AttributeName: 'orderId', AttributeType: 'S' }, + { AttributeName: 'status', AttributeType: 'S' }, + { AttributeName: 'createdAt', AttributeType: 'S' }, + ], + GlobalSecondaryIndexes: [ + { + IndexName: 'by-status', + KeySchema: [ + { AttributeName: 'status', KeyType: 'HASH' }, + { AttributeName: 'createdAt', KeyType: 'RANGE' }, + ], + Projection: { ProjectionType: 'ALL' }, + }, + ], + }), + ); + + await documents.send( + new BatchWriteCommand({ + RequestItems: { + [MANUAL]: Array.from({ length: 6 }, (_, i) => ({ + PutRequest: { + Item: { + customerId: 'c1', + orderId: `o${i}`, + status: i % 2 ? 'paid' : 'open', + total: i + 1, + createdAt: `2024-01-0${i + 1}T00:00:00Z`, + shipping: { city: i % 2 ? 'Paris' : 'Austin' }, + }, + }, + })), + }, + }), + ); + + manualLogger = jest.fn(); + manual = await createDynamoDBDataSource( + { + client: documents, + + chartCacheTtlMs: 0, + }, + { + introspection: { + collections: [ + { + name: 'manual_orders', + tableName: MANUAL, + partitionKey: { name: 'customerId', type: 'string' }, + sortKey: { name: 'orderId', type: 'string' }, + indexes: [ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'string' }, + sortKey: { name: 'createdAt', type: 'string' }, + }, + ], + fields: [ + { name: 'total', type: 'number' }, + { name: 'status', type: 'string' }, + { name: 'createdAt', type: 'date' }, + { name: 'blob', type: 'binary', nullable: true }, + { name: 'labels', type: 'array', subType: 'string', set: true, nullable: true }, + { name: 'shipping', type: 'object', fields: [{ name: 'city', type: 'string' }] }, + ], + }, + ], + }, + }, + )(manualLogger, jest.fn()); + }); + + afterAll(async () => { + await client.send(new DeleteTableCommand({ TableName: MANUAL })).catch(() => undefined); + }); + + it('declares the keys, the typed attributes and the flattened nested ones', () => { + expect(manual.getCollection('manual_orders').schema.fields).toMatchObject({ + customerId: { isPrimaryKey: true, columnType: 'String' }, + orderId: { isPrimaryKey: true, columnType: 'String' }, + total: { columnType: 'Number' }, + status: { columnType: 'String' }, + createdAt: { columnType: 'Date' }, + 'shipping@@@city': { columnType: 'String' }, + }); + }); + + it('exposes only the declared collections', () => { + expect(manual.collections.map(c => c.name)).toEqual(['manual_orders']); + }); + + it('reads the declared index and filters on a nested attribute', async () => { + manualLogger.mockClear(); + + const records = await manual.getCollection('manual_orders').list( + caller, + new PaginatedFilter({ + conditionTree: tree({ + aggregator: 'And', + conditions: [ + { field: 'status', operator: 'Equal', value: 'paid' }, + { field: 'createdAt', operator: 'GreaterThan', value: '2000-01-01T00:00:00Z' }, + { field: 'shipping@@@city', operator: 'Equal', value: 'Paris' }, + ], + }), + }), + new Projection('orderId', 'shipping@@@city'), + ); + + expect(records.map(r => r.orderId).sort()).toEqual(['o1', 'o3', 'o5']); + expect(manualLogger).not.toHaveBeenCalledWith( + 'Warn', + expect.stringContaining('Scan on table'), + ); + }); + + it('stores a binary attribute as bytes and reads it back as base64', async () => { + const collection = manual.getCollection('manual_orders'); + + await collection.create(caller, [{ customerId: 'c8', orderId: 'o1', blob: 'AQID' }]); + + const stored = await documents.send( + new GetCommand({ TableName: MANUAL, Key: { customerId: 'c8', orderId: 'o1' } }), + ); + expect(Array.from(stored.Item.blob as Uint8Array)).toEqual([1, 2, 3]); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('blob', 'Equal', 'AQID') }), + new Projection('orderId', 'blob'), + ); + expect(records).toEqual([{ orderId: 'o1', blob: 'AQID' }]); + }); + + it('writes a declared set back as a DynamoDB set', async () => { + const collection = manual.getCollection('manual_orders'); + const key = { customerId: 'c7', orderId: 'o1' }; + + await collection.create(caller, [{ ...key, labels: ['a', 'b'] }]); + await collection.update( + caller, + new Filter({ + conditionTree: tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c7' }, + { field: 'orderId', operator: 'Equal', value: 'o1' }, + ], + }), + }), + { labels: ['b', 'c'] }, + ); + + const stored = await documents.send(new GetCommand({ TableName: MANUAL, Key: key })); + expect(stored.Item.labels).toBeInstanceOf(Set); + expect([...stored.Item.labels].sort()).toEqual(['b', 'c']); + + await collection.update( + caller, + new Filter({ + conditionTree: tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c7' }, + { field: 'orderId', operator: 'Equal', value: 'o1' }, + ], + }), + }), + { labels: [] }, + ); + + const cleared = await documents.send(new GetCommand({ TableName: MANUAL, Key: key })); + expect(cleared.Item).toEqual(key); + }); + + it('creates, updates and deletes through the declared schema', async () => { + const collection = manual.getCollection('manual_orders'); + const key = tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c9' }, + { field: 'orderId', operator: 'Equal', value: 'o1' }, + ], + }); + + await collection.create(caller, [ + { customerId: 'c9', orderId: 'o1', total: 5, 'shipping@@@city': 'Rome' }, + ]); + await collection.update(caller, new Filter({ conditionTree: key }), { + 'shipping@@@city': 'Milan', + }); + + expect( + await collection.list( + caller, + new PaginatedFilter({ conditionTree: key }), + new Projection('total', 'shipping@@@city'), + ), + ).toEqual([{ total: 5, 'shipping@@@city': 'Milan' }]); + + await collection.delete(caller, new Filter({ conditionTree: key })); + + expect( + await collection.list( + caller, + new PaginatedFilter({ conditionTree: key }), + new Projection('total'), + ), + ).toEqual([]); + }); + }); + + describe('writes through virtual collections of a composite key table', () => { + const VC = `vc-${suffix}`; + let vc: DataSource; + + const original = () => ({ + 'c1|o1': { + customerId: 'c1', + orderId: 'o1', + status: 'open', + note: 'keep', + lines: [ + { sku: 'A', qty: 1, parts: [{ n: 1 }, { n: 2 }] }, + { sku: 'B', qty: 2, parts: [{ n: 3 }] }, + ], + }, + 'c1|o2': { + customerId: 'c1', + orderId: 'o2', + lines: [{ sku: 'A', qty: 10, parts: [{ n: 1 }] }], + }, + 'c2|o1': { customerId: 'c2', orderId: 'o1', lines: [{ sku: 'A', qty: 100, parts: [] }] }, + }); + + const keyOf = (id: string) => { + const [customerId, orderId] = id.split('|'); + + return { customerId, orderId }; + }; + + const raw = async (id: string) => + (await documents.send(new GetCommand({ TableName: VC, Key: keyOf(id) }))).Item; + + const at = (id: string, index: number, extra: Record = {}) => + tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: keyOf(id).customerId }, + { field: 'orderId', operator: 'Equal', value: keyOf(id).orderId }, + { field: 'index', operator: 'Equal', value: index }, + ...Object.entries(extra).map(([field, value]) => ({ + field, + operator: 'Equal', + value, + })), + ], + }); + + beforeAll(async () => { + await client.send( + new CreateTableCommand({ + TableName: VC, + BillingMode: 'PAY_PER_REQUEST', + KeySchema: [ + { AttributeName: 'customerId', KeyType: 'HASH' }, + { AttributeName: 'orderId', KeyType: 'RANGE' }, + ], + AttributeDefinitions: [ + { AttributeName: 'customerId', AttributeType: 'S' }, + { AttributeName: 'orderId', AttributeType: 'S' }, + ], + }), + ); + vc = await createDynamoDBDataSource({ + client: documents, + builder: configurator => + configurator.addCollectionFromTable({ name: 'vc_orders', tableName: VC }), + virtualCollections: [ + { parentCollection: 'vc_orders', collectionName: 'order_lines', arrayField: 'lines' }, + { parentCollection: 'order_lines', collectionName: 'line_parts', arrayField: 'parts' }, + ], + chartCacheTtlMs: 0, + })(jest.fn(), jest.fn()); + }); + + beforeEach(async () => { + await documents.send( + new BatchWriteCommand({ + RequestItems: { + [VC]: Object.values(original()).map(Item => ({ PutRequest: { Item } })), + }, + }), + ); + }); + + afterAll(async () => { + await client.send(new DeleteTableCommand({ TableName: VC })).catch(() => undefined); + }); + + it('edits the line of one order and leaves every other item and attribute alone', async () => { + await vc + .getCollection('order_lines') + .update(caller, new Filter({ conditionTree: at('c1|o1', 1) }), { qty: 20 }); + + const expected = original(); + expected['c1|o1'].lines[1].qty = 20; + + expect(await raw('c1|o1')).toEqual(expected['c1|o1']); + expect(await raw('c1|o2')).toEqual(expected['c1|o2']); + expect(await raw('c2|o1')).toEqual(expected['c2|o1']); + }); + + it('tells apart two items that share the sort key, or the partition key', async () => { + await vc + .getCollection('order_lines') + .update(caller, new Filter({ conditionTree: at('c2|o1', 0) }), { sku: 'Z' }); + + const expected = original(); + expected['c2|o1'].lines[0].sku = 'Z'; + + expect(await raw('c2|o1')).toEqual(expected['c2|o1']); + expect(await raw('c1|o1')).toEqual(expected['c1|o1']); + expect(await raw('c1|o2')).toEqual(expected['c1|o2']); + }); + + it('edits the element of an array inside an array', async () => { + await vc + .getCollection('line_parts') + .update(caller, new Filter({ conditionTree: at('c1|o1', 1, { order_linesIndex: 0 }) }), { + n: 99, + }); + + const expected = original(); + expected['c1|o1'].lines[0].parts[1].n = 99; + + expect(await raw('c1|o1')).toEqual(expected['c1|o1']); + expect(await raw('c1|o2')).toEqual(expected['c1|o2']); + }); + + it('deletes the line it names and keeps the order and its other lines', async () => { + await vc + .getCollection('order_lines') + .delete(caller, new Filter({ conditionTree: at('c1|o1', 0) })); + + const expected = original(); + expected['c1|o1'].lines.splice(0, 1); + + expect(await raw('c1|o1')).toEqual(expected['c1|o1']); + expect(await raw('c1|o2')).toEqual(expected['c1|o2']); + expect(await raw('c2|o1')).toEqual(expected['c2|o1']); + }); + + it('appends a line to the order its keys name', async () => { + const [created] = await vc + .getCollection('order_lines') + .create(caller, [{ customerId: 'c1', orderId: 'o2', sku: 'N', qty: 5 }]); + + const expected = original(); + expected['c1|o2'].lines.push({ sku: 'N', qty: 5 } as never); + + expect(created).toMatchObject({ customerId: 'c1', orderId: 'o2', index: 1 }); + expect(await raw('c1|o2')).toEqual(expected['c1|o2']); + expect(await raw('c1|o1')).toEqual(expected['c1|o1']); + expect(await raw('c2|o1')).toEqual(expected['c2|o1']); + }); + }); + + describe('introspect', () => { + it('describes a table so that a datasource starts from it without sampling', async () => { + const introspection = await introspect({ client: documents, tables: [ORDERS] }); + + expect(JSON.parse(JSON.stringify(introspection))).toEqual(introspection); + expect(introspection.collections[0]).toMatchObject({ + tableName: ORDERS, + partitionKey: { name: 'customerId', type: 'string' }, + sortKey: { name: 'orderId', type: 'string' }, + indexes: [{ name: 'by-status', type: 'GSI' }], + }); + + const offline = await createDynamoDBDataSource({ client: documents }, { introspection })( + jest.fn(), + jest.fn(), + ); + const records = await offline.getCollection(ORDERS).list( + caller, + new PaginatedFilter({ + conditionTree: tree({ + aggregator: 'And', + conditions: [ + { field: 'customerId', operator: 'Equal', value: 'c1' }, + { field: 'orderId', operator: 'Equal', value: 'o05' }, + ], + }), + }), + new Projection('customerId', 'orderId'), + ); + + expect(records).toEqual([{ customerId: 'c1', orderId: 'o05' }]); + }); + + it('keeps introspecting the other tables when the builder adjusts one', async () => { + const adjusted = await createDynamoDBDataSource({ + client: documents, + tables: [ORDERS], + builder: configurator => + configurator.addCollectionFromTable({ + name: 'shared_users', + tableName: SHARED, + entity: { sortKeyPrefix: 'USER#' }, + }), + })(jest.fn(), jest.fn()); + + expect(adjusted.collections.map(c => c.name).sort()).toEqual([ORDERS, 'shared_users'].sort()); + }); + }); + + describe('Present and Blank', () => { + const PB = `pb-${suffix}`; + + beforeAll(async () => { + await client.send( + new CreateTableCommand({ + TableName: PB, + BillingMode: 'PAY_PER_REQUEST', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }), + ); + + for (const Item of [ + { id: 'a', name: 'x' }, + { id: 'b' }, + { id: 'c', name: null }, + { id: 'd', name: '' }, + ]) { + // eslint-disable-next-line no-await-in-loop + await documents.send(new PutCommand({ TableName: PB, Item })); + } + }); + + afterAll(async () => { + await client.send(new DeleteTableCommand({ TableName: PB })).catch(() => undefined); + }); + + const idsFor = async (operator: string) => { + const source = await createDynamoDBDataSource({ client: documents, tables: [PB] })( + jest.fn(), + jest.fn(), + ); + const records = await source + .getCollection(PB) + .list( + caller, + new PaginatedFilter({ conditionTree: leaf('name', operator) }), + new Projection('id'), + ); + + return records.map(record => record.id).sort(); + }; + + it('keeps the items that hold a value, as DynamoDB evaluates it', async () => { + expect(await idsFor('Present')).toEqual(['a']); + }); + + it('keeps the items without a value, a null or an empty string', async () => { + expect(await idsFor('Blank')).toEqual(['b', 'c', 'd']); + }); + }); + + describe('native query', () => { + it('runs a PartiQL statement with parameters', async () => { + const rows = await dataSource.executeNativeQuery( + 'dynamo', + `SELECT orderId FROM "${ORDERS}" WHERE customerId = $customer AND orderId = $order`, + { customer: 'c1', order: 'o05' }, + ); + + expect(rows).toEqual([{ orderId: 'o05' }]); + }); + }); +}); diff --git a/packages/datasource-dynamodb/test/introspection-api.test.ts b/packages/datasource-dynamodb/test/introspection-api.test.ts new file mode 100644 index 0000000000..2556260a7f --- /dev/null +++ b/packages/datasource-dynamodb/test/introspection-api.test.ts @@ -0,0 +1,228 @@ +import { fakeClient } from './fake-client'; +import { createDynamoDBDataSource, introspect } from '../src'; +import { convertManualSchema } from '../src/schema/manual-schema'; + +const ordersTable = { + Table: { + TableName: 'Orders', + KeySchema: [ + { AttributeName: 'customerId', KeyType: 'HASH' }, + { AttributeName: 'orderId', KeyType: 'RANGE' }, + ], + AttributeDefinitions: [ + { AttributeName: 'customerId', AttributeType: 'S' }, + { AttributeName: 'orderId', AttributeType: 'S' }, + { AttributeName: 'status', AttributeType: 'S' }, + { AttributeName: 'createdAt', AttributeType: 'S' }, + ], + GlobalSecondaryIndexes: [ + { + IndexName: 'by-status', + KeySchema: [ + { AttributeName: 'status', KeyType: 'HASH' }, + { AttributeName: 'createdAt', KeyType: 'RANGE' }, + ], + Projection: { ProjectionType: 'ALL' }, + }, + ], + }, +}; + +const orders = [ + { + customerId: 'c1', + orderId: 'o1', + status: 'paid', + total: 12, + createdAt: '2024-01-15T10:00:00Z', + tags: new Set(['a']), + address: { city: 'Paris', geo: { lat: 1 } }, + lines: [{ sku: 'A' }], + }, + { customerId: 'c1', orderId: 'o2', status: null, total: 3, createdAt: '2024-01-16T10:00:00Z' }, +]; + +const awsClient = () => + fakeClient({ + ListTables: () => ({ TableNames: ['Orders'] }), + DescribeTable: () => ordersTable, + Scan: () => ({ Items: orders }), + }); + +const appTable = (name: string) => ({ + Table: { + TableName: name, + KeySchema: [ + { AttributeName: 'pk', KeyType: 'HASH' }, + { AttributeName: 'sk', KeyType: 'RANGE' }, + ], + AttributeDefinitions: [ + { AttributeName: 'pk', AttributeType: 'S' }, + { AttributeName: 'sk', AttributeType: 'S' }, + ], + }, +}); + +describe('introspect', () => { + it('describes each table as a collection definition', async () => { + const introspection = await introspect({ client: awsClient() }); + + expect(introspection.version).toBe(1); + expect(introspection.collections).toHaveLength(1); + expect(introspection.collections[0]).toMatchObject({ + name: 'Orders', + tableName: 'Orders', + partitionKey: { name: 'customerId', type: 'string' }, + sortKey: { name: 'orderId', type: 'string' }, + indexes: [ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'string' }, + sortKey: { name: 'createdAt', type: 'string' }, + }, + ], + }); + expect(introspection.collections[0].fields).toEqual( + expect.arrayContaining([ + { name: 'total', type: 'number', nullable: false }, + { name: 'tags', type: 'array', set: true, nullable: true }, + { + name: 'address', + type: 'object', + fields: [ + { name: 'city', type: 'string', nullable: true }, + { + name: 'geo', + type: 'object', + fields: [{ name: 'lat', type: 'number', nullable: true }], + }, + ], + }, + ]), + ); + expect(introspection.collections[0].fields.map(field => field.name)).not.toContain('orderId'); + }); + + it('survives a trip through JSON', async () => { + const introspection = await introspect({ client: awsClient() }); + + expect(JSON.parse(JSON.stringify(introspection))).toEqual(introspection); + }); + + it('builds the same collections as the live introspection, without calling AWS', async () => { + const live = await createDynamoDBDataSource({ client: awsClient() })(jest.fn(), jest.fn()); + const introspection = await introspect({ client: awsClient() }); + const offline = fakeClient({}); + + const dataSource = await createDynamoDBDataSource({ client: offline }, { introspection })( + jest.fn(), + jest.fn(), + ); + + expect(dataSource.getCollection('Orders').schema).toEqual(live.getCollection('Orders').schema); + expect(offline.send).not.toHaveBeenCalled(); + }); + + it('refuses an introspection of another version', async () => { + const factory = createDynamoDBDataSource( + { client: fakeClient({}) }, + { introspection: { version: 2, collections: [] } as never }, + ); + + await expect(factory(jest.fn(), jest.fn())).rejects.toThrow( + 'The introspection is in version 2, this datasource reads version 1', + ); + }); + + it('refuses an introspection with a builder', async () => { + const factory = createDynamoDBDataSource( + { client: fakeClient({}), builder: configurator => configurator }, + { introspection: { version: 1, collections: [] } }, + ); + + await expect(factory(jest.fn(), jest.fn())).rejects.toThrow( + 'An introspection and a builder cannot be used together', + ); + }); +}); + +describe('the builder next to the introspection', () => { + const client = () => + fakeClient({ + ListTables: () => ({ TableNames: ['App', 'Other'] }), + DescribeTable: (input: { TableName: string }) => appTable(input.TableName), + Scan: () => ({ Items: [] }), + }); + const twoEntities = (configurator: any) => + configurator + .addCollectionFromTable({ + name: 'users', + tableName: 'App', + entity: { sortKeyPrefix: 'USER#' }, + }) + .addCollectionFromTable({ + name: 'orders', + tableName: 'App', + entity: { sortKeyPrefix: 'ORDER#' }, + }); + + it('keeps introspecting the tables it does not name', async () => { + const dataSource = await createDynamoDBDataSource({ + client: client(), + builder: twoEntities, + })(jest.fn(), jest.fn()); + + expect(dataSource.collections.map(c => c.name)).toEqual(['Other', 'orders', 'users']); + }); + + it('introspects the table the builder names only once', async () => { + const aws = client(); + + await createDynamoDBDataSource({ client: aws, builder: twoEntities })(jest.fn(), jest.fn()); + + const described = aws.calls('DescribeTable').map(input => input.TableName); + expect(described.filter(name => name === 'App')).toHaveLength(2); + expect(described.filter(name => name === 'Other')).toHaveLength(1); + }); + + it('exposes only the builder collections when no table is listed', async () => { + const aws = client(); + + const dataSource = await createDynamoDBDataSource({ + client: aws, + tables: [], + builder: twoEntities, + })(jest.fn(), jest.fn()); + + expect(dataSource.collections.map(c => c.name)).toEqual(['orders', 'users']); + expect(aws.calls('ListTables')).toHaveLength(0); + }); + + it('introspects the listed tables next to the builder collections', async () => { + const dataSource = await createDynamoDBDataSource({ + client: client(), + tables: ['Other'], + builder: twoEntities, + })(jest.fn(), jest.fn()); + + expect(dataSource.collections.map(c => c.name)).toEqual(['Other', 'orders', 'users']); + }); +}); + +describe('an object field without nested fields', () => { + it('is a column of its own', () => { + const [table] = convertManualSchema({ + collections: [ + { + name: 'a', + tableName: 'A', + partitionKey: { name: 'id', type: 'string' }, + fields: [{ name: 'meta', type: 'object' }], + }, + ], + }); + + expect(table.schema.meta).toMatchObject({ type: 'object' }); + }); +}); diff --git a/packages/datasource-dynamodb/test/introspection/introspector.test.ts b/packages/datasource-dynamodb/test/introspection/introspector.test.ts new file mode 100644 index 0000000000..72c6a4c754 --- /dev/null +++ b/packages/datasource-dynamodb/test/introspection/introspector.test.ts @@ -0,0 +1,398 @@ +import { introspectTable, introspectTables } from '../../src/introspection/introspector'; +import { fakeClient } from '../fake-client'; + +const describeTable = (overrides: Record = {}) => ({ + Table: { + TableName: 'Orders', + KeySchema: [ + { AttributeName: 'customerId', KeyType: 'HASH' }, + { AttributeName: 'orderId', KeyType: 'RANGE' }, + ], + AttributeDefinitions: [ + { AttributeName: 'customerId', AttributeType: 'S' }, + { AttributeName: 'orderId', AttributeType: 'S' }, + { AttributeName: 'status', AttributeType: 'S' }, + { AttributeName: 'createdAt', AttributeType: 'S' }, + ], + GlobalSecondaryIndexes: [ + { + IndexName: 'by-status', + KeySchema: [ + { AttributeName: 'status', KeyType: 'HASH' }, + { AttributeName: 'createdAt', KeyType: 'RANGE' }, + ], + Projection: { ProjectionType: 'ALL' }, + }, + { + IndexName: 'keys-only', + KeySchema: [{ AttributeName: 'status', KeyType: 'HASH' }], + Projection: { ProjectionType: 'KEYS_ONLY' }, + }, + ], + LocalSecondaryIndexes: [ + { + IndexName: 'by-created', + KeySchema: [ + { AttributeName: 'customerId', KeyType: 'HASH' }, + { AttributeName: 'createdAt', KeyType: 'RANGE' }, + ], + Projection: { ProjectionType: 'ALL' }, + }, + ], + ...overrides, + }, +}); + +const items = [ + { + customerId: 'c1', + orderId: 'o1', + status: 'paid', + total: 12, + createdAt: '2024-01-15T10:00:00Z', + tags: new Set(['a']), + address: { city: 'Paris', geo: { lat: 1 } }, + note: 'x', + }, + { customerId: 'c1', orderId: 'o2', status: null, total: 3, createdAt: '2024-01-16T10:00:00Z' }, +]; + +describe('introspectTable', () => { + it('reads the key schema and the indexes from DescribeTable', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: items }), + }); + + const model = await introspectTable(client, { collectionName: 'orders', tableName: 'Orders' }); + + expect(model).toMatchObject({ + name: 'orders', + tableName: 'Orders', + partitionKey: { name: 'customerId', type: 'S' }, + sortKey: { name: 'orderId', type: 'S' }, + }); + expect(model.indexes).toEqual([ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'S' }, + sortKey: { name: 'createdAt', type: 'S' }, + projectsAllAttributes: true, + }, + { + name: 'keys-only', + type: 'GSI', + partitionKey: { name: 'status', type: 'S' }, + sortKey: undefined, + projectsAllAttributes: false, + }, + { + name: 'by-created', + type: 'LSI', + partitionKey: { name: 'customerId', type: 'S' }, + sortKey: { name: 'createdAt', type: 'S' }, + projectsAllAttributes: true, + }, + ]); + }); + + it('infers the attribute types from the sample, flattening nested maps', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: items }), + }); + + const { schema } = await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + }); + + expect(schema).toMatchObject({ + customerId: { type: 'string', nullable: false }, + orderId: { type: 'string', nullable: false }, + total: { type: 'number', nullable: false }, + createdAt: { type: 'date', nullable: false }, + tags: { type: 'array', nullable: true }, + 'address@@@city': { type: 'string', nullable: true }, + 'address@@@geo@@@lat': { type: 'number', nullable: true }, + note: { type: 'string', nullable: true }, + }); + expect(schema.address).toBeUndefined(); + }); + + it('marks an attribute null in the sample as nullable', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: items }), + }); + + const { schema } = await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + }); + + expect(schema.status).toMatchObject({ type: 'string', nullable: true }); + }); + + it('keeps a map as Json past the maximum depth', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: [{ customerId: 'c', orderId: 'o', a: { b: { c: 1 } } }] }), + }); + + const { schema } = await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + maxDepth: 2, + }); + + expect(schema['a@@@b']).toMatchObject({ type: 'object' }); + expect(schema['a@@@b@@@c']).toBeUndefined(); + }); + + it('exposes only the nested columns of a map seen empty in some items', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ + Items: [ + { customerId: 'c', orderId: 'o1', address: null }, + { customerId: 'c', orderId: 'o2', address: {} }, + { customerId: 'c', orderId: 'o3', address: { city: 'Paris' } }, + ], + }), + }); + + const { schema } = await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + }); + + expect(schema.address).toBeUndefined(); + expect(schema['address@@@city']).toMatchObject({ type: 'string', nullable: true }); + }); + + it('declares the keys even when the table is empty', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: [] }), + }); + + const { schema } = await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + }); + + expect(schema).toEqual({ + customerId: { type: 'string', nullable: false }, + orderId: { type: 'string', nullable: false }, + }); + }); + + it('types a numeric key as a number', async () => { + const client = fakeClient({ + DescribeTable: () => + describeTable({ + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'N' }], + GlobalSecondaryIndexes: undefined, + LocalSecondaryIndexes: undefined, + }), + Scan: () => ({ Items: [{ id: 1 }] }), + }); + + const model = await introspectTable(client, { collectionName: 'n', tableName: 'N' }); + + expect(model.partitionKey).toEqual({ name: 'id', type: 'N' }); + expect(model.schema.id).toMatchObject({ type: 'number' }); + expect(model.sortKey).toBeUndefined(); + expect(model.indexes).toEqual([]); + }); + + it('samples with a Scan limited to the sample size', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: items }), + }); + + await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + sampleSize: 50, + }); + + expect(client.calls('Scan')[0]).toMatchObject({ TableName: 'Orders', Limit: 50 }); + }); + + it('keeps scanning until the sample is filled', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: [ + { Items: [items[0]], LastEvaluatedKey: { customerId: 'c1', orderId: 'o1' } }, + { Items: [items[1]] }, + ], + }); + + const { schema } = await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + sampleSize: 10, + }); + + expect(client.calls('Scan')[1]).toMatchObject({ + ExclusiveStartKey: { customerId: 'c1', orderId: 'o1' }, + }); + expect(schema.total).toBeDefined(); + }); + + it('restricts the sample to a single-table entity by discriminator attribute', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: items }), + }); + + const model = await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + entity: { attribute: 'entityType', value: 'ORDER' }, + }); + + expect(client.calls('Scan')[0]).toMatchObject({ + FilterExpression: '#n0 = :v0', + ExpressionAttributeNames: { '#n0': 'entityType' }, + ExpressionAttributeValues: { ':v0': 'ORDER' }, + }); + expect(model.entity).toEqual({ attribute: 'entityType', value: 'ORDER' }); + }); + + it('restricts the sample to a single-table entity by sort key prefix', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: items }), + }); + + await introspectTable(client, { + collectionName: 'orders', + tableName: 'Orders', + entity: { sortKeyPrefix: 'ORDER#' }, + }); + + expect(client.calls('Scan')[0]).toMatchObject({ + FilterExpression: 'begins_with(#n0, :v0)', + ExpressionAttributeNames: { '#n0': 'orderId' }, + ExpressionAttributeValues: { ':v0': 'ORDER#' }, + }); + }); + + it('refuses a sort key prefix on a table without sort key', async () => { + const client = fakeClient({ + DescribeTable: () => + describeTable({ + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }), + Scan: () => ({ Items: [] }), + }); + + await expect( + introspectTable(client, { + collectionName: 'n', + tableName: 'N', + entity: { sortKeyPrefix: 'X#' }, + }), + ).rejects.toThrow("Table 'N' has no sort key, so 'sortKeyPrefix' cannot be used"); + }); + + it('refuses a binary key', async () => { + const client = fakeClient({ + DescribeTable: () => + describeTable({ + TableName: 'N', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'B' }], + }), + Scan: () => ({ Items: [] }), + }); + + await expect(introspectTable(client, { collectionName: 'n', tableName: 'N' })).rejects.toThrow( + "Key 'id' of table 'N' is binary, which is not supported", + ); + }); +}); + +describe('introspectTables', () => { + it('introspects every table, naming collections after tables', async () => { + const client = fakeClient({ + ListTables: () => ({ TableNames: ['Orders', 'Users'] }), + DescribeTable: (input: { TableName: string }) => + input.TableName === 'Orders' + ? describeTable() + : describeTable({ + TableName: 'Users', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }), + Scan: () => ({ Items: [] }), + }); + const logger = jest.fn(); + + const models = await introspectTables(client, {}, logger); + + expect(models.map(m => m.name).sort()).toEqual(['Orders', 'Users']); + }); + + it('follows ListTables pagination', async () => { + const client = fakeClient({ + ListTables: [{ TableNames: ['A'], LastEvaluatedTableName: 'A' }, { TableNames: ['B'] }], + DescribeTable: (input: { TableName: string }) => + describeTable({ + TableName: input.TableName, + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }), + Scan: () => ({ Items: [] }), + }); + + const models = await introspectTables(client, {}, jest.fn()); + + expect(client.calls('ListTables')[1]).toMatchObject({ ExclusiveStartTableName: 'A' }); + expect(models.map(m => m.name).sort()).toEqual(['A', 'B']); + }); + + it('only introspects the tables it is asked for', async () => { + const client = fakeClient({ + DescribeTable: () => describeTable(), + Scan: () => ({ Items: [] }), + }); + + await introspectTables(client, { tables: ['Orders'] }, jest.fn()); + + expect(client.calls('ListTables')).toHaveLength(0); + expect(client.calls('DescribeTable')).toHaveLength(1); + }); + + it('skips a table it cannot introspect and says why', async () => { + const client = fakeClient({ + ListTables: () => ({ TableNames: ['Good', 'Bad'] }), + DescribeTable: (input: { TableName: string }) => { + if (input.TableName === 'Bad') throw new Error('AccessDeniedException'); + + return describeTable({ + TableName: 'Good', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }); + }, + Scan: () => ({ Items: [] }), + }); + const logger = jest.fn(); + + const models = await introspectTables(client, {}, logger); + + expect(models.map(m => m.name)).toEqual(['Good']); + expect(logger).toHaveBeenCalledWith( + 'Warn', + "Introspector - Skipping table 'Bad': AccessDeniedException", + ); + }); +}); diff --git a/packages/datasource-dynamodb/test/review-findings.test.ts b/packages/datasource-dynamodb/test/review-findings.test.ts new file mode 100644 index 0000000000..28f778d5ed --- /dev/null +++ b/packages/datasource-dynamodb/test/review-findings.test.ts @@ -0,0 +1,216 @@ +import { + Aggregation, + Filter, + PaginatedFilter, + Projection, + Sort, +} from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { fakeClient } from './fake-client'; +import { buildModel, buildSimpleModel, leaf } from './helpers'; +import { introspectTable } from '../src/introspection/introspector'; +import { validateManualSchema } from '../src/schema/manual-schema'; + +const caller = { timezone: 'UTC' } as never; + +describe('pagination across timezones', () => { + it('does not resume a page from a position remembered for another timezone', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Scan: [ + { + Items: [ + { id: 'a', name: 'ab' }, + { id: 'b', name: 'ac' }, + ], + LastEvaluatedKey: { id: 'b' }, + }, + { Items: [{ id: 'c', name: 'ad' }] }, + ], + }); + const page = (timezone: string, skip: number) => + collection.list( + { timezone } as never, + new PaginatedFilter({ + conditionTree: leaf('name', 'ILike', 'a%'), + page: { skip, limit: 1 } as never, + }), + new Projection('id'), + ); + + await page('Europe/Paris', 0); + client.send.mockClear(); + await page('Asia/Tokyo', 1); + + expect(client.calls('Scan')[0].ExclusiveStartKey).toBeUndefined(); + }); +}); + +describe('caches after a failed write', () => { + it('forgets the cached charts when part of a bulk write failed', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { + Scan: () => ({ Items: [{ age: 1 }] }), + Put: [{}, Object.assign(new Error('throttled'), { name: 'ThrottlingException' })], + }, + { chartCacheTtlMs: 60000, scanConcurrency: 1 }, + ); + const sum = new Aggregation({ operation: 'Sum', field: 'age' }); + + await collection.aggregate(caller, new Filter({}), sum); + await expect(collection.create(caller, [{ id: 'a' }, { id: 'b' }])).rejects.toThrow( + 'throttled', + ); + await collection.aggregate(caller, new Filter({}), sum); + + expect(client.calls('Scan')).toHaveLength(2); + }); +}); + +describe('sorting a large result', () => { + it('sorts more records than a function call can take as arguments', async () => { + const items = Array.from({ length: 150000 }, (_, i) => ({ id: `u${i}`, age: i % 1000 })); + const { collection } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Items: items }) }, + { inMemoryProcessing: true }, + ); + + const records = await collection.list( + caller, + new PaginatedFilter({ + sort: new Sort({ field: 'age', ascending: false }), + page: { skip: 0, limit: 2 } as never, + }), + new Projection('age'), + ); + + expect(records).toEqual([{ age: 999 }, { age: 999 }]); + }); +}); + +describe('single-table discriminator', () => { + const entityModel = () => + buildModel({ + entity: { attribute: 'entityType', value: 'ORDER' }, + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + entityType: { type: 'string' }, + }, + }); + + it('refuses to create a record that claims another entity', async () => { + const { collection, client } = buildCollection(entityModel(), { Put: () => ({}) }); + + await expect( + collection.create(caller, [{ customerId: 'c', orderId: 'o', entityType: 'USER' }]), + ).rejects.toThrow("The attribute 'entityType' of 'orders' must be 'ORDER'"); + expect(client.calls('Put')).toHaveLength(0); + }); + + it('accepts the discriminator of its own entity', async () => { + const { collection, client } = buildCollection(entityModel(), { Put: () => ({}) }); + + await collection.create(caller, [{ customerId: 'c', orderId: 'o', entityType: 'ORDER' }]); + + expect(client.calls('Put')[0].Item.entityType).toBe('ORDER'); + }); + + it('refuses to move a record to another entity', async () => { + const { collection } = buildCollection(entityModel(), {}); + + await expect( + collection.update(caller, new Filter({ conditionTree: leaf('customerId', 'Equal', 'c') }), { + entityType: 'USER', + }), + ).rejects.toThrow( + "The attribute 'entityType' tells which entity a record of 'orders' belongs to", + ); + }); +}); + +describe('batch requests', () => { + it('asks for each key once', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + BatchGet: () => ({ Responses: { Users: [{ id: 'a' }] } }), + }); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('id', 'In', ['a', 'a']) }), + new Projection('id'), + ); + + expect(client.calls('BatchGet')[0].RequestItems.Users.Keys).toEqual([{ id: 'a' }]); + }); + + it('logs the capacity consumed by batch reads and writes', async () => { + const { collection, logger } = buildCollection( + buildSimpleModel(), + { + BatchGet: () => ({ + Responses: { Users: [{ id: 'a' }] }, + ConsumedCapacity: [{ CapacityUnits: 1.5 }, { CapacityUnits: 0.5 }], + }), + BatchWrite: () => ({ ConsumedCapacity: [{ CapacityUnits: 3 }] }), + }, + { logConsumedCapacity: true }, + ); + + await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('id', 'Equal', 'a') }), + new Projection('id'), + ); + await collection.delete(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'a') })); + + expect(logger).toHaveBeenCalledWith('Info', '[DynamoDB Capacity] users.batchGet: 2 units'); + expect(logger).toHaveBeenCalledWith('Info', '[DynamoDB Capacity] users.batchDelete: 3 units'); + }); +}); + +describe('introspection of unusual attribute names', () => { + it.each(['constructor', 'toString', '__proto__'])('infers an attribute named %s', async name => { + const item = Object.fromEntries([['id', 'a']]); + Object.defineProperty(item, name, { value: 'x', enumerable: true, configurable: true }); + const client = fakeClient({ + DescribeTable: () => ({ + Table: { + TableName: 'T', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }, + }), + Scan: () => ({ Items: [item] }), + }); + + const { schema } = await introspectTable(client, { collectionName: 't', tableName: 'T' }); + + expect(Object.keys(schema).sort()).toEqual(['id', name].sort()); + }); +}); + +describe('manual schema entity', () => { + const withEntity = (entity: Record) => ({ + collections: [ + { + name: 'c', + tableName: 'T', + partitionKey: { name: 'pk', type: 'string' as const }, + fields: [], + entity, + }, + ], + }); + + it.each([{ value: 'ORDER' }, { attribute: 'type' }])( + 'rejects a discriminator with only one of attribute and value: %p', + entity => { + expect(() => validateManualSchema(withEntity(entity))).toThrow( + "Collection 'c' entity must use either 'attribute'/'value' or 'sortKeyPrefix'", + ); + }, + ); +}); diff --git a/packages/datasource-dynamodb/test/schema/manual-schema.test.ts b/packages/datasource-dynamodb/test/schema/manual-schema.test.ts new file mode 100644 index 0000000000..1653bf6a6c --- /dev/null +++ b/packages/datasource-dynamodb/test/schema/manual-schema.test.ts @@ -0,0 +1,240 @@ +import type { ManualSchemaConfig } from '../../src/types'; + +import { convertManualSchema, validateManualSchema } from '../../src/schema/manual-schema'; + +const config = (overrides: Record = {}): ManualSchemaConfig => ({ + collections: [ + { + name: 'orders', + tableName: 'Orders', + partitionKey: { name: 'customerId', type: 'string' }, + sortKey: { name: 'orderId', type: 'string' }, + fields: [ + { name: 'total', type: 'number' }, + { name: 'status', type: 'string', nullable: true }, + { + name: 'address', + type: 'object', + fields: [ + { name: 'city', type: 'string' }, + { name: 'geo', type: 'object', fields: [{ name: 'lat', type: 'number' }] }, + ], + }, + { name: 'tags', type: 'array', subType: 'string' }, + ], + ...overrides, + }, + ], +}); + +describe('convertManualSchema', () => { + it('builds a table model from the definition', () => { + const [model] = convertManualSchema(config()); + + expect(model).toMatchObject({ + name: 'orders', + tableName: 'Orders', + partitionKey: { name: 'customerId', type: 'S' }, + sortKey: { name: 'orderId', type: 'S' }, + indexes: [], + }); + }); + + it('declares the keys even when they are not listed as fields', () => { + const [model] = convertManualSchema(config()); + + expect(model.schema.customerId).toEqual({ type: 'string', nullable: false }); + expect(model.schema.orderId).toEqual({ type: 'string', nullable: false }); + }); + + it('flattens nested objects into arrow-notation fields', () => { + const [model] = convertManualSchema(config()); + + expect(model.schema['address@@@city']).toEqual({ type: 'string', nullable: false }); + expect(model.schema['address@@@geo@@@lat']).toEqual({ type: 'number', nullable: false }); + expect(model.schema.address).toBeUndefined(); + }); + + it('keeps arrays as a single field and remembers their item schema', () => { + const [model] = convertManualSchema( + config({ + fields: [ + { + name: 'lines', + type: 'array', + subType: 'object', + fields: [{ name: 'sku', type: 'string' }], + }, + ], + }), + ); + + expect(model.schema.lines).toEqual({ + type: 'array', + nullable: false, + items: { sku: { type: 'string', nullable: false } }, + }); + }); + + it('types numeric keys as N', () => { + const [model] = convertManualSchema( + config({ partitionKey: { name: 'id', type: 'number' }, sortKey: undefined }), + ); + + expect(model.partitionKey).toEqual({ name: 'id', type: 'N' }); + expect(model.schema.id).toEqual({ type: 'number', nullable: false }); + }); + + it('carries indexes, entity, count and scan settings', () => { + const [model] = convertManualSchema( + config({ + indexes: [ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'string' }, + sortKey: { name: 'createdAt', type: 'string' }, + }, + ], + entity: { sortKeyPrefix: 'ORDER#' }, + enableCount: false, + scanPolicy: 'deny', + maxScannedItems: 50, + }), + ); + + expect(model.indexes).toEqual([ + { + name: 'by-status', + type: 'GSI', + partitionKey: { name: 'status', type: 'S' }, + sortKey: { name: 'createdAt', type: 'S' }, + projectsAllAttributes: true, + coversAllItems: false, + }, + ]); + expect(model).toMatchObject({ + entity: { sortKeyPrefix: 'ORDER#' }, + enableCount: false, + scanPolicy: 'deny', + maxScannedItems: 50, + }); + }); +}); + +describe('keys and fields of the same name', () => { + it('keeps the type configured for the key', () => { + const [model] = convertManualSchema( + config({ + partitionKey: { name: 'id', type: 'number' }, + sortKey: undefined, + fields: [{ name: 'id', type: 'string' }], + }), + ); + + expect(model.schema.id).toEqual({ type: 'number', nullable: false }); + }); +}); + +describe('validateManualSchema', () => { + it('accepts a valid schema', () => { + expect(() => validateManualSchema(config())).not.toThrow(); + }); + + it.each([ + [{ collections: [] }, 'The introspection must have at least one collection'], + [undefined, 'The introspection must have a collections array'], + ])('rejects %p', (schema, message) => { + expect(() => validateManualSchema(schema as never)).toThrow(message); + }); + + it('rejects a collection without table name', () => { + expect(() => validateManualSchema(config({ tableName: undefined }))).toThrow( + "Collection 'orders' must have a tableName", + ); + }); + + it('rejects a collection without partition key', () => { + expect(() => validateManualSchema(config({ partitionKey: undefined }))).toThrow( + "Collection 'orders' must have a partitionKey", + ); + }); + + it('rejects a sort key named like the partition key', () => { + expect(() => + validateManualSchema(config({ sortKey: { name: 'customerId', type: 'string' } })), + ).toThrow("Collection 'orders' uses 'customerId' as both partition key and sort key"); + }); + + it('rejects a key type other than string or number', () => { + expect(() => + validateManualSchema(config({ partitionKey: { name: 'customerId', type: 'object' } })), + ).toThrow("Key 'customerId' of collection 'orders' must be a string or a number"); + }); + + it('rejects duplicate collection names', () => { + const duplicated = config(); + duplicated.collections.push({ ...duplicated.collections[0] }); + + expect(() => validateManualSchema(duplicated)).toThrow("Duplicate collection name: 'orders'"); + }); + + it('rejects duplicate field names at the same level', () => { + expect(() => + validateManualSchema( + config({ + fields: [ + { name: 'a', type: 'string' }, + { name: 'a', type: 'number' }, + ], + }), + ), + ).toThrow("Duplicate field name: 'a' in collection 'orders'"); + }); + + it('rejects an unknown field type', () => { + expect(() => validateManualSchema(config({ fields: [{ name: 'a', type: 'uuid' }] }))).toThrow( + "Field 'a' in collection 'orders' has invalid type 'uuid'", + ); + }); + + it('accepts an array without sub type, as the introspection writes it', () => { + expect(() => + validateManualSchema(config({ fields: [{ name: 'a', type: 'array' }] })), + ).not.toThrow(); + }); + + it('accepts an object without nested fields, which is a column of its own', () => { + expect(() => + validateManualSchema(config({ fields: [{ name: 'a', type: 'object' }] })), + ).not.toThrow(); + }); + + it('rejects an array of objects without nested fields', () => { + expect(() => + validateManualSchema(config({ fields: [{ name: 'a', type: 'array', subType: 'object' }] })), + ).toThrow( + "Array field 'a' in collection 'orders' of objects must have nested field definitions", + ); + }); + + it('rejects a sort key prefix on a table without sort key', () => { + expect(() => + validateManualSchema(config({ sortKey: undefined, entity: { sortKeyPrefix: 'X#' } })), + ).toThrow("Collection 'orders' has no sortKey, so 'entity.sortKeyPrefix' cannot be used"); + }); + + it('rejects an entity naming both a discriminator and a prefix', () => { + expect(() => + validateManualSchema( + config({ entity: { attribute: 'type', value: 'O', sortKeyPrefix: 'ORDER#' } }), + ), + ).toThrow("Collection 'orders' entity must use either 'attribute'/'value' or 'sortKeyPrefix'"); + }); + + it('rejects an unknown scan policy', () => { + expect(() => validateManualSchema(config({ scanPolicy: 'sometimes' }))).toThrow( + "Collection 'orders' has an invalid scanPolicy 'sometimes'", + ); + }); +}); diff --git a/packages/datasource-dynamodb/test/second-round-findings.test.ts b/packages/datasource-dynamodb/test/second-round-findings.test.ts new file mode 100644 index 0000000000..0eaffa2af0 --- /dev/null +++ b/packages/datasource-dynamodb/test/second-round-findings.test.ts @@ -0,0 +1,294 @@ +import type { DataSource } from '@forestadmin/datasource-toolkit'; + +import { Aggregation, Filter, PaginatedFilter, Projection } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { fakeClient } from './fake-client'; +import { and, buildModel, buildSimpleModel, leaf, plain } from './helpers'; +import ArrayCollection from '../src/array-collection'; +import DynamoDBCollection from '../src/collection'; +import { introspectTables } from '../src/introspection/introspector'; +import { validateManualSchema } from '../src/schema/manual-schema'; +import { resolveSettings } from '../src/settings'; +import { planAccessPath } from '../src/utils/access-path'; +import Serializer from '../src/utils/serializer'; + +const caller = { timezone: 'UTC' } as never; +const sum = new Aggregation({ operation: 'Sum', field: 'age' }); + +describe('chart caches', () => { + it('drops the charts of every collection of a table when one of them writes', async () => { + const client = fakeClient({ Scan: () => ({ Items: [{ age: 1 }] }), Put: () => ({}) }); + const settings = resolveSettings({ chartCacheTtlMs: 60000, scanConcurrency: 1 }); + const build = (name: string, entity?: { sortKeyPrefix: string }) => + new DynamoDBCollection( + {} as DataSource, + buildModel({ + name, + entity, + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + age: { type: 'number' }, + }, + }), + client, + jest.fn(), + settings, + ); + const whole = build('whole'); + const orders = build('orders', { sortKeyPrefix: 'ORDER#' }); + + await whole.aggregate(caller, new Filter({}), sum); + await orders.create(caller, [{ customerId: 'c', orderId: 'ORDER#1' }]); + await whole.aggregate(caller, new Filter({}), sum); + + expect(client.calls('Scan')).toHaveLength(2); + }); + + it('is not poisoned by a caller that renames the groups it is given', async () => { + const { collection } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Items: [{ age: 1, name: 'a' }] }) }, + { chartCacheTtlMs: 60000, scanConcurrency: 1 }, + ); + const byName = new Aggregation({ operation: 'Sum', field: 'age', groups: [{ field: 'name' }] }); + + const first = await collection.aggregate(caller, new Filter({}), byName); + first[0].group['relation:id'] = first[0].group.name; + delete first[0].group.name; + const second = await collection.aggregate(caller, new Filter({}), byName); + + expect(second).toEqual([{ value: 1, group: { name: 'a' } }]); + }); +}); + +describe('array collections', () => { + const parentWith = (handlers: Record, model = buildModel()) => { + const parent = buildCollection( + { + ...model, + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + lines: { type: 'array' }, + details: { type: 'object' }, + }, + }, + handlers, + { scanConcurrency: 1 }, + ); + + return { + client: parent.client, + parent: parent.collection, + lines: (arrayField = 'lines') => + new ArrayCollection({} as DataSource, parent.collection, { + name: 'order_lines', + arrayField, + itemSchema: { sku: { type: 'string' }, metadata: { type: 'object' } }, + }), + }; + }; + + it('refuses a table key named like the position column', () => { + const { lines } = parentWith({}, buildModel({ sortKey: { name: 'index', type: 'S' } })); + + expect(() => lines()).toThrow("The key columns of 'order_lines' must have distinct names"); + }); + + it('replaces an object attribute as a whole when it rewrites the item, as SET would', async () => { + const order = { + customerId: 'c1', + orderId: 'o1', + lines: [{ sku: 'A', metadata: { a: 1, b: 2 } }], + }; + const { lines, client } = parentWith({ + Scan: () => ({ Items: [order] }), + Update: () => { + throw Object.assign(new Error('The document path provided is invalid'), { + name: 'ValidationException', + }); + }, + Get: () => ({ Item: JSON.parse(JSON.stringify(order)) }), + Put: () => ({}), + }); + + await lines().update(caller, new Filter({}), { metadata: { a: 3 } }); + + expect(client.calls('Put')[0].Item.lines[0].metadata).toEqual({ a: 3 }); + }); + + it('creates the first element of an array held in a map the item does not have', async () => { + const { lines, client } = parentWith({ + Update: () => { + throw Object.assign(new Error('The document path provided is invalid'), { + name: 'ValidationException', + }); + }, + Get: () => ({ Item: { customerId: 'c1', orderId: 'o1' } }), + Put: () => ({}), + }); + + const [created] = await lines('details@@@lines').create(caller, [ + { customerId: 'c1', orderId: 'o1', sku: 'A' }, + ]); + + expect(created).toMatchObject({ index: 0, sku: 'A' }); + expect(client.calls('Put')[0].Item).toEqual({ + customerId: 'c1', + orderId: 'o1', + details: { lines: [{ sku: 'A' }] }, + }); + }); +}); + +describe('attribute names holding the path separator', () => { + it('skips a table with such an attribute, and says why', async () => { + const client = fakeClient({ + DescribeTable: () => ({ + Table: { + TableName: 'T', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }, + }), + Scan: () => ({ Items: [{ id: 'a', 'a@@@b': 1 }] }), + }); + const logger = jest.fn(); + + const models = await introspectTables(client, { tables: ['T'] }, logger); + + expect(models).toEqual([]); + expect(logger).toHaveBeenCalledWith( + 'Warn', + "Introspector - Skipping table 'T': Attribute 'a@@@b' contains '@@@', which names nested attributes", + ); + }); + + it('rejects a manual field named so', () => { + expect(() => + validateManualSchema({ + collections: [ + { + name: 'c', + tableName: 'T', + partitionKey: { name: 'pk', type: 'string' }, + fields: [{ name: 'a@@@b', type: 'string' }], + }, + ], + }), + ).toThrow("Field 'a@@@b' in collection 'c' contains '@@@'"); + }); +}); + +describe('single-table creation and reads', () => { + const entityModel = () => + buildModel({ + entity: { attribute: 'entityType', value: 'ORDER' }, + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + entityType: { type: 'string' }, + }, + }); + + it('returns the discriminator the record was stored with', async () => { + const { collection } = buildCollection(entityModel(), { Put: () => ({}) }); + + const [created] = await collection.create(caller, [{ customerId: 'c', orderId: 'o' }]); + + expect(created.entityType).toBe('ORDER'); + }); + + it('does not let a first page of an entity collection bypass a deny policy', async () => { + const { collection } = buildCollection( + { ...entityModel(), scanPolicy: 'deny' }, + { Scan: () => ({ Items: [] }) }, + ); + + await expect( + collection.list( + caller, + new PaginatedFilter({ page: { skip: 0, limit: 15 } as never }), + new Projection('orderId'), + ), + ).rejects.toThrow("refused by scanPolicy 'deny'"); + }); +}); + +describe('a reversed range on the sort key', () => { + it('is not sent as a BETWEEN, which DynamoDB rejects', () => { + const plan = planAccessPath( + buildModel(), + and( + plain('customerId', 'Equal', 'c1'), + plain('orderId', 'GreaterThanOrEqual', 'o9'), + plain('orderId', 'LessThanOrEqual', 'o1'), + ), + ); + + expect(plan.keyCondition.sort?.operator).not.toBe('Between'); + expect(plan.remaining.toPlainObject()).toEqual(plain('orderId', 'LessThanOrEqual', 'o1')); + }); + + it('matches nothing when read', async () => { + const { collection } = buildCollection(buildModel(), { + Query: () => ({ + Items: [{ customerId: 'c1', orderId: 'o9' }], + }), + }); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: and( + plain('customerId', 'Equal', 'c1'), + plain('orderId', 'GreaterThanOrEqual', 'o9'), + plain('orderId', 'LessThanOrEqual', 'o1'), + ), + }), + new Projection('orderId'), + ); + + expect(records).toEqual([]); + }); +}); + +describe('writing records', () => { + it('refuses a path through __proto__ instead of changing every object', () => { + expect(() => Serializer.unflatten({ '__proto__@@@polluted': 'yes' })).toThrow( + "An attribute path cannot contain '__proto__'", + ); + expect(() => Serializer.setPath({}, ['__proto__', 'polluted'], 'yes')).toThrow( + "An attribute path cannot contain '__proto__'", + ); + expect(({} as Record).polluted).toBeUndefined(); + }); + + it('does not take an inherited property for a map to write into', () => { + const item = Serializer.unflatten({ 'constructor@@@name': 'x' }); + + expect(Object.keys(item)).toEqual(['constructor']); + expect((item.constructor as unknown as Record).name).toBe('x'); + expect(Object.prototype.constructor.name).toBe('Object'); + }); + + it('reads the item strongly consistent before rewriting it', async () => { + const { collection, client } = buildCollection(buildSimpleModel(), { + Update: () => { + throw Object.assign(new Error('The document path provided is invalid'), { + name: 'ValidationException', + }); + }, + Get: () => ({ Item: { id: 'u1' } }), + Put: () => ({}), + }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + 'address@@@city': 'Lyon', + }); + + expect(client.calls('Get')[0].ConsistentRead).toBe(true); + }); +}); diff --git a/packages/datasource-dynamodb/test/third-round-findings.test.ts b/packages/datasource-dynamodb/test/third-round-findings.test.ts new file mode 100644 index 0000000000..418ffdd28e --- /dev/null +++ b/packages/datasource-dynamodb/test/third-round-findings.test.ts @@ -0,0 +1,187 @@ +import { Filter, PaginatedFilter, Projection, Sort } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { and, buildModel, buildSimpleModel, leaf } from './helpers'; + +const caller = { timezone: 'UTC' } as never; + +describe('maxScannedItems on key lookups', () => { + it('counts the keys read by BatchGetItem', async () => { + const ids = Array.from({ length: 250 }, (_, i) => `u${i}`); + const { collection } = buildCollection( + buildSimpleModel(), + { + BatchGet: (input: any) => ({ + Responses: { + Users: (input.RequestItems.Users.Keys as Array<{ id: string }>).map(key => ({ + id: key.id, + })), + }, + }), + }, + { maxScannedItems: 50 }, + ); + + await expect( + collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('id', 'In', ids) }), + new Projection('id'), + ), + ).rejects.toThrow(/maxScannedItems is 50/); + }); + + it('reads the keys of a lookup that fits the limit', async () => { + const { collection } = buildCollection( + buildSimpleModel(), + { BatchGet: () => ({ Responses: { Users: [{ id: 'u1' }, { id: 'u2' }] } }) }, + { maxScannedItems: 50 }, + ); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('id', 'In', ['u1', 'u2']) }), + new Projection('id'), + ); + + expect(records).toEqual([{ id: 'u1' }, { id: 'u2' }]); + }); +}); + +describe('maxOffset on reads that sort in memory', () => { + it('refuses a skip beyond the limit, as the paged read does', async () => { + const { collection, client } = buildCollection( + buildSimpleModel(), + { Scan: () => ({ Items: [{ id: 'u1', name: 'a' }] }) }, + { maxOffset: 100, scanConcurrency: 1, inMemoryProcessing: true }, + ); + + await expect( + collection.list( + caller, + new PaginatedFilter({ + sort: new Sort({ field: 'name', ascending: true }), + page: { skip: 200, limit: 15 } as never, + }), + new Projection('id'), + ), + ).rejects.toThrow('Offset 200 exceeds the maximum allowed offset of 100'); + expect(client.calls('Scan')).toHaveLength(0); + }); +}); + +describe('binary attributes', () => { + it('filters them on the returned items, not in DynamoDB', async () => { + const model = buildSimpleModel({ + schema: { id: { type: 'string' }, blob: { type: 'binary' } }, + }); + const { collection, client } = buildCollection( + model, + { + Scan: () => ({ + Items: [ + { id: 'u1', blob: Uint8Array.from([1, 2, 3]) }, + { id: 'u2', blob: Uint8Array.from([9]) }, + ], + }), + }, + { scanConcurrency: 1 }, + ); + + const records = await collection.list( + caller, + new PaginatedFilter({ conditionTree: leaf('blob', 'Equal', 'AQID') }), + new Projection('id', 'blob'), + ); + + expect(records).toEqual([{ id: 'u1', blob: 'AQID' }]); + expect(client.calls('Scan')[0].FilterExpression).toBeUndefined(); + }); +}); + +describe('writing binary attributes', () => { + const model = () => + buildSimpleModel({ schema: { id: { type: 'string' }, blob: { type: 'binary' } } }); + + it('stores the bytes of the base64 text on create and returns the text', async () => { + const { collection, client } = buildCollection(model(), { Put: () => ({}) }); + + const [created] = await collection.create(caller, [{ id: 'u1', blob: 'AQID' }]); + + expect(Array.from(client.calls('Put')[0].Item.blob as Uint8Array)).toEqual([1, 2, 3]); + expect(created.blob).toBe('AQID'); + }); + + it('stores the bytes of the base64 text on update', async () => { + const { collection, client } = buildCollection(model(), { Update: () => ({}) }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + blob: 'AQID', + }); + + const [value] = Object.values(client.calls('Update')[0].ExpressionAttributeValues); + expect(Array.from(value as Uint8Array)).toEqual([1, 2, 3]); + }); + + it('keeps a null binary as null', async () => { + const { collection, client } = buildCollection(model(), { Update: () => ({}) }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + blob: null, + }); + + expect(Object.values(client.calls('Update')[0].ExpressionAttributeValues)).toContain(null); + }); +}); + +describe('a nested entity discriminator', () => { + const model = () => buildModel({ entity: { attribute: 'meta@@@type', value: 'ORDER' } }); + + it('is stamped under its path, not under a literal key', async () => { + const { collection, client } = buildCollection(model(), { Put: () => ({}) }); + + const [created] = await collection.create(caller, [{ customerId: 'c1', orderId: 'o1' }]); + const { Item } = client.calls('Put')[0]; + + expect(Item.meta).toEqual({ type: 'ORDER' }); + expect(Object.keys(Item)).not.toContain('meta@@@type'); + expect(created['meta@@@type']).toBe('ORDER'); + }); + + it('refuses another value supplied for it', async () => { + const { collection, client } = buildCollection(model(), { Put: () => ({}) }); + + await expect( + collection.create(caller, [{ customerId: 'c1', orderId: 'o1', 'meta@@@type': 'INVOICE' }]), + ).rejects.toThrow("The attribute 'meta@@@type' of 'orders' must be 'ORDER'"); + expect(client.calls('Put')).toHaveLength(0); + }); +}); + +describe('a nested entity discriminator left out of the schema', () => { + it('still matches the items of the entity when they are read by key', async () => { + const { collection } = buildCollection( + buildModel({ entity: { attribute: 'meta@@@type', value: 'ORDER' } }), + { + BatchGet: () => ({ + Responses: { + Orders: [ + { customerId: 'c1', orderId: 'o1', meta: { type: 'ORDER' } }, + { customerId: 'c1', orderId: 'o2', meta: { type: 'INVOICE' } }, + ], + }, + }), + }, + ); + + const records = await collection.list( + caller, + new PaginatedFilter({ + conditionTree: and(leaf('customerId', 'Equal', 'c1'), leaf('orderId', 'In', ['o1', 'o2'])), + }), + new Projection('orderId'), + ); + + expect(records).toEqual([{ orderId: 'o1' }]); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/access-path.test.ts b/packages/datasource-dynamodb/test/utils/access-path.test.ts new file mode 100644 index 0000000000..f9a06667e1 --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/access-path.test.ts @@ -0,0 +1,440 @@ +import { Sort } from '@forestadmin/datasource-toolkit'; + +import { planAccessPath, planKeyLookup } from '../../src/utils/access-path'; +import { and, buildModel, buildSimpleModel, leaf, or, plain } from '../helpers'; + +const gsi = (overrides = {}) => ({ + name: 'by-status', + type: 'GSI' as const, + partitionKey: { name: 'status', type: 'S' as const }, + sortKey: { name: 'createdAt', type: 'S' as const }, + projectsAllAttributes: true, + coversAllItems: true, + ...overrides, +}); + +describe('planAccessPath', () => { + it('queries the table when the partition key is pinned by an equality', () => { + const plan = planAccessPath(buildModel(), leaf('customerId', 'Equal', 'c1')); + + expect(plan).toMatchObject({ + kind: 'query', + keyCondition: { partition: { attribute: 'customerId', value: 'c1' } }, + }); + expect(plan.indexName).toBeUndefined(); + expect(plan.remaining).toBeUndefined(); + }); + + it('keeps the other conditions as filter', () => { + const plan = planAccessPath( + buildModel(), + and(plain('customerId', 'Equal', 'c1'), plain('status', 'Equal', 'paid')), + ); + + expect(plan.kind).toBe('query'); + expect(plan.remaining.toPlainObject()).toEqual(plain('status', 'Equal', 'paid')); + }); + + it('scans when nothing pins a partition key', () => { + const tree = leaf('status', 'Equal', 'paid'); + + const plan = planAccessPath(buildModel(), tree); + + expect(plan.kind).toBe('scan'); + expect(plan.remaining).toBe(tree); + }); + + it('scans without a condition tree', () => { + expect(planAccessPath(buildModel(), undefined)).toMatchObject({ kind: 'scan' }); + }); + + it('does not use an equality hidden inside an OR', () => { + const plan = planAccessPath( + buildModel(), + or(plain('customerId', 'Equal', 'c1'), plain('customerId', 'Equal', 'c2')), + ); + + expect(plan.kind).toBe('scan'); + }); + + it('turns a sort key range into a key condition', () => { + const plan = planAccessPath( + buildModel(), + and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'GreaterThan', 'o5')), + ); + + expect(plan.keyCondition.sort).toEqual({ + attribute: 'orderId', + operator: 'GreaterThan', + value: 'o5', + }); + expect(plan.remaining).toBeUndefined(); + }); + + it('uses BETWEEN for an inclusive sort key interval', () => { + const plan = planAccessPath( + buildModel(), + and( + plain('customerId', 'Equal', 'c1'), + plain('orderId', 'GreaterThanOrEqual', 'o1'), + plain('orderId', 'LessThanOrEqual', 'o9'), + ), + ); + + expect(plan.keyCondition.sort).toEqual({ + attribute: 'orderId', + operator: 'Between', + value: ['o1', 'o9'], + }); + expect(plan.remaining).toBeUndefined(); + }); + + it('keeps the second bound of an exclusive interval as filter', () => { + const plan = planAccessPath( + buildModel(), + and( + plain('customerId', 'Equal', 'c1'), + plain('orderId', 'GreaterThan', 'o1'), + plain('orderId', 'LessThan', 'o9'), + ), + ); + + expect(plan.keyCondition.sort).toMatchObject({ operator: 'GreaterThan', value: 'o1' }); + expect(plan.remaining.toPlainObject()).toEqual(plain('orderId', 'LessThan', 'o9')); + }); + + it('queries a global secondary index when its partition key is pinned', () => { + const model = buildModel({ indexes: [gsi()] }); + + const plan = planAccessPath(model, leaf('status', 'Equal', 'paid')); + + expect(plan).toMatchObject({ + kind: 'query', + indexName: 'by-status', + keyCondition: { partition: { attribute: 'status', value: 'paid' } }, + }); + }); + + it('ignores an index that does not project every attribute', () => { + const model = buildModel({ indexes: [gsi({ projectsAllAttributes: false })] }); + + expect(planAccessPath(model, leaf('status', 'Equal', 'paid')).kind).toBe('scan'); + }); + + it('prefers the table when both the table and an index could serve the query', () => { + const model = buildModel({ indexes: [gsi()] }); + + const plan = planAccessPath( + model, + and(plain('customerId', 'Equal', 'c1'), plain('status', 'Equal', 'paid')), + ); + + expect(plan.indexName).toBeUndefined(); + }); + + it('reads the sort key backwards for a descending sort on it', () => { + const plan = planAccessPath( + buildModel(), + leaf('customerId', 'Equal', 'c1'), + new Sort({ field: 'orderId', ascending: false }), + ); + + expect(plan).toMatchObject({ scanIndexForward: false, sortSatisfied: true }); + }); + + it('does not satisfy a sort on a non-key attribute', () => { + const plan = planAccessPath( + buildModel(), + leaf('customerId', 'Equal', 'c1'), + new Sort({ field: 'total', ascending: true }), + ); + + expect(plan).toMatchObject({ scanIndexForward: true, sortSatisfied: false }); + }); + + it('never claims a sorted result from a scan', () => { + const plan = planAccessPath( + buildModel(), + undefined, + new Sort({ field: 'orderId', ascending: true }), + ); + + expect(plan.sortSatisfied).toBe(false); + }); + + it('restricts a single-table entity to its sort key prefix', () => { + const model = buildModel({ entity: { sortKeyPrefix: 'ORDER#' } }); + + const plan = planAccessPath(model, leaf('customerId', 'Equal', 'c1')); + + expect(plan.keyCondition.sort).toEqual({ + attribute: 'orderId', + operator: 'StartsWith', + value: 'ORDER#', + }); + }); + + it('keeps the prefix as a filter when the user already ranges on the sort key', () => { + const model = buildModel({ entity: { sortKeyPrefix: 'ORDER#' } }); + + const plan = planAccessPath( + model, + and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'GreaterThan', 'ORDER#5')), + ); + + expect(plan.keyCondition.sort).toMatchObject({ operator: 'GreaterThan' }); + expect(plan.remaining.toPlainObject()).toEqual(plain('orderId', 'StartsWith', 'ORDER#')); + }); +}); + +describe('planKeyLookup', () => { + it('resolves an equality on the partition key of a simple table', () => { + expect(planKeyLookup(buildSimpleModel(), leaf('id', 'Equal', 'u1'))).toEqual({ + keys: [{ id: 'u1' }], + remaining: undefined, + }); + }); + + it('resolves an In on the partition key of a simple table', () => { + expect(planKeyLookup(buildSimpleModel(), leaf('id', 'In', ['u1', 'u2']))).toEqual({ + keys: [{ id: 'u1' }, { id: 'u2' }], + remaining: undefined, + }); + }); + + it('resolves a composite key given as an OR of equalities (relation emulation shape)', () => { + const tree = or( + and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'Equal', 'o1')), + and(plain('customerId', 'Equal', 'c2'), plain('orderId', 'Equal', 'o2')), + ); + + expect(planKeyLookup(buildModel(), tree).keys).toEqual([ + { customerId: 'c1', orderId: 'o1' }, + { customerId: 'c2', orderId: 'o2' }, + ]); + }); + + it('resolves a composite key given as an AND of equalities', () => { + const tree = and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'Equal', 'o1')); + + expect(planKeyLookup(buildModel(), tree).keys).toEqual([{ customerId: 'c1', orderId: 'o1' }]); + }); + + it('returns what is left once the key is resolved', () => { + const tree = and(plain('id', 'Equal', 'u1'), plain('age', 'GreaterThan', 3)); + + const lookup = planKeyLookup(buildSimpleModel(), tree); + + expect(lookup.keys).toEqual([{ id: 'u1' }]); + expect(lookup.remaining.toPlainObject()).toEqual(plain('age', 'GreaterThan', 3)); + }); + + it('is not a lookup when the sort key is missing from a composite table', () => { + expect(planKeyLookup(buildModel(), leaf('customerId', 'Equal', 'c1'))).toBeNull(); + }); + + it('is not a lookup for a non-key attribute', () => { + expect(planKeyLookup(buildSimpleModel(), leaf('name', 'Equal', 'a'))).toBeNull(); + }); + + it('is not a lookup without condition', () => { + expect(planKeyLookup(buildSimpleModel(), undefined)).toBeNull(); + }); + + it('is not a lookup when a key value is null', () => { + expect(planKeyLookup(buildSimpleModel(), leaf('id', 'Equal', null))).toBeNull(); + }); +}); + +describe('key value types', () => { + const numericModel = () => + buildModel({ + partitionKey: { name: 'accountId', type: 'N' }, + sortKey: { name: 'seq', type: 'N' }, + }); + + it('reads a numeric partition key given as a numeric string', () => { + const plan = planAccessPath(numericModel(), leaf('accountId', 'Equal', '42')); + + expect(plan.keyCondition.partition).toEqual({ attribute: 'accountId', value: 42 }); + }); + + it('reads a numeric sort key bound given as a numeric string', () => { + const plan = planAccessPath( + numericModel(), + and(plain('accountId', 'Equal', 1), plain('seq', 'GreaterThan', '7')), + ); + + expect(plan.keyCondition.sort).toEqual({ + attribute: 'seq', + operator: 'GreaterThan', + value: 7, + }); + }); + + it('reads a string partition key given as a number', () => { + const plan = planAccessPath(buildModel(), leaf('customerId', 'Equal', 12)); + + expect(plan.keyCondition.partition.value).toBe('12'); + }); + + it('keeps a non numeric string as it is for a numeric key', () => { + const plan = planAccessPath(numericModel(), leaf('accountId', 'Equal', 'abc')); + + expect(plan.keyCondition.partition.value).toBe('abc'); + }); + + it('types the keys of a lookup', () => { + const lookup = planKeyLookup( + buildSimpleModel({ partitionKey: { name: 'id', type: 'N' } }), + leaf('id', 'In', ['1', 2, '3']), + ); + + expect(lookup.keys).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }]); + }); +}); + +describe('secondary indexes that leave items out', () => { + const sparse = () => buildModel({ indexes: [gsi({ coversAllItems: false })] }); + + it('does not read an index by its partition key alone, as items lacking its sort key are not in it', () => { + expect(planAccessPath(sparse(), leaf('status', 'Equal', 'paid')).kind).toBe('scan'); + }); + + it('reads it once a condition on its sort key means the item must be in it', () => { + const plan = planAccessPath( + sparse(), + and(plain('status', 'Equal', 'paid'), plain('createdAt', 'GreaterThan', '2024')), + ); + + expect(plan).toMatchObject({ kind: 'query', indexName: 'by-status' }); + }); + + it('reads an index without sort key by its partition key', () => { + const model = buildModel({ + indexes: [gsi({ sortKey: undefined, coversAllItems: false })], + }); + + expect(planAccessPath(model, leaf('status', 'Equal', 'paid'))).toMatchObject({ + kind: 'query', + indexName: 'by-status', + }); + }); +}); + +describe('planKeyLookup soundness', () => { + it('is not a lookup when an arm of the OR carries a nested filter', () => { + const tree = or( + and(plain('id', 'Equal', 'a'), { + aggregator: 'Or', + conditions: [plain('age', 'Equal', 1), plain('age', 'Equal', 2)], + }), + plain('id', 'Equal', 'b'), + ); + + expect(planKeyLookup(buildSimpleModel(), tree)).toBeNull(); + }); + + it('lists each key once for a repeated value', () => { + expect(planKeyLookup(buildSimpleModel(), leaf('id', 'In', ['a', 'a', 'b'])).keys).toEqual([ + { id: 'a' }, + { id: 'b' }, + ]); + }); + + it('lists each key once for overlapping OR arms', () => { + const tree = or( + and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'Equal', 'o1')), + and(plain('customerId', 'Equal', 'c1'), plain('orderId', 'Equal', 'o1')), + ); + + expect(planKeyLookup(buildModel(), tree).keys).toEqual([{ customerId: 'c1', orderId: 'o1' }]); + }); +}); + +describe('a filter naming several partitions', () => { + it('queries each partition of an In on the partition key', () => { + const plan = planAccessPath(buildModel(), leaf('customerId', 'In', ['c1', 'c2', 'c1'])); + + expect(plan.kind).toBe('query'); + expect(plan.keyCondition.partition).toEqual({ + attribute: 'customerId', + value: 'c1', + values: ['c1', 'c2'], + }); + expect(plan.remaining).toBeUndefined(); + }); + + it('reads a single value In as one partition', () => { + const plan = planAccessPath(buildModel(), leaf('customerId', 'In', ['c1'])); + + expect(plan.keyCondition.partition).toEqual({ attribute: 'customerId', value: 'c1' }); + }); + + it('prefers an equality to an In on the same key', () => { + const plan = planAccessPath( + buildModel(), + and(plain('customerId', 'In', ['c1', 'c2']), plain('customerId', 'Equal', 'c2')), + ); + + expect(plan.keyCondition.partition.values).toBeUndefined(); + expect(plan.keyCondition.partition.value).toBe('c2'); + }); + + it('keeps the sort key range for every partition', () => { + const plan = planAccessPath( + buildModel(), + and(plain('customerId', 'In', ['c1', 'c2']), plain('orderId', 'GreaterThan', 'o5')), + ); + + expect(plan.keyCondition.sort).toMatchObject({ operator: 'GreaterThan', value: 'o5' }); + }); + + it('does not claim a sort order across partitions', () => { + const plan = planAccessPath( + buildModel(), + leaf('customerId', 'In', ['c1', 'c2']), + new Sort({ field: 'orderId', ascending: true }), + ); + + expect(plan.sortSatisfied).toBe(false); + }); + + it('scans instead of sending dozens of queries for a long list', () => { + const values = Array.from({ length: 51 }, (_, i) => `c${i}`); + + expect(planAccessPath(buildModel(), leaf('customerId', 'In', values)).kind).toBe('scan'); + }); + + it('types the values like the key', () => { + const plan = planAccessPath( + buildModel({ partitionKey: { name: 'accountId', type: 'N' } }), + leaf('accountId', 'In', ['1', 2]), + ); + + expect(plan.keyCondition.partition.values).toEqual([1, 2]); + }); +}); + +describe('planKeyLookup bounds', () => { + const composite = () => buildModel(); + + it('lists each value once before building the keys', () => { + const tree = and( + plain('customerId', 'In', ['c1', 'c1', 'c2']), + plain('orderId', 'In', ['o1', 'o1']), + ); + + expect(planKeyLookup(composite(), tree).keys).toEqual([ + { customerId: 'c1', orderId: 'o1' }, + { customerId: 'c2', orderId: 'o1' }, + ]); + }); + + it('does not build the product of two long lists, which would be millions of keys', () => { + const many = (prefix: string) => Array.from({ length: 10000 }, (_, i) => `${prefix}${i}`); + const tree = and(plain('customerId', 'In', many('c')), plain('orderId', 'In', many('o'))); + + expect(planKeyLookup(composite(), tree)).toBeNull(); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/aggregator.test.ts b/packages/datasource-dynamodb/test/utils/aggregator.test.ts new file mode 100644 index 0000000000..c41e9f9868 --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/aggregator.test.ts @@ -0,0 +1,206 @@ +import { Aggregation } from '@forestadmin/datasource-toolkit'; + +import StreamingAggregator from '../../src/utils/aggregator'; + +function aggregate( + aggregation: ConstructorParameters[0], + records: Record[], + options: { timezone?: string; limit?: number } = {}, +) { + const aggregator = new StreamingAggregator( + new Aggregation(aggregation), + options.timezone ?? 'UTC', + ); + records.forEach(record => aggregator.add(record)); + + return aggregator.result(options.limit); +} + +describe('StreamingAggregator', () => { + describe('without groups', () => { + it('counts every record when no field is given', () => { + expect(aggregate({ operation: 'Count' }, [{ a: 1 }, { a: null }, {}])).toEqual([ + { value: 3, group: {} }, + ]); + }); + + it('counts only the records having a value for the field', () => { + expect(aggregate({ operation: 'Count', field: 'a' }, [{ a: 1 }, { a: null }, {}])).toEqual([ + { value: 1, group: {} }, + ]); + }); + + it('sums numbers and ignores other values', () => { + expect( + aggregate({ operation: 'Sum', field: 'a' }, [{ a: 1 }, { a: 2.5 }, { a: 'x' }]), + ).toEqual([{ value: 3.5, group: {} }]); + }); + + it('averages over the records having a number', () => { + expect( + aggregate({ operation: 'Avg', field: 'a' }, [{ a: 2 }, { a: 4 }, { a: null }]), + ).toEqual([{ value: 3, group: {} }]); + }); + + it('takes the min and the max', () => { + const records = [{ a: 3 }, { a: 1 }, { a: 2 }]; + + expect(aggregate({ operation: 'Min', field: 'a' }, records)).toEqual([ + { value: 1, group: {} }, + ]); + expect(aggregate({ operation: 'Max', field: 'a' }, records)).toEqual([ + { value: 3, group: {} }, + ]); + }); + + it('returns zero for a count or a sum over nothing', () => { + expect(aggregate({ operation: 'Count' }, [])).toEqual([{ value: 0, group: {} }]); + expect(aggregate({ operation: 'Sum', field: 'a' }, [])).toEqual([{ value: 0, group: {} }]); + }); + + it('returns null for an average, min or max over nothing', () => { + expect(aggregate({ operation: 'Avg', field: 'a' }, [])).toEqual([{ value: null, group: {} }]); + expect(aggregate({ operation: 'Max', field: 'a' }, [])).toEqual([{ value: null, group: {} }]); + }); + }); + + describe('with groups', () => { + it('groups on one field and sorts by value, highest first', () => { + const result = aggregate({ operation: 'Count', groups: [{ field: 's' }] }, [ + { s: 'a' }, + { s: 'b' }, + { s: 'b' }, + ]); + + expect(result).toEqual([ + { value: 2, group: { s: 'b' } }, + { value: 1, group: { s: 'a' } }, + ]); + }); + + it('groups on several fields', () => { + const result = aggregate( + { operation: 'Sum', field: 'n', groups: [{ field: 'a' }, { field: 'b' }] }, + [ + { a: 1, b: 'x', n: 1 }, + { a: 1, b: 'x', n: 2 }, + { a: 1, b: 'y', n: 5 }, + ], + ); + + expect(result).toEqual([ + { value: 5, group: { a: 1, b: 'y' } }, + { value: 3, group: { a: 1, b: 'x' } }, + ]); + }); + + it('keeps falsy group keys', () => { + const result = aggregate({ operation: 'Count', groups: [{ field: 'k' }] }, [ + { k: 0 }, + { k: '' }, + { k: false }, + ]); + + expect(result.map(r => r.group.k).sort()).toEqual(['', 0, false].sort()); + }); + + it('groups missing values under null', () => { + const result = aggregate({ operation: 'Count', groups: [{ field: 'k' }] }, [{}, { k: null }]); + + expect(result).toEqual([{ value: 2, group: { k: null } }]); + }); + + it('truncates to the limit after sorting', () => { + const result = aggregate( + { operation: 'Count', groups: [{ field: 's' }] }, + [{ s: 'a' }, { s: 'b' }, { s: 'b' }, { s: 'c' }, { s: 'c' }, { s: 'c' }], + { limit: 2 }, + ); + + expect(result.map(r => r.group.s)).toEqual(['c', 'b']); + }); + + it('reads flattened nested fields by their arrow path', () => { + const result = aggregate({ operation: 'Count', groups: [{ field: 'address@@@city' }] }, [ + { 'address@@@city': 'Paris' }, + { 'address@@@city': 'Paris' }, + ]); + + expect(result).toEqual([{ value: 2, group: { 'address@@@city': 'Paris' } }]); + }); + + it('averages per group', () => { + const result = aggregate({ operation: 'Avg', field: 'n', groups: [{ field: 'g' }] }, [ + { g: 'a', n: 2 }, + { g: 'a', n: 4 }, + { g: 'b', n: 10 }, + ]); + + expect(result).toEqual([ + { value: 10, group: { g: 'b' } }, + { value: 3, group: { g: 'a' } }, + ]); + }); + }); + + describe('date groups', () => { + const dates = [ + { at: '2024-01-15T10:00:00Z' }, + { at: '2024-01-20T23:30:00Z' }, + { at: '2024-03-02T01:00:00Z' }, + ]; + + it.each([ + ['Year', ['2024-01-01']], + ['Quarter', ['2024-01-01']], + ['Month', ['2024-01-01', '2024-03-01']], + ['Day', ['2024-01-15', '2024-01-20', '2024-03-02']], + ['Week', ['2024-01-15', '2024-02-26']], + ])('buckets by %s', (operation, expected) => { + const result = aggregate( + { operation: 'Count', groups: [{ field: 'at', operation: operation as never }] }, + dates, + ); + + expect(result.map(r => r.group.at).sort()).toEqual([...expected].sort()); + }); + + it('buckets in the requested timezone', () => { + const result = aggregate( + { operation: 'Count', groups: [{ field: 'at', operation: 'Day' }] }, + [{ at: '2024-01-20T23:30:00Z' }], + { timezone: 'Asia/Tokyo' }, + ); + + expect(result[0].group.at).toBe('2024-01-21'); + }); + + it('accepts epoch milliseconds and seconds', () => { + const result = aggregate( + { operation: 'Count', groups: [{ field: 'at', operation: 'Day' }] }, + [{ at: Date.UTC(2024, 0, 15, 10) }, { at: Date.UTC(2024, 0, 15, 12) / 1000 }], + ); + + expect(result).toEqual([{ value: 2, group: { at: '2024-01-15' } }]); + }); + + it('keeps a date without time on its own day whatever the timezone', () => { + const result = aggregate( + { operation: 'Count', groups: [{ field: 'at', operation: 'Month' }] }, + [{ at: '2024-01-01' }], + { timezone: 'America/Los_Angeles' }, + ); + + expect(result).toEqual([{ value: 1, group: { at: '2024-01-01' } }]); + }); + + it('groups an unparsable date under null', () => { + const result = aggregate( + { operation: 'Count', groups: [{ field: 'at', operation: 'Day' }] }, + [{ at: 'not a date' }], + ); + + expect(result).toEqual([{ value: 1, group: { at: null } }]); + }); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/compare.test.ts b/packages/datasource-dynamodb/test/utils/compare.test.ts new file mode 100644 index 0000000000..aa474830de --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/compare.test.ts @@ -0,0 +1,67 @@ +import { Aggregation, Sort } from '@forestadmin/datasource-toolkit'; + +import StreamingAggregator from '../../src/utils/aggregator'; +import compareValues from '../../src/utils/compare'; +import compareRecords from '../../src/utils/sort'; + +describe('compareValues', () => { + it('orders date-times by the instant, not by their text', () => { + expect(compareValues('2024-01-01T00:30:00+02:00', '2023-12-31T23:00:00Z')).toBe(-1); + }); + + it('compares a date without time as the instant it starts at, in UTC', () => { + expect(compareValues('2024-01-01', '2024-01-01T00:30:00+02:00')).toBe(1); + expect(compareValues('2024-01-01', '2024-01-01T00:00:00Z')).toBe(0); + }); + + it('treats the same instant written two ways as equal', () => { + expect(compareValues('2024-01-01T02:00:00+02:00', '2024-01-01T00:00:00Z')).toBe(0); + }); + + it('orders other strings and numbers as usual', () => { + expect(compareValues('a', 'b')).toBe(-1); + expect(compareValues(2, 1)).toBe(1); + expect(compareValues(1, 1)).toBe(0); + }); +}); + +describe('sorting and aggregating dates with offsets', () => { + const earlier = '2024-01-01T00:30:00+02:00'; + const later = '2023-12-31T23:00:00Z'; + + it('sorts ascending by instant', () => { + const sort = new Sort({ field: 'at', ascending: true }); + const records = [{ at: later }, { at: earlier }]; + + records.sort((a, b) => compareRecords(a, b, sort)); + + expect(records.map(r => r.at)).toEqual([earlier, later]); + }); + + it.each([ + ['Min', earlier], + ['Max', later], + ])('takes the %s instant while keeping its text', (operation, expected) => { + const aggregator = new StreamingAggregator( + new Aggregation({ operation: operation as 'Min' | 'Max', field: 'at' }), + 'UTC', + ); + aggregator.add({ at: later }); + aggregator.add({ at: earlier }); + + expect(aggregator.result()).toEqual([{ value: expected, group: {} }]); + }); + + it('ranks groups by the instant of their Max, so a limit keeps the latest', () => { + const aggregator = new StreamingAggregator( + new Aggregation({ operation: 'Max', field: 'at', groups: [{ field: 'g' }] }), + 'UTC', + ); + aggregator.add({ g: 'earlier', at: '2024-01-01T00:30:00+02:00' }); + aggregator.add({ g: 'later', at: '2023-12-31T23:00:00Z' }); + + expect(aggregator.result(1)).toEqual([ + { value: '2023-12-31T23:00:00Z', group: { g: 'later' } }, + ]); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/concurrency.test.ts b/packages/datasource-dynamodb/test/utils/concurrency.test.ts new file mode 100644 index 0000000000..4a0d33491a --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/concurrency.test.ts @@ -0,0 +1,52 @@ +import { mapUntilSettled, mapWithConcurrency } from '../../src/utils/concurrency'; + +describe('mapWithConcurrency', () => { + it('keeps the order of the results', async () => { + const result = await mapWithConcurrency([3, 1, 2], 2, async n => n * 2); + + expect(result).toEqual([6, 2, 4]); + }); + + it('runs at most the given number of calls at once', async () => { + let running = 0; + let peak = 0; + + await mapWithConcurrency([1, 2, 3, 4, 5], 2, async () => { + running += 1; + peak = Math.max(peak, running); + await new Promise(resolve => { + setImmediate(resolve); + }); + running -= 1; + }); + + expect(peak).toBe(2); + }); +}); + +describe('mapUntilSettled', () => { + it('rejects only once the writes in flight have finished, and starts no new one', async () => { + const finished: number[] = []; + const started: number[] = []; + + await expect( + mapUntilSettled([1, 2, 3, 4, 5, 6], 2, async n => { + started.push(n); + + if (n === 1) throw new Error('boom'); + + await new Promise(resolve => { + setTimeout(resolve, 10); + }); + finished.push(n); + }), + ).rejects.toThrow('boom'); + + expect(finished).toContain(2); + expect(started).not.toContain(6); + }); + + it('returns the results when nothing fails', async () => { + expect(await mapUntilSettled([1, 2], 2, async n => n + 1)).toEqual([2, 3]); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/expression-builder.test.ts b/packages/datasource-dynamodb/test/utils/expression-builder.test.ts new file mode 100644 index 0000000000..2c6dac627b --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/expression-builder.test.ts @@ -0,0 +1,297 @@ +import { ConditionTreeFactory } from '@forestadmin/datasource-toolkit'; + +import ExpressionBuilder, { splitCondition } from '../../src/utils/expression-builder'; + +const leaf = (field: string, operator: string, value?: unknown) => + ConditionTreeFactory.fromPlainObject({ field, operator, value } as never); + +describe('ExpressionBuilder', () => { + describe('condition', () => { + it('aliases attribute names and values so reserved words are safe', () => { + const builder = new ExpressionBuilder(); + + const expression = builder.condition(leaf('status', 'Equal', 'paid')); + + expect(expression).toBe('#n0 = :v0'); + expect(builder.names).toEqual({ '#n0': 'status' }); + expect(builder.values).toEqual({ ':v0': 'paid' }); + }); + + it('reuses the alias of an attribute used twice', () => { + const builder = new ExpressionBuilder(); + + builder.condition( + ConditionTreeFactory.fromPlainObject({ + aggregator: 'And', + conditions: [ + { field: 'age', operator: 'GreaterThan', value: 1 }, + { field: 'age', operator: 'LessThan', value: 9 }, + ], + }), + ); + + expect(builder.names).toEqual({ '#n0': 'age' }); + }); + + it('turns arrow notation into a nested document path', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('address@@@city', 'Equal', 'Paris'))).toBe('#n0.#n1 = :v0'); + expect(builder.names).toEqual({ '#n0': 'address', '#n1': 'city' }); + }); + + it('wraps and/or branches in parentheses', () => { + const builder = new ExpressionBuilder(); + + const expression = builder.condition( + ConditionTreeFactory.fromPlainObject({ + aggregator: 'Or', + conditions: [ + { field: 'a', operator: 'Equal', value: 1 }, + { + aggregator: 'And', + conditions: [ + { field: 'b', operator: 'Equal', value: 2 }, + { field: 'c', operator: 'Equal', value: 3 }, + ], + }, + ], + }), + ); + + expect(expression).toBe('(#n0 = :v0 OR (#n1 = :v1 AND #n2 = :v2))'); + }); + + it.each([ + ['NotEqual', '#n0 <> :v0'], + ['LessThan', '#n0 < :v0'], + ['LessThanOrEqual', '#n0 <= :v0'], + ['GreaterThan', '#n0 > :v0'], + ['GreaterThanOrEqual', '#n0 >= :v0'], + ['StartsWith', 'begins_with(#n0, :v0)'], + ['Contains', 'contains(#n0, :v0)'], + ['NotContains', 'NOT contains(#n0, :v0)'], + ])('translates %s', (operator, expected) => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('f', operator, 5))).toBe(expected); + expect(builder.values).toEqual({ ':v0': 5 }); + }); + + it('treats Equal null as a missing or NULL attribute', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('f', 'Equal', null))).toBe( + '(attribute_not_exists(#n0) OR attribute_type(#n0, :v0))', + ); + expect(builder.values).toEqual({ ':v0': 'NULL' }); + }); + + it('translates Missing as a missing or NULL attribute', () => { + const missing = new ExpressionBuilder(); + + expect(missing.condition(leaf('f', 'Missing'))).toBe( + '(attribute_not_exists(#n0) OR attribute_type(#n0, :v0))', + ); + }); + + it('translates In into an IN list', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('f', 'In', ['a', 'b']))).toBe('#n0 IN (:v0, :v1)'); + expect(builder.values).toEqual({ ':v0': 'a', ':v1': 'b' }); + }); + + it('splits an In list of more than 100 values into several IN groups', () => { + const builder = new ExpressionBuilder(); + const values = Array.from({ length: 150 }, (_, i) => i); + + const expression = builder.condition(leaf('f', 'In', values)); + + expect(expression.startsWith('(#n0 IN (')).toBe(true); + expect(expression.match(/ IN \(/g)).toHaveLength(2); + expect(expression).toContain(') OR #n0 IN ('); + expect(Object.keys(builder.values)).toHaveLength(150); + }); + + it('lets In match null values too', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('f', 'In', ['a', null]))).toBe( + '(#n0 IN (:v0) OR (attribute_not_exists(#n0) OR attribute_type(#n0, :v1)))', + ); + }); + + it('turns an empty In into a contradiction', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('f', 'In', []))).toBe( + '(attribute_exists(#n0) AND attribute_not_exists(#n0))', + ); + }); + + it('negates NotIn', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('f', 'NotIn', ['a', 'b']))).toBe('NOT (#n0 IN (:v0, :v1))'); + }); + + it('requires every value for IncludesAll', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('tags', 'IncludesAll', ['a', 'b']))).toBe( + '(contains(#n0, :v0) AND contains(#n0, :v1))', + ); + }); + + it('translates Blank as missing, NULL or empty', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('f', 'Blank'))).toBe( + '(attribute_not_exists(#n0) OR attribute_type(#n0, :v0) OR #n0 = :v1)', + ); + expect(builder.values).toEqual({ ':v0': 'NULL', ':v1': '' }); + }); + + it('requires none of the values for IncludesNone', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('tags', 'IncludesNone', ['a', 'b']))).toBe( + '(NOT contains(#n0, :v0) AND NOT contains(#n0, :v1))', + ); + }); + + it('is always true for an empty IncludesAll or NotIn', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('tags', 'IncludesAll', []))).toBe( + '(attribute_exists(#n0) OR attribute_not_exists(#n0))', + ); + expect(builder.condition(leaf('tags', 'NotIn', []))).toBe( + '(attribute_exists(#n0) OR attribute_not_exists(#n0))', + ); + }); + + it('turns an In of null only into a missing check', () => { + const builder = new ExpressionBuilder(); + + expect(builder.condition(leaf('f', 'In', [null]))).toBe( + '(attribute_not_exists(#n0) OR attribute_type(#n0, :v0))', + ); + }); + + it('negates a NotIn that is split in several groups as a whole', () => { + const builder = new ExpressionBuilder(); + const values = Array.from({ length: 101 }, (_, i) => i); + + expect(builder.condition(leaf('f', 'NotIn', values)).startsWith('NOT (#n0 IN (')).toBe(true); + }); + + it('throws a typed error on an operator that cannot be pushed down', () => { + const builder = new ExpressionBuilder(); + + expect(() => builder.condition(leaf('f', 'ILike', '%a%'))).toThrow( + 'Operator "ILike" cannot be evaluated by DynamoDB', + ); + }); + + it('rejects a relation field', () => { + const builder = new ExpressionBuilder(); + + expect(() => builder.condition(leaf('rel:name', 'Equal', 'a'))).toThrow( + 'Filtering on the relation field "rel:name" is not supported', + ); + }); + }); + + describe('projection', () => { + it('aliases the top-level attribute of nested fields once', () => { + const builder = new ExpressionBuilder(); + + expect(builder.projection(['pk', 'address@@@city', 'address@@@zip'])).toBe('#n0, #n1'); + expect(builder.names).toEqual({ '#n0': 'pk', '#n1': 'address' }); + }); + }); + + describe('result', () => { + it('omits empty alias maps', () => { + expect(new ExpressionBuilder().result()).toEqual({}); + }); + + it('returns names and values when used', () => { + const builder = new ExpressionBuilder(); + builder.condition(leaf('f', 'Equal', 1)); + + expect(builder.result()).toEqual({ + ExpressionAttributeNames: { '#n0': 'f' }, + ExpressionAttributeValues: { ':v0': 1 }, + }); + }); + }); +}); + +describe('splitCondition', () => { + it.each(['Before', 'After'])( + 'leaves %s to the items, since DynamoDB compares dates as text whatever their offset', + operator => { + const tree = leaf('at', operator, '2024-01-01T00:00:00Z'); + + const { pushable, residual } = splitCondition(tree); + + expect(pushable).toBeUndefined(); + expect(residual).toBe(tree); + }, + ); + + it('pushes Present as the negation of Blank, which also excludes empty strings', () => { + const tree = leaf('f', 'Present'); + + const { pushable, residual } = splitCondition(tree); + + expect(pushable).toBe(tree); + expect(residual).toBeUndefined(); + }); + + it('returns everything as pushable when DynamoDB can evaluate it', () => { + const tree = leaf('f', 'Equal', 1); + + const { pushable, residual } = splitCondition(tree); + + expect(pushable).toBe(tree); + expect(residual).toBeUndefined(); + }); + + it('keeps unsupported leaves of an AND as residual', () => { + const tree = ConditionTreeFactory.fromPlainObject({ + aggregator: 'And', + conditions: [ + { field: 'a', operator: 'Equal', value: 1 }, + { field: 'b', operator: 'ILike', value: '%x%' }, + ], + }); + + const { pushable, residual } = splitCondition(tree); + + expect(pushable.toPlainObject()).toEqual({ field: 'a', operator: 'Equal', value: 1 }); + expect(residual.toPlainObject()).toEqual({ field: 'b', operator: 'ILike', value: '%x%' }); + }); + + it('sends a whole OR to the residual when one branch cannot be pushed down', () => { + const tree = ConditionTreeFactory.fromPlainObject({ + aggregator: 'Or', + conditions: [ + { field: 'a', operator: 'Equal', value: 1 }, + { field: 'b', operator: 'EndsWith', value: 'x' }, + ], + }); + + const { pushable, residual } = splitCondition(tree); + + expect(pushable).toBeUndefined(); + expect(residual).toBe(tree); + }); + + it('handles an absent tree', () => { + expect(splitCondition(undefined)).toEqual({ pushable: undefined, residual: undefined }); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/pagination-cache.test.ts b/packages/datasource-dynamodb/test/utils/pagination-cache.test.ts new file mode 100644 index 0000000000..76801e129e --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/pagination-cache.test.ts @@ -0,0 +1,74 @@ +import PaginationCache from '../../src/utils/pagination-cache'; + +describe('PaginationCache', () => { + it('returns nothing for an unknown query', () => { + expect(new PaginationCache().findBestToken('q', 100)).toBeUndefined(); + }); + + it('resumes from the closest stored offset not past the target', () => { + const cache = new PaginationCache(); + cache.storeToken('q', 100, { pk: 'a' }); + cache.storeToken('q', 300, { pk: 'c' }); + cache.storeToken('q', 500, { pk: 'e' }); + + expect(cache.findBestToken('q', 400)).toEqual({ offset: 300, startKey: { pk: 'c' } }); + }); + + it('never resumes past the target offset', () => { + const cache = new PaginationCache(); + cache.storeToken('q', 500, { pk: 'e' }); + + expect(cache.findBestToken('q', 400)).toBeUndefined(); + }); + + it('keeps queries apart', () => { + const cache = new PaginationCache(); + cache.storeToken('q1', 100, { pk: 'a' }); + + expect(cache.findBestToken('q2', 200)).toBeUndefined(); + }); + + it('drops entries older than the time to live', () => { + let now = 1000; + const cache = new PaginationCache({ ttlMs: 50, now: () => now }); + cache.storeToken('q', 100, { pk: 'a' }); + + now += 51; + + expect(cache.findBestToken('q', 200)).toBeUndefined(); + }); + + it('keeps at most maxEntriesPerQuery entries, dropping the oldest', () => { + const cache = new PaginationCache({ maxEntriesPerQuery: 2 }); + cache.storeToken('q', 100, { pk: 'a' }); + cache.storeToken('q', 200, { pk: 'b' }); + cache.storeToken('q', 300, { pk: 'c' }); + + expect(cache.findBestToken('q', 150)).toBeUndefined(); + expect(cache.findBestToken('q', 350)).toEqual({ offset: 300, startKey: { pk: 'c' } }); + }); + + it('forgets every query of a table when it is invalidated', () => { + const cache = new PaginationCache(); + cache.storeToken('q', 100, { pk: 'a' }, 'Orders'); + cache.storeToken('r', 100, { pk: 'a' }, 'Users'); + + cache.invalidateTable('Orders'); + + expect(cache.findBestToken('q', 200)).toBeUndefined(); + expect(cache.findBestToken('r', 200)).toBeDefined(); + }); + + it('hashes the same query to the same key and different ones apart', () => { + const cache = new PaginationCache(); + + expect(cache.hash({ a: 1, b: [2] })).toBe(cache.hash({ a: 1, b: [2] })); + expect(cache.hash({ a: 1 })).not.toBe(cache.hash({ a: 2 })); + }); + + it('exposes the interval between remembered positions', () => { + const cache = new PaginationCache({ cacheInterval: 7 }); + + expect(cache.getCacheInterval()).toBe(7); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/partiql.test.ts b/packages/datasource-dynamodb/test/utils/partiql.test.ts new file mode 100644 index 0000000000..6548dcdc6a --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/partiql.test.ts @@ -0,0 +1,46 @@ +import parameterize from '../../src/utils/partiql'; + +describe('parameterize', () => { + it('replaces placeholders by positional parameters, in order, repeats included', () => { + expect( + parameterize('SELECT * FROM "T" WHERE a = $x AND b = $y AND c = $x', { x: 1, y: 'z' }), + ).toEqual({ + statement: 'SELECT * FROM "T" WHERE a = ? AND b = ? AND c = ?', + parameters: [1, 'z', 1], + }); + }); + + it('leaves a placeholder inside a string literal alone', () => { + expect(parameterize('SELECT * FROM "T" WHERE name = \'$name\'', {})).toEqual({ + statement: 'SELECT * FROM "T" WHERE name = \'$name\'', + parameters: [], + }); + }); + + it('leaves a placeholder inside a quoted identifier alone', () => { + expect(parameterize('SELECT "$cost" FROM "T" WHERE a = $a', { a: 1 })).toEqual({ + statement: 'SELECT "$cost" FROM "T" WHERE a = ?', + parameters: [1], + }); + }); + + it('keeps reading a literal past an escaped quote', () => { + expect(parameterize("SELECT * FROM \"T\" WHERE a = 'it''s $x' AND b = $y", { y: 2 })).toEqual({ + statement: "SELECT * FROM \"T\" WHERE a = 'it''s $x' AND b = ?", + parameters: [2], + }); + }); + + it('refuses a placeholder without value', () => { + expect(() => parameterize('SELECT * FROM "T" WHERE a = $missing', {})).toThrow( + 'No value was given for the placeholder $missing', + ); + }); + + it('keeps a lone dollar sign', () => { + expect(parameterize('SELECT * FROM "T" WHERE a = $ AND b = $1', { 1: 'x' })).toEqual({ + statement: 'SELECT * FROM "T" WHERE a = $ AND b = ?', + parameters: ['x'], + }); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/serializer.test.ts b/packages/datasource-dynamodb/test/utils/serializer.test.ts new file mode 100644 index 0000000000..a795574df1 --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/serializer.test.ts @@ -0,0 +1,72 @@ +import Serializer from '../../src/utils/serializer'; + +describe('Serializer', () => { + describe('flatten', () => { + it('expands nested maps declared as columns into arrow-notation fields', () => { + const record = Serializer.flatten( + { pk: 'a', address: { city: 'Paris', geo: { lat: 1 } } }, + new Set(['pk', 'address@@@city', 'address@@@geo@@@lat']), + ); + + expect(record).toEqual({ pk: 'a', 'address@@@city': 'Paris', 'address@@@geo@@@lat': 1 }); + }); + + it('keeps a map as Json when no nested column is declared under it', () => { + const record = Serializer.flatten({ pk: 'a', meta: { x: 1 } }, new Set(['pk', 'meta'])); + + expect(record).toEqual({ pk: 'a', meta: { x: 1 } }); + }); + + it('turns sets into arrays', () => { + const record = Serializer.flatten({ tags: new Set(['a', 'b']) }, new Set(['tags'])); + + expect(record).toEqual({ tags: ['a', 'b'] }); + }); + + it('turns dates into ISO strings', () => { + const record = Serializer.flatten( + { at: new Date('2024-01-15T10:00:00.000Z') }, + new Set(['at']), + ); + + expect(record).toEqual({ at: '2024-01-15T10:00:00.000Z' }); + }); + + it('turns binary values into base64 strings', () => { + const record = Serializer.flatten({ blob: Buffer.from('hi') }, new Set(['blob'])); + + expect(record).toEqual({ blob: 'aGk=' }); + }); + + it('keeps GeoJSON points untouched', () => { + const point = { type: 'Point', coordinates: [1, 2] }; + + expect(Serializer.flatten({ loc: point }, new Set(['loc']))).toEqual({ loc: point }); + }); + + it('keeps null values', () => { + expect(Serializer.flatten({ a: null }, new Set(['a']))).toEqual({ a: null }); + }); + }); + + describe('unflatten', () => { + it('rebuilds nested maps from arrow-notation fields', () => { + expect( + Serializer.unflatten({ pk: 'a', 'address@@@city': 'Paris', 'address@@@zip': '75' }), + ).toEqual({ + pk: 'a', + address: { city: 'Paris', zip: '75' }, + }); + }); + + it('leaves plain fields alone', () => { + expect(Serializer.unflatten({ a: 1, b: [1, 2] })).toEqual({ a: 1, b: [1, 2] }); + }); + }); + + describe('toDocumentPath', () => { + it('splits arrow notation into path segments', () => { + expect(Serializer.toPathSegments('a@@@b@@@c')).toEqual(['a', 'b', 'c']); + }); + }); +}); diff --git a/packages/datasource-dynamodb/test/utils/type-converter.test.ts b/packages/datasource-dynamodb/test/utils/type-converter.test.ts new file mode 100644 index 0000000000..1c9f73c07c --- /dev/null +++ b/packages/datasource-dynamodb/test/utils/type-converter.test.ts @@ -0,0 +1,111 @@ +import TypeConverter from '../../src/utils/type-converter'; + +describe('TypeConverter', () => { + describe('inferTypeFromValue', () => { + it.each([ + [true, 'boolean'], + [12, 'number'], + ['hello', 'string'], + [null, 'null'], + [undefined, 'null'], + [['a'], 'array'], + [new Set(['a']), 'array'], + [{ a: 1 }, 'object'], + [new Date(), 'date'], + ['2024-01-15T10:30:00Z', 'date'], + ['2024-01-15', 'dateonly'], + ['10:30:00', 'timeonly'], + [{ type: 'Point', coordinates: [2.35, 48.85] }, 'point'], + [new Uint8Array([1, 2]), 'binary'], + ])('infers %p as %s', (value, expected) => { + expect(TypeConverter.inferTypeFromValue(value)).toBe(expected); + }); + }); + + describe('getMostSpecificType', () => { + it('keeps the type shared by every sample', () => { + expect(TypeConverter.getMostSpecificType(['number', 'number'])).toBe('number'); + }); + + it('ignores null samples when the others agree', () => { + expect(TypeConverter.getMostSpecificType(['string', 'null'])).toBe('string'); + }); + + it('prefers date over dateonly when both are seen', () => { + expect(TypeConverter.getMostSpecificType(['date', 'dateonly'])).toBe('date'); + }); + + it('falls back to object (Json) for incompatible types', () => { + expect(TypeConverter.getMostSpecificType(['number', 'string'])).toBe('object'); + }); + + it('returns null for no sample', () => { + expect(TypeConverter.getMostSpecificType([])).toBe('null'); + }); + }); + + describe('getColumnTypeFromDataType', () => { + it.each([ + ['string', 'String'], + ['number', 'Number'], + ['boolean', 'Boolean'], + ['date', 'Date'], + ['dateonly', 'Dateonly'], + ['timeonly', 'Time'], + ['point', 'Json'], + ['binary', 'String'], + ['object', 'Json'], + ['array', 'Json'], + ['null', 'Json'], + ])('maps %s to %s', (dataType, columnType) => { + expect(TypeConverter.getColumnTypeFromDataType(dataType as never)).toBe(columnType); + }); + + it('throws on an unknown data type', () => { + expect(() => TypeConverter.getColumnTypeFromDataType('weird' as never)).toThrow( + 'Unsupported data type: "weird"', + ); + }); + }); + + describe('operatorsForColumnType', () => { + it('gives strings the full text operators', () => { + const operators = TypeConverter.operatorsForColumnType('String'); + + expect([...operators]).toEqual( + expect.arrayContaining([ + 'Equal', + 'In', + 'Like', + 'ILike', + 'Contains', + 'IContains', + 'StartsWith', + 'EndsWith', + ]), + ); + }); + + it('does not give booleans range operators', () => { + expect(TypeConverter.operatorsForColumnType('Boolean').has('GreaterThan')).toBe(false); + }); + + it('gives numbers range operators', () => { + expect(TypeConverter.operatorsForColumnType('Number').has('GreaterThanOrEqual')).toBe(true); + }); + }); + + describe('isSortable', () => { + it('does not sort Json-like types', () => { + expect(TypeConverter.isSortable('object')).toBe(false); + expect(TypeConverter.isSortable('array')).toBe(false); + expect(TypeConverter.isSortable('point')).toBe(false); + }); + + it('sorts scalar types', () => { + expect(TypeConverter.isSortable('string')).toBe(true); + expect(TypeConverter.isSortable('number')).toBe(true); + expect(TypeConverter.isSortable('date')).toBe(true); + }); + }); +}); diff --git a/packages/datasource-dynamodb/test/validator-findings.test.ts b/packages/datasource-dynamodb/test/validator-findings.test.ts new file mode 100644 index 0000000000..d99921274f --- /dev/null +++ b/packages/datasource-dynamodb/test/validator-findings.test.ts @@ -0,0 +1,456 @@ +import type { DataSource } from '@forestadmin/datasource-toolkit'; + +import { Aggregation, Filter, PaginatedFilter, Projection } from '@forestadmin/datasource-toolkit'; + +import { buildCollection } from './collection-helpers'; +import { fakeClient } from './fake-client'; +import { and, buildModel, buildSimpleModel, leaf } from './helpers'; +import { createDynamoDBDataSource } from '../src'; +import ArrayCollection from '../src/array-collection'; +import { inferSchema } from '../src/introspection/introspector'; +import { convertManualSchema, validateManualSchema } from '../src/schema/manual-schema'; +import { resolveSettings } from '../src/settings'; +import Serializer from '../src/utils/serializer'; + +const caller = { timezone: 'UTC' } as never; + +describe('a number beyond the safe integer range', () => { + it('is read as text so that the response can be serialized', () => { + expect(Serializer.normalize(10n ** 20n)).toBe('100000000000000000000'); + }); + + it('does not make a list fail', async () => { + const { collection } = buildCollection( + buildSimpleModel({ schema: { id: { type: 'string' }, big: { type: 'number' } } }), + { Scan: () => ({ Items: [{ id: 'u1', big: 10n ** 20n }] }) }, + { scanConcurrency: 1 }, + ); + + const records = await collection.list( + caller, + new PaginatedFilter({}), + new Projection('id', 'big'), + ); + + expect(JSON.stringify(records)).toBe('[{"id":"u1","big":"100000000000000000000"}]'); + }); +}); + +describe('string and number sets', () => { + const model = () => + buildSimpleModel({ + schema: { id: { type: 'string' }, tags: { type: 'array', set: true } }, + }); + + it('are marked by the introspection when every sampled value is a Set', () => { + const schema = inferSchema( + [ + { id: 'a', tags: new Set(['x']), list: ['x'] }, + { id: 'b', tags: new Set(['y']), list: new Set(['z']) }, + ], + [{ name: 'id', type: 'S' }], + 5, + ); + + expect(schema.tags.set).toBe(true); + expect(schema.list.set).toBeUndefined(); + }); + + it('can be declared in a manual schema', () => { + const [table] = convertManualSchema({ + collections: [ + { + name: 'a', + tableName: 'A', + partitionKey: { name: 'id', type: 'string' }, + fields: [{ name: 'tags', type: 'array', subType: 'string', set: true }], + }, + ], + }); + + expect(table.schema.tags).toMatchObject({ type: 'array', set: true }); + }); + + it('are refused on a field that is not an array of strings or numbers', () => { + expect(() => + validateManualSchema({ + collections: [ + { + name: 'a', + tableName: 'A', + partitionKey: { name: 'id', type: 'string' }, + fields: [{ name: 'tags', type: 'string', set: true }], + }, + ], + }), + ).toThrow("Field 'tags' in collection 'a' cannot be a set"); + }); + + it('are written back as sets on create', async () => { + const { collection, client } = buildCollection(model(), { Put: () => ({}) }); + + await collection.create(caller, [{ id: 'u1', tags: ['a', 'b'] }]); + + const { tags } = client.calls('Put')[0].Item; + expect(tags).toBeInstanceOf(Set); + expect([...tags]).toEqual(['a', 'b']); + }); + + it('are written back as sets on update', async () => { + const { collection, client } = buildCollection(model(), { Update: () => ({}) }); + + await collection.update(caller, new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }), { + tags: ['a'], + }); + + const [value] = Object.values(client.calls('Update')[0].ExpressionAttributeValues); + expect(value).toBeInstanceOf(Set); + }); +}); + +describe('clearing a set', () => { + const model = () => + buildSimpleModel({ + schema: { + id: { type: 'string' }, + name: { type: 'string', nullable: true }, + tags: { type: 'array', set: true, nullable: true }, + }, + }); + const byId = () => new Filter({ conditionTree: leaf('id', 'Equal', 'u1') }); + + it('leaves the attribute out on create, since DynamoDB holds no empty set', async () => { + const { collection, client } = buildCollection(model(), { Put: () => ({}) }); + + const [created] = await collection.create(caller, [{ id: 'u1', tags: [] }]); + + expect(Object.keys(client.calls('Put')[0].Item)).toEqual(['id']); + expect(created.tags).toEqual([]); + }); + + it('removes the attribute on update', async () => { + const { collection, client } = buildCollection(model(), { Update: () => ({}) }); + + await collection.update(caller, byId(), { tags: [] }); + + const call = client.calls('Update')[0]; + expect(call.UpdateExpression).toBe('REMOVE #n0'); + expect(call.ExpressionAttributeNames).toMatchObject({ '#n0': 'tags' }); + expect(call.ExpressionAttributeValues).toBeUndefined(); + }); + + it('removes it next to the other attributes of the same update', async () => { + const { collection, client } = buildCollection(model(), { Update: () => ({}) }); + + await collection.update(caller, byId(), { name: 'x', tags: [] }); + + expect(client.calls('Update')[0].UpdateExpression).toBe('SET #n0 = :v0 REMOVE #n1'); + }); + + it('removes it in the client-side rewrite too', async () => { + const { collection, client } = buildCollection(model(), { + Update: () => { + throw Object.assign(new Error('The document path provided is invalid for update'), { + name: 'ValidationException', + }); + }, + Get: () => ({ Item: { id: 'u1', tags: new Set(['a']), name: 'n' } }), + Put: () => ({}), + }); + + await collection.update(caller, byId(), { name: 'x', tags: [] }); + + expect(client.calls('Put')[0].Item).toEqual({ id: 'u1', name: 'x' }); + }); +}); + +describe('a manual schema collection without fields', () => { + it('exposes its keys instead of failing', () => { + const [table] = convertManualSchema({ + collections: [{ name: 'a', tableName: 'A', partitionKey: { name: 'id', type: 'string' } }], + } as never); + + expect(Object.keys(table.schema)).toEqual(['id']); + }); +}); + +describe('the option naming the virtual collections', () => { + it('also reads the name the ticket gives it', async () => { + const dataSource = await createDynamoDBDataSource( + { + client: fakeClient({}), + + virtualArrayCollections: [ + { parentCollection: 'a', collectionName: 'a_items', arrayField: 'items' }, + ], + }, + { + introspection: { + collections: [ + { + name: 'a', + tableName: 'A', + partitionKey: { name: 'id', type: 'string' }, + fields: [ + { + name: 'items', + type: 'array', + subType: 'object', + fields: [{ name: 'k', type: 'string' }], + }, + ], + }, + ], + }, + }, + )(jest.fn(), jest.fn()); + + expect(dataSource.collections.map(c => c.name).sort()).toEqual(['a', 'a_items']); + }); +}); + +describe('the scanPolicy option', () => { + it('refuses a value that is not allow, warn or deny', () => { + expect(() => resolveSettings({ scanPolicy: 'Deny' as never })).toThrow( + "Invalid scanPolicy 'Deny': use allow, warn or deny", + ); + }); + + it.each(['allow', 'warn', 'deny'] as const)('accepts %s', scanPolicy => { + expect(resolveSettings({ scanPolicy }).scanPolicy).toBe(scanPolicy); + }); +}); + +describe('native queries', () => { + it.each([ + "INSERT INTO \"Users\" VALUE {'id': 'x'}", + "UPDATE \"Users\" SET name = 'x' WHERE id = 'a'", + 'DELETE FROM "Users" WHERE id = \'a\'', + ])('refuse the statement %s', async statement => { + const client = fakeClient({ ExecuteStatement: () => ({ Items: [] }) }); + const dataSource = await createDynamoDBDataSource( + { + client, + + liveQueryConnection: 'dynamo', + }, + { + introspection: { + collections: [ + { + name: 'u', + tableName: 'Users', + partitionKey: { name: 'id', type: 'string' }, + fields: [], + }, + ], + }, + }, + )(jest.fn(), jest.fn()); + + await expect(dataSource.executeNativeQuery('dynamo', statement, {})).rejects.toThrow( + 'Only SELECT statements can be run as native queries', + ); + expect(client.calls('ExecuteStatement')).toHaveLength(0); + }); +}); + +describe('a chart computed while a write happens', () => { + it('does not put its stale result back in the cache', async () => { + let release: (value: unknown) => void; + const firstScan = new Promise(resolve => { + release = resolve; + }); + let scans = 0; + const { collection } = buildCollection( + buildSimpleModel(), + { + Scan: () => { + scans += 1; + + return scans === 1 ? firstScan : { Items: [{ age: 2 }] }; + }, + Put: () => ({}), + }, + { chartCacheTtlMs: 60000, scanConcurrency: 1 }, + ); + const sum = new Aggregation({ operation: 'Sum', field: 'age' }); + + const started = collection.aggregate(caller, new Filter({}), sum); + await new Promise(resolve => { + setTimeout(resolve, 0); + }); + await collection.create(caller, [{ id: 'u9', age: 5 }]); + release({ Items: [{ age: 1 }] }); + await started; + + const after = await collection.aggregate(caller, new Filter({}), sum); + + expect(after).toEqual([{ value: 2, group: {} }]); + }); +}); + +describe('an attribute an item does not have', () => { + const model = () => + buildSimpleModel({ + schema: { + id: { type: 'string' }, + name: { type: 'string', nullable: true }, + 'address@@@city': { type: 'string', nullable: true }, + }, + }); + const items = [ + { id: 'a', name: 'x', address: { city: 'Paris' } }, + { id: 'b' }, + { id: 'c', name: null, address: {} }, + ]; + + it('is read as null', async () => { + const { collection } = buildCollection( + model(), + { Scan: () => ({ Items: items }) }, + { scanConcurrency: 1 }, + ); + + const records = await collection.list( + caller, + new PaginatedFilter({ sort: undefined }), + new Projection('id', 'name', 'address@@@city'), + ); + + expect(records.find(record => record.id === 'b')).toEqual({ + id: 'b', + name: null, + 'address@@@city': null, + }); + }); +}); + +describe('the binary and set attributes of an array element', () => { + const build = (handlers: Record) => { + const parent = buildCollection( + buildModel({ + schema: { + customerId: { type: 'string' }, + orderId: { type: 'string' }, + lines: { type: 'array' }, + }, + }), + handlers, + { scanConcurrency: 1 }, + ); + const collection = new ArrayCollection({} as DataSource, parent.collection, { + name: 'order_lines', + arrayField: 'lines', + itemSchema: { + sku: { type: 'string' }, + blob: { type: 'binary', nullable: true }, + labels: { type: 'array', set: true, nullable: true }, + }, + }); + + return { collection, client: parent.client }; + }; + + const order = { customerId: 'c1', orderId: 'o1', lines: [{ sku: 'A' }] }; + const first = new Filter({ + conditionTree: leaf('customerId', 'Equal', 'c1'), + }); + const valuesOf = (call: Record) => Object.values(call.ExpressionAttributeValues); + + it('stores bytes and a set when an element is appended', async () => { + const { collection, client } = build({ + Update: () => ({ Attributes: { lines: [{ sku: 'A' }, { sku: 'N' }] } }), + }); + + await collection.create(caller, [ + { customerId: 'c1', orderId: 'o1', sku: 'N', blob: 'AQID', labels: ['a', 'b'] }, + ]); + + const appended = valuesOf(client.calls('Update')[0]).find( + value => Array.isArray(value) && value.length, + ) as any[]; + const [element] = appended.filter(value => value.sku === 'N'); + expect(Array.from(element.blob)).toEqual([1, 2, 3]); + expect(element.labels).toBeInstanceOf(Set); + }); + + it('leaves an emptied set out of an appended element', async () => { + const { collection, client } = build({ + Update: () => ({ Attributes: { lines: [{ sku: 'N' }] } }), + }); + + await collection.create(caller, [{ customerId: 'c1', orderId: 'o1', sku: 'N', labels: [] }]); + + const [element] = valuesOf(client.calls('Update')[0]).find( + value => Array.isArray(value) && value.length, + ) as any[]; + expect(Object.keys(element)).toEqual(['sku']); + }); + + it('stores bytes and a set when an element is updated', async () => { + const { collection, client } = build({ + Query: () => ({ Items: [order] }), + Update: () => ({}), + }); + + await collection.update(caller, first, { blob: 'AQID', labels: ['x'] }); + + const [bytes, set] = valuesOf(client.calls('Update')[0]); + expect(Array.from(bytes as Uint8Array)).toEqual([1, 2, 3]); + expect(set).toBeInstanceOf(Set); + }); + + it('removes the attribute of an element whose set is emptied', async () => { + const { collection, client } = build({ + Query: () => ({ Items: [order] }), + Update: () => ({}), + }); + + await collection.update(caller, first, { labels: [] }); + + const call = client.calls('Update')[0]; + expect(call.UpdateExpression).toMatch(/^REMOVE #n0\[0\]\.#n1$|^REMOVE #n0\[0\]\.#n1 *$/); + expect(call.ExpressionAttributeValues).toBeUndefined(); + }); + + it('sets and removes in one statement, so the second does not look for a record the first changed', async () => { + const { collection, client } = build({ + Query: () => ({ Items: [{ ...order, lines: [{ sku: 'A', labels: new Set(['old']) }] }] }), + Update: () => ({}), + }); + + await collection.update( + caller, + new Filter({ + conditionTree: and(leaf('customerId', 'Equal', 'c1'), leaf('sku', 'Equal', 'A')), + }), + { sku: 'B', labels: [] }, + ); + + expect(client.calls('Update')).toHaveLength(1); + expect(client.calls('Update')[0].UpdateExpression).toMatch( + /^SET #n0\[0\]\.#n1 = :v0 REMOVE #n0\[0\]\.#n2$/, + ); + }); + + it('stores bytes and a set in the client-side rewrite too', async () => { + const { collection, client } = build({ + Query: () => ({ Items: [order] }), + Update: () => { + throw Object.assign(new Error('The document path provided is invalid for update'), { + name: 'ValidationException', + }); + }, + Get: () => ({ Item: { ...order, lines: [{ sku: 'A', labels: new Set(['old']) }] } }), + Put: () => ({}), + }); + + await collection.update(caller, first, { blob: 'AQID', labels: [] }); + + expect(client.calls('Put')).toHaveLength(1); + const [line] = client.calls('Put')[0].Item.lines; + expect(Array.from(line.blob)).toEqual([1, 2, 3]); + expect(line).not.toHaveProperty('labels'); + }); +}); diff --git a/packages/datasource-dynamodb/test/virtual-collections.test.ts b/packages/datasource-dynamodb/test/virtual-collections.test.ts new file mode 100644 index 0000000000..fc7d95dcb1 --- /dev/null +++ b/packages/datasource-dynamodb/test/virtual-collections.test.ts @@ -0,0 +1,166 @@ +import { createDynamoDBDataSource } from '../src'; +import { fakeClient } from './fake-client'; + +const orders = { + Table: { + TableName: 'Orders', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + }, +}; + +const build = ( + items: unknown[], + virtualCollections = [ + { parentCollection: 'Orders', collectionName: 'lines', arrayField: 'lines' }, + ], +) => { + const logger = jest.fn(); + const client = fakeClient({ + DescribeTable: () => orders, + Scan: () => ({ Items: items }), + }); + + return { + logger, + client, + dataSource: createDynamoDBDataSource({ client, tables: ['Orders'], virtualCollections })( + logger, + jest.fn(), + ), + }; +}; + +describe('virtual collections', () => { + it('exposes an array as a collection whose attributes are inferred from the items', async () => { + const { dataSource } = build([ + { + id: 'o1', + lines: [ + { sku: 'A', qty: 1 }, + { sku: 'B', qty: 2, note: 'x' }, + ], + }, + { id: 'o2', lines: [{ sku: 'C', qty: 3 }] }, + ]); + + const { fields } = (await dataSource).getCollection('lines').schema; + + expect(fields).toMatchObject({ + id: { isPrimaryKey: true, columnType: 'String', isReadOnly: true }, + index: { isPrimaryKey: true, columnType: 'Number', isReadOnly: true }, + sku: { columnType: 'String', isPrimaryKey: false }, + qty: { columnType: 'Number' }, + note: { columnType: 'String' }, + }); + }); + + it('samples the parents that have the array, applying the filter on the items read', async () => { + const { client, dataSource } = build([{ id: 'o1', lines: [{ sku: 'A' }] }, { id: 'o2' }]); + + const result = await dataSource; + + expect(client.calls('Scan')[0].FilterExpression).toBeUndefined(); + expect(result.getCollection('lines').schema.fields.sku).toBeDefined(); + }); + + it('skips a collection whose parent does not exist, and says so', async () => { + const { logger, dataSource } = build( + [], + [{ parentCollection: 'Missing', collectionName: 'lines', arrayField: 'lines' }], + ); + + const result = await dataSource; + + expect(result.collections.map(c => c.name)).toEqual(['Orders']); + expect(logger).toHaveBeenCalledWith( + 'Warn', + expect.stringContaining("Skipping virtual collection 'lines' for array 'Missing.lines'"), + ); + }); + + it('exposes an array of an array, identified by the position in each', async () => { + const { dataSource } = build( + [{ id: 'o1', lines: [{ sku: 'A', parts: [{ n: 1 }, { n: 2 }] }] }], + [ + { parentCollection: 'Orders', collectionName: 'lines', arrayField: 'lines' }, + { parentCollection: 'lines', collectionName: 'parts', arrayField: 'parts' }, + ], + ); + + const { fields } = (await dataSource).getCollection('parts').schema; + + expect( + Object.entries(fields) + .filter(([, field]) => field.type === 'Column' && field.isPrimaryKey) + .map(([name]) => name), + ).toEqual(['id', 'linesIndex', 'index']); + expect(fields.n).toMatchObject({ columnType: 'Number', isPrimaryKey: false }); + }); + + it('skips an array of an array declared before its parent, and says so', async () => { + const { logger, dataSource } = build( + [{ id: 'o1', lines: [{ sku: 'A', parts: [{ n: 1 }] }] }], + [ + { parentCollection: 'lines', collectionName: 'parts', arrayField: 'parts' }, + { parentCollection: 'Orders', collectionName: 'lines', arrayField: 'lines' }, + ], + ); + + const result = await dataSource; + + expect(result.collections.map(c => c.name).sort()).toEqual(['Orders', 'lines']); + expect(logger).toHaveBeenCalledWith( + 'Warn', + expect.stringContaining("Skipping virtual collection 'parts' for array 'lines.parts'"), + ); + }); + + it('uses the item schema a parent array collection declares when the sample is empty', async () => { + const client = fakeClient({ + DescribeTable: () => orders, + Scan: () => ({ Items: [] }), + }); + + const dataSource = await createDynamoDBDataSource( + { + client, + + virtualCollections: [ + { parentCollection: 'Orders', collectionName: 'lines', arrayField: 'lines' }, + { parentCollection: 'lines', collectionName: 'parts', arrayField: 'parts' }, + ], + }, + { + introspection: { + collections: [ + { + name: 'Orders', + tableName: 'Orders', + partitionKey: { name: 'id', type: 'string' }, + fields: [ + { + name: 'lines', + type: 'array', + subType: 'object', + fields: [ + { + name: 'parts', + type: 'array', + subType: 'object', + fields: [{ name: 'n', type: 'number' }], + }, + ], + }, + ], + }, + ], + }, + }, + )(jest.fn(), jest.fn()); + + expect(dataSource.getCollection('parts').schema.fields.n).toMatchObject({ + columnType: 'Number', + }); + }); +}); diff --git a/packages/datasource-dynamodb/tsconfig.eslint.json b/packages/datasource-dynamodb/tsconfig.eslint.json new file mode 100644 index 0000000000..9bdc52705d --- /dev/null +++ b/packages/datasource-dynamodb/tsconfig.eslint.json @@ -0,0 +1,3 @@ +{ + "extends": "../../tsconfig.eslint.json" +} diff --git a/packages/datasource-dynamodb/tsconfig.json b/packages/datasource-dynamodb/tsconfig.json new file mode 100644 index 0000000000..e0d66374ae --- /dev/null +++ b/packages/datasource-dynamodb/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": ["src/**/*"] +} diff --git a/yarn.lock b/yarn.lock index 7f8fc45c4e..4e1521e0f4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -210,6 +210,22 @@ "@smithy/util-utf8" "^4.2.1" tslib "^2.6.2" +"@aws-sdk/client-dynamodb@^3.750.0": + version "3.1155.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/client-dynamodb/-/client-dynamodb-3.1155.0.tgz#46a3e27da555ff2a8669306deb6a0f3d23bdc0ea" + integrity sha512-lbssxoO5CdXhza43hEzzFa0p5P2aa9IYAa5n2l4+g1uVFIKlPybBere3mBBMOUnVwsiVQgV0n2ERg9srNT/L8g== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/credential-provider-node" "^3.972.88" + "@aws-sdk/dynamodb-codec" "^3.973.50" + "@aws-sdk/middleware-endpoint-discovery" "^3.972.34" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/fetch-http-handler" "^5.8.0" + "@smithy/node-http-handler" "^4.12.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/client-kendra@^3.1141.0": version "3.1152.0" resolved "https://registry.yarnpkg.com/@aws-sdk/client-kendra/-/client-kendra-3.1152.0.tgz#d20ff97082bb6e371215c405a0a903620d144125" @@ -332,6 +348,20 @@ bowser "^2.11.0" tslib "^2.6.2" +"@aws-sdk/core@^3.978.5": + version "3.978.5" + resolved "https://registry.yarnpkg.com/@aws-sdk/core/-/core-3.978.5.tgz#1c9442d0cf4d7acb1c51a4a35a804033f995812a" + integrity sha512-ruI0AWqcrfL20XDS6SwZIJFPOBZ6N0ADa9nLzWlBhxk1aoUjbUX3JkTR9vEmk7o35XpAHi68ahZ4T6RE2U9XyA== + dependencies: + "@aws-sdk/types" "^3.975.0" + "@aws-sdk/xml-builder" "^3.972.41" + "@aws/lambda-invoke-store" "^0.3.0" + "@smithy/core" "^3.36.1" + "@smithy/signature-v4" "^5.7.4" + "@smithy/types" "^4.19.0" + bowser "^2.11.0" + tslib "^2.6.2" + "@aws-sdk/crc64-nvme@^3.972.1": version "3.972.1" resolved "https://registry.yarnpkg.com/@aws-sdk/crc64-nvme/-/crc64-nvme-3.972.1.tgz#c32bd754b8d0240e68d8768e0332b5512360114c" @@ -384,6 +414,17 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-env@^3.972.76": + version "3.972.76" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.76.tgz#4433bf572620128375d98f1ceca8912003124734" + integrity sha512-YxmL1Ar7bV9ZkJvtv+lj+MF6le76NnZnbmgySyZuxpZaCv0/1lZ3KxKy6RIj1h9ui2hWnVjZxbmTMeO0y/NrQA== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/credential-provider-http@^3.972.13": version "3.972.13" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.13.tgz#e3874a8ef3902ebbd3d36225fdb4b9b711939625" @@ -426,6 +467,19 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-http@^3.972.78": + version "3.972.78" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.78.tgz#52ad8d402d5b216183220f787aba61ca40be8fbf" + integrity sha512-3VHknDRngoebsaqyT1iR75b5cBT2zfEtODK6yYmYltljkK1HEAQLs2FEd1IpWqAeOYdMGaG9wikngWLTufY7RA== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/fetch-http-handler" "^5.8.0" + "@smithy/node-http-handler" "^4.12.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/credential-provider-ini@^3.972.11": version "3.972.11" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.11.tgz#43a1c4c926fbb1a56b350f3afa17c8f095de27ea" @@ -484,6 +538,25 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-ini@^3.973.21": + version "3.973.21" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.21.tgz#c64cf6bd91de0e0619a1d6de09886bee4165e017" + integrity sha512-aLx9hbCX6mGAyv4ZZd8UagJ4RASHVJw6BFDSIyuKmLkqNL6H47EjBPK6qaJ4B5TDV6mc+vMa0SJFYyo4mkvVQg== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/credential-provider-env" "^3.972.76" + "@aws-sdk/credential-provider-http" "^3.972.78" + "@aws-sdk/credential-provider-login" "^3.972.83" + "@aws-sdk/credential-provider-process" "^3.972.76" + "@aws-sdk/credential-provider-sso" "^3.973.20" + "@aws-sdk/credential-provider-web-identity" "^3.972.82" + "@aws-sdk/nested-clients" "^3.997.50" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/credential-provider-imds" "^4.5.2" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/credential-provider-login@^3.972.11": version "3.972.11" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.11.tgz#33fa3b2e95e77e48be601305773622d0aee30500" @@ -522,6 +595,18 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-login@^3.972.83": + version "3.972.83" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.83.tgz#d54c74c4f8de297044ea79c679f7fc984878d44b" + integrity sha512-9bJU6FjRacF4FK/QHf8Dyxhur7B1grRj8C1Yh4IkpoFW2exQflPdCv0VrbemLW9ZctLNle+AE7TETs9KNvuLYg== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/nested-clients" "^3.997.50" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/credential-provider-node@^3.972.12": version "3.972.12" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.12.tgz#6ca15d141f400851f0b742aab98db652eee4575c" @@ -574,6 +659,23 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-node@^3.972.88": + version "3.972.88" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.88.tgz#41f751b1711a8afcc6a44a573e54f332ca41adc1" + integrity sha512-AG0KID5qhQPzrFkh5b0Cws1Amytvzo0L0Xj6hMgJXEwD6ZXAqninDhSNAWJiN5zU4YVVZnuidnqwPwYJ04IamA== + dependencies: + "@aws-sdk/credential-provider-env" "^3.972.76" + "@aws-sdk/credential-provider-http" "^3.972.78" + "@aws-sdk/credential-provider-ini" "^3.973.21" + "@aws-sdk/credential-provider-process" "^3.972.76" + "@aws-sdk/credential-provider-sso" "^3.973.20" + "@aws-sdk/credential-provider-web-identity" "^3.972.82" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/credential-provider-imds" "^4.5.2" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/credential-provider-process@^3.972.11": version "3.972.11" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.11.tgz#3434eafdc92d706eb6838c1efc5e5d04f341a731" @@ -608,6 +710,17 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-process@^3.972.76": + version "3.972.76" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.76.tgz#0155e83b93379083bbb51686fc4662b467f1acc2" + integrity sha512-pdhKFTXv1zPYhh92MTlp0jVbLWvUEV9v+rziuEkoskGGQ2iJskxHTO38nA8UAbT/N4IM3n8OZJ9rIY5IBO15/A== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/credential-provider-sso@^3.972.11": version "3.972.11" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.11.tgz#ed8184019f0e046ea07d3ca19f2a52235afc5554" @@ -648,6 +761,19 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-sso@^3.973.20": + version "3.973.20" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.20.tgz#1d5ad65a5d1e86516bc27f134140ee0b82d27e0b" + integrity sha512-DlxrAB3MnjNLx/da7b89W+3DyJQYso/q7Ns15MsST2IV2nkAYQOYTUCaffNSjgrpMP99Dm2mV2jxzcbyF94ZfA== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/nested-clients" "^3.997.50" + "@aws-sdk/token-providers" "3.1155.0" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/credential-provider-web-identity@^3.972.11": version "3.972.11" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.11.tgz#976a5499f6828b00e068a3c81b7400278e71ad03" @@ -685,6 +811,18 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/credential-provider-web-identity@^3.972.82": + version "3.972.82" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.82.tgz#0919b59684666ff0c5649da7bb48839c99b277c1" + integrity sha512-LUo46IqPTJ4xKBsrLuoaxiv/D2GTBCcFDMD1y3jKIqPR/p7Rw14RJFZ0iRWftqZtyLPQX9JMGnSi3mBu4y/pNA== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/nested-clients" "^3.997.50" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/credential-providers@^3.186.0": version "3.997.0" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-providers/-/credential-providers-3.997.0.tgz#1b77e660142e13e5cf483fa70d810051bcea3424" @@ -711,6 +849,24 @@ "@smithy/types" "^4.12.1" tslib "^2.6.2" +"@aws-sdk/dynamodb-codec@^3.973.50": + version "3.973.50" + resolved "https://registry.yarnpkg.com/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.973.50.tgz#0042410ed4319b06f4c9299c74f1eae0bb698fb6" + integrity sha512-1jn3q2Ik/qUjPw1su5Abu6nzeDutzYbNUYKt+AqUh+45rb+MVWhgP1QWarTuxrm4MJKcSLVHLCrvi0rY5jmJZw== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + +"@aws-sdk/endpoint-cache@^3.972.11": + version "3.972.11" + resolved "https://registry.yarnpkg.com/@aws-sdk/endpoint-cache/-/endpoint-cache-3.972.11.tgz#0d4a52d6307e240d0f404c649f9a58b773511d03" + integrity sha512-8q1ICxcDjHId3bBryuu/j+1L9y5/3uQnwzLDt5j2ElcjZSoWmFtymdJy7OjLrluSMe0Z4mq5bcH4fxBXvlEHfw== + dependencies: + mnemonist "0.38.3" + tslib "^2.6.2" + "@aws-sdk/eventstream-handler-node@^3.972.36": version "3.972.36" resolved "https://registry.yarnpkg.com/@aws-sdk/eventstream-handler-node/-/eventstream-handler-node-3.972.36.tgz#ee96a93da3ea30242259b99df858a012cd7eb057" @@ -721,6 +877,17 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/lib-dynamodb@^3.750.0": + version "3.1155.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/lib-dynamodb/-/lib-dynamodb-3.1155.0.tgz#3b802780c7ac2258b35acdba5f9e2d349ea6b481" + integrity sha512-/84eZiKfFPMPozNPMxX346C2KdziZOqrXk7lsBmcteE8HmFTphapaATgCq0785ts4TmK1J3L/65EdqU8rzYtSw== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/util-dynamodb" "^3.996.9" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/middleware-bucket-endpoint@^3.972.4": version "3.972.4" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.972.4.tgz#03f07cce35d0983c5767e4a5ac26dafeabb7050c" @@ -734,6 +901,17 @@ "@smithy/util-config-provider" "^4.2.1" tslib "^2.6.2" +"@aws-sdk/middleware-endpoint-discovery@^3.972.34": + version "3.972.34" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.34.tgz#431383cc739b586537ff166c72ba13f84470f1dc" + integrity sha512-hwPWSvttg+bEpcM9e8HdVJF/oz79xInjnou23+GdsbDv75DVDLGhn2XhIbuNghp5SqyjtngXEI+K22Vq3VvSxw== + dependencies: + "@aws-sdk/endpoint-cache" "^3.972.11" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/middleware-eventstream@^3.972.31": version "3.972.31" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-eventstream/-/middleware-eventstream-3.972.31.tgz#9a15a621dde2c880fe084613378d55d9070a740d" @@ -940,6 +1118,20 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/nested-clients@^3.997.50": + version "3.997.50" + resolved "https://registry.yarnpkg.com/@aws-sdk/nested-clients/-/nested-clients-3.997.50.tgz#fcf3263fc47040aac27990970017d737a2a21683" + integrity sha512-nKb8qu4QsnpZTq/6JIXc72mzuzfh7BK4sUQkpQznPec2z4W7Ob2xYkcM4i5e1yGTBnbJVeFv/0sf3jcz85hclA== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/signature-v4-multi-region" "^3.996.49" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/fetch-http-handler" "^5.8.0" + "@smithy/node-http-handler" "^4.12.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/region-config-resolver@^3.972.4": version "3.972.4" resolved "https://registry.yarnpkg.com/@aws-sdk/region-config-resolver/-/region-config-resolver-3.972.4.tgz#aa178d3f3cbac33e2f5ade15a160dad2dfd062c6" @@ -997,6 +1189,16 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/signature-v4-multi-region@^3.996.49": + version "3.996.49" + resolved "https://registry.yarnpkg.com/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.49.tgz#2e7d874620cc19922f613f8f1e0c4c0cb38b9598" + integrity sha512-nvuAZNRrRRVCiARRKsYlQ0VnXiEQiq7vXPH+UA+LSo+kN3Djl9eo/V2r0udkw85PnsNshmv2Em1clld5M3j67w== + dependencies: + "@aws-sdk/types" "^3.975.0" + "@smithy/signature-v4" "^5.7.4" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/token-providers@3.1129.0": version "3.1129.0" resolved "https://registry.yarnpkg.com/@aws-sdk/token-providers/-/token-providers-3.1129.0.tgz#a4a11a789d7259874ccf3733e538fc70f8a289e4" @@ -1045,6 +1247,18 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/token-providers@3.1155.0": + version "3.1155.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/token-providers/-/token-providers-3.1155.0.tgz#201abec26c2815fb60bce72583d5fc30e4b63d36" + integrity sha512-iIcQnYLhWuq9qOLQbIWFrxF3PF0svVeZvHnUS4DUnO4zUFmFHIyGyjqrntXG+e/5PUgfavV0pYfDIKHxTzN2LA== + dependencies: + "@aws-sdk/core" "^3.978.5" + "@aws-sdk/nested-clients" "^3.997.50" + "@aws-sdk/types" "^3.975.0" + "@smithy/core" "^3.36.1" + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/token-providers@3.997.0": version "3.997.0" resolved "https://registry.yarnpkg.com/@aws-sdk/token-providers/-/token-providers-3.997.0.tgz#3941e72c86c70776f60030947ae88d428e8a1591" @@ -1082,6 +1296,14 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@aws-sdk/types@^3.975.0": + version "3.975.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.975.0.tgz#15fb614cf1877ee41c5f08edfd21818dc90edd1d" + integrity sha512-WQlhs6juByzWOzj5fwxOx0mPJaEqMcG/sDpl3Id61B3XhKE63xY+0QWcnG+sWR/sCfprmreyUo1lM7IKZmHMzw== + dependencies: + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@aws-sdk/util-arn-parser@^3.972.2": version "3.972.2" resolved "https://registry.yarnpkg.com/@aws-sdk/util-arn-parser/-/util-arn-parser-3.972.2.tgz#ef18ba889e8ef35f083f1e962018bc0ce70acef3" @@ -1089,6 +1311,13 @@ dependencies: tslib "^2.6.2" +"@aws-sdk/util-dynamodb@^3.996.9": + version "3.996.9" + resolved "https://registry.yarnpkg.com/@aws-sdk/util-dynamodb/-/util-dynamodb-3.996.9.tgz#493c00ffec4280450492966867197bbd26d50b83" + integrity sha512-16x2tRvl7OYpZ0W/DdFJieFriD13+RvuRBDbe5sj/tCEfK86HSGd7I2s5j0ivz8p6KWGkS+5wKRO9OliJkjUOQ== + dependencies: + tslib "^2.6.2" + "@aws-sdk/util-endpoints@^3.996.1": version "3.996.1" resolved "https://registry.yarnpkg.com/@aws-sdk/util-endpoints/-/util-endpoints-3.996.1.tgz#1fd0f3800c899f80be067318795301b85412484d" @@ -4393,6 +4622,14 @@ "@smithy/types" "^4.19.0" tslib "^2.6.2" +"@smithy/core@^3.36.1": + version "3.36.2" + resolved "https://registry.yarnpkg.com/@smithy/core/-/core-3.36.2.tgz#dc8348dbdd9604a297e93ab791266d6511c2038c" + integrity sha512-oRw0rB9QSUiCZfEs23G5F4Af+b2rQt1lItIICWu5IejFNd+uPM8SWqiKLMpQ6bX1Q7slsLbjuoE2niErFtkVPQ== + dependencies: + "@smithy/types" "^4.19.0" + tslib "^2.6.2" + "@smithy/credential-provider-imds@^4.2.10", "@smithy/credential-provider-imds@^4.2.9": version "4.2.10" resolved "https://registry.yarnpkg.com/@smithy/credential-provider-imds/-/credential-provider-imds-4.2.10.tgz#cae502b28257a110fc472a1531c19fbc4ba07037" @@ -13606,6 +13843,13 @@ mkdirp@1.0.4, mkdirp@^1.0.3, mkdirp@^1.0.4: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== +mnemonist@0.38.3: + version "0.38.3" + resolved "https://registry.yarnpkg.com/mnemonist/-/mnemonist-0.38.3.tgz#35ec79c1c1f4357cfda2fe264659c2775ccd7d9d" + integrity sha512-2K9QYubXx/NAjv4VLq1d1Ly8pWNC5L3BrixtdkyTegXWJIqY+zLNDhhX/A+ZwWt70tB1S8H4BE8FLYEFyNoOBw== + dependencies: + obliterator "^1.6.1" + mobx-react-lite@^4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/mobx-react-lite/-/mobx-react-lite-4.1.1.tgz#725d74b025235f73dc2ab815766ffe010be2cf8a" @@ -14628,6 +14872,11 @@ object.values@^1.2.1: define-properties "^1.2.1" es-object-atoms "^1.0.0" +obliterator@^1.6.1: + version "1.6.1" + resolved "https://registry.yarnpkg.com/obliterator/-/obliterator-1.6.1.tgz#dea03e8ab821f6c4d96a299e17aef6a3af994ef3" + integrity sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig== + ohash@^2.0.11: version "2.0.12" resolved "https://registry.yarnpkg.com/ohash/-/ohash-2.0.12.tgz#cb97c19888ff31e7b22214cd7924187fc97ef05c"