diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6ebb24549..0d766be0b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -71,6 +71,7 @@ jobs: - forest_admin_datasource_mambu_payments - forest_admin_datasource_graphql_hasura - forest_admin_datasource_pylon + - forest_admin_datasource_intercom services: mongodb: image: mongo:latest @@ -154,7 +155,7 @@ jobs: with: verbose: true oidc: true - files: ${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_active_record/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_customizer/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_toolkit/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rails/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mongoid/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rpc_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_rpc/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_zendesk/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_snowflake/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mambu_payments/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_graphql_hasura/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_pylon/coverage.json + files: ${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_active_record/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_customizer/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_toolkit/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rails/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mongoid/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rpc_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_rpc/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_zendesk/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_snowflake/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mambu_payments/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_graphql_hasura/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_pylon/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_intercom/coverage.json deploy: name: Release package diff --git a/.releaserc.js b/.releaserc.js index 998aa1a24..443aec802 100644 --- a/.releaserc.js +++ b/.releaserc.js @@ -32,7 +32,8 @@ module.exports = { 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb; '+ 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb; '+ 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb; '+ - 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/version.rb; ', + 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/version.rb; '+ + 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb; ', successCmd: '( cd packages/forest_admin_agent && gem build && gem push forest_admin_agent-*.gem );' + '( cd packages/forest_admin_datasource_active_record && gem build && gem push forest_admin_datasource_active_record-*.gem );' + @@ -47,7 +48,8 @@ module.exports = { '( cd packages/forest_admin_datasource_snowflake && gem build && gem push forest_admin_datasource_snowflake-*.gem );' + '( cd packages/forest_admin_datasource_mambu_payments && gem build && gem push forest_admin_datasource_mambu_payments-*.gem );' + '( cd packages/forest_admin_datasource_graphql_hasura && gem build && gem push forest_admin_datasource_graphql_hasura-*.gem );' + - '( cd packages/forest_admin_datasource_pylon && gem build && gem push forest_admin_datasource_pylon-*.gem );' , + '( cd packages/forest_admin_datasource_pylon && gem build && gem push forest_admin_datasource_pylon-*.gem );' + + '( cd packages/forest_admin_datasource_intercom && gem build && gem push forest_admin_datasource_intercom-*.gem );' , }, ], [ @@ -71,6 +73,7 @@ module.exports = { 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb', 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb', 'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/version.rb', + 'packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb', 'package.json' ], }, diff --git a/.rubocop.yml b/.rubocop.yml index 2bb2d3d15..4417897b9 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -43,6 +43,7 @@ Gemspec/RequireMFA: - 'packages/forest_admin_datasource_mambu_payments/forest_admin_datasource_mambu_payments.gemspec' - 'packages/forest_admin_datasource_graphql_hasura/forest_admin_datasource_graphql_hasura.gemspec' - 'packages/forest_admin_datasource_pylon/forest_admin_datasource_pylon.gemspec' + - 'packages/forest_admin_datasource_intercom/forest_admin_datasource_intercom.gemspec' # Offense count: 1 # This cop supports unsafe autocorrection (--autocorrect-all). @@ -135,6 +136,7 @@ Style/MutableConstant: - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb' - 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb' - 'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/version.rb' + - 'packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb' # Offense count: 38 # This cop supports safe autocorrection (--autocorrect). @@ -220,6 +222,7 @@ Style/StringLiterals: - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb' - 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb' - 'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/version.rb' + - 'packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb' # Offense count: 1 # This cop supports safe autocorrection (--autocorrect). @@ -264,6 +267,8 @@ Metrics/ParameterLists: - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb' - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/configuration.rb' + - 'packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/configuration.rb' + - 'packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb' - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/collections/base_collection.rb' - 'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/base_collection.rb' - 'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/configuration.rb' @@ -302,6 +307,7 @@ Metrics/ModuleLength: - 'packages/forest_admin_datasource_pylon/spec/**/*' - 'packages/forest_admin_rails/spec/**/*' - 'packages/forest_admin_rpc_agent/spec/**/*' + - 'packages/forest_admin_datasource_intercom/spec/**/*' - 'packages/forest_admin_datasource_mongoid/lib/forest_admin_datasource_mongoid/utils/helpers.rb' Metrics/MethodLength: diff --git a/packages/forest_admin_datasource_intercom/.gitignore b/packages/forest_admin_datasource_intercom/.gitignore new file mode 100644 index 000000000..06cfcfb83 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/.gitignore @@ -0,0 +1,8 @@ +*.gem +.bundle/ +Gemfile.lock +Gemfile-test.lock +coverage/ +pkg/ +tmp/ +.rspec_status diff --git a/packages/forest_admin_datasource_intercom/.rspec b/packages/forest_admin_datasource_intercom/.rspec new file mode 100644 index 000000000..34c5164d9 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/packages/forest_admin_datasource_intercom/Gemfile b/packages/forest_admin_datasource_intercom/Gemfile new file mode 100644 index 000000000..c229ff1d5 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/Gemfile @@ -0,0 +1,16 @@ +source 'https://rubygems.org' + +gemspec + +gem 'forest_admin_datasource_customizer' +gem 'forest_admin_datasource_toolkit' +gem 'rake', '~> 13.0' +gem 'rubocop', '1.86.1' +gem 'rubocop-performance', '1.26.1' +gem 'rubocop-rspec', '3.9.0' + +group :development, :test do + gem 'rspec', '~> 3.0' + gem 'simplecov', '~> 0.22', require: false + gem 'webmock', '~> 3.0' +end diff --git a/packages/forest_admin_datasource_intercom/Gemfile-test b/packages/forest_admin_datasource_intercom/Gemfile-test new file mode 100644 index 000000000..8b433e2b5 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/Gemfile-test @@ -0,0 +1,19 @@ +source 'https://rubygems.org' + +# Specify your gem's dependencies in forest_admin_datasource_intercom.gemspec +gemspec + +gem 'rake', '~> 13.0' +gem 'rubocop', '1.86.1' +gem 'rubocop-performance', '1.26.1' +gem 'rubocop-rspec', '3.9.0' + +group :development, :test do + gem 'forest_admin_datasource_customizer', path: '../forest_admin_datasource_customizer' + gem 'forest_admin_datasource_toolkit', path: '../forest_admin_datasource_toolkit' + gem 'rspec', '~> 3.0' + gem 'simplecov', '~> 0.22', require: false + gem 'simplecov-html', '~> 0.12.3' + gem 'simplecov_json_formatter', '~> 0.1.4' + gem 'webmock', '~> 3.0' +end diff --git a/packages/forest_admin_datasource_intercom/LICENSE b/packages/forest_admin_datasource_intercom/LICENSE new file mode 100644 index 000000000..1a15894fa --- /dev/null +++ b/packages/forest_admin_datasource_intercom/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2023-2026 Forest Admin + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/forest_admin_datasource_intercom/README.md b/packages/forest_admin_datasource_intercom/README.md new file mode 100644 index 000000000..b257ff9e2 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/README.md @@ -0,0 +1,801 @@ +# Forest — Intercom datasource + +Surface [Intercom](https://www.intercom.com) conversations, tickets, contacts, companies, teammates, +teams, ticket types and ticket states as Forest collections. + +This release is **read only**. Rows, record details, exact counts, server-side filtering, the +relations between collections and the conversation thread work; writes, business actions, notes, +tags, segments and group-by on the paginated collections arrive in the lots after it — see "What is +not here yet". + +## Installation + +```ruby +# Gemfile +gem 'forest_admin_datasource_intercom' +``` + +## Usage + +```ruby +# app/lib/forest_admin_rails/create_agent.rb +ForestAdminAgent::Builder::AgentFactory.instance.add_datasource( + ForestAdminDatasourceIntercom::Datasource.new( + access_token: ENV['INTERCOM_ACCESS_TOKEN'], + region: :eu # :us (default), :eu or :au + ) +) +``` + +The token is the access token of a private app, created in Intercom's Developer Hub under +*Configure › Authentication*. OAuth is out of scope: it belongs to a control plane distributing a +connector, not to an agent reading one workspace. + +`Client#me` is the health check — it returns the admin the token belongs to, and is the one call +that verifies the pinned API version was honoured. + +### Configuration + +| Option | Default | What it is for | +| --- | --- | --- | +| `access_token` | — | Required. The private app's bearer token. | +| `region` | `:us` | `:us`, `:eu`, `:au`. A workspace answers in its own region only. | +| `base_url` | from `region` | Wins over `region`. For a gateway or a mock server. Refused if it carries credentials: the `Authorization` header holds the Intercom token, so a credentialed egress proxy goes in `HTTPS_PROXY` instead. | +| `api_version` | `'2.16'` | Sent as `Intercom-Version` on every request. | +| `open_timeout` / `timeout` | `5` / `30` | A request that already has a page on screen. | +| `boot_open_timeout` / `boot_timeout` | `3` / `10` | The one read performed while the agent starts. | +| `retry_policy` | `RetryPolicy.new` | Statuses, verbs and backoff. | +| `boot_retry_policy` | `RetryPolicy.boot` | One quick retry; gives up rather than waiting a 429 out. | +| `rate_limiter` | `RateLimiter.new` | `nil` takes the pacing out of the stack. | +| `reference_cache_ttl` | `60` | Seconds the workspace's reference lists are held for. `0` re-reads them every time. | +| `adapter` | persistent, 25 connections | The Faraday adapter. `:net_http` opts out of keep-alive; `[name, options]` sizes the pool, which is read against the server's thread count. | +| `concurrent_reads` | `1` | How many records a read by id fetches at once. Every read in flight holds a connection of the pool — see [Reading records by id in parallel](#reading-records-by-id-in-parallel). | + +**Pin the region explicitly.** `api.intercom.io` does route to the right one, but a workspace under +GDPR wants its requests reaching the European host and nothing else. + +**The version is pinned on purpose.** Without the header a request follows the workspace's own +default version, which an operator can change on Intercom's side — and the payloads change shape +underneath. Intercom echoes the version it served, so `me` compares the two and logs a warning when +the pin was not honoured, rather than raising: running against a version we did not ask for still +beats not running. + +### Token permissions + +A read-only token is enough, and is what to recommend for this lot. A permission the token lacks +costs **columns or a collection, never the boot of the agent**: the three boot-time introspections +each degrade to no attribute column, a collection whose endpoint answers 403 fails its own page, and +a token that cannot read `/admins` or `/teams` leaves the `admin_names` / `team_names` column empty +rather than failing the page it is on. A token denied contacts or companies costs those two +collections and the `contact_name` column, and leaves everything else standing. + +A **relation is the exception**, and it is worth knowing before scoping a token: resolving one reads +the target endpoint, and that read is not guarded the way the names above are. A token denied +`/admins` fails any page projecting `admin_assignee:name`, and fails the related list behind +`IntercomTeam#admins` — the failure lands on the collection being read, not on the one that was +denied. Scope the token to the endpoints in the table below, or to none of them. + +## Collections + +| Collection | Endpoint | Paginated | Countable | +| --- | --- | --- | --- | +| `IntercomConversation` | `GET /conversations`, `POST /conversations/search`, `GET /conversations/{id}` | cursor | yes, exactly | +| `IntercomTicket` | `POST /tickets/search`, `GET /tickets/{id}` | cursor | yes, exactly | +| `IntercomAdmin` | `GET /admins` | read whole | yes, exactly | +| `IntercomTeam` | `GET /teams` | read whole | yes, exactly | +| `IntercomTeamMembership` | `GET /teams`, `GET /admins` | read whole | yes, exactly | +| `IntercomTicketType` | `GET /ticket_types` | read whole | yes, exactly | +| `IntercomTicketState` | `GET /ticket_states` | read whole | yes, exactly | +| `IntercomContact` | `GET /contacts`, `POST /contacts/search`, `GET /companies/{id}/contacts` | cursor | yes, exactly | +| `IntercomCompany` | `POST /companies/list`, `GET /companies?...`, `GET /companies/{id}` | **offset** | yes, exactly | + +Three tiers, and they behave differently on purpose. + +**Read whole** — admins, teams, team memberships, ticket types, ticket states. Their endpoints answer in one response, +so filtering, sorting, paging and counting them in memory is *exact*: the records in hand are every +record Intercom holds. These are the only collections that can be filtered, sorted and grouped in +this lot, and the only ones a chart may group by. The cost is bandwidth, not correctness. + +**Cursor** — conversations, tickets and contacts. What is in hand is a page of something far larger, +so nothing is filtered or sorted in memory. Three routes and no fourth: no condition walks the +listing, `id equals X` reads the record through its own endpoint, and anything else is translated +into Intercom's search DSL and walked through the search endpoint. What the translation cannot +express is **refused by name** — see [Filtering](#filtering). Contacts add two routes of their own, +both described under [Contacts](#contacts). + +**Offset** — companies, and nothing else. `POST /companies/list` takes a **page number**, which is +what a list view asks for: page 7 is one request rather than six pages walked to reach it, with no +cap and no truncation warning. It is the one place the [first limitation +below](#what-the-api-cannot-do-and-what-this-does-about-it) does not apply. What it pays for that is +filtering — there is no company search endpoint at all, so what a filter may say is a handful of +exact lookups and nothing else. See [Companies](#companies). + +## Relations + +Intercom joins nothing: a ticket carries an assignee id, and the teammate behind it is a second read +of a second endpoint. What makes these relations affordable is what the collection on the far end +costs to read by id, and the three tiers do not cost the same: + +| Target | Read by id | A page of rows costs | Fan-out | +| --- | --- | --- | --- | +| `IntercomAdmin`, `IntercomTeam`, `IntercomTicketState`, `IntercomTicketType` | read whole | **one request**, whatever the page holds | unbounded | +| `IntercomContact` | `id IN [...]`, 15 at a time | one request per 15 distinct contacts | unbounded | +| `IntercomCompany` | `GET /companies/{id}` | **one request per distinct account** | bounded, see below | + +The price is per target *collection*, not per relation: a ticket's `state` and `previous_state` are +one read of `/ticket_states`, over the ids both of them name. + +**The one relation with a ceiling is `IntercomContact#company`.** There is no bulk read for a +company — `/companies/scroll` is [deliberately rejected](#companies) — so each distinct account on +the page costs a request, and past +[`MAX_RELATION_READS`](lib/forest_admin_datasource_intercom/collections/offset_collection.rb) +distinct accounts the read is **refused by name** rather than resolved for the first slice and left +nil for the rest. Every page size a list view offers sits under that figure; an export or a segment +resolved whole does not, and the message says to leave the column out or read fewer rows at a time. +The alternative — a nil where an account exists — is the one answer this datasource must not give, +and a log line nobody reads is not a substitute for it. + +*Exactly*, with one more bound worth naming: "read whole" is what the endpoint answers, and +`fetch_all` stops after [`MAX_COLLECTED_PAGES`](lib/forest_admin_datasource_intercom/client.rb) pages +if Intercom paginates one of these on its own — it logs when it does. A workspace whose `/admins` or +`/teams` runs past that cap resolves the relations pointing at the records it dropped as empty. The +figure is sized for reference collections, which is what the first tier above is. + +| Collection | Relation | Target | Filterable through | +| --- | --- | --- | --- | +| `IntercomConversation` | `admin_assignee`, `closed_by` | `IntercomAdmin` | yes | +| `IntercomConversation` | `team_assignee` | `IntercomTeam` | yes | +| `IntercomTicket` | `admin_assignee` | `IntercomAdmin` | yes | +| `IntercomTicket` | `team_assignee` | `IntercomTeam` | yes | +| `IntercomTicket` | `ticket_type` | `IntercomTicketType` | yes | +| `IntercomTicket` | `state`, `previous_state` | `IntercomTicketState` | **no** — read and navigate only | +| `IntercomTeam` | `admins` | `IntercomAdmin` | no (many-to-many) | +| `IntercomAdmin` | `teams` | `IntercomTeam` | no (many-to-many) | +| `IntercomTeamMembership` | `team`, `admin` | `IntercomTeam`, `IntercomAdmin` | yes | +| `IntercomConversation` | `contact` | `IntercomContact` | yes | +| `IntercomTicket` | `contact` | `IntercomContact` | **spec, unprobed** — see below | +| `IntercomContact` | `owner` | `IntercomAdmin` | yes | +| `IntercomContact` | `company` | `IntercomCompany` | **no** — read and navigate only | +| `IntercomContact` | `conversations`, `tickets` | `IntercomConversation`, `IntercomTicket` | no (one-to-many) | +| `IntercomCompany` | `contacts` | `IntercomContact` | no (one-to-many) | + +Every one of them is **read-only**: this lot writes nothing, and Intercom exposes no endpoint that +writes a team membership at all. + +**`IntercomTeamMembership` exists because Intercom's does not.** The workspace carries the +membership on the team (`admin_ids`) and on the teammate (`team_ids`) both and exposes no resource +for the pair, while a many-to-many needs a collection to travel through. It is synthesized from +`GET /teams`, one record per pair, keyed `teamId:adminId`. Without it, both sides read as an array of +ids nobody can click. + +Two consequences of travelling through it are worth knowing. A **related list of teammates is +ordered by the membership, not by the teammate**: the agent hands the through collection the columns +of the collection the relation reaches, so an order on `name` or `email` cannot be resolved there and +is logged rather than silently dropped. And a `admin_ids` entry naming a teammate `/admins` does not +answer -- one who left, one outside the token's reach -- **drops out of the related list and of its +count** instead of appearing as an empty row: the memberships are read against `/admins`, which the +reference cache shares with `IntercomAdmin`. + +Alongside it, a team names its teammates (`admin_names`) and a teammate its teams (`team_names`) on +the row itself, so a list view reads without a join. **Those replace the arrays of ids** the first +lots published: one readable form plus a relation to navigate, rather than two ways to read one fact. +They are read only when a projection asks for them, and a token that cannot read the other side +costs the column and nothing else — never the page, and never the relation. + +**The 360 degrees is those last four rows.** From a ticket or a conversation, `contact` reaches the +person who wrote in; from them, `conversations` and `tickets` list everything they ever opened, and +`company` reaches their account, whose `contacts` lists their colleagues. Each of those lists is one +request: `/conversations/search` matches a conversation against one of its contact ids, and +`GET /companies/{id}/contacts` answers the contacts of an account — which is the one relation +`/contacts/search` could not have resolved, filtering no company field. + +**A conversation has several contacts, and the relation names the first of them** — the same one +`contact_name` and `contact_count` describe, so the column and the relation cannot disagree. The +others are a hop away: open that contact and read their conversations. The alternative, a +many-to-many through a join collection, would have been the honest cardinality at the price of three +collections of plumbing in the interface; naming the first contact and counting them is what lot 1 +already published, and lot 4 promotes it rather than replacing it. + +Two of these carry a caveat worth reading before scoping a token or writing a segment. The **ticket +side is a `spec` row the probe has not confirmed**: whether `/tickets/search` filters on +`contact_ids` at all is unmeasured, and if it does not, the relation stays navigable and the filter +moves to the refusal table — exactly what happened to the ticket `state`. And **the company +traversal is not offered**: `/contacts/search` filters no company field, so the relation is +published unfilterable and the agent refuses `company:name` before it reaches this datasource. Filter +from the company side instead: open the account and read its `contacts`. + +The same rule settled the ticket labels: `state_label` and `ticket_type_name` stay on the row, +`state_category` and `state_external_label` are gone — they are a hop away, on the `state` relation, +and neither was ever filterable, so no segment, scope or saved filter could rest on them. + +A relation reads its target **undecorated**, so a permission scope or a segment defined on the target +does not narrow what a relation resolves — the same way a native datasource joins a table without +applying the scopes of the collection mapped to it. + +One semantic worth stating plainly: **a row whose foreign key is null matches no relation filter**, +the way a join drops it, negated filters included. A ticket with no assignee is not "assigned to +someone other than Marie". + +## What the API cannot do, and what this does about it + +Where Forest asks for something Intercom has no equivalent for, this datasource **refuses with a +message naming the reason** rather than answering something that looks right and is not. Those +arrive as a 400 carrying the text. + +- **No offset pagination, except on companies.** Intercom hands out the page after a cursor and + documents that jumping to page N is unsupported, so reaching page 20 costs 20 sequential requests. + The walk is capped at 50 pages / 7 500 records and **every route out of it that is short of what + was asked for is logged**, naming the window it stopped in: the two caps, and the two defensive + stops — a page that advertises a next cursor and holds nothing, and a cursor already followed. + Intercom does neither of the last two today, which is exactly why they are reported rather than + taken for the end of the data. `POST /companies/list` is the exception and takes a page number, + which is why companies escape the walker and its caps entirely. +- **Duplicates on a moving dataset.** Intercom documents that records modified between two paginated + requests can be served twice; the walk deduplicates by id. The missed counterpart is inherent to + cursor pagination and cannot be repaired — it is documented rather than papered over. +- **One endpoint sorts, and it is `/contacts/search`.** Everything else comes back in the order the + API imposes: `POST /companies/list` has no order parameter at all, and the other two search + endpoints **accept a `sort` and ignore it** — measured, it raises nothing and changes nothing. + Since that is undetectable at runtime, no column of `IntercomConversation`, `IntercomTicket` or + `IntercomCompany` is declared sortable and a requested order is reported in the log. The + collections read whole sort in memory, exactly, and Contacts sort server-side on the columns the + measured table declares — see [Contacts](#contacts). One route of Contacts cannot carry it either: + a read by id cuts the window in the order the ids were named, so ordering what comes back would + order a slice picked by something else. That order is reported in the log too. +- **No aggregate endpoint.** Counting is free and exact — `total_count` counts what the query names, + not what a page held — so the record counter is one request. Anything beyond a count is refused on + the cursor collections: grouping over the pages a walk collected would look exact while answering + a fraction. +- **`per_page` is refused past 150**, with `invalid_per_page` and no silent downgrade, so the page + size is bounded before the request leaves. Tickets are bounded far lower still: **25**, because + the search response carries the whole timeline of every ticket and Intercom offers no field + selection. Provisional, pending measurement against real response sizes. +- **No `GET /tickets` at all.** Even an unfiltered ticket list goes through `POST /tickets/search` + with a predicate matching everything. +- **The envelope key is not always `data`.** Measured: `/tickets/search` answers under `tickets`, + `/admins` under `admins`, `/teams` under `teams`. A response carrying neither the expected key nor + `data` is refused rather than read as an empty page. + +## Filtering + +`POST /conversations/search` and `POST /tickets/search` answer the condition trees Forest sends, on +the fields Intercom really filters and with the operators each endpoint really validates. Anything +else is **refused with a message naming what to change** — a condition dropped on the way out comes +back as an unfiltered page that looks filtered, which is the one answer this datasource must not +give. A refusal costs no request: it is raised before anything leaves the process. + +### The table is data, and it says where each row comes from + +The fields a search endpoint filters are not the fields its specification lists. Measured: +`/tickets/search` refuses `company_id` with `invalid_field` although every ticket carries one. So +the source of truth is a committed table — `lib/forest_admin_datasource_intercom/query/search_fields.yml` +— one row per column, each carrying its provenance: + +| `source` | What it means | +| --- | --- | +| `measured` | observed against a real workspace, by `forest_admin_intercom_probe` or during the spike | +| `spec` | read off Intercom's documentation, and therefore still a candidate | + +Every `filter_operators` a column publishes is **derived** from that table, so a column cannot +advertise a filter the translator would then refuse, and a column the table does not carry +advertises nothing at all. That is the mechanism, and it holds whatever the rows say. + +**What the rows say today is mostly `spec`: 18 of 89 are measured, and no endpoint has been probed +end to end** — all three carry `measured_at: null`, which is what `Endpoint#measured?` reports. The +measured rows are the ones a spike went out of its way to check: the date operators on each +endpoint, which disagree between them, `id IN` on `/contacts/search`, `contact_ids` on +`/conversations/search`, and the refusals a read confirmed — `company_id` on `/tickets/search` +above all, alongside the columns the agent derives rather than reads. Everything else is Intercom's +documentation, and the disagreement above is why that is a candidate rather than a promise. Those +two figures are asserted against the file, so they cannot drift from it. + +So the first thing to do against a customer's workspace is to run the probe. The rows worth watching +first, in the order they will hurt: + +1. **`admin_assignee_id` and `team_assignee_id`**, on both search endpoints. Typed `string` here; + Intercom documents them as `Integer` and answers `data_invalid` on a value whose type it does not + accept. These carry the `admin_assignee` and `team_assignee` relations — the filter an ops team + reaches for first — so a wrong type here is the most expensive `spec` row in the file; +2. **`state_id` on `/tickets/search`** — the table carries no filter on it at all, which is what + keeps the `state` relation read-only. If the endpoint does filter one, a support queue becomes + filterable by state; +3. **`contact_ids` on `/tickets/search`** — the contact relation of a ticket rests on it; +4. **the operators Intercom answers on `custom_attributes.{name}`**, per data type, which is the only + thing keeping those columns display-only; +5. **whether `POST /conversations/search` honours `display_as=plaintext`** — it is sent either way, + and an ignored parameter costs a query string where the honoured one saves every filtered row + from coming back as markup. + +To measure a workspace of your own. The probe ships with the gem — it is what measures the +customer's workspace, and whoever runs it there has the gem installed rather than a clone of +`agent-ruby` — so `bundle install` puts it on the path of the application the datasource is +mounted in: + +```bash +INTERCOM_ACCESS_TOKEN=... bundle exec forest_admin_intercom_probe --endpoint tickets --out measured.yml +``` + +It sends one search per (field, operator) cell, reads Intercom's refusal codes — `invalid_field` for +a field the endpoint does not filter, `data_invalid` for an operator it refuses on that field — and +prints what the committed table promises that Intercom refuses, plus what Intercom accepts that the +table does not know about. It writes evidence rather than rewriting the table, which carries the +prose a generated file would drop. + +### A date filter is day-granular, and the day is the UTC one + +Intercom truncates a date search to the day, at the **UTC** boundary — measured, and against its own +documentation, which promises the workspace's timezone. `> V` answers from the start of the day +*after* V; `< V` answers before the start of V's own day. + +Sent as they come, the two bounds an interval is rewritten into cancel each other out: `today` +reaches the datasource as `> 00:00` and `< 23:59` of one day, which Intercom reads as "from +tomorrow" *and* "before today" — no rows at all, to the most ordinary filter there is. So each bound +is moved to the boundary that makes Intercom answer the day the filter named. + +What follows from that: + +- a bound naming a time of day matches **from the start of that day, or through the end of it**. It + is the granularity the Intercom interface itself filters on; +- a caller in UTC gets exactly the day they asked for; +- a caller in another timezone gets the UTC days their window overlaps — up to a day wider at each + end — and the agent logs that once per filter; +- a date column publishes `>` and `<` only, and no equality. Everything an operator actually uses — + `before`, `after`, `today`, `yesterday`, `past`, `future`, the whole `previous_*` family — is + rewritten by the agent into a pair of those bounds. An equality on an instant is what stays out, + and a day-granular filter could not have honoured it anyway. + +### What is filterable + +| Collection | Filterable on | +| --- | --- | +| `IntercomConversation` | `id`, `state`, `priority`, `open`, `read`, `title`, `admin_assignee_id`, `team_assignee_id`, `source_type`, `source_subject`, `source_body`, `source_delivered_as`, `source_author_email`, `closed_by_id`, `reopen_count`, `part_count`, `ai_agent_participated`, `contact_id`, and the dates `created_at`, `updated_at`, `waiting_since`, `snoozed_until`, `closed_at`, `first_closed_at`, `first_contact_reply_at`, `last_contact_reply_at`, `last_admin_reply_at` | +| `IntercomTicket` | `id`, `open`, `category`, `ticket_type_id`, `admin_assignee_id`, `team_assignee_id`, `contact_id`, `created_at`, `updated_at` | +| `IntercomContact` | `id`, `role`, `name`, `email`, `email_domain`, `phone`, `external_id`, `owner_id`, `unsubscribed_from_emails`, `has_hard_bounced`, `marked_email_as_spam`, `language_override`, `browser`, `browser_language`, `os`, `location_country`, `location_region`, `location_city`, and the dates `created_at`, `updated_at`, `signed_up_at`, `last_seen_at`, `last_contacted_at`, `last_replied_at`, `last_email_opened_at`, `last_email_clicked_at` | +| `IntercomCompany` | `id`, `company_id`, `name` — four lookups and no search endpoint, see [Companies](#companies) | + +**The primary key** is filterable like any other column, but a filter naming it *alone* is not +answered by a search: `id equals X` and `id in [...]` read the record endpoint directly, one request +per record. The search answers it only when something else is filtered alongside it — a permission +scope, a segment, or a second filter. + +**A ticket's `category`** is filtered on the label the ticket shows — `Customer`, `Back-office`, +`Tracker` — and sent as the word `/tickets/search` takes for it: `request`, `task`, `tracker`. Any +other value is refused rather than sent, since Intercom would compare it against a vocabulary the +column never shows. + +**Free-text search** is answered on two collections, each on the one column its endpoint matches text +on: + +| Collection | Searched on | +| --- | --- | +| `IntercomConversation` | `~` on `source.body` — the message that opened the conversation | +| `IntercomContact` | `~` on `email` — what an ops team types when they are looking for someone | + +Intercom matches `~` **per word, not as a substring**: searching `fact` does not find `facture`, and +searching `acme` does not find `camille@acme.test`. `IntercomTicket` exposes no text column its +endpoint matches and refuses a search by name; `IntercomCompany` has no search endpoint at all and +refuses one the same way. Both still show a search bar, so a search typed there — on the list, on a +related list, or in Forest's search across collections — comes back as that refusal rather than as an +empty page. `collection.disable_search` in the agent's customization removes the bar. + +### What is not filterable, and why + +- **every column of a company but two.** There is no `/companies/search`: Intercom looks a company + up by `name`, by `company_id`, by `tag_id` or by `segment_id`, one exact value at a time, and the + first two are the ones that name a column of the collection. Everything else — the industry, the + plan, the monthly spend — is refused by name. Filtering by tag or by segment belongs with the lot + that adds those collections; +- **a contact's `company_count`, `avatar` and `session_count`**, which the endpoint does not filter, + and **its `company_id` beyond a bare `equals`** — the search filters no company field, so + `company_id equals X` is answered by `GET /companies/{id}/contacts`, which is also what serves the + account's `contacts` relation, in one request — **and answers alone**. That + endpoint returns the contacts of the account whole and narrows nothing, so a `company_id equals X` + carrying anything else cannot be answered at all: there is no request that takes both halves. + Which means the related list of an account **is refused as soon as a permission scope or a segment + is defined on `IntercomContact`**, since that is what the agent intersects into the condition. The + refusal names the condition it could not carry alongside the account. Resolving it properly would + mean reading the account's contact ids first and handing `id IN [...]` plus the rest of the tree to + `/contacts/search` — which that endpoint does answer, and which is not in this lot; +- **a set of ids, counted.** `id in [...]` reads one record per id, so counting a set means reading + it, and past what a bulk read fetches the count is refused rather than answered with the number + the truncation left. A collection that advertises an exact count does not answer 25 to a question + about forty records. Contacts are the exception: Intercom counts them, 15 ids per search; +- **a list of more than 15 values** in an `in` or a `not in`, wherever it stands in the filter. + Intercom refuses it with a 400 that names neither the field nor the condition; +- **the custom attributes of a contact or a company.** They are filtered as + `custom_attributes.{name}`, by name — the ambiguity that keeps ticket attributes display-only does + not arise here — but which operators Intercom answers on each data type has not been measured, and + this package publishes no filter it has not seen work. They ship typed and display-only, and the + probe is what turns that around; +- **the columns a ticket derives from its parts** — `closed_at`, `closed_by_name`, `last_reply_at`, + `last_responder_name`, `last_responder_type`. They exist nowhere in Intercom; `/tickets/search` + filters none of them and ignores a sort on them without a word; +- **the account of a ticket** — `company_id`, refused by the endpoint itself with `invalid_field`; +- **the state of a ticket** — the measured table carries no filter on a state id, so `state_id`, + `previous_state_id` and the `state` relation are read and navigated rather than filtered. Whether + the endpoint filters one at all is one of the probe's open questions; +- **the ticket attributes** — filtered as `ticket_attribute.{id}`, and the same attribute carries a + different id per ticket type, so a union column has no single id to translate to. See + [Tickets](#tickets); +- **the tag names, the company name and the contact identity of a conversation** — read from + somewhere the search endpoint does not filter, or filtered by an id the column does not hold; +- **absence** — `present`, `blank` and `missing` are derived by the agent from an equality and + rewritten into a comparison with an empty value. Intercom's search matches values and has no + operator for the lack of one, so the rewritten condition is refused rather than sent as a + comparison against the empty string; +- **group-by**, on either cursor collection: there is no aggregate endpoint, and grouping over the + pages a walk collected would look exact while answering a fraction. + +### Through a relation + +A relation is published filterable as soon as *any* column of its target is, so the interface offers +`admin_assignee:name` the moment the relation exists — unless the endpoint filters nothing on the +foreign key. `IntercomTicket#state`, `IntercomTicket#previous_state` and `IntercomContact#company` are +published unfilterable for that reason: they are there to be read and navigated, and the agent refuses +a condition through them before it reaches this datasource. +What Intercom is really filtered on is the foreign key: the **target says which of its records +match**, over every record it holds rather than over a page, and the ids it names become the +condition the search carries. + +That is exact, and it has four visible edges: + +- **The target is read one record past what a group may hold, and no further.** Against a collection + read whole that costs nothing — every record is in hand — but Contacts are a page of something + far larger, and resolving `contact:email contains "@"` over a whole workspace to then refuse the + fan-out it comes to would spend a full cursor walk on a filter that was never going to be + answered. So the read is bounded, and the refusal says "more than fifteen" rather than a count it + deliberately did not go and measure. +- Intercom takes no membership operator on these fields, so several matches become **one equality per + match**, inside an `OR` — which counts against the fifteen conditions a group allows. A relation + condition matching more records than that is refused by name rather than sent and answered with a + 400 naming neither the limit nor the filter that hit it. That `OR` is **inlined into a parent that + aggregates the same way**, so it costs no level of nesting where it does not have to: the two + levels Intercom allows are spent on the filter that was written, not on the expansion of a + relation. Where inlining it would take the parent past fifteen conditions it stays nested, width + being the scarcer of the two. +- A condition the target matched **no record** with names no row, and the DSL cannot say so: the + search is skipped entirely rather than sent as a filter that would come back with everything. +- A relation whose foreign key the endpoint does not filter — the ticket `state`, a contact's + `company` — is refused with a message saying which of the two it is: the relation is there to be + read and navigated. Whether `/tickets/search` filters a state id or a contact id at all is one of + the probe's open questions; the answer lands in the table, not in an assumption. + +On the collections read whole the same condition costs nothing: they filter in memory, so the ids go +in as a plain membership and none of the DSL's limits apply. + +A **many-to-many is published unfilterable** — `admins` and `teams` — and a condition written on one +anyway, in a scope or a segment, is refused before it reaches this datasource: the agent's own +validator answers a 400 naming the field and its type. Filter on a column of the collection next +door instead. + +### The limits of a search, checked before the request leaves + +Intercom nests a search **two levels** deep and takes **fifteen conditions per group**. Past either +it answers a 400 whose body names neither the limit nor the part of the filter that reached it, so +both are checked here and refused with a message naming what to simplify. + +Fifteen is reached without trying: a scope, a segment and an operator's own filter add up, and a +condition naming several values arrives expanded into **one condition per value** — Intercom accepts +no membership operator on these fields. Branches carrying a single condition are unwrapped and spend +no level. + +## Conversations + +The row carries what a queue is read for: state, priority, assignee and team ids, the company, the +tags, and the lifecycle Intercom keeps in `statistics` — `closed_at`, `closed_by_id`, +`first_contact_reply_at`, `last_contact_reply_at`, `last_admin_reply_at`, `reopen_count`. + +**The timeline opens on `source`, not on the parts.** The message that started the conversation +lives there; a thread built from the parts alone opens on the first reply and loses what the +customer actually asked. Every entry keeps its `part_type` — an assignment, a note and a reply are +different events. + +Intercom returns the parts **only when retrieving a single conversation**, so: + +- a record detail gets its timeline for free; +- a list view asking for the `timeline` column pays one request per row, bounded to 10. The rows past + that keep a `nil`, which reads as *unknown* — never as an empty thread. + +A conversation is capped at its **500 most recent parts**; a very long thread is therefore partial, +and says so nowhere but here. + +**The internal notes of the team are in the thread**, next to what the customer was told. That is +what a thread is on Intercom, and publishing half of it would be the more surprising answer — but it +is worth knowing before opening the collection to a role that should not read them. + +The contact's name is denormalized onto the row by **one bulk read per page**, not one per row, and +only when the projection names it. A failure there costs that column, not the page. The e-mail is a +hop away, on the `contact` relation. + +## Tickets + +A ticket carries **no `statistics` block** — measured against a workspace of 81 142 tickets — so +neither a closure date nor a last responder exists as a field. Both are derived from the parts, +which ride along in the search response whether or not anything asks for them, and therefore cost +nothing: + +| Column | Derived from | +| --- | --- | +| `closed_at`, `closed_by_name` | the last transition into a state of category `resolved` | +| `last_reply_at`, `last_responder_name`, `last_responder_type` | the last `comment` part | + +Four things to know about them: + +- a ticket is not "closed" on Intercom, it enters a **resolved** state; +- the state-change event is matched on its **prefix**, not on `ticket_state_updated_by_admin`: a + workspace running workflows closes tickets through other variants, and an invisible closure is + worse than an absent column; +- a transition whose target equals the previous state is ignored — measured, they exist; +- **a resolved ticket showing no closure date may have been closed all the same**: past the 500-part + ceiling the transition falls out of the window. That case is detected and logged, since a Date + column cannot say "unknown". + +Both are **display only**, and not temporarily: `/tickets/search` filters on neither and ignores a +sort, so neither advertises an operator. + +**The thread is published too, and it is free here.** The same `timeline` column a conversation +carries — who said what, when, and through which kind of event, internal notes and state changes +included — built from the parts the response already holds. No request per row and no cap: where a +conversation read from a listing carries no parts at all and leaves the column `nil` for *unknown*, +a ticket always carries them, so an empty list means an empty thread. Both reads ask Intercom for +`display_as=plaintext`: the bodies are HTML written by end customers, and rendering third-party +markup inside Forest is neither safe nor useful. The 500-part ceiling applies here as well, which is +the same truncation that can hide a closure date. + +The attributes a workspace declares on its ticket types are introspected once at boot and published +as the **union** of every type's, keyed by name the way the payload is. Filtering one is a different +matter: Intercom filters an attribute by id (`ticket_attribute.{id}`), and the same name carries a +different id from one ticket type to the next — measured, `_default_title_` is `14162161` on one +type and `14162165` on another. A union column has no single id to translate to, so filtering on a +ticket attribute means one collection per ticket type — more collections in the interface, and a +schema that changes shape whenever the customer adds a type. Until that trade is worth paying for, +the attributes stay display-only and advertise no operator. The ids are kept per type so the day the +answer changes costs no second boot round trip. + +## Contacts + +The people who write in, users and leads alike. Cursor-paginated like conversations and tickets, +with two routes of its own and one thing no other collection has. + +**Intercom sorts this one.** `POST /contacts/search` is the only endpoint of the whole API that +takes a `sort` and applies it, so these are the only sortable columns of the datasource: `name`, +`email`, `created_at`, `updated_at`, `signed_up_at`, `last_seen_at`, `last_contacted_at`, +`last_replied_at`. The set is deliberately narrower than the documentation implies — nothing has +been measured, and a sort Intercom refuses is a list view that fails rather than one that comes back +unordered. A sort on any other column, or on two columns at once, is reported in the log and the +rows come back in the API's order: Intercom takes a single `{ field, order }`, and honouring the +first clause of two would order the page by something nobody asked for. + +An order is also what routes a plain list view through the search endpoint, the listing sorting +nothing: the read then carries the predicate matching everything that Tickets already send. + +**Its date operators are narrower than the other two endpoints'** — measured, 25 August 2026: +`/contacts/search` refuses `>=`, `<=` and `!=` on a date where `/conversations/search` and +`/tickets/search` take them. Nothing is lost that an operator can see, a Date column publishing the +two bounds alone everywhere in this datasource, but it is why the operator table is per endpoint. + +**A set of ids is read in one request** — `id IN [...]`, which this endpoint answers and no other +does — fifteen at a time, the most one `IN` takes, rather than one request per record. It is what +makes a related list of contacts affordable. + +**`company_id equals X` reads `GET /companies/{id}/contacts`.** The search filters no company field, +so without that route the contacts of an account would be a refusal rather than a list. It is a bare +equality only: an `and` also carrying a permission scope names a narrower set than the account does, +and answering it with the account alone would serve contacts the scope excludes. + +That is a limit worth knowing before scoping permissions, because it is not a slower route but no +route at all: the account endpoint returns its contacts whole and narrows nothing, and the search +filters no company field, so **a scope or a segment on this collection turns the related list of an +account into a refusal**. The message names the condition it could not carry alongside the account, +rather than telling the operator to open the account and read its contacts — which is what they were +doing. What would answer it is a read of the account's contact ids followed by `id IN [...]` plus the +rest of the tree on `/contacts/search`, which that endpoint takes; it is not in this lot. + +One more thing the two routes do not agree on, and it is visible: the `company_id` column names **the +first** of the accounts a contact belongs to, and the `company` relation resolves that same first +one — while `company_id equals X` returns **every** contact of X. So an account's related list can +show a contact whose `company` points somewhere else. The column is the payload's reading; the filter +is the account endpoint's, and it is the more useful of the two. + +**A merged contact reads as gone, not as an error.** Intercom drops it from the listing and from the +search, and the record lives on under the id it was merged into. A row pointing at the old id comes +back empty rather than failing the page. + +The custom attributes a workspace declares on its contacts are introspected once at boot from +`GET /data_attributes?model=contact`, typed from `data_type`, and published display-only. + +## Companies + +The accounts contacts belong to, and the collection that behaves least like the others. + +**Paginated by offset**, which is the tier above. **Looked up, not searched**: `name` and +`company_id` — the identifier the customer's own system gave the account, not Intercom's — are the +two filters it publishes, each an exact equality, and anything else is refused by name. A record is +read through `GET /companies/{id}`, and a set of ids one request each, capped at 25 with the +truncation logged. + +`GET /companies/scroll` exists and is **deliberately rejected**: one open scroll per application, +expiring after a minute, cannot serve two operators looking at a list at the same time. + +A contact carries its accounts as a list of ids and nothing else, so **projecting `company:name` on +a contact list costs one request per distinct account on the page**. Reading the account from the +contact's record page, or listing contacts from the account, both cost one. + +Which is why that projection is the one relation of the datasource with a ceiling: past +[`MAX_RELATION_READS`](lib/forest_admin_datasource_intercom/collections/offset_collection.rb) +distinct accounts on a single read, it is **refused rather than resolved for part of the rows**. Every +page size a list view offers stays under it. A read that does not — an export, which batches a +thousand rows at a time, or a segment resolved whole — has to leave the column out. See +[Relations](#relations). + +Custom attributes are introspected at boot the same way, from `GET /data_attributes?model=company`, +and published display-only for the same reason. + +## What a page costs, and what is held + +A list view is a handful of requests rather than one: Intercom joins nothing, so every relation on +the page is a read of its own, and they are issued one after another. Two things keep that from +being paid twice. + +**The workspace's reference lists are held for `reference_cache_ttl`** — the teammates, the teams, +the ticket types and the ticket states, the tier read whole. They are what four of a ticket's five +relations resolve through, and they change a few times a year; without a window, every page, every +count and every filter traversing one of them re-reads all four. What the window trades is bounded +staleness: a teammate added mid-session appears within the minute rather than on the next request. +Set `reference_cache_ttl: 0` to pay the requests instead, or drop what is held without waiting the +window out — a customizer that has just written to the workspace can call +`datasource.configuration.reference_cache.clear`. + +**Records are never held.** What the search and listing endpoints answer is the customer's data, and +serving a page of it from a store would show an operator a row they have just edited in its previous +state. The one deduplication that applies to records is scoped to a single page being built: a +ticket list asks for the same contacts twice — once for the `contact_name` column, once for the +`contact` relation — and the second read is answered by the first, for the length of that call and +no longer. + +**Connections are reused.** Several requests to one host means several TLS handshakes on Faraday's +default adapter, and they cost more than some of the requests they carry, so the client builds on +the persistent adapter. An adapter the process cannot load, named in `adapter` or the persistent +one itself, is refused when the agent starts. + +The pool holds 25 connections, and that figure is read against **the web server's thread count** +rather than against this datasource. A thread that finds the pool full waits half a second for a +connection and then fails — and it fails as a Faraday timeout, which reads as Intercom being slow +rather than as the pool being narrow. The searches a list view is built on travel over POST, which +the retry policy does not replay, so that failure costs an operator their page. Size the pool above +the threads that can reach it if the server runs wide: + +```ruby +ForestAdminDatasourceIntercom::Datasource.new( + access_token: ENV.fetch('INTERCOM_ACCESS_TOKEN'), + adapter: [:net_http_persistent, { pool_size: 50 }] +) +``` + +Measured on a page of 15 tickets projecting five relations +([`ticket_list_budget_spec.rb`](spec/forest_admin_datasource_intercom/collections/ticket_list_budget_spec.rb)): + +| | Requests | +| --- | --- | +| Before | 7 | +| First page after boot | 5 | +| Every page after it, within the window | 2 | + +What none of this touches is the search response itself. Intercom offers no field selection, so a +page of tickets carries every ticket's whole timeline whatever the projection says — which is why +[`MAX_TICKETS_PER_PAGE`](lib/forest_admin_datasource_intercom/collections/ticket.rb) is 25, and why +asking for 50 or 100 rows a page, or for page 5, costs that response two, four or three times over. + +### Reading records by id in parallel + +Intercom has no bulk read for a company, a conversation or a ticket, so some reads cost one request +per record: a related list or a page resolving a `company` relation (up to 100 companies), a record +read by id, and the timelines of a conversation list (up to 10 conversations). By default those +requests go **one after another** — at about 150 ms each, a page resolving 100 companies waits around +15 seconds. + +`concurrent_reads` sends that many of them at once. At 4, the same page waits around 4 seconds. The +order of the rows, the caps and the refusals are unchanged, and so is the rate limiting: every +request still goes through the limiter. + +It is 1 by default because **every read in flight holds a connection of the pool** for as long as it +waits. A request using four of them leaves four fewer for the other threads, and a thread that finds +the pool full fails as a timeout (see above). Size the pool against both figures together — at +least the server's thread count times `concurrent_reads`: + +```ruby +# Puma with 5 threads, 4 reads at once: 20 connections at the busiest. +ForestAdminDatasourceIntercom::Datasource.new( + access_token: ENV.fetch('INTERCOM_ACCESS_TOKEN'), + concurrent_reads: 4, + adapter: [:net_http_persistent, { pool_size: 25 }] +) +``` + +`concurrent_reads: 1` is also the way back to the sequential behaviour if a wider setting misbehaves. + +## Rate limits + +Intercom meters the app and, above it, the whole workspace — 25 000 requests a minute shared with +every other private app the customer runs — and allocates that budget in **10-second windows**: the +measured `x-ratelimit-limit` is 1667, not 10 000. A burst therefore takes a 429 while the minute's +budget is barely touched, which is why what matters is the instantaneous rate. + +The limiter is driven by the headers Intercom returns on every response rather than by a table: it +waits out the reset when the window is spent, and counts its own in-flight requests down so several +of them do not go out on the same stale figure. A reset further out than a window is a clock +disagreement rather than a window emptying — the request goes through and the log says so, once per +window. + +It sits **in front of** the 429 retry, not instead of it: the retry remains the defence against the +part of the workspace budget spent by traffic this process cannot see. Pass `rate_limiter: nil` to +meter on your own side instead. + +## Privacy + +The body of a conversation is raw personal data, and this datasource is built on that assumption. + +- **Nothing logs a body.** Logs carry the operation, the counts and Intercom's request id — never + content. A response that fails to parse is reported by name, never quoted: a JSON parser opens its + message with the characters it choked on, and on a 200 those are the payload. +- **`display_as=plaintext` on every conversation read.** The bodies are HTML written by end + customers; rendering third-party HTML inside Forest is neither safe nor useful. +- **The regional host is configurable** so a workspace's data stays in its region. +- Ticket list pages carry customer message bodies whether or not anything asks for them — Intercom + offers no field selection. Restrict the body columns with Forest's field-level permissions where + that matters. + +## Boot-time introspection + +Constructing the datasource performs exactly **four** reads. + +Three are of one kind: `GET /ticket_types` for the attribute columns of `IntercomTicket`, and +`GET /data_attributes?model=contact` and `?model=company` for those of `IntercomContact` and +`IntercomCompany`. A payload carries the values of the attributes that record happens to have been +given, never their definitions, which is why they cannot be discovered from the records. + +The fourth is `GET /me`, and it reads no column: Intercom echoes in a response header the API +version it served, and it serves the workspace's own default when the pin is not honoured — whose +payloads are shaped differently from the ones this expects. That echo is the only place the +substitution shows, so it is checked while the agent starts and reported as a warning. + +All four run on the boot connection — short timeouts, one quick retry — so a slow Intercom cannot +turn a Rails boot into minutes the operator sits through, and each degrades to a warning rather +than to a failed boot: a token missing a permission costs the columns it could not read, or the +version check, never the agent. + +`api_writable` is read alongside each attribute and kept, although every column of this lot is +published read-only: it is what tells an attribute the API may write from one Intercom fills in +itself, and reading it again later would be a second boot-time round trip. + +Everything else is read when a collection is listed, so an agent boots whatever Intercom is doing. + +## What is not here yet + +| Lot | What it brings | +| --- | --- | +| 3 | Writes and business actions on tickets and conversations: reply, close, snooze, reopen, assign, tag, convert | +| 4b | Writes on contacts and companies: create, update, archive, block, merge, attach and detach | +| 5 | Notes, tags, segments | +| 6 | Bounded group-by and the reporting export | + +Two questions this lot leaves in the table rather than in an assumption, both for +`forest_admin_intercom_probe` to answer against the customer's workspace: whether `/tickets/search` +filters on `contact_ids`, and which operators `/contacts/search` answers on a custom attribute. + +## Development + +```bash +cd packages/forest_admin_datasource_intercom +BUNDLE_GEMFILE=Gemfile-test bundle install +BUNDLE_GEMFILE=Gemfile-test bundle exec rspec +bundle exec rubocop # from the repository root +``` + +`exe/forest_admin_intercom_probe` ships with the gem rather than living in the repository alone: +what it measures is the customer's workspace, and whoever runs it there has the gem installed and +not a clone of this repository. From a checkout it runs in place, `exe/forest_admin_intercom_probe`. + +Specs stub the HTTP layer with WebMock. Every payload they feed in is **hand-written from the +OpenAPI 2.16 specification**, never captured from a workspace: a conversation body is personal data, +and a fixture is read by everyone who clones the repository. diff --git a/packages/forest_admin_datasource_intercom/Rakefile b/packages/forest_admin_datasource_intercom/Rakefile new file mode 100644 index 000000000..4c774a2bf --- /dev/null +++ b/packages/forest_admin_datasource_intercom/Rakefile @@ -0,0 +1,6 @@ +require 'bundler/gem_tasks' +require 'rspec/core/rake_task' + +RSpec::Core::RakeTask.new(:spec) + +task default: :spec diff --git a/packages/forest_admin_datasource_intercom/exe/forest_admin_intercom_probe b/packages/forest_admin_datasource_intercom/exe/forest_admin_intercom_probe new file mode 100755 index 000000000..c81efcdda --- /dev/null +++ b/packages/forest_admin_datasource_intercom/exe/forest_admin_intercom_probe @@ -0,0 +1,287 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Enumerates what Intercom's search endpoints really filter, and with which +# operators, against a real workspace. The answer is what `search_fields.yml` +# holds and what the schema derives its filters from -- and it cannot be read +# off the documentation: measured during lot 1, `/tickets/search` refuses +# `company_id` with `invalid_field` although a ticket carries one. +# +# INTERCOM_ACCESS_TOKEN=... forest_admin_intercom_probe --endpoint tickets +# INTERCOM_ACCESS_TOKEN=... forest_admin_intercom_probe --region eu --out measured.yml +# +# Shipped with the gem rather than left in the repository: what it measures is +# the customer's workspace, and the operator running it has the gem installed +# and not a clone of the agent. +# +# It sends one search per (field, operator) cell, asking for a single record, +# and reads Intercom's refusal codes: `invalid_field` for a field the endpoint +# does not filter at all -- the rest of its row is then skipped -- `data_invalid` +# for an operator it refuses on that field, `invalid_value` for a value shape it +# refuses. Read-only: a search changes nothing, and no payload is printed or +# written, only field names, operators and error codes. +# +# What it writes with `--out` is *evidence*, deliberately not the table itself: +# `search_fields.yml` carries the prose that says why a column stays refused, +# and a generated file would drop it. Merge the measurements into the table by +# hand, which is also where deciding to expose a newly discovered field belongs. + +$LOAD_PATH.unshift(File.expand_path('../lib', __dir__)) + +require 'forest_admin_datasource_intercom' +require 'optparse' + +module ProbeSearchFields + SearchFields = ForestAdminDatasourceIntercom::Query::SearchFields + + # Whether this file is being run as the command rather than loaded by a + # spec. The basenames and not the paths: RubyGems installs an executable + # behind a stub of its own and `load`s this file from it, so `$PROGRAM_NAME` + # is the stub while `__FILE__` is this file -- comparing the two would make + # the shipped command exit without doing anything, which is the one thing + # shipping it was for. A spec loading it runs under `rspec`, whose basename + # is not this one. + def self.invoked_as_command?(program_name) + File.basename(program_name.to_s) == File.basename(__FILE__) + end + + # A cell is probed with the value shapes its field plausibly takes, most + # likely first: a wrong shape is refused with `data_invalid` just like an + # unsupported operator, so a single attempt would report an operator as + # refused when only the value was wrong. + VALUES = { + 'date' => ['1', 1, '2026-01-01'], + 'number' => [0, '0'], + 'boolean' => [true, 'true'], + 'string' => ['forest-probe', 0], + 'text' => ['forest-probe'] + }.freeze + + LIST_OPERATORS = %w[IN NIN].freeze + + # What Intercom answers a search it refuses: `invalid_field`, `data_invalid` + # and `invalid_value` all travel on a 400. + REFUSAL_STATUSES = [400, 422].freeze + + # Raised when an answer measures nothing, so the run stops before writing. + class Inconclusive < StandardError; end + + # A candidate carries no declared type -- discovering it is the point -- so + # one is guessed from its name, and the fallbacks above cover a wrong guess. + def self.guess_type(field) + case field + when /(_at|_since|_until)\z/ then 'date' + when /\Acount_|_count\z|\Atime_to_|_time\z/ then 'number' + when /\A(open|read|is_|has_|.*_participated)/ then 'boolean' + else 'string' + end + end + + class Probe + def initialize(client, endpoint) + @client = client + @endpoint = endpoint + end + + # Every operator of the alphabet on every field of the row: what the table + # already declares, plus the candidates. Nothing is assumed from the + # declared operators -- an operator missing from the table is exactly what + # this is here to find. + def run(field, type) + results = {} + + SearchFields::KNOWN_OPERATORS.each do |operator| + outcome = probe(field, operator, type) + return { unfilterable: outcome } if outcome[:code] == 'invalid_field' + + results[operator] = outcome + end + + { operators: results } + end + + private + + def probe(field, operator, type) + last = nil + + VALUES.fetch(type, VALUES['string']).each do |value| + outcome = attempt(field, operator, operator_value(operator, value)) + # `invalid_field` is about the field, not about what was sent with it: + # another value shape cannot make a field the endpoint does not filter + # appear, so the row ends here rather than paying a request per shape. + return outcome if outcome[:ok] || outcome[:code] == 'invalid_field' + + last = outcome + end + + last + end + + # Only an answer about the search itself is a measurement. A 401, a 429 + # that outlasted the retries, a 5xx or a timeout says nothing about the + # field, and recorded as a refusal it would be wrong evidence. + def attempt(field, operator, value) + @client.search_page(@endpoint.path, query: { 'field' => field, 'operator' => operator, 'value' => value }, + per_page: 1, list_key: list_key) + { ok: true } + rescue ForestAdminDatasourceIntercom::APIError => e + raise Inconclusive, inconclusive_message(field, operator, e) unless REFUSAL_STATUSES.include?(e.status) + + { ok: false, code: error_code(e), status: e.status } + end + + def inconclusive_message(field, operator, error) + "#{@endpoint.path} answered #{field} #{operator} with #{error.status ? "HTTP #{error.status}" : "no status"}, " \ + 'which says nothing about the field. Nothing was written; check the token and the region, then run ' \ + 'the probe again.' + end + + def operator_value(operator, value) + LIST_OPERATORS.include?(operator) ? [value] : value + end + + # `/tickets/search` answers under `tickets` and `/conversations/search` + # under `conversations`; a body read under the wrong key raises before the + # outcome is classified. + def list_key = @endpoint.name + + def error_code(error) + errors = error.body.is_a?(Hash) ? error.body['errors'] : nil + first = Array(errors).first + + (first.is_a?(Hash) ? first['code'] : nil) || "http_#{error.status}" + end + end + + class Report + def initialize(endpoint) + @endpoint = endpoint + @rows = {} + end + + def record(field, column, result) + @rows[field] = { 'column' => column, 'result' => result } + end + + # What the run says about the committed table, which is the only reason to + # run it: an operator the table promises and Intercom refuses is a filter + # the interface offers and the read cannot honour. + def print_diff + @rows.each do |field, row| + column = row['column'] + result = row['result'] + + if result[:unfilterable] + declared = column ? " <- table declares it on column '#{column}'" : '' + puts " #{field.ljust(42)} NOT FILTERABLE (#{result[:unfilterable][:code]})#{declared}" + next + end + + print_operators(field, column, result[:operators]) + end + end + + def to_yaml_document + { 'endpoint' => @endpoint.name, 'path' => @endpoint.path, 'measured_at' => Time.now.utc.strftime('%Y-%m-%d'), + 'fields' => @rows.to_h { |field, row| [field, measured_row(row)] } }.to_yaml + end + + private + + def print_operators(field, column, outcomes) + supported = outcomes.select { |_, outcome| outcome[:ok] }.keys + declared = column ? Array(@endpoint.field(column)&.operators) : [] + puts " #{field.ljust(42)} #{supported.empty? ? "(no operator answered)" : supported.join(" ")}" + + report_drift(declared - supported, supported - declared) + end + + def report_drift(promised, discovered) + puts " ! the table promises #{promised.join(", ")} here, Intercom refuses it" unless promised.empty? + puts " + Intercom also accepts #{discovered.join(", ")}" unless discovered.empty? + end + + def measured_row(row) + result = row['result'] + return { 'filterable' => false, 'code' => result[:unfilterable][:code] } if result[:unfilterable] + + { 'filterable' => true, + 'operators' => result[:operators].select { |_, outcome| outcome[:ok] }.keys, + 'refused' => result[:operators].reject { |_, outcome| outcome[:ok] } + .transform_values { |outcome| outcome[:code] } } + end + end + + class CLI + def self.call(argv) + new(options(argv)).run + end + + def self.options(argv) + options = { endpoints: SearchFields.endpoints, region: 'us', token: ENV.fetch('INTERCOM_ACCESS_TOKEN', nil) } + + OptionParser.new do |parser| + parser.banner = 'Usage: INTERCOM_ACCESS_TOKEN=... forest_admin_intercom_probe [options]' + parser.on('--endpoint NAME', "one of #{SearchFields.endpoints.join(", ")}") { |v| options[:endpoints] = [v] } + parser.on('--region NAME', 'us (default), eu or au') { |v| options[:region] = v } + parser.on('--token TOKEN', 'defaults to $INTERCOM_ACCESS_TOKEN') { |v| options[:token] = v } + parser.on('--out PATH', 'write the measurements as YAML evidence') { |v| options[:out] = v } + end.parse!(argv) + + options + end + + def initialize(options) + @options = options + end + + def run + abort('No token: set INTERCOM_ACCESS_TOKEN or pass --token.') if @options[:token].to_s.empty? + + documents = @options[:endpoints].map { |name| probe_endpoint(SearchFields.fetch(name)) } + write(documents) if @options[:out] + 0 + rescue Inconclusive => e + abort("Probe stopped: #{e.message}") + end + + private + + def probe_endpoint(endpoint) + puts "\n#{endpoint.path} -- #{endpoint.fields.size} declared field(s), #{endpoint.candidates.size} candidate(s)" + probe = Probe.new(client, endpoint) + report = Report.new(endpoint) + + targets(endpoint).each { |field, column, type| report.record(field, column, probe.run(field, type)) } + report.print_diff + report.to_yaml_document + end + + # The declared fields first, with the type the table gives them; then the + # candidates, whose type is guessed. A candidate already covered by a + # declared field is not probed twice. + def targets(endpoint) + declared = endpoint.fields.values.map { |field| [field.field, field.column, field.type] } + candidates = (endpoint.candidates - declared.map(&:first)).map { |field| [field, nil, guess_type(field)] } + + declared + candidates + end + + def guess_type(field) = ProbeSearchFields.guess_type(field) + + def client + @client ||= ForestAdminDatasourceIntercom::Client.new( + ForestAdminDatasourceIntercom::Configuration.new(access_token: @options[:token], region: @options[:region]) + ) + end + + def write(documents) + File.write(@options[:out], documents.join("\n")) + puts "\nWritten to #{@options[:out]}. Merge the measurements into search_fields.yml by hand: " \ + 'the table carries the prose a generated file would drop.' + end + end +end + +exit(ProbeSearchFields::CLI.call(ARGV)) if ProbeSearchFields.invoked_as_command?($PROGRAM_NAME) diff --git a/packages/forest_admin_datasource_intercom/forest_admin_datasource_intercom.gemspec b/packages/forest_admin_datasource_intercom/forest_admin_datasource_intercom.gemspec new file mode 100644 index 000000000..f54347fa8 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/forest_admin_datasource_intercom.gemspec @@ -0,0 +1,38 @@ +lib = File.expand_path('lib', __dir__) +$LOAD_PATH.unshift lib unless $LOAD_PATH.include?(lib) + +require_relative 'lib/forest_admin_datasource_intercom/version' + +Gem::Specification.new do |spec| + spec.name = 'forest_admin_datasource_intercom' + spec.version = ForestAdminDatasourceIntercom::VERSION + spec.authors = ['Forest Admin'] + spec.email = ['contact@forestadmin.com'] + spec.homepage = 'https://www.forestadmin.com' + spec.summary = 'Intercom datasource for Forest Admin Ruby agent.' + spec.description = 'Surface Intercom conversations, tickets, contacts and companies as Forest Admin collections.' + spec.license = 'Apache-2.0' + spec.required_ruby_version = '>= 3.0.0' + + spec.metadata['homepage_uri'] = spec.homepage + spec.metadata['source_code_uri'] = 'https://github.com/ForestAdmin/agent-ruby' + spec.metadata['changelog_uri'] = 'https://github.com/ForestAdmin/agent-ruby/blob/main/CHANGELOG.md' + spec.metadata['rubygems_mfa_required'] = 'false' + + spec.files = Dir.chdir(__dir__) do + `git ls-files -z`.split("\x0").reject do |f| + (File.expand_path(f) == __FILE__) || + f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile]) + end + end + spec.bindir = 'exe' + spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) } + spec.require_paths = ['lib'] + + spec.add_dependency 'activesupport', '>= 6.1' + spec.add_dependency 'faraday', '~> 2.0' + spec.add_dependency 'faraday-net_http_persistent', '~> 2.0' + spec.add_dependency 'faraday-retry', '~> 2.0' + spec.add_dependency 'forest_admin_datasource_toolkit', ">= #{ForestAdminDatasourceIntercom::VERSION}" + spec.add_dependency 'zeitwerk', '~> 2.3' +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb new file mode 100644 index 000000000..cf72df082 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb @@ -0,0 +1,57 @@ +require_relative 'forest_admin_datasource_intercom/version' +require 'cgi/escape' +require 'json' +require 'logger' +require 'set' +require 'time' +require 'uri' +require 'yaml' +require 'zeitwerk' +# Required explicitly rather than relied on via the toolkit: `active_support` brings the autoload +# `Time.use_zone` needs, `active_support/time` brings `Time.zone` (see `Query::CallerZone`). +require 'active_support' +require 'active_support/time' +require 'faraday' +require 'faraday/retry' +require 'forest_admin_datasource_toolkit' + +loader = Zeitwerk::Loader.for_gem +loader.setup + +module ForestAdminDatasourceIntercom + class Error < StandardError; end + class ConfigurationError < Error; end + + # Descends from the toolkit's ValidationError so the agent answers 400 with the message, not a 500: + # the datasource refuses a filter Intercom cannot express exactly rather than approximating it. + class UnsupportedOperatorError < ForestAdminDatasourceToolkit::Exceptions::ValidationError; end + + # Raised when an Intercom API call fails. Carries the HTTP status and the + # parsed response body so callers -- smart actions in particular -- can + # surface Intercom's own error message instead of a generic string. + class APIError < Error + attr_reader :status, :body + + def initialize(message, status: nil, body: nil) + super(message) + @status = status + @body = body + end + end + + class << self + attr_writer :logger + + def logger + @logger ||= default_logger + end + + private + + def default_logger + return Rails.logger if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger + + Logger.new($stderr).tap { |l| l.progname = 'forest_admin_datasource_intercom' } + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/cache.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/cache.rb new file mode 100644 index 000000000..9b59dc17c --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/cache.rb @@ -0,0 +1,84 @@ +module ForestAdminDatasourceIntercom + # TTL store for workspace reference reads (teammates, teams, ticket types, ticket states), re-read + # on every page otherwise. Records are never cached here; see `Client#with_read_scope` for per-page dedup. + class Cache + Entry = Struct.new(:value, :expires_at) + + # Monotonic rather than wall clock: this measures an elapsed span, and a + # host whose clock steps -- an NTP correction, a suspend -- would otherwise + # expire every entry at once or hold them all far past the window. + def initialize(ttl:, now: nil) + @ttl = ttl.to_f + @now = now || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) } + @mutex = Mutex.new + @entries = {} + # Bumped by `clear`, and carried by every read that is about to fetch, so + # a response already in flight when the store was dropped cannot land in + # it afterwards. See `fetch`. + @generation = 0 + end + + def enabled? = @ttl.positive? + + # The block runs outside the mutex (it is an HTTP request): two concurrent misses both read, which is + # cheaper than serializing every thread. A raising block writes nothing, and the generation is read + # before the block so a response in flight across `clear` is not stored. + def fetch(key) + return yield unless enabled? + + hit = read(key) + return hit.value if hit + + generation = @mutex.synchronize { @generation } + value = yield + write(key, value, generation) + value + end + + # Drops everything held, without waiting the window out. Nothing inside this + # package calls it: the window is what expires an entry, and no read here + # writes. It is reachable through `Configuration#reference_cache` for the + # one case the window cannot serve -- a customizer that has just written a + # team or a ticket type into the workspace and wants the next page to show + # it. Documented in the README beside the ttl. + def clear + @mutex.synchronize do + @entries.clear + @generation += 1 + end + end + + private + + def read(key) + @mutex.synchronize do + entry = @entries[key] + next nil if entry.nil? + + if entry.expires_at <= @now.call + @entries.delete(key) + next nil + end + + entry + end + end + + # Expired entries are dropped on the way past rather than by a sweeper: the + # keys are a handful of endpoints, so the walk is short and there is no + # thread to own. + # + # A write from before a `clear` is dropped rather than stored: the store it + # was going to land in no longer exists. + def write(key, value, generation) + now = @now.call + + @mutex.synchronize do + next if generation != @generation + + @entries.delete_if { |_, entry| entry.expires_at <= now } + @entries[key] = Entry.new(value, now + @ttl) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb new file mode 100644 index 000000000..77b34b23c --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb @@ -0,0 +1,536 @@ +module ForestAdminDatasourceIntercom + # Hand-written on Faraday rather than the `intercom` gem, to keep raw payloads, quota headers and error bodies. + # Long by line count only: one method per endpoint plus shared envelope and error handling. + class Client # rubocop:disable Metrics/ClassLength + # `per_page=200` is refused with `invalid_per_page` -- "must be an integer + # between 0 and 150". There is no silent downgrade, so a page size is bounded + # before it is sent or the list view breaks rather than shrinks. + MAX_PER_PAGE = 150 + + # Bounds `fetch_all`, which asks for a whole reference collection rather than + # a window: those endpoints answer in one response, so reaching this many + # pages means Intercom started paginating on its own and the read is spending + # more than the answer is worth. + MAX_COLLECTED_PAGES = 10 + + # `next_cursor` is nil as soon as Intercom stops advertising a next page, so + # callers never have to know how the absence is spelled on the wire. + # `total_count` is exact, filter included, which is what makes Forest's + # record counter and its "number of" charts one request each. + # `total_pages` is filled by the one endpoint that paginates by offset and + # nil everywhere else: a cursor page has no notion of how many there are. + Page = Struct.new(:records, :next_cursor, :total_count, :total_pages, keyword_init: true) + + def initialize(configuration) + @configuration = configuration + # Named per instance rather than per class: two datasources are two + # workspaces, and a scope shared between them would hand one the other's + # answer. A handful of clients live in a process, so a symbol each costs + # nothing. + @scope_key = :"forest_admin_datasource_intercom_read_scope_#{object_id}" + # Both connections built here rather than memoized on first use. Neither + # reaches the network -- and what a lazy reader would hand the thread + # that lost the race is no longer a spare object but a pool of open + # sockets, nothing left to close it. The adapter is resolved once for the + # two of them. + @adapter = resolve_adapter + @connection = build_connection(retry_policy: @configuration.retry_policy, + timeout: @configuration.timeout, + open_timeout: @configuration.open_timeout) + # For what is read while the datasource is being constructed -- the + # custom-attribute introspection above all: short timeouts and one quick + # retry, so a slow Intercom cannot turn a Rails boot into minutes of + # waiting. Built apart from the one above, which keeps the patience every + # later request is entitled to. + @boot_connection = build_connection(retry_policy: @configuration.boot_retry_policy, + timeout: @configuration.boot_timeout, + open_timeout: @configuration.boot_open_timeout) + end + + # Deduplicates reads while one page is built (e.g. ticket contacts read for a column and a relation). + # Fiber-local (`Thread.current[]`) so concurrent requests on a fiber scheduler never share a scope; + # re-entrant; callers share returned objects, which must not be mutated. + def with_read_scope + return yield unless Thread.current[@scope_key].nil? + + Thread.current[@scope_key] = {} + begin + yield + ensure + Thread.current[@scope_key] = nil + end + end + + # Health check: the admin the token belongs to, plus its workspace. Enough + # to prove the credentials are usable, and the one call that verifies the + # pinned API version was honoured -- Intercom echoes the version it served + # in a response header. + # + # `boot: true` runs it on the short-timeout connection, for a caller + # checking the token while the agent is still starting. + def me(boot: false) + must_succeed('me') do + response = get('me', boot: boot) + verify_pinned_version(response) + response.body + end + end + + # `/tickets/search` answers under `tickets`, not `data` (measured), so the caller names its `list_key`. + def list_page(path, per_page:, starting_after: nil, params: {}, list_key: 'data', boot: false) + query = params.merge('per_page' => self.class.bounded_per_page(per_page)) + query['starting_after'] = starting_after unless blank?(starting_after) + + must_succeed(path) { to_page(get(path, query, boot: boot).body, path, list_key) } + end + + # `params` stays in the query string (`display_as`). `sort` is honoured by `/contacts/search` alone: + # the other search endpoints silently ignore it (measured). + def search_page(path, query:, per_page:, starting_after: nil, list_key: 'data', params: {}, sort: nil) + pagination = { 'per_page' => self.class.bounded_per_page(per_page) } + pagination['starting_after'] = starting_after unless blank?(starting_after) + body = { 'query' => query, 'pagination' => pagination } + body['sort'] = sort_clause(sort) if sort + + within_read_scope(['search', path, body, params, list_key]) do + must_succeed(path) { to_page(post(path, body, params: params).body, path, list_key) } + end + end + + # One page of an endpoint that paginates by **offset** rather than by + # cursor. `POST /companies/list` is the only one, and it is what lets the + # Companies collection answer page 7 of a list view with one request + # instead of walking six pages to reach it. Intercom counts pages from 1. + def offset_page(path, page:, per_page:, params: {}, list_key: 'data') + query = params.merge('page' => [page.to_i, 1].max, + 'per_page' => self.class.bounded_per_page(per_page)) + + must_succeed(path) { to_offset_page(post(path, {}, params: query).body, path, list_key) } + end + + # An exact lookup, and the shape surprise that comes with it: `GET + # /companies?name=` answers the company itself where `?tag_id=` answers a + # list. A record is read here as a page of one, so a caller writes one route + # rather than testing the envelope. + def lookup_page(path, params:, list_key: 'data') + must_succeed(path) do + body = get(path, params).body + next Page.new(records: [body], next_cursor: nil, total_count: 1) if single_record?(body, list_key) + + to_page(body, path, list_key) + end + end + + # One record from its own endpoint. Raises on a 404 like on any other + # failure: what a missing record means -- a stale link, a record outside the + # token's scope, a deletion -- is the caller's to decide, not the client's. + def fetch_record(path, id, params: {}, boot: false) + within_read_scope(record_key(path, id, params)) { read_record(path, id, params, boot: boot) } + end + + # Records read one request each, `concurrent_reads` at a time, in the order + # of `ids`; nil for one Intercom no longer answers. The read scope is read + # and written here, on the calling thread: a worker sees neither, a Hash + # shared between threads being no store. + def fetch_records(path, ids, params: {}) + scope = Thread.current[@scope_key] + found = {} + pending = [] + ids.uniq.each do |id| + key = record_key(path, id, params) + scope&.key?(key) ? found[id] = scope[key] : pending << id + end + + read_concurrently(pending) { |id| record_or_nil(path, id, params) }.each do |id, record| + found[id] = record + scope[record_key(path, id, params)] = record if scope && record + end + ids.map { |id| found[id] } + end + + # Every record of an unpaginated reference endpoint, held in `Cache` for the configured window. + # A cursor is still followed defensively. `boot` only picks the connection, so it is not part of the key; + # the list is frozen shallowly so no caller mutates what the next page reads. + def fetch_all(path, list_key: 'data', params: {}, boot: false) + @configuration.reference_cache.fetch(['fetch_all', path, list_key, params]) do + must_succeed(path) do + collect_pages(path, list_key: list_key, params: params, boot: boot).freeze + end + end + end + + # The page size Intercom accepts, whatever was asked for. + def self.bounded_per_page(size) + value = size.to_i + return 1 if value < 1 + + [value, MAX_PER_PAGE].min + end + + # The client holds the connections whose headers carry the access token in + # clear, and Faraday prints those headers on `inspect`. + def inspect + "#<#{self.class.name} url=#{@configuration.redacted_url.inspect}>" + end + + private + + # Readers rather than builders: both connections are built in the + # constructor, where what distinguishes them is argued. + attr_reader :connection, :boot_connection + + # The answer already given for `key` while this page was being built, or the + # block's. Outside a scope -- a smart action, a boot read -- the block runs + # as it always did: nothing is remembered where nothing opened a scope. + # + # A block that raises writes nothing, so a failed read is not replayed to + # the rest of the page as an answer. + def within_read_scope(key) + scope = Thread.current[@scope_key] + return yield if scope.nil? + return scope[key] if scope.key?(key) + + scope[key] = yield + end + + def record_key(path, id, params) = ['record', path, id, params] + + def read_record(path, id, params, boot: false) + operation = "#{path}/#{id}" + + must_succeed(operation) do + body = get("#{path}/#{Faraday::Utils.escape(id)}", params, boot: boot).body + body.is_a?(Hash) ? body : refuse_body_shape(operation, 'the response is not a record') + end + end + + def record_or_nil(path, id, params) + read_record(path, id, params) + rescue APIError => e + raise unless e.status == 404 + + nil + end + + # `items` mapped through the block, at most `concurrent_reads` at a time. + # Inline at a width of one, which is the behaviour of before this existed. + # Wider, a failure stops the workers taking new items, every worker is + # joined, and the failure raised is the first in the order of `items` -- + # the same one whatever order the responses came back in. + def read_concurrently(items, &read) + width = [@configuration.concurrent_reads, items.size].min + return items.to_h { |item| [item, yield(item)] } if width <= 1 + + outcomes = run_workers(items, width, &read) + failure = items.lazy.map { |item| outcomes[item] }.find { |outcome| outcome&.first == :error } + raise failure.last if failure + + items.to_h { |item| [item, outcomes[item].last] } + end + + def run_workers(items, width, &read) + queue = Queue.new + items.each { |item| queue << item } + queue.close + outcomes = {} + lock = Mutex.new + + Array.new(width) do + Thread.new do + while (item = queue.pop) + outcome = attempt(item, &read) + lock.synchronize { outcomes[item] = outcome } + queue.clear if outcome.first == :error + end + end + end.each(&:join) + outcomes + end + + def attempt(item) + [:ok, yield(item)] + rescue StandardError => e + [:error, e] + end + + # The raw response rather than its body: the quota headers are read by the + # throttle, and the version echo by `verify_pinned_version`. + def get(path, params = nil, boot: false) + (boot ? boot_connection : connection).get(path, params) + end + + def post(path, body, params: {}, boot: false) + http = boot ? boot_connection : connection + + http.post(path) do |request| + request.params.update(params) unless params.nil? || params.empty? + request.body = body + end + end + + # Intercom serves the version its workspace defaults to when the pin is not + # honoured, and the payloads differ between versions. The echo is the only + # way to notice, and noticing at boot is worth more than a schema that + # drifts silently -- so this reports rather than raises: the agent still runs + # against a version it did not ask for, which is better than not running. + def verify_pinned_version(response) + served = response.headers['intercom-version'] if response.respond_to?(:headers) + return if served.nil? || served.to_s == @configuration.api_version + + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] asked Intercom for API version #{@configuration.api_version} " \ + "and it served #{served}. Payload shapes may differ from the ones this datasource expects; check the " \ + "workspace's default version in the Developer Hub." + ) + end + + # Intercom spells an order `{ "field": "...", "order": "descending" }`, + # and answers `data_invalid` on anything else -- so the clause is written + # here rather than by the caller, whose vocabulary is Forest's. + def sort_clause(sort) + { 'field' => sort[:field].to_s, 'order' => sort[:ascending] == false ? 'descending' : 'ascending' } + end + + # A record rather than a listing: no list under either key, and an id where + # a record carries one. An empty listing is not one of these -- it answers + # `data` as an empty array, which is a page of nothing rather than a record. + def single_record?(body, list_key) + body.is_a?(Hash) && !body[list_key].is_a?(Array) && !body['data'].is_a?(Array) && body.key?('id') + end + + def collect_pages(path, list_key:, params:, boot:) + records = [] + cursor = nil + pages = 0 + + loop do + query = cursor.nil? ? params : params.merge('starting_after' => cursor) + body = get(path, query.empty? ? nil : query, boot: boot).body + records.concat(extract_entities(body, path, list_key)) + pages += 1 + cursor = next_cursor(body, path) + break if cursor.nil? + + if pages >= MAX_COLLECTED_PAGES + log_collection_cap(path, pages, records.size) + break + end + end + + records + end + + # The records under the key the endpoint uses, or under `data`. Anything + # else is refused rather than read as an empty page: `Array()` would turn the + # envelope into `[key, value]` pairs a collection would serialize into rows + # holding nothing -- a page that looks answered and is empty -- and a + # reference collection silently read as empty is a state column with no + # values and an assignee shown as a raw id. + def extract_entities(body, operation, list_key) + return [] if body.nil? || body == '' + + entities = body.is_a?(Hash) ? (body[list_key] || body['data']) : nil + return entities if entities.is_a?(Array) + + detail = list_key == 'data' ? "'data' is not a list" : "neither '#{list_key}' nor 'data' is a list" + refuse_body_shape(operation, detail) + end + + def log_collection_cap(path, pages, collected) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] Stopped reading #{path} after #{pages} page(s) / " \ + "#{collected} record(s); the rest is left out. This endpoint is read whole on purpose, so a workspace " \ + 'this large needs the collection bounded rather than listed.' + ) + end + + # Intercom wraps a listing in `{ "type": "list", "data": [...], + # "total_count": N, "pages": { "next": { "starting_after": "..." } } }`. + def to_page(body, operation, list_key) + Page.new(records: extract_entities(body, operation, list_key), + next_cursor: next_cursor(body, operation), + total_count: extract_count(body), + total_pages: extract_total_pages(body)) + end + + # The offset tier addresses the page after this one by number, and Intercom + # advertises it the same way -- `pages.next` there is a page, not a cursor. + # Reading it as one is what made the Companies list refuse a perfectly + # ordinary answer, so this page is built without it. + def to_offset_page(body, operation, list_key) + Page.new(records: extract_entities(body, operation, list_key), + next_cursor: nil, + total_count: extract_count(body), + total_pages: extract_total_pages(body)) + end + + # Absent on the last page. An older API version (served despite the pin) spells it as a url, so the + # cursor is read from its query string; an unreadable advertised cursor is refused, not taken as the end. + def next_cursor(body, operation) + advertised = body.is_a?(Hash) && body['pages'].is_a?(Hash) ? body['pages']['next'] : nil + return nil if advertised.nil? + + cursor = case advertised + when Hash then advertised['starting_after'] + when String then cursor_from_url(advertised) + end + + presence(cursor) || refuse_body_shape(operation, "'pages.next' carries no cursor this can follow") + end + + def cursor_from_url(url) + Faraday::Utils.parse_query(URI.parse(url).query.to_s)['starting_after'] + rescue URI::InvalidURIError + nil + end + + # nil rather than 0 when Intercom sends no count: zero is an answer, and + # this is the absence of one. + def extract_count(body) + count = body['total_count'] if body.is_a?(Hash) + count.is_a?(Numeric) ? count.to_i : nil + end + + # How many pages the offset tier has to read through, when the endpoint + # counts them. nil on a cursor page, which counts nothing. + def extract_total_pages(body) + pages = body['pages'] if body.is_a?(Hash) + total = pages['total_pages'] if pages.is_a?(Hash) + total.is_a?(Numeric) ? total.to_i : nil + end + + def refuse_body_shape(operation, detail) + raise APIError.new("Intercom API call failed: #{operation}: unexpected response shape, #{detail}", status: nil) + end + + def presence(value) + blank?(value) ? nil : value + end + + def blank?(value) + value.nil? || value.to_s.empty? + end + + # `cause: nil`: the chained original would quote the payload, the URL, or the request with its bearer token. + def must_succeed(operation) + yield + rescue Faraday::Error, JSON::ParserError => e + raise api_error(operation, e), cause: nil + rescue APIError + # Already mapped, with its status intact; re-wrapping would erase it -- + # a 404 read as "no such record" rather than as a failure, above all. + raise + rescue StandardError => e + raise APIError, "Intercom API call failed: #{operation}: #{e.class}: #{e.message}", cause: nil + end + + # Builds an APIError preserving the HTTP status and Intercom's own error body + # so a smart action can show the operator the real reason instead of + # "failed". + def api_error(operation, error) + response = response_of(error) + status = response[:status] + body = parse_body(response[:body]) + + APIError.new("Intercom API call failed: #{operation}: #{failure_detail(error, status, body)}", + status: status, body: body) + end + + # Faraday hands the status and the body back in a plain hash on most errors + # and in its own `Env` on a parsing error; both answer `[]`. + def response_of(error) + response = error.respond_to?(:response) ? error.response : nil + return { status: nil, body: nil } unless response.respond_to?(:[]) + + { status: response[:status], body: response[:body] } + end + + # A body that could not be parsed is named, never quoted: the parser's own + # message opens with the characters it choked on, and on a 200 those are the + # payload -- a conversation body, most of the time (R10). + def failure_detail(error, status, body) + return unreadable_detail(status) if parse_failure?(error) + return "#{error.class}: #{error.message}" unless status + + "HTTP #{status} #{error_message(body)}".strip + end + + def unreadable_detail(status) + detail = 'the response could not be read as JSON' + status ? "#{detail} (HTTP #{status})" : detail + end + + def parse_failure?(error) + error.is_a?(Faraday::ParsingError) || error.is_a?(JSON::ParserError) + end + + # The request id is appended after truncation since Intercom support asks for it. A non-error body is + # never echoed -- it may be a payload such as a conversation body (R10); only its size is reported. + def error_message(parsed) + return "(unreadable body, #{parsed.to_s.bytesize} bytes)" unless parsed.is_a?(Hash) + + message = join_errors(parsed['errors']) + message = parsed.to_json if message.empty? + + append_request_id(message[0, 500], parsed['request_id']) + end + + def join_errors(errors) + Array(errors).filter_map do |error| + next error unless error.is_a?(Hash) + + [error['code'], error['message']].compact.join(': ') + end.join('; ') + end + + def append_request_id(message, request_id) + return message unless request_id + + "#{message} (request_id: #{request_id})" + end + + def parse_body(body) + return body unless body.is_a?(String) && !body.empty? + + JSON.parse(body) + rescue JSON::ParserError + body + end + + # Resolved once, at boot: an adapter this process cannot load is refused + # there rather than on the first request. The persistent adapter registers + # itself only once its gem is required. + def resolve_adapter + name, options = Array(@configuration.adapter || Configuration::DEFAULT_ADAPTER) + require 'faraday/net_http_persistent' if name == :net_http_persistent + Faraday::Adapter.lookup_middleware(name) + + [name, options || {}] + rescue LoadError, Faraday::Error + raise ConfigurationError, + "ForestAdminDatasourceIntercom adapter #{name.inspect} cannot be loaded by this process. Add the gem " \ + 'that provides it, or set an adapter this process can load.' + end + + # `raise_error` outside the JSON parser (raises with a parsed body); `retry` innermost to see raw 429s; + # the throttle inside `retry` so replays wait and 429 headers reach the one shared limiter. + def build_connection(retry_policy:, timeout:, open_timeout:) + name, options = @adapter + + Faraday.new(url: @configuration.url) do |f| + f.request :json + f.response :raise_error + f.response :json + f.request :retry, **retry_policy.to_faraday_options + f.use Throttle, limiter: @configuration.rate_limiter if @configuration.rate_limiter + f.adapter(name, **options) + f.headers['Authorization'] = "Bearer #{@configuration.access_token}" + f.headers['Accept'] = 'application/json' + f.headers['Intercom-Version'] = @configuration.api_version + f.headers['User-Agent'] = "forest_admin_datasource_intercom/#{VERSION}" + f.options.open_timeout = open_timeout + f.options.timeout = timeout + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/admin.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/admin.rb new file mode 100644 index 000000000..9cba4a2d3 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/admin.rb @@ -0,0 +1,81 @@ +module ForestAdminDatasourceIntercom + module Collections + # The teammates of the workspace: who a conversation or a ticket is assigned + # to. Without this collection an assignee is a raw id on every row. + class Admin < FetchAllCollection + def initialize(datasource) + super(datasource, 'IntercomAdmin') + end + + protected + + # `/admins` puts its records under `admins`, not under the `data` envelope + # the paginated listings use. + def fetch_all + client.fetch_all('admins', list_key: 'admins') + end + + def serialize(admin) + attrs = admin.is_a?(Hash) ? admin : {} + + { 'id' => stringify_id(attrs['id']), + 'name' => attrs['name'], + 'email' => attrs['email'], + 'job_title' => attrs['job_title'], + 'away_mode_enabled' => attrs['away_mode_enabled'], + 'away_mode_reassign' => attrs['away_mode_reassign'], + 'has_inbox_seat' => attrs['has_inbox_seat'], + # Not a column: it is what the names below are read from, and what the + # membership collection turns into the relation. + 'team_ids' => Array(attrs['team_ids']).map { |id| stringify_id(id) }, + 'team_names' => nil } + end + + # The teams the teammate belongs to, by name, and only when a projection + # asked for them: one read of `/teams` for the whole page. A token that + # cannot read them costs the column and nothing else. + def enrich(records, rows, projection) + return unless column_asked?(projection, 'team_names') + + names = team_names + records.each_with_index do |record, index| + rows[index]['team_names'] = Array(record['team_ids']).filter_map { |id| names[id] } + end + end + + private + + def team_names + client.fetch_all('teams', list_key: 'teams') + .to_h { |team| [stringify_id(team['id']), team['name']] } + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} could not read the teams of the workspace (HTTP " \ + "#{e.status || "-"}); the names are left empty. The relation to IntercomTeam is unaffected." + ) + {} + end + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('name', 'String') + add_column('email', 'String') + add_column('job_title', 'String') + # Whether the teammate is away, and whether their conversations get + # reassigned while they are: the two an ops lead looks at before + # assigning anything. + add_column('away_mode_enabled', 'Boolean') + add_column('away_mode_reassign', 'Boolean') + add_column('has_inbox_seat', 'Boolean') + # A list, so neither filterable nor sortable -- as the array of ids it + # replaces was. It reads the teammate without a join; the relation below + # is what navigates it, through the membership collection that gives the + # many-to-many the join Intercom does not expose. + add_column('team_names', 'Json') + add_many_to_many('teams', foreign_collection: 'IntercomTeam', + through_collection: 'IntercomTeamMembership', + origin_key: 'admin_id', foreign_key: 'team_id') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb new file mode 100644 index 000000000..90f28278c --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb @@ -0,0 +1,161 @@ +module ForestAdminDatasourceIntercom + module Collections + # Read-only for now: writes and business actions arrive with lot 3. A relation to + # a collection not served yet is a schema the agent refuses to boot on. + class BaseCollection < ForestAdminDatasourceToolkit::Collection + include Relations + + ColumnSchema = ForestAdminDatasourceToolkit::Schema::ColumnSchema + Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + Equivalent = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeEquivalent + Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + + def initialize(datasource, name) + super + define_schema + end + + def client + datasource.client + end + + # How many ids this collection resolves per `list`, and in total, as a relation target. + # Nil means unbounded: the tier read whole answers any number of ids in one request. + def ids_per_read = nil + def max_resolvable_ids = nil + + protected + + def define_schema = raise(NotImplementedError, "#{self.class} did not implement define_schema") + + # A missing field yields nil rather than an absent key; relation paths are left to + # `embed_relations`. No projection means every column, not every serialized key: + # some keys are internal plumbing (e.g. the ids behind a membership). + def project(record, projection) + asked = Array(projection).map(&:to_s) + return record.slice(*column_names) if asked.empty? + + asked.reject { |field| field.include?(':') }.to_h { |field| [field, record[field]] } + end + + # Whether a projection asks for a column. No projection at all asks for + # every declared column, which is how `project` reads it -- an enrichment + # guarded on the column being named would leave nil the very column the + # projection publishes. + def column_asked?(projection, column) + asked = Array(projection).map(&:to_s) + + asked.empty? || asked.include?(column) + end + + # Whether a projection asks for any of a group of columns, read the same + # way: no projection at all asks for every one of them. + def any_column_asked?(projection, columns) + asked = Array(projection).map(&:to_s) + + asked.empty? || columns.any? { |column| asked.include?(column) } + end + + # The window a list view asked for, cut out of records already in hand. + # + # A filter with no page -- or a page naming no limit -- asks for every + # record it matched, and there is nothing to cut. How far the read that + # collected them went is a different question, answered by the walker and + # its caps. + def page_window(records, filter) + page = filter&.page + return records if page.nil? + + offset = page.offset.to_i.clamp(0, nil) + limit = page_limit(page) + return records.drop(offset) if limit.nil? + + records[offset, limit] || [] + end + + # Nil when the page names no limit, and an explicit zero kept as one: the + # toolkit's `Page#apply` reads it as an empty window, never as "all". + def page_limit(page) + page.limit&.to_i&.clamp(0, nil) + end + + def column_names + @column_names ||= fields.select { |_, field| field.is_a?(ColumnSchema) }.keys + end + + # The columns a workspace's own attributes became. None here; the + # collections that carry any override this through `CustomAttributes`. + # The translator reads them to answer a filter on one with the refusal + # covering the family rather than with the message for a column nobody + # declared. + def attribute_column_names = [] + + # The timezone in-memory date comparisons are evaluated in. The caller's, + # since that is whose "today" the filter was written against. + def timezone_for(caller) + caller.respond_to?(:timezone) ? caller.timezone : nil + end + + # Ids reach this datasource as strings -- a filter value from Forest, a + # segment, a url -- while Intercom types them inconsistently: a team id is + # a string, the same team's id inside `admin_ids` is a number. Left as it + # comes, an integer id would never match the string the filter carries. + def stringify_id(value) + value&.to_s + end + + # Intercom nests its lists twice -- `{"type": "contact.list", "contacts": + # [...]}` -- and answers a null instead of an empty list when there is + # nothing. + def nested_list(container, key) + return [] unless container.is_a?(Hash) + + list = container[key] + list.is_a?(Array) ? list : [] + end + + # Intercom dates travel as epoch seconds; Forest reads a Date column as an + # ISO8601 string, and a filter carries one too, so comparing the two is the + # ordering itself. UTC deliberately: that is where Intercom stores and + # truncates, and rendering a local time here would hide the very shift that + # makes a day-granular date filter wrong. + def stamp(seconds) + return nil unless seconds.is_a?(Numeric) && seconds.positive? + + Time.at(seconds).utc.iso8601 + end + + # --- Reading a sort clause ------------------------------------------- + # + # Forest hands a clause keyed with symbols or with strings depending on + # where it was written, and all three tiers have to read one. + + def primary_key + @primary_key ||= fields.find do |_name, field| + field.respond_to?(:is_primary_key) && field.is_primary_key + end&.first + end + + def sort_field(clause) = clause[:field] || clause['field'] + + # `key?` rather than `||`: a descending clause carries `false`, which an + # `||` fallback reads as "absent" -- so an explicit `?sort=-id` would be + # taken for the ascending default the agent injects. + def ascending?(clause) + clause.key?(:ascending) ? clause[:ascending] : clause['ascending'] + end + + # The ascending primary-key sort the agent injects when a request names + # none. Not an order anybody asked for, so a tier that cannot honour one + # stays quiet about this one. + def default_pk_sort?(clauses) + return false unless clauses.size == 1 + + clause = clauses.first + return false unless sort_field(clause).to_s == primary_key + + ascending?(clause) != false + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company.rb new file mode 100644 index 000000000..33cb248dc --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company.rb @@ -0,0 +1,69 @@ +module ForestAdminDatasourceIntercom + module Collections + # No `/companies/search`: `GET /companies` answers exact lookups on `name`, `company_id`, + # `tag_id` and `segment_id`. Tag and segment filters wait for those collections (lot 5). + class Company < OffsetCollection + include Company::Serializer + include CustomAttributes + + # The column each lookup is written on, and the query parameter Intercom + # answers it under. They happen to share a name; keeping the mapping + # explicit is what lets a column be renamed without silently dropping the + # lookup. + LOOKUPS = { 'name' => 'name', 'company_id' => 'company_id' }.freeze + + def initialize(datasource, attributes: []) + @attributes = attributes + super(datasource, 'IntercomCompany') + end + + protected + + def list_path = 'companies/list' + def record_endpoint = 'companies' + def lookups = LOOKUPS + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + # Intercom's id and the workspace's own are two different things, and an + # ops team knows the second one: it is what their billing system calls + # the account. + add_column('company_id', 'String') + add_column('name', 'String') + define_profile_columns + define_activity_columns + # Before the attribute columns, so an attribute whose name lands on the + # relation is skipped with a warning rather than taking the boot with + # it. + add_one_to_many('contacts', foreign_collection: 'IntercomContact', origin_key: 'company_id') + register_attribute_columns + end + + def define_profile_columns + add_column('plan_name', 'String') + add_column('size', 'Number') + add_column('industry', 'String') + add_column('website', 'String') + add_column('monthly_spend', 'Number') + end + + def define_activity_columns + add_column('user_count', 'Number') + add_column('session_count', 'Number') + add_column('created_at', 'Date') + add_column('updated_at', 'Date') + add_column('last_request_at', 'Date') + # When the account was created in the customer's own system, which is + # not when Intercom heard about it. + add_column('remote_created_at', 'Date') + end + + # Typed from `GET /data_attributes?model=company`, and unfilterable for + # the same reason as everything else here: this collection is looked up, + # not searched. + def attribute_kind = 'company' + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company/serializer.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company/serializer.rb new file mode 100644 index 000000000..c449d1070 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company/serializer.rb @@ -0,0 +1,44 @@ +module ForestAdminDatasourceIntercom + module Collections + class Company < OffsetCollection + # One Intercom company flattened into the row the schema declares. + module Serializer + protected + + def serialize(company) + attrs = company.is_a?(Hash) ? company : {} + plan = attrs['plan'].is_a?(Hash) ? attrs['plan'] : {} + + identity(attrs).merge( + 'plan_name' => plan['name'], + 'user_count' => attrs['user_count'], + 'session_count' => attrs['session_count'] + ).merge(dates_of(attrs)).merge(attribute_values(attrs['custom_attributes'])) + end + + private + + def identity(attrs) + { + 'id' => stringify_id(attrs['id']), + 'company_id' => stringify_id(attrs['company_id']), + 'name' => attrs['name'], + 'size' => attrs['size'], + 'industry' => attrs['industry'], + 'website' => attrs['website'], + 'monthly_spend' => attrs['monthly_spend'] + } + end + + def dates_of(attrs) + { + 'created_at' => stamp(attrs['created_at']), + 'updated_at' => stamp(attrs['updated_at']), + 'last_request_at' => stamp(attrs['last_request_at']), + 'remote_created_at' => stamp(attrs['remote_created_at']) + } + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb new file mode 100644 index 000000000..90eafd5b7 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb @@ -0,0 +1,270 @@ +module ForestAdminDatasourceIntercom + module Collections + # `/contacts/search` is the only endpoint that honours a sort and answers `id IN`. It filters + # no company field, hence the `GET /companies/{id}/contacts` route. A merged contact vanishes + # from search and listing. Long by line count only: most of it declares the columns. + class Contact < CursorCollection # rubocop:disable Metrics/ClassLength + include Contact::Serializer + include CustomAttributes + + # `/contacts/search` demands a query, so a read with no condition of its + # own -- a list view asking for an order -- sends the least noisy + # predicate that matches everything. Every contact has a creation date, + # and a bound at the epoch keeps whatever the day-granular truncation does + # to it harmless. The same predicate `Ticket` sends, for the same reason. + MATCH_EVERY_CONTACT = { 'field' => 'created_at', 'operator' => '>', 'value' => '0' }.freeze + + # `ContactIdentity::CONTACT_CHUNK` must equal IDS_PER_READ, or the enrichment and the + # relation send different request bodies and the page pays for both. + IDS_PER_READ = Query::FilterValue::MAX_LIST_SIZE + MAX_IDS_READ = 300 + + def initialize(datasource, attributes: []) + @attributes = attributes + super(datasource, 'IntercomContact') + # Answered on the e-mail address, which is what an ops team types when + # they are looking for someone. Per word, not as a substring -- see the + # README. + enable_search + end + + # This endpoint answers `id IN [...]`, so fifteen ids cost one request + # rather than fifteen: a relation pointing here resolves a whole page of + # rows for a handful of requests, and there is no fan-out to refuse. That + # is why `max_resolvable_ids` is nil where the other two tiers bound it -- + # the read is proportional to the rows already collected, and the walk + # that collected them is bounded. + def ids_per_read = IDS_PER_READ + def max_resolvable_ids = nil + + protected + + def list_endpoint = 'contacts' + def searchable = 'contacts' + def search_column = 'email' + def match_all_query = MATCH_EVERY_CONTACT + def max_id_reads = MAX_IDS_READ + def routed_operators = { 'company_id' => [Operators::EQUAL] } + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + define_identity_columns + define_date_columns + define_reachability_columns + define_device_columns + # Before the attribute columns rather than after: a workspace attribute + # whose name lands on a relation is then skipped with a warning, the way + # one landing on a column already is. Declared after, it would collide + # and take the boot with it. + define_relations + register_attribute_columns + end + + def define_identity_columns + add_column('role', 'String') + add_column('name', 'String') + add_column('email', 'String') + add_column('email_domain', 'String') + add_column('phone', 'String') + add_column('external_id', 'String') + add_column('avatar', 'String') + add_column('owner_id', 'String') + add_column('session_count', 'Number') + # The first of the accounts the contact belongs to, and how many there + # are: the same reading a conversation gives its contacts, and the + # foreign key the `company` relation is built on. + add_column('company_id', 'String') + add_column('company_count', 'Number') + end + + def define_date_columns + add_column('created_at', 'Date') + add_column('updated_at', 'Date') + add_column('signed_up_at', 'Date') + add_column('last_seen_at', 'Date') + add_column('last_contacted_at', 'Date') + add_column('last_replied_at', 'Date') + add_column('last_email_opened_at', 'Date') + add_column('last_email_clicked_at', 'Date') + end + + def define_reachability_columns + add_column('unsubscribed_from_emails', 'Boolean') + add_column('has_hard_bounced', 'Boolean') + add_column('marked_email_as_spam', 'Boolean') + end + + def define_device_columns + add_column('language_override', 'String') + add_column('browser', 'String') + add_column('browser_language', 'String') + add_column('os', 'String') + add_column('location_country', 'String') + add_column('location_region', 'String') + add_column('location_city', 'String') + end + + # Typed from `GET /data_attributes?model=contact` rather than guessed from + # a payload, and published unfilterable: which operators Intercom answers + # on `custom_attributes.{name}` has not been measured, and this package + # offers no filter it has not seen work. `api_writable` travels on the + # introspected attribute for lot 4b, not on the column -- everything here + # is read-only. + def attribute_kind = 'contact' + + # The owner is a teammate, read whole in one request, and + # `/contacts/search` filters on the key -- so that relation is readable, + # navigable and filterable alike. + # + # The company is none of those last two: the endpoint filters no company + # field, so the traversal is refused by name (see `check_relation_filterable!` + # on the tier). The two lists are the 360 degrees this lot exists for. + def define_relations + add_many_to_one('owner', foreign_collection: 'IntercomAdmin', foreign_key: 'owner_id') + add_many_to_one('company', foreign_collection: 'IntercomCompany', foreign_key: 'company_id') + add_one_to_many('conversations', foreign_collection: 'IntercomConversation', origin_key: 'contact_id') + add_one_to_many('tickets', foreign_collection: 'IntercomTicket', origin_key: 'contact_id') + end + + # The contacts of an account, which `/contacts/search` cannot answer and + # `GET /companies/{id}/contacts` can. Anything else goes the usual way. + def fetch_records(caller, filter, sort = nil) + company = company_lookup(filter) + unless company + refuse_narrowed_company!(filter) if narrowed_company?(filter) + return super + end + + warn_unordered_company_contacts(Array(filter&.sort)) if sort + offset, limit = translate_page(filter&.page) + + walker.walk(offset: offset, limit: limit) do |per_page, cursor| + read_company_page(company, per_page: per_page, cursor: cursor) + end + end + + def count_records(caller, filter) + company = company_lookup(filter) + unless company + refuse_narrowed_company!(filter) if narrowed_company?(filter) + return super + end + + exact_count(read_company_page(company, per_page: 1, cursor: nil)) + end + + # A bare equality and nothing else: an `and` also carrying a scope names a + # narrower set than the account does, and answering it with the account + # alone would serve contacts the scope excludes. + def company_lookup(filter) + tree = filter&.condition_tree + return nil unless tree.is_a?(Leaf) && tree.field.to_s == 'company_id' + return nil unless tree.operator == Operators::EQUAL && blank_search?(filter) + + tree.value&.to_s + end + + # `company_id` plus another condition (e.g. a scope): `GET /companies/{id}/contacts` narrows + # nothing and `/contacts/search` filters no company field, so no request answers both. + def narrowed_company?(filter) + tree = filter&.condition_tree + return false if tree.nil? || tree.is_a?(Leaf) + + tree.some_leaf { |leaf| leaf.field.to_s == 'company_id' && leaf.operator == Operators::EQUAL } + end + + def refuse_narrowed_company!(filter) + raise UnsupportedOperatorError, + "#{name} cannot answer this filter: the contacts of an account are read through " \ + 'GET /companies/{id}/contacts, which returns them whole and narrows nothing, and ' \ + "#{search_endpoint.path} filters no company field -- so the account and the " \ + "#{narrowing_cause(filter)} cannot be asked for in one request. Read the account's contacts " \ + 'without it, or filter on a column this collection is searched on: ' \ + "#{search_endpoint.filterable_columns.join(", ")}." + end + + # A scope and a segment are what put a condition next to the account's in + # practice, and the operator can act on neither the same way -- so the + # message names what it can see rather than guessing. + def narrowing_cause(filter) + others = [] + filter.condition_tree.some_leaf do |leaf| + others << leaf.field.to_s unless leaf.field.to_s == 'company_id' + false + end + + others.empty? ? 'condition filtered alongside it' : "condition on #{others.uniq.join(", ")}" + end + + # An account Intercom no longer answers for -- deleted, or moved outside + # the token's reach between the moment the row was rendered and the + # moment its related list was opened -- reads as an account with no + # contact rather than as a failed page, the way a record read by its id + # already does. + def read_company_page(company, per_page:, cursor:) + client.list_page("companies/#{Faraday::Utils.escape(company)}/contacts", + per_page: [per_page, max_page_size].min, starting_after: cursor) + rescue APIError => e + raise unless e.status == 404 + + Client::Page.new(records: [], next_cursor: nil, total_count: 0) + end + + # One request per fifteen ids instead of one per id: this endpoint answers + # `id IN [...]`, which is what lot 1 already reads the contacts of a page + # through. A contact that was merged away is simply absent from the + # answer -- the row reads as gone, not as a failure. + # + # Put back in the order the ids were named: the search answers in its own, + # and the window was cut on the ids. + def records_by_ids(ids) + wanted = ids.first(max_id_reads) + warn_truncated_ids(ids.size) if ids.size > wanted.size + + records = wanted.each_slice(IDS_PER_READ).flat_map do |chunk| + client.search_page(search_endpoint.path, per_page: chunk.size, + query: { 'field' => 'id', 'operator' => 'IN', + 'value' => chunk }).records + end + position = wanted.each_with_index.to_h + records.sort_by { |record| position.fetch(record['id'].to_s, wanted.size) } + end + + # The same searches, each asked for one record: Intercom counts the slice + # in `total_count`, so nothing needs reading to be counted. + def count_by_ids(ids) + refuse_id_count!(ids.size) if ids.size > max_id_reads + + ids.each_slice(IDS_PER_READ).sum do |chunk| + exact_count(client.search_page(search_endpoint.path, per_page: 1, + query: { 'field' => 'id', 'operator' => 'IN', + 'value' => chunk })) + end + end + + # `sort` here is the clause `server_sort` found `/contacts/search` does + # honour, which is why nothing has reported it yet: it is this route that + # cannot carry the order, not the column. Saying Intercom does not sort + # this collection would be wrong -- it is the one collection it sorts -- + # so this is the counterpart of `warn_unordered_ids`, for the other route + # that leaves the search behind. + def warn_unordered_company_contacts(clauses) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked to sort on " \ + "#{clauses.map { |clause| sort_field(clause) }.join(", ")} while reading the contacts of an account, " \ + 'which Intercom answers through GET /companies/{id}/contacts -- a route that takes no order, unlike ' \ + 'the search this collection is otherwise read through. The rows come back in the order the API imposes.' + ) + end + + def warn_truncated_ids(asked) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked for #{asked} records by id and read the first " \ + "#{max_id_reads}: Intercom reads them #{IDS_PER_READ} at a time. The result is truncated." + ) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact/serializer.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact/serializer.rb new file mode 100644 index 000000000..509351034 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact/serializer.rb @@ -0,0 +1,106 @@ +module ForestAdminDatasourceIntercom + module Collections + class Contact < CursorCollection + # One Intercom contact flattened into the row the schema declares. + # Nothing here reads a sub-resource: every value comes from the payload + # the search already returned. + module Serializer + protected + + def serialize(contact) + attrs = contact.is_a?(Hash) ? contact : {} + + native(attrs) + .merge(account_of(attrs['companies'])) + .merge(location_of(attrs['location'])) + .merge(attribute_values(attrs['custom_attributes'])) + end + + private + + def native(attrs) + identity(attrs).merge(dates_of(attrs)).merge(flags(attrs)).merge(device_of(attrs)) + end + + def identity(attrs) + { + 'id' => stringify_id(attrs['id']), + 'role' => attrs['role'], + 'name' => attrs['name'], + 'email' => attrs['email'], + # Derived rather than read, and filterable all the same: the search + # endpoint carries a field of its own for it, which is what turns + # "everyone at this customer" into a filter instead of a wildcard. + 'email_domain' => domain_of(attrs['email']), + 'phone' => attrs['phone'], + 'external_id' => attrs['external_id'], + 'avatar' => attrs['avatar'], + 'owner_id' => stringify_id(attrs['owner_id']), + 'session_count' => attrs['session_count'] + } + end + + def dates_of(attrs) + { + 'created_at' => stamp(attrs['created_at']), + 'updated_at' => stamp(attrs['updated_at']), + 'signed_up_at' => stamp(attrs['signed_up_at']), + 'last_seen_at' => stamp(attrs['last_seen_at']), + 'last_contacted_at' => stamp(attrs['last_contacted_at']), + 'last_replied_at' => stamp(attrs['last_replied_at']), + 'last_email_opened_at' => stamp(attrs['last_email_opened_at']), + 'last_email_clicked_at' => stamp(attrs['last_email_clicked_at']) + } + end + + def flags(attrs) + { + 'unsubscribed_from_emails' => attrs['unsubscribed_from_emails'], + 'has_hard_bounced' => attrs['has_hard_bounced'], + 'marked_email_as_spam' => attrs['marked_email_as_spam'] + } + end + + def device_of(attrs) + { + 'language_override' => attrs['language_override'], + 'browser' => attrs['browser'], + 'browser_language' => attrs['browser_language'], + 'os' => attrs['os'] + } + end + + # A contact belongs to several accounts, and the row names the first of + # them and counts them -- the same reading a conversation gives its + # contacts. The whole list is a hop away, on the `company` relation. + def account_of(companies) + list = nested_list(companies, 'data') + first = list.first.is_a?(Hash) ? list.first : {} + + { 'company_id' => stringify_id(first['id']), 'company_count' => account_count(companies, list) } + end + + # Intercom caps the accounts it nests on a contact and says how many + # there really are, so the count is read rather than measured on the + # list -- a contact belonging to twelve accounts must not read as + # belonging to the ten the payload had room for. + def account_count(companies, list) + declared = companies['total_count'] if companies.is_a?(Hash) + + declared.is_a?(Numeric) ? declared : list.size + end + + def location_of(location) + attrs = location.is_a?(Hash) ? location : {} + + { 'location_country' => attrs['country'], 'location_region' => attrs['region'], + 'location_city' => attrs['city'] } + end + + def domain_of(email) + email.to_s[/@(.+)\z/, 1] + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb new file mode 100644 index 000000000..1f550cd92 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb @@ -0,0 +1,77 @@ +module ForestAdminDatasourceIntercom + module Collections + # Intercom nests only contact ids, so `contact_name` costs one bulk read per page, never per row. + module ContactIdentity + COLUMNS = %w[contact_name].freeze + + # Must match the `contact` relation's chunking so `Client#with_read_scope` dedups the two + # reads into one request (guarded by `ticket_list_budget_spec`). + CONTACT_CHUNK = Contact::IDS_PER_READ + + private + + def define_contact_columns + add_column('contact_id', 'String') + add_column('contact_count', 'Number') + add_column('contact_name', 'String') + end + + # A group conversation, or a ticket opened for several people, has more + # than one contact: the row names the first and counts them, rather than + # presenting one of several as the one. The `contact` relation resolves + # that same first contact, so the column and the relation cannot disagree; + # the others are reached through the contact's own conversations. + def contact_columns_for(attrs) + ids = nested_list(attrs['contacts'], 'contacts').filter_map { |contact| stringify_id(contact['id']) } + + { 'contact_id' => ids.first, 'contact_count' => ids.size, + # Filled by the bulk read below, and left nil when the projection did + # not ask for it. + 'contact_name' => nil } + end + + def first_contact_id(record) + contact = nested_list((record || {})['contacts'], 'contacts').first + contact.is_a?(Hash) ? stringify_id(contact['id']) : nil + end + + # The Contacts endpoint as the table spells it, rather than a path written + # a second time here: this is the same `/contacts/search` the Contacts + # collection reads itself through, and a table that renamed it would + # otherwise leave this one behind. + def contact_search_path + @contact_search_path ||= Query::SearchFields.fetch('contacts').path + end + + def embed_contact_identity(records, rows, projection) + return unless any_column_asked?(projection, COLUMNS) + + identities = contact_identities(records) + records.each_with_index do |record, index| + identity = identities[first_contact_id(record)] || {} + rows[index]['contact_name'] = identity['name'] if rows[index].key?('contact_name') + end + end + + # A failure costs the column and nothing else: an identity that could not + # be read is not a page that could not be served. + def contact_identities(records) + ids = records.filter_map { |record| first_contact_id(record) }.uniq + return {} if ids.empty? + + ids.each_slice(CONTACT_CHUNK).with_object({}) do |chunk, indexed| + page = client.search_page(contact_search_path, per_page: chunk.size, + query: { 'field' => 'id', 'operator' => 'IN', + 'value' => chunk }) + page.records.each { |contact| indexed[contact['id'].to_s] = contact } + end + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} could not read the contacts of this page (HTTP " \ + "#{e.status || "-"}); the name column is left empty for it." + ) + {} + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb new file mode 100644 index 000000000..0839e05e5 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb @@ -0,0 +1,153 @@ +module ForestAdminDatasourceIntercom + module Collections + # Records come back under `conversations`, not `data`. Every read sends `display_as=plaintext`: bodies + # are customer-written HTML, unsafe to render inside Forest (R10). + # Long by line count only: most of it declares the columns, one call each. + class Conversation < CursorCollection + include ContactIdentity + include Conversation::Serializer + # The shared thread, then the hooks this collection puts over it. + include Collections::Timeline + include Conversation::Timeline + + # How many conversations of one page may have their timeline read. The + # parts are absent from the listing response -- Intercom returns them only + # when retrieving a single conversation -- so a timeline asked for in a + # list view costs one request per row. Bounded rather than turned into a + # page the operator waits half a minute for; rows past the cap are left at + # nil, which reads as "unknown", never as "this conversation is empty". + MAX_TIMELINE_READS = 10 + + def initialize(datasource) + super(datasource, 'IntercomConversation') + # The one collection of this datasource Intercom matches text on: `~` on + # `source.body`, which is the message that opened the conversation. + enable_search + end + + protected + + def list_endpoint = 'conversations' + def list_key = 'conversations' + def searchable = 'conversations' + def search_column = 'source_body' + + # Sent on the search too, where Intercom does not document it: the bodies + # are HTML written by end customers (R10), and a parameter it ignores costs + # a query string while the one it honours saves every filtered row from + # coming back as markup. + def read_params = { 'display_as' => 'plaintext' } + + # The contact identity and the timeline, each read only when the projection + # names it: neither is on the conversation payload, and a page that never + # asked for them must not pay for them. + def enrich(records, rows, projection) + embed_contact_identity(records, rows, projection) + embed_timeline(records, rows, projection) + end + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('title', 'String') + # Left plain strings rather than enums: the values a workspace really + # serves are worth measuring before the interface offers them as a + # closed list, and nothing filters on them in this lot anyway. + add_column('state', 'String') + add_column('priority', 'String') + add_column('open', 'Boolean') + add_column('read', 'Boolean') + add_column('created_at', 'Date') + add_column('updated_at', 'Date') + add_column('waiting_since', 'Date') + add_column('snoozed_until', 'Date') + define_assignment_columns + define_contact_columns + define_source_columns + define_statistics_columns + add_column('tag_names', 'Json') + add_column('ai_agent_participated', 'Boolean') + add_column('timeline', 'Json') + define_relations + end + + # /conversations/search filters on all three keys and on a contact id (measured), so `contact:email` is + # resolved through /contacts/search. No company relation: Intercom filters no company field here. A group + # conversation names its first contact. + def define_relations + add_many_to_one('admin_assignee', foreign_collection: 'IntercomAdmin', foreign_key: 'admin_assignee_id') + add_many_to_one('team_assignee', foreign_collection: 'IntercomTeam', foreign_key: 'team_assignee_id') + add_many_to_one('closed_by', foreign_collection: 'IntercomAdmin', foreign_key: 'closed_by_id') + add_many_to_one('contact', foreign_collection: 'IntercomContact', foreign_key: 'contact_id') + end + + # Who the conversation sits with, and which account it belongs to. The + # conversation carries its company as a whole object, so the account name + # is free here -- unlike on a ticket, which carries the id alone. + def define_assignment_columns + add_column('admin_assignee_id', 'String') + add_column('team_assignee_id', 'String') + add_column('company_id', 'String') + add_column('company_name', 'String') + end + + # The message that opened the conversation lives in `source`, not in the + # parts. A timeline built from the parts alone loses it, which is the one + # message nobody opens a conversation without wanting to read. + def define_source_columns + add_column('source_type', 'String') + add_column('source_subject', 'String') + add_column('source_body', 'String') + add_column('source_author_name', 'String') + add_column('source_author_email', 'String') + add_column('source_delivered_as', 'String') + end + + # Intercom keeps the lifecycle of a conversation in `statistics`, which is + # where the closure date and the reply timestamps come from. Flattened onto + # the row: they cost nothing, they are exact, and they are what an ops lead + # reads a queue for. + def define_statistics_columns + add_column('closed_at', 'Date') + add_column('first_closed_at', 'Date') + add_column('closed_by_id', 'String') + add_column('first_contact_reply_at', 'Date') + add_column('last_contact_reply_at', 'Date') + add_column('last_admin_reply_at', 'Date') + add_column('reopen_count', 'Number') + add_column('part_count', 'Number') + end + + # A record read through the record endpoint already carries its parts, so + # its timeline is free; one read from the listing does not, and pays a + # request. Rows past the cap keep the nil the projection put there. + def embed_timeline(records, rows, projection) + return unless column_asked?(projection, 'timeline') + + unread = [] + records.each_with_index do |record, index| + if parts_of(record) + rows[index]['timeline'] = build_timeline(record) + else + unread << index + end + end + read = unread.first(MAX_TIMELINE_READS) + details = client.fetch_records(list_endpoint, read.map { |index| records[index]['id'] }, params: read_params) + read.zip(details) { |index, detail| rows[index]['timeline'] = detail && build_timeline(detail) } + + missing = unread.size - read.size + warn_truncated_timelines(missing) if missing.positive? + end + + def warn_truncated_timelines(missing) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} left the timeline of #{missing} row(s) unread: Intercom " \ + 'returns the parts only when retrieving one conversation, so a list view pays a request per row and ' \ + "this reads at most #{MAX_TIMELINE_READS}. Those rows show no timeline rather than an empty one." + ) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/serializer.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/serializer.rb new file mode 100644 index 000000000..e241b168c --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/serializer.rb @@ -0,0 +1,75 @@ +module ForestAdminDatasourceIntercom + module Collections + class Conversation < CursorCollection + # One Intercom conversation flattened into the row the schema declares. + # Nothing here reads a sub-resource: every value comes from the payload the + # listing already returned. + module Serializer + protected + + def serialize(conversation) + attrs = conversation.is_a?(Hash) ? conversation : {} + + native(attrs) + .merge(contact_columns_for(attrs)) + .merge(source_of(attrs['source'])) + .merge(statistics_of(attrs['statistics'])) + end + + private + + def native(attrs) + company = attrs['company'].is_a?(Hash) ? attrs['company'] : {} + + { + 'id' => stringify_id(attrs['id']), + 'title' => attrs['title'], + 'state' => attrs['state'], + 'priority' => attrs['priority'], + 'open' => attrs['open'], + 'read' => attrs['read'], + 'created_at' => stamp(attrs['created_at']), + 'updated_at' => stamp(attrs['updated_at']), + 'waiting_since' => stamp(attrs['waiting_since']), + 'snoozed_until' => stamp(attrs['snoozed_until']), + 'admin_assignee_id' => stringify_id(attrs['admin_assignee_id']), + 'team_assignee_id' => stringify_id(attrs['team_assignee_id']), + 'company_id' => stringify_id(company['id']), + 'company_name' => company['name'], + 'tag_names' => nested_list(attrs['tags'], 'tags').filter_map { |tag| tag['name'] if tag.is_a?(Hash) }, + 'ai_agent_participated' => attrs['ai_agent_participated'] + } + end + + def source_of(source) + attrs = source.is_a?(Hash) ? source : {} + author = attrs['author'].is_a?(Hash) ? attrs['author'] : {} + + { 'source_type' => attrs['type'], + 'source_subject' => attrs['subject'], + # Plaintext, because `display_as=plaintext` rides on every read: the + # bodies are HTML written by end customers. + 'source_body' => attrs['body'], + 'source_author_name' => author['name'], + 'source_author_email' => author['email'], + 'source_delivered_as' => attrs['delivered_as'] } + end + + # `statistics` is null on a conversation Intercom has computed nothing + # for yet; every column then reads as absent rather than as zero. + def statistics_of(statistics) + attrs = statistics.is_a?(Hash) ? statistics : {} + + { 'closed_at' => stamp(attrs['last_close_at']), + 'first_closed_at' => stamp(attrs['first_close_at']), + 'closed_by_id' => stringify_id(attrs['last_closed_by_id']), + 'first_contact_reply_at' => stamp(attrs['first_contact_reply_at']), + 'last_contact_reply_at' => stamp(attrs['last_contact_reply_at']), + 'last_admin_reply_at' => stamp(attrs['last_admin_reply_at']), + 'reopen_count' => attrs['count_reopens'], + 'part_count' => attrs['count_conversation_parts'] } + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/timeline.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/timeline.rb new file mode 100644 index 000000000..bb7fe62cd --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/timeline.rb @@ -0,0 +1,40 @@ +module ForestAdminDatasourceIntercom + module Collections + class Conversation < CursorCollection + # What a conversation adds to the shared thread: where its parts live, and + # the entry that opens it. + # + # The opening message lives in `source`, not in the parts -- a timeline + # built from the parts alone opens on the first reply and loses what the + # customer actually asked. + module Timeline + # The pseudo type of the opening entry. Not an Intercom part type: it is + # the source, and calling it `comment` would make it indistinguishable + # from the replies that follow. + SOURCE_PART_TYPE = 'conversation_started'.freeze + + private + + # nil rather than an empty list when the payload carries no parts at all: + # a listing response has none, and reading that as "this conversation is + # empty" is exactly the answer that looks complete without being it. + def parts_of(conversation) + container = (conversation || {})['conversation_parts'] + return nil unless container.is_a?(Hash) + + parts = container['conversation_parts'] + parts.is_a?(Array) ? parts : nil + end + + def opening_entry(attrs) + source = attrs['source'] + return nil unless source.is_a?(Hash) + + entry(part_type: SOURCE_PART_TYPE, created_at: attrs['created_at'], author: source['author'], + body: source['body'], attachments: source['attachments']) + .merge('id' => stringify_id(source['id'])) + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb new file mode 100644 index 000000000..abbf3ca11 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb @@ -0,0 +1,363 @@ +module ForestAdminDatasourceIntercom + module Collections + # Cursor-paginated tier: only a page is in hand, so nothing is filtered, sorted or counted in + # memory. Whatever the search DSL cannot express is refused, never served unfiltered. + # Long by line count only: half of it is the refusals and their remedies. + class CursorCollection < BaseCollection # rubocop:disable Metrics/ClassLength + include RecordsById + + # Countable, and exactly: unlike the pages a walk collected, `total_count` + # is the whole dataset the filter names. + def initialize(datasource, name) + super + enable_count + end + + # One request per id here, so a relation pointing at this tier resolves a + # slice at a time and is refused past what a slice may hold -- see + # `Relations#target_rows`. The collection that reads its ids in bulk + # widens both; see `Contact`. + def ids_per_read = max_id_reads + def max_resolvable_ids = max_id_reads + + # The read scope spans the whole page rather than any one step of it: the + # enrichment and the relation embed below ask Intercom for the same + # contacts, from two places that cannot see each other. See + # `Client#with_read_scope`. + def list(caller, filter, projection) + client.with_read_scope do + records = fetch_records(caller, filter, server_sort(filter)) + # Serialized whole and projected afterwards rather than the other way + # round: a projection reaching through a relation names no foreign + # key, and the key is where the relation is read from. + serialized = records.map { |record| serialize(record) } + rows = serialized.map { |record| project(record, projection) } + + enrich(records, rows, projection) + embed_relations(caller, serialized, rows, projection) + rows + end + end + + # Count only, and never a group: Intercom exposes no aggregate endpoint, + # and grouping over the pages a walk happened to collect would look exact + # while answering a fraction. Refused here rather than through the + # contract's NotImplementedError, which reads as an oversight. + def aggregate(caller, filter, aggregation, _limit = nil) + refuse_unsupported_aggregation!(aggregation) + + [{ 'group' => {}, 'value' => count_records(caller, filter) }] + end + + protected + + def list_endpoint = raise(NotImplementedError, "#{self.class} did not implement list_endpoint") + def record_endpoint = list_endpoint + def list_key = 'data' + def read_params = {} + + # The row of the measured table this collection is filtered through: what + # its columns may advertise, and what the translator is allowed to write. + def searchable = raise(NotImplementedError, "#{self.class} did not implement searchable") + + def search_endpoint + @search_endpoint ||= Query::SearchFields.fetch(searchable) + end + + # The column a free-text search is answered on, for a collection whose + # endpoint has one. Nil elsewhere, and a search is then refused rather than + # answered by a page that ignored it. + def search_column = nil + + def serialize(_entity) = raise(NotImplementedError, "#{self.class} did not implement serialize") + + # Hook for what a row needs beyond its own payload. Left empty here: what + # it costs is the collection's business, not this base's. + def enrich(_records, _rows, _projection); end + + # Bounded per collection rather than by the API maximum: Intercom offers no + # field selection, so a collection whose rows carry their whole timeline + # pays for it by the page. See Ticket. + def max_page_size = Client::MAX_PER_PAGE + + # One page of the collection. A listing for conversations, a search for + # tickets -- Intercom exposes no `GET /tickets` at all -- so the endpoint + # and its shape belong to the collection, while walking it does not. + def read_page(per_page:, cursor:, query: nil, sort: nil) + size = [per_page, max_page_size].min + # The listing endpoint neither filters nor sorts, and a collection whose + # records are only reachable through the search has no listing at all. + # Either way what routes the read to the search is a query, so the one + # that matches everything stands in for the condition there is none of. + query ||= match_all_query if sort || searchable_only? + + if query.nil? + client.list_page(list_endpoint, per_page: size, starting_after: cursor, + params: read_params, list_key: list_key) + else + client.search_page(search_endpoint.path, query: query, per_page: size, starting_after: cursor, + params: read_params, list_key: list_key, sort: sort) + end + end + + # Operators come from the measured table only, so no column offers a filter the translator + # would refuse. Only `/contacts/search` honours a sort; the other search endpoints silently + # ignore one, so their columns are never sortable. + def add_column(name, type, is_primary_key: false) + add_field(name, ColumnSchema.new(column_type: type, + filter_operators: column_operators(name, is_primary_key), + is_primary_key: is_primary_key, + is_read_only: true, + is_sortable: sortable_column?(name), + is_groupable: false)) + end + + def sortable_column?(name) + search_endpoint.field(name)&.sortable? == true + end + + # The Intercom query standing in for the condition a read has none of. + # Two collections need one and for different reasons: Intercom exposes no + # `GET /tickets` at all, and an order is applied by the search endpoint + # alone. Nil where the listing endpoint answers both, which is where a + # sort is reported as ignored rather than dropped. + def match_all_query = nil + + def searchable_only? = false + + # Bounded, unlike the tier read whole: one more than a group may hold is + # all it takes to know the fan-out will not fit, and reading further would + # walk a whole collection to refuse it afterwards. + def match_page + ForestAdminDatasourceToolkit::Components::Query::Page.new( + offset: 0, limit: Query::ConditionTreeTranslator::MAX_GROUP_SIZE + 1 + ) + end + + def walker + @walker ||= Pagination::CursorWalker.new + end + + # How many records a bulk read by id may fetch. A hook rather than the + # constant, since the collection that reads its ids in bulk answers far + # more of them for the same request count. + def max_id_reads = MAX_ID_READS + + # Columns answered by a route of the collection's own rather than by its + # search endpoint, with the operators that route takes. + def routed_operators = {} + + private + + def column_operators(name, is_primary_key) + return [Operators::EQUAL, Operators::IN] if is_primary_key + return routed_operators[name] if routed_operators.key?(name) + + field = search_endpoint.field(name) + + field ? Query::OperatorTable.forest_operators(field) : [] + end + + def fetch_records(caller, filter, sort = nil) + ids = id_lookup(filter) + # The window is cut out of the ids rather than out of the records they + # read: Intercom reads them one request each, so paging after the read + # would pay for a whole page to hand back a slice of it -- and page 2 of + # a set larger than the cap would come back empty, the records it names + # having been dropped by the truncation before the window was applied. + if ids + # Which is also why an order cannot be honoured on this route: the + # window is cut in the order the ids were named, so sorting what comes + # back would order a slice picked by something else. Reported rather + # than dropped in silence, like every other order this tier cannot + # apply -- `server_sort` stayed quiet, having found a column Intercom + # does sort. + warn_unordered_ids(sort) if sort + return records_by_ids(page_window(ids, filter)) + end + + query = translate(caller, filter) + # A condition through a relation the target matched no record with names + # no row, and Intercom's DSL cannot say so: the read is skipped rather + # than sent as a filter that would come back with everything. + return [] if query == NOTHING + + listed_records(filter, query, sort) + end + + # The Intercom query a filter comes down to, or nil for a list view, which + # walks the listing endpoint instead. The free-text search is folded into + # the condition tree rather than added to the translated query: written as + # one tree, it is checked against the nesting Intercom allows like every + # other condition, instead of adding a level nothing counted. + def translate(caller, filter) + tree = rewrite_relation_conditions(caller, combined_tree(filter)) do |key, ids, leaf| + relation_group(key, ids, leaf) + end + return NOTHING if tree == NOTHING + + Query::ConditionTreeTranslator.call(tree, endpoint: search_endpoint, collection: name, + timezone: timezone_for(caller), + attribute_columns: attribute_column_names) + end + + # An OR of equalities: the DSL has no IN on foreign keys, and caps a group at 15 conditions, + # so a larger match is refused here where the relation can be named. A RelationBranch so it + # can be inlined into a same-aggregator parent, saving a nesting level. + def relation_group(key, ids, leaf) + refuse_fan_out!(leaf, key, ids) if ids.size > Query::ConditionTreeTranslator::MAX_GROUP_SIZE + return Leaf.new(key, Operators::EQUAL, ids.first) if ids.size == 1 + + RelationBranch.new('Or', ids.map { |id| Leaf.new(key, Operators::EQUAL, id) }, leaf.field) + end + + # Inlining a relation group into its parent trades a level of nesting for + # width, and Intercom bounds both. Past the conditions a group may hold, + # the nested form is the one it answers. + def absorb_relation_group?(size) + size <= Query::ConditionTreeTranslator::MAX_GROUP_SIZE + end + + # A relation this endpoint filters nothing through. It is navigable all the + # same -- the read costs nothing, the target being read whole -- and saying + # which of the two it is, is the whole point of the message. + def relation_key_filterable?(key) = !search_endpoint.field(key).nil? + + def check_relation_filterable!(leaf, relation) + key = relation.foreign_key + refuse_unfilterable_key!(leaf, key) unless relation_key_filterable?(key) + end + + def refuse_unfilterable_key!(leaf, key) + raise UnsupportedOperatorError, + "#{name} cannot filter #{leaf.field.inspect}: the relation resolves to #{key.inspect}, on " \ + "which #{search_endpoint.path} takes no filter. The relation is there to be read and " \ + "navigated; filter on one of: #{search_endpoint.filterable_columns.join(", ")}." + end + + def refuse_fan_out!(leaf, key, _ids) + # "more than", never a count: the target is read one record past what a + # group may hold, so what is known is that it does not fit -- printing + # 16 where a workspace holds three thousand would read as a number the + # operator could go and narrow by one. + raise UnsupportedOperatorError, + "#{name} cannot filter #{leaf.field.inspect}: it names more than " \ + "#{Query::ConditionTreeTranslator::MAX_GROUP_SIZE} records, " \ + "#{search_endpoint.path} answers #{key.inspect} one value at a time, and Intercom takes " \ + "#{Query::ConditionTreeTranslator::MAX_GROUP_SIZE} conditions per group. Narrow the condition " \ + "on the relation, or filter on #{key.inspect} itself." + end + + def combined_tree(filter) + conditions = [filter&.condition_tree, search_condition(filter)].compact + return conditions.first if conditions.size < 2 + + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeFactory.intersect(conditions) + end + + # A free-text search reaches Intercom as a condition on the one column its + # endpoint matches text on -- per word, not as a substring, which the + # README says rather than the interface implying otherwise. + def search_condition(filter) + return nil if blank_search?(filter) + + refuse_search! if search_column.nil? + + Leaf.new(search_column, Operators::CONTAINS, filter.search.to_s.strip) + end + + # The records of this tier carry bodies written by end customers, so every + # read of one asks Intercom for plain text (R10). `BaseCollection` reads + # this on the way to the record endpoint. + def record_read_params = read_params + + def listed_records(filter, query, sort = nil) + offset, limit = translate_page(filter&.page) + + walker.walk(offset: offset, limit: limit) do |per_page, cursor| + read_page(per_page: per_page, cursor: cursor, query: query, sort: sort) + end + end + + # A filter with no page asks for every record it matched; the walker reads + # that as the nil limit it bounds with its own caps. + def translate_page(page) + return [0, nil] if page.nil? + + [page.offset.to_i.clamp(0, nil), page_limit(page)] + end + + # Exact, and one request: `total_count` counts what the filter names, not + # what a page happened to hold. An id lookup counts the records it found, + # which is cheaper still. + def count_records(caller, filter) + ids = id_lookup(filter) + return count_by_ids(ids) if ids + + query = translate(caller, filter) + return 0 if query == NOTHING + + exact_count(read_page(per_page: 1, cursor: nil, query: query)) + end + + def refuse_search! + raise UnsupportedOperatorError, + "#{name} cannot answer a free-text search: #{search_endpoint.path} matches values field by field, " \ + 'and this collection exposes no text column it searches. Filter on a column instead of searching.' + end + + # The order Intercom will really apply, written the way the client sends + # it -- or nil, and the order the operator asked for is then reported + # rather than dropped in silence. + # + # The ascending primary-key sort the agent injects when a request names + # none is neither honoured nor reported: it is not an order anybody asked + # for, and `/contacts/search` does not sort on an id anyway. + def server_sort(filter) + clauses = Array(filter&.sort) + return nil if clauses.empty? || default_pk_sort?(clauses) + + honoured = honourable_sort(clauses) + warn_ignored_sort(clauses) if honoured.nil? + honoured + end + + # One clause, on a column the measured table says the endpoint sorts. + # Intercom takes a single `{ field, order }` and nothing composite, so a + # second clause is not half-honoured: honouring the first alone would + # order the page by something the operator did not ask for. + def honourable_sort(clauses) + return nil unless clauses.size == 1 + + clause = clauses.first + field = search_endpoint.field(sort_field(clause).to_s) + return nil unless field&.sortable? + + { field: field.field, ascending: ascending?(clause) } + end + + # Intercom accepts a `sort` on the other two endpoints and ignores it + # without a word -- measured -- so an order asked for and not applied has + # to be reported here or nowhere. + def warn_ignored_sort(clauses) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked to sort on " \ + "#{clauses.map { |clause| sort_field(clause) }.join(", ")}, which Intercom does not sort this " \ + 'collection on -- and it ignores a sort it refuses without reporting it. The rows come back in the ' \ + 'order the API imposes.' + ) + end + + # `sort` here is the clause `server_sort` found the endpoint does honour, + # which is why nothing has reported it yet: it is this route, not the + # column, that cannot carry it. + def warn_unordered_ids(sort) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked to sort on #{sort[:field].inspect} while reading " \ + 'records by id, which Intercom reads by id and in no order. The rows come back in the order the ids ' \ + 'were named.' + ) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/custom_attributes.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/custom_attributes.rb new file mode 100644 index 000000000..e5f57ad5f --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/custom_attributes.rb @@ -0,0 +1,55 @@ +module ForestAdminDatasourceIntercom + module Collections + # An attribute whose name is already a column or relation (measured: a contact attribute named `id`) is + # skipped with a warning, so relations must be declared first. Dates arrive as epoch seconds. + module CustomAttributes + private + + def register_attribute_columns + @attribute_columns = @attributes.reject { |attribute| collides?(attribute) } + @attribute_columns.each { |attribute| add_column(attribute.column_name, attribute.column_type) } + end + + def attribute_columns = @attribute_columns || [] + + # What the translator needs of them: the names the schema published, so a + # filter reaching one is refused with the reason the table carries for + # the whole family rather than with the message for a column that is not + # in it. + def attribute_column_names = attribute_columns.map(&:column_name) + + # What the log calls these, which is the workspace's own vocabulary: a + # ticket attribute is declared per ticket type, a contact attribute per + # model. + def attribute_kind = raise(NotImplementedError, "#{self.class} did not implement attribute_kind") + + def collides?(attribute) + return false unless fields.key?(attribute.column_name) + + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} skips the #{attribute_kind} attribute " \ + "#{attribute.name.inspect}: a native column or relation already carries the name " \ + "#{attribute.column_name.inspect}, and overwriting it would show the attribute where the operator " \ + 'expects the record field. Rename it in Intercom to publish it.' + ) + true + end + + # Nil rather than absent when the record lacks the attribute: the column exists on every row. + def attribute_values(values) + held = values.is_a?(Hash) ? values : {} + + attribute_columns.to_h do |attribute| + [attribute.column_name, coerce_attribute(held[attribute.name], attribute)] + end + end + + def coerce_attribute(value, attribute) + return nil if value.nil? + return stamp(value) if attribute.column_type == 'Date' && value.is_a?(Numeric) + + value + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb new file mode 100644 index 000000000..a184a0744 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb @@ -0,0 +1,196 @@ +module ForestAdminDatasourceIntercom + module Collections + # Collections Intercom returns whole in one unpaginated response, so filtering, sorting and + # counting in memory is exact. The response is cached for `reference_cache_ttl` (0 disables). + class FetchAllCollection < BaseCollection # rubocop:disable Metrics/ClassLength + # The filters a column may advertise, per column type. Restricted to what + # the toolkit can evaluate in memory, since the in-memory pass is the only + # pass there is here: an operator with no equivalence makes `match` answer + # nil, which `apply` reads as "no match" and would empty the page instead + # of filtering it. + OPERATOR_CANDIDATES = { + 'String' => [Operators::EQUAL, Operators::NOT_EQUAL, Operators::IN, Operators::NOT_IN, + Operators::PRESENT, Operators::BLANK, Operators::CONTAINS, Operators::I_CONTAINS, + Operators::NOT_CONTAINS, Operators::STARTS_WITH, Operators::ENDS_WITH], + 'Boolean' => [Operators::EQUAL, Operators::NOT_EQUAL, Operators::IN, Operators::NOT_IN, + Operators::PRESENT, Operators::BLANK] + }.freeze + + # The operators `ConditionTreeLeaf#match` evaluates natively; anything else + # needs an equivalence for the column's type to be evaluable at all. + IN_MEMORY_OPERATORS = [Operators::IN, Operators::EQUAL, Operators::LESS_THAN, Operators::GREATER_THAN, + Operators::MATCH, Operators::STARTS_WITH, Operators::ENDS_WITH, + Operators::LONGER_THAN, Operators::SHORTER_THAN, Operators::INCLUDES_ALL, + Operators::NOT_IN, Operators::NOT_EQUAL, Operators::NOT_CONTAINS].freeze + + # Candidates are re-checked against the toolkit rather than trusted, so an + # equivalence it stops providing takes the filter out of the schema instead + # of turning every page using it into an empty one. + def self.operators_for(column_type) + Array(OPERATOR_CANDIDATES[column_type]).select do |operator| + Equivalent.equivalent_tree?(operator, IN_MEMORY_OPERATORS, column_type) + end + end + + # Countable, unlike the cursor-paginated collections: the count answered + # here is taken over every record Intercom holds rather than over the pages + # a walk happened to collect. + def initialize(datasource, name) + super + enable_count + end + + def list(caller, filter, projection) + client.with_read_scope do + records = sort_in_memory(filtered_records(caller, filter), filter&.sort) + window = page_window(records, filter) + rows = window.map { |record| project(record, projection) } + + enrich(window, rows, projection) + embed_relations(caller, window, rows, projection) + rows + end + end + + # Relations are embedded first: a group through one (`team:name`) would otherwise read nil + # and fold every row into one group. + def aggregate(caller, filter, aggregation, limit = nil) + client.with_read_scope do + records = filtered_records(caller, filter) + rows = records.map(&:dup) + embed_relations(caller, records, rows, aggregation.projection) + + aggregation.apply(rows, timezone_for(caller), limit) + .map { |row| { 'group' => row[:group], 'value' => row[:value] } } + end + end + + protected + + # Read-only: an editable column would offer a Save reaching an unimplemented `update`. + def add_column(name, type, is_primary_key: false) + add_field(name, ColumnSchema.new(column_type: type, + filter_operators: self.class.operators_for(type), + is_primary_key: is_primary_key, + is_read_only: true, + is_sortable: type != 'Json', + is_groupable: type != 'Json')) + end + + # Hook for what a row needs beyond the endpoint this collection reads -- + # a name held on the other side of a membership, say. Called with the rows + # of the page only, and with the projection, so a column nobody asked for + # costs no request. + def enrich(_records, _rows, _projection); end + + # Every record of the collection, straight from its endpoint. + def fetch_all = raise(NotImplementedError, "#{self.class} did not implement fetch_all") + + def serialize(_entity) = raise(NotImplementedError, "#{self.class} did not implement serialize") + + private + + # The complete dataset, serialized and narrowed to the rows the filter + # keeps: what `list` pages and what `aggregate` counts are the same rows. + def filtered_records(caller, filter) + records = fetch_all.map { |entity| serialize(entity) } + tree = filter&.condition_tree + return records if tree.nil? + + # A condition through a relation becomes a membership on the foreign key: + # this tier filters in memory, where a list of ids costs no more than one + # id and none of Intercom's group limits apply -- they bound its search + # DSL, which nothing here goes through. + tree = rewrite_relation_conditions(caller, tree) { |key, ids, _| Leaf.new(key, Operators::IN, ids) } + return [] if tree == NOTHING + + refuse_unevaluable!(tree) + tree.apply(records, self, timezone_for(caller)) + end + + # A condition this collection cannot evaluate is refused rather than + # applied. `match` answers nil for an operator with no in-memory + # equivalence and `apply` reads that as "no match", so an unevaluable + # condition would hand back an empty page that looks like a filter + # matching nothing -- indistinguishable, to the operator, from a real + # answer. The schema advertises no such operator; a scope, a segment or a + # customizer can still send one. + def refuse_unevaluable!(tree) + offender = nil + tree.some_leaf do |leaf| + offender = leaf unless evaluable?(leaf) + !offender.nil? + end + return if offender.nil? + + raise UnsupportedOperatorError, + "#{name} cannot filter '#{offender.field}' with '#{offender.operator}': it is read whole from " \ + 'Intercom and filtered in memory, which supports only the operators its columns advertise. ' \ + 'Change the condition, or the scope or segment carrying it.' + end + + def evaluable?(leaf) + schema = fields[leaf.field] + schema.is_a?(ColumnSchema) && schema.filter_operators.include?(leaf.operator) + end + + # Not `sort`/`Sort#apply`: `sort` is unstable and `<=>` answers nil on nulls, booleans and + # mixed types. Ties fall back to Intercom's order. + def sort_in_memory(records, sort) + clauses = sort_clauses(sort) + return records if clauses.empty? + + records.each_with_index.sort do |(left, left_index), (right, right_index)| + compare_clauses(left, right, clauses).nonzero? || (left_index <=> right_index) + end.map(&:first) + end + + # Unsortable clauses are dropped with a warning. They do arrive: `Filter#nest` prefixes the + # condition tree but not the sort, so a many-to-many membership gets the target's columns. + def sort_clauses(sort) + known, unknown = Array(sort).partition { |clause| sortable_column?(sort_field(clause)) } + warn_unsortable(unknown) unless unknown.empty? + + known.map { |clause| [sort_field(clause), ascending?(clause) != false] } + end + + def sortable_column?(field) + schema = fields[field] + schema.is_a?(ColumnSchema) && schema.is_sortable + end + + def warn_unsortable(clauses) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked to sort on " \ + "#{clauses.map { |clause| sort_field(clause).inspect }.join(", ")}, which it does not carry as a " \ + 'sortable column; the rows ' \ + 'come back in the order Intercom returned them. A related list through a many-to-many is ordered by the ' \ + 'columns of the collection it travels through, not by those of the collection it reaches.' + ) + end + + def compare_clauses(left, right, clauses) + clauses.each do |field, ascending| + comparison = compare_values(left[field], right[field]) + next if comparison.zero? + + return ascending ? comparison : -comparison + end + + 0 + end + + # Nulls sort last ascending and first descending, the way a database orders + # them; values `<=>` cannot compare -- two booleans, for one -- are + # compared through their string form rather than left undefined, which puts + # `false` before `true`, again like a database. + def compare_values(left, right) + return 0 if left.nil? && right.nil? + return 1 if left.nil? + return -1 if right.nil? + + (left <=> right) || (left.to_s <=> right.to_s) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb new file mode 100644 index 000000000..14049d5d3 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb @@ -0,0 +1,302 @@ +module ForestAdminDatasourceIntercom + module Collections + # `POST /companies/list` paginates by offset. Companies have no search endpoint + # (`/companies/scroll` allows one open scroll per app, expiring in a minute), so filters are + # limited to exact lookups and everything else is refused. Long by line count only: refusals. + class OffsetCollection < BaseCollection # rubocop:disable Metrics/ClassLength + include RecordsById + + # What one page holds when the read names no window: a relation resolving + # its target, a segment, a customizer. A list view always names one. + UNBOUNDED_PAGE_SIZE = Client::MAX_PER_PAGE + + # And how many such pages are read before the answer is cut short. The + # figure only ever applies to a read with no window of its own. + MAX_COLLECTED_PAGES = 10 + + # What a read that *does* name a window may collect before it is cut + # short. A window is its own bound, so the page count above must not apply + # to it: a window needing eleven pages would come back one page short + # while the warning blamed it for naming no window -- which it did name. + # This is the backstop for a window nothing sane asked for, and it is the + # cursor walker's own record budget, that tier being bounded the same way. + MAX_COLLECTED_RECORDS = Pagination::CursorWalker::MAX_RECORDS + + # How many records of this collection one relation read may resolve. + # Higher than `MAX_ID_READS`, which bounds a single request batch, and + # deliberately above every page size a list view offers: a page naming one + # account per row resolves, and a read naming more -- an export, a + # segment resolved whole -- is refused by name rather than answered with + # a nil where an account exists. There is no bulk read for a company, so + # each one costs a request and the figure is what an operator waits for. + MAX_RELATION_READS = 100 + + # Search is declared only to be refused by name: left to the customizer, + # it would be emulated as equalities on columns no lookup answers. + def initialize(datasource, name) + super + enable_count + enable_search + end + + def ids_per_read = MAX_ID_READS + def max_resolvable_ids = MAX_RELATION_READS + + def list(caller, filter, projection) + warn_ignored_sort(filter&.sort) + + client.with_read_scope do + records = fetch_records(filter) + serialized = records.map { |record| serialize(record) } + rows = serialized.map { |record| project(record, projection) } + + embed_relations(caller, serialized, rows, projection) + rows + end + end + + # Count only, and never a group: `total_count` is exact on every listing, + # while grouping the page in hand would answer a fraction as if it were + # the whole. + def aggregate(_caller, filter, aggregation, _limit = nil) + refuse_unsupported_aggregation!(aggregation) + + [{ 'group' => {}, 'value' => count_records(filter) }] + end + + protected + + # The endpoint that lists the collection by offset, the one that reads a + # record, and the lookups Intercom answers on the listing path. + def list_path = raise(NotImplementedError, "#{self.class} did not implement list_path") + def record_endpoint = raise(NotImplementedError, "#{self.class} did not implement record_endpoint") + def lookup_path = record_endpoint + def lookups = {} + + def serialize(_entity) = raise(NotImplementedError, "#{self.class} did not implement serialize") + + # A column advertises a filter only where Intercom looks the collection up + # by it, so a column cannot offer a filter this tier would then refuse. + # The primary key is the exception and it is not a filter: `id equals X` + # and `id in [...]` are answered by the record endpoint. + # + # Nothing is sortable: the listing takes no order and ordering a page in + # hand would order a fraction of the collection. + def add_column(name, type, is_primary_key: false) + add_field(name, ColumnSchema.new(column_type: type, + filter_operators: column_operators(name, is_primary_key), + is_primary_key: is_primary_key, + is_read_only: true, + is_sortable: false, + is_groupable: false)) + end + + private + + def column_operators(name, is_primary_key) + return [Operators::EQUAL, Operators::IN] if is_primary_key + + lookups.key?(name) ? [Operators::EQUAL] : [] + end + + def fetch_records(filter) + refuse_search! unless blank_search?(filter) + + ids = id_lookup(filter) + return records_by_ids(page_window(ids, filter)) if ids + + lookup = lookup_condition(filter) + return page_window(looked_up_records(lookup), filter) if lookup + + refuse_condition!(filter.condition_tree) unless filter&.condition_tree.nil? + + listed_records(filter) + end + + def count_records(filter) + refuse_search! unless blank_search?(filter) + + ids = id_lookup(filter) + return count_by_ids(ids) if ids + + lookup = lookup_condition(filter) + return looked_up_count(lookup) if lookup + + refuse_condition!(filter.condition_tree) unless filter&.condition_tree.nil? + + exact_count(read_offset_page(page: 1, per_page: 1)) + end + + # The window a list view asked for, read as the page Intercom counts from + # 1. An offset that does not fall on a page boundary is served by reading + # the page it lands in and the ones after it until the window is filled -- + # exactly, rather than by rounding the offset to something the API likes. + def listed_records(filter) + offset, limit = window(filter&.page) + return [] if limit&.zero? + + per_page = Client.bounded_per_page(limit || UNBOUNDED_PAGE_SIZE) + skip = offset % per_page + + collected = collect_pages(first_page: (offset / per_page) + 1, per_page: per_page, + wanted: limit && (skip + limit)) + + limit ? (collected[skip, limit] || []) : collected.drop(skip) + end + + def collect_pages(first_page:, per_page:, wanted:) + records = [] + page = first_page + read = 0 + + loop do + answer = read_offset_page(page: page, per_page: per_page) + records.concat(answer.records) + read += 1 + break if last_page?(answer, page) || (wanted && records.size >= wanted) + break if cap_reached?(read, records.size, wanted) + + page += 1 + end + + # Intercom numbers its pages by `per_page`, so the last one read cannot + # be asked smaller: what it brings past the budget is cut here instead. + wanted ? records.first(MAX_COLLECTED_RECORDS) : records + end + + def read_offset_page(page:, per_page:) + client.offset_page(list_path, page: page, per_page: per_page) + end + + def last_page?(answer, page) + answer.records.empty? || (answer.total_pages && page >= answer.total_pages) + end + + # Which of the two caps applies, and the reason that goes with it: a read + # with no window of its own is stopped by the page count, a read that + # named one by the record budget. Telling a list view it named no window + # is exactly the confusion these two keep apart. + def cap_reached?(read, collected, wanted) + if wanted + return false if collected < MAX_COLLECTED_RECORDS + + warn_capped("#{collected} record(s)", + 'this read named a window larger than one answer may hold') + else + return false if read < MAX_COLLECTED_PAGES + + warn_capped("#{read} page(s) / #{collected} record(s)", + 'this read named no window of its own, and a list view always does') + end + + true + end + + def warn_capped(reached, reason) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] Stopped reading #{name} after #{reached}; the rest is left out: " \ + "#{reason}." + ) + end + + # A filter with no page asks for every record it matched; nil is how the + # window says so. + def window(page) + return [0, nil] if page.nil? + + [page.offset.to_i.clamp(0, nil), page_limit(page)] + end + + def lookup_condition(filter) + tree = filter&.condition_tree + return nil unless tree.is_a?(Leaf) && tree.operator == Operators::EQUAL + + parameter = lookups[tree.field.to_s] + parameter && { parameter => tree.value.to_s } + end + + # Read as a single page; overflow is reported. Intercom answers a lookup matching nothing + # with a 404, read here as an empty page. + def looked_up_records(params) + answer = lookup(params) + return [] unless answer + + warn_truncated_lookup(params) if answer.next_cursor + answer.records + end + + # Intercom's own count when it gave one. A lookup spread over several + # pages without one cannot be counted from the first page alone. + def looked_up_count(params) + answer = lookup(params) + return 0 unless answer + return answer.total_count if answer.total_count + return exact_count(answer) if answer.next_cursor + + answer.records.size + end + + def lookup(params) + client.lookup_page(lookup_path, params: params) + rescue APIError => e + raise unless e.status == 404 + + nil + end + + def refuse_search! + raise UnsupportedOperatorError, + "#{name} cannot answer a free-text search: Intercom exposes no search endpoint for this collection " \ + "and looks a record up by id, #{lookups.keys.join(", ")} alone, one exact value at a time. Filter " \ + 'on one of those instead of searching.' + end + + def refuse_condition!(tree) + leaves = [] + tree.some_leaf do |leaf| + leaves << leaf + false + end + offender = leaves.find { |leaf| !published?(leaf) } + refused = if offender + "filter #{offender.field.inspect}" + else + "combine conditions on #{leaves.map(&:field).uniq.join(", ")}" + end + + raise UnsupportedOperatorError, + "#{name} cannot #{refused}: Intercom exposes no search endpoint for this " \ + "collection and looks a record up by #{lookups.keys.join(", ")} alone -- one exact value at a time, " \ + 'with no combination and no other operator. Filter on one of those, or reach the record from the ' \ + 'collection next door.' + end + + def published?(leaf) + schema = fields[leaf.field] + schema.respond_to?(:filter_operators) && schema.filter_operators.include?(leaf.operator) + end + + # Intercom takes no order on this listing at all -- there is no parameter + # for one -- so an order asked for and not applied is reported here or + # nowhere. The ascending primary-key sort the agent injects when a request + # names none is not one of those. + def warn_ignored_sort(sort) + clauses = Array(sort) + return if clauses.empty? || default_pk_sort?(clauses) + + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked to sort on " \ + "#{clauses.map { |clause| sort_field(clause) }.join(", ")}, and Intercom takes no order " \ + 'on this listing. The rows come back in the order the API imposes.' + ) + end + + def warn_truncated_lookup(params) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} looked up #{params.inspect} and Intercom advertised more " \ + 'records than one page holds; the rest is left out. This lookup is meant for a key that names one record.' + ) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/records_by_id.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/records_by_id.rb new file mode 100644 index 000000000..9cd743ab3 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/records_by_id.rb @@ -0,0 +1,103 @@ +module ForestAdminDatasourceIntercom + module Collections + # `id equals X` / `id in [...]` answered by the record endpoint, plus exact counting -- shared + # by the two paginated tiers, where a count over collected pages would answer a fraction. + module RecordsById + Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + Aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation + + # How many records a bulk read by id may fetch. Intercom exposes no "read + # these records" endpoint, so a set of ids is one request each -- the + # fan-out is bounded rather than turned into a rate limit halfway through + # a page. The collection that reads its ids in bulk raises the figure. + MAX_ID_READS = 25 + + protected + + # The endpoint one record is read from. + def record_endpoint = raise(NotImplementedError, "#{self.class} did not implement record_endpoint") + + # What rides along on that read. `display_as=plaintext` for the tiers + # whose records carry bodies written by end customers; nothing elsewhere. + def record_read_params = {} + + def max_id_reads = MAX_ID_READS + + # A bare primary-key leaf only: answering an `and` carrying a scope, or a search, with the + # ids alone would serve records it excludes. Deduplicated so an id is not read twice. + def id_lookup(filter) + tree = filter&.condition_tree + return nil unless tree.is_a?(Leaf) && tree.field.to_s == primary_key + return nil unless blank_search?(filter) + + case tree.operator + when Operators::EQUAL then [tree.value].compact.map(&:to_s).uniq + when Operators::IN then Array(tree.value).compact.map(&:to_s).uniq + end + end + + # A record the operator can no longer reach -- deleted, or outside the + # token's scope -- reads as "no record" rather than as a failed page. + def records_by_ids(ids) + wanted = ids.first(max_id_reads) + warn_truncated_ids(ids.size) if ids.size > wanted.size + + client.fetch_records(record_endpoint, wanted, params: record_read_params).compact + end + + # How many of a set of ids name a record, which means reading them: + # Intercom has no "how many of these exist" endpoint. Past what a bulk + # read may fetch the count is refused rather than answered with the number + # the truncation left -- both paginated tiers advertise an exact count, + # and "25" where the question named forty records is not one. + def count_by_ids(ids) + refuse_id_count!(ids.size) if ids.size > max_id_reads + + records_by_ids(ids).size + end + + # The count Intercom answered, or nothing at all. Counting the pages a + # read collected would answer a fraction of the collection as if it were + # the whole of it, which is the one thing the paginated tiers do not do. + def exact_count(page) + return page.total_count if page.total_count + + raise UnsupportedOperatorError, + "#{name} cannot be counted: Intercom answered this listing without a total_count, and counting the " \ + 'pages the agent read would answer a fraction of the collection as if it were the whole of it.' + end + + def refuse_unsupported_aggregation!(aggregation) + return if aggregation.is_a?(Aggregation) && aggregation.operation.to_s.casecmp('count').zero? && + Array(aggregation.groups).empty? && aggregation.field.nil? + + raise UnsupportedOperatorError, + "#{name} can only be counted: Intercom exposes no aggregate endpoint, and grouping or summing the " \ + 'pages the agent read would answer a fraction of the collection as if it were the whole of it. ' \ + 'Chart it on a collection read whole, or wait for the bounded group-by of the reporting lot.' + end + + def refuse_id_count!(asked) + raise UnsupportedOperatorError, + "#{name} cannot count #{asked} records by id: Intercom exposes no endpoint that counts a set of ids, " \ + "so counting them means reading them, and this reads #{max_id_reads} at most. Counting the ones it " \ + 'read would answer a number smaller than the question. Narrow the condition, or count the ' \ + 'collection with a filter it answers instead of a list of ids.' + end + + def warn_truncated_ids(asked) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked for #{asked} records by id and read the first " \ + "#{max_id_reads}: Intercom reads them one request each. The result is truncated." + ) + end + + def blank_search?(filter) + search = filter.respond_to?(:search) ? filter.search : nil + + search.nil? || search.to_s.strip.empty? + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb new file mode 100644 index 000000000..fdc65c5a7 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb @@ -0,0 +1,294 @@ +module ForestAdminDatasourceIntercom + module Collections + # Intercom joins nothing, so relation filters must be answered here: the agent publishes a + # many-to-one filterable as soon as any target column is. A null foreign key matches no + # relation filter, negated ones included, like a join. + module Relations # rubocop:disable Metrics/ModuleLength + ManyToOneSchema = ForestAdminDatasourceToolkit::Schema::Relations::ManyToOneSchema + ManyToManySchema = ForestAdminDatasourceToolkit::Schema::Relations::ManyToManySchema + OneToManySchema = ForestAdminDatasourceToolkit::Schema::Relations::OneToManySchema + Filter = ForestAdminDatasourceToolkit::Components::Query::Filter + Projection = ForestAdminDatasourceToolkit::Components::Query::Projection + Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + Branch = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + + # What a relation condition comes to when the target matched no record: no + # row can satisfy it. It is a value rather than an empty condition because + # the two tiers spell "match nothing" differently -- in memory an `in []` + # says it, while Intercom's search DSL has no way to. + NOTHING = :matches_nothing + + # The group a relation condition expands into, told apart from a group the + # operator wrote. Two things hang on the difference: this one may be + # inlined into a parent aggregating the same way -- the level it adds is + # one nobody budgeted for -- and a tier refusing it can name the relation + # rather than a shape the operator never wrote. + class RelationBranch < Branch + attr_reader :relation_field + + def initialize(aggregator, conditions, relation_field) + @relation_field = relation_field + super(aggregator, conditions) + end + end + + protected + + # Every relation of this package points at a collection keyed by `id` and + # is read-only: nothing in this lot writes, and an editable relation would + # offer an association the collection cannot perform. + # + # Published unfilterable where the endpoint filters nothing on the key, so + # the interface does not offer the traversal `check_relation_filterable!` + # would refuse. + def add_many_to_one(name, foreign_collection:, foreign_key:) + add_field(name, ManyToOneSchema.new(foreign_collection: foreign_collection, + foreign_key: foreign_key, + foreign_key_target: 'id', + is_read_only: true, + is_filterable: relation_key_filterable?(foreign_key))) + end + + # Served by the agent as `origin_key equals ` on the target, which must answer it. + def add_one_to_many(name, foreign_collection:, origin_key:) + add_field(name, OneToManySchema.new(foreign_collection: foreign_collection, + origin_key: origin_key, + origin_key_target: 'id', + is_read_only: true)) + end + + def add_many_to_many(name, foreign_collection:, through_collection:, origin_key:, foreign_key:) + add_field(name, ManyToManySchema.new(foreign_collection: foreign_collection, + through_collection: through_collection, + origin_key: origin_key, origin_key_target: 'id', + foreign_key: foreign_key, foreign_key_target: 'id', + is_read_only: true)) + end + + # `records` are the unprojected rows, where the foreign keys live. One read per target + # collection per page, shared by every relation pointing at it. + def embed_relations(caller, records, rows, projection) + asked = many_to_one_asked(projection) + return if asked.empty? + + indexed = indexed_targets(caller, records, asked) + + asked.each do |name, relation, wanted| + rows_of_target = indexed[relation.foreign_collection] + + # Nil rather than absent when an id names no record: a teammate who + # left the workspace reads as no teammate, not as a broken row. Sliced + # back to what this relation asked for, the read having been widened to + # the union of what every relation on that collection did. + records.each_with_index do |record, index| + target_row = rows_of_target[record[relation.foreign_key]] + rows[index][name] = target_row&.slice(*wanted) + end + end + end + + # Trades each `relation:field` leaf for a condition on the foreign key, built by the block + # from the ids the target matched. + def rewrite_relation_conditions(caller, node, &builder) + case node + when Branch then rewrite_branch(caller, node, &builder) + when Leaf then relation_leaf?(node) ? rewrite_relation_leaf(caller, node, &builder) : node + else node + end + end + + private + + def relation_leaf?(node) + node.is_a?(Leaf) && node.field.to_s.include?(':') + end + + def relations_asked(projection) + Projection.new(Array(projection).map(&:to_s)).relations + end + + # The many-to-one relations the projection named, each with the columns it + # asked of its target. The target's own key travels with them whether or + # not it was asked for: it is what the rows are indexed by here, and what + # makes the nested row a link rather than a label in the interface. + def many_to_one_asked(projection) + relations_asked(projection).filter_map do |relation_name, sub_projection| + relation = fields[relation_name] + next unless relation.is_a?(ManyToOneSchema) + + columns = Array(sub_projection).map(&:to_s) + refuse_two_hop_projection!(relation_name, relation, columns) + + [relation_name, relation, columns.union([relation.foreign_key_target])] + end + end + + # `admin_assignee:teams:name` is a projection Forest's own parser builds + # and this cannot answer: it nests one target row under the relation name, + # not a tree of them, so the second hop would be dropped by the target's + # projection and the column would come back missing from a row that looks + # complete. Refused by name, like the filter that reaches that deep -- + # `refuse_two_hops!` is its counterpart. + def refuse_two_hop_projection!(relation_name, relation, columns) + deep = columns.select { |column| column.include?(':') } + return if deep.empty? + + raise UnsupportedOperatorError, + "#{name} cannot read #{deep.map { |column| "#{relation_name}:#{column}".inspect }.join(", ")}: it " \ + 'reaches through two relations, and this datasource resolves one. Read the column from a record ' \ + "page of #{relation.foreign_collection}, one hop away." + end + + def indexed_targets(caller, records, asked) + asked.group_by { |_, relation, _| relation.foreign_collection } + .transform_values { |group| target_rows(caller, records, group) } + end + + # Past what the target resolves at all, refused: a partly resolved page would show nil where + # a record exists. + def target_rows(caller, records, group) + relation = group.first[1] + ids = group.flat_map { |_, rel, _| records.filter_map { |record| record[rel.foreign_key] } }.uniq + return {} if ids.empty? + + collection = foreign_collection(relation) + refuse_relation_fan_out!(group, collection, ids.size) if beyond_reach?(collection, ids.size) + + target = relation.foreign_key_target + wanted = Projection.new(group.flat_map { |_, _, columns| columns }.uniq) + + read_targets(caller, collection, target, ids, wanted).to_h { |row| [row[target], row] } + end + + def beyond_reach?(collection, count) + limit = collection.max_resolvable_ids + + !limit.nil? && count > limit + end + + def read_targets(caller, collection, target, ids, wanted) + chunk = collection.ids_per_read + slices = chunk ? ids.each_slice(chunk).to_a : [ids] + + slices.flat_map do |slice| + collection.list(caller, Filter.new(condition_tree: Leaf.new(target, Operators::IN, slice)), wanted) + end + end + + # Names the relations rather than the key they resolve to: what the + # operator can act on is the column they put in the view or the export, + # and how many rows they asked for at once. + def refuse_relation_fan_out!(group, collection, count) + asked = group.map { |relation_name, _, _| relation_name }.join(', ') + + raise UnsupportedOperatorError, + "#{name} cannot resolve #{asked} over this read: it names #{count} distinct #{collection.name} " \ + 'records and Intercom has no bulk read for that collection, so this resolves at most ' \ + "#{collection.max_resolvable_ids} of them per read -- one request each. Read fewer rows at a time, " \ + 'or leave the relation out of the projection: a relation resolved for part of a page would show no ' \ + 'record where one exists.' + end + + def rewrite_branch(caller, branch, &builder) + rewritten = Array(branch.conditions).map { |node| rewrite_relation_conditions(caller, node, &builder) } + + if branch.aggregator.to_s.casecmp('or').zero? + kept = rewritten.reject { |node| node == NOTHING } + kept.empty? ? NOTHING : Branch.new(branch.aggregator, absorb_groups(branch.aggregator, kept)) + elsif rewritten.include?(NOTHING) + NOTHING + else + Branch.new(branch.aggregator, absorb_groups(branch.aggregator, rewritten)) + end + end + + # `or(x, or(a, b))` becomes `or(x, a, b)`, saving the nesting level a relation expansion + # adds. Only relation groups are inlined, never groups the operator wrote. + def absorb_groups(aggregator, nodes) + absorbed = nodes.flat_map do |node| + inlinable_group?(node, aggregator) ? Array(node.conditions) : [node] + end + + absorb_relation_group?(absorbed.size) ? absorbed : nodes + end + + def inlinable_group?(node, aggregator) + node.is_a?(RelationBranch) && node.aggregator.to_s.casecmp(aggregator.to_s).zero? + end + + # Hook for a tier that bounds how many conditions a group may hold: past + # that, the nested form is the one that fits, and the level it costs is + # spent rather than the width. + def absorb_relation_group?(_size) = true + + def rewrite_relation_leaf(caller, leaf) + name, path = leaf.field.to_s.split(':', 2) + relation = fields[name] + refuse_unknown_relation!(leaf, name) unless relation.is_a?(ManyToOneSchema) + refuse_two_hops!(leaf) if path.include?(':') + + check_relation_filterable!(leaf, relation) + + ids = matching_ids(caller, relation, Leaf.new(path, leaf.operator, leaf.value)) + return NOTHING if ids.empty? + + yield(relation.foreign_key, ids, leaf) + end + + # Hooks for a tier that cannot filter on every foreign key it declares a + # relation on. Answered before the target is read: a refusal that spends a + # request first costs exactly what it refuses to do. + def relation_key_filterable?(_key) = true + + def check_relation_filterable!(_leaf, _relation); end + + # Which records of the target the condition names, asked of the target + # itself: it owns what its columns can be filtered with, and it answers + # over every record Intercom holds rather than over a page of them. + def matching_ids(caller, relation, leaf) + target = relation.foreign_key_target + filter = Filter.new(condition_tree: leaf, page: match_page) + + foreign_collection(relation) + .list(caller, filter, Projection.new([target])) + .filter_map { |row| row[target] } + .uniq + end + + # How much of the target a relation condition may read; unbounded for in-memory targets. + def match_page = nil + + # The target as the datasource holds it, undecorated -- so a permission + # scope or a segment defined on the target does not narrow what a relation + # resolves. That is how a native datasource behaves too: it joins the table + # without applying the scopes of the collection mapped to it. + def foreign_collection(relation) + datasource.get_collection(relation.foreign_collection) + end + + # Either a name this collection carries no relation under -- a condition + # from a scope or a segment written against another schema -- or a + # many-to-many, which is published unfilterable: resolving one would mean + # reading the collection it travels through once per value. + def refuse_unknown_relation!(leaf, relation_name) + filterable = fields.select { |_, field| field.is_a?(ManyToOneSchema) }.keys + + raise UnsupportedOperatorError, + "#{name} cannot filter #{leaf.field.inspect}: #{relation_name.inspect} is not a relation it filters " \ + "through. #{filterable.empty? ? "It has none." : "Those it does: #{filterable.join(", ")}."} Filter " \ + 'on a column of the collection next door instead.' + end + + # Two hops would mean resolving a relation of a relation, one read per + # level, and nothing in this datasource publishes a filter that deep -- a + # many-to-many is published unfilterable. It is refused by name rather than + # half-answered. + def refuse_two_hops!(leaf) + raise UnsupportedOperatorError, + "#{name} cannot filter #{leaf.field.inspect}: it reaches through two relations, and this datasource " \ + 'resolves one. Filter on a column of the collection next door instead.' + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team.rb new file mode 100644 index 000000000..a0e111bcf --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team.rb @@ -0,0 +1,68 @@ +module ForestAdminDatasourceIntercom + module Collections + # The inbox teams a conversation can be assigned to, rather than a single + # teammate. + class Team < FetchAllCollection + def initialize(datasource) + super(datasource, 'IntercomTeam') + end + + protected + + # Like `/admins`, `/teams` uses its own key instead of the `data` envelope. + def fetch_all + client.fetch_all('teams', list_key: 'teams') + end + + def serialize(team) + attrs = team.is_a?(Hash) ? team : {} + + { 'id' => stringify_id(attrs['id']), + 'name' => attrs['name'], + # Not a column: it is what the names below are read from, and what the + # membership collection turns into the relation. Intercom types these + # as numbers here and as strings on the admin itself, so they are + # stringified for both sides to carry the same id. + 'admin_ids' => Array(attrs['admin_ids']).map { |id| stringify_id(id) }, + 'admin_names' => nil } + end + + # The teammates of the team, by name, and only when a projection asked for + # them: one read of `/admins` for the whole page, never one per team. A + # token that cannot read the teammates costs the column and nothing else. + def enrich(records, rows, projection) + return unless column_asked?(projection, 'admin_names') + + names = admin_names + records.each_with_index do |record, index| + rows[index]['admin_names'] = Array(record['admin_ids']).filter_map { |id| names[id] } + end + end + + private + + def admin_names + client.fetch_all('admins', list_key: 'admins') + .to_h { |admin| [stringify_id(admin['id']), admin['name']] } + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} could not read the teammates of the workspace (HTTP " \ + "#{e.status || "-"}); the names are left empty. The relation to IntercomAdmin is unaffected." + ) + {} + end + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('name', 'String') + # A list, so neither filterable nor sortable -- as the array of ids it + # replaces was. It reads the team without a join; the relation below is + # what navigates it, and filtering happens on the Admins collection. + add_column('admin_names', 'Json') + add_many_to_many('admins', foreign_collection: 'IntercomAdmin', + through_collection: 'IntercomTeamMembership', + origin_key: 'team_id', foreign_key: 'admin_id') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team_membership.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team_membership.rb new file mode 100644 index 000000000..02b4c9c64 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team_membership.rb @@ -0,0 +1,51 @@ +module ForestAdminDatasourceIntercom + module Collections + # Synthetic through-collection for the Team <-> Admin many-to-many, which Intercom has no + # resource for. Pairs whose teammate `/admins` no longer answers are dropped: `admin_ids` + # keeps naming a teammate who left until Intercom drops the pair. + class TeamMembership < FetchAllCollection + def initialize(datasource) + super(datasource, 'IntercomTeamMembership') + end + + protected + + def fetch_all + admin_ids = client.fetch_all('admins', list_key: 'admins') + .filter_map { |admin| stringify_id(admin['id']) if admin.is_a?(Hash) }.to_set + client.fetch_all('teams', list_key: 'teams').flat_map { |team| pairs_of(team, admin_ids) } + end + + # Keyed by both ids rather than by a counter: a related list is read over + # two requests, and a key that changed between them would move the rows + # under the operator. + def serialize(pair) + { 'id' => "#{pair["team_id"]}:#{pair["admin_id"]}", + 'team_id' => pair['team_id'], + 'admin_id' => pair['admin_id'] } + end + + private + + def pairs_of(team, admin_ids) + return [] unless team.is_a?(Hash) + + team_id = stringify_id(team['id']) + return [] if team_id.nil? + + Array(team['admin_ids']).filter_map do |admin_id| + id = stringify_id(admin_id) + { 'team_id' => team_id, 'admin_id' => id } if admin_ids.include?(id) + end + end + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('team_id', 'String') + add_column('admin_id', 'String') + add_many_to_one('team', foreign_collection: 'IntercomTeam', foreign_key: 'team_id') + add_many_to_one('admin', foreign_collection: 'IntercomAdmin', foreign_key: 'admin_id') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb new file mode 100644 index 000000000..6cbbd7fb7 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb @@ -0,0 +1,142 @@ +module ForestAdminDatasourceIntercom + module Collections + # Read through POST /tickets/search: Intercom has no GET /tickets, and records come under `tickets`. + # Every response carries each ticket's whole timeline with no field selection (measured: 155 parts on one + # ticket), hence the small page size and the free part-derived columns. + class Ticket < CursorCollection + include ContactIdentity + include Ticket::Serializer + include Ticket::DerivedColumns + include CustomAttributes + # The same thread a conversation publishes, and free here: the parts are + # in the response whether or not anything asks for them. + include Timeline + + # Intercom accepts 150. This is not that: it is what keeps one page of + # tickets, timelines included, a response an agent can hold and an operator + # can wait for. Provisional until measured against real response sizes on + # the customer's workspace. + MAX_TICKETS_PER_PAGE = 25 + + # `/tickets/search` demands a query, so a list view sends the least noisy + # predicate that matches everything. Every ticket has a creation date, and + # a bound at the epoch keeps whatever the day-granular truncation does to + # it harmless. + MATCH_EVERY_TICKET = { 'field' => 'created_at', 'operator' => '>', 'value' => '0' }.freeze + + def initialize(datasource, attributes: []) + @attributes = attributes + super(datasource, 'IntercomTicket') + # Declared so the search reaches `refuse_search!` instead of being + # emulated by the customizer as an equality on every String column, + # which `/tickets/search` answers with an empty page. + enable_search + end + + protected + + def record_endpoint = 'tickets' + def list_key = 'tickets' + def searchable = 'tickets' + def max_page_size = MAX_TICKETS_PER_PAGE + + # The part bodies are HTML written by end customers, and rendering + # third-party HTML inside Forest is neither safe nor useful (R10). Sent on + # the search, where Intercom does not document it: a parameter it ignores + # costs a query string, while the one it honours saves every row of the + # thread from coming back as markup. + def read_params = { 'display_as' => 'plaintext' } + + # Intercom exposes no `GET /tickets`, so a list view searches too: with the + # filter it was given, or with the predicate that matches everything when + # it was given none. + def searchable_only? = true + def match_all_query = MATCH_EVERY_TICKET + + def enrich(records, rows, projection) + embed_contact_identity(records, rows, projection) + embed_derived_columns(records, rows, projection) + embed_timeline(records, rows, projection) + end + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + # The number the support team says out loud, next to the id the API + # answers by. + add_column('ticket_id', 'String') + # `request` / `task` / `tracker` on the wire, never the labels the + # Intercom interface shows -- the same mismatch a filter on it will have + # to respect. + add_column('category', 'String') + add_column('open', 'Boolean') + add_column('is_shared', 'Boolean') + add_column('created_at', 'Date') + add_column('updated_at', 'Date') + add_column('admin_assignee_id', 'String') + add_column('team_assignee_id', 'String') + # The ticket carries its company as an id alone, unlike a conversation + # which carries the whole object: the account name would cost a request + # per row, so it is not offered here. Measured: the id is Intercom's own, + # not the customer's external one, which is what a relation will have to + # target in lot 4. + add_column('company_id', 'String') + define_state_columns + define_type_columns + define_contact_columns + define_derived_columns + add_column('part_count', 'Number') + add_column('timeline', 'Json') + # Before the attribute columns rather than after: a workspace attribute + # whose name lands on a relation is then skipped with a warning, the way + # one landing on a column already is. Declared after, it would collide + # and take the boot with it. + define_relations + register_attribute_columns + end + + # The state arrives embedded as a whole object, so its label costs nothing: + # a queue reads without a join. One label and not three -- the category and + # the customer-facing label are read through the `state` relation, which is + # where every field of a state lives. Neither of the two was ever + # filterable, so nothing that could be saved in a segment or a scope + # depended on them. + def define_state_columns + add_column('state_id', 'String') + add_column('state_label', 'String') + add_column('previous_state_id', 'String') + end + + def define_type_columns + add_column('ticket_type_id', 'String') + add_column('ticket_type_name', 'String') + end + + # /tickets/search filters on admin_assignee_id, team_assignee_id and ticket_type_id, on no state id, and + # `contact_ids` is unconfirmed; a traversal the endpoint cannot filter is refused by name, not dropped. + def define_relations + add_many_to_one('admin_assignee', foreign_collection: 'IntercomAdmin', foreign_key: 'admin_assignee_id') + add_many_to_one('team_assignee', foreign_collection: 'IntercomTeam', foreign_key: 'team_assignee_id') + add_many_to_one('state', foreign_collection: 'IntercomTicketState', foreign_key: 'state_id') + add_many_to_one('previous_state', foreign_collection: 'IntercomTicketState', + foreign_key: 'previous_state_id') + add_many_to_one('ticket_type', foreign_collection: 'IntercomTicketType', foreign_key: 'ticket_type_id') + add_many_to_one('contact', foreign_collection: 'IntercomContact', foreign_key: 'contact_id') + end + + # Free: the parts come with the search response regardless. An empty list is an empty thread, unlike a + # listed conversation, whose timeline stays nil (unknown). + def embed_timeline(records, rows, projection) + return unless column_asked?(projection, 'timeline') + + records.each_with_index { |record, index| rows[index]['timeline'] = build_timeline(record) } + end + + # The attribute columns of every ticket type, in union. Read at boot by + # `TicketAttributesIntrospector`, which is also where a workspace's own + # name is turned into one a Forest query string can carry. + def attribute_kind = 'ticket' + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/derived_columns.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/derived_columns.rb new file mode 100644 index 000000000..6f7799e4c --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/derived_columns.rb @@ -0,0 +1,119 @@ +module ForestAdminDatasourceIntercom + module Collections + class Ticket < CursorCollection + # Derived from the parts: a ticket has no `statistics` block (measured on 81 142 tickets). Display only + # for good: /tickets/search filters on neither and silently ignores a sort on them. + module DerivedColumns + # A ticket is not "closed" on Intercom, it enters a state whose category + # is resolved. + RESOLVED = 'resolved'.freeze + + # Matched on the prefix, never on the full `ticket_state_updated_by_admin` + # the sample showed: a workspace running workflows closes tickets through + # other variants of the same event, and a closure nobody can see is worse + # than a column nobody offers. + STATE_CHANGE_PREFIX = 'ticket_state_updated'.freeze + + # A reply to the customer. A `note` is an internal touch, not an answer: + # counting it would name as "last responder" someone who never wrote to + # the person waiting. + REPLY_PART = 'comment'.freeze + + # The columns a truncated timeline leaves unknown rather than empty, and + # therefore the ones whose presence in a projection is worth a warning. + CLOSURE_COLUMNS = %w[closed_at closed_by_name].freeze + + private + + def define_derived_columns + add_column('closed_at', 'Date') + add_column('closed_by_name', 'String') + add_column('last_reply_at', 'Date') + add_column('last_responder_name', 'String') + # `admin` or `contact`: whether the last word came from the team or + # from the customer is what tells a queue who owes the next one. + add_column('last_responder_type', 'String') + end + + def derived_columns_for(attrs) + parts = parts_of(attrs) + closure = last_closure(parts) + reply = last_reply(parts) + + { 'closed_at' => stamp(closure&.dig('created_at')), + 'closed_by_name' => author_of(closure)['name'], + 'last_reply_at' => stamp(reply&.dig('created_at')), + 'last_responder_name' => author_of(reply)['name'], + 'last_responder_type' => author_of(reply)['type'] } + end + + # The hook of the base's `enrich`: nothing to read here, since + # `serialize` already derived everything. What is left is telling the + # operator when a value is missing because the timeline was truncated + # rather than because the event never happened. + def embed_derived_columns(records, _rows, projection) + return unless any_column_asked?(projection, CLOSURE_COLUMNS) + + unknown = records.count { |record| closure_unknown?(record) } + warn_unknown_closures(unknown) if unknown.positive? + end + + # A resolved ticket with no closure in hand, on a timeline Intercom + # truncated: the date is *unknown*, not absent. A Date column cannot say + # that, so the log does. + def closure_unknown?(record) + state = record['ticket_state'].is_a?(Hash) ? record['ticket_state'] : {} + return false unless state['category'] == RESOLVED + + last_closure(parts_of(record)).nil? && truncated?(record) + end + + # Intercom keeps the 500 most recent parts of a ticket. Past that, the + # transition that closed it may have fallen out of the window. + def truncated?(record) + total = parts_total(record) + total ? total > parts_of(record).size : false + end + + def last_closure(parts) + closures = parts.select { |part| state_change?(part) && part['ticket_state'] == RESOLVED } + + closures.max_by { |part| part['created_at'].to_i } + end + + # A part can record a transition to the state the ticket was already in + # -- measured -- and that is not an event. + def state_change?(part) + part['part_type'].to_s.start_with?(STATE_CHANGE_PREFIX) && + part['ticket_state'] != part['previous_ticket_state'] + end + + def last_reply(parts) + parts.select { |part| part['part_type'] == REPLY_PART }.max_by { |part| part['created_at'].to_i } + end + + def author_of(part) + author = (part || {})['author'] + author.is_a?(Hash) ? author : {} + end + + def parts_of(record) + nested_list((record || {})['ticket_parts'], 'ticket_parts') + end + + def parts_total(record) + container = (record || {})['ticket_parts'] + container.is_a?(Hash) ? container['total_count'] : nil + end + + def warn_unknown_closures(unknown) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name}: #{unknown} resolved ticket(s) of this page show no " \ + 'closure date because Intercom truncated their timeline at 500 parts, not because they were never ' \ + 'closed. The column is unknown for those rows.' + ) + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb new file mode 100644 index 000000000..3325853aa --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb @@ -0,0 +1,70 @@ +module ForestAdminDatasourceIntercom + module Collections + class Ticket < CursorCollection + # One Intercom ticket flattened into the row the schema declares. Nothing + # here reads a sub-resource: the state, the type and the attributes all + # travel with the ticket. + module Serializer + protected + + # Intercom keys the attribute values by **name**, which is what lets a + # single collection display the union of every ticket type's -- and what + # stops it from filtering on them, the filter being written by an id + # that differs from one type to the next. A ticket of another type + # simply does not carry the key, and the column reads as empty. + def serialize(ticket) + attrs = ticket.is_a?(Hash) ? ticket : {} + + native(attrs) + .merge(state_of(attrs)) + .merge(type_of(attrs['ticket_type'])) + .merge(contact_columns_for(attrs)) + .merge(ticket_attribute_values(attrs)) + .merge(derived_columns_for(attrs)) + end + + private + + def native(attrs) + { 'id' => stringify_id(attrs['id']), + 'ticket_id' => stringify_id(attrs['ticket_id']), + 'category' => attrs['category'], + 'open' => attrs['open'], + 'is_shared' => attrs['is_shared'], + 'created_at' => stamp(attrs['created_at']), + 'updated_at' => stamp(attrs['updated_at']), + 'admin_assignee_id' => stringify_id(attrs['admin_assignee_id']), + 'team_assignee_id' => stringify_id(attrs['team_assignee_id']), + 'company_id' => stringify_id(attrs['company_id']), + 'part_count' => parts_total(attrs) } + end + + def state_of(attrs) + state = attrs['ticket_state'].is_a?(Hash) ? attrs['ticket_state'] : {} + + # `internal_label` is what the support team reads. The category and the + # customer-facing label are a hop away, on the `state` relation. + { 'state_id' => stringify_id(state['id']), + 'state_label' => state['internal_label'], + 'previous_state_id' => stringify_id(attrs['previous_ticket_state_id']) } + end + + def ticket_attribute_values(attrs) + values = attribute_values(attrs['ticket_attributes']) + type_id = stringify_id(attrs.dig('ticket_type', 'id')) if attrs['ticket_type'].is_a?(Hash) + + attribute_columns.each do |attribute| + values[attribute.column_name] = nil if Array(attribute.conflicting_ticket_types).include?(type_id) + end + values + end + + def type_of(ticket_type) + attrs = ticket_type.is_a?(Hash) ? ticket_type : {} + + { 'ticket_type_id' => stringify_id(attrs['id']), 'ticket_type_name' => attrs['name'] } + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_state.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_state.rb new file mode 100644 index 000000000..f9845d7e7 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_state.rb @@ -0,0 +1,40 @@ +module ForestAdminDatasourceIntercom + module Collections + # The ticket states of the workspace. A ticket carries its state as an id, so + # without this collection a support queue reads as a column of numbers. + class TicketState < FetchAllCollection + def initialize(datasource) + super(datasource, 'IntercomTicketState') + end + + protected + + def fetch_all + client.fetch_all('ticket_states') + end + + # Two labels rather than one: `internal_label` is what the support team + # sees, `external_label` what the customer is shown. An operator reading a + # queue needs the first, and needs to know what the second says. + def serialize(ticket_state) + attrs = ticket_state.is_a?(Hash) ? ticket_state : {} + + { 'id' => stringify_id(attrs['id']), + 'category' => attrs['category'], + 'internal_label' => attrs['internal_label'], + 'external_label' => attrs['external_label'], + 'archived' => attrs['archived'] } + end + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('category', 'String') + add_column('internal_label', 'String') + add_column('external_label', 'String') + add_column('archived', 'Boolean') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_type.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_type.rb new file mode 100644 index 000000000..7b6367198 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_type.rb @@ -0,0 +1,47 @@ +module ForestAdminDatasourceIntercom + module Collections + # The ticket types the workspace defines. They are what makes a ticket's type + # readable, and they are also where the ticket attributes are declared -- + # which is what the ticket collection reads them for. + class TicketType < FetchAllCollection + def initialize(datasource) + super(datasource, 'IntercomTicketType') + end + + protected + + def fetch_all + client.fetch_all('ticket_types') + end + + # `ticket_type_attributes` is deliberately left out: it is a nested list of + # attribute definitions, useful to the ticket collection and meaningless as + # a column. An attribute of the same name carries a different id from one + # type to the next (measured), which is exactly why the ticket collection + # has to read the definitions rather than assume them. + def serialize(ticket_type) + attrs = ticket_type.is_a?(Hash) ? ticket_type : {} + + { 'id' => stringify_id(attrs['id']), + 'name' => attrs['name'], + 'description' => attrs['description'], + 'category' => attrs['category'], + 'icon' => attrs['icon'], + 'archived' => attrs['archived'] } + end + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('name', 'String') + add_column('description', 'String') + # `request` / `task` / `tracker` on the wire, not the labels the Intercom + # interface shows -- the same mismatch the ticket filter has to respect. + add_column('category', 'String') + add_column('icon', 'String') + add_column('archived', 'Boolean') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/timeline.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/timeline.rb new file mode 100644 index 000000000..00ee623d9 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/timeline.rb @@ -0,0 +1,40 @@ +module ForestAdminDatasourceIntercom + module Collections + # The thread of a conversation or ticket. `part_type` is kept, and internal notes are included next to + # customer replies -- that is what a thread is on Intercom. Intercom keeps only the 500 most recent parts. + module Timeline + private + + def build_timeline(entity) + attrs = entity.is_a?(Hash) ? entity : {} + + [opening_entry(attrs)].compact + (parts_of(attrs) || []).map { |part| part_entry(part) } + end + + # What comes before the parts. A conversation opens on its `source`, which + # is not a part at all; a ticket opens on its first part like any other + # event, so there is nothing to prepend. + def opening_entry(_attrs) = nil + + def part_entry(part) + attrs = part.is_a?(Hash) ? part : {} + + entry(part_type: attrs['part_type'], created_at: attrs['created_at'], author: attrs['author'], + body: attrs['body'], attachments: attrs['attachments']) + .merge('id' => stringify_id(attrs['id']), 'redacted' => attrs['redacted']) + end + + def entry(part_type:, created_at:, author:, body:, attachments:) + writer = author.is_a?(Hash) ? author : {} + + { 'part_type' => part_type, + 'created_at' => stamp(created_at), + 'author_type' => writer['type'], + 'author_name' => writer['name'], + 'author_email' => writer['email'], + 'body' => body, + 'attachment_count' => Array(attachments).size } + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/configuration.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/configuration.rb new file mode 100644 index 000000000..2ceeea4d2 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/configuration.rb @@ -0,0 +1,226 @@ +module ForestAdminDatasourceIntercom + # Long by line count only: every option of the datasource is declared here + # with the reason it has the default it has, and next to the check that + # refuses the shapes it cannot mean. Splitting the checks off would put an + # option and its meaning in two files. + class Configuration # rubocop:disable Metrics/ClassLength + # A workspace is hosted in one region and answers in that region only. The + # host is therefore a configuration parameter rather than a constant: + # `api.intercom.io` does route to the right region, but a customer under + # GDPR wants its requests to reach the European host and nothing else. + REGION_HOSTS = { + us: 'https://api.intercom.io', + eu: 'https://api.eu.intercom.io', + au: 'https://api.au.intercom.io' + }.freeze + + DEFAULT_REGION = :us + + # Where a cleartext base_url reaches nobody else's network, so the bearer + # header crossing it in clear is not worth a word. + LOOPBACK_HOSTS = %w[localhost 127.0.0.1 ::1 [::1] 0.0.0.0].freeze + + # Without an explicit version a request follows the workspace's own default, + # which an operator can change on Intercom's side -- and the payloads change + # shape under us. Pinned to what the spike ran against; 2.14 and 2.16 both + # answered, and the response echoes the version back, so `Client#me` + # verifies at boot that the pin was honoured. + DEFAULT_API_VERSION = '2.16'.freeze + + # Reference lists are re-read on every page otherwise (four sequential round trips per ticket list); + # a minute bounds staleness. `0` disables the store. Records are never cached -- see `Cache`. + DEFAULT_REFERENCE_CACHE_TTL = 60 + + # A ticket list is ~7 requests to one host, so keep-alive saves a TLS handshake each. The pool is sized + # against the server's thread count (Puma default 5): a thread waiting >0.5s for a connection gets a + # non-retried timeout blamed on Intercom. `adapter: :net_http` opts out; an unloadable adapter is refused. + DEFAULT_ADAPTER = [:net_http_persistent, { pool_size: 25 }].freeze + + # How many records a read by id fetches at once -- a related list of + # companies, a page of timelines -- Intercom answering those one request + # each. One by default: every extra read in flight holds a connection of + # the pool above, so widening this is the integrator's call, made against + # `pool_size` and the server's thread count together. + DEFAULT_CONCURRENT_READS = 1 + + attr_reader :access_token, :region, :base_url, :api_version, :open_timeout, :timeout, + :retry_policy, :rate_limiter, :boot_open_timeout, :boot_timeout, :boot_retry_policy, + :reference_cache_ttl, :adapter, :reference_cache, :concurrent_reads + + # `rate_limiter: nil` leaves the 429 retry as the only rate-limit handling. The `boot_` trio governs + # reads made while the datasource is constructed (attribute introspection), during the Rails boot. + def initialize(access_token:, region: nil, base_url: nil, api_version: DEFAULT_API_VERSION, + open_timeout: 5, timeout: 30, retry_policy: RetryPolicy.new, + rate_limiter: RateLimiter.new, boot_open_timeout: 3, boot_timeout: 10, + boot_retry_policy: RetryPolicy.boot, + reference_cache_ttl: DEFAULT_REFERENCE_CACHE_TTL, adapter: nil, + concurrent_reads: DEFAULT_CONCURRENT_READS) + @access_token = access_token + @region = (region || DEFAULT_REGION).to_s.downcase.to_sym + @base_url = base_url + @api_version = api_version.to_s + @open_timeout = open_timeout + @timeout = timeout + @retry_policy = retry_policy + @rate_limiter = rate_limiter + @boot_open_timeout = boot_open_timeout + @boot_timeout = boot_timeout + @boot_retry_policy = boot_retry_policy + @reference_cache_ttl = reference_cache_ttl + @adapter = normalize_adapter(adapter) + @concurrent_reads = concurrent_reads + validate! + # One store per Configuration, hence per token, like the rate limiter: + # what it holds is a workspace's own lists, and two workspaces do not + # share them. Built here rather than lazily -- a memoized reader would + # hand two stores to the two threads that first raced for it, and one of + # them would then write into a store nothing else reads. + @reference_cache = Cache.new(ttl: @reference_cache_ttl) + end + + # An explicit `base_url` wins over the region: it is what points the client + # at a mock server or an egress proxy, neither of which is a region. + def url + @url ||= (@base_url || REGION_HOSTS.fetch(@region)).chomp('/') + end + + # The url with any credentials in it masked. `URI` accepts them -- + # `https://user:pass@proxy/...` -- and the refusal that names such a url + # would otherwise print a second secret exactly where this class keeps the + # first one from going. A query string is masked whole for the same + # reason: it is where a gateway takes its key. Read by `Client#inspect` + # too, which prints the same url. + def redacted_url + @redacted_url ||= url.sub(%r{\A([a-zA-Z][\w+.-]*://)[^/@]*@}, '\1[FILTERED]@') + .sub(/\?.*\z/m, '?[FILTERED]') + end + + # `access_token` is a bearer credential, and nothing prints a Configuration + # on purpose: what reaches an `inspect` is a Rails error page, or a + # `logger.debug` of something holding one. The default would put the token + # in clear there. `Client` and `Datasource` mask their own for the same + # reason -- together they cut every path from an object this package hands + # out to the credential. + def inspect + "#<#{self.class.name} url=#{redacted_url.inspect} api_version=#{@api_version.inspect} " \ + 'access_token=[FILTERED]>' + end + + private + + def validate! + raise ConfigurationError, 'ForestAdminDatasourceIntercom missing required config: access_token' if + blank?(@access_token) + + validate_region! + validate_base_url! + raise ConfigurationError, 'ForestAdminDatasourceIntercom api_version cannot be empty' if blank?(@api_version) + + validate_reference_cache_ttl! + validate_concurrent_reads! + end + + def validate_concurrent_reads! + return if @concurrent_reads.is_a?(Integer) && @concurrent_reads.positive? + + raise ConfigurationError, + 'ForestAdminDatasourceIntercom concurrent_reads must be a whole number of reads, one or more, got ' \ + "#{@concurrent_reads.inspect}" + end + + # Refuses nil, non-numeric strings and Infinity rather than coercing them into a silent 0 (cache off) + # or a never-expiring cache. + def validate_reference_cache_ttl! + return if @reference_cache_ttl.is_a?(Numeric) && @reference_cache_ttl.finite? && + !@reference_cache_ttl.negative? + + raise ConfigurationError, + 'ForestAdminDatasourceIntercom reference_cache_ttl must be a finite number of seconds, zero or ' \ + "more, got #{@reference_cache_ttl.inspect}" + end + + # Faraday's adapter registry is keyed by symbol: a string name (as YAML hands over) silently lost + # keep-alive. Whether the name resolves is left to the client, which falls back. + def normalize_adapter(adapter) + return nil if adapter.nil? + + name, options = adapter.is_a?(Array) ? adapter : [adapter, nil] + refuse_adapter!(adapter) unless name.respond_to?(:to_sym) + refuse_adapter!(adapter) unless options.nil? || namable_hash?(options) + + options.nil? ? name.to_sym : [name.to_sym, options.to_h { |key, value| [key.to_sym, value] }] + end + + def namable_hash?(options) + options.is_a?(Hash) && options.keys.all? { |key| key.respond_to?(:to_sym) } + end + + def refuse_adapter!(adapter) + raise ConfigurationError, + 'ForestAdminDatasourceIntercom adapter must be an adapter name or a [name, options] pair, got ' \ + "#{adapter.inspect}" + end + + def validate_region! + return if @base_url || REGION_HOSTS.key?(@region) + + raise ConfigurationError, + "ForestAdminDatasourceIntercom unknown region #{@region.inspect}: " \ + "expected one of #{REGION_HOSTS.keys.map(&:inspect).join(", ")}, or an explicit base_url." + end + + # A base url that is not absolute makes Faraday resolve every path against + # the process's working directory instead of Intercom, which surfaces much + # later as a connection failure naming nothing. + def validate_base_url! + return if @base_url.nil? + + uri = URI.parse(@base_url) + refuse_credentials! if uri.userinfo + return warn_cleartext!(uri) if uri.is_a?(URI::HTTP) && !blank?(uri.host) + + raise ConfigurationError, + "ForestAdminDatasourceIntercom base_url must be an absolute http(s) url, got #{redacted_url.inspect}" + rescue URI::InvalidURIError + # Without the parser's error, whose message quotes the url unredacted. + raise ConfigurationError, + "ForestAdminDatasourceIntercom base_url is not a valid url: #{redacted_url.inspect}", cause: nil + end + + # Faraday turns credentials in the url into a Basic `Authorization` + # header, which the Intercom bearer token then replaces: they would be + # dropped on every request without a word. An egress proxy takes its + # credentials from `HTTPS_PROXY`, which Faraday sends as + # `Proxy-Authorization` instead. + def refuse_credentials! + raise ConfigurationError, + "ForestAdminDatasourceIntercom base_url #{redacted_url.inspect} carries credentials, which cannot " \ + 'travel: the Authorization header carries the Intercom access token. For a credentialed egress ' \ + 'proxy, set HTTPS_PROXY=https://user:password@proxy:port instead.' + end + + # `URI::HTTPS` is a `URI::HTTP`, so plain http passes above -- and it is + # worth passing: a mock server in a test suite is one, and that is half of + # what `base_url` is for. What it costs is `Authorization: Bearer ` + # travelling in clear to whatever sits at the other end, and that token + # reads the whole workspace. Named rather than allowed in silence; a + # loopback host is nobody else's network, so it says nothing there. + def warn_cleartext!(uri) + return if uri.scheme == 'https' || loopback?(uri.host) + + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] base_url #{redacted_url.inspect} is not https, and every request " \ + 'carries the Intercom access token as a bearer header. Anything on the path can read it and use it ' \ + 'against the workspace. Use https, or terminate TLS before the network this crosses.' + ) + end + + def loopback?(host) + LOOPBACK_HOSTS.include?(host.to_s.downcase) || host.to_s.downcase.end_with?('.localhost') + end + + def blank?(value) + value.nil? || value.to_s.strip.empty? + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb new file mode 100644 index 000000000..79bafc6a4 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb @@ -0,0 +1,78 @@ +module ForestAdminDatasourceIntercom + class Datasource < ForestAdminDatasourceToolkit::Datasource + attr_reader :client, :configuration + + def initialize(access_token:, **options) + super() + @configuration = Configuration.new(access_token: access_token, **options) + @client = Client.new(@configuration) + + verify_api_version + register_collections + end + + # The datasource is what a Rails error page or a `logger.debug` is likeliest + # to print, and it holds the client whose connections carry the access token. + # Every collection will reach that token the same way, through the + # `@datasource` the toolkit's Collection keeps, so cutting the chain here + # covers them too -- and spares the recursive dump the default `inspect` + # walks into, a datasource and its collections pointing at each other. + def inspect + "#<#{self.class.name} collections=#{collections.keys.inspect}>" + end + + private + + # The reference collections first: they are what turns an assignee id into a + # teammate and a state id into a label. No request is made here -- each + # collection reads its endpoint when it is listed, so a datasource boots + # whatever Intercom is doing, and a workspace the token cannot read costs + # rows rather than the agent. + def register_collections + add_collection(Collections::Admin.new(self)) + add_collection(Collections::Team.new(self)) + # The join Intercom does not expose: without it the membership of a team is + # an array of ids on either side, since a many-to-many needs a collection + # to travel through. + add_collection(Collections::TeamMembership.new(self)) + add_collection(Collections::TicketType.new(self)) + add_collection(Collections::TicketState.new(self)) + # Contacts and Companies before the collections that point at them, so a + # relation is declared next to a target the datasource already holds. + # Each carries the custom attributes its workspace defines, read at boot: + # they cannot be discovered from a payload, a contact carrying the values + # of the attributes it happens to have been given. + add_collection(Collections::Contact.new(self, attributes: model_attributes('contact'))) + add_collection(Collections::Company.new(self, attributes: model_attributes('company'))) + add_collection(Collections::Conversation.new(self)) + # The attributes a workspace defines on its ticket types, which are + # columns of the Tickets collection and cannot be discovered from a ticket + # payload either -- a ticket carries the values of its own type only. + add_collection(Collections::Ticket.new(self, attributes: ticket_attributes)) + end + + # Intercom silently serves the workspace's default API version when the pin is not honoured; + # `Client#me` checks the echoed version. A token that cannot reach `/me` costs the check, not the boot. + def verify_api_version + @client.me(boot: true) + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] could not read /me at boot (HTTP #{e.status || "-"}); the API " \ + 'version Intercom serves was not checked against the pinned one, and a workspace serving another one ' \ + 'answers payloads of another shape.' + ) + end + + # The three attribute reads of the boot, alongside the version check above. + # Each degrades to no attribute column rather than to a failed boot: a + # token missing a permission costs the columns it could not read, never + # the agent. + def ticket_attributes + Schema::TicketAttributesIntrospector.new(@client).attributes + end + + def model_attributes(model) + Schema::DataAttributesIntrospector.new(@client, model: model).attributes + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/pagination/cursor_walker.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/pagination/cursor_walker.rb new file mode 100644 index 000000000..80f29dfb0 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/pagination/cursor_walker.rb @@ -0,0 +1,123 @@ +module ForestAdminDatasourceIntercom + module Pagination + # Intercom only paginates by cursor (no jump to page N), so an offset/limit window costs one sequential + # request per page. Capped for operator patience (R9), and every truncation is logged, never silent. + class CursorWalker + # 50 pages of 150 records. Intercom's quota lets these be generous: the + # walk is bounded by patience, and by the point past which a list view is + # not being read but scraped. + MAX_PAGES = 50 + MAX_RECORDS = 7_500 + + def initialize(max_pages: MAX_PAGES, max_records: MAX_RECORDS) + @max_pages = max_pages + @max_records = max_records + end + + # Yields `(per_page, cursor)` and expects a Client::Page back. A nil limit means every record past + # the offset, so a cap stopping the walk is known to be a truncation and is logged. + def walk(offset:, limit:, &page_source) + offset = offset.to_i.clamp(0, nil) + limit = limit&.to_i + return [] if limit && !limit.positive? + + records = collect(offset, limit, &page_source) + + limit ? (records[offset, limit] || []) : records.drop(offset) + end + + private + + # The walk itself: pages are collected until the window is covered, the + # source says there is nothing left, or a cap stops it. The slicing is + # `walk`'s; this only decides how far to go. + def collect(offset, limit) + needed = limit && (offset + limit) + records = [] + cursor = nil + seen_ids = Set.new + seen_cursors = Set.new + pages = 0 + + loop do + page = yield(batch_size(needed, records.size), cursor) + records.concat(fresh(page.records, seen_ids)) + pages += 1 + + # The only silent exits: Intercom said there is no page left, or the + # window asked for is covered. Everything below hands back less than + # was asked for, and says so. + break if page.next_cursor.nil? + break if needed && records.size >= needed + break if cut_short?(page, seen_cursors, pages, records: records.size, window: window_of(offset, limit)) + + cursor = page.next_cursor + end + + records + end + + # Whether the walk is handing back less than it was asked for, and which + # of the two reasons it is. Both are logged here rather than by the + # caller: a page that looks like the whole answer and is not is the + # failure this datasource exists to avoid, so there is no route out of + # this walk that is short and quiet. + def cut_short?(page, seen_cursors, pages, records:, window:) + if stalled?(page, seen_cursors) + log_stalled(window: window, pages: pages, collected: records) + elsif capped?(pages, records) + log_truncation(window: window, pages: pages, collected: records) + else + return false + end + + true + end + + # Intercom documents duplicate or missed records when items change mid-walk; duplicates are dropped + # here (missed ones cannot be). A record with no id is kept. + def fresh(records, seen_ids) + records.select { |record| record['id'].nil? || seen_ids.add?(record['id']) } + end + + # An empty page with a next cursor, or a cursor already followed: Intercom does neither today, but a + # walk driven by a remote value must stop on its own terms -- and the stop is logged as a truncation. + def stalled?(page, seen_cursors) + page.records.empty? || !seen_cursors.add?(page.next_cursor) + end + + def capped?(pages, collected) + pages >= @max_pages || collected >= @max_records + end + + # Bounded by the record budget left, and by the window still missing when + # there is one, so the walk never collects past @max_records. `Client` + # bounds it again to what Intercom accepts. + def batch_size(needed, collected) + budget = @max_records - collected + budget = [needed - collected, budget].min if needed + Client.bounded_per_page(budget) + end + + def log_truncation(window:, pages:, collected:) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] Stopped paginating after #{pages} page(s) / " \ + "#{collected} record(s) while fetching #{window}; results are truncated. " \ + 'Narrow the filter to reach records past this point.' + ) + end + + def log_stalled(window:, pages:, collected:) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] Stopped paginating after #{pages} page(s) / " \ + "#{collected} record(s) while fetching #{window}: Intercom advertised a next page this could not " \ + 'follow -- an empty page, or a cursor already read. Results are truncated.' + ) + end + + def window_of(offset, limit) + limit ? "offset=#{offset} limit=#{limit}" : "every record past offset=#{offset}" + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/caller_zone.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/caller_zone.rb new file mode 100644 index 000000000..eddbdd70a --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/caller_zone.rb @@ -0,0 +1,48 @@ +module ForestAdminDatasourceIntercom + module Query + # The timezone a filter was written in, and what reading a value in it + # costs when the caller names one nothing knows. + # + # Split from FilterValue for the reason DayBounds was: that class knows + # what shape a field expects on the wire, and whether a value carries the + # caller's wall clock or the server's is a different question. + class CallerZone + # Kept stripped rather than only checked stripped: as it came, a + # `" Europe/Paris "` passes the blank guard and then fails the zone + # lookup, and a day boundary lands an offset away from where the filter + # meant it. + attr_reader :name + + def initialize(identifier) + stripped = identifier.to_s.strip + @name = stripped.empty? ? 'UTC' : stripped + end + + # A day with no time of day is the caller's day, not the server's: it is + # the timezone the filter was written in that says when that day starts. + def start_of_day(date) + read('the day boundary') { Time.zone.local(date.year, date.month, date.day).to_i } + end + + # An offset-less timestamp is the caller's wall clock: FilterFactory writes previous-period bounds as + # `%Y-%m-%d %H:%M:%S`, and a midnight shifted by any offset truncates onto another UTC day. + def timestamp(value) + read('the timestamp') { Time.zone.parse(value).to_i } + end + + private + + # Falling back to UTC silently would move a boundary by the offset, which + # is the failure a timezone is read for in the first place. + def read(what, &block) + Time.use_zone(@name, &block) + rescue ArgumentError + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] unknown timezone #{@name.inspect}, reading #{what} of a date " \ + 'filter in UTC instead.' + ) + Time.use_zone('UTC', &block) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb new file mode 100644 index 000000000..06d2d5970 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb @@ -0,0 +1,169 @@ +module ForestAdminDatasourceIntercom + module Query + # Whatever it cannot translate it refuses, naming the field or operator: a dropped condition would serve + # an unfiltered page that looks filtered. What each endpoint filters comes from `search_fields.yml`. + class ConditionTreeTranslator + Branch = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + + AGGREGATORS = { 'and' => 'AND', 'or' => 'OR' }.freeze + + # Intercom nests a search two levels deep and takes fifteen conditions per + # group. Both are checked here rather than left to the API: over either, + # Intercom answers a 400 whose body names neither the limit nor the part of + # the filter that reached it, and an operator reading it has no way of + # knowing that their segment plus their scope plus their own filter is what + # went over. + MAX_DEPTH = 2 + MAX_GROUP_SIZE = 15 + + def self.call(condition_tree, endpoint:, collection:, timezone: nil, attribute_columns: []) + return nil if condition_tree.nil? + + new(endpoint: endpoint, collection: collection, timezone: timezone, + attribute_columns: attribute_columns).translate(condition_tree) + end + + # `attribute_columns` are the columns a workspace's own attributes became, + # which no row of the table can name: they are discovered at boot. They + # share one refusal, carried by the endpoint, and passing them here is + # what lets it be read instead of the generic message. + def initialize(endpoint:, collection:, timezone: nil, attribute_columns: []) + @endpoint = endpoint + @collection = collection + @attribute_columns = Array(attribute_columns).map(&:to_s) + @value = FilterValue.new(collection: collection, timezone: timezone) + end + + def translate(node, depth = 1) + case node + when Branch then translate_branch(node, depth) + when Leaf then translate_leaf(node) + else raise UnsupportedOperatorError, "#{@collection} cannot read #{node.class} as a condition." + end + end + + private + + def translate_branch(branch, depth) + conditions = Array(branch.conditions) + refuse_empty_branch!(branch) if conditions.empty? + + # Read before the unwrap below, so a branch is refused on the aggregator + # it carries rather than on how many conditions it holds. + operator = aggregator(branch) + + # A branch holding one condition needs no group of its own. The agent + # builds a tree one branch at a time -- a scope, then a segment, then the + # operator's own filter -- and the nesting Intercom allows is shallow + # enough that a wrapper around nothing is a level worth not spending. + return translate(conditions.first, depth) if conditions.size == 1 + + refuse_too_deep!(branch, depth) if depth > MAX_DEPTH + refuse_too_wide!(branch, conditions.size) if conditions.size > MAX_GROUP_SIZE + + { 'operator' => operator, 'value' => conditions.map { |condition| translate(condition, depth + 1) } } + end + + # What reaches this depth is a group inside a group inside a group. The + # message names the shape rather than a number, since the tree an operator + # can act on is the segment and the scope they wrote, not the one the agent + # assembled out of them. + def refuse_too_deep!(branch, depth) + raise UnsupportedOperatorError, + "#{@collection} cannot answer this filter: Intercom nests a search #{MAX_DEPTH} levels deep and this " \ + "one reaches #{depth}. #{deepening_cause(branch)}" + end + + # A group the operator wrote is theirs to flatten. A group a relation + # expanded into is not: it is the several records the relation matched, + # written one equality each because Intercom takes no membership operator + # on a foreign key -- and telling them to flatten a nesting they never + # wrote is a refusal they cannot act on. + def deepening_cause(branch) + field = branch.respond_to?(:relation_field) ? branch.relation_field : nil + + if field.nil? + 'A group inside a group inside a group is one level too many -- flatten the segment, the scope or the ' \ + 'filter carrying the innermost one.' + else + "The innermost group is what #{field.inspect} expanded into, one equality per record it matched: narrow " \ + 'that condition until it names a single record, or lift it out of the groups nesting it.' + end + end + + # Fifteen is reached without trying: a scope, a segment and a filter add up, + # and a condition naming several values is expanded into one condition per + # value on the way here, Intercom taking no membership operator on these + # fields. + def refuse_too_wide!(branch, size) + raise UnsupportedOperatorError, + "#{@collection} cannot answer this filter: Intercom takes #{MAX_GROUP_SIZE} conditions per group and " \ + "this #{branch.aggregator} carries #{size}. A filter naming several values counts one condition per " \ + 'value here, so narrowing the list, the segment or the scope is what brings it back under the limit.' + end + + def aggregator(branch) + AGGREGATORS[branch.aggregator.to_s.downcase] || + raise(UnsupportedOperatorError, + "#{@collection} cannot read #{branch.aggregator.inspect} as a condition tree aggregator; " \ + "expected 'And' or 'Or'.") + end + + def translate_leaf(leaf) + field = @endpoint.field(leaf.field.to_s) || refuse_unfilterable!(leaf.field.to_s) + spelling = OperatorTable.intercom_operator(field, leaf.operator) || refuse_operator!(leaf, field) + + { 'field' => field.field, 'operator' => spelling, 'value' => @value.call(leaf, field, spelling) } + end + + # A column the endpoint does not filter, and the reason it does not, taken + # from the table when it carries one: those reasons are the difference + # between "no" and a message an operator can do something with. + def refuse_unfilterable!(column) + raise UnsupportedOperatorError, "#{@collection} cannot filter #{column.inspect}: #{unfilterable_reason(column)}" + end + + def unfilterable_reason(column) + return relation_reason(column) if column.include?(':') + + refusal = @endpoint.refusal(column) || attribute_refusal(column) + return refusal.reason if refusal + + "#{@endpoint.path} takes no filter on it. Filter on one of: #{@endpoint.filterable_columns.join(", ")}." + end + + # A relation reaches the translator as `relation:field`, and it should not: + # `Relations#rewrite_relation_conditions` trades every one of them for a + # condition on the foreign key before a tree gets this far, or refuses it + # by name -- with the relations the collection does declare. So this is + # the message for a caller that skipped that pass, and it names what this + # endpoint filters rather than a relation nobody resolved. + def relation_reason(_column) + "#{@endpoint.path} filters columns, not paths through a relation -- those are resolved against the " \ + 'collection they point at before a condition reaches here. Filter on one of: ' \ + "#{@endpoint.filterable_columns.join(", ")}." + end + + # A column the table cannot carry a row for, the workspace having named it: + # the endpoint's one refusal for the whole family answers for it. + def attribute_refusal(column) + @attribute_columns.include?(column) ? @endpoint.attribute_refusal : nil + end + + def refuse_operator!(leaf, field) + supported = OperatorTable.forest_operators(field) + + raise UnsupportedOperatorError, + "#{@collection} cannot filter #{leaf.field.inspect} with #{leaf.operator.inspect}: " \ + "#{@endpoint.path} answers #{supported.join(", ")} on #{field.field.inspect} and nothing else." + end + + def refuse_empty_branch!(branch) + raise UnsupportedOperatorError, + "#{@collection} was given a #{branch.aggregator} branch carrying no condition, which names no record " \ + 'and no filter.' + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/day_bounds.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/day_bounds.rb new file mode 100644 index 000000000..234a11ce4 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/day_bounds.rb @@ -0,0 +1,47 @@ +module ForestAdminDatasourceIntercom + module Query + # Intercom truncates a date search to the UTC day (measured; the docs promise the workspace timezone): + # `> V` starts at the day after V's, `< V` stops before V's day. Each bound is moved one day outward (a `<` + # already on a day boundary stays) so Intercom answers the day the filter named: the window is day-granular. + class DayBounds + UTC_DAY = 86_400 + + def initialize(collection:, timezone:) + @collection = collection + @timezone = timezone + end + + def call(seconds, spelling) + day = seconds - (seconds % UTC_DAY) + report_utc_day(seconds, day) + + return day - UTC_DAY if spelling == '>' + + seconds == day ? seconds : day + UTC_DAY + end + + private + + # A window written in another timezone is answered on the UTC days it + # overlaps, which is up to a day wider at each end. Reported once per + # filter rather than per bound: what an operator needs to know is that the + # day boundary is not theirs, not how many bounds crossed it. + def report_utc_day(seconds, day) + return if @reported || same_day_locally?(seconds, day) + + @reported = true + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{@collection} was filtered on a date written in #{@timezone}, and " \ + 'Intercom truncates a date search to the UTC day whatever the workspace timezone says. The rows come ' \ + 'back for the UTC days the window overlaps, which is up to a day wider at each end.' + ) + end + + def same_day_locally?(seconds, day) + Time.use_zone(@timezone) { Time.zone.at(seconds).to_date } == Time.at(day).utc.to_date + rescue ArgumentError + true + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/filter_value.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/filter_value.rb new file mode 100644 index 000000000..eceabe069 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/filter_value.rb @@ -0,0 +1,177 @@ +module ForestAdminDatasourceIntercom + module Query + # Intercom types its filter values, and a wrongly shaped one fails with `data_invalid`, the same code as an + # unsupported operator, so values are converted here rather than by reading error codes. + # Long by line count only: most of it is the refusals, one per shape. + class FilterValue # rubocop:disable Metrics/ClassLength + # What the frontend sends for a Dateonly column, and what a segment written + # in Ruby carries: a day with no time of day, which is midnight in the + # timezone of whoever wrote the filter rather than in the server's. + DATE_ONLY = /\A\d{4}-\d{2}-\d{2}\z/ + + # What `Date._parse` has to have found for a string to name a day at all. + DATE_PARTS = %i[year mon mday].freeze + + INTEGER = /\A-?\d+\z/ + + # Measured on every list, whatever group it stands in: Intercom counts + # each value as an element of a composite query and answers a 400 naming + # neither the field nor the condition past 15. + MAX_LIST_SIZE = 15 + + def initialize(collection:, timezone: nil) + @collection = collection + @zone = CallerZone.new(timezone) + @day_bounds = DayBounds.new(collection: collection, timezone: @zone.name) + end + + def call(leaf, field, spelling) + return list(leaf, field) if OperatorTable.list_operator?(spelling) + + refuse_absence!(leaf, field) if blank?(leaf.value) + + value = scalar(leaf.value, leaf, field) + field.type == 'date' ? @day_bounds.call(value, spelling) : value + end + + private + + # Dropping the blanks would answer a different question: `not_in [nil, + # 'open']` was asked to exclude the records carrying neither and would come + # back including them. An empty list is as bad the other way round, being a + # filter that matches everything. + def list(leaf, field) + values = Array(leaf.value) + refuse_empty_list!(leaf) if values.empty? + refuse_long_list!(leaf, values.size) if values.size > MAX_LIST_SIZE + refuse_absence!(leaf, field) if values.any? { |value| blank?(value) } + + values.map { |value| scalar(value, leaf, field) } + end + + def scalar(value, leaf, field) + case field.type + when 'date' then epoch(value, leaf) + when 'number' then number(value, leaf) + when 'boolean' then boolean(value, leaf) + else field.value_map ? mapped(value.to_s, leaf, field) : value.to_s + end + end + + # A value outside the map is refused rather than sent as typed: Intercom + # would compare it against a vocabulary the column never shows. + def mapped(value, leaf, field) + field.value_map.fetch(value) { refuse_value!(leaf, value, "one of #{field.value_map.keys.join(", ")}") } + end + + # Intercom stores and compares its dates as epoch seconds. What arrives + # here is an ISO8601 string most of the time -- the frontend sends one, and + # so does every interval operator the toolkit rewrites into a pair of + # bounds -- but a scope or a segment written in Ruby carries a Time or a + # Date, and neither has a timezone of its own. + def epoch(value, leaf) + case value + when DateTime then value.to_time.to_i + when Date then @zone.start_of_day(value) + when Time then value.to_i + when Numeric then seconds(value, leaf) + when String then parse(value, leaf) + else refuse_value!(leaf, value, 'a date') + end + end + + # An Infinity or a NaN -- a cast that overflowed above this datasource -- + # makes `Integer()` raise a FloatDomainError naming a float where the + # operator asked for a date, and a Complex a RangeError, which is the + # same class one level up. Refused like every other value the field + # cannot take, for the reason `number` refuses them too. + def seconds(value, leaf) + Integer(value) + rescue RangeError, TypeError + refuse_value!(leaf, value, 'a date') + end + + # Refuses a string lacking a year, month or day instead of letting a parser complete it: `Time.zone.parse` + # answers today for `last tuesday`, and `Time.parse` fills in the server's date for `12:00`. + def parse(value, leaf) + return @zone.start_of_day(Date.parse(value)) if DATE_ONLY.match?(value) + + refuse_value!(leaf, value, 'a date') unless dated?(value) + + @zone.timestamp(value) + rescue ArgumentError, TypeError + refuse_value!(leaf, value, 'a date') + end + + def dated?(value) + parts = Date._parse(value) + + DATE_PARTS.all? { |part| parts.key?(part) } + end + + # The agent casts every Number column with `to_f`, so an integer field + # would be filtered with `42.0` -- a form none of its values carry. A float + # with nothing after the point travels as the integer it is. + # + # A cast that overflowed to Infinity, or a NaN, is refused rather than + # passed on: the JSON encoder raises on both, a step later, as a 500 naming + # nothing the operator can act on. + def number(value, leaf) + case value + when Integer then value + when Float then finite(value, leaf) + when String then value.match?(INTEGER) ? value.to_i : finite(Float(value, exception: false), leaf) + else refuse_value!(leaf, value, 'a number') + end + end + + def finite(value, leaf) + refuse_value!(leaf, value, 'a number') unless value.is_a?(Float) && value.finite? + + value == value.to_i ? value.to_i : value + end + + def boolean(value, leaf) + case value + when true, false then value + when 'true' then true + when 'false' then false + else refuse_value!(leaf, value, 'a true or a false') + end + end + + def blank?(value) = value.nil? || value.to_s.empty? + + # `present`, `blank` and `missing` are derived by the agent from an + # equality, above this datasource, and rewritten into a comparison with an + # empty value. Intercom's search matches values and has no spelling for the + # absence of one, so the rewritten condition is refused here rather than + # sent as a comparison against the empty string -- which Intercom would + # answer as if it were a value of its own. + def refuse_absence!(leaf, field) + raise UnsupportedOperatorError, + "#{@collection} cannot filter #{leaf.field.inspect} for absence: Intercom's search matches values " \ + "and has no operator for the lack of one, so a #{leaf.operator} condition on " \ + "#{field.field.inspect} cannot be translated. Filter on a value instead." + end + + def refuse_empty_list!(leaf) + raise UnsupportedOperatorError, + "#{@collection} was asked to filter #{leaf.field.inspect} with #{leaf.operator} and an empty list, " \ + 'which names no record and no filter. Pass at least one value.' + end + + def refuse_long_list!(leaf, size) + raise UnsupportedOperatorError, + "#{@collection} cannot filter #{leaf.field.inspect} with #{leaf.operator} and #{size} values: " \ + "Intercom takes #{MAX_LIST_SIZE} at most in one list. Narrow the list, or split the filter." + end + + def refuse_value!(leaf, value, expected) + raise UnsupportedOperatorError, + "#{@collection} cannot filter #{leaf.field.inspect} with #{value.inspect}: Intercom expects " \ + "#{expected} on this field." + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/operator_table.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/operator_table.rb new file mode 100644 index 000000000..da4e0ff15 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/operator_table.rb @@ -0,0 +1,54 @@ +module ForestAdminDatasourceIntercom + module Query + # Read by both the schema and the translator, so a column never advertises an operator the translator + # refuses. Which operators an endpoint accepts per field lives in `search_fields.yml`. + module OperatorTable + Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + + EQUALITY = { Operators::EQUAL => '=', Operators::NOT_EQUAL => '!=', + Operators::IN => 'IN', Operators::NOT_IN => 'NIN' }.freeze + + # `contains` and `i_contains` both map to `~`, Intercom's only substring operator. `not_i_contains` is + # absent: the toolkit's `Rules` rejects it on a String column (PRD-989). + SUBSTRING = { Operators::CONTAINS => '~', Operators::I_CONTAINS => '~', + Operators::NOT_CONTAINS => '!~', + Operators::STARTS_WITH => '^', Operators::ENDS_WITH => '$' }.freeze + + BOUNDS = { Operators::GREATER_THAN => '>', Operators::LESS_THAN => '<' }.freeze + + # Dates publish only the two bounds although both endpoints accept `=`, `!=`, `>=`, `<=` (measured): + # declaring `equal` makes the toolkit republish `in`, which its validator refuses (PRD-989). The toolkit + # still derives `today`, `past`, `previous_*` and the rest from the two bounds. + MAPS = { + 'string' => EQUALITY, + 'boolean' => EQUALITY, + 'number' => EQUALITY.merge(BOUNDS), + 'text' => EQUALITY.merge(SUBSTRING), + 'date' => BOUNDS + }.freeze + + # The operators that take a list rather than a value, on the wire. + LIST_OPERATORS = %w[IN NIN].freeze + + class << self + def types = MAPS.keys + + # What the column publishes: the Forest operators whose Intercom spelling + # this endpoint accepts on this field, and no other. + def forest_operators(field) + MAPS.fetch(field.type).select { |_, spelling| field.operators.include?(spelling) }.keys + end + + # nil when the endpoint does not accept that operator on that field, + # which the translator turns into a refusal naming what it does accept. + def intercom_operator(field, forest_operator) + spelling = MAPS.fetch(field.type)[forest_operator] + + spelling if spelling && field.operators.include?(spelling) + end + + def list_operator?(spelling) = LIST_OPERATORS.include?(spelling) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb new file mode 100644 index 000000000..267c13c18 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb @@ -0,0 +1,225 @@ +module ForestAdminDatasourceIntercom + module Query + # Loads `search_fields.yml`, which `forest_admin_intercom_probe` rewrites from a real workspace. Every row + # is validated on load, so a bad rewrite fails at boot rather than producing an unexplainable schema. + module SearchFields # rubocop:disable Metrics/ModuleLength + PATH = File.expand_path('search_fields.yml', __dir__) + + # The operators Intercom's search DSL spells, and nothing else: `=`, `!=`, + # `>`, `<`, `>=`, `<=`, the substring pair `~` / `!~`, the anchors `^` / + # `$`, and the membership pair. Which of them an endpoint honours on a + # given field is the table's business; this is only the alphabet. + KNOWN_OPERATORS = ['=', '!=', '>', '<', '>=', '<=', '~', '!~', '^', '$', 'IN', 'NIN'].freeze + # Read off the operator table rather than listed again here: a type with + # no spelling of its own would pass this validation and raise when the + # schema asked what to publish on it. + KNOWN_TYPES = OperatorTable.types + KNOWN_SOURCES = %w[measured spec].freeze + + # `source: spec` marks a row the probe has not confirmed. Only `/contacts/search` honours `sortable`; the + # other endpoints silently ignore a sort, so absent means unsortable. `value_map` (`values` in the file) + # maps a shown value onto the searched one, e.g. a ticket shows `Customer` but searches `request`. + Field = Struct.new(:column, :field, :type, :operators, :source, :sortable, :value_map, keyword_init: true) do + def measured? = source == 'measured' + def sortable? = sortable == true + end + + # A column that stays unfilterable, and why. The reason travels into the + # refusal the operator reads, so it names what to filter on instead + # wherever there is something to name. + Refusal = Struct.new(:column, :reason, :source, keyword_init: true) do + def measured? = source == 'measured' + end + + Endpoint = Struct.new(:name, :path, :measured_at, :fields, :refused, :candidates, :attribute_refusal, + keyword_init: true) do + # Whether the probe has run against a real workspace for this endpoint. + # False means every `spec` row is still a candidate. + def measured? = !measured_at.nil? + + def field(column) = fields[column] + def refusal(column) = refused[column] + def filterable_columns = fields.keys + def sortable_columns = fields.values.select(&:sortable?).map(&:column) + def unmeasured_fields = fields.values.reject(&:measured?) + end + + # Long by line count only: it is one parse method and one validation per + # thing the file can get wrong, and a validation that does not say what + # is wrong is one nobody can act on. + class << self # rubocop:disable Metrics/ClassLength + def fetch(name) + table[name.to_s] || + raise(ConfigurationError, "Unknown Intercom search endpoint #{name.inspect}; " \ + "the table declares #{table.keys.join(", ")}.") + end + + def endpoints = table.keys + + def table + @table ||= build(YAML.safe_load_file(PATH)) + end + + # Public so a spec can feed it a table of its own: this validation is the + # reason the file can be rewritten by a script without the package + # trusting whatever comes back. + def build(raw) + raw.fetch('endpoints').to_h { |name, definition| [name, endpoint(name, definition)] }.freeze + end + + private + + def endpoint(name, definition) + filterable = fields(name, definition['fields']) + refused = refusals(name, definition['refused']) + validate_no_overlap!(name, filterable, refused) + + Endpoint.new( + name: name, + path: definition.fetch('path'), + measured_at: definition['measured_at'], + fields: filterable, + refused: refused, + candidates: Array(definition['candidates']).freeze, + attribute_refusal: attribute_refusal(name, definition) + ).freeze + end + + # One refusal for the workspace-defined attribute columns, whose names are unknown until boot: + # `ticket_attributes` (per ticket type) or `custom_attributes` (per model). + def attribute_refusal(endpoint, definition) + column = %w[ticket_attributes custom_attributes].find { |key| definition.key?(key) } + return nil if column.nil? + + row = definition.fetch(column) + refusal = Refusal.new(column: column, reason: squish(row.fetch('reason')), source: row.fetch('source')) + validate_source!(endpoint, column, refusal) + validate_attributes_unfilterable!(endpoint, column, row) + + refusal.freeze + end + + # `filterable: true` is a state nothing here implements: the whole + # block is a refusal, and a rewrite flipping the flag would publish + # nothing new while making the file say the opposite of what it does. + def validate_attributes_unfilterable!(endpoint, column, row) + return if row.fetch('filterable') == false + + malformed!(endpoint, column, + "filterable #{row["filterable"].inspect} is not something this reads; the attribute columns " \ + 'are published for display only, and the block exists to say why') + end + + # A column filed as both filterable and refused. `Endpoint#field` is + # consulted first, so the filterable row would win and the refusal -- + # with the reason an operator reads -- would be ignored in silence. The + # file is rewritten by a script, so this is the shape a bad rewrite + # takes, and it fails at load rather than producing a schema nobody can + # explain. + def validate_no_overlap!(endpoint, filterable, refused) + both = filterable.keys & refused.keys + return if both.empty? + + malformed!(endpoint, both.join(', '), + 'it is declared filterable and refused at once; the refusal would be ignored') + end + + def fields(endpoint, declared) + (declared || {}).to_h do |column, row| + field = Field.new(column: column, field: row.fetch('field'), type: row.fetch('type'), + operators: Array(row['operators']).freeze, source: row.fetch('source'), + sortable: row.fetch('sortable', false), value_map: row['values']).freeze + validate_field!(endpoint, field) + + [column, field] + end.freeze + end + + def refusals(endpoint, declared) + (declared || {}).to_h do |column, row| + refusal = Refusal.new(column: column, reason: squish(row.fetch('reason')), + source: row.fetch('source')).freeze + validate_source!(endpoint, column, refusal) + + [column, refusal] + end.freeze + end + + def validate_field!(endpoint, field) + validate_source!(endpoint, field.column, field) + validate_type!(endpoint, field) + validate_operators!(endpoint, field) + validate_publishable!(endpoint, field) + validate_sortable!(endpoint, field) + validate_values!(endpoint, field) + end + + # Operators the DSL spells, none of which this column's *type* can carry: + # a `date` row declaring `~` passes the alphabet above and publishes + # nothing, `OperatorTable` mapping no Forest operator onto it. The result + # is a column the table calls filterable that no filter can reach, and + # the translator refusing every request on it -- the same + # advertise-then-refuse this package exists to prevent, one layer lower. + def validate_publishable!(endpoint, field) + return unless OperatorTable.forest_operators(field).empty? + + malformed!(endpoint, field.column, + "none of #{field.operators.join(", ")} is an operator Intercom answers on a " \ + "#{field.type}, so the column would publish no filter at all") + end + + # Anything but a boolean, `"true"` above all: YAML reads it as a string, + # which is truthy in Ruby and would publish a sortable column out of a + # typo the file cannot otherwise show. + def validate_sortable!(endpoint, field) + return if [true, false].include?(field.sortable) + + malformed!(endpoint, field.column, "sortable #{field.sortable.inspect} is neither true nor false") + end + + def validate_values!(endpoint, field) + map = field.value_map + return if map.nil? || (map.is_a?(Hash) && !map.empty? && map.to_a.flatten.all?(String)) + + malformed!(endpoint, field.column, "values #{map.inspect} is not a map of strings to strings") + end + + def validate_type!(endpoint, field) + return if KNOWN_TYPES.include?(field.type) + + malformed!(endpoint, field.column, "type #{field.type.inspect} is not one of #{KNOWN_TYPES.join(", ")}") + end + + # An empty operator list is how a table stops short of saying anything: + # it would publish a filterable column no operator can reach. A column + # Intercom does not filter belongs in the refused table, where it comes + # with the reason an operator reads. + def validate_operators!(endpoint, field) + if field.operators.empty? + malformed!(endpoint, field.column, + 'it declares no operator; a column Intercom cannot filter belongs in the refused table') + end + + unknown = field.operators - KNOWN_OPERATORS + return if unknown.empty? + + malformed!(endpoint, field.column, "Intercom's search DSL has no operator #{unknown.join(", ")}") + end + + def validate_source!(endpoint, column, row) + return if KNOWN_SOURCES.include?(row.source) + + malformed!(endpoint, column, "source #{row.source.inspect} is neither #{KNOWN_SOURCES.join(" nor ")}") + end + + def malformed!(endpoint, column, detail) + raise ConfigurationError, "#{File.basename(PATH)} is malformed at #{endpoint}.#{column}: #{detail}." + end + + # A YAML folded block keeps the newlines the file needs to stay readable; + # the reason travels into a one-line message. + def squish(text) = text.to_s.split.join(' ') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml new file mode 100644 index 000000000..10eb45aa1 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml @@ -0,0 +1,663 @@ +# The table of (endpoint x field x operator): what Intercom's search endpoints +# really filter, and with which operators. It is the single source of truth of +# this datasource's filtering -- the schema derives every column's +# filter_operators from it, so no collection can advertise a filter the +# translator would then refuse. +# +# `source` is the provenance of a row, and it is not decoration: +# +# measured -- observed against a real workspace, by `forest_admin_intercom_probe` +# or during a spike, and recorded here with the date; +# spec -- read off Intercom's documentation and nothing else. +# +# The distinction exists because the two disagree. Measured during lot 1: +# `company_id` is documented on a ticket and refused by `/tickets/search` with +# `invalid_field`. A `spec` row is therefore a candidate, never a promise, and +# the probe is what turns it into one. Run it against the customer's workspace +# and commit what it writes. +# +# `type` decides how a column's Forest operators are derived from the Intercom +# ones, and how a value reaches the wire: +# +# string -- an exact-match field +# text -- a field Intercom also matches per word (`~`) +# date -- epoch seconds on the wire, truncated to the UTC day on read +# boolean, number +version: 1 + +endpoints: + conversations: + path: conversations/search + # Nothing has been probed yet: fill this in with the probe's output. + measured_at: null + fields: + # The primary key, and the one column whose Forest operators are not + # derived from this table: every collection publishes `equal` and `in` on + # its key whatever is written here, the toolkit refusing a collection + # whose key carries neither. It has to be declared all the same, or the + # schema advertises a filter the translator has no row to write -- which + # is what a record detail becomes the moment a permission scope turns + # `id equals X` into an `and` the record endpoint cannot answer alone. + # + # A bare `id equals X` still reads the record endpoint, one request + # instead of a search. This row is for the compound case. + id: + field: id + type: string + operators: ['=', 'IN'] + source: spec + state: + field: state + type: string + operators: ['=', '!='] + source: spec + open: + field: open + type: boolean + operators: ['='] + source: spec + read: + field: read + type: boolean + operators: ['='] + source: spec + priority: + field: priority + type: string + operators: ['=', '!='] + source: spec + title: + field: title + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + admin_assignee_id: + field: admin_assignee_id + type: string + operators: ['=', '!='] + source: spec + team_assignee_id: + field: team_assignee_id + type: string + operators: ['=', '!='] + source: spec + source_type: + field: source.type + type: string + operators: ['=', '!='] + source: spec + source_delivered_as: + field: source.delivered_as + type: string + operators: ['=', '!='] + source: spec + source_subject: + field: source.subject + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + # The full-text search of the collection: Intercom matches `~` per word, + # not as a substring, which the README states rather than the interface + # implying otherwise. + source_body: + field: source.body + type: text + operators: ['~', '!~'] + source: spec + source_author_email: + field: source.author.email + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + # The date operators are measured, and they are not the same on every + # endpoint: `/contacts/search` refuses `>=`, `<=` and `!=` where this one + # accepts them (25 August 2026, API 2.16). Whatever this table allows, a + # Date column publishes the two bounds alone -- see the translator. + created_at: + field: created_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: measured + updated_at: + field: updated_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: measured + waiting_since: + field: waiting_since + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + snoozed_until: + field: snoozed_until + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + closed_at: + field: statistics.last_close_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + first_closed_at: + field: statistics.first_close_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + closed_by_id: + field: statistics.last_closed_by_id + type: string + operators: ['=', '!='] + source: spec + first_contact_reply_at: + field: statistics.first_contact_reply_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + last_contact_reply_at: + field: statistics.last_contact_reply_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + last_admin_reply_at: + field: statistics.last_admin_reply_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + reopen_count: + field: statistics.count_reopens + type: number + operators: ['=', '!=', '>', '<'] + source: spec + part_count: + field: statistics.count_conversation_parts + type: number + operators: ['=', '!=', '>', '<'] + source: spec + ai_agent_participated: + field: ai_agent_participated + type: boolean + operators: ['='] + source: spec + # The foreign key of the `contact` relation, and the reason lot 4 could + # turn the denormalized contact columns into one: Intercom matches a + # conversation against one of the contacts it carries. The column is + # singular and the wire field plural on purpose -- the row names its first + # contact, the endpoint asks "is this contact one of yours". + # + # `!=` is deliberately absent: on a field holding a list, "not that + # contact" is a question Intercom's DSL does not clearly answer, and the + # one thing this table may not do is publish a filter whose meaning is a + # guess. + contact_id: + field: contact_ids + type: string + operators: ['='] + source: measured + # Columns this collection publishes and will not filter, with the reason an + # operator reads when they try. A column absent from both tables is refused + # by name too -- these are the ones whose refusal is permanent and has an + # explanation worth giving. + refused: + tag_names: + reason: >- + Intercom filters conversations by tag id, and this column holds the + tag names. Filtering it would mean resolving a name to an id per + request, and a name the workspace renamed would silently match + nothing. + source: spec + company_name: + reason: >- + The conversation carries its company as an object read from the + payload; the search endpoint filters no company field. + source: spec + company_id: + reason: >- + The search endpoint filters no company field. Measured on + `/tickets/search`, which refuses `company_id` with `invalid_field` + although a ticket carries one; the same is assumed here until probed. + source: spec + timeline: + reason: >- + Built by the agent from the parts of a conversation, which the search + endpoint does not read. + source: measured + contact_name: + reason: >- + Read from the Contacts endpoint, not from the conversation. Filter + through the `contact` relation, which resolves against + `/contacts/search` and rewrites onto the contact id -- or on + `source_author_email`, which is on the conversation itself. + source: spec + contact_count: + reason: Counted by the agent from the contacts the payload carries. + source: measured + source_author_name: + reason: >- + The endpoint matches the author of the opening message by e-mail + alone -- `source.author.email` is a field, `source.author.name` is + not. Filter on `source_author_email`. + source: spec + # What `forest_admin_intercom_probe` enumerates on top of the fields above: + # names the documentation mentions, or that an ops team would plausibly + # search on. A candidate that turns out to be filterable becomes a field + # above -- with a column to expose it on, or with none, in which case it is + # a column the next lot may add. + candidates: + - source.id + - source.author.id + - source.author.type + - source.author.name + - statistics.time_to_assignment + - statistics.time_to_admin_reply + - statistics.time_to_first_close + - statistics.median_time_to_reply + - statistics.first_assignment_at + - statistics.first_admin_reply_at + - statistics.last_assignment_at + - statistics.count_assignments + - conversation_rating.score + - conversation_rating.remark + - conversation_rating.contact_id + - ai_agent.resolution_state + - ai_agent.last_answer_type + - ai_agent.rating + - channel_initiated + - tag_ids + - teammate_ids + - company_id + - topics + + tickets: + path: tickets/search + measured_at: null + fields: + # The primary key, declared here for the same reason as on conversations: + # the schema publishes `equal` and `in` on it whatever this table says, + # so a row has to exist for the translator to write. `GET /tickets/{id}` + # answers the key on its own; the search answers it in an `and`. + id: + field: id + type: string + operators: ['=', 'IN'] + source: spec + open: + field: open + type: boolean + operators: ['='] + source: spec + category: + field: category + type: string + operators: ['=', '!='] + source: spec + # A ticket reads its category as a label; the search takes another + # word for each (OpenAPI 2.16, "Searching by Category"). + values: + Customer: request + Back-office: task + Tracker: tracker + ticket_type_id: + field: ticket_type_id + type: string + operators: ['=', '!='] + source: spec + admin_assignee_id: + field: admin_assignee_id + type: string + operators: ['=', '!='] + source: spec + team_assignee_id: + field: team_assignee_id + type: string + operators: ['=', '!='] + source: spec + created_at: + field: created_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: measured + updated_at: + field: updated_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: measured + # The foreign key of the `contact` relation. `spec` rather than + # `measured`, and the difference is visible to an operator: it is what + # decides whether the contacts of a ticket can be filtered at all, and the + # probe has not run on this endpoint. If Intercom refuses it, this row + # moves to the refused table and the relation stays navigable without + # being filterable -- the way `state` already is. + contact_id: + field: contact_ids + type: string + operators: ['='] + source: spec + refused: + company_id: + reason: >- + Measured during lot 1: `/tickets/search` refuses `company_id` with + `invalid_field`, although a ticket carries one. Filtering tickets by + account is not something this endpoint does. + source: measured + closed_at: + reason: >- + Derived by the agent from the parts of the ticket. The search endpoint + filters nothing of the sort, and ignores a sort on it without a word. + source: measured + closed_by_name: + reason: Derived by the agent from the parts of the ticket. + source: measured + last_reply_at: + reason: Derived by the agent from the parts of the ticket. + source: measured + last_responder_name: + reason: Derived by the agent from the parts of the ticket. + source: measured + last_responder_type: + reason: Derived by the agent from the parts of the ticket. + source: measured + state_label: + reason: >- + Read off the state object the ticket embeds. Whether the endpoint + filters a ticket state at all, and under which name, is one of the + probe's questions. + source: spec + ticket_type_name: + reason: >- + Read off the type object the ticket embeds. Filter on + `ticket_type_id`, which the endpoint does take. + source: spec + ticket_id: + reason: >- + The number the support team says out loud, which is not the id the + API answers by. The search endpoint carries no field for it. Filter + on `id`, which takes the value the url and the record page use. + source: spec + is_shared: + reason: >- + Whether the ticket is visible to the customer. On the payload and not + in the search DSL; the probe lists it as a candidate. + source: spec + state_id: + reason: >- + Read off the state object the ticket embeds. The endpoint takes no + filter on a ticket state under any name this table knows, which is + what keeps the `state` relation navigable without being filterable. + Whether it filters one at all is the probe's second question. Filter + on `open` or on `category` meanwhile. + source: spec + previous_state_id: + reason: >- + The state the ticket left, read off its payload. Filterable no more + than the state it is in, and for the same reason. + source: spec + contact_name: + reason: >- + Read from the Contacts endpoint, not from the ticket. Filter through + the `contact` relation, which resolves against `/contacts/search` and + rewrites onto `contact_ids`. + source: spec + contact_count: + reason: Counted by the agent from the contacts the payload carries. + source: spec + part_count: + reason: Counted by the agent from the parts the payload carries. + source: measured + timeline: + reason: >- + Built by the agent from the parts of the ticket, which the search + endpoint does not read. Filter on `last_reply_at` or on the state + instead. + source: measured + # R7, and it is a product decision rather than an implementation choice: a + # ticket attribute is filtered as `ticket_attribute.{id}`, and a same-named + # attribute carries a different id per ticket type (measured: + # `_default_title_` is 14162161 on one type and 14162165 on another). One + # union column cannot know which id a row's type uses, so filtering it + # server-side would take one collection per ticket type -- more collections + # in the interface, and a schema that changes shape when the customer adds a + # type. Until the customer says that trade is worth it, the attributes stay + # display-only, as lot 1 published them. + ticket_attributes: + filterable: false + reason: >- + An Intercom ticket attribute is filtered through an id that differs from + one ticket type to the next, so a column showing the attribute of every + type at once cannot say which id to filter on. Filter on the ticket type + and a native field instead. + source: measured + + candidates: + - ticket_id + - state + - ticket_state.category + - ticket_state.id + - state_id + - is_shared + - previous_state_id + - company_ids + - source.author.email + - source.subject + - source.body + - title + - description + + # The one endpoint of the whole API that sorts. `sortable: true` is what a + # column reads to publish `isSortable`, and it appears nowhere else in this + # file on purpose: `/conversations/search` accepts a `sort` and ignores it + # without a word (measured), so a row that stayed silent there would be a + # column promising an order Intercom never applies. + # + # The set is deliberately narrower than what the documentation implies. It + # says the results may be sorted by any attribute; nothing has been measured, + # and a sort Intercom refuses is a list view that fails rather than one that + # comes back unordered. These are the fields an ops team orders a contact list + # by, and the probe is what may widen the set. + contacts: + path: contacts/search + measured_at: null + fields: + # Measured rather than taken from the documentation: lot 1 reads the + # contacts of a page through `id IN [...]` on this endpoint, which is the + # membership operator every other field refuses. + id: + field: id + type: string + operators: ['=', 'IN'] + source: measured + role: + field: role + type: string + operators: ['=', '!='] + source: spec + name: + field: name + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + sortable: true + email: + field: email + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + sortable: true + # Derived by the agent from the address, and filtered by Intercom all the + # same: the endpoint carries a field of its own for it, which is what + # makes "everyone at this customer" a filter rather than a wildcard. + email_domain: + field: email_domain + type: string + operators: ['=', '!='] + source: spec + phone: + field: phone + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + external_id: + field: external_id + type: string + operators: ['=', '!='] + source: spec + owner_id: + field: owner_id + type: string + operators: ['=', '!='] + source: spec + # Measured on 25 August 2026, API 2.16, and this is the asymmetry the + # per-endpoint table exists for: `/conversations/search` and + # `/tickets/search` accept `>=`, `<=` and `!=` on a date where this + # endpoint answers `data_invalid`. `IN` is refused on a date everywhere. + # + # Nothing is lost that an operator can see: a Date column publishes the + # two bounds alone whatever this table allows -- see the operator table -- + # and the toolkit derives `before`, `after`, `today` and the whole + # `previous_*` family from them. + created_at: + field: created_at + type: date + operators: ['>', '<'] + source: measured + sortable: true + updated_at: + field: updated_at + type: date + operators: ['>', '<'] + source: measured + sortable: true + signed_up_at: + field: signed_up_at + type: date + operators: ['>', '<'] + source: spec + sortable: true + last_seen_at: + field: last_seen_at + type: date + operators: ['>', '<'] + source: spec + sortable: true + last_contacted_at: + field: last_contacted_at + type: date + operators: ['>', '<'] + source: spec + sortable: true + last_replied_at: + field: last_replied_at + type: date + operators: ['>', '<'] + source: spec + sortable: true + last_email_opened_at: + field: last_email_opened_at + type: date + operators: ['>', '<'] + source: spec + last_email_clicked_at: + field: last_email_clicked_at + type: date + operators: ['>', '<'] + source: spec + unsubscribed_from_emails: + field: unsubscribed_from_emails + type: boolean + operators: ['='] + source: spec + has_hard_bounced: + field: has_hard_bounced + type: boolean + operators: ['='] + source: spec + marked_email_as_spam: + field: marked_email_as_spam + type: boolean + operators: ['='] + source: spec + language_override: + field: language_override + type: string + operators: ['=', '!='] + source: spec + browser: + field: browser + type: string + operators: ['=', '!='] + source: spec + browser_language: + field: browser_language + type: string + operators: ['=', '!='] + source: spec + os: + field: os + type: string + operators: ['=', '!='] + source: spec + location_country: + field: location.country + type: string + operators: ['=', '!='] + source: spec + location_region: + field: location.region + type: string + operators: ['=', '!='] + source: spec + location_city: + field: location.city + type: string + operators: ['=', '!='] + source: spec + refused: + avatar: + reason: >- + The url of the contact's picture. Nothing filters an image, and + Intercom's search does not carry the field. + source: spec + company_id: + reason: >- + The first of the companies the contact belongs to, read off the + payload. `/contacts/search` filters no company field, so a company + list is reached from the company side -- open the company and read + its contacts. + source: spec + company_count: + reason: Counted by the agent from the companies the payload carries. + source: spec + session_count: + reason: >- + On the contact payload and not in the search DSL. Filter on + `last_seen_at` instead, which the endpoint does take. + source: spec + # Contact custom attributes are filtered as `custom_attributes.{name}` -- + # by name, unlike a ticket attribute, which carries a different id per + # ticket type. So the obstacle is not R7 here, it is measurement: the + # operator set Intercom answers on a custom attribute of each data type has + # not been probed, and this package does not publish a filter it has not + # seen work. They ship typed and display-only, and the probe is what turns + # that around. + custom_attributes: + filterable: false + reason: >- + Contact custom attributes are published for reading only: which + operators Intercom answers on `custom_attributes.{name}` has not been + measured on a real workspace, and this datasource does not offer a + filter it has not seen work. Filter on a native field instead. + source: spec + candidates: + - tag_ids + - segment_id + - company_id + - companies.id + - companies.name + - android_app_name + - android_last_seen_at + - ios_app_name + - ios_last_seen_at + - utm_source + - utm_campaign + - utm_medium + - referrer + - session_count diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/rate_limiter.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/rate_limiter.rb new file mode 100644 index 000000000..81fdc7aa9 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/rate_limiter.rb @@ -0,0 +1,148 @@ +module ForestAdminDatasourceIntercom + # Paces on Intercom's x-ratelimit-* headers. Workspace budget is 25 000/min but allocated in 10s windows + # (measured x-ratelimit-limit 1667), so the instantaneous rate matters. Paces, does not queue: callers + # woken at a reset can burst unmetered until the next response; the 429 retry behind this absorbs it. + class RateLimiter + # Intercom's allocation window. Only used as the ceiling below: the reset + # instant itself always comes from the response. + WINDOW = 10.0 + + # How far past the reset a request may be held. A little over one window, so + # a full window can be waited out, and no more: past this the wait is not + # Intercom's window emptying but a clock disagreeing. + DEFAULT_MAX_WAIT = 12.0 + + attr_reader :max_wait + + def initialize(max_wait: DEFAULT_MAX_WAIT, now: nil, sleeper: nil) + @max_wait = max_wait.to_f + # Wall clock rather than monotonic on purpose: `X-RateLimit-Reset` is an + # absolute epoch second on Intercom's clock, so the two have to be + # comparable. `clamp_wait` is what keeps a skewed clock from turning that + # comparison into a long sleep. + @now = now || -> { Time.now.to_f } + @sleeper = sleeper || ->(seconds) { sleep(seconds) } + @mutex = Mutex.new + @remaining = nil + @reset_at = nil + @limit = nil + @warned_at = nil + end + + # Blocks until the current window has room, then returns. Called once per + # attempt, retries included: a replayed request spends the budget a first + # one did. + def acquire + wait, declined = @mutex.synchronize { plan_wait } + + warn_saturated(declined) if declined + return if wait <= 0 + + @sleeper.call(wait) + end + + # What a response says about the window it was answered in. Called on every + # response, the 429 included -- that one carries the most useful reset of + # all. + def observe(headers) + limit = integer_header(headers, 'x-ratelimit-limit') + remaining = integer_header(headers, 'x-ratelimit-remaining') + reset_at = integer_header(headers, 'x-ratelimit-reset') + return if remaining.nil? && reset_at.nil? + + @mutex.synchronize { record(limit, remaining, reset_at) } + end + + private + + # Decrements locally so in-flight requests do not all pass on the same stale `remaining`; a + # `remaining` of 1 still allows this request. The declined wait is returned only once per window. + def plan_wait + unless exhausted? + @remaining -= 1 if @remaining + return [0, nil] + end + + wait = clamp_wait(@reset_at - @now.call) + return [0, nil] if wait <= 0 + return [0, first_warning? ? wait : nil] if wait > @max_wait + + [wait, nil] + end + + # Nothing left in a window that has not refilled yet. An unknown state -- + # before the first response -- is not exhaustion: the first request is what + # discovers the budget. + def exhausted? + !@remaining.nil? && @remaining <= 0 && !@reset_at.nil? + end + + # Intercom's reset is a timestamp from its clock, and the two clocks can + # disagree by more than the window is long. A wait longer than a window plus + # its own slack is that disagreement rather than a window emptying, so it is + # cut back to something a caller can afford to wait. + def clamp_wait(seconds) + return 0.0 if seconds <= 0 + + [seconds, WINDOW + @max_wait].min + end + + # A window is adopted whole. A response answered in an older window than the + # one already recorded is ignored: replies come back out of order, and one + # from the previous window would otherwise resurrect a budget already spent. + # + # Within the same window the smaller `remaining` wins, so the local + # decrements of in-flight requests are not undone by a response that left + # Intercom before they were made. + def record(limit, remaining, reset_at) + return if reset_at && @reset_at && reset_at < @reset_at + + new_window = reset_at && (@reset_at.nil? || reset_at > @reset_at) + @reset_at = reset_at if reset_at + @limit = limit if limit + return if remaining.nil? + + @remaining = new_window || @remaining.nil? ? remaining : [remaining, @remaining].min + end + + # One line per window. What the warning reports is a saturation that lasts, + # so a line per request puts one on every request it describes -- hundreds + # of them, burying the first, which is the only one the operator needed. + def first_warning? + now = @now.call + return false if @warned_at && now - @warned_at < WINDOW + + @warned_at = now + true + end + + def warn_saturated(wait) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] the Intercom rate-limit window is spent (limit #{@limit || "unknown"} " \ + "per #{WINDOW.round}s) and its reset is #{wait.round(1)}s out, past the #{@max_wait.round(1)}s this waits. " \ + 'Letting the request through -- Intercom may answer 429, which the client retries. A reset this far out ' \ + "usually means this host's clock disagrees with Intercom's. Reported once per #{WINDOW.round}s." + ) + end + + def integer_header(headers, name) + value = header_value(headers, name) + return nil if value.nil? || value.to_s.strip.empty? + + Integer(value.to_s.strip, exception: false) + end + + # Faraday hands over headers that look themselves up case-insensitively, but + # what reaches here is whatever the middleware was given -- a plain hash + # included -- and HTTP header names are case-insensitive on the wire. + def header_value(headers, name) + return nil if headers.nil? + + direct = headers[name] + return direct unless direct.nil? + return nil unless headers.respond_to?(:find) + + headers.find { |key, _value| key.to_s.downcase == name }&.last + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/retry_policy.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/retry_policy.rb new file mode 100644 index 000000000..499d4d756 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/retry_policy.rb @@ -0,0 +1,95 @@ +module ForestAdminDatasourceIntercom + # Everything governing how the client reacts to a failed request, in one + # place: which statuses and exceptions are worth another attempt, on which + # verbs, and how long to wait. + class RetryPolicy + # Intercom allocates its quota in 10-second windows, so a 429 is recovered + # from within one of them -- unlike an API metering by the minute. The cap + # still has to cover a whole window: faraday-retry gives up outright when + # Retry-After exceeds max_interval, which would turn the 429 retry into an + # immediate give-up exactly when it matters. + DEFAULT_MAX_INTERVAL = 12 + + STATUSES = [429, 500, 502, 503, 504].freeze + + # Named rather than referenced: the persistent adapter is a gem the client + # treats as optional -- it falls back to Faraday's default when the process + # cannot load it -- so this file cannot name the class outright. Both places + # that need it resolve it at the moment they are asked, and answer that it + # cannot have been raised when nothing defined it. + PERSISTENT_ADAPTER_ERROR = 'Net::HTTP::Persistent::Error'.freeze + + # faraday-retry's defaults plus ConnectionFailed, and the persistent adapter's own error, which it + # re-raises unwrapped for messages it does not translate. + EXCEPTIONS = [ + Errno::ETIMEDOUT, 'Timeout::Error', Faraday::TimeoutError, + Faraday::RetriableResponse, Faraday::ConnectionFailed, + PERSISTENT_ADAPTER_ERROR + ].freeze + + # The verbs that change nothing, so any transient failure is worth another + # attempt. Narrower than faraday-retry's idempotent default: a 502 or a + # dropped connection on the way back from a POST Intercom did perform would + # be replayed into a second reply on the conversation, or a second ticket. + RETRYABLE_METHODS = %i[get head options].freeze + + # The paths Intercom answers a *read* on through POST: its three search + # endpoints, and the one listing that paginates by offset. Nothing that + # writes is spelled this way -- a reply is `POST /conversations/{id}/reply`, + # a ticket `POST /tickets` -- so the exemption below cannot reach one. + READ_ONLY_POST_PATHS = %r{/(search|list)\z} + + # Widens `methods` (ORed by faraday-retry) to a 429 on any verb, and to a dropped connection on a + # read-only POST -- keep-alive makes an idle socket closed by the server an ordinary failure. + RETRY_IF = lambda do |env, exception| + env[:status] == 429 || (RetryPolicy.connection_failure?(exception) && + READ_ONLY_POST_PATHS.match?(env[:url]&.path.to_s)) + end + + # Whether another attempt would be dialling again rather than re-sending. + # See `PERSISTENT_ADAPTER_ERROR` for why one of the two is named. + def self.connection_failure?(exception) + return true if exception.is_a?(Faraday::ConnectionFailed) + return false unless Object.const_defined?(PERSISTENT_ADAPTER_ERROR) + + exception.is_a?(Object.const_get(PERSISTENT_ADAPTER_ERROR)) + end + + # The cap for a call that must not hold the boot, deliberately below a + # rate-limit window where DEFAULT_MAX_INTERVAL sits above it: a Retry-After + # past the cap makes faraday-retry abandon outright, which is what turns a + # 429 at boot into an immediate give-up rather than a window of waiting per + # attempt. + BOOT_MAX_INTERVAL = 2 + + BACKOFF_FACTOR = 2 + + attr_reader :max_retries, :interval, :max_interval + + # One retry rather than none, for what is read once and never revisited: a + # transient failure there costs its result for the whole life of the + # process, and half a second absorbs the hiccup without waiting a 429 out. + def self.boot + new(max_retries: 1, interval: 0.5, max_interval: BOOT_MAX_INTERVAL) + end + + def initialize(max_retries: 3, interval: 0.5, max_interval: DEFAULT_MAX_INTERVAL) + @max_retries = max_retries + @interval = interval + @max_interval = max_interval + end + + def to_faraday_options + { + max: @max_retries, + interval: @interval, + max_interval: @max_interval, + backoff_factor: BACKOFF_FACTOR, + retry_statuses: STATUSES, + exceptions: EXCEPTIONS, + methods: RETRYABLE_METHODS, + retry_if: RETRY_IF + } + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/attribute_naming.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/attribute_naming.rb new file mode 100644 index 000000000..baa3e8f28 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/attribute_naming.rb @@ -0,0 +1,52 @@ +module ForestAdminDatasourceIntercom + module Schema + # What the two introspectors do identically: turn the name a workspace gave + # an attribute into a name a Forest schema can carry, and say which + # attribute was left out when two of them land on one column. + # + # They differ in where the attributes come from -- `/ticket_types` per + # ticket type, `/data_attributes` per model -- and in the data types they + # map, which is why `COLUMN_TYPES` stays with each of them. + module AttributeNaming + # What a column name may not contain, and it has nothing to do with + # Intercom: Forest lists the fields of a request in a **comma-separated** + # query parameter, and uses a colon to name a field through a relation. + # A workspace names its attributes in free text -- measured, one is called + # `ID de l'objet en question (immo, facture, user)` -- and a comma in + # there splits the projection into fields no collection has, which the + # agent rejects as a 400 before the page is ever read. + UNSAFE_IN_A_COLUMN_NAME = /[,:]/ + + private + + # Intercom hands these back HTML-escaped -- `Ce que j'ai vérifié` -- + # which is an artefact of where they were typed, not part of the name. + def column_name_for(name) + CGI.unescapeHTML(name).gsub(UNSAFE_IN_A_COLUMN_NAME, ' ').squeeze(' ').strip + end + + # An unknown data type reads as a string rather than being dropped: + # showing the value Intercom sent beats hiding a column because its type + # is new. + def column_type_for(definition) + self.class::COLUMN_TYPES.fetch(definition['data_type'].to_s, self.class::DEFAULT_COLUMN_TYPE) + end + + # Two attributes reading as one column: the second is left out rather than + # sharing the first's entry, which would show its values under the first's + # name -- wrong values rather than missing ones, and worse. + # + # `attribute_kind` is the workspace's own vocabulary for these, which is + # what the operator has to go and rename. + def warn_collision(name, kept, column) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] the #{attribute_kind} attribute #{name.inspect} is left out: it " \ + "reads as the column #{column.inspect}, which #{kept.inspect} already carries. Rename one of them in " \ + 'Intercom to publish both.' + ) + end + + def attribute_kind = raise(NotImplementedError, "#{self.class} did not implement attribute_kind") + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/data_attributes_introspector.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/data_attributes_introspector.rb new file mode 100644 index 000000000..564d03deb --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/data_attributes_introspector.rb @@ -0,0 +1,85 @@ +module ForestAdminDatasourceIntercom + module Schema + # Declared per model, so each column maps to exactly one attribute. Still display-only: which operators + # Intercom answers on `custom_attributes.{name}` is unmeasured. `api_writable` is carried for lot 4b. + class DataAttributesIntrospector + include AttributeNaming + + # Intercom's attribute data types, mapped onto what Forest can render. + COLUMN_TYPES = { + 'string' => 'String', 'integer' => 'Number', 'float' => 'Number', 'decimal' => 'Number', + 'boolean' => 'Boolean', 'date' => 'Date', 'datetime' => 'Date' + }.freeze + + DEFAULT_COLUMN_TYPE = 'String'.freeze + + # `name` is the key `custom_attributes` uses, `column_name` the one the + # schema publishes; they differ whenever the workspace's own name cannot + # travel through Forest's query string. + Attribute = Struct.new(:name, :column_name, :column_type, :data_type, :api_writable, keyword_init: true) + + def initialize(client, model:) + @client = client + @model = model + end + + # Degrades to nothing rather than to a failure: a token without the + # permission on this model costs the custom-attribute columns, never the + # boot of the agent. + def attributes + @attributes ||= build + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] could not read the #{@model} attributes (HTTP #{e.status || "-"}); " \ + 'the collection boots without its custom-attribute columns.' + ) + @attributes = [] + end + + private + + def build + # Read on the boot connection: this happens while Rails is starting, and + # a slow Intercom must not turn that into minutes the operator sits + # through. + @client.fetch_all('data_attributes', params: { 'model' => @model }, boot: true) + .each_with_object({}) { |definition, union| collect(definition, union) } + .values + end + + # The standard attributes are left out: they are columns this datasource + # declares by hand, with the filters the search table measured, and + # publishing them a second time under their `custom_attributes` name would + # show one fact twice -- the unfilterable copy winning nothing. + def collect(definition, union) + return unless definition.is_a?(Hash) && definition['custom'] && !definition['archived'] + + name = definition['name'].to_s + return if name.empty? + + column = column_name_for(name) + return if column.empty? + + add(union, name, column, definition) + end + + def add(union, name, column, definition) + entry = union[column] + return union[column] = attribute_from(name, column, definition) if entry.nil? + return if entry.name == name + + warn_collision(name, entry.name, column) + end + + def attribute_from(name, column, definition) + Attribute.new(name: name, column_name: column, column_type: column_type_for(definition), + data_type: definition['data_type'], api_writable: definition['api_writable'] == true) + end + + # These are declared per model, so that is what the log calls them: it is + # the workspace's own vocabulary, and where the operator goes to rename + # the one that was left out. + def attribute_kind = @model + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/ticket_attributes_introspector.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/ticket_attributes_introspector.rb new file mode 100644 index 000000000..55f4dd428 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/ticket_attributes_introspector.rb @@ -0,0 +1,130 @@ +module ForestAdminDatasourceIntercom + module Schema + # Declared per ticket type: one name has a different id on each type (measured: 14162161 vs 14162165) and + # Intercom filters by id only, so the union columns ship unfilterable (R7). Ids are kept per type for a + # future per-type filter translation, sparing a second boot-time round trip. + class TicketAttributesIntrospector + include AttributeNaming + + # Intercom's attribute data types, mapped onto what Forest can render. A + # `list` is a single choice among values the workspace defined, so it reads + # as a string rather than as a Json blob; `files` is a list of attachments + # and has no scalar form at all. + COLUMN_TYPES = { + 'string' => 'String', 'list' => 'String', 'integer' => 'Number', 'decimal' => 'Number', + 'boolean' => 'Boolean', 'datetime' => 'Date', 'date' => 'Date', 'files' => 'Json' + }.freeze + + DEFAULT_COLUMN_TYPE = 'String'.freeze + + # `name` is the key the payload uses, `column_name` the one the schema + # publishes; they differ whenever the workspace's own name cannot travel + # through Forest's query string. + # + # `conflicting_ticket_types` names the ticket types declaring the same name + # with another type: their value would not be what the column holds. + Attribute = Struct.new(:name, :column_name, :column_type, :data_type, :ids_by_ticket_type, + :conflicting_ticket_types, keyword_init: true) + + def initialize(client) + @client = client + end + + # The union, one entry per attribute name. Degrades to nothing rather than + # to a failure: a token without the ticket-types permission costs the + # attribute columns, never the boot of the agent. + def attributes + @attributes ||= build + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] could not read the ticket types (HTTP #{e.status || "-"}); " \ + 'the Tickets collection boots without its attribute columns.' + ) + @attributes = [] + end + + private + + def build + # Read on the boot connection: this happens while Rails is starting, and + # a slow Intercom must not turn that into minutes the operator sits + # through. + @client.fetch_all('ticket_types', boot: true) + .each_with_object({}) { |ticket_type, union| collect(ticket_type, union) } + .values + end + + def collect(ticket_type, union) + return unless ticket_type.is_a?(Hash) + + type_id = ticket_type['id'].to_s + definitions(ticket_type).each do |definition| + entry = entry_for(definition, union, type_id) + next if entry.nil? + + entry.ids_by_ticket_type[type_id] = definition['id'].to_s + end + end + + # The union is keyed by column name rather than by the workspace's own, + # since that is what has to be unique in a schema. Two different attributes + # landing on one column would otherwise share an entry, and the second's + # values would be read under the first's name -- wrong values rather than + # missing ones, which is worse. + def entry_for(definition, union, type_id) + name = definition['name'].to_s + # An archived attribute is not offered any more, and a nameless one has + # nothing to be a column of. + return nil if name.empty? || definition['archived'] + + column = column_name_for(name) + return nil if column.empty? + + entry = union[column] + return union[column] = attribute_from(name, column, definition) if entry.nil? + return same_name(entry, definition, type_id) if entry.name == name + + warn_collision(name, entry.name, column) + nil + end + + # The same name declared with another type by another ticket type: that + # ticket type's value is left out of the column rather than shown under a + # type it does not have. + def same_name(entry, definition, type_id) + column_type = column_type_for(definition) + return entry if column_type == entry.column_type + + entry.conflicting_ticket_types << type_id + warn_type_conflict(entry, column_type, type_id) + nil + end + + def warn_type_conflict(entry, column_type, type_id) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] the ticket attribute #{entry.name.inspect} is a #{column_type} on " \ + "ticket type #{type_id} and a #{entry.column_type} elsewhere; the column stays a #{entry.column_type} " \ + 'and reads as empty on that ticket type. Give the two attributes different names to publish both.' + ) + end + + def attribute_from(name, column, definition) + Attribute.new(name: name, column_name: column, column_type: column_type_for(definition), + data_type: definition['data_type'], ids_by_ticket_type: {}, conflicting_ticket_types: []) + end + + def definitions(ticket_type) + container = ticket_type['ticket_type_attributes'] + return [] unless container.is_a?(Hash) + + list = container['data'] + list.is_a?(Array) ? list.grep(Hash) : [] + end + + # These are declared per ticket type, and that is what the log calls + # them: the workspace's own vocabulary, and where the operator goes to + # rename the one that was left out. + def attribute_kind = 'ticket' + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/throttle.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/throttle.rb new file mode 100644 index 000000000..fa8fdad3b --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/throttle.rb @@ -0,0 +1,19 @@ +module ForestAdminDatasourceIntercom + # Holds a request until the rate-limit window has room, and feeds the window + # back what the response says about it. A middleware rather than a call in + # each client method: there is one code path for every request here, where the + # client has one per endpoint, and this one also covers the requests the client + # never issues itself -- the replays `retry` performs. + class Throttle < Faraday::Middleware + def initialize(app, limiter:) + super(app) + @limiter = limiter + end + + def call(env) + @limiter.acquire + + @app.call(env).on_complete { |response_env| @limiter.observe(response_env[:response_headers]) } + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb new file mode 100644 index 000000000..bcf5c5876 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb @@ -0,0 +1,3 @@ +module ForestAdminDatasourceIntercom + VERSION = "0.1.0" +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/cache_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/cache_spec.rb new file mode 100644 index 000000000..b03ff4109 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/cache_spec.rb @@ -0,0 +1,110 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Cache do + subject(:cache) { described_class.new(ttl: 60, now: -> { clock[:seconds] }) } + + # A clock the examples move by hand: the store measures an elapsed span, and + # a spec that slept through one would be a spec nobody runs. + let(:clock) { { seconds: 1_000.0 } } + + it 'reads through on the first miss' do + expect(cache.fetch('key') { 'answer' }).to eq('answer') + end + + it 'answers the second read from the store' do + cache.fetch('key') { 'first' } + + expect(cache.fetch('key') { 'second' }).to eq('first') + end + + it 'tells two keys apart' do + cache.fetch('a') { 'first' } + + expect(cache.fetch('b') { 'second' }).to eq('second') + end + + it 'holds an entry for the whole window' do + cache.fetch('key') { 'first' } + clock[:seconds] += 59 + + expect(cache.fetch('key') { 'second' }).to eq('first') + end + + it 'reads the source again once the window has passed' do + cache.fetch('key') { 'first' } + clock[:seconds] += 61 + + expect(cache.fetch('key') { 'second' }).to eq('second') + end + + # A failure is not an answer: caching one would keep an outage alive for the + # length of the window. + it 'stores nothing when the block raises' do + expect { cache.fetch('key') { raise APIError, 'boom' } }.to raise_error(APIError) + + expect(cache.fetch('key') { 'answer' }).to eq('answer') + end + + it 'stores a nil answer rather than reading through it again' do + nothing = nil + cache.fetch('key') { nothing } + + expect(cache.fetch('key') { 'second' }).to be_nil + end + + it 'forgets everything on clear' do + cache.fetch('key') { 'first' } + cache.clear + + expect(cache.fetch('key') { 'second' }).to eq('second') + end + + # The block runs outside the mutex, so a read can still be in flight when + # the store is dropped -- which is what calling `clear` from inside one + # stands for here. Its answer is what Intercom held *before* the write the + # `clear` was called for, and storing it would hold that stale answer for a + # further window: exactly what `clear` exists to prevent. + it 'drops the write of a read that started before clear' do + cache.fetch('key') do + cache.clear + 'read before the clear' + end + + expect(cache.fetch('key') { 'read after it' }).to eq('read after it') + end + + # And the store is usable again afterwards: the generation guard refuses the + # writes that predate the clear, not every write that follows it. + it 'holds what is read after a clear' do + cache.fetch('key') { 'first' } + cache.clear + cache.fetch('key') { 'second' } + + expect(cache.fetch('key') { 'third' }).to eq('second') + end + + # Expired entries are dropped on the way past rather than by a sweeper, so + # a store written to for hours does not grow by one entry per window. + it 'drops expired entries as it writes' do + cache.fetch('a') { 'first' } + clock[:seconds] += 61 + cache.fetch('b') { 'second' } + + expect(cache.instance_variable_get(:@entries).keys).to eq(['b']) + end + + # The escape hatch a deployment sets when it would rather pay the request. + context 'with a ttl of zero' do + subject(:cache) { described_class.new(ttl: 0) } + + it 'is disabled' do + expect(cache).not_to be_enabled + end + + it 'reads through every time' do + cache.fetch('key') { 'first' } + + expect(cache.fetch('key') { 'second' }).to eq('second') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/client_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/client_spec.rb new file mode 100644 index 000000000..fd81a4f6f --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/client_spec.rb @@ -0,0 +1,855 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Client do + subject(:client) { described_class.new(configuration) } + + let(:retry_policy) { RetryPolicy.new(max_retries: 2, interval: 0) } + let(:configuration) { Configuration.new(access_token: 's3cr3t', retry_policy: retry_policy, rate_limiter: nil) } + let(:base) { configuration.url } + + def json(payload, status = 200, headers = {}) + { status: status, + body: payload.is_a?(String) ? payload : payload.to_json, + headers: { 'Content-Type' => 'application/json' }.merge(headers) } + end + + describe 'authentication and version pinning' do + before { stub_request(:get, "#{base}/me").to_return(json({ 'type' => 'admin' })) } + + it 'sends the access token as a bearer token' do + client.me + + expect(WebMock).to have_requested(:get, "#{base}/me") + .with(headers: { 'Authorization' => 'Bearer s3cr3t', 'Accept' => 'application/json' }) + end + + # Without the header the request follows the workspace's own default + # version, which an operator can change on Intercom's side. + it 'pins the API version on every request' do + client.me + + expect(WebMock).to have_requested(:get, "#{base}/me").with(headers: { 'Intercom-Version' => '2.16' }) + end + + it 'advertises a versioned user agent' do + client.me + + expect(WebMock).to have_requested(:get, "#{base}/me") + .with(headers: { 'User-Agent' => "forest_admin_datasource_intercom/#{VERSION}" }) + end + end + + describe '#me' do + it 'returns the admin the token belongs to' do + stub_request(:get, "#{base}/me").to_return(json('type' => 'admin', 'id' => '1', 'email' => 'a@b.test')) + + expect(client.me).to include('id' => '1', 'email' => 'a@b.test') + end + + it 'reaches the regional host it was configured for' do + eu = described_class.new(Configuration.new(access_token: 's3cr3t', region: :eu, rate_limiter: nil)) + stub_request(:get, 'https://api.eu.intercom.io/me').to_return(json('type' => 'admin')) + + eu.me + + expect(WebMock).to have_requested(:get, 'https://api.eu.intercom.io/me') + end + end + + describe 'the version Intercom actually served' do + # Intercom echoes the version it served. A mismatch means the payloads may + # not be the ones this datasource expects, which is worth saying out loud + # -- and worth saying rather than raising: running against a version we + # did not ask for beats not running. + it 'warns when it differs from the pinned one' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/me").to_return(json({ 'type' => 'admin' }, 200, 'intercom-version' => '2.14')) + + client.me + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/asked.*2\.16.*served 2\.14/m) + end + + it 'stays quiet when the pin was honoured' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/me").to_return(json({ 'type' => 'admin' }, 200, 'intercom-version' => '2.16')) + + client.me + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + + it 'stays quiet when Intercom echoes nothing' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/me").to_return(json('type' => 'admin')) + + client.me + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + end + + describe 'failures' do + it "carries Intercom's status, parsed body and error text" do + body = { 'type' => 'error.list', 'request_id' => 'req_1', + 'errors' => [{ 'code' => 'unauthorized', 'message' => 'Access Token Invalid' }] } + stub_request(:get, "#{base}/me").to_return(json(body, 401)) + + expect { client.me }.to raise_error(APIError) { |error| + expect(error.message).to eq('Intercom API call failed: me: HTTP 401 unauthorized: ' \ + 'Access Token Invalid (request_id: req_1)') + expect(error.status).to eq(401) + expect(error.body).to eq(body) + } + end + + it 'joins the several errors one response can carry' do + body = { 'errors' => [{ 'code' => 'parameter_invalid', 'message' => 'per_page' }, + { 'code' => 'parameter_invalid', 'message' => 'starting_after' }] } + stub_request(:get, "#{base}/me").to_return(json(body, 400)) + + expect { client.me }.to raise_error(APIError, /per_page; parameter_invalid: starting_after/) + end + + it 'falls back to the whole body when the shape is not the documented one' do + stub_request(:get, "#{base}/me").to_return(json({ 'oops' => true }, 500)) + + expect { client.me }.to raise_error(APIError, /\{"oops":true\}/) + end + + it 'keeps a body that is not JSON at all, which is what a gateway answers' do + stub_request(:get, "#{base}/me").to_return(status: 502, body: 'bad gateway') + + expect { client.me }.to raise_error(APIError) { |error| + expect(error.status).to eq(502) + expect(error.body).to eq('bad gateway') + } + end + + # No status to report: the request never reached Intercom, so there is + # nothing of its to surface. + it 'reports a dropped connection without a status' do + stub_request(:get, "#{base}/me").to_raise(Faraday::ConnectionFailed.new('closed')) + + expect { client.me }.to raise_error(APIError) { |error| + expect(error.message).to include('Faraday::ConnectionFailed') + expect(error.status).to be_nil + } + end + + it 'replays a 429 rather than surfacing it' do + stub_request(:get, "#{base}/me") + .to_return(json({ 'errors' => [{ 'code' => 'rate_limit_exceeded' }] }, 429)) + .then.to_return(json('type' => 'admin')) + + expect(client.me).to eq('type' => 'admin') + end + + it 'gives up on a 429 that outlasts the retries, saying which endpoint' do + stub_request(:get, "#{base}/me").to_return(json({ 'errors' => [{ 'code' => 'rate_limit_exceeded' }] }, 429)) + + expect { client.me }.to raise_error(APIError, /me: HTTP 429 rate_limit_exceeded/) + end + + # A body that failed to parse is a payload, not an error: on a 200 it is + # customer content, and this message is shown in the interface and + # collected by whatever watches the agent (R10). + it 'names a body it could not read rather than quoting it' do + stub_request(:get, "#{base}/me") + .to_return(json('Bonjour, voici mon RIB FR76 3000 4000 0500 0012 3456 789')) + + expect { client.me }.to raise_error(APIError) { |error| + expect(error.message).to eq('Intercom API call failed: me: the response could not be read as JSON') + # Faraday hands a parsing error an unfinished response, so there is no + # body to keep here -- which suits this one: the point is that the + # payload does not travel with the error. + expect(error.body).to be_nil + } + end + + # The same guard, one level down: a parser raising on its own would + # otherwise reach the catch-all, whose message is the exception's -- and + # a JSON parser quotes what it choked on. + it 'says as little when a parser raises outside Faraday' do + stub_request(:get, "#{base}/me").to_raise(JSON::ParserError.new("unexpected token 'mon RIB FR76'")) + + expect { client.me }.to raise_error(APIError) { |error| + expect(error.message).to include('could not be read as JSON') + expect(error.message).not_to include('RIB') + } + end + + # Whatever else goes wrong on the way, a caller of this client only ever + # has to rescue APIError -- and the message names the operation, since a + # failure with no endpoint in it is a failure nobody can place. + it 'still names the operation when the failure is not one it expected' do + stub_request(:get, "#{base}/me").to_raise(ArgumentError.new('unexpected')) + + expect { client.me }.to raise_error(APIError, /me: ArgumentError: unexpected/) + end + + # Ruby chains the rescued error as `cause`, which `full_message` and error + # trackers print: the parser's message quotes the payload, and Faraday's + # error carries the request, bearer token included. + it 'chains neither the payload nor the token through the cause' do + stub_request(:get, "#{base}/me") + .to_return(json('Bonjour, voici mon RIB FR76 3000 4000 0500 0012 3456 789')) + + expect { client.me }.to raise_error(APIError) { |error| + expect(error.cause).to be_nil + expect(error.full_message).not_to include('RIB') + } + end + + it 'chains no cause on an HTTP failure either' do + stub_request(:get, "#{base}/me").to_return(json({ 'errors' => [{ 'code' => 'unauthorized' }] }, 401)) + + expect { client.me }.to raise_error(APIError) { |error| + expect(error.cause).to be_nil + expect(error.full_message).not_to include('s3cr3t') + } + end + + it 'chains no cause from the catch-all' do + stub_request(:get, "#{base}/me").to_raise(ArgumentError.new('unexpected')) + + expect { client.me }.to raise_error(APIError) { |error| expect(error.cause).to be_nil } + end + end + + describe '#list_page' do + def list_body(data, next_page: nil, total: 2) + body = { 'type' => 'list', 'data' => data, 'total_count' => total, + 'pages' => { 'type' => 'pages', 'page' => 1, 'per_page' => 50 } } + body['pages']['next'] = next_page unless next_page.nil? + body + end + + it 'reads the records, the next cursor and the exact count off one response' do + body = list_body([{ 'id' => '1' }], next_page: { 'starting_after' => 'cursor_2' }) + stub_request(:get, "#{base}/conversations").with(query: { 'per_page' => '150' }).to_return(json(body)) + + page = client.list_page('conversations', per_page: 150) + + expect(page.records).to eq([{ 'id' => '1' }]) + expect(page.next_cursor).to eq('cursor_2') + expect(page.total_count).to eq(2) + end + + it 'sends the cursor the previous page advertised' do + stub_request(:get, "#{base}/conversations") + .with(query: { 'per_page' => '50', 'starting_after' => 'cursor_2' }) + .to_return(json(list_body([]))) + + client.list_page('conversations', per_page: 50, starting_after: 'cursor_2') + + expect(WebMock).to have_requested(:get, "#{base}/conversations") + .with(query: { 'per_page' => '50', 'starting_after' => 'cursor_2' }) + end + + it 'bounds the page size before sending it, Intercom refusing rather than clamping' do + stub_request(:get, "#{base}/conversations").with(query: { 'per_page' => '150' }) + .to_return(json(list_body([]))) + + client.list_page('conversations', per_page: 500) + + expect(WebMock).to have_requested(:get, "#{base}/conversations").with(query: { 'per_page' => '150' }) + end + + it 'carries the parameters an endpoint of its own needs' do + stub_request(:get, "#{base}/conversations") + .with(query: { 'per_page' => '150', 'display_as' => 'plaintext' }).to_return(json(list_body([]))) + + client.list_page('conversations', per_page: 150, params: { 'display_as' => 'plaintext' }) + + expect(WebMock).to have_requested(:get, "#{base}/conversations") + .with(query: hash_including('display_as' => 'plaintext')) + end + + # The last page simply carries no `pages.next`, which is what stops a walk. + it 'reports no next cursor on the last page' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json(list_body([{ 'id' => '1' }]))) + + expect(client.list_page('conversations', per_page: 150).next_cursor).to be_nil + end + + # An older API version spells `pages.next` as a url, and one can be served + # despite the pin -- reading the cursor out of it beats taking the page for + # the last one and truncating the answer. + it 'reads the cursor out of a next page spelled as a url' do + url = "#{base}/conversations?per_page=50&starting_after=cursor_9" + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json(list_body([], next_page: url))) + + expect(client.list_page('conversations', per_page: 50).next_cursor).to eq('cursor_9') + end + + # An advertised page taken for the last one is a silently truncated + # answer, so every unreadable shape is refused rather than dropped. + it 'refuses a next-page url carrying no cursor' do + body = list_body([], next_page: "#{base}/conversations?per_page=50") + stub_request(:get, "#{base}/conversations").with(query: hash_including({})).to_return(json(body)) + + expect { client.list_page('conversations', per_page: 50) } + .to raise_error(APIError, /pages\.next' carries no cursor/) + end + + it 'refuses a next-page url it cannot parse' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json(list_body([], next_page: 'http://[bad'))) + + expect { client.list_page('conversations', per_page: 50) } + .to raise_error(APIError, /pages\.next' carries no cursor/) + end + + it 'refuses a next page it can read neither way, rather than truncating silently' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json(list_body([], next_page: 42))) + + expect { client.list_page('conversations', per_page: 50) } + .to raise_error(APIError, /unexpected response shape.*pages\.next/m) + end + + # `Array()` on the envelope would hand the collection rows built out of + # [key, value] pairs: a page that looks answered and holds nothing. + it 'refuses a response whose data is not a list' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json({ 'type' => 'list', 'data' => { 'id' => '1' } })) + + expect { client.list_page('conversations', per_page: 50) } + .to raise_error(APIError, /unexpected response shape.*'data' is not a list/m) + end + + it 'refuses a response carrying no data at all' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json({ 'type' => 'list', 'total_count' => 0 })) + + expect { client.list_page('conversations', per_page: 50) }.to raise_error(APIError, /'data' is not a list/) + end + + it 'serves an empty page as an empty page, zero being an answer' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json(list_body([], total: 0))) + + expect(client.list_page('conversations', per_page: 50)) + .to have_attributes(records: [], next_cursor: nil, total_count: 0) + end + + # nil rather than 0: zero is an answer, and this is the absence of one. + it 'reports no count when Intercom sends none' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json({ 'type' => 'list', 'data' => [] })) + + expect(client.list_page('conversations', per_page: 50).total_count).to be_nil + end + + it 'reads a page through the boot connection when asked to' do + stub_request(:get, "#{base}/ticket_types").with(query: hash_including({})) + .to_return(json(list_body([{ 'id' => '1' }]))) + + expect(client.list_page('ticket_types', per_page: 50, boot: true).records.size).to eq(1) + end + + it 'names the endpoint when the read fails' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json({ 'errors' => [{ 'code' => 'not_found' }] }, 404)) + + expect { client.list_page('conversations', per_page: 50) } + .to raise_error(APIError, /conversations: HTTP 404 not_found/) + end + end + + describe '#fetch_all' do + it 'reads the records under the key the endpoint uses' do + stub_request(:get, "#{base}/admins") + .to_return(json('type' => 'admin.list', 'admins' => [{ 'id' => '1' }, { 'id' => '2' }])) + + expect(client.fetch_all('admins', list_key: 'admins').size).to eq(2) + end + + # Intercom is not consistent about it: /admins and /teams use their own + # key, /ticket_types the `data` envelope every paginated listing uses. + it 'falls back to the data envelope' do + stub_request(:get, "#{base}/ticket_types").to_return(json('type' => 'list', 'data' => [{ 'id' => '1' }])) + + expect(client.fetch_all('ticket_types')).to eq([{ 'id' => '1' }]) + end + + it 'asks for no page: these endpoints answer whole' do + stub_request(:get, "#{base}/teams").to_return(json('teams' => [])) + + client.fetch_all('teams', list_key: 'teams') + + expect(WebMock).to have_requested(:get, "#{base}/teams").with(query: {}) + end + + # A reference collection read as empty is a state column with no values and + # an assignee shown as a raw id -- worse than a failure naming the shape. + it 'refuses a response holding neither key' do + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => { 'id' => '1' })) + + expect { client.fetch_all('admins', list_key: 'admins') } + .to raise_error(APIError, /neither 'admins' nor 'data' is a list/) + end + + it 'reads an empty body as no record' do + stub_request(:get, "#{base}/admins").to_return(status: 200, body: '') + + expect(client.fetch_all('admins', list_key: 'admins')).to eq([]) + end + + # No pagination parameter in the specification is not a promise that a + # large workspace answers in one response, and a truncated reference + # collection would show an operator a state list missing its last states. + it 'follows a cursor if one is advertised anyway' do + stub_request(:get, "#{base}/tags").with(query: {}) + .to_return(json('data' => [{ 'id' => '1' }], + 'pages' => { 'next' => { 'starting_after' => 'c2' } })) + stub_request(:get, "#{base}/tags").with(query: { 'starting_after' => 'c2' }) + .to_return(json('data' => [{ 'id' => '2' }])) + + expect(client.fetch_all('tags').map { |tag| tag['id'] }).to eq(%w[1 2]) + end + + it 'stops at its page cap and says what it left out' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/tags").with(query: hash_including({})) + .to_return(json('data' => [{ 'id' => '1' }], + 'pages' => { 'next' => { 'starting_after' => 'c' } })) + + client.fetch_all('tags') + + expect(WebMock).to have_requested(:get, "#{base}/tags") + .with(query: hash_including({})).times(described_class::MAX_COLLECTED_PAGES) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/Stopped reading tags/) + end + + it 'reads through the boot connection when asked to' do + stub_request(:get, "#{base}/ticket_types").to_return(json('data' => [])) + + expect(client.fetch_all('ticket_types', boot: true)).to eq([]) + end + + it 'names the endpoint when the read fails' do + stub_request(:get, "#{base}/admins").to_return(json({ 'errors' => [{ 'code' => 'forbidden' }] }, 403)) + + expect { client.fetch_all('admins', list_key: 'admins') } + .to raise_error(APIError, /admins: HTTP 403 forbidden/) + end + end + + describe '.bounded_per_page' do + # Intercom answers `invalid_per_page` past 150 instead of clamping, so a + # page size is bounded before it is sent or the list view breaks. + it 'caps a page size at what Intercom accepts' do + expect(described_class.bounded_per_page(200)).to eq(150) + end + + it 'leaves an acceptable size alone' do + expect(described_class.bounded_per_page(50)).to eq(50) + end + + it 'asks for one record rather than none, an empty page being no answer' do + expect([described_class.bounded_per_page(0), described_class.bounded_per_page(-5)]).to eq([1, 1]) + end + end + + describe 'the boot connection' do + # What is read while the datasource is being constructed waits far less + # than a request that already has a page on screen: the wait there is + # minutes of Rails boot the operator sits through. + it 'honours the configured boot timeouts' do + booted = described_class.new(Configuration.new(access_token: 's3cr3t', boot_open_timeout: 1, boot_timeout: 2)) + conn = booted.send(:boot_connection) + + expect(conn.options).to have_attributes(open_timeout: 1, timeout: 2) + end + + it 'keeps the patience of a regular request on the regular connection' do + expect(client.send(:connection).options).to have_attributes(open_timeout: 5, timeout: 30) + end + + it 'reads through it when asked to' do + stub_request(:get, "#{base}/me").to_return(json('type' => 'admin')) + + expect(client.me(boot: true)).to eq('type' => 'admin') + end + end + + describe 'pacing' do + let(:limiter) { instance_double(RateLimiter, acquire: nil, observe: nil) } + let(:paced) do + described_class.new(Configuration.new(access_token: 's3cr3t', retry_policy: retry_policy, + rate_limiter: limiter)) + end + + it 'asks the limiter for room, and feeds it the window back' do + stub_request(:get, "#{base}/me").to_return(json({ 'type' => 'admin' }, 200, + 'x-ratelimit-remaining' => '1666')) + + paced.me + + expect(limiter).to have_received(:acquire) + expect(limiter).to have_received(:observe).with(hash_including('x-ratelimit-remaining' => '1666')) + end + + # The throttle sits inside the retry, so a replay waits for the window + # like a first attempt rather than going out on a budget already spent. + it 'asks again for every replay, not once per call' do + stub_request(:get, "#{base}/me") + .to_return(json({}, 429)).then.to_return(json('type' => 'admin')) + + paced.me + + expect(limiter).to have_received(:acquire).twice + end + end + + # The workspace's own lists, held for the configured window rather than + # re-read by every page that resolves a relation through one of them. + describe '#fetch_all caching' do + before { stub_request(:get, "#{base}/teams").to_return(json('teams' => [{ 'id' => '1' }])) } + + it 'reads the endpoint once within the window' do + 2.times { client.fetch_all('teams', list_key: 'teams') } + + expect(WebMock).to have_requested(:get, "#{base}/teams").once + end + + it 'answers the second read with the same records' do + first = client.fetch_all('teams', list_key: 'teams') + + expect(client.fetch_all('teams', list_key: 'teams')).to eq(first) + end + + # `boot` picks the connection a read travels on -- shorter timeouts while + # the agent starts -- not what comes back, so the ticket types read at boot + # are the ticket types a list view reads. + it 'shares one entry between the boot read and the later one' do + client.fetch_all('teams', list_key: 'teams', boot: true) + client.fetch_all('teams', list_key: 'teams') + + expect(WebMock).to have_requested(:get, "#{base}/teams").once + end + + # `/data_attributes` answers the contact attributes and the company ones + # under `?model=`, and both are read whole. + it 'tells two reads of one endpoint apart by their params' do + stub_request(:get, "#{base}/data_attributes").with(query: hash_including({})) + .to_return(json('data' => [])) + + client.fetch_all('data_attributes', params: { 'model' => 'contact' }) + client.fetch_all('data_attributes', params: { 'model' => 'company' }) + + expect(WebMock).to have_requested(:get, "#{base}/data_attributes").with(query: { 'model' => 'contact' }) + expect(WebMock).to have_requested(:get, "#{base}/data_attributes").with(query: { 'model' => 'company' }) + end + + it 'hands back a frozen list, nothing downstream owning what the next page reads' do + expect(client.fetch_all('teams', list_key: 'teams')).to be_frozen + end + + # A 403 rather than a 500: the latter is retried, and the retry would eat + # the answer this example checks is read afresh. + it 'caches no failure' do + stub_request(:get, "#{base}/admins").to_return(json({}, 403), json('admins' => [{ 'id' => '1' }])) + + expect { client.fetch_all('admins', list_key: 'admins') }.to raise_error(APIError) + expect(client.fetch_all('admins', list_key: 'admins').size).to eq(1) + end + + context 'with the store off' do + let(:configuration) do + Configuration.new(access_token: 's3cr3t', retry_policy: retry_policy, rate_limiter: nil, + reference_cache_ttl: 0) + end + + it 'reads the endpoint on every call' do + 2.times { client.fetch_all('teams', list_key: 'teams') } + + expect(WebMock).to have_requested(:get, "#{base}/teams").twice + end + end + end + + # The other half: the same read issued twice while a single page is being + # built, from two callers that cannot see each other. + describe '#with_read_scope' do + before do + stub_request(:post, "#{base}/contacts/search").to_return(json('data' => [{ 'id' => 'c1' }])) + stub_request(:get, %r{#{base}/contacts/c1}).to_return(json('id' => 'c1')) + end + + def search + client.search_page('contacts/search', query: { 'field' => 'id', 'operator' => 'IN', 'value' => ['c1'] }, + per_page: 1) + end + + it 'issues one request for two identical searches' do + client.with_read_scope { 2.times { search } } + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").once + end + + it 'answers the second search with the first page' do + pages = client.with_read_scope { [search, search] } + + expect(pages.first.records).to eq(pages.last.records) + end + + it 'issues one request for two reads of one record' do + client.with_read_scope { 2.times { client.fetch_record('contacts', 'c1') } } + + expect(WebMock).to have_requested(:get, "#{base}/contacts/c1").once + end + + it 'tells two searches apart by their query' do + client.with_read_scope do + search + client.search_page('contacts/search', query: { 'field' => 'id', 'operator' => 'IN', 'value' => ['c2'] }, + per_page: 1) + end + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").twice + end + + it 'remembers nothing outside a scope' do + 2.times { search } + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").twice + end + + # A page is one scope, and the next page is another: what travels here is + # the customer's own records, and holding them would show an operator a row + # they have just edited in its previous state. + it 'remembers nothing from one scope to the next' do + 2.times { client.with_read_scope { search } } + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").twice + end + + # A related list resolves relations of its own, and the inner `ensure` + # would otherwise close the scope the outer read is still building in. + it 'joins the scope already open rather than opening a second one' do + client.with_read_scope do + search + client.with_read_scope { search } + search + end + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").once + end + + it 'closes the scope even when the page fails' do + expect { client.with_read_scope { raise APIError, 'boom' } }.to raise_error(APIError) + + 2.times { search } + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").twice + end + + it 'remembers no failure' do + stub_request(:post, "#{base}/companies/search").to_return(json({}, 500), json('data' => [])) + + client.with_read_scope do + expect { client.search_page('companies/search', query: {}, per_page: 1) }.to raise_error(APIError) + client.search_page('companies/search', query: {}, per_page: 1) + end + + expect(WebMock).to have_requested(:post, "#{base}/companies/search").twice + end + end + + # Seven requests to one host is seven TLS handshakes on the default adapter. + describe 'connection reuse' do + it 'builds its connections on the persistent adapter' do + stub_request(:get, "#{base}/me").to_return(json('type' => 'admin')) + client.me + + expect(WebMock).to have_requested(:get, "#{base}/me").with(headers: { 'Connection' => 'keep-alive' }) + end + + it 'refuses at boot an adapter this process cannot load, naming it' do + expect do + described_class.new(Configuration.new(access_token: 's3cr3t', rate_limiter: nil, adapter: :no_such_adapter)) + end.to raise_error(ConfigurationError, /adapter :no_such_adapter cannot be loaded/) + end + + it 'takes the adapter the configuration names' do + configured = described_class.new( + Configuration.new(access_token: 's3cr3t', rate_limiter: nil, adapter: :net_http) + ) + stub_request(:get, "#{base}/me").to_return(json('type' => 'admin')) + + configured.me + + expect(WebMock).to have_requested(:get, "#{base}/me") + end + + # Faraday looks a name up by symbol: a string has to be normalised to + # reach the persistent adapter. + it 'keeps the connection alive for a name spelled as a string' do + configured = described_class.new( + Configuration.new(access_token: 's3cr3t', rate_limiter: nil, adapter: 'net_http_persistent') + ) + stub_request(:get, "#{base}/me").to_return(json('type' => 'admin')) + + configured.me + + expect(WebMock).to have_requested(:get, "#{base}/me").with(headers: { 'Connection' => 'keep-alive' }) + end + end + + # What a pooled connection costs: a socket the server closed while it was + # idle fails the next request that reuses it, where a client opening one per + # request could not meet the case. On a GET the retry policy already + # absorbed it; the searches every list view is built on travel on POST. + describe 'a connection dropped under keep-alive' do + it 'retries a search, which reads despite travelling on POST' do + stub_request(:post, "#{base}/contacts/search") + .to_raise(Faraday::ConnectionFailed).then + .to_return(json('data' => [{ 'id' => 'c1' }])) + + page = client.search_page('contacts/search', query: {}, per_page: 1) + + expect(page.records.size).to eq(1) + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").twice + end + + # The other read Intercom answers on POST, and the reason the exemption is + # scoped to paths rather than to the verb. + it 'retries the offset listing' do + stub_request(:post, "#{base}/companies/list").with(query: hash_including({})) + .to_raise(Faraday::ConnectionFailed).then + .to_return(json('data' => [])) + + expect(client.offset_page('companies/list', page: 1, per_page: 1).records).to eq([]) + expect(WebMock).to have_requested(:post, "#{base}/companies/list") + .with(query: hash_including({})).twice + end + + # The persistent adapter re-raises its own error for what it does not + # recognise -- a host found down while a pooled connection is reset -- so + # the policy names the class it cannot reference. Required here rather + # than at the top of the file: nothing guarantees the optional gem is + # loaded before this example runs. + it 'retries the adapter error a dropped connection can surface as' do + require 'net/http/persistent' + stub_request(:get, "#{base}/me") + .to_raise(Net::HTTP::Persistent::Error.new('host down: api.intercom.io:443')).then + .to_return(json('type' => 'admin')) + + client.me + + expect(WebMock).to have_requested(:get, "#{base}/me").twice + end + end + + describe '#inspect' do + it 'never prints the token its connections carry' do + expect(client.inspect).to include(base) + expect(client.inspect).not_to include('s3cr3t') + end + end + + describe 'reading records by id' do + def stub_record(id, status = 200) + stub_request(:get, "#{base}/companies/#{id}") + .to_return(json(status == 200 ? { 'type' => 'company', 'id' => id } : { 'type' => 'error.list' }, status)) + end + + # A response that waits until `width` requests are in flight together, and + # says whether they were: the only way to tell reads issued at once from + # reads issued one after another. + def stub_overlapping(ids, width, wait: 2) + lock = Mutex.new + all_in = ConditionVariable.new + in_flight = 0 + + ids.each do |id| + stub_request(:get, "#{base}/companies/#{id}").to_return do + overlapped = lock.synchronize do + in_flight += 1 + all_in.broadcast + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + wait + while in_flight < width && (left = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)).positive? + all_in.wait(lock, left) + end + in_flight >= width + end + json('type' => 'company', 'id' => id, 'overlapped' => overlapped) + end + end + end + + it 'answers in the order of the ids, with nil for a record Intercom no longer answers' do + stub_record('a') + stub_record('gone', 404) + stub_record('b') + + expect(client.fetch_records('companies', %w[a gone b]).map { |record| record&.fetch('id') }) + .to eq(['a', nil, 'b']) + end + + it 'reads them one at a time by default' do + stub_overlapping(%w[a b], 2, wait: 0.2) + + expect(client.fetch_records('companies', %w[a b]).map { |record| record['overlapped'] }).to eq([false, true]) + end + + it 'reads nothing the page already read, and keeps what it read for the rest of the page' do + stub_record('a') + stub_record('b') + + client.with_read_scope do + client.fetch_record('companies', 'a') + client.fetch_records('companies', %w[a b]) + client.fetch_record('companies', 'b') + end + + expect(WebMock).to have_requested(:get, "#{base}/companies/a").once + expect(WebMock).to have_requested(:get, "#{base}/companies/b").once + end + + context 'with concurrent reads' do + let(:configuration) do + Configuration.new(access_token: 's3cr3t', retry_policy: retry_policy, rate_limiter: nil, + concurrent_reads: 3) + end + + it 'has that many requests in flight at once' do + stub_overlapping(%w[a b c], 3) + + expect(client.fetch_records('companies', %w[a b c]).map { |record| record['overlapped'] }) + .to eq([true, true, true]) + end + + it 'still answers in the order of the ids' do + stub_overlapping(%w[a b c], 3) + + expect(client.fetch_records('companies', %w[c a b]).map { |record| record['id'] }).to eq(%w[c a b]) + end + + it 'reads a record named twice once' do + stub_record('a') + + client.fetch_records('companies', %w[a a]) + + expect(WebMock).to have_requested(:get, "#{base}/companies/a").once + end + + # Whatever order the responses came back in, the failure raised is the + # one the earliest id met, and no worker outlives the call. + it 'raises the failure of the earliest id once every worker is done' do + stub_record('a') + stub_record('b', 403) + stub_record('c', 404) + threads = Thread.list.size + + expect { client.fetch_records('companies', %w[a b c]) } + .to raise_error(APIError) { |error| expect(error.status).to eq(403) } + expect(Thread.list.size).to eq(threads) + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/admin_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/admin_spec.rb new file mode 100644 index 000000000..9ef97e218 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/admin_spec.rb @@ -0,0 +1,119 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::Admin do + subject(:collection) { described_class.new(datasource) } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + + def filter + ForestAdminDatasourceToolkit::Components::Query::Filter.new + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def stub_admins(*admins) + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => admins)) + end + + def stub_teams(*teams) + stub_request(:get, "#{base}/teams").to_return(json('type' => 'team.list', 'teams' => teams)) + end + + it 'is named IntercomAdmin' do + expect(collection.name).to eq('IntercomAdmin') + end + + it 'exposes the columns an ops lead reads before assigning anything' do + expect(collection.fields.keys) + .to eq(%w[id name email job_title away_mode_enabled away_mode_reassign has_inbox_seat team_names teams]) + end + + it 'declares id as the primary key' do + expect(collection.fields['id']).to have_attributes(is_primary_key: true, column_type: 'String') + end + + # `/admins` puts its records under `admins` rather than under the `data` + # envelope the paginated listings use. + it 'reads the endpoint and flattens the teammate' do + stub_admins('type' => 'admin', 'id' => '1', 'name' => 'Alice', 'email' => 'alice@acme.test', + 'job_title' => 'Support', 'away_mode_enabled' => true, 'away_mode_reassign' => false, + 'has_inbox_seat' => true, 'team_ids' => [814_865]) + stub_teams('id' => '814865', 'name' => 'Support') + + expect(collection.list(nil, filter, nil)) + .to eq([{ 'id' => '1', 'name' => 'Alice', 'email' => 'alice@acme.test', 'job_title' => 'Support', + 'away_mode_enabled' => true, 'away_mode_reassign' => false, 'has_inbox_seat' => true, + 'team_names' => ['Support'] }]) + end + + # A projection naming nothing asks for every declared column, which is how + # `project` reads it -- so the enrichment has to read it that way too, or the + # one column the projection publishes comes back nil. + it 'fills the derived names when the projection names no column at all' do + stub_admins('id' => '1', 'name' => 'Alice', 'team_ids' => [814_865]) + stub_teams('id' => '814865', 'name' => 'Support') + + expect(collection.list(nil, filter, nil).first['team_names']).to eq(['Support']) + end + + # Intercom types a team id as a number here and as a string on the team + # itself; a filter value from Forest always arrives as a string. Both sides + # of the membership therefore carry the same id, without which the relation + # would resolve to nothing rather than to an error. + it 'stringifies the ids so both sides of the membership match' do + stub_admins('id' => 493_881, 'team_ids' => [814_865]) + stub_teams('id' => '814865', 'name' => 'Support') + + expect(collection.list(nil, filter, %w[id team_names]).first) + .to eq({ 'id' => '493881', 'team_names' => ['Support'] }) + end + + describe 'the teams of a teammate' do + it 'names them for the whole page in one read, rather than one read per row' do + stub_admins({ 'id' => '1', 'team_ids' => [814_865] }, { 'id' => '2', 'team_ids' => [814_865, 814_866] }) + stub_teams({ 'id' => '814865', 'name' => 'Support' }, { 'id' => '814866', 'name' => 'Billing' }) + + expect(collection.list(nil, filter, %w[id team_names])) + .to eq([{ 'id' => '1', 'team_names' => ['Support'] }, + { 'id' => '2', 'team_names' => %w[Support Billing] }]) + expect(WebMock).to have_requested(:get, "#{base}/teams").once + end + + # A column nobody asked for costs no request. + it 'reads nothing when no projection asks for the names' do + stub_admins('id' => '1', 'team_ids' => [814_865]) + + collection.list(nil, filter, %w[id name]) + + expect(WebMock).not_to have_requested(:get, "#{base}/teams") + end + + # A missing permission costs the column, never the page -- and never the + # relation, which reads the teams from the other side. + it 'leaves the names empty when the teams cannot be read' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_admins('id' => '1', 'team_ids' => [814_865]) + stub_request(:get, "#{base}/teams").to_return(json({ 'type' => 'error.list' }, 403)) + + expect(collection.list(nil, filter, %w[id team_names]).first) + .to eq({ 'id' => '1', 'team_names' => [] }) + end + + it 'reads a teammate with no team as one with no team, not as one with a null' do + stub_admins('id' => '1', 'team_ids' => nil) + stub_teams('id' => '814865', 'name' => 'Support') + + expect(collection.list(nil, filter, %w[team_names]).first['team_names']).to eq([]) + end + + it 'is a many-to-many through the membership collection, read-only' do + expect(collection.fields['teams']) + .to have_attributes(type: 'ManyToMany', foreign_collection: 'IntercomTeam', + through_collection: 'IntercomTeamMembership', + origin_key: 'admin_id', foreign_key: 'team_id', is_read_only: true) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb new file mode 100644 index 000000000..98635a198 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb @@ -0,0 +1,559 @@ +module ForestAdminDatasourceIntercom + # The offset tier is exercised through Companies, the one collection Intercom + # paginates that way -- and the only one it does not search at all. + RSpec.describe Collections::Company do + subject(:collection) { datasource.get_collection('IntercomCompany') } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, *conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + def filter(condition_tree: nil, page: nil, sort: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree, page: page, + sort: sort) + end + + def page(offset, limit) + ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: limit) + end + + def sort(*clauses) + ForestAdminDatasourceToolkit::Components::Query::Sort.new(clauses) + end + + def count(condition_tree: nil) + collection.aggregate(nil, filter(condition_tree: condition_tree), + ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count')) + end + + def company(id, overrides = {}) + { 'type' => 'company', 'id' => id, 'company_id' => "erp-#{id}", 'name' => "Company #{id}", + 'plan' => { 'type' => 'plan', 'id' => '9', 'name' => 'Paid' }, 'size' => 85, + 'industry' => 'Manufacturing', 'website' => 'https://acme.test', 'monthly_spend' => 49, + 'session_count' => 26, 'user_count' => 10, 'created_at' => 1_700_000_000, + 'updated_at' => 1_700_000_600, 'last_request_at' => 1_700_000_900, + 'remote_created_at' => 1_394_531_169, 'custom_attributes' => {} }.merge(overrides) + end + + # `pages.next` is what the real endpoint sends on every page but the last, + # and it points at a page number rather than at a cursor -- the shape the + # collection has to read for a list view to open at all. + def stub_page(*companies, page: 1, per_page: 15, total_pages: 1, total: nil) + pages = { 'type' => 'pages', 'page' => page, 'per_page' => per_page, 'total_pages' => total_pages } + pages['next'] = "#{base}/companies?per_page=#{per_page}&page=#{page + 1}" if page < total_pages + + stub_request(:post, "#{base}/companies/list") + .with(query: { 'page' => page.to_s, 'per_page' => per_page.to_s }) + .to_return(json('type' => 'list', 'data' => companies, 'total_count' => total || companies.size, + 'pages' => pages)) + end + + def rows(projection = %w[id], **options) + collection.list(nil, filter(**options), projection) + end + + describe 'schema' do + it 'is named IntercomCompany' do + expect(collection.name).to eq('IntercomCompany') + end + + it 'publishes the columns of an account' do + expect(collection.fields.keys) + .to include('id', 'company_id', 'name', 'plan_name', 'size', 'industry', 'website', + 'monthly_spend', 'user_count', 'session_count', 'created_at', 'remote_created_at') + end + + it 'publishes every column read-only and unsortable' do + columns = collection.fields.values.grep(ForestAdminDatasourceToolkit::Schema::ColumnSchema) + + expect(columns.map(&:is_read_only).uniq).to eq([true]) + expect(columns.map(&:is_sortable).uniq).to eq([false]) + end + + # Four lookups is what `GET /companies` answers, and two of them name a + # column of this collection. A tag and a segment are collections of their + # own, and filtering by them belongs with the lot that adds them. + it 'offers a filter on the two keys Intercom looks a company up by' do + expect(collection.fields['name'].filter_operators).to eq(%w[equal]) + expect(collection.fields['company_id'].filter_operators).to eq(%w[equal]) + expect(collection.fields['id'].filter_operators).to eq(%w[equal in]) + end + + it 'offers no filter on anything else' do + %w[plan_name size industry website monthly_spend user_count created_at].each do |column| + expect(collection.fields[column].filter_operators).to be_empty, "#{column} advertises a filter" + end + end + + it 'declares the contacts of the account' do + expect(collection.fields['contacts'].origin_key).to eq('company_id') + end + + it 'is countable, total_count being exact on every answer' do + expect(collection.is_countable?).to be(true) + end + end + + describe 'the custom attributes' do + before do + stub_data_attributes('company', + { 'name' => 'arr', 'data_type' => 'float', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + end + + it 'publishes one column per attribute, typed and display-only' do + expect(collection.fields['arr'].column_type).to eq('Number') + expect(collection.fields['arr'].filter_operators).to be_empty + end + + it 'skips an attribute whose name a native column already carries, and says which' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_data_attributes('company', { 'name' => 'name', 'data_type' => 'string', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + stub_page(company('co1', 'custom_attributes' => { 'name' => 'Not the account name' }), + page: 1, per_page: 15) + + expect(rows(nil, page: page(0, 15)).first['name']).to eq('Company co1') + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/skips the company attribute "name"/) + end + + it 'reads a date attribute as a date' do + stub_data_attributes('company', { 'name' => 'renewal', 'data_type' => 'date', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + stub_page(company('co1', 'custom_attributes' => { 'renewal' => 1_700_000_000 }), page: 1, per_page: 15) + + expect(rows(%w[id renewal], page: page(0, 15)).first['renewal']).to eq('2023-11-14T22:13:20Z') + end + + it 'reads its value off the payload' do + stub_page(company('co1', 'custom_attributes' => { 'arr' => 12_000 })) + + expect(rows(%w[id arr], page: page(0, 15))).to eq([{ 'id' => 'co1', 'arr' => 12_000 }]) + end + end + + describe 'pagination by offset' do + # The one place R1 does not apply: Intercom counts pages, which is what a + # list view asks for. No cursor walked, no page read to be thrown away. + it 'asks for the page the window names, in one request' do + stub_page(company('co1'), page: 3, per_page: 15, total_pages: 3) + + expect(rows(%w[id], page: page(30, 15))).to eq([{ 'id' => 'co1' }]) + expect(WebMock).to have_requested(:post, "#{base}/companies/list") + .with(query: { 'page' => '3', 'per_page' => '15' }).once + end + + it 'reads a zero limit as an empty window, without a request' do + expect(rows(%w[id], page: page(0, 0))).to be_empty + expect(WebMock).not_to have_requested(:post, "#{base}/companies/list") + end + + # An offset that does not fall on a page boundary is served exactly, by + # reading the page it lands in and the next -- never by rounding the + # window to something the API likes better. + it 'reads across two pages when the window straddles them' do + stub_page(company('a'), company('b'), page: 1, per_page: 2, total_pages: 3) + stub_page(company('c'), company('d'), page: 2, per_page: 2, total_pages: 3) + + expect(rows(%w[id], page: page(1, 2)).map { |row| row['id'] }).to eq(%w[b c]) + end + + it 'stops at the last page rather than asking for one past it' do + stub_page(company('a'), page: 1, per_page: 15, total_pages: 1) + + expect(rows(%w[id], page: page(0, 15)).size).to eq(1) + expect(WebMock).not_to have_requested(:post, "#{base}/companies/list") + .with(query: { 'page' => '2', 'per_page' => '15' }) + end + + # The same advertisement, spelled as an object rather than as a url. Both + # name a page and neither carries a cursor, and a list view opens on both. + it 'reads a page whose next is advertised as an object' do + stub_request(:post, "#{base}/companies/list") + .with(query: { 'page' => '1', 'per_page' => '2' }) + .to_return(json('type' => 'list', 'data' => [company('a'), company('b')], 'total_count' => 4, + 'pages' => { 'type' => 'pages', 'page' => 1, 'per_page' => 2, 'total_pages' => 2, + 'next' => { 'page' => 2 } })) + stub_page(company('c'), company('d'), page: 2, per_page: 2, total_pages: 2) + + expect(rows(%w[id], page: page(1, 2)).map { |row| row['id'] }).to eq(%w[b c]) + end + + it 'stops on a page Intercom answers empty' do + stub_page(page: 1, per_page: 2, total_pages: 9) + + expect(rows(%w[id], page: page(0, 2))).to be_empty + end + + # A read naming no window -- a segment, a customizer -- is the only one + # that can run long, and it is the only one this bounds. + it 'reads full pages when the read names no window' do + stub_page(company('a'), page: 1, per_page: 150, total_pages: 1) + + expect(rows(%w[id]).map { |row| row['id'] }).to eq(%w[a]) + end + + it 'stops such a read after the pages it allows, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + (1..10).each { |number| stub_page(company("c#{number}"), page: number, per_page: 150, total_pages: 99) } + + expect(rows(%w[id]).size).to eq(10) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/Stopped reading IntercomCompany.*named no window of its own/) + end + + # A window is its own bound, so the page cap must not apply to it: a + # window needing eleven pages would come back one page short while the + # warning blamed it for naming no window -- which it did name. The record + # budget is what bounds a window nothing sane asked for, and it says so + # in its own words. + it 'reads past the page cap for a window that needs it' do + (1..12).each { |number| stub_page(company("c#{number}"), page: number, per_page: 150, total_pages: 12) } + + expect(rows(%w[id], page: page(0, 1800)).size).to eq(12) + end + + it 'stops a window on the record budget, and says which cap it hit' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_const('ForestAdminDatasourceIntercom::Collections::OffsetCollection::MAX_COLLECTED_RECORDS', 2) + (1..4).each { |number| stub_page(company("c#{number}"), page: number, per_page: 150, total_pages: 99) } + + expect(rows(%w[id], page: page(0, 1800)).size).to eq(2) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/named a window larger than one answer may hold/) + end + + it 'answers no more than the record budget when the last page read goes past it' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_const('ForestAdminDatasourceIntercom::Collections::OffsetCollection::MAX_COLLECTED_RECORDS', 3) + (1..3).each do |number| + stub_page(company("a#{number}"), company("b#{number}"), page: number, per_page: 150, total_pages: 99) + end + + expect(rows(%w[id], page: page(0, 1800)).map { |row| row['id'] }).to eq(%w[a1 b1 a2]) + end + + it 'counts what Intercom counted, in one request' do + stub_page(company('a'), page: 1, per_page: 1, total_pages: 40, total: 597) + + expect(count).to eq([{ 'group' => {}, 'value' => 597 }]) + end + + # Counting the pages read would answer a fraction of the collection as if + # it were the whole of it. + it 'refuses to count a listing Intercom answered without a total' do + stub_request(:post, "#{base}/companies/list").with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => [company('a')], + 'pages' => {})) + + expect { count }.to raise_error(UnsupportedOperatorError, /cannot be counted/) + end + end + + describe 'the four lookups, and everything past them' do + it 'reads a record detail through its own endpoint' do + stub_request(:get, "#{base}/companies/co1").to_return(json(company('co1'))) + + expect(rows(%w[id], condition_tree: leaf('id', operators::EQUAL, 'co1'))).to eq([{ 'id' => 'co1' }]) + end + + it 'reads a set of ids one request each' do + stub_request(:get, "#{base}/companies/co1").to_return(json(company('co1'))) + stub_request(:get, "#{base}/companies/co2").to_return(json(company('co2'))) + + expect(rows(%w[id], condition_tree: leaf('id', operators::IN, %w[co1 co2])).map { |row| row['id'] }) + .to eq(%w[co1 co2]) + end + + it 'reads a company the token can no longer reach as no record' do + stub_request(:get, "#{base}/companies/co1").to_return(json({ 'type' => 'error.list' }, 404)) + + expect(rows(%w[id], condition_tree: leaf('id', operators::EQUAL, 'co1'))).to be_empty + end + + it 'raises on a failure that is not a missing record' do + stub_request(:get, "#{base}/companies/co1").to_return(json({ 'type' => 'error.list' }, 500)) + + expect { rows(%w[id], condition_tree: leaf('id', operators::EQUAL, 'co1')) }.to raise_error(APIError) + end + + it 'truncates a set of ids larger than it will read, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, %r{#{base}/companies/co\d+}).to_return(json(company('co1'))) + + rows(%w[id], condition_tree: leaf('id', operators::IN, (1..30).map { |n| "co#{n}" })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/asked for 30 records by id/) + end + + it 'counts the records the ids named' do + stub_request(:get, "#{base}/companies/co1").to_return(json(company('co1'))) + + expect(count(condition_tree: leaf('id', operators::EQUAL, 'co1'))).to eq([{ 'group' => {}, 'value' => 1 }]) + end + + # Counting a set of ids means reading it, so past what a bulk read fetches + # the count is refused rather than answered with the number the truncation + # left: this collection advertises an exact count, and 25 where the + # question named forty records is not one. + it 'refuses to count more ids than it will read' do + expect { count(condition_tree: leaf('id', operators::IN, (1..40).map { |n| "co#{n}" })) } + .to raise_error(UnsupportedOperatorError, /cannot count 40 records by id/) + end + + # The `in` this comes from is a membership: it matches a record once, + # where a read by id would fetch a repeated value twice and hand it back + # as two rows carrying one id. + it 'reads a value named twice once' do + stub_request(:get, "#{base}/companies/co1").to_return(json(company('co1'))) + + expect(rows(%w[id], condition_tree: leaf('id', operators::IN, %w[co1 co1]))).to eq([{ 'id' => 'co1' }]) + expect(WebMock).to have_requested(:get, "#{base}/companies/co1").once + end + end + + # A relation pointing at this collection resolves it by id, and Intercom has + # no bulk read for a company: the fan-out is sliced by what one batch holds + # and refused past what the tier resolves at all, rather than answered with a + # nil where an account exists. + describe 'as the target of a relation' do + subject(:contacts) { datasource.get_collection('IntercomContact') } + + # An unfiltered, unsorted contact list reads the listing endpoint; the + # search is for the routes that filter. + def stub_contact_list(*records) + stub_request(:get, "#{base}/contacts") + .with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => records, 'total_count' => records.size, 'pages' => {})) + end + + def contact_of(company_id) + { 'type' => 'contact', 'id' => "c#{company_id}", + 'companies' => { 'type' => 'list', 'data' => [{ 'type' => 'company', 'id' => company_id }], + 'total_count' => 1 } } + end + + it 'slices the fan-out into batches rather than truncating it' do + stub_contact_list(*(1..30).map { |n| contact_of("co#{n}") }) + stub_request(:get, %r{#{base}/companies/co\d+}).to_return do |request| + json(company(request.uri.path.split('/').last)) + end + + rows = contacts.list(nil, filter, %w[id company:name]) + + expect(rows.filter_map { |row| row['company'] }.size).to eq(30) + end + + it 'refuses a fan-out past what it resolves, naming the relation' do + stub_contact_list(*(1..120).map { |n| contact_of("co#{n}") }) + + expect { contacts.list(nil, filter, %w[id company:name]) } + .to raise_error(UnsupportedOperatorError, /cannot resolve company over this read: it names 120/) + end + + # `GET /companies?name=` answers the company itself where a listing would + # answer an envelope: a record is read as a page of one rather than as a + # shape every caller has to test for. + it 'looks a company up by name, and reads the record Intercom answers' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Acme' }) + .to_return(json(company('co1'))) + + expect(rows(%w[id], condition_tree: leaf('name', operators::EQUAL, 'Acme'))).to eq([{ 'id' => 'co1' }]) + end + + it 'looks one up by the identifier the workspace gave it' do + stub_request(:get, "#{base}/companies").with(query: { 'company_id' => 'erp-co1' }) + .to_return(json('type' => 'list', 'data' => [company('co1')], + 'total_count' => 1, 'pages' => {})) + + expect(rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, 'erp-co1'))) + .to eq([{ 'id' => 'co1' }]) + end + + it 'counts what a lookup found' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Acme' }) + .to_return(json(company('co1'))) + + expect(count(condition_tree: leaf('name', operators::EQUAL, 'Acme'))) + .to eq([{ 'group' => {}, 'value' => 1 }]) + end + + # Intercom answers this route with a 404 where a search endpoint would + # answer an empty list, and a filter matching nothing is the most + # ordinary thing a list view does: it reads as no record rather than as a + # failure the operator sees as "Unexpected error". + it 'answers a lookup naming no record with an empty page' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Nope' }) + .to_return(json({ 'type' => 'error.list', + 'errors' => [{ 'code' => 'company_not_found', + 'message' => 'Company Not Found' }] }, + 404)) + + expect(rows(%w[id], condition_tree: leaf('name', operators::EQUAL, 'Nope'))).to eq([]) + end + + it 'counts a lookup naming no record as none' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Nope' }) + .to_return(json({ 'type' => 'error.list', + 'errors' => [{ 'code' => 'company_not_found', + 'message' => 'Company Not Found' }] }, + 404)) + + expect(count(condition_tree: leaf('name', operators::EQUAL, 'Nope'))) + .to eq([{ 'group' => {}, 'value' => 0 }]) + end + + # A 404 is the only status read as an absence: anything else is a failure + # this must not answer an empty page to. + it 'raises on a lookup Intercom refused for another reason' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Acme' }) + .to_return(json({ 'type' => 'error.list' }, 500)) + + expect { rows(%w[id], condition_tree: leaf('name', operators::EQUAL, 'Acme')) } + .to raise_error(APIError, /HTTP 500/) + end + + it 'reports a lookup Intercom answered with more than one page' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Acme' }) + .to_return(json('type' => 'list', 'data' => [company('co1')], + 'pages' => { 'next' => { 'starting_after' => 'zzz' } })) + + rows(%w[id], condition_tree: leaf('name', operators::EQUAL, 'Acme')) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/advertised more records/) + end + + it 'counts a lookup spread over several pages by the total Intercom gave' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Acme' }) + .to_return(json('type' => 'list', 'data' => [company('co1')], + 'total_count' => 3, + 'pages' => { 'next' => { 'starting_after' => 'zzz' } })) + + expect(count(condition_tree: leaf('name', operators::EQUAL, 'Acme'))) + .to eq([{ 'group' => {}, 'value' => 3 }]) + end + + it 'counts a single-page lookup with no total by what it holds' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Acme' }) + .to_return(json('type' => 'list', + 'data' => [company('co1'), company('co2')], + 'pages' => {})) + + expect(count(condition_tree: leaf('name', operators::EQUAL, 'Acme'))) + .to eq([{ 'group' => {}, 'value' => 2 }]) + end + + it 'refuses to count a lookup spread over several pages with no total' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Acme' }) + .to_return(json('type' => 'list', 'data' => [company('co1')], + 'pages' => { 'next' => { 'starting_after' => 'zzz' } })) + + expect { count(condition_tree: leaf('name', operators::EQUAL, 'Acme')) } + .to raise_error(UnsupportedOperatorError, /cannot be counted/) + end + + # A filter this collection cannot look up is refused by name. A page + # served in answer to a filter it ignored is the one failure this + # datasource is built to avoid. + it 'refuses a filter on a column Intercom does not look up' do + expect { rows(%w[id], condition_tree: leaf('industry', operators::EQUAL, 'Manufacturing')) } + .to raise_error(UnsupportedOperatorError, /cannot filter "industry".*name, company_id alone/m) + end + + it 'refuses an operator the lookup has no equivalent for' do + expect { rows(%w[id], condition_tree: leaf('name', operators::CONTAINS, 'Acm')) } + .to raise_error(UnsupportedOperatorError, /cannot filter "name"/) + end + + it 'refuses a combination, the lookup answering one value at a time' do + expect do + rows(%w[id], condition_tree: branch('And', leaf('name', operators::EQUAL, 'Acme'), + leaf('company_id', operators::EQUAL, 'erp-co1'))) + end.to raise_error(UnsupportedOperatorError, /cannot combine conditions on name, company_id.*one exact value/m) + end + + it 'names the condition it cannot look up, not the lookup standing next to it' do + expect do + rows(%w[id], condition_tree: branch('And', leaf('name', operators::EQUAL, 'Acme'), + leaf('plan_name', operators::EQUAL, 'Pro'))) + end.to raise_error(UnsupportedOperatorError, /cannot filter "plan_name"/) + end + + it 'refuses to count what it refuses to list' do + expect { count(condition_tree: leaf('industry', operators::EQUAL, 'Manufacturing')) } + .to raise_error(UnsupportedOperatorError, /cannot filter "industry"/) + end + end + + describe 'what it will not do' do + it 'refuses to group, Intercom exposing no aggregate endpoint' do + expect do + collection.aggregate(nil, filter, ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count', groups: [{ field: 'industry' }])) + end.to raise_error(UnsupportedOperatorError, /can only be counted/) + end + + # There is no order parameter on this listing at all, so an order asked + # for and not applied is reported here or nowhere. + it 'reports an order it cannot apply' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_page(company('a'), page: 1, per_page: 15) + + rows(%w[id], page: page(0, 15), sort: sort({ field: 'name', ascending: true })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/takes no order on this listing/) + end + + it 'says nothing of the ascending primary-key order the agent injects' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_page(company('a'), page: 1, per_page: 15) + + rows(%w[id], page: page(0, 15), sort: sort({ field: 'id', ascending: true })) + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + + it 'reports an explicit descending order on the key, which it cannot apply either' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_page(company('a'), page: 1, per_page: 15) + + rows(%w[id], page: page(0, 15), sort: sort({ field: 'id', ascending: false })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/sort on id/) + end + end + + describe 'the row' do + it 'flattens the payload, the plan read off the object that carries it' do + stub_page(company('co1'), page: 1, per_page: 15) + + expect(rows(nil, page: page(0, 15)).first) + .to include('id' => 'co1', 'company_id' => 'erp-co1', 'name' => 'Company co1', 'plan_name' => 'Paid', + 'size' => 85, 'monthly_spend' => 49, 'user_count' => 10, + 'created_at' => '2023-11-14T22:13:20Z', 'remote_created_at' => '2014-03-11T09:46:09Z') + end + + it 'reads a company carrying no plan without failing' do + stub_page(company('co1', 'plan' => nil), page: 1, per_page: 15) + + expect(rows(nil, page: page(0, 15)).first).to include('plan_name' => nil) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb new file mode 100644 index 000000000..2b9b4e511 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb @@ -0,0 +1,621 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::Contact do + subject(:collection) { datasource.get_collection('IntercomContact') } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, *conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + def filter(condition_tree: nil, page: nil, sort: nil, search: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree, page: page, + sort: sort, search: search) + end + + def page(offset, limit) + ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: limit) + end + + def sort(*clauses) + ForestAdminDatasourceToolkit::Components::Query::Sort.new(clauses) + end + + def contact(id, overrides = {}) + { 'type' => 'contact', 'id' => id, 'role' => 'user', 'name' => "Contact #{id}", + 'email' => "#{id}@acme.test", 'phone' => nil, 'external_id' => "ext-#{id}", 'owner_id' => 493_881, + 'created_at' => 1_700_000_000, 'updated_at' => 1_700_000_600, 'session_count' => 3, + 'unsubscribed_from_emails' => false, 'has_hard_bounced' => false, 'marked_email_as_spam' => false, + 'browser' => 'chrome', 'browser_language' => 'fr', 'os' => 'OS X', 'language_override' => nil, + 'location' => { 'type' => 'location', 'country' => 'France', 'region' => 'IdF', 'city' => 'Paris' }, + 'companies' => { 'type' => 'list', 'data' => [{ 'type' => 'company', 'id' => 'co1' }], + 'total_count' => 2 }, + 'custom_attributes' => {} }.merge(overrides) + end + + def stub_list(*contacts, cursor: nil) + pages = cursor ? { 'next' => { 'starting_after' => cursor } } : {} + stub_request(:get, "#{base}/contacts").with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => contacts, + 'total_count' => contacts.size, 'pages' => pages)) + end + + def stub_search(*contacts, total: nil) + stub_request(:post, "#{base}/contacts/search") + .to_return(json('type' => 'list', 'data' => contacts, 'total_count' => total || contacts.size, + 'pages' => {})) + end + + def rows(projection = %w[id], **options) + collection.list(nil, filter(**options), projection) + end + + describe 'schema' do + it 'is named IntercomContact' do + expect(collection.name).to eq('IntercomContact') + end + + it 'publishes the columns of a contact, the account it belongs to included' do + expect(collection.fields.keys) + .to include('id', 'role', 'name', 'email', 'email_domain', 'phone', 'external_id', 'avatar', + 'owner_id', 'company_id', 'company_count', 'session_count', 'created_at', + 'last_seen_at', 'unsubscribed_from_emails', 'location_country') + end + + it 'publishes every column read-only, this lot writing nothing' do + columns = collection.fields.values.grep(ForestAdminDatasourceToolkit::Schema::ColumnSchema) + + expect(columns.map(&:is_read_only).uniq).to eq([true]) + end + + # The measured asymmetry: `/contacts/search` refuses `>=`, `<=`, `!=` and + # `IN` on a date where the other two endpoints take them. A Date column + # publishes the two bounds alone anyway -- declaring `equal` would make + # the toolkit republish `in`, which its own validator then refuses + # (PRD-989) -- so what the restriction really has to guarantee is that + # nothing wider reaches the wire. + it 'offers the two bounds on a date, and nothing the endpoint refuses' do + expect(collection.fields['created_at'].filter_operators).to eq(%w[greater_than less_than]) + expect(collection.fields['last_seen_at'].filter_operators).to eq(%w[greater_than less_than]) + end + + it 'offers on a text column exactly what the table measured' do + expect(collection.fields['email'].filter_operators) + .to eq(%w[equal not_equal contains i_contains not_contains starts_with ends_with]) + expect(collection.fields['role'].filter_operators).to eq(%w[equal not_equal]) + end + + # A column the table does not carry advertises nothing, which is how a + # refusal is spelled in a schema. + it 'advertises no filter on a column the endpoint does not filter' do + %w[avatar company_count session_count].each do |column| + expect(collection.fields[column].filter_operators).to be_empty, "#{column} advertises a filter" + end + end + + # The search filters no company field, but the account endpoint answers a + # bare equality -- and the agent sends exactly that to open the related + # list of an account. + it 'offers an equality on company_id, which the account endpoint answers' do + expect(collection.fields['company_id'].filter_operators).to eq(%w[equal]) + end + + # The one collection of the whole API Intercom sorts, and the reason this + # tier reads a `sortable` flag off the measured table at all. + it 'is sortable on the columns the table measured, and on no others' do + sortable = collection.fields.select { |_, f| f.respond_to?(:is_sortable) && f.is_sortable }.keys + + expect(sortable).to contain_exactly('name', 'email', 'created_at', 'updated_at', 'signed_up_at', + 'last_seen_at', 'last_contacted_at', 'last_replied_at') + end + + it 'declares the relations the 360 degrees is walked through' do + expect(collection.fields['owner']).to be_a(ForestAdminDatasourceToolkit::Schema::Relations::ManyToOneSchema) + expect(collection.fields['company']).to be_a(ForestAdminDatasourceToolkit::Schema::Relations::ManyToOneSchema) + expect(collection.fields['conversations']) + .to be_a(ForestAdminDatasourceToolkit::Schema::Relations::OneToManySchema) + expect(collection.fields['tickets'].origin_key).to eq('contact_id') + end + + it 'publishes the company unfilterable, contacts/search filtering no company field' do + expect(collection.fields['owner'].is_filterable).to be(true) + expect(collection.fields['company'].is_filterable).to be(false) + end + + it 'is countable, total_count being exact on every answer' do + expect(collection.is_countable?).to be(true) + end + + it 'is searchable on the address an ops team types' do + expect(collection.is_searchable?).to be(true) + end + end + + describe 'the custom attributes' do + subject(:collection) { datasource.get_collection('IntercomContact') } + + before do + stub_data_attributes('contact', + { 'name' => 'paid_subscriber', 'data_type' => 'boolean', 'custom' => true, + 'api_writable' => true, 'archived' => false }, + { 'name' => 'email', 'data_type' => 'string', 'custom' => false, + 'api_writable' => false, 'archived' => false }) + end + + it 'publishes one column per custom attribute, typed from the introspection' do + expect(collection.fields['paid_subscriber'].column_type).to eq('Boolean') + end + + # Display-only: which operators Intercom answers on + # `custom_attributes.{name}` has not been measured, and this package + # publishes no filter it has not seen work. + it 'publishes it unfilterable and unsortable' do + expect(collection.fields['paid_subscriber'].filter_operators).to be_empty + expect(collection.fields['paid_subscriber'].is_sortable).to be(false) + end + + # Measured on a real workspace, and it took the agent's boot with it: the + # toolkit refuses a field declared twice, so an attribute landing on a + # native column has to be skipped -- and skipped rather than renamed, + # since the operator expects the record's own key under `id`. + it 'skips an attribute whose name a native column already carries, and says which' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_data_attributes('contact', { 'name' => 'id', 'data_type' => 'string', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + stub_list(contact('1', 'custom_attributes' => { 'id' => 'erp-42' })) + + expect { collection }.not_to raise_error + expect(rows(nil).first['id']).to eq('1') + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/skips the contact attribute "id"/) + end + + # Relations are declared before these columns for exactly this reason. + it 'skips one landing on a relation name too' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_data_attributes('contact', { 'name' => 'company', 'data_type' => 'string', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + + expect(collection.fields['company']) + .to be_a(ForestAdminDatasourceToolkit::Schema::Relations::ManyToOneSchema) + end + + # A custom date arrives as epoch seconds like every other Intercom date, + # and a Date column handed an integer renders as one. + it 'reads a date attribute as a date' do + stub_data_attributes('contact', { 'name' => 'renewal', 'data_type' => 'date', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + stub_list(contact('1', 'custom_attributes' => { 'renewal' => 1_700_000_000 })) + + expect(rows(%w[id renewal]).first['renewal']).to eq('2023-11-14T22:13:20Z') + end + + it 'reads its value off the payload, nil where the contact carries none' do + stub_list(contact('1', 'custom_attributes' => { 'paid_subscriber' => true }), contact('2')) + + expect(rows(%w[id paid_subscriber])) + .to eq([{ 'id' => '1', 'paid_subscriber' => true }, { 'id' => '2', 'paid_subscriber' => nil }]) + end + end + + describe '#list' do + it 'walks the listing endpoint when nothing is filtered, sorted or searched' do + stub_list(contact('1'), contact('2')) + + expect(rows).to eq([{ 'id' => '1' }, { 'id' => '2' }]) + expect(WebMock).not_to have_requested(:post, "#{base}/contacts/search") + end + + it 'flattens the payload onto the row' do + stub_list(contact('1')) + + expect(rows(nil).first) + .to include('id' => '1', 'role' => 'user', 'email' => '1@acme.test', 'email_domain' => 'acme.test', + 'owner_id' => '493881', 'session_count' => 3, 'created_at' => '2023-11-14T22:13:20Z', + 'location_country' => 'France', 'location_city' => 'Paris') + end + + # A contact belongs to several accounts: the row names the first and says + # how many there are, which Intercom counts itself -- the nested list is + # capped and a contact of twelve accounts must not read as one of ten. + it 'names the first account and takes the count from Intercom' do + stub_list(contact('1')) + + expect(rows(nil).first).to include('company_id' => 'co1', 'company_count' => 2) + end + + it 'counts the accounts it can see when Intercom sends no count' do + stub_list(contact('1', 'companies' => { 'type' => 'list', 'data' => [{ 'id' => 'co1' }] })) + + expect(rows(nil).first).to include('company_count' => 1) + end + + it 'leaves the account columns empty on a contact belonging to none' do + stub_list(contact('1', 'companies' => nil)) + + expect(rows(nil).first).to include('company_id' => nil, 'company_count' => 0) + end + + it 'reads no domain out of an address that has none' do + stub_list(contact('1', 'email' => nil)) + + expect(rows(nil).first).to include('email_domain' => nil) + end + + it 'translates a filter into the search DSL' do + stub_search(contact('1')) + + expect(rows(%w[id], condition_tree: leaf('role', operators::EQUAL, 'lead'))).to eq([{ 'id' => '1' }]) + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'role', 'operator' => '=', 'value' => 'lead' })) + end + + it 'answers a free-text search on the address, per word' do + stub_search(contact('1')) + + rows(%w[id], search: 'acme.test') + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'email', 'operator' => '~', 'value' => 'acme.test' })) + end + end + + describe 'the one collection Intercom sorts' do + # A list view asking for an order has no condition to send, and the + # listing endpoint does not sort: the order is what routes the read + # through the search, with the predicate that matches everything. + it 'sends the order to the search endpoint, with the match-all predicate' do + stub_search(contact('1')) + + rows(%w[id], sort: sort({ field: 'last_seen_at', ascending: false })) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => described_class::MATCH_EVERY_CONTACT, + 'sort' => { 'field' => 'last_seen_at', 'order' => 'descending' })) + end + + it 'sends it alongside the filter when there is one' do + stub_search(contact('1')) + + rows(%w[id], condition_tree: leaf('role', operators::EQUAL, 'user'), + sort: sort({ field: 'name', ascending: true })) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'role', 'operator' => '=', 'value' => 'user' }, + 'sort' => { 'field' => 'name', 'order' => 'ascending' })) + end + + # The ascending primary-key sort the agent injects when a request names + # none is not an order anybody asked for, and this endpoint does not sort + # on an id anyway. + it 'keeps the listing route for the default primary-key order, and says nothing' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_list(contact('1')) + + rows(%w[id], sort: sort({ field: 'id', ascending: true })) + + expect(WebMock).to have_requested(:get, "#{base}/contacts").with(query: hash_including({})) + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + + it 'reports an order on a column Intercom does not sort, rather than dropping it' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_list(contact('1')) + + rows(%w[id], sort: sort({ field: 'browser', ascending: true })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/sort on browser/) + expect(WebMock).not_to have_requested(:post, "#{base}/contacts/search") + end + + # Intercom takes a single `{ field, order }`: honouring the first clause + # alone would order the page by something the operator did not ask for. + it 'reports a composite order rather than honouring half of it' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_list(contact('1')) + + rows(%w[id], sort: sort({ field: 'name', ascending: true }, { field: 'email', ascending: false })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/sort on name, email/) + end + end + + describe 'reading records by id' do + # One request per hundred ids rather than one per id: this endpoint + # answers `id IN [...]`, which is what makes a related list of contacts + # affordable at all. + it 'reads a set of ids in one request through the search' do + stub_search(contact('1'), contact('2')) + + expect(rows(%w[id], condition_tree: leaf('id', operators::IN, %w[1 2]))) + .to eq([{ 'id' => '1' }, { 'id' => '2' }]) + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'id', 'operator' => 'IN', 'value' => %w[1 2] })).once + end + + it 'reads a record detail the same way' do + stub_search(contact('1')) + + expect(rows(%w[id], condition_tree: leaf('id', operators::EQUAL, '1'))).to eq([{ 'id' => '1' }]) + end + + # A contact merged into another disappears from the search: the row reads + # as gone rather than as an error, which is what a merge means. + it 'answers no row for a contact that was merged away' do + stub_search + + expect(rows(%w[id], condition_tree: leaf('id', operators::EQUAL, 'merged'))).to be_empty + end + + it 'counts what the ids named' do + stub_search(contact('1')) + + expect(collection.aggregate(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count'))) + .to eq([{ 'group' => {}, 'value' => 1 }]) + end + + # Counted by Intercom rather than by reading: each slice of fifteen ids -- + # the most one `IN` takes -- is one search asked for a single record, and + # its `total_count` is the slice's. + it 'counts a set of ids from the total of each slice, reading one record per search' do + stub_search(contact('1'), total: 15) + + expect(collection.aggregate(nil, filter(condition_tree: leaf('id', operators::IN, (1..45).map(&:to_s))), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count'))) + .to eq([{ 'group' => {}, 'value' => 45 }]) + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('pagination' => { 'per_page' => 1 })).times(3) + end + + it 'never names more ids in one search than Intercom takes' do + stub_search(contact('1')) + + rows(%w[id], condition_tree: leaf('id', operators::IN, (1..40).map(&:to_s))) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").times(3) + expect(WebMock).not_to(have_requested(:post, "#{base}/contacts/search").with do |request| + JSON.parse(request.body).dig('query', 'value').size > 15 + end) + end + + it 'refuses a count Intercom answered without a total' do + stub_request(:post, "#{base}/contacts/search") + .to_return(json('type' => 'list', 'data' => [contact('1')], 'pages' => {})) + + expect do + collection.aggregate(nil, filter(condition_tree: leaf('id', operators::IN, %w[1 2])), + ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count')) + end.to raise_error(UnsupportedOperatorError, /without a total_count/) + end + + it 'truncates a set larger than it will read, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_search(contact('1')) + + rows(%w[id], condition_tree: leaf('id', operators::IN, (1..400).map(&:to_s))) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/asked for 400 records by id/) + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").times(20) + end + + # The window is cut in the order the ids were named, so ordering what + # comes back would order a slice picked by something else. This is the one + # route of the collection Intercom does sort where an order cannot be + # honoured, so `server_sort` stayed quiet and this has to speak. + it 'reports an order this route cannot apply, on a column Intercom does sort' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_search(contact('1')) + + rows(%w[id], condition_tree: leaf('id', operators::IN, %w[1 2]), + sort: sort({ field: 'name', ascending: true })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/sort on "name" while reading/) + end + + # The window was cut on the ids, so the rows keep their order rather than + # the one the search answered in. + it 'answers in the order the ids were named, whatever order the search used' do + stub_search(contact('3'), contact('1'), contact('2')) + + expect(rows(%w[id], condition_tree: leaf('id', operators::IN, %w[1 2 3])).map { |row| row['id'] }) + .to eq(%w[1 2 3]) + end + + # The `in` this comes from is a membership: it matches a record once. + it 'reads a value named twice once' do + stub_search(contact('1')) + + rows(%w[id], condition_tree: leaf('id', operators::IN, %w[1 1])) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'id', 'operator' => 'IN', 'value' => %w[1] })) + end + + # Counting a set of ids means reading it, so past what a bulk read fetches + # the count is refused rather than answered with the number the truncation + # left -- this collection advertises an exact count. + it 'refuses to count more ids than it will read' do + expect do + collection.aggregate(nil, filter(condition_tree: leaf('id', operators::IN, (1..400).map(&:to_s))), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count')) + end.to raise_error(UnsupportedOperatorError, /cannot count 400 records by id/) + end + end + + describe 'the contacts of an account' do + # `/contacts/search` filters no company field, and `GET + # /companies/{id}/contacts` is what answers the one relation an ops team + # walks the most. Without this route it would be a refusal. + it 'reads them from the company endpoint rather than from the search' do + stub_request(:get, "#{base}/companies/co1/contacts").with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => [contact('1')], + 'total_count' => 1, 'pages' => {})) + + expect(rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, 'co1'))) + .to eq([{ 'id' => '1' }]) + expect(WebMock).not_to have_requested(:post, "#{base}/contacts/search") + end + + it 'counts them from the same endpoint' do + stub_request(:get, "#{base}/companies/co1/contacts").with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => [contact('1')], + 'total_count' => 42, 'pages' => {})) + + expect(collection.aggregate(nil, filter(condition_tree: leaf('company_id', operators::EQUAL, 'co1')), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count'))) + .to eq([{ 'group' => {}, 'value' => 42 }]) + end + + it 'reports an order this route cannot apply' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/companies/co1/contacts").with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => [contact('1')], + 'total_count' => 1, 'pages' => {})) + + rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, 'co1'), + sort: sort({ field: 'name', ascending: true })) + + # Named for what it is: this collection *is* the one Intercom sorts, so + # the reason is the route, not the column. + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/sort on name while reading the contacts of an account/) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/a route that takes no order, unlike the search/) + end + + # An account Intercom no longer answers for -- deleted, or moved out of + # the token's reach between the moment the row was rendered and the + # moment its related list was opened -- reads as an account with no + # contact rather than as a failed page. + it 'reads an account Intercom no longer answers for as one with no contact' do + stub_request(:get, "#{base}/companies/gone/contacts").with(query: hash_including({})) + .to_return(json({ 'type' => 'error.list' }, 404)) + + expect(rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, 'gone'))).to eq([]) + expect(collection.aggregate(nil, filter(condition_tree: leaf('company_id', operators::EQUAL, 'gone')), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count'))) + .to eq([{ 'group' => {}, 'value' => 0 }]) + end + + it 'raises where Intercom refused the account for another reason' do + stub_request(:get, "#{base}/companies/co1/contacts").with(query: hash_including({})) + .to_return(json({ 'type' => 'error.list' }, 500)) + + expect { rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, 'co1')) } + .to raise_error(APIError, /HTTP 500/) + end + + # An `and` also carrying a scope names a narrower set than the account + # does, and answering it with the account alone would serve contacts the + # scope excludes. There is no request that answers both halves either: + # the account endpoint narrows nothing and the search filters no company + # field, which is what a related list runs into the moment a scope or a + # segment exists on this collection. + it 'refuses to take the route for anything but a bare equality' do + expect do + rows(%w[id], condition_tree: branch('And', leaf('company_id', operators::EQUAL, 'co1'), + leaf('role', operators::EQUAL, 'user'))) + end.to raise_error(UnsupportedOperatorError, %r{GET /companies/\{id\}/contacts}) + end + + # The refusal names the half that does not fit rather than telling the + # operator to read the account's contacts -- which is what they asked for. + it 'names the condition filtered alongside the account' do + expect do + rows(%w[id], condition_tree: branch('And', leaf('company_id', operators::EQUAL, 'co1'), + leaf('role', operators::EQUAL, 'user'))) + end.to raise_error(UnsupportedOperatorError, /condition on role/) + end + + it 'refuses the count of a narrowed account the same way' do + expect do + collection.aggregate(nil, + filter(condition_tree: branch('And', leaf('company_id', operators::EQUAL, 'co1'), + leaf('role', operators::EQUAL, 'user'))), + ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count')) + end.to raise_error(UnsupportedOperatorError, %r{GET /companies/\{id\}/contacts}) + end + end + + describe 'a condition through a relation' do + def stub_admins(*admins) + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => admins)) + end + + # The owner is a teammate, read whole in one request, and + # `/contacts/search` filters on the key: readable, navigable and + # filterable alike. + it 'resolves the owner against the teammates and filters on the key' do + stub_admins({ 'id' => '493881', 'name' => 'Marie' }) + stub_search(contact('1')) + + rows(%w[id], condition_tree: leaf('owner:name', operators::EQUAL, 'Marie')) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'owner_id', 'operator' => '=', 'value' => '493881' })) + end + + it 'nests the owner under the relation when the projection names it' do + stub_admins({ 'id' => '493881', 'name' => 'Marie' }) + stub_list(contact('1')) + + expect(rows(%w[id owner:name]).first).to eq('id' => '1', + 'owner' => { 'id' => '493881', 'name' => 'Marie' }) + end + + # The endpoint filters no company field, so the relation is there to be + # read and navigated. Refused by name, before the target is read: a + # refusal that spends a request costs exactly what it refuses to do. + it 'refuses a condition through the company, and says what to filter instead' do + expect { rows(%w[id], condition_tree: leaf('company:name', operators::EQUAL, 'Acme')) } + .to raise_error(UnsupportedOperatorError, + %r{resolves to "company_id", on which contacts/search takes no filter}) + expect(WebMock).not_to have_requested(:post, /companies/) + end + end + + describe 'what it will not do' do + it 'refuses to group, Intercom exposing no aggregate endpoint' do + expect do + collection.aggregate(nil, filter, ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count', groups: [{ field: 'role' }])) + end.to raise_error(UnsupportedOperatorError, /can only be counted/) + end + + it 'counts what the filter names in one request' do + stub_search(contact('1'), total: 812) + + expect(collection.aggregate(nil, filter(condition_tree: leaf('role', operators::EQUAL, 'user')), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count'))) + .to eq([{ 'group' => {}, 'value' => 812 }]) + end + end + + describe 'paging' do + it 'asks Intercom for the window the list view named' do + stub_list(contact('1'), contact('2'), contact('3')) + + expect(rows(%w[id], page: page(1, 2)).map { |row| row['id'] }).to eq(%w[2 3]) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb new file mode 100644 index 000000000..a5d8bbd57 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb @@ -0,0 +1,765 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::Conversation do + subject(:collection) { datasource.get_collection('IntercomConversation') } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + def filter(condition_tree: nil, page: nil, sort: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree, page: page, + sort: sort) + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, *conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + # The columns alone: a relation carries neither operators nor an order, and + # what it may be filtered through is asserted on its own below. + def columns + collection.fields.select { |_, field| field.type == 'Column' } + end + + # Hand-written from the OpenAPI 2.16 spec, never captured from a workspace: + # a conversation body is personal data. + def conversation(id, overrides = {}) + { + 'type' => 'conversation', 'id' => id, 'title' => "Facture #{id}", 'state' => 'closed', + 'priority' => 'priority', 'open' => false, 'read' => true, + 'created_at' => 1_700_000_000, 'updated_at' => 1_700_003_600, + 'waiting_since' => nil, 'snoozed_until' => nil, + 'admin_assignee_id' => 493_881, 'team_assignee_id' => 814_865, + 'company' => { 'type' => 'company', 'id' => '696dd52099f73812610d9c7b', 'name' => 'Acme' }, + 'contacts' => { 'type' => 'contact.list', + 'contacts' => [{ 'type' => 'contact', 'id' => 'c1' }, { 'type' => 'contact', 'id' => 'c2' }] }, + 'tags' => { 'type' => 'tag.list', 'tags' => [{ 'id' => 't1', 'name' => 'billing' }] }, + 'ai_agent_participated' => true, + 'source' => { 'type' => 'conversation', 'id' => 's1', 'delivered_as' => 'customer_initiated', + 'subject' => 'Ma facture', 'body' => 'Bonjour, ou est ma facture ?', + 'author' => { 'type' => 'user', 'id' => 'c1', 'name' => 'Camille', + 'email' => 'camille@acme.test' }, + 'attachments' => [] }, + 'statistics' => { 'type' => 'conversation_statistics', 'first_close_at' => 1_700_002_000, + 'last_close_at' => 1_700_003_000, 'last_closed_by_id' => '493881', + 'first_contact_reply_at' => 1_700_000_050, 'last_contact_reply_at' => 1_700_001_000, + 'last_admin_reply_at' => 1_700_002_500, 'count_reopens' => 1, + 'count_conversation_parts' => 4 } + }.merge(overrides) + end + + def parts(*entries) + { 'conversation_parts' => { 'type' => 'conversation_part.list', 'conversation_parts' => entries } } + end + + def part(part_type, overrides = {}) + { 'type' => 'conversation_part', 'id' => 'p1', 'part_type' => part_type, 'body' => 'Je regarde.', + 'created_at' => 1_700_002_500, 'redacted' => false, 'attachments' => [], + 'author' => { 'type' => 'admin', 'id' => '493881', 'name' => 'Alice', + 'email' => 'alice@acme.test' } }.merge(overrides) + end + + # Intercom puts the records under `conversations`, not under the `data` + # envelope -- measured on `/tickets/search`, and the listings follow the same + # habit. + def stub_list(*records, next_cursor: nil, total: nil, query: hash_including({})) + body = { 'type' => 'conversation.list', 'conversations' => records, + 'total_count' => total || records.size, 'pages' => { 'type' => 'pages', 'page' => 1 } } + body['pages']['next'] = { 'starting_after' => next_cursor } if next_cursor + + stub_request(:get, "#{base}/conversations").with(query: query).to_return(json(body)) + end + + def stub_search(*records, total: nil, next_cursor: nil) + body = { 'type' => 'conversation.list', 'conversations' => records, + 'total_count' => total || records.size, 'pages' => { 'type' => 'pages', 'page' => 1 } } + body['pages']['next'] = { 'starting_after' => next_cursor } if next_cursor + + stub_request(:post, "#{base}/conversations/search").with(query: hash_including({})).to_return(json(body)) + end + + def stub_record(id, payload, status = 200) + stub_request(:get, "#{base}/conversations/#{id}").with(query: hash_including({})).to_return(json(payload, status)) + end + + def ids(rows) + rows.map { |row| row['id'] } + end + + describe 'schema' do + it 'is named IntercomConversation' do + expect(collection.name).to eq('IntercomConversation') + end + + # Neither search endpoint takes a sort, and Intercom ignores the one it is + # sent without a word, so no column of this tier may advertise one. + it 'declares every column unsortable' do + expect(columns.values.map(&:is_sortable).uniq).to eq([false]) + end + + # Derived from the measured table, never written by hand: a column + # advertises exactly what the search endpoint answers on it. + it 'advertises the filters the search endpoint answers, and only those' do + expect(collection.fields['state'].filter_operators).to eq(%w[equal not_equal]) + expect(collection.fields['created_at'].filter_operators).to eq(%w[greater_than less_than]) + expect(collection.fields['source_body'].filter_operators).to eq(%w[contains i_contains not_contains]) + end + + # A column the table does not carry advertises nothing, which is how a + # refusal is spelled in a schema: the tag names, the company, the timeline + # and the contact identity are all read from somewhere the endpoint does + # not filter. + it 'advertises no filter on a column the endpoint does not filter' do + %w[tag_names company_name contact_name timeline contact_count].each do |column| + expect(collection.fields[column].filter_operators).to be_empty, "#{column} advertises a filter" + end + end + + it 'is searchable, Intercom matching text on the body of the first message' do + expect(collection.is_searchable?).to be(true) + end + + # The record detail is `id equals X`, answered by the record endpoint + # rather than by a filter. + it 'answers the primary key with equal and in' do + expect(collection.fields['id'].filter_operators).to eq(%w[equal in]) + end + + it 'is countable, since total_count is exact' do + expect(collection.is_countable?).to be(true) + end + + # No aggregate endpoint, so no group-by may be offered. + it 'declares no column groupable' do + expect(columns.values.map(&:is_groupable).uniq).to eq([false]) + end + + # This lot writes nothing: an editable column would offer a Save that + # reaches an `update` the collection does not implement. + it 'declares every column read-only' do + expect(collection.fields.values.map(&:is_read_only).uniq).to eq([true]) + end + end + + # All three targets are read whole in one request, and + # `/conversations/search` takes a filter on each of the three keys -- so + # these relations can be read, navigated and filtered through alike, unlike + # the state of a ticket. + describe 'the relations to the reference collections' do + it 'points every id at the collection that reads it' do + expect(collection.fields['admin_assignee']) + .to have_attributes(type: 'ManyToOne', foreign_collection: 'IntercomAdmin', + foreign_key: 'admin_assignee_id', foreign_key_target: 'id', is_read_only: true) + expect(collection.fields['team_assignee']) + .to have_attributes(foreign_collection: 'IntercomTeam', foreign_key: 'team_assignee_id') + expect(collection.fields['closed_by']) + .to have_attributes(foreign_collection: 'IntercomAdmin', foreign_key: 'closed_by_id') + end + + # The account is on the payload as a whole object, so the name is already a + # column; the Companies collection arrives with lot 4, and a relation whose + # target is missing is a schema the agent refuses to boot on. + it 'declares no relation towards the account' do + expect(collection.fields.keys).not_to include('company') + end + + it 'nests the teammate who closed it, reading the teammates once for the page' do + stub_list(conversation('1'), conversation('2')) + stub_request(:get, "#{base}/admins") + .to_return(json('type' => 'admin.list', 'admins' => [{ 'id' => '493881', 'name' => 'Alice' }])) + + rows = collection.list(nil, filter, ['id', 'closed_by:name']) + + expect(rows).to eq([{ 'id' => '1', 'closed_by' => { 'name' => 'Alice', 'id' => '493881' } }, + { 'id' => '2', 'closed_by' => { 'name' => 'Alice', 'id' => '493881' } }]) + expect(WebMock).to have_requested(:get, "#{base}/admins").once + end + + it 'filters on the foreign key the target resolved to' do + stub_request(:get, "#{base}/admins") + .to_return(json('type' => 'admin.list', 'admins' => [{ 'id' => '493881', 'name' => 'Alice' }])) + stub_search(conversation('1')) + + collection.list(nil, filter(condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Alice')), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/conversations/search") + .with(query: hash_including({}), + body: hash_including('query' => { 'field' => 'admin_assignee_id', 'operator' => '=', + 'value' => '493881' })) + end + end + + # The other half of the 360 degrees: `IntercomContact#conversations` is a + # one-to-many, and the agent serves it by listing this collection on the key + # -- so what makes that list possible is this endpoint filtering on a + # contact id at all. The column is singular and the wire field plural: the + # row names its first contact, the endpoint asks whether a contact is one of + # the conversation's. + describe 'the conversations of a contact' do + it 'filters on the contact id the relation resolves to' do + stub_search(conversation('1')) + + collection.list(nil, filter(condition_tree: leaf('contact_id', operators::EQUAL, 'c1')), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/conversations/search") + .with(query: hash_including({}), + body: hash_including('query' => { 'field' => 'contact_ids', 'operator' => '=', 'value' => 'c1' })) + end + + it 'counts them in one request' do + stub_search(conversation('1'), total: 37) + + expect(collection.aggregate(nil, filter(condition_tree: leaf('contact_id', operators::EQUAL, 'c1')), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count'))) + .to eq([{ 'group' => {}, 'value' => 37 }]) + end + end + + describe '#list' do + # No projection at all asks for every declared column, `contact_name` and + # `timeline` included, so an all-columns read pays for both enrichments -- + # the identity read of the page, and the detail read the parts are only + # returned by. Guarding them on a column being *named* would leave nil the + # very columns the row publishes. + before do + stub_request(:post, "#{base}/contacts/search") + .to_return(json('type' => 'list', 'data' => [{ 'id' => 'c1', 'name' => 'Camille' }])) + stub_record('1', conversation('1')) + end + + it 'reads the listing endpoint as plain text and pages by cursor' do + stub_list(conversation('1')) + + collection.list(nil, filter, nil) + + expect(WebMock).to have_requested(:get, "#{base}/conversations") + .with(query: hash_including('display_as' => 'plaintext')) + end + + it 'flattens the payload into the row the schema declares' do + stub_list(conversation('1')) + + row = collection.list(nil, filter, nil).first + + expect(row).to include('id' => '1', 'title' => 'Facture 1', 'state' => 'closed', 'open' => false, + 'company_id' => '696dd52099f73812610d9c7b', 'company_name' => 'Acme', + 'admin_assignee_id' => '493881', 'team_assignee_id' => '814865', + 'tag_names' => %w[billing], 'ai_agent_participated' => true) + end + + # Epoch seconds are what Intercom sends; a Date column and a date filter + # both read ISO8601, and UTC is where Intercom truncates. + it 'reads the dates as ISO8601 in UTC' do + row = (stub_list(conversation('1')) && collection.list(nil, filter, nil)).first + + expect(row['created_at']).to eq('2023-11-14T22:13:20Z') + end + + it 'flattens the lifecycle Intercom keeps in statistics' do + stub_list(conversation('1')) + + expect(collection.list(nil, filter, nil).first) + .to include('closed_at' => '2023-11-14T23:03:20Z', 'closed_by_id' => '493881', + 'last_admin_reply_at' => '2023-11-14T22:55:00Z', 'reopen_count' => 1, 'part_count' => 4) + end + + # A conversation Intercom has computed nothing for yet answers a null + # statistics; the columns then read as absent rather than as zero. + it 'reads a missing statistics block as absent, not as zero' do + stub_list(conversation('1', 'statistics' => nil)) + + expect(collection.list(nil, filter, nil).first) + .to include('closed_at' => nil, 'reopen_count' => nil) + end + + # A group conversation has several contacts: the row names the first, says + # how many there are, and the relation resolves that same first one -- the + # column and the relation cannot disagree. + it 'names the first contact and counts them' do + stub_list(conversation('1')) + + expect(collection.list(nil, filter, nil).first) + .to include('contact_id' => 'c1', 'contact_count' => 2) + end + + it 'narrows the row to the projection' do + stub_list(conversation('1')) + + expect(collection.list(nil, filter, %w[id state])).to eq([{ 'id' => '1', 'state' => 'closed' }]) + end + + # The counterpart of the guard above: a row that publishes a column has + # that column filled, rather than carrying the nil the projection put + # there. + it 'fills the enriched columns of an all-columns read' do + stub_list(conversation('1')) + + row = collection.list(nil, filter, nil).first + + expect(row['contact_name']).to eq('Camille') + expect(row['timeline']).to be_an(Array) + end + + it 'walks the cursor until the window is covered' do + first = { 'conversations' => [conversation('1'), conversation('2')], + 'pages' => { 'next' => { 'starting_after' => 'c2' } } } + stub_request(:get, "#{base}/conversations").with(query: hash_including('per_page' => '3')) + .to_return(json(first)) + stub_request(:get, "#{base}/conversations").with(query: hash_including('starting_after' => 'c2')) + .to_return(json('conversations' => [conversation('3')])) + + page = ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: 2, limit: 1) + + expect(ids(collection.list(nil, filter(page: page), %w[id]))).to eq(%w[3]) + end + + it 'reads a zero limit as an empty window, without a request' do + page = ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: 0, limit: 0) + + expect(collection.list(nil, filter(page: page), %w[id])).to be_empty + expect(WebMock).not_to have_requested(:get, /conversations/) + end + end + + describe '#list of one record' do + # What a record detail is. It goes to the record endpoint rather than to + # the listing, which is also what brings the parts along. + it 'reads id equals X through the record endpoint' do + stub_record('1', conversation('1')) + + expect(ids(collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), %w[id]))) + .to eq(%w[1]) + end + + # A stale link, or a record outside the token's scope: no record, not a + # failed page. + it 'reads a 404 as no record' do + stub_record('gone', { 'errors' => [{ 'code' => 'not_found' }] }, 404) + + expect(collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, 'gone')), %w[id])).to eq([]) + end + + it 'still raises on a failure that is not a missing record' do + stub_record('1', { 'errors' => [{ 'code' => 'forbidden' }] }, 403) + + expect { collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), %w[id]) } + .to raise_error(APIError) + end + + # A permission scope turns the record detail into `id equals X and `, which the record endpoint cannot answer: the ids name a wider + # set than the scope does, and reading them alone would serve a record the + # scope excludes. It goes to the search, where the key is a field like any + # other -- and the whole condition travels, or none of it does. + it 'reads the key through the search once a scope is filtered alongside it' do + search = stub_search(conversation('1')) + tree = branch('And', leaf('id', operators::EQUAL, '1'), leaf('state', operators::EQUAL, 'closed')) + + expect(ids(collection.list(nil, filter(condition_tree: tree), %w[id]))).to eq(%w[1]) + expect(search).to have_been_requested + end + + it 'sends the scope and the key as the one query, neither dropped' do + stub_search(conversation('1')) + tree = branch('And', leaf('id', operators::IN, %w[1 2]), leaf('open', operators::EQUAL, false)) + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expected = { 'operator' => 'AND', + 'value' => [{ 'field' => 'id', 'operator' => 'IN', 'value' => %w[1 2] }, + { 'field' => 'open', 'operator' => '=', 'value' => false }] } + + expect(a_request(:post, "#{base}/conversations/search") + .with(query: hash_including({}), body: hash_including('query' => expected))).to have_been_made + end + end + + describe '#list of several records by id' do + # What a pointing collection asks for when it reads related records in + # bulk. Intercom has no "read these records" endpoint, so it is one + # request per id -- and therefore bounded. + it 'reads each id through the record endpoint' do + %w[1 2].each do |id| + stub_record(id, conversation(id)) + end + + rows = collection.list(nil, filter(condition_tree: leaf('id', operators::IN, %w[1 2])), %w[id]) + + expect(ids(rows)).to eq(%w[1 2]) + end + + # A pointing collection pages through the records it named, and every page + # must name different ones: cut out of the records instead of out of the + # ids, the window would render the same rows on page 1 and page 2 -- and + # a page past the cap would come back empty, its ids having been dropped + # by the truncation before the window was applied. + it 'reads only the ids the page window names' do + stub_record('2', conversation('2')) + page = ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: 1, limit: 1) + tree = leaf('id', operators::IN, %w[1 2 3]) + + expect(ids(collection.list(nil, filter(condition_tree: tree, page: page), %w[id]))).to eq(%w[2]) + end + + it 'reads the first of too many and says the result is truncated' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + asked = (1..(Collections::CursorCollection::MAX_ID_READS + 3)).map(&:to_s) + stub_request(:get, %r{/conversations/\d+}).to_return(json(conversation('1'))) + + rows = collection.list(nil, filter(condition_tree: leaf('id', operators::IN, asked)), %w[id]) + + expect(rows.size).to eq(Collections::CursorCollection::MAX_ID_READS) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/read the first 25/) + end + end + + describe 'a filter Intercom answers' do + # A condition switches the read from the listing to the search endpoint, + # which is the only one that takes a filter. + it 'searches instead of listing, with the query the translator wrote' do + search = stub_search + tree = leaf('state', operators::EQUAL, 'open') + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expect(search.with(body: hash_including('query' => { 'field' => 'state', 'operator' => '=', + 'value' => 'open' }))).to have_been_made + end + + # The bodies are HTML written by end customers (R10), and a filtered read + # must not come back as markup where an unfiltered one comes back as text. + it 'asks the search for plain text too' do + search = stub_search + + collection.list(nil, filter(condition_tree: leaf('open', operators::EQUAL, true)), %w[id]) + + expect(search.with(query: hash_including('display_as' => 'plaintext'))).to have_been_made + end + + it 'walks the search cursor for the window a list view asked for' do + stub_request(:post, "#{base}/conversations/search") + .with(query: hash_including({})) + .to_return(json({ 'conversations' => [conversation('1'), conversation('2')], + 'pages' => { 'next' => { 'starting_after' => 'c2' } } })) + stub_request(:post, "#{base}/conversations/search") + .with(query: hash_including({}), + body: hash_including('pagination' => hash_including('starting_after' => 'c2'))) + .to_return(json('conversations' => [conversation('3')])) + page = ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: 2, limit: 1) + + rows = collection.list(nil, filter(condition_tree: leaf('open', operators::EQUAL, true), page: page), %w[id]) + + expect(ids(rows)).to eq(%w[3]) + end + + # Per word rather than as a substring, which the README says out loud. + it 'answers a free-text search on the body of the message that opened the conversation' do + search = stub_search + searched = ForestAdminDatasourceToolkit::Components::Query::Filter.new(search: ' facture ') + + collection.list(nil, searched, %w[id]) + + expect(search.with(body: hash_including('query' => { 'field' => 'source.body', 'operator' => '~', + 'value' => 'facture' }))).to have_been_made + end + + # Written as one tree rather than added to the translated query: the + # nesting Intercom allows is then checked over the whole of it. + it 'ands a free-text search with the condition it came with' do + search = stub_search + searched = ForestAdminDatasourceToolkit::Components::Query::Filter.new( + search: 'facture', condition_tree: leaf('open', operators::EQUAL, true) + ) + + collection.list(nil, searched, %w[id]) + + expect(search.with { |request| JSON.parse(request.body)['query']['operator'] == 'AND' }).to have_been_made + end + + # The date bounds Intercom answers are the ones the day rule moved, which + # is what makes an interval answer the day it names. + it 'sends a date bound on the UTC day boundary that answers the day asked for' do + search = stub_search + tree = leaf('created_at', operators::GREATER_THAN, '2026-09-01T08:30:00Z') + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expect(search.with(body: hash_including('query' => hash_including('value' => Time.utc(2026, 8, + 31).to_i)))) + .to have_been_made + end + end + + describe 'a filter it cannot honour' do + # A condition dropped on the way to Intercom comes back as an unfiltered + # page that looks filtered, which is the one answer this datasource must + # not give. + it 'refuses a condition on a column the endpoint does not filter' do + expect { collection.list(nil, filter(condition_tree: leaf('tag_names', operators::EQUAL, 'billing')), %w[id]) } + .to raise_error(UnsupportedOperatorError, /cannot filter "tag_names"/) + end + + it 'refuses an operator the endpoint does not answer on that column' do + expect { collection.list(nil, filter(condition_tree: leaf('state', operators::CONTAINS, 'op')), %w[id]) } + .to raise_error(UnsupportedOperatorError, /answers equal, not_equal on "state"/) + end + + it 'makes no request at all when it refuses' do + expect { collection.list(nil, filter(condition_tree: leaf('tag_names', operators::EQUAL, 'x')), %w[id]) } + .to raise_error(UnsupportedOperatorError) + expect(a_request(:post, "#{base}/conversations/search")).not_to have_been_made + end + end + + describe 'a sort Intercom ignores' do + before { allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) } + + # Measured: a sort sent to this endpoint raises nothing and changes + # nothing, so an order the operator asked for and did not get can only be + # reported here. + it 'reports the order it did not get' do + stub_list(conversation('1')) + sort = ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: 'created_at', ascending: false }]) + + collection.list(nil, filter(sort: sort), %w[id]) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/ignores a sort/) + end + + # `?sort=-id` is an order the operator asked for, not the ascending default + # the agent injects when a request names none. + it 'reports an explicit descending order on the primary key' do + stub_list(conversation('1')) + sort = ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: 'id', ascending: false }]) + + collection.list(nil, filter(sort: sort), %w[id]) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/ignores a sort/) + end + + it 'stays quiet on the primary-key order the agent injects by default' do + stub_list(conversation('1')) + sort = ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: 'id', ascending: true }]) + + collection.list(nil, filter(sort: sort), %w[id]) + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + end + + describe '#aggregate' do + def aggregation(operation, field: nil, groups: []) + ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: operation, field: field, + groups: groups) + end + + # One request, and exact on the whole collection rather than on the page + # the walk happened to read. + it 'counts through total_count' do + stub_list(conversation('1'), total: 81_142, query: hash_including('per_page' => '1')) + + expect(collection.aggregate(nil, filter, aggregation('Count'))) + .to eq([{ 'group' => {}, 'value' => 81_142 }]) + end + + it 'counts the records an id lookup found' do + stub_record('1', conversation('1')) + + expect(collection.aggregate(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), + aggregation('Count')).first['value']).to eq(1) + end + + # Grouping over the pages a walk collected would look exact while + # answering a fraction. + it 'refuses a group-by' do + expect { collection.aggregate(nil, filter, aggregation('Count', groups: [{ field: 'state' }])) } + .to raise_error(UnsupportedOperatorError, /can only be counted/) + end + + it 'refuses a sum' do + expect { collection.aggregate(nil, filter, aggregation('Sum', field: 'reopen_count')) } + .to raise_error(UnsupportedOperatorError, /can only be counted/) + end + + # `total_count` is exact on a search too, so a filtered count is one + # request over the whole filtered set rather than over a page of it. + it 'counts a filtered collection through the search, in one request' do + stub_search(total: 1_234) + + value = collection.aggregate(nil, filter(condition_tree: leaf('state', operators::EQUAL, 'open')), + aggregation('Count')).first['value'] + + expect(value).to eq(1_234) + end + + it 'refuses to count what it refuses to list' do + expect do + collection.aggregate(nil, filter(condition_tree: leaf('tag_names', operators::EQUAL, 'billing')), + aggregation('Count')) + end.to raise_error(UnsupportedOperatorError, /cannot filter "tag_names"/) + end + + # Counting the pages a walk collected would answer a fraction as if it + # were the whole, so a listing with no total_count is a listing this + # cannot count. + it 'refuses to count a listing Intercom answered without a total_count' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json('conversations' => [], + 'pages' => { 'type' => 'pages' })) + + expect { collection.aggregate(nil, filter, aggregation('Count')) } + .to raise_error(UnsupportedOperatorError, /without a total_count/) + end + end + + describe 'the contact identity' do + before do + stub_list(conversation('1')) + stub_request(:post, "#{base}/contacts/search") + .to_return(json('type' => 'list', + 'data' => [{ 'id' => 'c1', 'name' => 'Camille', 'email' => 'camille@acme.test' }])) + end + + # One label on the row plus the relation to navigate, which is the rule + # lot 2.5 set for the ticket labels: `contact_email` is gone, it is a hop + # away on `contact:email`. + it 'reads the identity of the page in one request and puts it on the row' do + row = collection.list(nil, filter, %w[id contact_name]).first + + expect(row).to include('contact_name' => 'Camille') + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").once + end + + it 'no longer publishes the e-mail the relation carries' do + expect(collection.fields.keys).not_to include('contact_email', 'contact_ids') + end + + it 'asks for the contacts of the page by id' do + collection.list(nil, filter, %w[id contact_name]) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'id', 'operator' => 'IN', 'value' => %w[c1] })) + end + + # A page that never asked for the identity must not pay for it. + it 'reads nothing when the projection does not name it' do + collection.list(nil, filter, %w[id state]) + + expect(WebMock).not_to have_requested(:post, "#{base}/contacts/search") + end + + # An identity that could not be read is not a page that could not be + # served: it costs the two columns. + it 'degrades to empty columns when the read fails' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:post, "#{base}/contacts/search").to_return(json({ 'errors' => [] }, 403)) + + row = collection.list(nil, filter, %w[id contact_name]).first + + expect(row['contact_name']).to be_nil + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/could not read the contacts/) + end + end + + describe 'the timeline' do + let(:conversation_with_parts) do + conversation('1').merge(parts(part('assignment', 'id' => 'p1', 'body' => nil, 'created_at' => 1_700_000_100), + part('comment', 'id' => 'p2', 'created_at' => 1_700_002_500))) + end + + # The message that opened the conversation lives in `source`, not in the + # parts: a timeline built from the parts alone loses what the customer + # actually asked. + it 'opens on the source message' do + stub_record('1', conversation_with_parts) + + timeline = collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), + %w[id timeline]).first['timeline'] + + expect(timeline.first).to include('part_type' => 'conversation_started', + 'body' => 'Bonjour, ou est ma facture ?', + 'author_name' => 'Camille', 'created_at' => '2023-11-14T22:13:20Z') + end + + # An assignment, a note and a reply are not the same event; a thread that + # flattens them reads as a conversation that never happened that way. + it 'keeps the part type of every entry' do + stub_record('1', conversation_with_parts) + + timeline = collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), + %w[timeline]).first['timeline'] + + expect(timeline.map { |entry| entry['part_type'] }).to eq(%w[conversation_started assignment comment]) + end + + it 'costs no request on a record read, the parts riding along with it' do + stub_record('1', conversation_with_parts) + + collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), %w[timeline]) + + expect(WebMock).to have_requested(:get, "#{base}/conversations/1") + .with(query: hash_including({})).once + end + + # Intercom returns the parts only when retrieving one conversation, so a + # list view pays a request per row. + it 'reads the record when a listed row has no parts' do + stub_list(conversation('1')) + stub_record('1', conversation_with_parts) + + rows = collection.list(nil, filter, %w[id timeline]) + + expect(rows.first['timeline'].size).to eq(3) + end + + it 'reads nothing when the projection does not name it' do + stub_list(conversation('1')) + + collection.list(nil, filter, %w[id state]) + + expect(WebMock).not_to have_requested(:get, "#{base}/conversations/1").with(query: hash_including({})) + end + + # A conversation deleted between the page and the read of its timeline: + # the row keeps a nil timeline rather than failing the whole page. + it 'leaves the timeline unread when the record has gone' do + stub_list(conversation('1')) + stub_record('1', { 'errors' => [{ 'code' => 'not_found' }] }, 404) + + expect(collection.list(nil, filter, %w[id timeline]).first['timeline']).to be_nil + end + + it 'still raises when the timeline read fails for another reason' do + stub_list(conversation('1')) + stub_record('1', { 'errors' => [{ 'code' => 'forbidden' }] }, 403) + + expect { collection.list(nil, filter, %w[id timeline]) }.to raise_error(APIError) + end + + # Rows past the cap keep a nil timeline -- unknown -- rather than an empty + # list, which would read as "this conversation has no message". + it 'bounds the fan-out and says what it left unread' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + listed = (1..(described_class::MAX_TIMELINE_READS + 2)).map { |index| conversation(index.to_s) } + stub_list(*listed) + stub_request(:get, %r{/conversations/\d+}).to_return(json(conversation_with_parts)) + + rows = collection.list(nil, filter, %w[id timeline]) + + expect(rows.count { |row| row['timeline'].nil? }).to eq(2) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/left the timeline of 2 row/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/fetch_all_collection_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/fetch_all_collection_spec.rb new file mode 100644 index 000000000..f901ef278 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/fetch_all_collection_spec.rb @@ -0,0 +1,309 @@ +module ForestAdminDatasourceIntercom + # The in-memory tier is exercised through Admin, a real collection carrying one + # column of each kind it has to handle: strings, booleans and a list. + RSpec.describe Collections::FetchAllCollection do + subject(:collection) { Collections::Admin.new(datasource) } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def filter(condition_tree: nil, page: nil, sort: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree, page: page, + sort: sort) + end + + def page(offset, limit) + ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: limit) + end + + def sort(*clauses) + ForestAdminDatasourceToolkit::Components::Query::Sort.new(clauses) + end + + def aggregation(operation, field: nil, groups: []) + ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: operation, field: field, + groups: groups) + end + + def admin(id, overrides = {}) + { 'type' => 'admin', 'id' => id, 'name' => "Admin #{id}", 'email' => "#{id}@acme.test", + 'away_mode_enabled' => false, 'has_inbox_seat' => true, 'team_ids' => [] }.merge(overrides) + end + + def stub_admins(*admins) + stub_request(:get, "#{base}/admins") + .to_return(status: 200, body: { 'type' => 'admin.list', 'admins' => admins }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end + + def ids(records) + records.map { |record| record['id'] } + end + + # An operator is evaluable in memory when `ConditionTreeLeaf#match` handles + # it natively or the toolkit can rewrite it into operators that it does. + def evaluable?(operator, column_type) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeEquivalent + .equivalent_tree?(operator, described_class::IN_MEMORY_OPERATORS, column_type) + end + + # A projection naming nothing asks for every declared column, `team_names` + # among them, so a list with no projection reads the teams as well. + before do + stub_request(:get, "#{base}/teams") + .to_return(status: 200, body: { 'type' => 'team.list', 'teams' => [] }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end + + # The count and the group are taken over every record Intercom holds, not + # over a page of them, which is what makes them exact. + it 'is countable' do + expect(collection.is_countable?).to be(true) + end + + describe 'columns' do + it 'declares a scalar column filterable, sortable and groupable' do + expect(collection.fields['name']) + .to have_attributes(is_sortable: true, is_groupable: true) + expect(collection.fields['name'].filter_operators).to include(operators::EQUAL) + end + + # This lot writes nothing: an editable column would offer a Save that + # reaches an `update` the collection does not implement. + it 'declares every column read-only' do + expect(collection.fields.values.map(&:is_read_only).uniq).to eq([true]) + end + + # A list has no in-memory counterpart for any of the three. + it 'declares a Json column neither filterable nor sortable' do + expect(collection.fields['team_names']) + .to have_attributes(column_type: 'Json', is_sortable: false, is_groupable: false, filter_operators: []) + end + + # A filter the UI offers and the collection then answers by emptying the + # page is the failure this whole datasource is built to avoid. + it 'advertises only operators it can actually evaluate' do + advertised = collection.fields.select { |_, field| field.type == 'Column' }.flat_map do |_name, column| + column.filter_operators.map { |operator| [operator, column.column_type] } + end + + expect(advertised.reject { |operator, type| evaluable?(operator, type) }).to be_empty + end + end + + describe '#list' do + it 'reads every record of the endpoint and serializes it' do + stub_admins(admin('1'), admin('2')) + + expect(ids(collection.list(nil, filter, nil))).to eq(%w[1 2]) + end + + it 'narrows the record to the projection' do + stub_admins(admin('1')) + + expect(collection.list(nil, filter, %w[id email])).to eq([{ 'id' => '1', 'email' => '1@acme.test' }]) + end + + # The workspace's own lists are held for the configured window rather than + # re-read by every page: they are what four relations of a ticket list + # resolve through, and they change a few times a year. See `Cache`. + it 'serves the next list from the store, within the window' do + stub_admins(admin('1')) + + 2.times { collection.list(nil, filter, nil) } + + expect(WebMock).to have_requested(:get, "#{base}/admins").once + end + + # The escape hatch, for a deployment that would rather pay the request + # than hold a list for a minute. + it 'reads the endpoint again on the next list when the store is off' do + unstored = Collections::Admin.new( + Datasource.new(access_token: 's3cr3t', rate_limiter: nil, reference_cache_ttl: 0) + ) + stub_admins(admin('1')) + + 2.times { unstored.list(nil, filter, nil) } + + expect(WebMock).to have_requested(:get, "#{base}/admins").twice + end + + it 'propagates a failure rather than answering with no record' do + stub_request(:get, "#{base}/admins").to_return(status: 500, body: '{}', + headers: { 'Content-Type' => 'application/json' }) + + expect { collection.list(nil, filter, nil) }.to raise_error(APIError) + end + end + + describe '#list with a filter' do + before { stub_admins(admin('1', 'name' => 'Alice'), admin('2', 'name' => 'Bob', 'has_inbox_seat' => false)) } + + def filtered(field, operator, value = nil) + ids(collection.list(nil, filter(condition_tree: leaf(field, operator, value)), nil)) + end + + it 'keeps the rows a string condition names' do + expect(filtered('name', operators::EQUAL, 'Alice')).to eq(%w[1]) + end + + it 'keeps the rows a boolean condition names' do + expect(filtered('has_inbox_seat', operators::EQUAL, false)).to eq(%w[2]) + end + + it 'answers an operator it advertises through an equivalence' do + expect(filtered('name', operators::I_CONTAINS, 'ali')).to eq(%w[1]) + end + + it 'answers nothing when nothing matches, rather than everything' do + expect(filtered('name', operators::EQUAL, 'Nobody')).to be_empty + end + + # `match` answers nil for an operator with no equivalence and `apply` reads + # that as "no match", so this would otherwise be an empty page an operator + # cannot tell from a real answer. The schema advertises no such operator; a + # scope or a segment can still send one. + it 'refuses a condition it cannot evaluate instead of emptying the page' do + expect { filtered('team_ids', operators::EQUAL, 'x') } + .to raise_error(UnsupportedOperatorError, /cannot filter 'team_ids' with 'equal'/) + end + + it 'refuses a condition on a column it does not carry' do + expect { filtered('unknown', operators::EQUAL, 'x') } + .to raise_error(UnsupportedOperatorError, /cannot filter 'unknown'/) + end + + it 'names a refusal after the operator, so the message says what to change' do + expect { filtered('name', operators::LESS_THAN, 'x') } + .to raise_error(UnsupportedOperatorError, /'less_than'/) + end + end + + describe '#list with a sort' do + before do + stub_admins(admin('2', 'name' => 'Bob'), admin('1', 'name' => 'Alice'), admin('3', 'name' => nil)) + end + + it 'orders on the column asked for' do + expect(ids(collection.list(nil, filter(sort: sort({ field: 'name', ascending: true })), nil))) + .to eq(%w[1 2 3]) + end + + # Nulls last ascending, first descending, the way a database orders them. + it 'puts a null first on a descending order' do + expect(ids(collection.list(nil, filter(sort: sort({ field: 'name', ascending: false })), nil))) + .to eq(%w[3 2 1]) + end + + it 'keeps the order Intercom returned for rows the sort cannot separate' do + rows = collection.list(nil, filter(sort: sort({ field: 'away_mode_enabled', ascending: true })), nil) + + expect(ids(rows)).to eq(%w[2 1 3]) + end + + # An order asked for and not honoured is reported everywhere else in this + # datasource, and a related list through a many-to-many reaches this tier + # with the columns of the collection it *reaches* rather than the one it + # travels through -- `Filter#nest` prefixes the condition tree and not the + # sort. Dropped in silence, that is the one order nothing would report. + it 'drops a clause naming a column it does not carry, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + rows = collection.list(nil, filter(sort: sort({ field: 'unknown', ascending: true })), nil) + + expect(ids(rows)).to eq(%w[2 1 3]) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/sort on "unknown", which it does not carry as a sortable column/) + end + + # A Json column holds a list, which has no order of its own. + it 'drops a clause on a column it cannot sort, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + rows = collection.list(nil, filter(sort: sort({ field: 'team_ids', ascending: true })), nil) + + expect(ids(rows)).to eq(%w[2 1 3]) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/sort on "team_ids", which it does not carry as a sortable column/) + end + + it 'keeps the clauses it does carry alongside the one it drops' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + rows = collection.list( + nil, filter(sort: sort({ field: 'unknown', ascending: true }, { field: 'name', ascending: true })), nil + ) + + expect(ids(rows)).to eq(%w[1 2 3]) + end + end + + describe '#list with a page' do + before { stub_admins(admin('1'), admin('2'), admin('3')) } + + it 'cuts the window out of the records in hand' do + expect(ids(collection.list(nil, filter(page: page(1, 1)), nil))).to eq(%w[2]) + end + + it 'reads a page with no limit as every record past the offset' do + expect(ids(collection.list(nil, filter(page: page(1, nil)), nil))).to eq(%w[2 3]) + end + + # What the toolkit's `Page#apply` answers for a zero limit. + it 'reads a zero limit as an empty window, not as every record' do + expect(collection.list(nil, filter(page: page(1, 0)), nil)).to be_empty + end + + it 'answers an offset past the end with no record' do + expect(collection.list(nil, filter(page: page(50, 10)), nil)).to be_empty + end + end + + describe '#aggregate' do + before { stub_admins(admin('1', 'name' => 'Alice'), admin('2', 'name' => 'Bob', 'has_inbox_seat' => false)) } + + it 'counts every record, exactly' do + expect(collection.aggregate(nil, filter, aggregation('Count'))) + .to eq([{ 'group' => {}, 'value' => 2 }]) + end + + it 'counts the rows a filter keeps' do + filtered = filter(condition_tree: leaf('has_inbox_seat', operators::EQUAL, true)) + + expect(collection.aggregate(nil, filtered, aggregation('Count')).first['value']).to eq(1) + end + + it 'groups on a column, which is exact for the same reason' do + rows = collection.aggregate(nil, filter, aggregation('Count', groups: [{ field: 'has_inbox_seat' }])) + + expect(rows.sum { |row| row['value'] }).to eq(2) + expect(rows.size).to eq(2) + end + end + + describe 'the hooks a collection has to implement' do + let(:incomplete) do + Class.new(described_class) do + def initialize(datasource) + super(datasource, 'Incomplete') + end + + def define_schema + add_column('id', 'String', is_primary_key: true) + end + end + end + + it 'says which one is missing rather than failing obscurely' do + expect { incomplete.new(datasource).list(nil, filter, nil) } + .to raise_error(NotImplementedError, /did not implement fetch_all/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_membership_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_membership_spec.rb new file mode 100644 index 000000000..f5e1fbf5c --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_membership_spec.rb @@ -0,0 +1,228 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::TeamMembership do + subject(:collection) { datasource.get_collection('IntercomTeamMembership') } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + def filter(condition_tree: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree) + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, *conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def stub_teams(*teams) + stub_request(:get, "#{base}/teams").to_return(json('type' => 'team.list', 'teams' => teams)) + end + + def stub_admins(*admins) + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => admins)) + end + + before { stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) } + + it 'is named IntercomTeamMembership' do + expect(collection.name).to eq('IntercomTeamMembership') + end + + it 'exposes the pair and a relation to either side of it' do + expect(collection.fields.keys).to eq(%w[id team_id admin_id team admin]) + end + + # One record per pair, keyed by both ids: a related list is read over two + # requests, and a key that changed in between would move the rows under the + # operator. + it 'reads one record per pair from the team side' do + stub_teams({ 'id' => '814865', 'name' => 'Support', 'admin_ids' => [493_881, 493_882] }, + { 'id' => '814866', 'name' => 'Billing', 'admin_ids' => [493_882] }) + + expect(collection.list(nil, filter, nil)) + .to eq([{ 'id' => '814865:493881', 'team_id' => '814865', 'admin_id' => '493881' }, + { 'id' => '814865:493882', 'team_id' => '814865', 'admin_id' => '493882' }, + { 'id' => '814866:493882', 'team_id' => '814866', 'admin_id' => '493882' }]) + end + + # A teammate who left the workspace is still named by `admin_ids` until + # Intercom drops the pair. Kept, the pair would be counted by a related + # list that cannot show it. + it 'drops a pair whose teammate /admins no longer answers, from the list and the count' do + stub_teams('id' => '814865', 'admin_ids' => [493_881, 493_883]) + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') + + expect(collection.list(nil, filter, %w[id])).to eq([{ 'id' => '814865:493881' }]) + expect(collection.aggregate(nil, filter, aggregation)).to eq([{ 'group' => {}, 'value' => 1 }]) + end + + it 'holds no record for a team nobody belongs to' do + stub_teams({ 'id' => '814865', 'admin_ids' => [] }, { 'id' => '814866', 'admin_ids' => nil }) + + expect(collection.list(nil, filter, nil)).to be_empty + end + + # Intercom types the id as a number inside `admin_ids` and as a string on the + # teammate itself: the pair carries the form the other side answers by, or + # the relation resolves to nothing. + it 'stringifies both ids' do + stub_teams('id' => 814_865, 'admin_ids' => [493_881]) + + expect(collection.list(nil, filter, nil).first) + .to eq({ 'id' => '814865:493881', 'team_id' => '814865', 'admin_id' => '493881' }) + end + + describe 'a projection through a relation' do + it 'nests the teammate under the relation, reading them once for the page' do + stub_teams('id' => '814865', 'admin_ids' => [493_881, 493_882]) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + + expect(collection.list(nil, filter, ['id', 'admin:name'])) + .to eq([{ 'id' => '814865:493881', 'admin' => { 'name' => 'Alice', 'id' => '493881' } }, + { 'id' => '814865:493882', 'admin' => { 'name' => 'Bruno', 'id' => '493882' } }]) + expect(WebMock).to have_requested(:get, "#{base}/admins").once + end + end + + # A relation is not a column: ordering by it would compare nil on every + # row, so the clause is dropped and reported. + it 'drops a sort on a relation rather than ordering by nil, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_teams('id' => '814865', 'admin_ids' => [493_882, 493_881]) + order = ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: 'team', ascending: true }]) + + rows = collection.list(nil, ForestAdminDatasourceToolkit::Components::Query::Filter.new(sort: order), %w[id]) + + expect(rows.map { |row| row['id'] }).to eq(%w[814865:493882 814865:493881]) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/sort on "team"/) + end + + # The leaderboard of a team's teammates is this aggregation: the through + # collection grouped by the far side's label. + describe 'a group through a relation' do + it 'groups by the related record, not into one nil bucket' do + stub_teams({ 'id' => '814865', 'name' => 'Support', 'admin_ids' => [493_881, 493_882] }, + { 'id' => '814866', 'name' => 'Billing', 'admin_ids' => [493_882] }) + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new( + operation: 'Count', groups: [{ field: 'team:name' }] + ) + + expect(collection.aggregate(nil, filter, aggregation)) + .to contain_exactly({ 'group' => { 'team:name' => 'Support' }, 'value' => 2 }, + { 'group' => { 'team:name' => 'Billing' }, 'value' => 1 }) + end + end + + # This tier filters in memory, over every record Intercom holds, so a + # condition through a relation becomes a plain membership on the foreign key + # -- none of the limits of Intercom's search DSL apply here, nothing going + # through it. + describe 'a condition through a relation' do + it 'keeps the pairs whose teammate the target matched' do + stub_teams('id' => '814865', 'admin_ids' => [493_881, 493_882]) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + + rows = collection.list(nil, filter(condition_tree: leaf('admin:name', operators::EQUAL, 'Alice')), %w[id]) + + expect(rows).to eq([{ 'id' => '814865:493881' }]) + end + + it 'answers nothing when the target matched no record, rather than everything' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + stub_admins('id' => '493881', 'name' => 'Alice') + + rows = collection.list(nil, filter(condition_tree: leaf('admin:name', operators::EQUAL, 'Zoe')), %w[id]) + + expect(rows).to be_empty + end + + it 'counts the pairs a relation condition keeps, exactly' do + stub_teams('id' => '814865', 'admin_ids' => [493_881, 493_882]) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') + + rows = collection.aggregate(nil, filter(condition_tree: leaf('admin:name', operators::EQUAL, 'Alice')), + aggregation) + + expect(rows).to eq([{ 'group' => {}, 'value' => 1 }]) + end + + # The shape the agent really builds: a scope, then a segment, then the + # operator's own filter, one branch at a time. + it 'keeps a relation condition standing next to a condition on its own column' do + stub_teams({ 'id' => '814865', 'admin_ids' => [493_881] }, { 'id' => '814866', 'admin_ids' => [493_881] }) + stub_admins('id' => '493881', 'name' => 'Alice') + tree = branch('And', leaf('team_id', operators::EQUAL, '814866'), + leaf('admin:name', operators::EQUAL, 'Alice')) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])) + .to eq([{ 'id' => '814866:493881' }]) + end + + # An `and` carrying a condition nothing can satisfy matches nothing itself. + it 'answers nothing when a relation condition inside an and matches nothing' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + stub_admins('id' => '493881', 'name' => 'Alice') + tree = branch('And', leaf('team_id', operators::EQUAL, '814865'), + leaf('admin:name', operators::EQUAL, 'Zoe')) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])).to be_empty + end + + # An `or` drops it and keeps its siblings: what the others name is still + # named. + it 'keeps the siblings of a relation condition inside an or' do + stub_teams({ 'id' => '814865', 'admin_ids' => [493_881] }, { 'id' => '814866', 'admin_ids' => [493_882] }) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + tree = branch('Or', leaf('admin:name', operators::EQUAL, 'Zoe'), + leaf('team_id', operators::EQUAL, '814866')) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])) + .to eq([{ 'id' => '814866:493882' }]) + end + + it 'answers nothing when every branch of an or matches nothing' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + stub_admins('id' => '493881', 'name' => 'Alice') + tree = branch('Or', leaf('admin:name', operators::EQUAL, 'Zoe'), + leaf('admin:name', operators::EQUAL, 'Yann')) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])).to be_empty + end + + # The operators a relation condition may carry are the target's own: it is + # the one that evaluates them, and the one whose refusal is worth reading. + it 'refuses an operator the target cannot evaluate, naming it' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + stub_admins('id' => '493881', 'name' => 'Alice') + + expect { collection.list(nil, filter(condition_tree: leaf('admin:team_names', operators::EQUAL, 'x')), nil) } + .to raise_error(UnsupportedOperatorError, /IntercomAdmin cannot filter 'team_names'/) + end + + it 'refuses a relation it does not declare, naming the ones it has' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + + expect { collection.list(nil, filter(condition_tree: leaf('owner:name', operators::EQUAL, 'x')), nil) } + .to raise_error(UnsupportedOperatorError, /"owner" is not a relation it filters through.*team, admin/m) + end + + it 'refuses a path reaching through two relations' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + + expect { collection.list(nil, filter(condition_tree: leaf('admin:teams:name', operators::EQUAL, 'x')), nil) } + .to raise_error(UnsupportedOperatorError, /reaches through two relations/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_spec.rb new file mode 100644 index 000000000..0dae47446 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_spec.rb @@ -0,0 +1,172 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::Team do + subject(:collection) { datasource.get_collection('IntercomTeam') } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + + def filter(condition_tree: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree) + end + + def admins_named(name) + operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new('admins:name', operators::EQUAL, name) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def stub_teams(*teams) + stub_request(:get, "#{base}/teams").to_return(json('type' => 'team.list', 'teams' => teams)) + end + + def stub_admins(*admins) + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => admins)) + end + + it 'is named IntercomTeam' do + expect(collection.name).to eq('IntercomTeam') + end + + it 'exposes the team, its teammates by name, and the relation to them' do + expect(collection.fields.keys).to eq(%w[id name admin_names admins]) + end + + it 'reads the endpoint under its own key and stringifies the ids' do + stub_teams('type' => 'team', 'id' => '814865', 'name' => 'Support', 'admin_ids' => [493_881]) + + expect(collection.list(nil, filter, %w[id name])).to eq([{ 'id' => '814865', 'name' => 'Support' }]) + end + + describe 'the teammates of a team' do + it 'names them for the whole page in one read, rather than one read per row' do + stub_teams({ 'id' => '814865', 'name' => 'Support', 'admin_ids' => [493_881, 493_882] }, + { 'id' => '814866', 'name' => 'Billing', 'admin_ids' => [493_882] }) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + + expect(collection.list(nil, filter, %w[id admin_names])) + .to eq([{ 'id' => '814865', 'admin_names' => %w[Alice Bruno] }, + { 'id' => '814866', 'admin_names' => %w[Bruno] }]) + expect(WebMock).to have_requested(:get, "#{base}/admins").once + end + + it 'names them when the projection names no column at all' do + stub_teams('id' => '814865', 'name' => 'Support', 'admin_ids' => [493_881]) + stub_admins('id' => '493881', 'name' => 'Alice') + + expect(collection.list(nil, filter, nil).first['admin_names']).to eq(['Alice']) + end + + it 'reads nothing when no projection asks for the names' do + stub_teams('id' => '814865', 'name' => 'Support', 'admin_ids' => [493_881]) + + collection.list(nil, filter, %w[id name]) + + expect(WebMock).not_to have_requested(:get, "#{base}/admins") + end + + # A missing permission costs the column, never the page -- and never the + # relation, which reads the teammates from the other side. + it 'leaves the names empty when the teammates cannot be read' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + stub_request(:get, "#{base}/admins").to_return(json({ 'type' => 'error.list' }, 403)) + + expect(collection.list(nil, filter, %w[id admin_names]).first) + .to eq({ 'id' => '814865', 'admin_names' => [] }) + end + end + + # Intercom carries the membership on the team and on the teammate both and + # exposes no resource for the pair, so the many-to-many travels through the + # membership collection this datasource synthesizes. + describe 'the relation to the teammates' do + it 'is a many-to-many through the membership collection' do + expect(collection.fields['admins']) + .to have_attributes(type: 'ManyToMany', foreign_collection: 'IntercomAdmin', + through_collection: 'IntercomTeamMembership', + origin_key: 'team_id', foreign_key: 'admin_id') + end + + # Intercom exposes no endpoint that writes a membership, and an editable + # relation would offer an association that could only fail. + it 'is read-only' do + expect(collection.fields['admins'].is_read_only).to be(true) + end + + # What the toolkit looks for on the collection a many-to-many travels + # through. Without the two, a related list falls back to a filter on the + # foreign collection that nothing there can answer. + it 'is reachable from both ends of the membership' do + utils = ForestAdminDatasourceToolkit::Utils::Collection + + expect(utils.get_through_target(collection, 'admins')).to eq('admin') + expect(utils.get_through_origin(collection, 'admins')).to eq('team') + end + + # A many-to-many is published unfilterable, so nothing the interface offers + # reaches here -- a scope or a segment still can, and it is refused rather + # than resolved by reading the membership once per value. + it 'refuses a condition through the membership' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + + expect { collection.list(nil, filter(condition_tree: admins_named('Alice')), nil) } + .to raise_error(UnsupportedOperatorError, /"admins" is not a relation it filters through. It has none/) + end + + # The path a related list really takes: through the membership, whose own + # relation towards the teammate is what carries the rows back. + it 'lists the teammates of one team, through the membership' do + stub_teams({ 'id' => '814865', 'admin_ids' => [493_881] }, { 'id' => '814866', 'admin_ids' => [493_882] }) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + query = ForestAdminDatasourceToolkit::Components::Query + + rows = ForestAdminDatasourceToolkit::Utils::Collection.list_relation( + collection, %w[814865], 'admins', nil, query::Filter.new, query::Projection.new(%w[id name]) + ) + + expect(rows).to eq([{ 'id' => '493881', 'name' => 'Alice' }]) + end + + # `admin_ids` names a teammate `/admins` does not answer -- one who left + # the workspace, one outside the token's reach, one past the page cap of a + # large workspace. The membership nests a nil under the relation, which is + # right on a row an operator reads; unwrapped into a related list it would + # be a row holding nothing, and the serializer reads a row by key. + it 'drops a teammate the workspace no longer answers, rather than listing an empty row' do + stub_teams('id' => '814865', 'admin_ids' => [493_881, 999_999]) + stub_admins('id' => '493881', 'name' => 'Alice') + query = ForestAdminDatasourceToolkit::Components::Query + + rows = ForestAdminDatasourceToolkit::Utils::Collection.list_relation( + collection, %w[814865], 'admins', nil, query::Filter.new, query::Projection.new(%w[id name]) + ) + + expect(rows).to eq([{ 'id' => '493881', 'name' => 'Alice' }]) + end + + # The membership is handed the columns of the collection the relation + # reaches, `Filter#nest` prefixing the condition tree and not the sort. It + # cannot order on them, and says so rather than answering an order it did + # not honour. + it 'reports the order it cannot honour on a related list' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_teams('id' => '814865', 'admin_ids' => [493_881, 493_882]) + stub_admins({ 'id' => '493881', 'name' => 'Zoe' }, { 'id' => '493882', 'name' => 'Alice' }) + query = ForestAdminDatasourceToolkit::Components::Query + + ForestAdminDatasourceToolkit::Utils::Collection.list_relation( + collection, %w[814865], 'admins', nil, + query::Filter.new(sort: query::Sort.new([{ field: 'name', ascending: true }])), + query::Projection.new(%w[id name]) + ) + + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/sort on "name", which it does not carry as a sortable column/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_list_budget_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_list_budget_spec.rb new file mode 100644 index 000000000..bda6fd3c4 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_list_budget_spec.rb @@ -0,0 +1,160 @@ +module ForestAdminDatasourceIntercom + # What one page of a list view costs in requests, counted end to end. + # + # Every other spec of this package asserts what a collection answers; this one + # asserts what it spends to answer it, because that is what an operator waits + # through. Intercom joins nothing, so a page projecting five relations used to + # be seven sequential round trips -- the page itself, the contacts read twice + # over, and the four workspace lists re-read on every page. + # + # The figures below are the budget. A change that raises one of them is a + # change that makes the list view slower, and it should have to say so here. + RSpec.describe Collections::Ticket, '#list' do + let(:base) { Configuration::REGION_HOSTS[:us] } + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + + def json(payload) + { status: 200, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def ticket(id) + { 'type' => 'ticket', 'id' => id, 'ticket_id' => "1#{id}", 'category' => 'Customer', 'open' => true, + 'created_at' => 1_700_000_000, 'admin_assignee_id' => 493_881, 'team_assignee_id' => 12, + 'ticket_state' => { 'id' => '19', 'internal_label' => 'En cours' }, + 'previous_ticket_state_id' => '14', + 'ticket_type' => { 'id' => '1', 'name' => 'Bug' }, + 'contacts' => { 'type' => 'contact.list', 'contacts' => [{ 'id' => "c#{id}" }] }, + 'ticket_parts' => { 'type' => 'ticket_part.list', 'ticket_parts' => [] } } + end + + # Columns and relations both, the way Forest projects a list view: a + # many-to-one is displayed through a column of its target. + def projection + ForestAdminDatasourceToolkit::Components::Query::Projection.new( + %w[id ticket_id category open created_at state_label contact_name + admin_assignee:id admin_assignee:name team_assignee:id team_assignee:name + state:id state:internal_label previous_state:id previous_state:internal_label + ticket_type:id ticket_type:name contact:id contact:name] + ) + end + + def list_page(collection = 'IntercomTicket') + filter = ForestAdminDatasourceToolkit::Components::Query::Filter.new( + page: ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: 0, limit: 15) + ) + + datasource.get_collection(collection).list(nil, filter, projection) + end + + def search_answer + json('type' => 'ticket.list', 'total_count' => 500, 'pages' => { 'page' => 1 }, + 'tickets' => (1..15).map { |index| ticket(index.to_s) }) + end + + # Counted at the HTTP layer rather than per stub: what matters is the number + # of round trips, whichever endpoint they reach. + # + # WebMock registers callbacks globally and offers no way to remove one, so + # the reset takes every callback with it. Nothing else in this suite + # registers any -- this is the only spec that counts requests rather than + # asserting them -- and leaving this one in place would have it counting + # into a dead hash for the rest of the run. + def requests_of + calls = Hash.new(0) + WebMock.after_request { |request, _| calls["#{request.method.to_s.upcase} #{request.uri.path}"] += 1 } + yield + calls + ensure + WebMock::CallbackRegistry.reset + end + + before do + # The suite stubs `/ticket_types` empty for the boot read every datasource + # issues; this one installs the type its tickets carry, so the relation + # resolves to a row rather than to nil and the budget below is the budget + # of a page that answered. + stub_ticket_types({ 'id' => '1', 'name' => 'Bug' }) + stub_request(:get, "#{base}/admins") + .to_return(json('type' => 'admin.list', + 'admins' => [{ 'id' => '493881', 'name' => 'Alice', 'team_ids' => [12] }])) + stub_request(:get, "#{base}/teams") + .to_return(json('type' => 'team.list', 'teams' => [{ 'id' => '12', 'name' => 'Support' }])) + stub_request(:get, "#{base}/ticket_states") + .to_return(json('type' => 'list', + 'data' => [{ 'id' => '19', 'internal_label' => 'En cours' }, + { 'id' => '14', 'internal_label' => 'Soumis' }])) + stub_request(:post, "#{base}/tickets/search").with(query: hash_including({})).to_return(search_answer) + stub_request(:post, "#{base}/contacts/search") + .to_return(json('type' => 'list', + 'data' => (1..15).map { |index| { 'id' => "c#{index}", 'name' => "Contact #{index}" } })) + # The boot reads, so the counts below are the page's own. + datasource + end + + # A budget is only a budget if the page it counts is a page. Without this, + # a change resolving every relation to nil would spend the same requests and + # read here as an improvement -- the five reads would still be issued, and + # nothing would be asserting that any of them landed on a row. + it 'answers a page whose five relations are resolved' do + row = list_page.first + + expect(row).to include( + 'id' => '1', 'state_label' => 'En cours', 'contact_name' => 'Contact 1', + 'admin_assignee' => { 'id' => '493881', 'name' => 'Alice' }, + 'team_assignee' => { 'id' => '12', 'name' => 'Support' }, + 'state' => { 'id' => '19', 'internal_label' => 'En cours' }, + 'previous_state' => { 'id' => '14', 'internal_label' => 'Soumis' }, + 'ticket_type' => { 'id' => '1', 'name' => 'Bug' }, + 'contact' => { 'id' => 'c1', 'name' => 'Contact 1' } + ) + end + + # The first page after a boot pays for the workspace lists it is the first to + # need -- three of the four, `/ticket_types` being held from the boot read + # that introspects the ticket attributes. The contacts are read once for the + # column and the relation both, where that used to be two requests. + it 'spends five requests on the first page of tickets, five relations projected' do + calls = requests_of { list_page } + + expect(calls).to eq('POST /tickets/search' => 1, 'POST /contacts/search' => 1, + 'GET /admins' => 1, 'GET /teams' => 1, 'GET /ticket_states' => 1) + end + + # And every page after it, for as long as the window holds: the page itself, + # and the contacts of the page. + it 'spends two requests on the next page' do + list_page + calls = requests_of { list_page } + + expect(calls).to eq('POST /tickets/search' => 1, 'POST /contacts/search' => 1) + end + + # The one that cost a request per page and answered the same rows every time. + it 'reads the contacts of a page once for the column and the relation both' do + list_page + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").once + end + + it 'reads each workspace list once across three pages' do + calls = requests_of { 3.times { list_page } } + + expect(calls).to eq('POST /tickets/search' => 3, 'POST /contacts/search' => 3, + 'GET /admins' => 1, 'GET /teams' => 1, 'GET /ticket_states' => 1) + end + + # Without the two stores, the seven round trips the change was made against. + context 'with the stores off' do + let(:datasource) do + Datasource.new(access_token: 's3cr3t', rate_limiter: nil, reference_cache_ttl: 0) + end + + it 'spends a request per relation target on every page' do + calls = requests_of { 3.times { list_page } } + + expect(calls).to include('GET /admins' => 3, 'GET /teams' => 3, + 'GET /ticket_states' => 3, 'GET /ticket_types' => 3) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb new file mode 100644 index 000000000..ab6034792 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb @@ -0,0 +1,779 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::Ticket do + subject(:collection) { described_class.new(datasource, attributes: attributes) } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + # Not read off the datasource: that would build it, and boot the ticket-type + # introspection before the stub of it exists. + let(:base) { Configuration::REGION_HOSTS[:us] } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + let(:attributes) { [attribute('_default_title_'), attribute('Due', column_type: 'Date')] } + + # `column_name` is what the schema publishes and `name` the key the payload + # uses; they differ when the workspace's own name cannot travel through a + # Forest query string. + def attribute(name, column_name: nil, column_type: 'String') + Schema::TicketAttributesIntrospector::Attribute.new(name: name, column_name: column_name || name, + column_type: column_type, data_type: 'string', + ids_by_ticket_type: { '1' => '9001' }) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def filter(condition_tree: nil, page: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree, page: page) + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, *conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + # Hand-written from the shape measured on a real workspace: the state comes + # embedded, the company as a bare id, and the parts ride along. + def ticket(id, overrides = {}) + { 'type' => 'ticket', 'id' => id, 'ticket_id' => "1#{id}", 'category' => 'Customer', + 'open' => true, 'is_shared' => false, 'created_at' => 1_700_000_000, 'updated_at' => 1_700_003_600, + 'admin_assignee_id' => 493_881, 'team_assignee_id' => 0, + 'company_id' => '696dd52099f73812610d9c7b', + 'ticket_state' => { 'type' => 'ticket_state', 'id' => '19', 'category' => 'in_progress', + 'internal_label' => 'En cours Tech', 'external_label' => 'Investigation en cours' }, + 'previous_ticket_state_id' => '14', + 'ticket_type' => { 'type' => 'ticket_type', 'id' => '1', 'name' => 'Bug' }, + 'contacts' => { 'type' => 'contact.list', 'contacts' => [{ 'type' => 'contact', 'id' => 'c1' }] }, + 'ticket_attributes' => { '_default_title_' => 'Facture manquante' }, + 'ticket_parts' => { 'type' => 'ticket_part.list', 'total_count' => 0, 'ticket_parts' => [] } } + .merge(overrides) + end + + def parts(*entries, total: nil) + { 'ticket_parts' => { 'type' => 'ticket_part.list', 'total_count' => total || entries.size, + 'ticket_parts' => entries } } + end + + def state_change(to, from: 'in_progress', at: 1_700_002_000, by: 'Alice', part_type: nil) + { 'type' => 'ticket_part', 'id' => "s#{at}", 'part_type' => part_type || 'ticket_state_updated_by_admin', + 'ticket_state' => to, 'previous_ticket_state' => from, 'created_at' => at, + 'author' => { 'type' => 'admin', 'id' => '1', 'name' => by, 'email' => 'alice@acme.test' } } + end + + def comment(at:, by: 'Alice', type: 'admin', part_type: 'comment') + { 'type' => 'ticket_part', 'id' => "c#{at}", 'part_type' => part_type, 'body' => 'Je regarde.', + 'created_at' => at, 'author' => { 'type' => type, 'id' => '1', 'name' => by } } + end + + def stub_admins(*admins) + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => admins)) + end + + def stub_ticket_states(*states) + stub_request(:get, "#{base}/ticket_states").to_return(json('type' => 'list', 'data' => states)) + end + + def stub_search(*records, total: nil, body: nil) + answer = { 'type' => 'ticket.list', 'tickets' => records, 'total_count' => total || records.size, + 'pages' => { 'type' => 'pages', 'page' => 1 } } + + request = stub_request(:post, "#{base}/tickets/search").with(query: hash_including({})) + request = request.with(body: hash_including(body)) if body + request.to_return(json(answer)) + end + + def rows(projection = nil, **options) + collection.list(nil, filter(**options), projection) + end + + # The columns alone: a relation carries neither operators nor an order. + def columns + collection.fields.select { |_, field| field.type == 'Column' } + end + + # No projection at all asks for every declared column, `contact_name` + # included, so the reads below pay the identity read of the page -- guarding + # it on the column being *named* would leave nil the very column the row + # publishes. The blocks that assert on the identity itself stub over this. + before do + stub_request(:post, "#{base}/contacts/search") + .to_return(json('type' => 'list', 'data' => [{ 'id' => 'c1', 'name' => 'Camille' }])) + end + + describe 'schema' do + it 'is named IntercomTicket' do + expect(collection.name).to eq('IntercomTicket') + end + + # The state travels embedded, so its label costs nothing and the row does + # not depend on IntercomTicketState to be readable. One label and not + # three: the category and the customer-facing label are a hop away, on the + # relation, and neither was ever filterable -- so no segment or scope could + # rest on them. + it 'flattens the embedded state into one label, the rest being a hop away' do + expect(collection.fields.keys).to include('state_id', 'state_label', 'previous_state_id') + expect(collection.fields.keys).not_to include('state_category', 'state_external_label') + end + + it 'carries the attributes of every ticket type in union' do + expect(collection.fields.keys).to include('_default_title_', 'Due') + expect(collection.fields['Due'].column_type).to eq('Date') + end + + # `/tickets/search` ignores a sort without saying so, on every column. + it 'declares every column unsortable' do + expect(columns.values.map(&:is_sortable).uniq).to eq([false]) + end + + it 'advertises the filters the search endpoint answers, and only those' do + expect(collection.fields['category'].filter_operators).to eq(%w[equal not_equal]) + expect(collection.fields['created_at'].filter_operators).to eq(%w[greater_than less_than]) + end + + # Measured during lot 1: `/tickets/search` refuses `company_id` with + # `invalid_field` although a ticket carries one. Filtering tickets by + # account is not something this endpoint does. + it 'advertises no filter on the account, which the endpoint refuses' do + expect(collection.fields['company_id'].filter_operators).to be_empty + end + + # Derived by the agent from the parts of the ticket. A column advertising a + # filter the read cannot honour is what this lot exists to prevent. + it 'advertises no filter on the columns derived from the parts' do + %w[closed_at closed_by_name last_reply_at last_responder_name last_responder_type].each do |column| + expect(collection.fields[column].filter_operators).to be_empty, "#{column} advertises a filter" + end + end + + # R7: an attribute is filtered through an id that differs from one ticket + # type to the next, so the union column cannot say which id to use. + it 'advertises no filter on a ticket attribute while the arbitration stands' do + expect(collection.fields['Due'].filter_operators).to be_empty + expect(collection.fields['_default_title_'].filter_operators).to be_empty + end + + # The schema advertises no filter, but a scope, a segment or a customizer + # can still send one -- and the reason it is refused with is the + # arbitration itself rather than "this column is not in the table", which + # is what an attribute column would otherwise get: its name is the + # workspace's, so no row can carry it. + it 'refuses a filter on a ticket attribute with the arbitration as its reason' do + expect { rows(%w[id], condition_tree: leaf('Due', operators::EQUAL, 'x')) } + .to raise_error(UnsupportedOperatorError, /differs from one ticket type to the next/) + end + + # Intercom matches text field by field, and this endpoint exposes none + # this collection carries. Declared searchable all the same, so the search + # reaches the collection and its refusal rather than the customizer's + # emulation. + it 'answers the search itself' do + expect(collection).to be_is_searchable + end + + # An attribute overwriting a native column would show the attribute where + # the operator expects the ticket field. + it 'skips an attribute whose name a native column already carries' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + collection = described_class.new(datasource, attributes: [attribute('category')]) + + expect(collection.fields['category'].column_type).to eq('String') + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/skips the ticket attribute/) + end + end + + describe '#list' do + # There is no GET /tickets at all: even an unfiltered list view goes + # through the search endpoint with a predicate that matches everything. + it 'reads the search endpoint with a predicate matching every ticket' do + stub_search(ticket('1')) + + rows(%w[id]) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(body: hash_including('query' => { 'field' => 'created_at', 'operator' => '>', 'value' => '0' })) + end + + # Not the 150 the API accepts: a page carries every ticket's whole + # timeline, and there is no way to ask Intercom for less. + it 'asks for far fewer tickets than the API would allow' do + stub_search(ticket('1')) + + rows(%w[id]) + + pagination = hash_including('per_page' => described_class::MAX_TICKETS_PER_PAGE) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(body: hash_including('pagination' => pagination)) + end + + it 'reads the records under the tickets key' do + stub_search(ticket('1'), ticket('2')) + + expect(rows(%w[id]).map { |row| row['id'] }).to eq(%w[1 2]) + end + + it 'flattens the ticket into the row the schema declares' do + stub_search(ticket('1')) + + expect(rows.first) + .to include('id' => '1', 'ticket_id' => '11', 'category' => 'Customer', 'open' => true, + 'state_id' => '19', 'state_label' => 'En cours Tech', + 'previous_state_id' => '14', 'ticket_type_name' => 'Bug', + 'company_id' => '696dd52099f73812610d9c7b', 'admin_assignee_id' => '493881') + end + + # Intercom keys the values by attribute name, which is what lets one + # collection display the union -- and what stops it from filtering on them. + it 'reads an attribute value by its name' do + stub_search(ticket('1')) + + expect(rows.first['_default_title_']).to eq('Facture manquante') + end + + # Forest lists the fields of a request in a comma-separated query + # parameter, so a column name carrying one splits the projection into + # fields no collection has -- a 400 before the page is ever read. The + # introspector renames such an attribute; the value is still read under the + # name Intercom keys it by. + it 'reads a renamed attribute under the name the payload uses' do + renamed = attribute('ID de l\'objet (immo, facture)', column_name: "ID de l'objet (immo facture)") + collection = described_class.new(datasource, attributes: [renamed]) + stub_search(ticket('1', 'ticket_attributes' => { 'ID de l\'objet (immo, facture)' => 'immo_42' })) + + row = collection.list(nil, filter, nil).first + + expect(row["ID de l'objet (immo facture)"]).to eq('immo_42') + end + + # The invariant behind the rename, asserted on the whole schema rather than + # on one column. + it 'publishes no column name a Forest query string could not carry' do + collection = described_class.new(datasource, attributes: [attribute('Scope', column_name: 'Scope')]) + + expect(collection.fields.keys.grep(/[,:]/)).to be_empty + end + + it 'leaves an attribute of another ticket type absent rather than empty' do + stub_search(ticket('1')) + + expect(rows.first['Due']).to be_nil + end + + # The introspector names the ticket types declaring the name with another + # type; their value is not what the column holds. + it 'leaves empty the value of a ticket type declaring the attribute with another type' do + severity = attribute('Severity', column_type: 'Number') + severity.conflicting_ticket_types = ['2'] + collection = described_class.new(datasource, attributes: [severity]) + stub_search(ticket('1', 'ticket_attributes' => { 'Severity' => 3 }), + ticket('2', 'ticket_type' => { 'id' => '2', 'name' => 'Task' }, + 'ticket_attributes' => { 'Severity' => 'high' })) + + expect(collection.list(nil, filter, nil).map { |row| row['Severity'] }).to eq([3, nil]) + end + + it 'reads a date attribute as ISO8601 like every other Intercom date' do + stub_search(ticket('1', 'ticket_attributes' => { 'Due' => 1_700_000_000 })) + + expect(rows.first['Due']).to eq('2023-11-14T22:13:20Z') + end + + # A record detail goes to its own endpoint, which is not the search one. + it 'reads one ticket through the record endpoint' do + stub_request(:get, "#{base}/tickets/1").with(query: hash_including({})).to_return(json(ticket('1'))) + + expect(rows(%w[id], condition_tree: leaf('id', operators::EQUAL, '1')).map { |row| row['id'] }).to eq(%w[1]) + end + + # The search carries the filter it was given, in place of the predicate + # that matches everything. + it 'searches with the query the translator wrote' do + search = stub_search(ticket('1'), body: { 'query' => { 'field' => 'category', 'operator' => '=', + 'value' => 'request' } }) + + rows(%w[id], condition_tree: leaf('category', operators::EQUAL, 'Customer')) + + expect(search).to have_been_made + end + + # The search spells each category differently from the ticket it returns. + it 'searches a category by the word the search takes, not the label the ticket shows' do + search = stub_search(ticket('1'), body: { 'query' => { 'field' => 'category', 'operator' => '!=', + 'value' => 'task' } }) + + rows(%w[id], condition_tree: leaf('category', operators::NOT_EQUAL, 'Back-office')) + + expect(search).to have_been_made + end + + it 'refuses a category the ticket never shows, rather than sending it as typed' do + expect { rows(%w[id], condition_tree: leaf('category', operators::EQUAL, 'request')) } + .to raise_error(UnsupportedOperatorError, /with "request".*one of Customer, Back-office, Tracker/) + end + + it 'refuses a condition on a column the endpoint does not filter, by name' do + expect { rows(%w[id], condition_tree: leaf('state_category', operators::EQUAL, 'resolved')) } + .to raise_error(UnsupportedOperatorError, /cannot filter "state_category"/) + end + + # The account, measured as refused by the endpoint itself. + it 'refuses a condition on the account with the measurement as its reason' do + expect { rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, '696dd')) } + .to raise_error(UnsupportedOperatorError, /invalid_field/) + end + + it 'refuses a free-text search, having no text column the endpoint matches' do + searched = ForestAdminDatasourceToolkit::Components::Query::Filter.new(search: 'facture') + + expect { collection.list(nil, searched, %w[id]) } + .to raise_error(UnsupportedOperatorError, /cannot answer a free-text search/) + end + end + + # Every target is read whole in one request, so a relation resolves for a + # whole page at the price of a single read -- which is what makes eight of + # them affordable at all. + describe 'the relations to the reference collections' do + it 'points every id at the collection that reads it' do + expect(collection.fields['admin_assignee']) + .to have_attributes(type: 'ManyToOne', foreign_collection: 'IntercomAdmin', + foreign_key: 'admin_assignee_id', foreign_key_target: 'id', is_read_only: true) + expect(collection.fields['team_assignee']) + .to have_attributes(foreign_collection: 'IntercomTeam', foreign_key: 'team_assignee_id') + expect(collection.fields['state']) + .to have_attributes(foreign_collection: 'IntercomTicketState', foreign_key: 'state_id') + expect(collection.fields['previous_state']) + .to have_attributes(foreign_collection: 'IntercomTicketState', foreign_key: 'previous_state_id') + expect(collection.fields['ticket_type']) + .to have_attributes(foreign_collection: 'IntercomTicketType', foreign_key: 'ticket_type_id') + end + + # Declared before the attribute columns, so a workspace attribute whose + # name lands on a relation is skipped with a warning the way one landing on + # a column is. Declared after, it would collide and take the boot with it. + it 'keeps the relation when a ticket attribute carries its name' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + collection = described_class.new(datasource, attributes: [attribute('state')]) + + expect(collection.fields['state'].type).to eq('ManyToOne') + end + + it 'nests the assignee under the relation, reading the teammates once for the page' do + stub_search(ticket('1'), ticket('2')) + stub_admins('id' => '493881', 'name' => 'Alice', 'email' => 'alice@acme.test') + + rows = collection.list(nil, filter, ['id', 'admin_assignee:name']) + + expect(rows).to eq([{ 'id' => '1', 'admin_assignee' => { 'name' => 'Alice', 'id' => '493881' } }, + { 'id' => '2', 'admin_assignee' => { 'name' => 'Alice', 'id' => '493881' } }]) + expect(WebMock).to have_requested(:get, "#{base}/admins").once + end + + # A teammate who left the workspace: the row reads as unassigned rather + # than as a page that could not be served. + it 'nests nothing when the id names no teammate' do + stub_search(ticket('1')) + stub_admins('id' => '493882', 'name' => 'Bruno') + + expect(collection.list(nil, filter, ['id', 'admin_assignee:name']).first) + .to eq({ 'id' => '1', 'admin_assignee' => nil }) + end + + it 'reads no relation the projection did not name' do + stub_search(ticket('1')) + + collection.list(nil, filter, %w[id state_label]) + + expect(WebMock).not_to have_requested(:get, "#{base}/admins") + end + + # Forest's own parser builds a two-hop path, and this nests one target row + # under the relation name rather than a tree of them: the second hop would + # be dropped by the target's projection and the column would come back + # missing from a row that looks complete. Refused by name, like the filter + # that reaches that deep. + it 'refuses a projection reaching through two relations' do + stub_search(ticket('1')) + + expect { collection.list(nil, filter, ['id', 'admin_assignee:teams:name']) } + .to raise_error(UnsupportedOperatorError, /reaches through two relations/) + end + + # The price of a relation is one read per target *collection*, not one per + # relation: `state` and `previous_state` name the same endpoint, and it is + # read once, over the ids both of them point at. + it 'reads a collection two relations point at once, not once per relation' do + stub_search(ticket('1')) + stub_ticket_states({ 'id' => '19', 'internal_label' => 'En cours Tech' }, + { 'id' => '14', 'internal_label' => 'Recu' }) + + rows = collection.list(nil, filter, ['id', 'state:internal_label', 'previous_state:internal_label']) + + expect(rows.first).to eq({ 'id' => '1', + 'state' => { 'internal_label' => 'En cours Tech', 'id' => '19' }, + 'previous_state' => { 'internal_label' => 'Recu', 'id' => '14' } }) + expect(WebMock).to have_requested(:get, "#{base}/ticket_states").once + end + + # One read for two relations means one projection for two, and each of them + # still gets the columns it asked for rather than the union. + it 'nests under each relation only the columns that relation asked for' do + stub_search(ticket('1')) + stub_ticket_states({ 'id' => '19', 'category' => 'in_progress', 'internal_label' => 'En cours Tech' }, + { 'id' => '14', 'category' => 'submitted', 'internal_label' => 'Recu' }) + + rows = collection.list(nil, filter, ['id', 'state:category', 'previous_state:internal_label']) + + expect(rows.first).to eq({ 'id' => '1', + 'state' => { 'category' => 'in_progress', 'id' => '19' }, + 'previous_state' => { 'internal_label' => 'Recu', 'id' => '14' } }) + end + end + + # A relation is published filterable as soon as any column of its target is, + # so the interface offers `admin_assignee:name` the moment the relation + # exists. What Intercom is really filtered on is the foreign key: the target + # says which of its records match -- over every record it holds, not over a + # page -- and the ids it names are what the search carries. + # `IntercomContact#tickets` is a one-to-many the agent serves by listing this + # collection on the key. The row it rests on is `spec`, not `measured`: + # whether `/tickets/search` filters on a contact id at all is one of the + # probe's questions, and the answer moves the row into the table or into the + # refusals. + describe 'the tickets of a contact' do + it 'filters on the contact id the relation resolves to' do + stub_search(ticket('1')) + + collection.list(nil, filter(condition_tree: leaf('contact_id', operators::EQUAL, 'c1')), %w[id]) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(body: hash_including('query' => { 'field' => 'contact_ids', 'operator' => '=', + 'value' => 'c1' })) + end + end + + # The exchange itself, which lot 1 paid for and did not publish: the parts + # ride along in every ticket response, so the thread costs no request. + describe 'the thread' do + it 'names who said what, when, and through which kind of event' do + stub_search(ticket('1', parts(comment(at: 1_700_001_000, by: 'Camille', type: 'contact'), + comment(at: 1_700_002_000, by: 'Alice')))) + + expect(rows(%w[id timeline]).first['timeline']) + .to eq([{ 'id' => 'c1700001000', 'part_type' => 'comment', 'created_at' => '2023-11-14T22:30:00Z', + 'author_type' => 'contact', 'author_name' => 'Camille', 'author_email' => nil, + 'body' => 'Je regarde.', 'attachment_count' => 0, 'redacted' => nil }, + { 'id' => 'c1700002000', 'part_type' => 'comment', 'created_at' => '2023-11-14T22:46:40Z', + 'author_type' => 'admin', 'author_name' => 'Alice', 'author_email' => nil, + 'body' => 'Je regarde.', 'attachment_count' => 0, 'redacted' => nil }]) + end + + # An assignment, an internal note and a reply are not the same event, and + # a thread that flattens them reads as an exchange that never happened the + # way it did. Which also means the team's internal notes are in there. + it 'keeps the internal notes and the state changes, each under its own type' do + stub_search(ticket('1', parts(comment(at: 1_700_001_000, part_type: 'note'), + state_change('resolved', at: 1_700_002_000)))) + + expect(rows(%w[id timeline]).first['timeline'].map { |entry| entry['part_type'] }) + .to eq(%w[note ticket_state_updated_by_admin]) + end + + # Empty means empty here, and says so: the parts are in every response, + # unlike a conversation read from a listing, whose timeline stays nil + # because nothing is known. + it 'answers an empty thread on a ticket nothing happened to' do + stub_search(ticket('1')) + + expect(rows(%w[id timeline]).first['timeline']).to eq([]) + end + + it 'builds nothing when the projection does not name it' do + stub_search(ticket('1', parts(comment(at: 1_700_001_000)))) + + expect(rows(%w[id]).first).to eq('id' => '1') + end + + # The bodies are HTML written by end customers, and rendering third-party + # HTML inside Forest is neither safe nor useful (R10). + it 'asks Intercom for plain text rather than markup' do + stub_search(ticket('1')) + + rows(%w[id timeline]) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(query: { 'display_as' => 'plaintext' }) + end + end + + describe 'a condition through a relation' do + it 'filters on the foreign key the target resolved to' do + stub_admins('id' => '493881', 'name' => 'Alice') + stub_search(ticket('1')) + + rows(%w[id], condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Alice')) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(body: hash_including('query' => { 'field' => 'admin_assignee_id', 'operator' => '=', + 'value' => '493881' })) + end + + # Intercom takes no membership operator on these fields, so several matches + # become several conditions -- which is also why the group has a ceiling. + it 'writes a group of equalities when the target matched several records' do + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Alice' }) + stub_search(ticket('1')) + + rows(%w[id], condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Alice')) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(body: hash_including('query' => { + 'operator' => 'OR', + 'value' => [{ 'field' => 'admin_assignee_id', 'operator' => '=', + 'value' => '493881' }, + { 'field' => 'admin_assignee_id', 'operator' => '=', + 'value' => '493882' }] + })) + end + + # No row can satisfy it, and Intercom's DSL has no way of saying so: the + # search is skipped rather than sent as a filter that would come back with + # every ticket. + it 'answers nothing, and reads nothing, when the target matched no record' do + stub_admins('id' => '493881', 'name' => 'Alice') + + expect(rows(%w[id], condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Zoe'))).to be_empty + expect(WebMock).not_to have_requested(:post, %r{#{base}/tickets/search}) + end + + it 'counts none of them either, without a request' do + stub_admins('id' => '493881', 'name' => 'Alice') + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') + + counted = collection.aggregate(nil, filter(condition_tree: leaf('admin_assignee:name', + operators::EQUAL, 'Zoe')), aggregation) + + expect(counted).to eq([{ 'group' => {}, 'value' => 0 }]) + expect(WebMock).not_to have_requested(:post, %r{#{base}/tickets/search}) + end + + # A relation group nested inside the tree the agent assembled: a scope, a + # segment and the operator's own filter, and the group counts as one of + # them. + it 'nests the group inside the condition it was written next to' do + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Alice' }) + stub_search(ticket('1')) + tree = branch('And', leaf('category', operators::EQUAL, 'Customer'), + leaf('admin_assignee:name', operators::EQUAL, 'Alice')) + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(body: hash_including('query' => { + 'operator' => 'AND', + 'value' => [{ 'field' => 'category', 'operator' => '=', + 'value' => 'request' }, + { 'operator' => 'OR', + 'value' => [{ 'field' => 'admin_assignee_id', + 'operator' => '=', 'value' => '493881' }, + { 'field' => 'admin_assignee_id', + 'operator' => '=', 'value' => '493882' }] }] + })) + end + + # Intercom nests a search two levels deep, and the group a relation expands + # into is a level nobody wrote. Inlined into a parent aggregating the same + # way, it costs none -- without which a scope plus a "match any" filter + # carrying one relation condition would be refused for a nesting the + # operator cannot find in their own filter. + it 'inlines the group into a parent that aggregates the same way' do + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Alice' }) + stub_search(ticket('1')) + tree = branch('And', leaf('category', operators::EQUAL, 'Customer'), + branch('Or', leaf('open', operators::EQUAL, true), + leaf('admin_assignee:name', operators::EQUAL, 'Alice'))) + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(body: hash_including('query' => { + 'operator' => 'AND', + 'value' => [{ 'field' => 'category', 'operator' => '=', + 'value' => 'request' }, + { 'operator' => 'OR', + 'value' => [{ 'field' => 'open', 'operator' => '=', + 'value' => true }, + { 'field' => 'admin_assignee_id', + 'operator' => '=', 'value' => '493881' }, + { 'field' => 'admin_assignee_id', + 'operator' => '=', 'value' => '493882' }] }] + })) + end + + # Inlining trades a level of nesting for width, and Intercom bounds both. + # Past fifteen conditions the nested form is the one that fits, so the + # group stays where it was rather than emptying the budget it was spared. + it 'leaves the group nested when inlining it would pass fifteen conditions' do + stub_admins(*(1..3).map { |index| { 'id' => index.to_s, 'name' => 'Alice' } }) + stub_search(ticket('1')) + others = (1..14).map { |index| leaf('ticket_type_id', operators::NOT_EQUAL, "t#{index}") } + tree = branch('Or', *others, leaf('admin_assignee:name', operators::EQUAL, 'Alice')) + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expect(WebMock).to(have_requested(:post, %r{#{base}/tickets/search}).with do |request| + query = JSON.parse(request.body)['query'] + query['value'].size == 15 && query['value'].last['operator'] == 'OR' && + query['value'].last['value'].size == 3 + end) + end + + it 'reads nothing when a relation condition inside an and matches nothing' do + stub_admins('id' => '493881', 'name' => 'Alice') + tree = branch('And', leaf('category', operators::EQUAL, 'Customer'), + leaf('admin_assignee:name', operators::EQUAL, 'Zoe')) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])).to be_empty + expect(WebMock).not_to have_requested(:post, %r{#{base}/tickets/search}) + end + + # Fifteen conditions per group is Intercom's limit, and a relation reaches + # it without trying. Refused by name rather than sent and answered with a + # 400 naming neither the limit nor the filter that hit it. + # + # "more than fifteen" rather than a count: the target is read one record + # past what a group holds and no further, so what is known is that it does + # not fit -- see `match_page`. + it 'refuses a relation condition matching more records than a group holds' do + stub_admins(*(1..20).map { |index| { 'id' => index.to_s, 'name' => 'Alice' } }) + + expect { rows(%w[id], condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Alice')) } + .to raise_error(UnsupportedOperatorError, /names more than 15 records.*15 conditions per group/m) + end + + # `/tickets/search` filters no state id -- the table carries none -- so the + # state relation is there to be read and navigated, and the message says + # which of the two it is rather than naming a column the operator never + # wrote. Refused before the target is read, a refusal that spends a request + # costing exactly what it refuses. + it 'refuses a condition through a relation the endpoint filters nothing on' do + expect { rows(%w[id], condition_tree: leaf('state:category', operators::EQUAL, 'in_progress')) } + .to raise_error(UnsupportedOperatorError, %r{resolves to "state_id", on which tickets/search takes no}) + expect(WebMock).not_to have_requested(:get, /ticket_states/) + end + + it 'publishes unfilterable the relations the endpoint filters nothing on' do + filterable = collection.fields.select { |_, f| f.respond_to?(:is_filterable) }.transform_values(&:is_filterable) + + expect(filterable).to include('state' => false, 'previous_state' => false, + 'admin_assignee' => true, 'ticket_type' => true) + end + end + + describe '#aggregate' do + it 'counts a filtered collection through the total_count of its search' do + stub_search(total: 12, body: { 'query' => { 'field' => 'open', 'operator' => '=', 'value' => true } }) + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') + + expect(collection.aggregate(nil, filter(condition_tree: leaf('open', operators::EQUAL, true)), + aggregation).first['value']).to eq(12) + end + + it 'counts through the total_count of the search, exactly' do + stub_search(ticket('1'), total: 81_142) + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') + + expect(collection.aggregate(nil, filter, aggregation)).to eq([{ 'group' => {}, 'value' => 81_142 }]) + end + end + + describe 'the derived columns' do + # A ticket has no statistics block -- measured on 81 142 tickets -- so the + # closure date exists nowhere but in the parts, which ride along anyway. + it 'reads the closure from the last transition into a resolved state' do + stub_search(ticket('1', **parts(state_change('in_progress', from: 'submitted', at: 1_700_001_000), + state_change('resolved', at: 1_700_002_000, by: 'Alice')))) + + expect(rows.first).to include('closed_at' => '2023-11-14T22:46:40Z', 'closed_by_name' => 'Alice') + end + + # This workspace runs workflows: a closure done by automation carries + # another variant of the same event, and matching the admin one in full + # would make it invisible. + it 'reads a closure whatever the variant of the state-change event' do + stub_search(ticket('1', **parts(state_change('resolved', at: 1_700_002_000, + part_type: 'ticket_state_updated_by_workflow')))) + + expect(rows.first['closed_at']).to eq('2023-11-14T22:46:40Z') + end + + # Measured: a part can record a transition to the state the ticket was + # already in, and that is not an event. + it 'ignores a transition that changed nothing' do + stub_search(ticket('1', **parts(state_change('resolved', from: 'resolved', at: 1_700_002_000)))) + + expect(rows.first['closed_at']).to be_nil + end + + it 'keeps the last closure of a ticket that was reopened' do + stub_search(ticket('1', **parts(state_change('resolved', at: 1_700_001_000), + state_change('in_progress', from: 'resolved', at: 1_700_001_500), + state_change('resolved', at: 1_700_002_000)))) + + expect(rows.first['closed_at']).to eq('2023-11-14T22:46:40Z') + end + + it 'names the last responder and which side they are on' do + stub_search(ticket('1', **parts(comment(at: 1_700_001_000, by: 'Alice'), + comment(at: 1_700_002_000, by: 'Camille', type: 'contact')))) + + expect(rows.first) + .to include('last_reply_at' => '2023-11-14T22:46:40Z', 'last_responder_name' => 'Camille', + 'last_responder_type' => 'contact') + end + + # An internal note is a touch, not an answer: it would name as last + # responder someone who never wrote to the person waiting. + it 'ignores an internal note' do + stub_search(ticket('1', **parts(comment(at: 1_700_001_000, by: 'Alice'), + comment(at: 1_700_002_000, by: 'Bob', part_type: 'note')))) + + expect(rows.first['last_responder_name']).to eq('Alice') + end + + it 'leaves both columns empty on a ticket nothing happened to' do + stub_search(ticket('1')) + + expect(rows.first).to include('closed_at' => nil, 'last_responder_name' => nil) + end + + # Intercom keeps the 500 most recent parts. A resolved ticket whose + # transition fell out of that window has an *unknown* closure date, not an + # absent one -- a Date column cannot say it, so the log does. + it 'reports a resolved ticket whose timeline was truncated' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + resolved = { 'ticket_state' => { 'id' => '20', 'category' => 'resolved', 'internal_label' => 'Resolu' } } + stub_search(ticket('1', **resolved, **parts(comment(at: 1_700_001_000), total: 500))) + + rows(%w[id closed_at]) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/truncated their timeline/) + end + + it 'stays quiet when the closure is simply absent from a complete timeline' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_search(ticket('1', **parts(comment(at: 1_700_001_000)))) + + rows(%w[id closed_at]) + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_state_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_state_spec.rb new file mode 100644 index 000000000..88ab24ded --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_state_spec.rb @@ -0,0 +1,38 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::TicketState do + subject(:collection) { described_class.new(datasource) } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + + def filter + ForestAdminDatasourceToolkit::Components::Query::Filter.new + end + + def stub_ticket_states(*states) + stub_request(:get, "#{base}/ticket_states") + .to_return(status: 200, body: { 'type' => 'list', 'data' => states }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end + + it 'is named IntercomTicketState' do + expect(collection.name).to eq('IntercomTicketState') + end + + # Two labels rather than one: what the support team sees, and what the + # customer is shown. + it 'exposes both labels of a state' do + expect(collection.fields.keys).to eq(%w[id category internal_label external_label archived]) + end + + it 'reads the endpoint and serializes a state' do + stub_ticket_states('type' => 'ticket_state', 'id' => '3', 'category' => 'submitted', + 'internal_label' => 'Waiting on triage', 'external_label' => 'We are on it', + 'archived' => false) + + expect(collection.list(nil, filter, nil)) + .to eq([{ 'id' => '3', 'category' => 'submitted', 'internal_label' => 'Waiting on triage', + 'external_label' => 'We are on it', 'archived' => false }]) + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_type_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_type_spec.rb new file mode 100644 index 000000000..d924ff84b --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_type_spec.rb @@ -0,0 +1,50 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::TicketType do + subject(:collection) { described_class.new(datasource) } + + # The reference store is off here, and only here: `/ticket_types` is the one + # reference endpoint the boot also reads -- for the ticket attributes -- so + # the suite's boot stub would be the answer these examples get, rather than + # the payload each of them installs. What the store does is covered by + # `Cache` and by the client, on an endpoint nothing reads at boot. + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil, reference_cache_ttl: 0) } + let(:base) { datasource.configuration.url } + + def filter + ForestAdminDatasourceToolkit::Components::Query::Filter.new + end + + def stub_types(*types) + stub_request(:get, "#{base}/ticket_types") + .to_return(status: 200, body: { 'type' => 'list', 'data' => types }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end + + it 'is named IntercomTicketType' do + expect(collection.name).to eq('IntercomTicketType') + end + + it 'exposes what makes a ticket type readable' do + expect(collection.fields.keys).to eq(%w[id name description category icon archived]) + end + + # This endpoint uses the `data` envelope, unlike /admins and /teams. + it 'reads the endpoint through the data envelope' do + stub_types('type' => 'ticket_type', 'id' => '1', 'name' => 'Bug', 'description' => 'A bug', + 'category' => 'Customer', 'icon' => '🐛', 'archived' => false) + + expect(collection.list(nil, filter, nil)) + .to eq([{ 'id' => '1', 'name' => 'Bug', 'description' => 'A bug', 'category' => 'Customer', + 'icon' => '🐛', 'archived' => false }]) + end + + # The attribute definitions nested here are what the ticket collection reads + # to build its columns -- an attribute of the same name carries a different + # id from one type to the next -- and they are meaningless as a column. + it 'leaves the nested attribute definitions out of the schema' do + stub_types('id' => '1', 'ticket_type_attributes' => { 'type' => 'list', 'data' => [{ 'id' => '9' }] }) + + expect(collection.list(nil, filter, nil).first.keys).not_to include('ticket_type_attributes') + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/configuration_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/configuration_spec.rb new file mode 100644 index 000000000..790d255b5 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/configuration_spec.rb @@ -0,0 +1,243 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Configuration do + subject(:configuration) { described_class.new(access_token: 's3cr3t') } + + it 'defaults to the US host, since that is where a workspace lands unasked' do + expect(configuration.url).to eq('https://api.intercom.io') + end + + it 'pins the API version the spike ran against' do + expect(configuration.api_version).to eq('2.16') + end + + it 'points at the regional host it is given' do + expect(described_class.new(access_token: 's3cr3t', region: :eu).url).to eq('https://api.eu.intercom.io') + end + + it 'takes the region as a string too' do + expect(described_class.new(access_token: 's3cr3t', region: 'AU').url).to eq('https://api.au.intercom.io') + end + + it 'lets an explicit base_url win over the region, for a proxy or a mock server' do + configured = described_class.new(access_token: 's3cr3t', region: :eu, base_url: 'https://intercom.test/api/') + + expect(configured.url).to eq('https://intercom.test/api') + end + + describe 'validation' do + it 'refuses a missing access token' do + expect { described_class.new(access_token: nil) } + .to raise_error(ConfigurationError, /missing required config: access_token/) + end + + it 'refuses a blank access token' do + expect { described_class.new(access_token: ' ') } + .to raise_error(ConfigurationError, /access_token/) + end + + it 'names the regions it knows when handed one it does not' do + expect { described_class.new(access_token: 's3cr3t', region: :moon) } + .to raise_error(ConfigurationError, /unknown region :moon.*:us, :eu, :au/m) + end + + # A relative base_url makes Faraday resolve paths against the working + # directory, which surfaces much later as a failure naming nothing. + it 'refuses a base_url that is not absolute' do + expect { described_class.new(access_token: 's3cr3t', base_url: 'api.intercom.io') } + .to raise_error(ConfigurationError, /must be an absolute http\(s\) url/) + end + + it 'refuses a base_url that is not a url at all' do + expect { described_class.new(access_token: 's3cr3t', base_url: 'http://[bad') } + .to raise_error(ConfigurationError, /not a valid url/) + end + + it 'refuses an empty api_version, which would let the workspace default decide' do + expect { described_class.new(access_token: 's3cr3t', api_version: '') } + .to raise_error(ConfigurationError, /api_version cannot be empty/) + end + + it 'refuses a number of concurrent reads that is not a whole number of one or more' do + [0, -1, 2.5, '4', nil].each do |width| + expect { described_class.new(access_token: 's3cr3t', concurrent_reads: width) } + .to raise_error(ConfigurationError, /concurrent_reads must be a whole number/), width.inspect + end + end + end + + describe 'defaults' do + it 'paces requests and retries unless told otherwise' do + expect(configuration).to have_attributes(rate_limiter: an_instance_of(RateLimiter), + retry_policy: an_instance_of(RetryPolicy)) + end + + it 'is patient on a request and impatient on the boot' do + expect(configuration).to have_attributes(timeout: 30, open_timeout: 5, boot_timeout: 10, + boot_open_timeout: 3) + end + + # Every read in flight holds a connection of the pool: widening is the + # integrator's call. + it 'reads records one at a time' do + expect(configuration.concurrent_reads).to eq(1) + end + + it 'takes the pacing out of the stack when handed no limiter' do + expect(described_class.new(access_token: 's3cr3t', rate_limiter: nil).rate_limiter).to be_nil + end + end + + # `URI::HTTPS` is a `URI::HTTP`, so a cleartext base_url is accepted -- a + # mock server is one, and that is half of what the parameter is for. What it + # costs is the bearer header crossing that network in clear, and the token + # reads the whole workspace. + describe 'a cleartext base_url' do + before { allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) } + + it 'is accepted, and says the access token travels in clear over it' do + described_class.new(access_token: 's3cr3t', base_url: 'http://proxy.internal/intercom') + + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/is not https, and every request carries the Intercom access token/) + end + + it 'says nothing about a loopback host, which is nobody else s network' do + described_class.new(access_token: 's3cr3t', base_url: 'http://localhost:4010') + described_class.new(access_token: 's3cr3t', base_url: 'http://intercom.localhost') + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + + it 'says nothing about https' do + described_class.new(access_token: 's3cr3t', base_url: 'https://intercom.test') + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + end + + describe 'the reference store' do + it 'holds the workspace lists for a minute by default' do + expect(configuration.reference_cache_ttl).to eq(60) + expect(configuration.reference_cache).to be_enabled + end + + # A deployment that would rather pay the request than hold a list. + it 'is off at a ttl of zero' do + configured = described_class.new(access_token: 's3cr3t', reference_cache_ttl: 0) + + expect(configured.reference_cache).not_to be_enabled + end + + # One store per token, like the rate limiter: what it holds is one + # workspace's own lists. + it 'is one store, handed to every read of this configuration' do + store = configuration.reference_cache + + expect(configuration.reference_cache).to be(store) + end + + it 'refuses a window that runs backwards' do + expect { described_class.new(access_token: 's3cr3t', reference_cache_ttl: -1) } + .to raise_error(ConfigurationError, /reference_cache_ttl must be a finite number/) + end + + # A window that never closes is not a long window: it holds the workspace + # lists until the process restarts, which is the one thing the store was + # built not to do. + it 'refuses a window that never closes' do + expect { described_class.new(access_token: 's3cr3t', reference_cache_ttl: Float::INFINITY) } + .to raise_error(ConfigurationError, /reference_cache_ttl must be a finite number/) + end + + # Each of these used to coerce to a window of zero and turn the store off + # without a word -- an unset environment variable, above all, on the one + # option whose whole point is to be on. + [nil, 'sixty', '60', Float::NAN, [60]].each do |value| + it "refuses #{value.inspect}, rather than reading it as a window of zero" do + expect { described_class.new(access_token: 's3cr3t', reference_cache_ttl: value) } + .to raise_error(ConfigurationError, /reference_cache_ttl must be a finite number/) + end + end + end + + describe 'the adapter' do + # Seven requests to one host is seven TLS handshakes without keep-alive. + it 'defaults to none, the client resolving the persistent one' do + expect(configuration.adapter).to be_nil + expect(described_class::DEFAULT_ADAPTER.first).to eq(:net_http_persistent) + end + + # A pool narrower than the threads that can reach it queues them for half + # a second and then fails the request -- as a Faraday timeout, over a POST + # the retry policy does not replay, so an operator loses their page and + # the message accuses Intercom. The figure is only worth having while it + # stays clear of the thread counts a Rails agent is deployed with; a + # change lowering it is a change that makes that failure reachable. + it 'pools well above the threads a web server can point at it' do + expect(described_class::DEFAULT_ADAPTER.last[:pool_size]).to be >= 25 + end + + it 'takes the one it was given' do + configured = described_class.new(access_token: 's3cr3t', adapter: :net_http) + + expect(configured.adapter).to eq(:net_http) + end + + # Faraday's registry is keyed by symbol, so a name spelled as a string -- + # what a YAML config hands over -- missed every lookup and lost the + # keep-alive it was asking for, behind a warning naming the adapter it was + # falling back to as the one that was unavailable. + it 'reads a name spelled as a string, and the options with it' do + configured = described_class.new(access_token: 's3cr3t', + adapter: ['net_http_persistent', { 'pool_size' => 25 }]) + + expect(configured.adapter).to eq([:net_http_persistent, { pool_size: 25 }]) + end + + # Refused at boot like every other option, rather than degrading into a + # warning on whichever request happened to be first. + [{ pool_size: 3 }, 7, %i[net_http_persistent pool_size], [:net_http_persistent, { 3 => 1 }]].each do |value| + it "refuses #{value.inspect}" do + expect { described_class.new(access_token: 's3cr3t', adapter: value) } + .to raise_error(ConfigurationError, /adapter must be an adapter name or a \[name, options\] pair/) + end + end + end + + describe '#inspect' do + it 'never prints the bearer token' do + expect(configuration.inspect).to include('[FILTERED]') + expect(configuration.inspect).not_to include('s3cr3t') + end + + it 'still names the host and version, which is what one inspects it for' do + expect(configuration.inspect).to include('https://api.intercom.io', '2.16') + end + + # Faraday turns them into a Basic Authorization header, which the bearer + # token replaces: they would be dropped on every request without a word. + it 'refuses a base_url carrying credentials, without printing them' do + expect { described_class.new(access_token: 's3cr3t', base_url: 'https://bob:hunter2@proxy.test/api') } + .to raise_error(ConfigurationError) { |error| + expect(error.message).to include('https://[FILTERED]@proxy.test/api', 'HTTPS_PROXY') + expect(error.message).not_to include('hunter2', 'bob') + } + end + + it 'prints no credentials when refusing a url it cannot parse' do + expect { described_class.new(access_token: 's3cr3t', base_url: 'https://bob:hunter2@proxy test/api') } + .to raise_error(ConfigurationError, /not a valid url/) { |error| + expect(error.message).not_to include('hunter2') + expect(error.cause).to be_nil + } + end + + it 'masks the query string of a base_url, where a gateway takes its key' do + configured = described_class.new(access_token: 's3cr3t', base_url: 'https://proxy.test/api?api_key=hunter2') + + expect(configured.inspect).not_to include('hunter2') + expect(configured.inspect).to include('https://proxy.test/api?[FILTERED]') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_relations_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_relations_spec.rb new file mode 100644 index 000000000..4870ae965 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_relations_spec.rb @@ -0,0 +1,78 @@ +module ForestAdminDatasourceIntercom + # Read through the customizer, as the agent does: a key the agent filters a + # related list on and the collection does not publish is a crash in the + # operators equivalence decorator, long before any request leaves. Read from + # the sort layer, the last one under the hooks, which need the agent loaded. + RSpec.describe Datasource do + let(:base) { Configuration::REGION_HOSTS[:us] } + let(:stack) do + customizer = ForestAdminDatasourceCustomizer::DatasourceCustomizer.new + customizer.add_datasource(described_class.new(access_token: 's3cr3t', rate_limiter: nil), {}) + customizer.datasource(nil) + customizer.stack.sort + end + let(:caller) do + ForestAdminDatasourceToolkit::Components::Caller.new( + id: 1, email: 'ops@acme.test', first_name: 'Ops', last_name: 'Team', team: 'Ops', rendering_id: 1, + tags: {}, timezone: 'Europe/Paris', permission_level: 'admin' + ) + end + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + let(:collection_utils) { ForestAdminDatasourceToolkit::Utils::Collection } + + def json(payload) + { status: 200, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def operators_of(collection_name, field) + stack.get_collection(collection_name).schema[:fields][field].filter_operators + end + + it 'publishes, on every to-many relation, the operators the agent filters its keys with' do + stack.collections.each_value do |collection| + collection.schema[:fields].each do |name, relation| + case relation + when ForestAdminDatasourceToolkit::Schema::Relations::OneToManySchema + expect(operators_of(relation.foreign_collection, relation.origin_key)) + .to include(operators::EQUAL), "#{collection.name}.#{name} filters on an unpublished key" + when ForestAdminDatasourceToolkit::Schema::Relations::ManyToManySchema + expect(operators_of(relation.through_collection, relation.origin_key)) + .to include(operators::EQUAL), "#{collection.name}.#{name} filters on an unpublished key" + expect(operators_of(relation.through_collection, relation.foreign_key)) + .to include(operators::PRESENT), "#{collection.name}.#{name} filters on an unpublished key" + end + end + end + end + + describe 'the contacts of an account' do + let(:company) { stack.get_collection('IntercomCompany') } + let(:filter) { ForestAdminDatasourceToolkit::Components::Query::Filter.new } + + before do + stub_request(:get, "#{base}/companies/co1/contacts").with(query: hash_including({})) + .to_return(json('type' => 'list', + 'data' => [{ 'type' => 'contact', + 'id' => 'c1' }], + 'total_count' => 1, 'pages' => {})) + end + + it 'lists them from the account endpoint' do + records = collection_utils.list_relation( + company, { 'id' => 'co1' }, 'contacts', caller, filter, + ForestAdminDatasourceToolkit::Components::Query::Projection.new(%w[id]) + ) + + expect(records).to eq([{ 'id' => 'c1' }]) + end + + it 'counts them from the account endpoint' do + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') + + expect(collection_utils.aggregate_relation(company, { 'id' => 'co1' }, 'contacts', caller, filter, + aggregation)) + .to eq([{ 'group' => {}, 'value' => 1 }]) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_search_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_search_spec.rb new file mode 100644 index 000000000..492695f25 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_search_spec.rb @@ -0,0 +1,44 @@ +module ForestAdminDatasourceIntercom + # A collection that leaves `searchable` off has its search emulated by the + # customizer, as an equality on every String column. On a collection whose + # endpoint matches no text that is a page that looks searched and is not, so + # the search has to reach the collection and be refused there. + RSpec.describe Datasource do + let(:base) { Configuration::REGION_HOSTS[:us] } + let(:stack) do + customizer = ForestAdminDatasourceCustomizer::DatasourceCustomizer.new + customizer.add_datasource(described_class.new(access_token: 's3cr3t', rate_limiter: nil), {}) + customizer.datasource(nil) + customizer.stack.search + end + + def searched(text) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(search: text) + end + + def projection + ForestAdminDatasourceToolkit::Components::Query::Projection.new(%w[id]) + end + + describe 'a free-text search through the customizer' do + it 'refuses a search on tickets by name, sending nothing' do + expect { stack.get_collection('IntercomTicket').list(nil, searched('facture'), projection) } + .to raise_error(UnsupportedOperatorError, /IntercomTicket cannot answer a free-text search/) + expect(WebMock).not_to have_requested(:post, "#{base}/tickets/search") + end + + it 'refuses a search on companies by name, sending nothing' do + expect { stack.get_collection('IntercomCompany').list(nil, searched('acme'), projection) } + .to raise_error(UnsupportedOperatorError, /IntercomCompany cannot answer a free-text search/) + expect(WebMock).not_to have_requested(:post, "#{base}/companies/list") + end + + it 'refuses to count a searched company list' do + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') + + expect { stack.get_collection('IntercomCompany').aggregate(nil, searched('acme'), aggregation) } + .to raise_error(UnsupportedOperatorError, /cannot answer a free-text search/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb new file mode 100644 index 000000000..290bc4123 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb @@ -0,0 +1,107 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Datasource do + subject(:datasource) { described_class.new(access_token: 's3cr3t') } + + it 'boots without reaching Intercom' do + expect { datasource }.not_to raise_error + end + + # The reference collections come first: they are what turns an assignee id + # into a teammate and a state id into a label. Contacts and Companies + # follow, before the two collections whose relations point at them. + # The membership sits with the two collections it joins: a many-to-many + # needs a collection to travel through, and Intercom exposes none. + it 'publishes the collections of the lot' do + expect(datasource.collections.keys) + .to eq(%w[IntercomAdmin IntercomTeam IntercomTeamMembership IntercomTicketType IntercomTicketState + IntercomContact IntercomCompany IntercomConversation IntercomTicket]) + end + + # The four reads a boot performs, and no fifth: `/me`, which is where + # Intercom echoes the API version it served, and the attributes a workspace + # declares on its ticket types, on its contacts and on its companies -- + # columns of those collections, since a payload carries the values of the + # attributes that record happens to have been given, never their + # definitions. + it 'checks the version and introspects the workspace attributes, and reads nothing else' do + datasource + + expect(WebMock).to have_requested(:get, %r{/me}).once + expect(WebMock).to have_requested(:get, /ticket_types/).once + expect(WebMock).to have_requested(:get, /data_attributes/).with(query: { 'model' => 'contact' }).once + expect(WebMock).to have_requested(:get, /data_attributes/).with(query: { 'model' => 'company' }).once + expect(WebMock).not_to have_requested(:get, %r{conversations|admins|teams|companies/list}) + end + + # Intercom serves the workspace's own default when the pin is not honoured + # and the payload shapes differ between versions. The echo is the only + # place that shows, and reading it at boot is what turns the check from + # code that exists into code that runs. + it 'reports a workspace serving another API version than the pinned one' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_me(version: '2.11') + + datasource + + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/asked Intercom for API version 2\.16 and it served 2\.11/) + end + + # Degrades like the attribute reads: a token that cannot reach `/me` costs + # the check, never the agent. + it 'boots without the check when the token cannot read /me' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, %r{/me}).to_return(status: 403, body: '{}', + headers: { 'Content-Type' => 'application/json' }) + + expect(datasource.collections.keys).to include('IntercomTicket') + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(%r{could not read /me at boot \(HTTP 403\)}) + end + + # A token without that permission costs the attribute columns, never the + # agent. + it 'boots without the attribute columns when the introspection is refused' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, /ticket_types/).to_return(status: 403, body: '{}', + headers: { 'Content-Type' => 'application/json' }) + + expect(datasource.get_collection('IntercomTicket').fields.keys).not_to include('_default_title_') + end + + # The same guarantee on the other two models, and it is what the acceptance + # criterion of lot 4 asks for: a token missing a permission costs the + # columns it could not read, and the collection still boots. + it 'boots the contact and company collections when their introspection is refused' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, /data_attributes/).to_return(status: 403, body: '{}', + headers: { 'Content-Type' => 'application/json' }) + + expect(datasource.get_collection('IntercomContact').fields.keys).to include('email') + expect(datasource.get_collection('IntercomCompany').fields.keys).to include('name') + end + + it 'configures a client from the options it is handed' do + stub_me(base: 'https://api.eu.intercom.io') + stub_ticket_types(base: 'https://api.eu.intercom.io') + stub_data_attributes('contact', base: 'https://api.eu.intercom.io') + stub_data_attributes('company', base: 'https://api.eu.intercom.io') + configured = described_class.new(access_token: 's3cr3t', region: :eu, rate_limiter: nil) + + expect(configured.configuration.url).to eq('https://api.eu.intercom.io') + expect(configured.client).to be_a(Client) + end + + it 'refuses to boot on a configuration it cannot use' do + expect { described_class.new(access_token: nil) }.to raise_error(ConfigurationError) + end + + it 'names the collections it holds when printed' do + expect(datasource.inspect).to include('IntercomAdmin', 'IntercomTicketState') + end + + it 'never prints the token the client carries' do + expect(datasource.inspect).not_to include('s3cr3t') + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/pagination/cursor_walker_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/pagination/cursor_walker_spec.rb new file mode 100644 index 000000000..06df6b725 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/pagination/cursor_walker_spec.rb @@ -0,0 +1,193 @@ +module ForestAdminDatasourceIntercom + module Pagination + RSpec.describe CursorWalker do + subject(:walker) { described_class.new } + + let(:asked) { [] } + + def record(id) + { 'id' => id } + end + + def page(records, next_cursor: nil) + Client::Page.new(records: records, next_cursor: next_cursor, total_count: nil) + end + + # A page source Intercom's own shape: each page advertises the cursor of + # the next one, and the last advertises nothing. + def source(*pages) + queue = pages.dup + + lambda do |per_page, cursor| + asked << [per_page, cursor] + queue.shift || page([]) + end + end + + def ids(records) + records.map { |r| r['id'] } + end + + it 'serves a window one page already covers' do + records = walker.walk(offset: 0, limit: 2, &source(page([record('a'), record('b')], next_cursor: 'c1'))) + + expect(ids(records)).to eq(%w[a b]) + end + + it 'asks only for the records the window still needs' do + walker.walk(offset: 0, limit: 3, &source(page([record('a'), record('b'), record('c')]))) + + expect(asked).to eq([[3, nil]]) + end + + it 'walks pages until the window is covered, then slices the offset out' do + pages = source(page([record('a'), record('b')], next_cursor: 'c1'), + page([record('c'), record('d')], next_cursor: 'c2')) + + records = walker.walk(offset: 2, limit: 2, &pages) + + expect(ids(records)).to eq(%w[c d]) + expect(asked).to eq([[4, nil], [2, 'c1']]) + end + + it 'follows the cursor each page advertises' do + walker.walk(offset: 0, limit: 4, &source(page([record('a')], next_cursor: 'c1'), + page([record('b')], next_cursor: 'c2'), + page([record('c')]))) + + expect(asked.map(&:last)).to eq([nil, 'c1', 'c2']) + end + + it 'stops where Intercom stops advertising a next page' do + records = walker.walk(offset: 0, limit: 10, &source(page([record('a')]))) + + expect(ids(records)).to eq(%w[a]) + expect(asked.size).to eq(1) + end + + # Stopping here is a truncation like a cap: Intercom said there was a next + # page and this could not follow it, so the window is short of what was + # asked -- and a short answer is never silent. + it 'stops on an empty page, and says the result is truncated' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + records = walker.walk(offset: 0, limit: 10, &source(page([], next_cursor: 'c1'))) + + expect(records).to be_empty + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/advertised a next page this could not follow/) + end + + # None of this happens against Intercom today, but a walk driven by a + # remote value stops on its own terms rather than on the caps only. + it 'stops on a cursor it has already followed, and says the result is truncated' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + pages = source(page([record('a')], next_cursor: 'loop'), + page([record('b')], next_cursor: 'loop'), + page([record('c')], next_cursor: 'loop')) + + walker.walk(offset: 0, limit: 10, &pages) + + expect(asked.size).to eq(2) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/advertised a next page this could not follow/) + end + + # The window was covered: there is nothing short about the answer, so the + # cursor that did not move is not worth a word. + it 'stays quiet when a stalled cursor comes after the window was covered' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + walker.walk(offset: 0, limit: 1, &source(page([record('a')], next_cursor: 'loop'))) + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + + # Intercom documents duplicates on a dataset that moves between two + # paginated requests, and conversations move constantly. Two rows carrying + # one id is what a list view renders as two identical lines. + it 'drops a record a previous page already served' do + pages = source(page([record('a'), record('b')], next_cursor: 'c1'), + page([record('b'), record('c')])) + + records = walker.walk(offset: 0, limit: 10, &pages) + + expect(ids(records)).to eq(%w[a b c]) + end + + it 'keeps records carrying no id rather than deciding they are not records' do + anonymous = source(page([{ 'email' => 'a@b.test' }, { 'email' => 'c@d.test' }])) + records = walker.walk(offset: 0, limit: 10, &anonymous) + + expect(records.size).to eq(2) + end + + it 'returns nothing, and asks nothing, for a limit of zero' do + records = walker.walk(offset: 0, limit: 0, &source(page([record('a')]))) + + expect(records).to be_empty + expect(asked).to be_empty + end + + it 'reads an offset past the end as an empty window rather than an error' do + records = walker.walk(offset: 50, limit: 10, &source(page([record('a')]))) + + expect(records).to be_empty + end + + it 'treats a negative offset as the beginning' do + records = walker.walk(offset: -5, limit: 1, &source(page([record('a')]))) + + expect(ids(records)).to eq(%w[a]) + end + + describe 'a limit of nil, which asks for everything past the offset' do + it 'walks to the end' do + pages = source(page([record('a')], next_cursor: 'c1'), page([record('b')])) + + expect(ids(walker.walk(offset: 0, limit: nil, &pages))).to eq(%w[a b]) + end + + it 'asks for the largest page Intercom accepts' do + walker.walk(offset: 0, limit: nil, &source(page([record('a')]))) + + expect(asked.first.first).to eq(Client::MAX_PER_PAGE) + end + end + + describe 'caps' do + before { allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) } + + it 'stops after the page it is allowed, and says the result is truncated' do + capped = described_class.new(max_pages: 2) + pages = source(page([record('a')], next_cursor: 'c1'), + page([record('b')], next_cursor: 'c2'), + page([record('c')], next_cursor: 'c3')) + + capped.walk(offset: 0, limit: nil, &pages) + + expect(asked.size).to eq(2) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/truncated/) + end + + it 'stops on the record budget, and never asks for more than it has left' do + capped = described_class.new(max_records: 3) + pages = source(page([record('a'), record('b')], next_cursor: 'c1'), + page([record('c'), record('d')], next_cursor: 'c2')) + + capped.walk(offset: 0, limit: nil, &pages) + + expect(asked).to eq([[3, nil], [1, 'c1']]) + end + + # A walk that covered the window it was given hands back exactly that, + # and has nothing to report -- unlike one a cap cut short. + it 'stays quiet when the window was covered' do + walker.walk(offset: 0, limit: 1, &source(page([record('a')], next_cursor: 'c1'))) + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb new file mode 100644 index 000000000..cd891d7de --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb @@ -0,0 +1,211 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Query::ConditionTreeTranslator do + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + let(:nodes) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes } + let(:endpoint) { Query::SearchFields.fetch('conversations') } + + def translate(tree, timezone: 'UTC') + described_class.call(tree, endpoint: endpoint, collection: 'IntercomConversation', timezone: timezone) + end + + def leaf(field, operator, value = nil) + nodes::ConditionTreeLeaf.new(field, operator, value) + end + + def branch(aggregator, *conditions) + nodes::ConditionTreeBranch.new(aggregator, conditions) + end + + describe 'a leaf' do + it 'writes the Intercom field, operator and value the endpoint takes' do + expect(translate(leaf('state', operators::EQUAL, 'open'))) + .to eq({ 'field' => 'state', 'operator' => '=', 'value' => 'open' }) + end + + # The column is the operator's name for it; the field is Intercom's. They + # are not the same on a statistic, which the column flattens onto the row. + it 'filters a flattened statistic through the field Intercom nests it in' do + expect(translate(leaf('closed_at', operators::LESS_THAN, '2026-09-01T00:00:00Z'))['field']) + .to eq('statistics.last_close_at') + end + + it 'answers nil for no condition at all, which is a list view' do + expect(translate(nil)).to be_nil + end + end + + # A bare `id equals X` never reaches here -- the collection reads the record + # endpoint instead, one request rather than a search. What does reach here + # is the key nested in an `and`: a permission scope, a segment, or a second + # filter alongside it, which the record endpoint cannot answer on its own. + describe 'the primary key, once something else is filtered alongside it' do + it 'writes the key as the search field it is' do + tree = branch('And', leaf('id', operators::EQUAL, '42'), leaf('state', operators::EQUAL, 'open')) + + expect(translate(tree)).to eq({ 'operator' => 'AND', + 'value' => [{ 'field' => 'id', 'operator' => '=', 'value' => '42' }, + { 'field' => 'state', 'operator' => '=', 'value' => 'open' }] }) + end + + it 'writes a membership on the key as the list Intercom takes' do + tree = branch('And', leaf('id', operators::IN, %w[1 2]), leaf('open', operators::EQUAL, true)) + + expect(translate(tree)['value'].first) + .to eq({ 'field' => 'id', 'operator' => 'IN', 'value' => %w[1 2] }) + end + end + + describe 'a branch' do + it 'groups its conditions under the aggregator Intercom spells in capitals' do + tree = branch('Or', leaf('state', operators::EQUAL, 'open'), leaf('state', operators::EQUAL, 'snoozed')) + + expect(translate(tree)).to eq({ 'operator' => 'OR', + 'value' => [{ 'field' => 'state', 'operator' => '=', 'value' => 'open' }, + { 'field' => 'state', 'operator' => '=', 'value' => 'snoozed' }] }) + end + + # The agent builds a tree one branch at a time -- a scope, then a segment, + # then the operator's filter -- and Intercom allows two levels of nesting. + # A wrapper around a single condition is a level worth not spending. + it 'unwraps a branch carrying one condition rather than spending a level on it' do + tree = branch('And', branch('And', leaf('open', operators::EQUAL, true))) + + expect(translate(tree)).to eq({ 'field' => 'open', 'operator' => '=', 'value' => true }) + end + + it 'nests a group inside a group' do + tree = branch('And', leaf('open', operators::EQUAL, true), + branch('Or', leaf('state', operators::EQUAL, 'open'), + leaf('state', operators::EQUAL, 'snoozed'))) + + expect(translate(tree)['value'].last['operator']).to eq('OR') + end + + it 'refuses an aggregator that is neither and nor or' do + expect { translate(branch('Xor', leaf('open', operators::EQUAL, true))) } + .to raise_error(UnsupportedOperatorError, /cannot read "Xor" as a condition tree aggregator/) + end + + # A branch with nothing in it names no record and no filter, so sending it + # would answer a filtered question with the whole collection. + it 'refuses a branch carrying no condition' do + expect { translate(branch('And')) } + .to raise_error(UnsupportedOperatorError, /carrying no condition/) + end + + it 'refuses a node that is neither a leaf nor a branch' do + expect { translate(Object.new) } + .to raise_error(UnsupportedOperatorError, /cannot read Object as a condition/) + end + end + + # The headline failure this lot had to avoid: Intercom truncates a date + # search to the UTC day, so the pair of bounds the toolkit rewrites `today` + # into reads as "from tomorrow" and "before today" -- an empty answer to the + # most ordinary filter there is -- unless each bound is moved to the day + # boundary that makes Intercom answer the day the filter named. + describe 'the two bounds an interval is rewritten into' do + it 'asks for the day the interval names rather than cancelling out' do + tree = branch('And', leaf('created_at', operators::GREATER_THAN, '2026-09-01T00:00:00Z'), + leaf('created_at', operators::LESS_THAN, '2026-09-01T23:59:59Z')) + + expect(translate(tree)['value'].map { |bound| Time.at(bound['value']).utc.iso8601 }) + .to eq(['2026-08-31T00:00:00Z', '2026-09-02T00:00:00Z']) + end + end + + # Over either limit Intercom answers a 400 whose body names neither the + # limit nor the part of the filter that reached it. + describe 'the limits of the search DSL, checked before the request leaves' do + def leaves(count) + Array.new(count) { |index| leaf('state', operators::EQUAL, "state-#{index}") } + end + + it 'takes a group nested one level inside another' do + tree = branch('And', leaf('open', operators::EQUAL, true), + branch('Or', *leaves(2))) + + expect(translate(tree)['value'].last['operator']).to eq('OR') + end + + it 'refuses a group inside a group inside a group, naming the shape' do + tree = branch('And', leaf('open', operators::EQUAL, true), + branch('Or', leaf('read', operators::EQUAL, true), + branch('And', *leaves(2)))) + + expect { translate(tree) } + .to raise_error(UnsupportedOperatorError, + /nests a search 2 levels deep and this one reaches 3.*flatten the segment/m) + end + + # The innermost group is sometimes not one the operator wrote: a relation + # expands into one equality per record it matched, and telling them to + # flatten a nesting they never wrote is a refusal they cannot act on. + it 'names the relation when the innermost group is what one expanded into' do + relation_group = Collections::Relations::RelationBranch.new('Or', leaves(2), 'admin_assignee:name') + tree = branch('Or', leaf('open', operators::EQUAL, true), + branch('And', leaf('read', operators::EQUAL, true), relation_group)) + + expect { translate(tree) } + .to raise_error(UnsupportedOperatorError, + /"admin_assignee:name" expanded into, one equality per record it matched/) + end + + # A branch carrying a single condition is unwrapped, so it spends no level: + # the agent wraps a scope and a segment one branch at a time. + it 'does not spend a level on the branches the agent wraps around one condition' do + tree = branch('And', branch('And', branch('And', leaf('open', operators::EQUAL, true)))) + + expect(translate(tree)).to eq({ 'field' => 'open', 'operator' => '=', 'value' => true }) + end + + it 'takes a group of fifteen conditions' do + expect(translate(branch('Or', *leaves(15)))['value'].size).to eq(15) + end + + # A scope, a segment and a filter add up; and a condition naming several + # values arrives expanded into one condition per value, Intercom taking no + # membership operator on these fields. + it 'refuses a group of sixteen, and says what brings it back under' do + expect { translate(branch('Or', *leaves(16))) } + .to raise_error(UnsupportedOperatorError, /takes 15 conditions per group and this Or carries 16/) + end + end + + describe 'what it will not translate' do + # The whole point of the lot: a condition dropped on the way to Intercom + # comes back as an unfiltered page that looks filtered. + it 'refuses a column the endpoint does not filter, and names what it does' do + expect { translate(leaf('contact_name', operators::EQUAL, 'Camille')) } + .to raise_error(UnsupportedOperatorError, /cannot filter "contact_name".*Contacts endpoint/m) + end + + it 'refuses a column nothing declares, listing the ones it takes' do + expect { translate(leaf('nope', operators::EQUAL, 'x')) } + .to raise_error(UnsupportedOperatorError, /takes no filter on it. Filter on one of: id, state, open/) + end + + # A `relation:field` is traded for a condition on the foreign key before a + # tree reaches this translation, or refused by the collection with the + # relations it does declare. Reaching here means that pass was skipped, so + # the message names what the endpoint filters rather than a relation + # nobody resolved. + it 'refuses a path through a relation, naming what it does filter' do + expect { translate(leaf('contact:email', operators::EQUAL, 'camille@acme.test')) } + .to raise_error(UnsupportedOperatorError, /filters columns, not paths through a relation/) + end + + it 'refuses an operator the endpoint does not answer on that field' do + expect { translate(leaf('state', operators::CONTAINS, 'op')) } + .to raise_error(UnsupportedOperatorError, /answers equal, not_equal on "state" and nothing else/) + end + + # Published on a date column by the toolkit and refused by this endpoint: + # the datasource cannot express an equality on a day it truncates. + it 'refuses an equality on a date, which is not one of the two bounds' do + expect { translate(leaf('created_at', operators::EQUAL, '2026-09-01T00:00:00Z')) } + .to raise_error(UnsupportedOperatorError, /answers greater_than, less_than/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/filter_value_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/filter_value_spec.rb new file mode 100644 index 000000000..9688aece6 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/filter_value_spec.rb @@ -0,0 +1,284 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Query::FilterValue do + subject(:formatter) { described_class.new(collection: 'IntercomConversation', timezone: timezone) } + + let(:timezone) { 'Europe/Paris' } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + let(:nodes) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes } + + def field(type, operators = ['=']) + Query::SearchFields::Field.new(column: 'c', field: 'c', type: type, operators: operators, source: 'spec') + end + + def call(type, value, spelling: '=', operator: nil) + leaf = nodes::ConditionTreeLeaf.new('c', operator || operators::EQUAL, value) + + formatter.call(leaf, field(type), spelling) + end + + # Intercom truncates a date search to the UTC day -- measured, and against + # its own documentation, which promises the workspace timezone. `>` answers + # from the start of the day after the value, `<` before the start of the + # day of the value. Sent as they come, the two bounds of an interval cancel + # each other out and `today` answers nothing. + describe 'a date, and the UTC day Intercom truncates it to' do + def bound(value, spelling) + Time.at(call('date', value, spelling: spelling, + operator: spelling == '>' ? operators::GREATER_THAN : operators::LESS_THAN)) + .utc.iso8601 + end + + it 'moves a lower bound back a day, so Intercom answers from the day it names' do + expect(bound('2026-09-01T08:30:00Z', '>')).to eq('2026-08-31T00:00:00Z') + end + + it 'moves an upper bound forward a day, so Intercom answers through the day it names' do + expect(bound('2026-09-01T08:30:00Z', '<')).to eq('2026-09-02T00:00:00Z') + end + + # An upper bound already sitting on a day boundary names the day to leave + # out, which is what Intercom answers on its own. + it 'leaves an upper bound already on a UTC day boundary where it is' do + expect(bound('2026-09-01T00:00:00Z', '<')).to eq('2026-09-01T00:00:00Z') + end + + # The pair the toolkit rewrites `today` into, from a caller in UTC: what + # comes back is that day and nothing else. + it 'answers the day itself for the two bounds of an interval' do + expect(bound('2026-09-01T00:00:00Z', '>')).to eq('2026-08-31T00:00:00Z') + expect(bound('2026-09-01T23:59:59Z', '<')).to eq('2026-09-02T00:00:00Z') + end + + it 'keeps the offset an ISO8601 timestamp carries' do + expect(bound('2026-09-01T10:30:00+02:00', '<')).to eq('2026-09-02T00:00:00Z') + end + + # `FilterFactory` writes the bounds of a previous period with strftime + # and no offset at all, so a chart comparing to the previous month sends + # `2026-08-01 00:00:00` meaning the caller's midnight. Read in the + # process timezone it is a different instant, and a midnight moved by any + # offset lands on another UTC day once truncated -- a whole day of rows + # beside the ones the chart named. + it 'reads a timestamp carrying no offset in the timezone of the caller' do + # Paris midnight on 1 September is 2026-08-31T22:00:00Z, whose UTC day + # is the 31st, so `>` must answer from the 31st and sit on the 30th. + expect(bound('2026-09-01 00:00:00', '>')).to eq('2026-08-30T00:00:00Z') + end + + it 'reads a whole previous-period window in the timezone of the caller' do + expect(bound('2026-08-01 00:00:00', '>')).to eq('2026-07-30T00:00:00Z') + expect(bound('2026-09-01 00:00:00', '<')).to eq('2026-09-01T00:00:00Z') + end + + # A day with no time of day is the caller's day: it is the timezone the + # filter was written in that says when that day starts. + it 'reads a bare date as midnight in the timezone of the caller' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + expect(bound('2026-09-01', '>')).to eq('2026-08-30T00:00:00Z') + end + + it 'reads a Ruby Date, a Time, a DateTime and epoch seconds the same way' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + expect(bound(Date.new(2026, 9, 1), '>')).to eq('2026-08-30T00:00:00Z') + expect(bound(Time.utc(2026, 9, 1, 8, 30), '<')).to eq('2026-09-02T00:00:00Z') + expect(bound(DateTime.new(2026, 9, 1, 8, 30, 0), '<')).to eq('2026-09-02T00:00:00Z') + expect(bound(1_788_251_400, '<')).to eq('2026-09-02T00:00:00Z') + end + + # A window written in another timezone is answered on the UTC days it + # overlaps, and an operator has no way of guessing that from the rows. + it 'reports the UTC day boundary once, when it is not the day of the caller' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + bound('2026-08-31T22:00:00Z', '>') + bound('2026-08-31T23:00:00Z', '>') + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).once.with(/truncates a date search/) + end + + it 'stays quiet when the window and the UTC day are the same day anyway' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + bound('2026-09-01T08:30:00Z', '>') + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + + context 'when the caller names a timezone nothing knows' do + let(:timezone) { 'Middle-Earth/Shire' } + + # Falling back to UTC silently would move a day boundary by the offset, + # which is the failure a timezone is read for in the first place. + it 'reads the day boundary in UTC and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + expect(bound('2026-09-01', '>')).to eq('2026-08-31T00:00:00Z') + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/unknown timezone/) + end + + it 'reads a timestamp carrying no offset in UTC and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + expect(bound('2026-09-01 00:00:00', '>')).to eq('2026-08-31T00:00:00Z') + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/unknown timezone/) + end + end + + context 'when the caller names no timezone at all' do + let(:timezone) { ' ' } + + it 'reads the day boundary in UTC' do + expect(bound('2026-09-01', '>')).to eq('2026-08-31T00:00:00Z') + end + end + + it 'refuses a string that is not a date' do + expect { call('date', 'last tuesday', spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + + it 'refuses a value that is not a date at all' do + expect { call('date', { 'day' => 1 }, spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + + # `Time.parse` accepts a time of day on its own and fills the date in from + # the server's clock, so this would have travelled as a bound on whichever + # day the request happened to run -- a filter whose answer changes at + # midnight and never says why. + it 'refuses a time of day rather than completing it from today' do + expect { call('date', '12:00', spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + + # A month names a range, and which end of it the operator meant is theirs + # to say rather than this class's to guess. + it 'refuses a date missing its day' do + expect { call('date', 'Jan 2026', spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + + # It carries the three parts a date needs and still names no day: the + # parser is what says so, and its complaint is not a bound to send on. + it 'refuses a date whose parts are out of range' do + expect { call('date', '2026-13-45', spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + + # Reading either as epoch seconds raises a FloatDomainError, which would + # leave the read with an error naming a float where the operator asked + # for a date. The number branch already refuses them; a date is no + # different. + it 'refuses a cast that overflowed to Infinity, and a NaN' do + expect { call('date', Float::INFINITY, spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + expect { call('date', Float::NAN, spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + + it 'reads a finite number as the epoch seconds Intercom stores' do + expect(bound(1_767_225_600, '>')).to eq('2025-12-31T00:00:00Z') + end + end + + describe 'a number' do + # The agent casts every Number column with `to_f`, so an integer field + # would otherwise be filtered with `42.0`, a form none of its values carry. + it 'sends a whole float as the integer it is' do + expect(call('number', 42.0)).to eq(42) + end + + it 'keeps a decimal, and an integer, as they are' do + expect(call('number', 42.5)).to eq(42.5) + expect(call('number', 42)).to eq(42) + end + + it 'reads an integer written as a string' do + expect(call('number', '42')).to eq(42) + end + + it 'reads a decimal written as a string' do + expect(call('number', '42.5')).to eq(42.5) + end + + # `to_i` raises on both, and so does the JSON encoder a step later, as a + # 500 naming nothing the operator can act on. + it 'refuses a cast that overflowed and a value that is not a number' do + expect { call('number', Float::INFINITY) }.to raise_error(UnsupportedOperatorError, /expects a number/) + expect { call('number', 'many') }.to raise_error(UnsupportedOperatorError, /expects a number/) + expect { call('number', []) }.to raise_error(UnsupportedOperatorError, /expects a number/) + end + end + + describe 'a boolean' do + it 'sends a flag as a boolean, whichever way the filter spelled it' do + expect(call('boolean', true)).to be(true) + expect(call('boolean', false)).to be(false) + expect(call('boolean', 'true')).to be(true) + expect(call('boolean', 'false')).to be(false) + end + + it 'refuses anything else, rather than reading it as truthy' do + expect { call('boolean', 'yes') }.to raise_error(UnsupportedOperatorError, /expects a true or a false/) + end + end + + describe 'a list' do + it 'formats every value of an IN the way the field takes it' do + expect(call('number', ['4', 5.0], spelling: 'IN', operator: operators::IN)).to eq([4, 5]) + end + + # A filter matching everything is not what an empty list was asked for. + it 'refuses an empty list' do + expect { call('string', [], spelling: 'IN', operator: operators::IN) } + .to raise_error(UnsupportedOperatorError, /empty list/) + end + + # Measured, on IN and NIN alike: a longer list is a 400 naming neither the + # field nor the condition. + it 'refuses a list longer than Intercom takes, naming the limit' do + expect { call('string', (1..16).map(&:to_s), spelling: 'IN', operator: operators::IN) } + .to raise_error(UnsupportedOperatorError, /"c" with in and 16 values: Intercom takes 15 at most/) + expect { call('string', (1..16).map(&:to_s), spelling: 'NIN', operator: operators::NOT_IN) } + .to raise_error(UnsupportedOperatorError, /16 values/) + end + + it 'takes a list of exactly as many values as Intercom does' do + expect(call('string', (1..15).map(&:to_s), spelling: 'IN', operator: operators::IN).size).to eq(15) + end + + # `not_in [nil, 'open']` was asked to exclude the records carrying neither + # and would come back including the blank ones. + it 'refuses a list holding a blank, rather than dropping it' do + expect { call('string', [nil, 'open'], spelling: 'NIN', operator: operators::NOT_IN) } + .to raise_error(UnsupportedOperatorError, /cannot filter "c" for absence/) + end + end + + describe 'a condition on the absence of a value' do + # `present`, `blank` and `missing` are derived from an equality above this + # datasource and rewritten into a comparison with an empty value. Intercom + # would answer it as if the empty string were a value of its own. + it 'refuses the rewritten comparison and says to filter on a value' do + expect { call('string', nil) } + .to raise_error(UnsupportedOperatorError, /matches values and has no operator for the lack of one/) + end + + it 'refuses an empty string the same way' do + expect { call('string', '') }.to raise_error(UnsupportedOperatorError, /for absence/) + end + end + + describe 'a string' do + it 'sends it as it came' do + expect(call('string', 'open')).to eq('open') + end + + it 'sends a text the same way' do + expect(call('text', 'facture')).to eq('facture') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/operator_table_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/operator_table_spec.rb new file mode 100644 index 000000000..d0f04c0ee --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/operator_table_spec.rb @@ -0,0 +1,110 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Query::OperatorTable do + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + let(:rules) { ForestAdminDatasourceToolkit::Validations::Rules } + let(:equivalent) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeEquivalent } + + def field(type, operators) + Query::SearchFields::Field.new(column: 'c', field: 'c', type: type, operators: operators, source: 'spec') + end + + describe '.forest_operators' do + it 'publishes only what the endpoint accepts on that field' do + expect(described_class.forest_operators(field('string', ['=']))).to eq([operators::EQUAL]) + expect(described_class.forest_operators(field('string', ['=', '!=']))) + .to eq([operators::EQUAL, operators::NOT_EQUAL]) + end + + # Intercom documents one substring operator and no case semantics for it, + # and the frontend sends either spelling depending on the column. + it 'reads both spellings of contains onto the one operator Intercom has' do + published = described_class.forest_operators(field('text', ['~'])) + + expect(published).to eq([operators::CONTAINS, operators::I_CONTAINS]) + end + + # The point of the whole table: a date column carries the two bounds even + # where the endpoint accepts more, because declaring an equality on a Date + # makes the toolkit republish `in`, which its own validator refuses. + it 'keeps a date column to the two bounds whatever the endpoint accepts' do + published = described_class.forest_operators(field('date', ['>', '<', '>=', '<=', '=', '!='])) + + expect(published).to eq([operators::GREATER_THAN, operators::LESS_THAN]) + end + + it 'publishes nothing for a field the endpoint answers no known operator on' do + expect(described_class.forest_operators(field('string', ['~']))).to be_empty + end + end + + describe '.intercom_operator' do + it 'spells a Forest operator the way the search DSL does' do + expect(described_class.intercom_operator(field('number', ['>']), operators::GREATER_THAN)).to eq('>') + end + + it 'answers nil for an operator the endpoint does not accept on that field' do + expect(described_class.intercom_operator(field('string', ['=']), operators::NOT_EQUAL)).to be_nil + end + + it 'answers nil for an operator no field of that type carries' do + expect(described_class.intercom_operator(field('boolean', ['=']), operators::CONTAINS)).to be_nil + end + end + + describe 'the operators every published column ends up with' do + # The invariant PRD-989 says nothing checks: everything the agent publishes + # from what a column declares must be an operator its own validator allows. + # A column advertising a filter the agent then rejects is a 400 in the + # interface for a reason that has nothing to do with Intercom. + it 'is a set the toolkit validator allows, for every column of every endpoint' do + column_types = { 'string' => 'String', 'text' => 'String', 'date' => 'Date', + 'boolean' => 'Boolean', 'number' => 'Number' } + + Query::SearchFields.endpoints.each do |name| + Query::SearchFields.fetch(name).fields.each_value do |searchable| + declared = described_class.forest_operators(searchable) + column_type = column_types.fetch(searchable.type) + published = operators.all.select { |o| equivalent.equivalent_tree?(o, declared, column_type) } + + expect(published - rules.get_allowed_operators_for_column_type(column_type)) + .to be_empty, "#{name}.#{searchable.column} publishes an operator Rules refuses" + end + end + end + + # Walked from the schema rather than from the table, which is the only way + # to see the primary key: `add_column` writes its operators by hand -- the + # toolkit refuses a collection whose key carries neither `equal` nor `in` + # -- so the loop above, reading the table, could never reach the one + # column no row of the table is derived from. A column the schema + # publishes and the table cannot spell is a filter the translator refuses + # at read time, which is what a record detail hits the moment a scope + # nests `id equals X` in an `and`. + it 'is a set the table can express, for every column the datasource publishes' do + cursor_collections.each do |collection| + endpoint = collection.send(:search_endpoint) + + collection.fields.each do |column, schema| + published = Array(schema.respond_to?(:filter_operators) ? schema.filter_operators : nil) + next if published.empty? || collection.send(:routed_operators).key?(column) + + searchable = endpoint.field(column) + expect(searchable).not_to be_nil, + "#{collection.name}.#{column} publishes #{published.join(", ")} and " \ + "#{endpoint.path} carries no row to translate it" + expect(published - described_class.forest_operators(searchable)) + .to be_empty, "#{collection.name}.#{column} publishes an operator the translator would refuse" + end + end + end + end + + # Only the two collections a search endpoint backs: the reference ones are + # read whole and filtered in memory, and publish no operator from a table. + def cursor_collections + datasource = Datasource.new(access_token: 's3cr3t', rate_limiter: nil) + + datasource.collections.each_value.grep(Collections::CursorCollection) + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb new file mode 100644 index 000000000..a8021276e --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb @@ -0,0 +1,288 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Query::SearchFields do + def table(fields: {}, refused: {}, measured_at: nil, attributes: nil) + definition = { 'path' => 'tickets/search', 'measured_at' => measured_at, 'fields' => fields, + 'refused' => refused, 'candidates' => [] } + definition['ticket_attributes'] = attributes if attributes + + described_class.build('endpoints' => { 'tickets' => definition })['tickets'] + end + + def attributes_row(overrides = {}) + { 'filterable' => false, 'reason' => 'display only', 'source' => 'spec' }.merge(overrides) + end + + def field_row(overrides = {}) + { 'field' => 'created_at', 'type' => 'date', 'operators' => ['>', '<'], 'source' => 'spec' }.merge(overrides) + end + + describe 'the committed table' do + # The schema derives its filters from this file, so a malformed row is a + # boot failure rather than a column nobody can explain. Reading it here is + # what turns that guarantee into a test. + it 'reads the three search endpoints' do + expect(described_class.endpoints).to eq(%w[conversations tickets contacts]) + end + + # The asymmetry the per-endpoint table exists for, asserted on the file + # that ships rather than on a table a spec built: a date is bounded on + # every endpoint, and only two of them take the closed bounds and the + # inequality. + it 'carries the measured date restrictions of /contacts/search' do + expect(described_class.fetch('contacts').field('created_at').operators).to eq(['>', '<']) + expect(described_class.fetch('conversations').field('created_at').operators) + .to include('>=', '<=', '!=') + end + + # Intercom sorts one endpoint and ignores the sort it is sent on the other + # two, so a sortable column outside contacts would promise an order that + # never happens. + it 'declares a sortable column on the one endpoint that sorts' do + expect(described_class.fetch('contacts').sortable_columns).not_to be_empty + expect(described_class.fetch('conversations').sortable_columns).to be_empty + expect(described_class.fetch('tickets').sortable_columns).to be_empty + end + + it 'names the path each endpoint is searched through' do + expect(described_class.fetch('conversations').path).to eq('conversations/search') + expect(described_class.fetch('tickets').path).to eq('tickets/search') + end + + # The one thing lot 1 measured about the field lists: `/tickets/search` + # refuses `company_id`, which the specification lists on a ticket. + it 'refuses company_id on tickets, with the measurement as its reason' do + refusal = described_class.fetch('tickets').refusal('company_id') + + expect(refusal.reason).to include('invalid_field') + expect(refusal).to be_measured + end + + # The columns the agent derives from the parts of a ticket: Intercom + # filters none of them, and lot 1 published them without an operator. + it 'refuses every column derived from the parts of a ticket' do + refused = described_class.fetch('tickets').refused.keys + + expect(refused).to include('closed_at', 'closed_by_name', 'last_reply_at', 'last_responder_name', + 'last_responder_type') + end + + # The columns a workspace's own attributes become cannot have a row each + # -- their names are discovered at boot -- so the endpoint carries one + # refusal for the whole family, and the translator reads it in place of + # the message for a column nobody declared. + it 'keeps the ticket attributes unfilterable while the arbitration stands' do + refusal = described_class.fetch('tickets').attribute_refusal + + expect(refusal.reason).to include('differs from one ticket type to the next') + expect(refusal).to be_measured + end + + it 'carries the same refusal for the custom attributes of a contact' do + expect(described_class.fetch('contacts').attribute_refusal.reason) + .to include('custom_attributes.{name}') + end + + # `/conversations/search` has no attribute family of its own, and a nil + # here is what makes the translator fall back on the column message. + it 'carries none where the endpoint declares no attribute family' do + expect(described_class.fetch('conversations').attribute_refusal).to be_nil + end + + # Until the probe runs against the customer's workspace, the date rows are + # the only ones a measurement backs. + it 'reports which rows nothing has measured yet' do + conversations = described_class.fetch('conversations') + + expect(conversations).not_to be_measured + expect(conversations.unmeasured_fields.map(&:column)).not_to include('created_at', 'updated_at') + end + + it 'declares no column both filterable and refused' do + described_class.endpoints.each do |name| + endpoint = described_class.fetch(name) + + expect(endpoint.filterable_columns & endpoint.refused.keys).to be_empty + end + end + + # A candidate is what the probe enumerates on top of the table; one that + # is already declared would be probed twice and read as a discovery. + it 'names no candidate already declared as a field' do + described_class.endpoints.each do |name| + endpoint = described_class.fetch(name) + + expect(endpoint.candidates & endpoint.fields.values.map(&:field)).to be_empty + end + end + end + + # The table is checked in one direction by the schema itself: a column + # publishes exactly the operators its row allows, so it cannot advertise a + # filter the translator would refuse. This is the other direction, and it + # is what the refusal reasons are worth: a column the table says nothing + # about falls back on the generic "takes no filter on it", which is the one + # refusal an operator cannot act on -- and a row whose column was dropped + # survives its own collection, which is how `state_external_label` outlived + # the schema by two lots. + # + # The datasource boots here with no workspace attribute, those columns + # being named by the workspace rather than by the table; they are covered + # by `attribute_refusal` instead. + describe 'the columns of the collections it answers for' do + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + + { 'IntercomConversation' => 'conversations', 'IntercomTicket' => 'tickets', + 'IntercomContact' => 'contacts' }.each do |collection_name, endpoint_name| + it "names every column of #{collection_name}, as filterable or as refused" do + endpoint = described_class.fetch(endpoint_name) + columns = datasource.get_collection(collection_name).fields + .select { |_, field| field.is_a?(ForestAdminDatasourceToolkit::Schema::ColumnSchema) } + .keys + named = endpoint.filterable_columns + endpoint.refused.keys + + expect(columns - named).to be_empty + expect(named - columns).to be_empty + end + end + end + + # The README is where an operator reads what they may filter on before the + # interface shows it to them, so it is checked against the table rather than + # left to drift from it. + describe 'the README section the table feeds' do + let(:filterable) do + File.read(File.expand_path('../../../README.md', __dir__), encoding: 'UTF-8')[ + /### What is filterable\n(.*?)\n### /m, 1 + ] + end + + it 'lists exactly the columns each endpoint filters' do + { 'IntercomConversation' => 'conversations', 'IntercomTicket' => 'tickets', + 'IntercomContact' => 'contacts' }.each do |collection, endpoint| + row = filterable.lines.find { |line| line.start_with?("| `#{collection}` |") } + listed = row.to_s.scan(/`([a-z_]+)`/).flatten + + expect(listed).to match_array(described_class.fetch(endpoint).filterable_columns) + end + end + + # How much of the table a measurement backs is the first thing the README + # says about it, and a figure typed by hand is a figure that drifts the + # next time a row moves. + it 'counts the measured rows the way the file does' do + rows = described_class.endpoints.flat_map do |name| + endpoint = described_class.fetch(name) + + endpoint.fields.values + endpoint.refused.values + end + readme = File.read(File.expand_path('../../../README.md', __dir__), encoding: 'UTF-8') + + expect(readme).to include("#{rows.count(&:measured?)} of #{rows.size} are measured") + end + end + + describe 'a table that cannot be trusted' do + it 'refuses an operator Intercom has no spelling for' do + expect { table(fields: { 'created_at' => field_row('operators' => ['~=']) }) } + .to raise_error(ConfigurationError, /has no operator ~=/) + end + + it 'refuses a type nothing knows how to send' do + expect { table(fields: { 'created_at' => field_row('type' => 'timestamp') }) } + .to raise_error(ConfigurationError, /type "timestamp" is not one of/) + end + + # An empty list would publish a filterable column no operator can reach. + it 'refuses a field declaring no operator, and says where it belongs' do + expect { table(fields: { 'created_at' => field_row('operators' => []) }) } + .to raise_error(ConfigurationError, /belongs in the refused table/) + end + + # YAML reads an unquoted `true` as a boolean and a quoted one as a string, + # which is truthy in Ruby: a typo would publish a sortable column the + # endpoint never sorts. + it 'refuses a sortable flag that is not a boolean' do + expect { table(fields: { 'created_at' => field_row('sortable' => 'true') }) } + .to raise_error(ConfigurationError, /sortable "true" is neither true nor false/) + end + + it 'refuses a value map that does not map strings to strings' do + category = { 'field' => 'category', 'type' => 'string', 'operators' => ['='], 'source' => 'spec' } + + [[], {}, { 'Customer' => 1 }].each do |values| + expect { table(fields: { 'category' => category.merge('values' => values) }) } + .to raise_error(ConfigurationError, /is not a map of strings to strings/) + end + end + + it 'refuses a provenance that is neither measured nor read off the documentation' do + expect { table(fields: { 'created_at' => field_row('source' => 'guessed') }) } + .to raise_error(ConfigurationError, /source "guessed" is neither measured nor spec/) + end + + it 'refuses a refusal with no provenance of its own' do + expect { table(refused: { 'company_id' => { 'reason' => 'no', 'source' => 'hearsay' } }) } + .to raise_error(ConfigurationError, /tickets.company_id/) + end + + # Operators the DSL does spell, none of which the column's *type* can + # carry: `OperatorTable` maps no Forest operator onto `~` for a date, so + # the column would be called filterable and publish nothing, and the + # translator would refuse every request on it. + it 'refuses operators the column type cannot publish' do + expect { table(fields: { 'created_at' => field_row('operators' => ['~']) }) } + .to raise_error(ConfigurationError, /none of ~ is an operator Intercom answers on a date/) + end + + # `Endpoint#field` is consulted before `#refusal`, so the filterable row + # would win and the reason an operator reads would be dropped in silence. + # The file is script-rewritten, which is the shape a bad rewrite takes. + it 'refuses a column declared filterable and refused at once' do + expect do + table(fields: { 'created_at' => field_row }, + refused: { 'created_at' => { 'reason' => 'no', 'source' => 'spec' } }) + end.to raise_error(ConfigurationError, /filterable and refused at once/) + end + + # The block is a refusal from end to end, so a rewrite flipping the flag + # would leave the file saying the opposite of what the package does with + # it -- publish nothing. + it 'refuses an attribute family declared filterable' do + expect { table(attributes: attributes_row('filterable' => true)) } + .to raise_error(ConfigurationError, /tickets.ticket_attributes: filterable true is not something/) + end + + it 'refuses an attribute family with no provenance of its own' do + expect { table(attributes: attributes_row('source' => 'hearsay')) } + .to raise_error(ConfigurationError, /tickets.ticket_attributes: source "hearsay"/) + end + + it 'names the endpoint and the column it choked on' do + expect { table(fields: { 'created_at' => field_row('type' => 'timestamp') }) } + .to raise_error(ConfigurationError, /search_fields\.yml is malformed at tickets\.created_at/) + end + end + + describe 'what it hands to the schema' do + it 'carries the Intercom field a column is filtered through' do + expect(described_class.fetch('conversations').field('closed_at').field).to eq('statistics.last_close_at') + end + + it 'reads a refusal reason as one line, whatever the YAML wrapping' do + reason = table(refused: { 'company_id' => { 'reason' => "one\ntwo\n", 'source' => 'spec' } }) + .refusal('company_id').reason + + expect(reason).to eq('one two') + end + + it 'refuses an endpoint nothing declares, rather than filtering nothing' do + expect { described_class.fetch('companies') } + .to raise_error(ConfigurationError, /Unknown Intercom search endpoint "companies"/) + end + + it 'is measured once the probe has stamped a date on it' do + expect(table(measured_at: '2026-09-01')).to be_measured + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/rate_limiter_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/rate_limiter_spec.rb new file mode 100644 index 000000000..6edb14d84 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/rate_limiter_spec.rb @@ -0,0 +1,157 @@ +module ForestAdminDatasourceIntercom + RSpec.describe RateLimiter do + # Sleeping moves the clock, the way it does outside a spec: a window waited + # out is a window that has refilled by the time the next request asks. + subject(:limiter) { build_limiter } + + let(:time) { [1_000.0] } + let(:slept) { [] } + + def build_limiter(**options) + sleeper = lambda do |seconds| + slept << seconds + time[0] += seconds + end + + described_class.new(now: -> { time[0] }, sleeper: sleeper, **options) + end + + # What Intercom answers with: the limit of the 10-second window, what is + # left of it, and the epoch second it refills at. + def headers(remaining:, reset_in: 4, limit: 1667) + { 'x-ratelimit-limit' => limit.to_s, + 'x-ratelimit-remaining' => remaining.to_s, + 'x-ratelimit-reset' => (time[0] + reset_in).to_i.to_s } + end + + it 'lets the first request through: the budget is what that request discovers' do + limiter.acquire + + expect(slept).to be_empty + end + + it 'lets a request through while the window still has room' do + limiter.observe(headers(remaining: 5)) + limiter.acquire + + expect(slept).to be_empty + end + + it 'waits for the reset once the window is spent' do + limiter.observe(headers(remaining: 0, reset_in: 4)) + limiter.acquire + + expect(slept).to eq([4.0]) + end + + it 'lets requests through again once the reset has passed' do + limiter.observe(headers(remaining: 0, reset_in: -1)) + limiter.acquire + + expect(slept).to be_empty + end + + # Several requests can be in flight before any of them answers, and a + # `remaining` that only moves on a response lets all of them through on the + # same stale figure. + it 'counts its own requests down rather than trusting the last response' do + limiter.observe(headers(remaining: 2, reset_in: 3)) + 3.times { limiter.acquire } + + expect(slept).to eq([3.0]) + end + + it 'sends the last request a window allows rather than waiting for the next window' do + limiter.observe(headers(remaining: 1, reset_in: 3)) + limiter.acquire + + expect(slept).to be_empty + end + + it 'ignores a response from a window older than the one it knows' do + limiter.observe(headers(remaining: 0, reset_in: 5)) + limiter.observe(headers(remaining: 100, reset_in: -10)) + limiter.acquire + + expect(slept).to eq([5.0]) + end + + it 'adopts a new window whole, generous remaining included' do + limiter.observe(headers(remaining: 0, reset_in: 2)) + limiter.observe(headers(remaining: 50, reset_in: 12)) + limiter.acquire + + expect(slept).to be_empty + end + + # A response that left Intercom before the requests now in flight were made + # must not hand their budget back. + it 'keeps the smaller remaining inside one window' do + limiter.observe(headers(remaining: 2, reset_in: 3)) + limiter.acquire + limiter.observe(headers(remaining: 99, reset_in: 3)) + 2.times { limiter.acquire } + + expect(slept).to eq([3.0]) + end + + it 'reads the headers whatever their case' do + limiter.observe('X-RateLimit-Remaining' => '0', 'X-RateLimit-Reset' => (time[0] + 6).to_i.to_s) + limiter.acquire + + expect(slept).to eq([6.0]) + end + + it 'ignores a response carrying no rate-limit headers at all' do + limiter.observe('content-type' => 'application/json') + limiter.acquire + + expect(slept).to be_empty + end + + it 'ignores a remaining that is not a number' do + limiter.observe('x-ratelimit-remaining' => 'many', 'x-ratelimit-reset' => (time[0] + 3).to_i.to_s) + limiter.acquire + + expect(slept).to be_empty + end + + it 'sleeps a reset out when it fits within the bound it was given' do + capped = build_limiter(max_wait: 2.0) + capped.observe(headers(remaining: 0, reset_in: 1)) + capped.acquire + + expect(slept).to eq([1.0]) + end + + describe 'a reset further out than one window' do + # Intercom's reset is a timestamp from its clock. One that far out is the + # two clocks disagreeing, not a window emptying, so the request goes + # through and the log says why -- waiting an hour would read as a hang. + before do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + limiter.observe(headers(remaining: 0, reset_in: 3_600)) + end + + it 'lets the request through instead of waiting it out' do + limiter.acquire + + expect(slept).to be_empty + end + + it 'says so once, not once per request' do + 3.times { limiter.acquire } + + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).once.with(/rate-limit window is spent.*limit 1667/m) + end + end + + it 'sleeps for real when handed no sleeper' do + real = described_class.new(now: -> { 999.96 }) + real.observe('x-ratelimit-remaining' => '0', 'x-ratelimit-reset' => '1000') + + expect { real.acquire }.not_to raise_error + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/retry_policy_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/retry_policy_spec.rb new file mode 100644 index 000000000..e4e6ca430 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/retry_policy_spec.rb @@ -0,0 +1,106 @@ +module ForestAdminDatasourceIntercom + RSpec.describe RetryPolicy do + describe '#to_faraday_options' do + subject(:options) { described_class.new.to_faraday_options } + + it 'retries the statuses worth another attempt' do + expect(options[:retry_statuses]).to eq([429, 500, 502, 503, 504]) + end + + # A 502 on the way back from a POST Intercom did perform would be replayed + # into a second reply on the conversation. + it 'only replays the verbs that change nothing' do + expect(options[:methods]).to eq(%i[get head options]) + end + + it 'replays a 429 whatever the verb, Intercom having rejected it unprocessed' do + expect(options[:retry_if].call({ status: 429 }, nil)).to be(true) + end + + it 'leaves any other status to the methods list' do + expect(options[:retry_if].call({ status: 502 }, nil)).to be(false) + end + + # faraday-retry abandons outright when Retry-After exceeds max_interval, + # so the cap has to cover Intercom's whole 10-second window. + it 'waits out a full rate-limit window' do + expect(options[:max_interval]).to be > RateLimiter::WINDOW + end + + it 'absorbs a dropped connection, which faraday-retry does not by default' do + expect(options[:exceptions]).to include(Faraday::ConnectionFailed) + end + + # The persistent adapter re-raises its own error for the part of its + # surface it does not translate, and the gem is optional -- so the class + # is named rather than referenced, and faraday-retry skips a name nothing + # defined. + it 'absorbs the adapter error a dropped connection can also surface as' do + expect(options[:exceptions]).to include('Net::HTTP::Persistent::Error') + end + + context 'when a pooled connection was dropped' do + let(:dropped) { Faraday::ConnectionFailed.new('closed') } + + def env(path) + { status: nil, url: URI.parse("https://api.intercom.io/#{path}") } + end + + # Keep-alive is what makes this ordinary rather than impossible: a socket + # the server closed while it was idle fails the next request that reuses + # it, and every list view of this datasource is built on one of these. + it 'replays the reads Intercom answers through POST' do + %w[contacts/search tickets/search conversations/search companies/list].each do |path| + expect(options[:retry_if].call(env(path), dropped)).to be(true) + end + end + + # The guarantee the exemption had to leave intact, and the reason it is + # scoped to paths rather than to the verb. + it 'never replays a POST that writes' do + %w[conversations tickets contacts conversations/123/reply].each do |path| + expect(options[:retry_if].call(env(path), dropped)).to be(false) + end + end + + it 'leaves a GET to the methods list, which already replays it' do + expect(options[:retry_if].call(env('me'), dropped)).to be(false) + expect(options[:methods]).to include(:get) + end + end + end + + describe '.connection_failure?' do + it "knows Faraday's own" do + expect(described_class).to be_connection_failure(Faraday::ConnectionFailed.new('closed')) + end + + # Raised when resetting a pooled connection finds the host down, and it + # reaches Faraday unwrapped: the adapter translates the messages it + # recognises -- a timeout, a refused connection -- and re-raises the rest. + it "knows the persistent adapter's, which it cannot name outright" do + require 'net/http/persistent' + + expect(described_class).to be_connection_failure(Net::HTTP::Persistent::Error.new('host down')) + end + + it 'takes nothing else for one' do + expect(described_class).not_to be_connection_failure(Faraday::TimeoutError.new('slow')) + end + end + + describe '.boot' do + subject(:options) { described_class.boot.to_faraday_options } + + it 'retries once: a boot read is never revisited, and never worth a long wait' do + expect(options[:max]).to eq(1) + end + + # Below a rate-limit window on purpose: past the cap faraday-retry gives + # up at once, which is what keeps a 429 from holding the Rails boot. + it 'gives up rather than waiting a 429 out' do + expect(options[:max_interval]).to be < RateLimiter::WINDOW + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/data_attributes_introspector_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/data_attributes_introspector_spec.rb new file mode 100644 index 000000000..6896ccf33 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/data_attributes_introspector_spec.rb @@ -0,0 +1,136 @@ +module ForestAdminDatasourceIntercom + module Schema + RSpec.describe DataAttributesIntrospector do + subject(:introspector) { described_class.new(Client.new(configuration), model: 'contact') } + + let(:configuration) { Configuration.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { configuration.url } + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def attribute(name, overrides = {}) + { 'type' => 'data_attribute', 'model' => 'contact', 'name' => name, 'label' => name, + 'data_type' => 'string', 'custom' => true, 'archived' => false, + 'api_writable' => true }.merge(overrides) + end + + def stub_attributes(*attributes, model: 'contact') + stub_request(:get, "#{base}/data_attributes").with(query: { 'model' => model }) + .to_return(json('type' => 'list', 'data' => attributes)) + end + + it 'reads the attributes of the model it was built for' do + stub_attributes(attribute('paid_subscriber'), model: 'company') + + expect(described_class.new(Client.new(configuration), model: 'company').attributes.map(&:name)) + .to eq(['paid_subscriber']) + end + + # The standard attributes are columns the collection declares by hand, + # with the filters the search table measured. Publishing them again under + # their `custom_attributes` name would show one fact twice, the + # unfilterable copy winning nothing. + it 'leaves out the attributes Intercom defines itself' do + stub_attributes(attribute('paid_subscriber'), attribute('email', 'custom' => false)) + + expect(introspector.attributes.map(&:name)).to eq(['paid_subscriber']) + end + + it 'leaves out an archived attribute, which the workspace stopped offering' do + stub_attributes(attribute('paid_subscriber'), attribute('old_plan', 'archived' => true)) + + expect(introspector.attributes.map(&:name)).to eq(['paid_subscriber']) + end + + it 'leaves out an attribute with no name to be a column of' do + stub_attributes(attribute(''), 'not a hash') + + expect(introspector.attributes).to be_empty + end + + # Read and carried although every column of this lot is published + # read-only: it is what lot 4b needs to tell an attribute it may write + # from one Intercom fills in itself, and reading it again then would be a + # second boot-time round trip. + it 'carries api_writable for the lot that writes' do + stub_attributes(attribute('paid_subscriber'), attribute('lifetime_value', 'api_writable' => false)) + + expect(introspector.attributes.map { |a| [a.name, a.api_writable] }) + .to eq([['paid_subscriber', true], ['lifetime_value', false]]) + end + + it 'maps each Intercom data type onto what Forest renders' do + stub_attributes(attribute('a', 'data_type' => 'integer'), attribute('b', 'data_type' => 'float'), + attribute('c', 'data_type' => 'boolean'), attribute('d', 'data_type' => 'date')) + + expect(introspector.attributes.map(&:column_type)).to eq(%w[Number Number Boolean Date]) + end + + # Showing the value Intercom sent beats hiding a column because its type + # is new. + it 'reads an unknown data type as a string rather than dropping the column' do + stub_attributes(attribute('paid_subscriber', 'data_type' => 'quantum')) + + expect(introspector.attributes.first.column_type).to eq('String') + end + + # Forest lists the fields of a request in a comma-separated query + # parameter and names a field through a relation with a colon: either one + # in a column name breaks the projection before the page is read. + it 'takes the commas and colons out of a column name, keeping the name the payload uses' do + stub_attributes(attribute('Plan, tier: current')) + + expect(introspector.attributes.map { |a| [a.name, a.column_name] }) + .to eq([['Plan, tier: current', 'Plan tier current']]) + end + + it 'unescapes a name Intercom handed back escaped' do + stub_attributes(attribute('Ce que j'ai vérifié')) + + expect(introspector.attributes.first.column_name).to eq("Ce que j'ai vérifié") + end + + # Two attributes landing on one column would share an entry, and the + # second's values would be read under the first's name -- wrong values + # rather than missing ones. + it 'leaves out an attribute colliding with one already kept, and says which' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_attributes(attribute('Plan, tier'), attribute('Plan: tier')) + + expect(introspector.attributes.map(&:name)).to eq(['Plan, tier']) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/"Plan: tier" is left out.*"Plan tier"/) + end + + it 'keeps an attribute whose name is nothing but separators out of the schema' do + stub_attributes(attribute(',,')) + + expect(introspector.attributes).to be_empty + end + + # A token without the permission costs the columns, never the boot. + it 'answers no attribute when Intercom refuses the read, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/data_attributes").with(query: { 'model' => 'contact' }) + .to_return(json( + { 'type' => 'error.list', + 'errors' => [{ 'code' => 'forbidden' }] }, 403 + )) + + expect(introspector.attributes).to eq([]) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/could not read the contact attributes \(HTTP 403\)/) + end + + it 'reads Intercom once, however many times it is asked' do + stub_attributes(attribute('paid_subscriber')) + + 2.times { introspector.attributes } + + expect(WebMock).to have_requested(:get, /data_attributes/).once + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/ticket_attributes_introspector_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/ticket_attributes_introspector_spec.rb new file mode 100644 index 000000000..924f79b39 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/ticket_attributes_introspector_spec.rb @@ -0,0 +1,186 @@ +module ForestAdminDatasourceIntercom + module Schema + RSpec.describe TicketAttributesIntrospector do + subject(:introspector) { described_class.new(Client.new(configuration)) } + + let(:configuration) { Configuration.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { configuration.url } + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def attribute(name, id, data_type: 'string', archived: false) + { 'id' => id, 'name' => name, 'data_type' => data_type, 'archived' => archived } + end + + def ticket_type(id, name, *attributes) + { 'type' => 'ticket_type', 'id' => id, 'name' => name, + 'ticket_type_attributes' => { 'type' => 'list', 'data' => attributes } } + end + + def stub_types(*types) + stub_request(:get, "#{base}/ticket_types").to_return(json('type' => 'list', 'data' => types)) + end + + it 'reads one entry per attribute name' do + stub_types(ticket_type('1', 'Bug', attribute('Severity', '9001')), + ticket_type('2', 'Task', attribute('Due', '9002', data_type: 'datetime'))) + + expect(introspector.attributes.map(&:name)).to contain_exactly('Severity', 'Due') + end + + # Measured: two ticket types share the names `_default_title_` and + # `_default_description_` while carrying different attribute ids. A union + # column has no single id to be filtered by, which is why these ship + # unfilterable -- and why the ids are kept, since that is what a filter + # per ticket type will need. + it 'keeps the id each ticket type gives the same attribute name' do + stub_types(ticket_type('1', 'Bug', attribute('_default_title_', '14162161')), + ticket_type('2', 'Task', attribute('_default_title_', '14162165'))) + + expect(introspector.attributes.map(&:ids_by_ticket_type)) + .to eq([{ '1' => '14162161', '2' => '14162165' }]) + end + + # Forest lists the fields of a request in a comma-separated query + # parameter: a comma in a column name splits the projection into fields no + # collection has, and the agent rejects the page with a 400 before reading + # anything. Measured on a real workspace, several attributes carry one. + it 'takes the commas out of a column name, keeping the name the payload uses' do + stub_types(ticket_type('1', 'Bug', attribute("ID de l'objet (immo, facture, user)", '9001'))) + + expect(introspector.attributes.first) + .to have_attributes(name: "ID de l'objet (immo, facture, user)", + column_name: "ID de l'objet (immo facture user)") + end + + # A colon is how Forest names a field through a relation. + it 'takes a colon out too' do + stub_types(ticket_type('1', 'Bug', attribute('Scope: mobile', '9001'))) + + expect(introspector.attributes.first.column_name).to eq('Scope mobile') + end + + # Intercom hands the names back HTML-escaped, which is an artefact of where + # they were typed rather than part of the name. + it 'unescapes what Intercom escaped' do + stub_types(ticket_type('1', 'Bug', attribute('Ce que j'ai vérifié & validé', '9001'))) + + expect(introspector.attributes.first.column_name).to eq("Ce que j'ai vérifié & validé") + end + + it 'leaves a name that needs nothing alone' do + stub_types(ticket_type('1', 'Bug', attribute('Severity', '9001'))) + + expect(introspector.attributes.first).to have_attributes(name: 'Severity', column_name: 'Severity') + end + + # Two different attributes landing on one column would otherwise share an + # entry, and the second's values would be read under the first's name -- + # wrong values rather than missing ones. + it 'leaves out a second attribute that reads as an existing column' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_types(ticket_type('1', 'Bug', attribute('Scope, mobile', '9001'), attribute('Scope mobile', '9002'))) + + expect(introspector.attributes.map(&:name)).to eq(['Scope, mobile']) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/is left out/) + end + + # The same name with another type: one column cannot hold both, so the + # second ticket type's value is left out rather than read as the first's. + it 'keeps the first type of a name two ticket types declare differently, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_types(ticket_type('1', 'Bug', attribute('Severity', '9001', data_type: 'integer')), + ticket_type('2', 'Task', attribute('Severity', '9002', data_type: 'string'))) + + severity = introspector.attributes.first + expect([severity.column_type, severity.ids_by_ticket_type, severity.conflicting_ticket_types]) + .to eq(['Number', { '1' => '9001' }, ['2']]) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/"Severity" is a String on ticket type 2 and a Number elsewhere/) + end + + it 'leaves out an attribute whose name is nothing but separators' do + stub_types(ticket_type('1', 'Bug', attribute(' , : ', '9001'))) + + expect(introspector.attributes).to be_empty + end + + it 'maps the Intercom data types onto what Forest renders' do + stub_types(ticket_type('1', 'Bug', attribute('n', '1', data_type: 'integer'), + attribute('d', '2', data_type: 'decimal'), + attribute('b', '3', data_type: 'boolean'), + attribute('t', '4', data_type: 'datetime'), + attribute('l', '5', data_type: 'list'), + attribute('f', '6', data_type: 'files'))) + + expect(introspector.attributes.map(&:column_type)).to eq(%w[Number Number Boolean Date String Json]) + end + + # Showing the value Intercom sent beats hiding a column because its type + # is one this datasource has not met yet. + it 'reads an unknown data type as a string rather than dropping the column' do + stub_types(ticket_type('1', 'Bug', attribute('x', '1', data_type: 'quantum'))) + + expect(introspector.attributes.map(&:column_type)).to eq(%w[String]) + end + + it 'leaves out an archived attribute, which is not offered any more' do + stub_types(ticket_type('1', 'Bug', attribute('Gone', '1', archived: true), attribute('Here', '2'))) + + expect(introspector.attributes.map(&:name)).to eq(%w[Here]) + end + + it 'leaves out an attribute with no name to be a column of' do + stub_types(ticket_type('1', 'Bug', attribute('', '1'))) + + expect(introspector.attributes).to be_empty + end + + # Read at boot, outside the APIError rescue: a malformed element costs that + # element, never the agent. + it 'skips a ticket type or an attribute that is not an object, keeping the rest' do + stub_types(nil, 7, ticket_type('1', 'Bug', nil, attribute('Severity', '9001'))) + + expect(introspector.attributes.map(&:name)).to eq(['Severity']) + end + + it 'reads a ticket type declaring no attribute as declaring none' do + stub_types({ 'id' => '1', 'name' => 'Bug' }) + + expect(introspector.attributes).to be_empty + end + + # It runs while Rails is starting, so it waits far less than a request + # that already has a page on screen. + it 'reads through the boot connection' do + slow = Configuration.new(access_token: 's3cr3t', rate_limiter: nil, boot_timeout: 2, timeout: 30) + client = Client.new(slow) + stub_types + + described_class.new(client).attributes + + expect(client.send(:boot_connection).options.timeout).to eq(2) + end + + it 'reads once and remembers, a schema being built once' do + stub_types(ticket_type('1', 'Bug', attribute('Severity', '9001'))) + + 2.times { introspector.attributes } + + expect(WebMock).to have_requested(:get, "#{base}/ticket_types").once + end + + # A token without the ticket-types permission costs the attribute columns, + # never the boot of the agent. + it 'degrades to no attribute when the read is refused' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/ticket_types").to_return(json({ 'errors' => [{ 'code' => 'forbidden' }] }, 403)) + + expect(introspector.attributes).to eq([]) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/boots without its attribute/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/throttle_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/throttle_spec.rb new file mode 100644 index 000000000..a2209a2a7 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/throttle_spec.rb @@ -0,0 +1,40 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Throttle do + let(:limiter) { instance_double(RateLimiter, acquire: nil, observe: nil) } + let(:connection) do + Faraday.new(url: 'https://api.intercom.test') do |f| + f.use described_class, limiter: limiter + end + end + + before do + stub_request(:get, 'https://api.intercom.test/me') + .to_return(status: 200, body: '{}', + headers: { 'Content-Type' => 'application/json', 'x-ratelimit-remaining' => '7' }) + end + + it 'asks for room before the request leaves' do + connection.get('me') + + expect(limiter).to have_received(:acquire) + end + + it 'hands the window back what the response says about it' do + connection.get('me') + + expect(limiter).to have_received(:observe).with(hash_including('x-ratelimit-remaining' => '7')) + end + + # The 429 carries the most useful reset of all, so the observation cannot be + # limited to the responses that succeeded. + it 'observes a rejected response too' do + stub_request(:get, 'https://api.intercom.test/me') + .to_return(status: 429, body: '{}', + headers: { 'Content-Type' => 'application/json', 'x-ratelimit-remaining' => '0' }) + + connection.get('me') + + expect(limiter).to have_received(:observe).with(hash_including('x-ratelimit-remaining' => '0')) + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom_spec.rb new file mode 100644 index 000000000..f38f4adfe --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom_spec.rb @@ -0,0 +1,49 @@ +RSpec.describe ForestAdminDatasourceIntercom do + describe 'VERSION' do + # The release `sed` in .releaserc.js only matches `VERSION = "x.y.z"`, and a + # format it misses is a version that stays behind with no CI failure to say so. + it 'is a double-quoted semantic version' do + expect(described_class::VERSION).to match(/\A\d+\.\d+\.\d+\z/) + end + end + + describe '.logger' do + around do |example| + previous = described_class.logger + example.run + described_class.logger = previous + end + + it 'defaults to a logger named after the package' do + described_class.logger = nil + + expect(described_class.logger.progname).to eq('forest_admin_datasource_intercom') + end + + it 'takes the logger it is handed' do + logger = Logger.new(File::NULL) + described_class.logger = logger + + expect(described_class.logger).to be(logger) + end + end + + describe 'errors' do + it 'reports a filter Intercom cannot express as a validation error' do + expect(described_class::UnsupportedOperatorError.new('nope')) + .to be_a(ForestAdminDatasourceToolkit::Exceptions::ValidationError) + end + + it 'carries the status and parsed body of a failed call' do + error = described_class::APIError.new('boom', status: 429, body: { 'type' => 'error.list' }) + + expect(error).to have_attributes(message: 'boom', status: 429, body: { 'type' => 'error.list' }) + end + + it 'leaves the status and body unset when the call failed before answering' do + error = described_class::APIError.new('timeout') + + expect(error).to have_attributes(status: nil, body: nil) + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb b/packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb new file mode 100644 index 000000000..4d659a75a --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb @@ -0,0 +1,215 @@ +require 'tmpdir' + +load File.expand_path('../exe/forest_admin_intercom_probe', __dir__) + +module ForestAdminDatasourceIntercom + RSpec.describe ProbeSearchFields do + let(:base) { Configuration::REGION_HOSTS[:us] } + let(:endpoint) { Query::SearchFields.fetch('tickets') } + let(:client) { Client.new(Configuration.new(access_token: 's3cr3t', rate_limiter: nil)) } + + # The gem installs this as `forest_admin_intercom_probe`, and RubyGems runs + # it through a stub that `load`s it -- so `$PROGRAM_NAME` is the stub and + # not this file. A guard comparing the two paths would ship a command that + # exits without doing anything. + describe '.invoked_as_command?' do + it 'runs behind the stub RubyGems installs' do + expect(described_class.invoked_as_command?('/usr/local/bin/forest_admin_intercom_probe')).to be(true) + end + + it 'runs when the file is the command itself' do + expect(described_class.invoked_as_command?(File.expand_path('../exe/forest_admin_intercom_probe', + __dir__))).to be(true) + end + + it 'stays out of the way of whatever else loads it' do + expect(described_class.invoked_as_command?('/usr/local/bin/rspec')).to be(false) + end + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + # Every probe is a search asking for one record; what a stub answers is + # therefore an empty page or one of Intercom's refusal codes. + def stub_search(code: nil, operator: nil, value: nil) + body = if code + { 'type' => 'error.list', 'errors' => [{ 'code' => code, 'message' => 'nope' }] } + else + { 'type' => 'list', 'tickets' => [], 'total_count' => 0 } + end + + stub_request(:post, "#{base}/tickets/search").with { |request| matches?(request, operator, value) } + .to_return(json(body, code ? 400 : 200)) + end + + def matches?(request, operator, value) + query = JSON.parse(request.body)['query'] + + (operator.nil? || query['operator'] == operator) && (value.nil? || query['value'] == value) + end + + describe 'the type guessed for a candidate field' do + # A candidate is a name and nothing else -- discovering what it is is the + # point -- so the value shape sent with it is guessed from that name. + it 'reads a timestamp, a count and a flag off the name' do + expect(described_class.guess_type('statistics.last_close_at')).to eq('date') + expect(described_class.guess_type('count_reopens')).to eq('number') + expect(described_class.guess_type('open')).to eq('boolean') + expect(described_class.guess_type('state')).to eq('string') + end + end + + describe described_class::Probe do + subject(:probe) { described_class.new(client, endpoint) } + + it 'keeps the operators the endpoint answers and drops the ones it refuses' do + stub_search(code: 'data_invalid') + stub_search(operator: '=') + stub_search(operator: '!=') + + result = probe.run('category', 'string') + + expect(result[:operators].select { |_, outcome| outcome[:ok] }.keys).to eq(['=', '!=']) + end + + # A field the endpoint does not filter at all is worth one request, not + # twelve: `invalid_field` ends the row. + it 'stops at the first invalid_field and reports the field unfilterable' do + stub_search(code: 'invalid_field') + + result = probe.run('company_id', 'string') + + expect(result[:unfilterable][:code]).to eq('invalid_field') + expect(a_request(:post, "#{base}/tickets/search")).to have_been_made.once + end + + # A wrong value shape is refused with the same code as an unsupported + # operator, so a single attempt would report a filter Intercom does answer + # as refused. + it 'retries a refused cell with the other value shapes its type takes' do + stub_search(code: 'data_invalid') + stub_search(operator: '>', value: '2026-01-01') + + result = probe.run('created_at', 'date') + + expect(result[:operators]['>'][:ok]).to be(true) + expect(result[:operators]['<'][:ok]).to be(false) + end + + it 'names the refusal by its HTTP status when Intercom sends no error code' do + stub_request(:post, "#{base}/tickets/search").to_return(json({ 'nope' => true }, 400)) + + result = probe.run('category', 'string') + + expect(result[:operators]['='][:code]).to eq('http_400') + end + + # A failure that is not about the search is not a measurement: recorded, + # it would publish a valid filter as refused. + [401, 429, 500].each do |status| + it "stops on an HTTP #{status} rather than recording it as a refusal" do + stub_request(:post, "#{base}/tickets/search").to_return(json({ 'errors' => [{ 'code' => 'x' }] }, status)) + + expect { probe.run('category', 'string') } + .to raise_error(ProbeSearchFields::Inconclusive, /HTTP #{status}, which says nothing about the field/) + end + end + end + + describe described_class::Report do + subject(:report) { described_class.new(endpoint) } + + def run_probe(field, type) + ProbeSearchFields::Probe.new(client, endpoint).run(field, type) + end + + # The only reason to run the probe: an operator the table promises and + # Intercom refuses is a filter the interface offers and the read cannot + # honour. + it 'reports an operator the table promises and Intercom refuses' do + stub_search(code: 'data_invalid') + stub_search(operator: '=') + report.record('category', 'category', run_probe('category', 'string')) + + expect { report.print_diff }.to output(/! the table promises != here, Intercom refuses it/).to_stdout + end + + it 'reports an operator the table does not know about yet' do + stub_search(code: 'data_invalid') + ['=', '!=', '~'].each { |operator| stub_search(operator: operator) } + report.record('category', 'category', run_probe('category', 'string')) + + expect { report.print_diff }.to output(/\+ Intercom also accepts ~/).to_stdout + end + + it 'names the column a field the endpoint refuses is declared on' do + stub_search(code: 'invalid_field') + report.record('category', 'category', run_probe('category', 'string')) + + expect { report.print_diff }.to output(/NOT FILTERABLE \(invalid_field\).*column 'category'/).to_stdout + end + + it 'writes the measurement as evidence, refusal codes included' do + stub_search(code: 'data_invalid') + stub_search(operator: '=') + report.record('category', 'category', run_probe('category', 'string')) + + written = YAML.safe_load(report.to_yaml_document) + + expect(written['fields']['category']['operators']).to eq(['=']) + expect(written['fields']['category']['refused']['!=']).to eq('data_invalid') + end + + it 'writes a field the endpoint refuses as unfilterable' do + stub_search(code: 'invalid_field') + report.record('company_id', nil, run_probe('company_id', 'string')) + + written = YAML.safe_load(report.to_yaml_document) + + expect(written['fields']['company_id']).to eq({ 'filterable' => false, 'code' => 'invalid_field' }) + end + end + + describe described_class::CLI do + # A probe with no token would abort on the first request with an Intercom + # 401, which reads as a workspace problem rather than as a missing option. + it 'refuses to start without a token' do + expect { described_class.call(['--endpoint', 'tickets']) } + .to raise_error(SystemExit).and output(/No token/).to_stderr + end + + it 'refuses an endpoint the table does not declare' do + expect { described_class.call(['--endpoint', 'companies', '--token', 's3cr3t']) } + .to raise_error(ConfigurationError, /Unknown Intercom search endpoint/) + end + + it 'probes every declared endpoint and writes the evidence where it was asked to' do + out = File.join(Dir.tmpdir, 'intercom-probe.yml') + stub_search(code: 'invalid_field') + stub_request(:post, "#{base}/conversations/search") + .to_return(json({ 'type' => 'error.list', 'errors' => [{ 'code' => 'invalid_field' }] }, 400)) + stub_request(:post, "#{base}/contacts/search") + .to_return(json({ 'type' => 'error.list', 'errors' => [{ 'code' => 'invalid_field' }] }, 400)) + + expect { described_class.call(['--token', 's3cr3t', '--out', out]) }.to output(/NOT FILTERABLE/).to_stdout + expect(YAML.safe_load_file(out)['endpoint']).to eq('conversations') + ensure + FileUtils.rm_f(out) + end + + it 'stops without writing anything when an answer measures nothing' do + out = File.join(Dir.tmpdir, 'intercom-probe-inconclusive.yml') + stub_request(:post, "#{base}/tickets/search") + .to_return(json({ 'errors' => [{ 'code' => 'unauthorized' }] }, 401)) + + expect { described_class.call(['--endpoint', 'tickets', '--token', 's3cr3t', '--out', out]) } + .to raise_error(SystemExit).and output(/Probe stopped: .*HTTP 401/).to_stderr.and output.to_stdout + expect(File).not_to exist(out) + ensure + FileUtils.rm_f(out) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/spec_helper.rb b/packages/forest_admin_datasource_intercom/spec/spec_helper.rb new file mode 100644 index 000000000..bdf9ae346 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/spec_helper.rb @@ -0,0 +1,78 @@ +require 'simplecov' +# JSON output is consumed by the qlty CI coverage step; HTML is for local +# inspection. simplecov-html and simplecov_json_formatter are required only +# in Gemfile-test, so guard the require for local Gemfile runs. +begin + require 'simplecov_json_formatter' + require 'simplecov-html' + SimpleCov.formatters = [SimpleCov::Formatter::JSONFormatter, SimpleCov::Formatter::HTMLFormatter] +rescue LoadError + # Local Gemfile run without the CI formatters; default text output is fine. +end + +SimpleCov.start do + add_filter '/spec/' + enable_coverage :branch + minimum_coverage 90 +end + +SimpleCov.coverage_dir 'coverage' + +require 'webmock/rspec' +require 'forest_admin_datasource_customizer' +require 'forest_admin_datasource_intercom' + +# Every payload the specs feed in is hand-written from the Intercom OpenAPI 2.16 +# spec, never captured from a workspace: a conversation body is personal data, +# and a fixture is read by everyone who clones the repo. +WebMock.disable_net_connect!(allow_localhost: true) + +# A datasource checks the API version it was served and introspects the +# ticket-type attributes and the contact and company attributes while it +# registers its collections, so every spec building one issues those four +# reads. The base url is not taken from the datasource on purpose: reading it +# would build the datasource, and boot the very reads this stubs. +module IntercomBootStubs + def stub_me(base: ForestAdminDatasourceIntercom::Configuration::REGION_HOSTS[:us], + version: ForestAdminDatasourceIntercom::Configuration::DEFAULT_API_VERSION) + stub_request(:get, "#{base}/me") + .to_return(status: 200, body: { 'type' => 'admin', 'id' => '1', 'email' => 'ops@example.test' }.to_json, + headers: { 'Content-Type' => 'application/json', 'Intercom-Version' => version }) + end + + def stub_ticket_types(*types, base: ForestAdminDatasourceIntercom::Configuration::REGION_HOSTS[:us]) + stub_request(:get, "#{base}/ticket_types") + .to_return(status: 200, body: { 'type' => 'list', 'data' => types }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end + + def stub_data_attributes(model, *attributes, + base: ForestAdminDatasourceIntercom::Configuration::REGION_HOSTS[:us]) + stub_request(:get, "#{base}/data_attributes").with(query: { 'model' => model }) + .to_return(status: 200, + body: { 'type' => 'list', 'data' => attributes }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end +end + +RSpec.configure do |config| + config.include IntercomBootStubs + config.expect_with :rspec do |c| + c.syntax = :expect + end + config.mock_with :rspec do |m| + m.verify_partial_doubles = true + end + config.disable_monkey_patching! + config.warnings = false + config.order = :random + Kernel.srand config.seed + + config.before do + WebMock.reset! + stub_me + stub_ticket_types + stub_data_attributes('contact') + stub_data_attributes('company') + end +end diff --git a/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/utils/collection.rb b/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/utils/collection.rb index 537cdf97c..264b9ca7b 100644 --- a/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/utils/collection.rb +++ b/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/utils/collection.rb @@ -56,10 +56,11 @@ def self.other_inverse?(field, relation_field) field.origin_key == relation_field.foreign_key end + # ValidationError: an unknown field comes from the request, so it is a 400, not a 500. def self.get_field_schema(collection, field_name) fields = collection.schema[:fields] unless field_name.include?(':') - raise ForestException, "Column not found #{collection.name}.#{field_name}" unless fields.key?(field_name) + raise ValidationError, "Column not found #{collection.name}.#{field_name}" unless fields.key?(field_name) return fields[field_name] end @@ -67,10 +68,11 @@ def self.get_field_schema(collection, field_name) association_name = field_name.split(':')[0] relation_schema = fields[association_name] - raise ForestException, "Relation not found #{collection.name}.#{association_name}" unless relation_schema + raise ValidationError, "Relation not found #{collection.name}.#{association_name}" unless relation_schema if relation_schema.type != 'ManyToOne' && relation_schema.type != 'OneToOne' - raise ForestException, "Unexpected field type #{relation_schema.type}: #{collection.name}.#{association_name}" + raise ValidationError, + "Unexpected field type #{relation_schema.type}: #{collection.name}.#{association_name}" end get_field_schema( @@ -161,7 +163,8 @@ def self.list_relation(collection, primary_key_values, relation_name, caller, fo projection.nest(prefix: foreign_relation) ) - return records.map { |r| r[foreign_relation] } + # A through row whose target is gone yields no row, not a nil the serializer would read. + return records.filter_map { |r| r[foreign_relation] } end end diff --git a/packages/forest_admin_datasource_toolkit/spec/lib/forest_admin_datasource_toolkit/utils/collection_spec.rb b/packages/forest_admin_datasource_toolkit/spec/lib/forest_admin_datasource_toolkit/utils/collection_spec.rb index fb7d88535..aa3b716bc 100644 --- a/packages/forest_admin_datasource_toolkit/spec/lib/forest_admin_datasource_toolkit/utils/collection_spec.rb +++ b/packages/forest_admin_datasource_toolkit/spec/lib/forest_admin_datasource_toolkit/utils/collection_spec.rb @@ -221,7 +221,7 @@ module Utils expect do described_class.get_field_schema(collection_person, 'foo') - end.to raise_error(ForestException, 'Column not found Person.foo') + end.to raise_error(ValidationError, 'Column not found Person.foo') end it 'get_field_schema should work with simple column' do @@ -233,14 +233,14 @@ module Utils expect do described_class.get_field_schema(collection_person, 'unknown:foo') - end.to raise_error(ForestException, 'Relation not found Person.unknown') + end.to raise_error(ValidationError, 'Relation not found Person.unknown') end it 'get_field_schema should throw with invalid relation type' do expect do described_class.get_field_schema(collection_book, 'myBookPersons:bookId') - end.to raise_error(ForestException, 'Unexpected field type OneToMany: Book.myBookPersons') + end.to raise_error(ValidationError, 'Unexpected field type OneToMany: Book.myBookPersons') end it 'get_field_schema should work with relation column' do @@ -311,6 +311,20 @@ module Utils ForestAdminDatasourceToolkit::Components::Query::Projection.new)).to eq([1]) end + it 'list_relation should drop a through row whose target resolved to nothing' do + book_person_class = Struct.new(:bookId, :personId, :myPerson, :myBook) + stub_const('BookPerson', book_person_class) + allow(collection_book_person).to receive(:list).and_return( + [ + BookPerson.new(1, 1, 1, 1), + BookPerson.new(1, 2, nil, 1) + ] + ) + + expect(described_class.list_relation(collection_book, [1], 'myPersons', caller, ForestAdminDatasourceToolkit::Components::Query::Filter.new, + ForestAdminDatasourceToolkit::Components::Query::Projection.new)).to eq([1]) + end + it 'aggregate_relation should work with one to many relation' do allow(collection_book_person).to receive(:aggregate).and_return(1)