From ae3dabd12cd73d4070cff25afedf3615cc2900d4 Mon Sep 17 00:00:00 2001 From: Benita Volkmann Date: Fri, 18 Sep 2026 18:17:10 +0200 Subject: [PATCH 1/3] Self-contained sync rules and sync streams sections --- .claude/CLAUDE.md | 11 +- .claude/agents/document-reviewer.md | 1 + .claude/skills/doc-author/SKILL.md | 2 +- .claude/skills/pr-to-docs/SKILL.md | 2 +- docs.json | 29 +- package.json | 3 +- scripts/check-links.mjs | 177 ++++++++++++ snippets/sync-shared/case-sensitivity.mdx | 40 +++ snippets/sync-shared/client-id.mdx | 65 +++++ snippets/sync-shared/compatibility.mdx | 217 ++++++++++++++ .../sync-shared/reducing-bucket-count.mdx | 240 ++++++++++++++++ .../sync-shared/schemas-and-connections.mdx | 60 ++++ snippets/sync-shared/sharded-databases.mdx | 44 +++ snippets/sync-shared/storage-version-4.mdx | 170 +++++++++++ snippets/sync-shared/types.mdx | 164 +++++++++++ sync/advanced/case-sensitivity.mdx | 39 +-- sync/advanced/client-id.mdx | 64 +---- sync/advanced/compatibility.mdx | 216 +------------- sync/advanced/multiple-client-versions.mdx | 52 ++-- sync/advanced/partitioned-tables.mdx | 65 ++--- sync/advanced/reducing-bucket-count.mdx | 239 +--------------- sync/advanced/schemas-and-connections.mdx | 59 +--- sync/advanced/sharded-databases.mdx | 43 +-- sync/advanced/storage-version-4.mdx | 169 +---------- sync/advanced/sync-data-by-time.mdx | 267 ++++++------------ sync/grammar/sync-rules/index.mdx | 4 +- sync/rules/case-sensitivity.mdx | 15 + sync/rules/client-id.mdx | 15 + sync/rules/compatibility.mdx | 15 + sync/rules/data-queries.mdx | 2 +- sync/rules/migrate-to-sync-streams.mdx | 4 +- sync/rules/multiple-client-versions.mdx | 39 +++ sync/rules/organize-data-into-buckets.mdx | 2 +- sync/rules/overview.mdx | 25 +- sync/rules/parameter-queries.mdx | 6 +- sync/rules/partitioned-tables.mdx | 37 +++ sync/rules/prioritized-sync.mdx | 126 +++++++++ sync/rules/reducing-bucket-count.mdx | 16 ++ sync/rules/schemas-and-connections.mdx | 15 + sync/rules/sharded-databases.mdx | 15 + sync/rules/storage-version-4.mdx | 15 + sync/rules/supported-sql.mdx | 157 ++++++++++ sync/rules/sync-data-by-time.mdx | 174 ++++++++++++ sync/rules/types.mdx | 16 ++ sync/streams/bucket-count.mdx | 4 - sync/streams/client-usage.mdx | 2 +- sync/streams/examples.mdx | 2 +- sync/streams/overview.mdx | 6 - sync/streams/parameters.mdx | 2 +- sync/streams/prioritized-sync.mdx | 111 +------- sync/streams/quickstart.mdx | 1 - sync/supported-sql.mdx | 122 ++------ sync/types.mdx | 163 +---------- 53 files changed, 2074 insertions(+), 1475 deletions(-) create mode 100644 scripts/check-links.mjs create mode 100644 snippets/sync-shared/case-sensitivity.mdx create mode 100644 snippets/sync-shared/client-id.mdx create mode 100644 snippets/sync-shared/compatibility.mdx create mode 100644 snippets/sync-shared/reducing-bucket-count.mdx create mode 100644 snippets/sync-shared/schemas-and-connections.mdx create mode 100644 snippets/sync-shared/sharded-databases.mdx create mode 100644 snippets/sync-shared/storage-version-4.mdx create mode 100644 snippets/sync-shared/types.mdx create mode 100644 sync/rules/case-sensitivity.mdx create mode 100644 sync/rules/client-id.mdx create mode 100644 sync/rules/compatibility.mdx create mode 100644 sync/rules/multiple-client-versions.mdx create mode 100644 sync/rules/partitioned-tables.mdx create mode 100644 sync/rules/prioritized-sync.mdx create mode 100644 sync/rules/reducing-bucket-count.mdx create mode 100644 sync/rules/schemas-and-connections.mdx create mode 100644 sync/rules/sharded-databases.mdx create mode 100644 sync/rules/storage-version-4.mdx create mode 100644 sync/rules/supported-sql.mdx create mode 100644 sync/rules/sync-data-by-time.mdx create mode 100644 sync/rules/types.mdx diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 17b261d2e..ca29c2dab 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -142,18 +142,19 @@ Update `docs.json` when adding, moving, or removing pages. Add redirects for mov ## Sync Streams and Sync Rules -Sync Streams are the default for new documentation. Keep legacy Sync Rules documentation accurate, but do not add new Sync Rules teaching, examples, or proactive references. +Sync Rules are deprecated. New documentation and updates cover Sync Streams only. Many customers still use Sync Rules, so their documentation stays online but frozen. -When existing content shows both in tabs, preserve equivalent results and filters. Do not add new parallel Sync Rules examples. - -For existing prose that mentions both, use "[Sync Streams](/sync/streams/overview) (or legacy [Sync Rules](/sync/rules/overview))" once per page or major section. Later mentions should omit Sync Rules. +- **Sync Rules docs are frozen.** `sync/rules/` and `sync/grammar/sync-rules/` take error fixes only: no new features, examples, or pages. Each page opens with an `` callout that starts "Sync Rules are deprecated." and links to its Sync Streams version. The sidebar group stays "Sync Rules (Legacy)". +- **Keep the engines apart.** Never place Sync Streams and Sync Rules content side by side: no engine tabs, no "(or legacy Sync Rules)" asides, no pointers to a Sync Rules equivalent. Outside `sync/rules/`, mention Sync Rules only to state a behavior difference that affects Sync Rules users, and remove other mentions when you edit a page. +- **Shared pages.** A page that applies to both engines keeps one body in `snippets/sync-shared/.mdx`, imported by a Sync Streams wrapper at the original path and a Sync Rules wrapper at `sync/rules/.mdx` that adds the callout and `noindex: true`. Edit the snippet, not the wrappers, and keep it valid for both engines. +- **Counterpart comments.** Every split twin, wrapper, and shared snippet starts with an MDX comment naming its counterpart. Read it before editing, apply a fix to both sides where content is shared, and keep the comment when restructuring. ## Verification - Verify technical claims and run code examples before publication. Select other checks appropriate to the change. - Run `vale ` for changed MDX pages. Add new technical terms to `.github/vale/config/vocabularies/PowerSync/accept.txt`; do not add ordinary misspellings. - After link or navigation changes, run `npx mintlify broken-links`. Mintlify requires Node 20.17–24; if needed, use `PATH="/opt/homebrew/opt/node@24/bin:$PATH" npx mintlify broken-links`. -- For anchor and snippet checks, use `pnpm check:links`. Validate repository instruction links as file paths, since the site checker does not cover all of them. +- For anchor and snippet checks, use `pnpm check:links`. It wraps the Mintlify anchor check in `scripts/check-links.mjs` so that anchors defined in imported snippets resolve. Validate repository instruction links as file paths, since the site checker does not cover all of them. - Use [the lint command](commands/lint-docs.md) for the check workflow and [the reviewer](agents/document-reviewer.md) for editorial review. Passing linters does not establish technical accuracy or style compliance. ## Git Workflow diff --git a/.claude/agents/document-reviewer.md b/.claude/agents/document-reviewer.md index 06f904eae..05794244e 100644 --- a/.claude/agents/document-reviewer.md +++ b/.claude/agents/document-reviewer.md @@ -17,6 +17,7 @@ Apply the canonical standards in three passes and report findings from each: 1. **Accuracy:** claims, platform scope, versions, and consistency with the surrounding page. Before reporting a claim as unverified, check the sources the PR or the user cites, such as the source PR, divergence issue, release notes, or code at the merged commit, and follow the links inside them. If nothing is cited, look up the release the text names. Report a claim as unverified only when no source covers it or a source contradicts it. Give evidence, do not invent problems, and do not approve unverified claims as correct. 2. **Necessity:** list every sentence that describes what the product prints, displays, logs, or says in an error. Treat each one as a finding to remove unless it passes the restating rule in [Content Strategy](../CLAUDE.md#content-strategy), and report it even when the sentence is accurate. Describing visible output is not a mechanism, consequence, or signal. Also flag internal mechanics, rare exceptions, and repetition. Flag missing context only when readers need it, and do not require every entry to explain a mechanism, consequence, signal, action, and trade-off. 3. **Clarity and format:** plain technical English, clear actors and actions, and suitable examples and components. +4. **Sync Rules containment:** flag any Sync Rules mention, example, or tab outside `sync/rules/` and `snippets/sync-shared/` that does not state a behavior difference, and any new Sync Rules content anywhere. Check that split twins, wrappers, and shared snippets keep their counterpart comment. ## Default Output diff --git a/.claude/skills/doc-author/SKILL.md b/.claude/skills/doc-author/SKILL.md index 6ac236dc1..84de0db5d 100644 --- a/.claude/skills/doc-author/SKILL.md +++ b/.claude/skills/doc-author/SKILL.md @@ -18,6 +18,6 @@ Use the canonical Working Process for scope changes and unresolved decisions. 1. Identify the reader, desired outcome, and affected feature or concept. 2. Research the implementation and existing coverage. Read the most relevant related pages and `docs.json`; avoid unnecessary duplication. 3. If a plan is needed, state the proposed pages, structure, and unresolved questions before drafting. -4. Write the update under the canonical standards. Keep the existing structure unless the task requires a change. +4. Write the update under the canonical standards. Cover Sync Streams only and keep Sync Rules content out of it; Sync Rules pages take error fixes only. Keep the existing structure unless the task requires a change. 5. Self-review for accuracy, reader understanding, minimum useful detail, and navigation fit. Run the canonical verification checks relevant to the change. 6. Present the result and any unresolved draft TODOs, or complete the delivery workflow already authorized by the user. diff --git a/.claude/skills/pr-to-docs/SKILL.md b/.claude/skills/pr-to-docs/SKILL.md index d5203a4a8..b74ea0ed0 100644 --- a/.claude/skills/pr-to-docs/SKILL.md +++ b/.claude/skills/pr-to-docs/SKILL.md @@ -33,4 +33,4 @@ Ask before expanding scope, documenting a deprecation that needs migration decis ## 4. Draft and Verify -Apply the canonical writing standards, navigation requirements, and verification checks. Preserve existing structure unless restructuring is part of the approved plan. Flag unresolved facts using the canonical draft-TODO convention and report what must be resolved before publication. +Apply the canonical writing standards, navigation requirements, and verification checks. Document new behavior for Sync Streams only; do not add Sync Rules examples or mentions. Preserve existing structure unless restructuring is part of the approved plan. Flag unresolved facts using the canonical draft-TODO convention and report what must be resolved before publication. diff --git a/docs.json b/docs.json index 50a4eec91..7d5d58fa2 100644 --- a/docs.json +++ b/docs.json @@ -235,7 +235,30 @@ "sync/rules/data-queries", "sync/rules/many-to-many-join-tables", "sync/rules/client-parameters", - "sync/grammar/sync-rules/index" + { + "group": "Reference", + "pages": [ + "sync/rules/supported-sql", + "sync/grammar/sync-rules/index", + "sync/rules/types" + ] + }, + { + "group": "Advanced", + "pages": [ + "sync/rules/reducing-bucket-count", + "sync/rules/prioritized-sync", + "sync/rules/client-id", + "sync/rules/case-sensitivity", + "sync/rules/compatibility", + "sync/rules/storage-version-4", + "sync/rules/sync-data-by-time", + "sync/rules/schemas-and-connections", + "sync/rules/multiple-client-versions", + "sync/rules/partitioned-tables", + "sync/rules/sharded-databases" + ] + } ] } ] @@ -800,10 +823,6 @@ "source": "/usage/sync-rules/operators-and-functions", "destination": "/sync/supported-sql" }, - { - "source": "/sync/rules/supported-sql", - "destination": "/sync/supported-sql" - }, { "source": "/usage/sync-rules/advanced-topics", "destination": "/sync/advanced/overview" diff --git a/package.json b/package.json index d83b56d31..490f97ce2 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "packageManager": "pnpm@11.3.0+sha512.2c403d6594527287672b1f7056343a1f7c3634036a67ffabfcc2b3d7595d843768f8787148d1b57cf7956c90606bbd192857c363af19e96d2d0ec9ec5741d215", "scripts": { "dev": "mintlify dev", - "check:links": "mintlify broken-links --check-anchors --check-snippets" + "check:links": "node scripts/check-links.mjs", + "check:links:mintlify": "mintlify broken-links --check-anchors --check-snippets" }, "devDependencies": { "mintlify": "^4.2.520" diff --git a/scripts/check-links.mjs b/scripts/check-links.mjs new file mode 100644 index 000000000..a2fa6c6c8 --- /dev/null +++ b/scripts/check-links.mjs @@ -0,0 +1,177 @@ +#!/usr/bin/env node +/** + * Snippet-aware link check. + * + * Runs `mintlify broken-links --check-anchors --check-snippets`, then re-validates + * every reported `#anchor` against the headings of the target page *including* + * headings that come from snippets the page imports. The Mintlify checker only + * reads page files, so a page whose body lives in a snippet (for example the pages + * shared between the Sync Streams and Sync Rules sections, see snippets/sync-shared/) + * would otherwise fail for every inbound anchor link. + * + * Reported links without a fragment, and anchors that still cannot be found, are + * printed in the Mintlify format and make the script exit with status 1. + */ +import { spawnSync } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; + +const root = process.cwd(); +const localBin = path.join(root, 'node_modules', '.bin', 'mintlify'); +const bin = existsSync(localBin) ? localBin : 'mintlify'; + +const run = spawnSync(bin, ['broken-links', '--check-anchors', '--check-snippets'], { + cwd: root, + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, +}); +if (run.error) { + console.error(`Could not run ${bin}: ${run.error.message}`); + process.exit(1); +} + +const output = ((run.stdout ?? '') + (run.stderr ?? '')) + .replace(/\x1B\[[0-9;?]*[A-Za-z]/g, '') + .replace(/\r/g, ''); + +// Parse blocks of "\n ⎿ \n ⎿ ". +const flagged = []; +let currentFile = null; +for (const rawLine of output.split('\n')) { + const line = rawLine.trim(); + if (!line) continue; + if (/checking for broken links/.test(line)) continue; + if (/^found \d+ broken link/.test(line)) continue; + if (/^success/i.test(line)) continue; + const link = line.match(/^⎿\s*(\S.*)$/); + if (link) { + if (currentFile) flagged.push({ file: currentFile, link: link[1].trim() }); + continue; + } + currentFile = line; +} + +if (run.status === 0 && flagged.length === 0) { + console.log('success no broken links found'); + process.exit(0); +} +if (flagged.length === 0) { + // Non-zero exit without a parsable report: show what Mintlify printed. + console.log(output.trim()); + process.exit(run.status ?? 1); +} + +function pageFile(urlPath) { + const p = urlPath.replace(/^\//, '').replace(/\/$/, ''); + for (const candidate of [`${p}.mdx`, `${p}.md`, `${p}/index.mdx`, `${p}/index.md`]) { + if (existsSync(path.join(root, candidate))) return candidate; + } + return null; +} + +function snippetFile(spec) { + const rel = spec.startsWith('/') ? spec.slice(1) : path.posix.join('snippets', spec); + return existsSync(path.join(root, rel)) ? rel : null; +} + +// Page source plus the source of every snippet it imports, recursively. +function collectSource(file, seen = new Set()) { + if (seen.has(file)) return ''; + seen.add(file); + const src = readFileSync(path.join(root, file), 'utf8').replace(/^---\n[\s\S]*?\n---\n/, ''); + let out = src; + for (const m of src.matchAll(/^import\s+\w+\s+from\s+['"]([^'"]+)['"]/gm)) { + const f = snippetFile(m[1]); + if (f) out += `\n${collectSource(f, seen)}`; + } + for (const m of src.matchAll(/]*>/g, ' ') + .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') + .replace(/[`*]/g, '') + .trim() + .toLowerCase(); + const strict = base.replace(/[^a-z0-9\s_-]/g, '').trim().replace(/\s+/g, '-'); + const keepPunctuation = base.replace(/[^a-z0-9\s_/:-]/g, '').trim().replace(/\s+/g, '-'); + return new Set([strict, keepPunctuation, strict.replace(/-+/g, '-'), keepPunctuation.replace(/-+/g, '-')]); +} + +const anchorCache = new Map(); +function anchorsFor(file) { + if (anchorCache.has(file)) return anchorCache.get(file); + const src = collectSource(file).replace(/```[\s\S]*?```/g, ''); + const anchors = new Set(); + for (const m of src.matchAll(/^#{1,6}\s+(.+?)\s*$/gm)) { + let heading = m[1]; + const explicit = heading.match(/\{#([^}]+)\}\s*$/); + if (explicit) { + anchors.add(explicit[1]); + heading = heading.replace(/\{#[^}]+\}\s*$/, ''); + } + for (const slug of slugCandidates(heading)) anchors.add(slug); + } + for (const m of src.matchAll(/]*\btitle=["']([^"']+)["']/g)) { + for (const slug of slugCandidates(m[1])) anchors.add(slug); + } + for (const m of src.matchAll(/\bid=["']([^"']+)["']/g)) anchors.add(m[1]); + for (const m of src.matchAll(/<(?:ResponseField|ParamField)\b[^>]*\bname=["']([^"']+)["']/g)) { + anchors.add(`param-${m[1].replace(/_/g, '-')}`); + } + anchorCache.set(file, anchors); + return anchors; +} + +const unresolved = []; +for (const item of flagged) { + const hash = item.link.indexOf('#'); + if (hash < 0) { + unresolved.push(item); // a path problem, not an anchor problem + continue; + } + const target = item.link.slice(0, hash).split('?')[0]; + let anchor = item.link.slice(hash + 1); + try { + anchor = decodeURIComponent(anchor); + } catch { + // keep the raw fragment + } + const file = target === '' ? item.file : pageFile(target); + if (!file) { + unresolved.push(item); + continue; + } + const anchors = anchorsFor(file); + const withoutSuffix = anchor.replace(/-\d+$/, ''); + if (anchors.has(anchor) || (withoutSuffix !== anchor && anchors.has(withoutSuffix))) continue; + unresolved.push(item); +} + +const plural = (n, word) => `${n} ${word}${n === 1 ? '' : 's'}`; +if (unresolved.length === 0) { + console.log(`success no broken links found (${plural(flagged.length, 'anchor')} resolved through imported snippets)`); + process.exit(0); +} + +const byFile = new Map(); +for (const item of unresolved) { + if (!byFile.has(item.file)) byFile.set(item.file, []); + byFile.get(item.file).push(item.link); +} +console.log(`found ${plural(unresolved.length, 'broken link')} in ${plural(byFile.size, 'file')}\n`); +for (const [file, links] of byFile) { + console.log(file); + for (const link of links) console.log(` ⎿ ${link}`); + console.log(); +} +process.exit(1); diff --git a/snippets/sync-shared/case-sensitivity.mdx b/snippets/sync-shared/case-sensitivity.mdx new file mode 100644 index 000000000..da99add96 --- /dev/null +++ b/snippets/sync-shared/case-sensitivity.mdx @@ -0,0 +1,40 @@ +{/* Shared body: rendered by sync/advanced/case-sensitivity.mdx (Sync Streams section) and sync/rules/case-sensitivity.mdx (Sync Rules (Legacy) section). Keep the content valid for both engines. */} + +### Case in Sync Rules + +PowerSync converts all table/collection and column/field names to lower-case by default in Sync Rule queries (this is how Postgres also behaves). To preserve the case, surround the names with double quotes, for example: + +```sql +SELECT "ID" as id, "Description", "ListID" FROM "TODOs" WHERE "TODOs"."ListID" = bucket.list_id +``` + +When using `SELECT *`, the original case is preserved for the returned columns/fields. + +### Client-Side Case + +On the client side, the case of table and column names in the [client-side schema](/intro/setup-guide#define-your-client-side-schema) must match the case produced by Sync Rules exactly. For the above example, use the following in Dart: + +```dart + Table('TODOs', [ + Column.text('Description'), + Column.text('ListID') + ]) +``` + +SQLite itself is case-insensitive. When querying and modifying the data on the client, any case may be used. For example, the above table may be queried using `SELECT description FROM todos WHERE listid = ?`. + +Operations (`PUT`/`PATCH`/`DELETE`) are stored in the upload queue using the case as defined in the schema above for table and column names, not the case used in queries. + +As another example, in this Sync Rule query: + +```sql +SELECT ID, todo_description as Description FROM todo_items as TODOs +``` + +Each identifier in the example is unquoted and converted to lower case. That means the client-side schema would be: + +```dart +Table('todos', [ + Column.text('description') +]) +``` diff --git a/snippets/sync-shared/client-id.mdx b/snippets/sync-shared/client-id.mdx new file mode 100644 index 000000000..f6ec13cd5 --- /dev/null +++ b/snippets/sync-shared/client-id.mdx @@ -0,0 +1,65 @@ +{/* Shared body: rendered by sync/advanced/client-id.mdx (Sync Streams section) and sync/rules/client-id.mdx (Sync Rules (Legacy) section). Keep the content valid for both engines. */} + +For tables where the client will create new rows: + +- Postgres, MySQL and SQL Server: use a UUID for `id`. Use the `uuid()` helper to generate a random UUID (v4) on the client. +- MongoDB: use an `ObjectId` for `_id`. Generate an `ObjectId()` in your app code and store it in the client's `id` column as a string; this will map to MongoDB's `_id`. + +To use a different column/field from the server-side database as the record ID on the client, use a column/field alias in your [Sync Streams](/sync/streams/overview) query (or [Sync Rules](/sync/rules/overview) data query): + +```sql +SELECT client_id as id FROM my_data +``` + + + MongoDB uses `_id` as the name of the ID field in collections. You must use `SELECT _id as id` (and include any other columns you need) in [Sync Streams](/sync/streams/overview) queries and [Sync Rules](/sync/rules/overview) data queries when using MongoDB as the backend source database. When inserting new documents from the client, prefer `ObjectId` values for `_id` (stored in the client's `id` column). + + +Custom transformations can also be used for the ID column. This is useful in certain scenarios for example when dealing with join tables, because PowerSync doesn't currently support composite primary keys. For example: + +```sql +-- Concatenate multiple columns into a single id column +SELECT *, item_id || '.' || category_id as id FROM item_categories + +-- the source database schema for the above example is CREATE TABLE item_categories(item_id uuid, category_id uuid, PRIMARY KEY(item_id, category_id)); +``` + + + For multiple columns with the same name (e.g. if there was an `id` column in `*`), the last column wins. Prefer writing the `*` before other columns for this reason. + + If you want to upload data to a table with a custom record ID, ensure that `uploadData()` isn't blindly using a field named `id` when handling CRUD operations. See the [Sequential ID mapping tutorial](/client-sdks/advanced/sequential-id-mapping#update-client-to-use-uuids) for an example where the record ID is aliased to `uuid` on the backend. + + +PowerSync does not perform any validation that IDs are unique. Duplicate IDs on a client could occur in any of these scenarios: + +1. A non-unique column is used for the ID. +2. Multiple table partitions are used (Postgres), with the same ID present in different partitions. +3. Multiple data queries returning the same record. This is typically not an issue if the queries return the same values (same transformations used in each query). + +We recommend using a unique index on the fields in the source database to ensure uniqueness — this will prevent (1) at least. + +If the client does sync multiple records with the same ID, only one will be present in the final database. This would typically be the one modified last, but this is subject to change — do not depend on any specific record being picked. + +### Postgres: Strategies for Auto-Incrementing IDs + +With auto-incrementing / sequential IDs (e.g. `sequence` type in Postgres), the issue is that the ID can only be generated on the server, and not on the client while offline. If this _must_ be used, there are some options, depending on the use case. + +#### Option 1: Generate ID when server receives record + +If the client does not use the ID as a reference (foreign key) elsewhere, insert any unique value on the client in the `id` field, then generate a new ID when the server receives it. + +#### Option 2: Pre-create records on the server + +For some use cases, it could work to have the server pre-create a set of e.g. 100 draft records for each user. While offline, the client can populate these records without needing to generate new IDs. This is similar to providing an employee with a paper book of blank invoices — each with an invoice number pre-printed. + +This does mean that a user has a limit on how many records can be populated while offline. + +Care must be taken if a user can populate the same records from different devices while offline — ideally each device must have a unique set of pre-created records. + +#### Option 3: Use an ID mapping + +Use UUIDs on the client, then map them to sequential IDs when performing an update on the server. This allows using a sequential primary key for each record, with a UUID as a secondary ID. + +This mapping must be performed wherever the UUIDs are referenced, including for every foreign key column. + +For more information, have a look at [Sequential ID Mapping](/client-sdks/advanced/sequential-id-mapping). diff --git a/snippets/sync-shared/compatibility.mdx b/snippets/sync-shared/compatibility.mdx new file mode 100644 index 000000000..6642437f1 --- /dev/null +++ b/snippets/sync-shared/compatibility.mdx @@ -0,0 +1,217 @@ +{/* Shared body: rendered by sync/advanced/compatibility.mdx (Sync Streams section) and sync/rules/compatibility.mdx (Sync Rules (Legacy) section). Keep the content valid for both engines. */} + +To ensure consistency, it is important that the PowerSync Service does not interpret the same source row in different ways after updating to a new version. +At the same time, we want to fix bugs or other inaccuracies that have accumulated during the development of the Service. + +## Overview + +To make this trade‑off explicit, you choose whether to keep the existing behavior or turn on newer fixes that slightly change how data is processed. + +Use the `config` block in your Sync Config YAML to choose the behavior. There are two ways to turn fixes on: + +1. Set an `edition` to enable the full set of fixes for that edition. This is the recommended approach for new projects. +2. Toggle individual options for more fine‑grained control. + +For older projects, the previous behavior remains the default. New projects should enable all current fixes. + +### Configuration + +For new projects, it is recommended to enable all current fixes by setting `edition: `: + +```yaml +config: + edition: 3 # Recommended to set to the latest available edition (see 'Supported fixes' table below) + +streams: + # ... +``` + +Or, specify options individually: + +```yaml +config: + timestamps_iso8601: true + versioned_bucket_ids: true + fixed_json_extract: true + custom_postgres_types: true +``` + +## Sync Streams Requirement + +**New Sync Streams configurations should use `edition: 3`**, which enables the new compiler with an expanded SQL feature set (including `JOIN`, CTEs, multiple queries per stream, `BETWEEN`, `CASE`, and more): + +```yaml +config: + edition: 3 + +streams: + my_stream: + query: SELECT * FROM my_table WHERE user_id = auth.user_id() +``` + + +**Upgrading from alpha**: If you have existing Sync Streams using `edition: 2`, upgrade to `edition: 3` to enable the new compiler with an expanded SQL feature set (including `JOIN`, CTEs, multiple queries per stream, `BETWEEN`, `CASE`, and more). See [Supported SQL](/sync/supported-sql) for the full list of supported features. + + +## Storage Version + +A storage version tells the PowerSync Service how to organize prepared sync data in the [bucket storage database](/architecture/powersync-service#bucket-storage). + +Changing the version does not rewrite the current data in place. When you next deploy the Sync Config, PowerSync prepares a new copy using the selected version. Clients continue using the current copy until the new one is ready. This avoids taking the instance offline for a bucket storage migration. + +### Optional `config.storage_version` + +You can choose the bucket storage version in the `config` block: + +```yaml +config: + edition: 3 + storage_version: 4 + +streams: + todos: + query: SELECT * FROM todos WHERE owner_id = auth.user_id() +``` + +When you omit `storage_version`, the PowerSync Service uses its default, which is version 2 in v1.26.0. On PowerSync Cloud, PowerSync manages the default. For self-hosted deployments, set `config.storage_version` explicitly to select a different version. + +Set `storage_version` when you need to: + +- Use [storage version 4](/sync/advanced/storage-version-4), which is in Beta and enables incremental reprocessing and S3 object storage. +- Delay a storage upgrade. When the default moves to a newer version, pin `storage_version` to the version your data already uses. This keeps later Sync Config deployments on that format. Remove the pin when you are ready for the new format. +- Prepare for a Service downgrade. Select a version supported by the older Service, deploy the Sync Config, and wait for the new copy to finish before downgrading. + +### Available Versions + +All PowerSync Cloud instances use MongoDB bucket storage, so they are compatible with all available storage versions. Self-hosted instances with Postgres bucket storage can use versions 1 and 2 only. + +| Version | Bucket storage | Status | +| --- | --- | --- | +| `1` | MongoDB or Postgres | Legacy format, retained for existing deployments. | +| `2` | MongoDB or Postgres | Stable. The default in v1.26.0. | +| `3` | MongoDB | Experimental. The unstable predecessor of version 4, with the same format. Do not use it in production. Deploy with version 4 instead. | +| `4` | MongoDB | Stable. Enables [incremental reprocessing and S3 object storage](/sync/advanced/storage-version-4) (Beta). | + +Version numbers follow a pattern. Even numbers are stable formats: they stay backwards compatible and later Service versions continue to support them. Stable makes no guarantee that a format is bug-free. Odd numbers are experimental formats: their layout can change without notice and support can be removed in a later release, so use them only for testing, never in production. + +## Supported Fixes + +This table lists all fixes currently supported: + +| Name | Explanation | Added in Service version | Fixed in edition | +|----------------------------|------------------------------------|--------------|------------------| +| `timestamps_iso8601` | [Link](#timestamps_iso8601) | 1.15.0 | 2 | +| `versioned_bucket_ids` | [Link](#versioned_bucket_ids) | 1.15.0 | 2 | +| `fixed_json_extract` | [Link](#fixed_json_extract) | 1.15.0 | 2 | +| `custom_postgres_types` | [Link](#custom_postgres_types) | 1.15.3 | 2 | +| `unstable_sqlite_expression_engine` | [Link](#unstable_sqlite_expression_engine). | 1.22.0 | None (unstable) | + +### `timestamps_iso8601` + +PowerSync is supposed to encode timestamps according to the ISO-8601 standard. +Without this fix, the service encoded timestamps from MongoDB and Postgres source databases incorrectly. +To ensure time values from Postgres compare lexicographically, they're also padded to six digits of accuracy when encoded. +Since MongoDB only stores values with an accuracy of milliseconds, only three digits of accuracy are used. + +For instance, the value `2025-09-22T14:29:30` would be encoded as follows: + +- For Postgres: `2025-09-22 14:29:30` without the fix, `2025-09-22T14:29:30.000000` with the fix applied. +- For MongoDB: `2025-09-22 14:29:30.000` without the fix, `2025-09-22T14:29:30.000` with the fix applied. + +Note that MySQL has never been affected by this issue, and thus behaves the same regardless of the option used. + +#### Configurable Sub-Second Datetime Precision + +When the `timestamps_iso8601` option is enabled, PowerSync will sync date and time values with a higher +precision depending on the source database. +You can use the `timestamp_max_precision` option to configure the actual precision to use. +For instance, a Postgres timestamp value would sync as `2025-09-22T14:29:30.000000` by default. +If you don't want that level of precision, you can use the following options to make it sync as `2025-09-22T14:29:30.000`: + +```yaml sync-config.yaml +config: + edition: 3 + timestamp_max_precision: milliseconds +``` + +Valid options for `timestamp_max_precision` are `seconds`, `milliseconds`, `microseconds` and `nanoseconds`. When an explicit +value is given, all synced time values will use that precision. +If a source value has a higher precision, it will be truncated (it is not rounded). +If a source value has a lower precision, it will be padded (so setting the option to `microseconds` with a MongoDB source database +will sync values as `2025-09-22T14:29:30.123000`, with the last three sub-second digits always being set to zero). + +If no option is given, the default precision depends on the source database: + +| Source database | Default precision | Max precision | Notes | +|-----------------|-------------------|---------------|---------------------------------------------------------------------------------------------------------| +| MongoDB | Milliseconds | Milliseconds | | +| Postgres | Microseconds | Microseconds | | +| MySQL | Milliseconds | Microseconds | Defaults to milliseconds, but can be expanded with the option. | +| SQL Server | Nanoseconds | Nanoseconds | SQL Server supports 7 digits of accuracy, the sync service pads values to always use 9 for nanoseconds. | + +### `versioned_bucket_ids` + +Sync Rules define buckets, which rows to sync are then assigned to. When you run a full defragmentation or +redeploy Sync Rules, the same bucket identifiers are re-used when processing data again. + +Because the second iteration uses different checksums for the same bucket ids, clients may sync data +twice before realizing that something is off and starting from scratch. + +Applying this fix improves client-side progress estimation and is more efficient, since data would not get +downloaded twice. + +For how bucket identifiers are represented in bucket storage at the persistence layer (including automatic use of versioned bucket names with newer storage formats), see [Storage version](#storage-version). + +### `fixed_json_extract` + +This fixes the `json_extract` functions as well as the `->` and `->>` operators in Sync Rules to behave similar +to recent SQLite versions: We only split on `.` if the path starts with `$.`. + +For instance, `'json_extract({"foo.bar": "baz"}', 'foo.bar')` would evaluate to: + +1. `baz` with the option enabled. +2. `null` with the option disabled. + +### `custom_postgres_types` + +If you have custom Postgres types in your backend source database schema, older versions of the PowerSync Service +would not recognize these values and sync them with the textual wire representation used by Postgres. +This is especially noticeable when defining `DOMAIN` types with e.g. a `REAL` inner type: The wrapped +`DOMAIN` type should get synced as a real value as well, but it would actually get synced as a string. + +With this fix applied: + +- `DOMAIN TYPE`s are synced as their inner type. +- Array types of custom types get parsed correctly, and sync as a JSON array. +- Custom types get parsed and synced as a JSON object containing their members. +- Ranges sync as a JSON object corresponding to the following TypeScript definition: + ```TypeScript + export type Range = + | { + lower: T | null; + upper: T | null; + lower_exclusive: boolean; + upper_exclusive: boolean; + } + | 'empty'; + ``` +- Multi-ranges sync as an array of ranges. + +### `unstable_sqlite_expression_engine` + + +This option is experimental: When enabled, updates to the PowerSync Service might change how rows are processed +and this option may be removed in a future version of the Service. + + +Sync Streams support scalar SQL operators (like `+`, `-` and `||`) and [functions](/sync/supported-sql#functions). +SQL in Sync Streams should behave exactly as it would in SQLite, but the Service uses a custom implementation which differs +from SQLite for some edge cases. + +To perfectly align the behavior of the Service and SQLite, enabling this option makes the Service use an actual +SQLite database to evaluate Sync Streams. +Some known issues with the JavaScript evaluator that are fixed by this option are: + +- Exact null handling: `NOT NULL` evaluates to `TRUE` without this option, enabling it yields `NULL`. +- Without this option, `substr()` and `length()` operate on UTF-16 code units. Enabling it makes them operate on + Unicode code points. diff --git a/snippets/sync-shared/reducing-bucket-count.mdx b/snippets/sync-shared/reducing-bucket-count.mdx new file mode 100644 index 000000000..84568d37c --- /dev/null +++ b/snippets/sync-shared/reducing-bucket-count.mdx @@ -0,0 +1,240 @@ +{/* Shared body: rendered by sync/advanced/reducing-bucket-count.mdx (Sync Streams section) and sync/rules/reducing-bucket-count.mdx (Sync Rules (Legacy) section). Keep the content valid for both engines. */} + +import BucketCountExampleApp from '/snippets/bucket-count-example-app.mdx'; + +If a user syncs too many buckets, or you hit a `PSYNC_S2305` error, this page shows how to find the cause and bring the count down. For how buckets are counted in the first place, see [Bucket Count](/sync/streams/bucket-count). + +PowerSync enforces two limits per user, both with a default of 1,000. One is the number of unique buckets. The other is the number of parameter query results, counted before duplicates are removed. Exceeding either fails the sync with a `PSYNC_S2305` error. The fix is different for each, so start by finding out which one you hit from the error message. See [Limits](/sync/streams/bucket-count#limits) for the full difference. + +## Diagnosing High Bucket Count + +### Reading the Error Message First + +The `PSYNC_S2305` message tells you which limit you reached. The fix is different for each, so read it first. + +- `Too many buckets` means you reached the bucket limit. Reduce the number of unique buckets. Any strategy below helps. +- `Too many parameter query results` means you reached the parameter limit. Reduce the rows your parameter lookups return. Only some strategies help here: [Denormalizing the Scope Key](#denormalizing-the-scope-key) and [Querying the Membership Table Directly](#querying-the-membership-table-directly) cut the lookups themselves, so they lower both counts. + +```mermaid +flowchart TD + E["PSYNC_S2305 error"] --> M{"Which message?"} + M -->|"Too many buckets"| Bk["Reduce unique buckets"] + M -->|"Too many parameter query results"| Pr["Reduce parameter rows"] + Bk --> D["Denormalize the scope key,
or merge streams"] + Pr --> D +``` + +### The Contributor Breakdown + +The `PSYNC_S2305` log includes a breakdown of the streams that contribute the most. + +- For a bucket-limit error, it lists streams by bucket count, highest first. +- For a parameter-limit error, it lists the streams that returned the most rows, and then the stream that exceeded the limit. Each listed stream shows how many rows it returned. The failing stream instead shows how much budget was left when it failed. + + +For a parameter-limit error, the last stream in the breakdown is the one that ran when the limit was reached. This stream is not always the cause. PowerSync adds up parameter results across streams in order. The last stream is only the one that exceeded the limit. Check every stream in the breakdown, not just the last one. + + +### Checkpoint Logs + +Checkpoint logs record the counts for each connection. Find them in your [instance logs](/maintenance-ops/monitoring-and-alerting). For example: + +```text +New checkpoint: 800178 | write: null | buckets: 7 | param_results: 6 ["5#org_data|0[\"ef718ff3...\"]","5#org_data|1[\"1ddeddba...\"]", ...] +``` + +- `buckets` is the number of unique buckets for this connection. +- `param_results` is the total number of parameter rows for this connection. +- The array lists the bucket names. Each name already includes its parameter value. The list stops after 20 names. + +### Sync Diagnostics Client + +The [Sync Diagnostics Client](/tools/diagnostics-client) shows the buckets for one user. It does not load for a user who is over the limit, because that user's sync fails before the data loads. Use the instance logs and the error breakdown for those users. The client shows the bucket count, which may not be the limit you reached. Confirm the limit from the error message. + + + +## Reducing Bucket Count + +Start with the strategy that matches your query pattern. Most high counts come from hierarchical or many-to-many data, where denormalizing the scope key gives the biggest reduction. + +### Multiple Queries per Stream + +**Reduces:** bucket count. + +Use `queries` instead of separate streams to group related tables. All queries in a stream that filter the same way share one bucket per value. See [multiple queries per stream](/sync/streams/queries#multiple-queries-per-stream). + +**Before**: 5 separate streams, each with a direct `auth.user_id()` filter, create 5 buckets per user. + +**After**: 1 stream with 5 queries creates 1 bucket per user. + +```yaml +streams: + user_settings: # [!code --] + query: SELECT * FROM settings WHERE user_id = auth.user_id() # [!code --] + user_prefs: # [!code --] + query: SELECT * FROM preferences WHERE user_id = auth.user_id() # [!code --] + user_org_list: # [!code --] + query: SELECT * FROM org_membership WHERE user_id = auth.user_id() # [!code --] + user_region: # [!code --] + query: SELECT * FROM region_members WHERE user_id = auth.user_id() # [!code --] + user_profile: # [!code --] + query: SELECT * FROM profiles WHERE user_id = auth.user_id() # [!code --] + user_data: # [!code ++] + queries: # [!code ++] + - SELECT * FROM settings WHERE user_id = auth.user_id() # [!code ++] + - SELECT * FROM preferences WHERE user_id = auth.user_id() # [!code ++] + - SELECT * FROM org_membership WHERE user_id = auth.user_id() # [!code ++] + - SELECT * FROM region_members WHERE user_id = auth.user_id() # [!code ++] + - SELECT * FROM profiles WHERE user_id = auth.user_id() # [!code ++] +``` + +### Denormalizing the Scope Key + +**Reduces:** bucket count and parameter query results. + +This is the most effective fix for parent-child data. When chained queries through org → project → task create too many buckets, filter every table with the same top-level parameter, such as `org_id`. A bucket's key must be a column on the table you sync (see [The Partition Key Must Exist on the Row](#the-partition-key-must-exist-on-the-row) below). So this works only if the child tables have that column. If tasks only have `project_id`, add `org_id` to the tasks table. + +**Before**: chained queries create 10 + 500 = 510 buckets for 10 orgs with 50 projects each. Projects and tasks share buckets because they use the same filter. Orgs use a different filter, so they add their own buckets. + +**After**: add `org_id` to the tasks table, drop the `user_projects` CTE, and filter every table by org. This creates 10 buckets. + +```yaml +streams: + org_projects_tasks: + with: + user_orgs: SELECT org_id FROM org_membership WHERE user_id = auth.user_id() + user_projects: SELECT id FROM projects WHERE org_id IN (SELECT org_id FROM org_membership WHERE user_id = auth.user_id()) # [!code --] + queries: + - SELECT * FROM orgs WHERE id IN user_orgs + - SELECT * FROM projects WHERE id IN user_projects # [!code --] + - SELECT * FROM projects WHERE org_id IN user_orgs # [!code ++] + - SELECT * FROM tasks WHERE project_id IN user_projects # [!code --] + - SELECT * FROM tasks WHERE org_id IN user_orgs # [!code ++] +``` + +### Querying the Membership Table Directly + +**Reduces:** bucket count and parameter query results. + +When a subquery or JOIN through a membership table creates N buckets, query the membership table directly with a direct auth filter. Use no subquery and no JOIN. You often need fields from the related table, such as the org name, alongside each membership row. Denormalize those fields onto the membership table so they are available without a JOIN. + +**Before**: N org memberships create N buckets. + +**After**: 1 bucket per user, with org fields denormalized onto `org_membership`. + +```yaml +streams: + org_data: # [!code --] + query: SELECT * FROM orgs WHERE id IN (SELECT org_id FROM org_membership WHERE user_id = auth.user_id()) # [!code --] + my_org_memberships: # [!code ++] + query: SELECT * FROM org_membership WHERE user_id = auth.user_id() # [!code ++] +``` + +### Many-to-Many via a JSON Array Column + +**Reduces:** bucket count. + +A join through a link table creates one bucket per row of the table you select from. For assets linked to projects through `project_assets`, you get one bucket per asset. + +Add a denormalized `project_ids` JSON array column to `assets`, maintained with database triggers. Then use `json_each()` to traverse it. This lets PowerSync key the bucket by project ID instead of asset ID. + +**Before**: one bucket per asset. 2,000 assets create 2,000 buckets. + +**After**: key by project. 50 projects create 50 buckets. + +```yaml +streams: + assets_in_projects: + with: + user_projects: SELECT id FROM projects WHERE org_id IN (SELECT org_id FROM org_membership WHERE user_id = auth.user_id()) + query: SELECT assets.* FROM assets JOIN project_assets ON project_assets.asset_id = assets.id WHERE project_assets.project_id IN user_projects # [!code --] + query: SELECT assets.* FROM assets INNER JOIN json_each(assets.project_ids) AS p INNER JOIN user_projects ON p.value = user_projects.id # [!code ++] +``` + +The `INNER JOIN user_projects` syncs only assets that belong to at least one of the user's projects. The bucket key is the project ID, so the count matches the number of projects, not assets. + +### Subscription Parameters for On-Demand Sync + +**Reduces:** bucket count. + +Buckets are created per active subscription, not from every possible value. Use `subscription.parameter('project_id')` so the count is bounded by how many subscriptions the client has active. + +**Before**: a subquery returns all of the user's projects. 50 projects create 50 buckets. + +**After**: the client subscribes per project on demand. 3 open projects create 3 buckets. + +```yaml +streams: + project_tasks: + with: + user_projects: SELECT id FROM projects WHERE org_id IN (SELECT org_id FROM org_membership WHERE user_id = auth.user_id()) + query: SELECT * FROM tasks WHERE project_id IN user_projects # [!code --] + query: SELECT * FROM tasks WHERE project_id = subscription.parameter('project_id') AND project_id IN user_projects # [!code ++] +``` + +The client subscribes when the user opens a project and unsubscribes when they leave. This works only when the user does not need every record available offline at the same time. + +## Edge Cases and Gotchas + +### The Partition Key Must Exist on the Row + +A bucket's key must be a value that physically exists on a row of the table you sync. You cannot split a table into buckets by a column it does not have. This is why denormalizing the scope key onto child tables is the standard fix. If tasks only have `project_id`, you cannot key their buckets by `org_id` until you add `org_id` to the tasks table. + +### Subscription Parameters Choose Buckets, Not Re-Partition Them + +A subscription parameter lets the client choose which existing buckets to sync. It does not change how those buckets are defined. + +For a parameter to select a bucket, its value must match a value on the row being synced. For example, each task has a `project_id`, so you can use that column to group tasks into project buckets: + +```yaml +streams: + project_tasks: + query: SELECT * FROM tasks WHERE project_id = subscription.parameter('project_id') +``` + +Assets are different. An asset can belong to multiple projects, so the asset row does not have a single `project_id`. Passing a `project_id` as a subscription parameter therefore cannot make PowerSync group those assets by project. The asset row has no project ID to match against. + +If you want to sync assets by project, the asset row needs to contain a project reference first. For example, you could add a `project_ids` array column as described in [Reducing Bucket Count](#reducing-bucket-count). + +### Correlated Joins Behave Like Subqueries + +A correlated JOIN and an `IN (subquery)` compile to the same internal form. They create the same number of buckets. Rewriting one as the other does not reduce the count. + +### CTEs Cannot Reference Each Other + +Each CTE must be self-contained. A CTE cannot reference another CTE by name. If it does, the deploy fails. Inline the nested subquery instead. See [CTE limitations](/sync/streams/ctes#limitations). + +### Global Buckets Multiply Storage and Cost + +A stream with no filter creates one global bucket that every user syncs. Under `auto_subscribe: true`, every write to that table fans out to every user. This drives up synced data volume and cost. Scope global buckets carefully, and only mark truly shared reference data as global. + +### Bucket Storage Does Not Shrink When You Archive + +Buckets are append-only. Marking a row as archived does not remove it from bucket storage on its own. A row leaves storage only when it stops matching the data query, through a hard delete or a filter on the table's own column. Storage reclaims space during [compaction](/maintenance-ops/compacting-buckets). Filtering through a parent table does not shrink a child table's stored data. + +## Increasing the Limit + +Raise the limit only after you exhaust the reduction strategies above. + +Before you raise it, weigh the cost. Sync overhead scales roughly linearly with the number of buckets per user. Doubling the bucket count roughly doubles sync latency for a single operation. It also roughly doubles CPU and memory use on the server and the client. Many operations inside a single bucket scale much more efficiently than many buckets. The 1,000 default exists to encourage fewer, larger buckets and to protect the service from excessive counts. + +On PowerSync Cloud, you can request a higher limit on [Team and Enterprise](https://www.powersync.com/pricing) plans, up to 10,000. The limit applies per user, so your instance can still track far more buckets in total. + +For self-hosted deployments, set the limits under `api.parameters`: + +```yaml service.yaml +api: + parameters: + max_buckets_per_connection: 5000 + max_parameter_query_results: 5000 +``` + +Set both. Raising one without the other still leaves you capped by the limit you did not change. + +## Related Pages + +- [Bucket Count](/sync/streams/bucket-count) explains how buckets are counted and the two limits. +- [Writing Queries](/sync/streams/queries) covers the query syntax that determines your bucket count. +- [Common Table Expressions (CTEs)](/sync/streams/ctes) covers shared filtering logic. +- [Troubleshooting](/debugging/troubleshooting#psync_s2305-too-many-buckets-/-parameter-query-results) covers the `PSYNC_S2305` error. +- [Performance and Limits](/resources/performance-and-limits) lists the Service limits. diff --git a/snippets/sync-shared/schemas-and-connections.mdx b/snippets/sync-shared/schemas-and-connections.mdx new file mode 100644 index 000000000..f922962c8 --- /dev/null +++ b/snippets/sync-shared/schemas-and-connections.mdx @@ -0,0 +1,60 @@ +{/* Shared body: rendered by sync/advanced/schemas-and-connections.mdx (Sync Streams section) and sync/rules/schemas-and-connections.mdx (Sync Rules (Legacy) section). Keep the content valid for both engines. */} + +## Schemas (Postgres) + +When no schema is specified, the Postgres `public` schema is used for every query. A different schema can be specified as a prefix: + +```sql +-- Note: the schema must be in double quotes +SELECT * FROM "other"."assets" +``` + +## Wildcard Schemas (Postgres) + + + Wildcard schemas require [Sync Streams](/sync/streams/overview) and PowerSync Service v1.24.0 or later. They are currently only supported for Postgres connections. + + +Use `%` as a wildcard in the schema name to match tables with the same name across multiple schemas. `"%"` matches every schema, and a prefix such as `"tenant_%"` matches every schema whose name starts with `tenant_`. The wildcard can only be the last character of the schema name. Postgres system schemas (`pg_*` and `information_schema`) are never matched. + +Combine a wildcard schema with the [`schema()` function](/sync/supported-sql#functions), which returns the schema each row was replicated from, to filter rows by schema. This supports schema-per-tenant databases (a single database with one identical schema per tenant): one stream covers every tenant schema, and each client syncs only its own tenant's data, resolved from a JWT claim. + +```yaml +config: + edition: 3 + +streams: + work_orders: + query: SELECT * FROM "%".work_orders WHERE work_orders.schema() = auth.parameter('tenant_schema') +``` + +In this example, rows are grouped into a bucket per schema, and each client syncs only the bucket matching the `tenant_schema` claim in its JWT. Rows from all matched schemas sync into a single client-side table, named after the table in the query (`work_orders` here). + + + Each matched table must be part of the [PowerSync publication](/configuration/source-db/setup#3-create-powersync-publication). Tables that are not in the publication are skipped. + + +## High Availability / Replicated Databases (Postgres) + +When the source Postgres database is replicated, for example with Amazon RDS Multi-AZ deployments, specify a single connection with multiple host endpoints. Each host endpoint will be tried in sequence, with the first available primary connection being used. + +For this, each endpoint must point to the same physical database, with the same replication slots. This is the case when block-level replication is used between the databases, but not when streaming physical or logical replication is used. In those cases, replication slots are unique on each host, and all data would be re-synced in a fail-over event. + +## Multiple Separate Database Connections (Planned) + + + This feature will be available in a future release. See this [item on our roadmap](https://roadmap.powersync.com/c/84-support-for-sharding-multiple-database-connections). + + +In the future, it will be possible to configure PowerSync with multiple separate source database connections, where each connection is concurrently replicated. + +You should not add multiple connections to multiple replicas of the same database — this would cause data duplication. Only use this when the data on each connection does not overlap. + +It will be possible for each connection to be configured with a "tag", to distinguish these connections in Sync Rules. The same tag may be used for multiple connections (if the schema is the same in each). + +By default, queries will reference the "default" tag. To use a different connection or connections, assign a different tag, and specify it in the query as a schema prefix. In this case, the schema itself must also be specified. + +```sql +-- Note the usage of quotes here +SELECT * FROM "secondconnection.public"."assets" +``` diff --git a/snippets/sync-shared/sharded-databases.mdx b/snippets/sync-shared/sharded-databases.mdx new file mode 100644 index 000000000..9ea54cd92 --- /dev/null +++ b/snippets/sync-shared/sharded-databases.mdx @@ -0,0 +1,44 @@ +{/* Shared body: rendered by sync/advanced/sharded-databases.mdx (Sync Streams section) and sync/rules/sharded-databases.mdx (Sync Rules (Legacy) section). Keep the content valid for both engines. */} + +In the case of Postgres, PowerSync cannot replicate Postgres [foreign tables](https://www.postgresql.org/docs/current/ddl-foreign-data.html). + +However, PowerSync does have options available to support sharded databases in general. + + + When using MongoDB, MySQL, or SQL Server as the backend source database, PowerSync does not currently support connecting to sharded clusters. + + +The primary options are: + +1. Use a separate PowerSync Service instance per database. +2. Add a connection for each database in the same PowerSync Service instance ([planned](https://roadmap.powersync.com/c/84-support-for-sharding-multiple-database-connections); this capability will be available in a future release). + +Where feasible, using separate PowerSync Service instances would give better performance and give more control over how changes are rolled out, especially around Sync Rule reprocessing. + +Some specific scenarios: + +#### 1\. Different Tables on Different Databases + +This is common when separate "services" use separate databases, but multiple tables across those databases need to be synced to the same users. + +Use a single PowerSync Service instance, with a separate connection for each source database ([planned](https://roadmap.powersync.com/c/84-support-for-sharding-multiple-database-connections); this capability will be available in a future release). Use a unique [connection tag](/sync/advanced/schemas-and-connections) for each source database, allowing them to be distinguished in your [Sync Streams](/sync/streams/overview) or [Sync Rules](/sync/rules/overview). + +#### 2a. All Data for a Single Customer Is Contained in a Single Shard + +This is common when sharding per customer account / organization. + +In this case, use a separate PowerSync Service instance for each database. + +#### 2b. Most Customer Data Is in a Single Shard, but Some Data Is in a Shared Database + +If the amount of shared data is small, still use a separate PowerSync Service instance for each database, but also add the shared database connection to each PowerSync Service instance using a separate connection tag ([planned](https://roadmap.powersync.com/c/84-support-for-sharding-multiple-database-connections); this capability will be available in a future release). + +#### 2c. Data for a Single Customer Is Split Across Multiple Shards + +This is more complicated than the other cases listed above. Please [reach out to us](/resources/contact-us) if this is your architecture. + +#### 3\. Only Some Tables Are Sharded + +In some cases, most tables would be on a shared server, with only a few large tables being sharded. + +For this case, use a single PowerSync Service instance. Add each shard as a new connection on this instance ([planned](https://roadmap.powersync.com/c/84-support-for-sharding-multiple-database-connections); this capability will be available in a future release) — all with the same connection tag, so that the same [Sync Streams](/sync/streams/overview) or [Sync Rules](/sync/rules/overview) applies to each. diff --git a/snippets/sync-shared/storage-version-4.mdx b/snippets/sync-shared/storage-version-4.mdx new file mode 100644 index 000000000..17a4df47d --- /dev/null +++ b/snippets/sync-shared/storage-version-4.mdx @@ -0,0 +1,170 @@ +{/* Shared body: rendered by sync/advanced/storage-version-4.mdx (Sync Streams section) and sync/rules/storage-version-4.mdx (Sync Rules (Legacy) section). Keep the content valid for both engines. */} + +Storage version 4 is a new version of the format the PowerSync Service uses to store the data it syncs to clients. It is in [Beta](/resources/feature-status) as of PowerSync Service v1.26.0. + +Compared to version 2, it provides: + +- Faster sync and faster reprocessing after a deployment. +- [Incremental reprocessing](#incremental-reprocessing): a Sync Streams deployment reprocesses only the streams you added or changed. Clients no longer download all their data again after every deployment. +- [S3 object storage](#s3-object-storage): larger blocks of synced data move from the storage database to S3. This reduces load on the storage database when many clients sync at once or sync large amounts of data. + +## Availability + +Storage version 4 is compatible with all PowerSync Cloud instances, which already use MongoDB [bucket storage](/architecture/powersync-service#bucket-storage). Self-hosted instances must also use MongoDB bucket storage. Postgres bucket storage is not currently supported. + +The PowerSync Cloud and self-hosted columns below apply during the Beta only. Once storage version 4 is generally available, it will become the default for all supported instances. S3 object storage is then also enabled on all PowerSync Cloud instances. For self-hosted deployments, follow the [S3 setup instructions](#self-hosted-s3-setup). + +| | Source database | Sync Config | PowerSync Cloud (Beta) | Self-hosted (Beta) | +| --- | --- | --- | --- | --- | +| Storage version 4 | Any | Sync Streams or Sync Rules | Free plan: automatic. Other plans: [opt in](#opt-in). | [Opt in](#opt-in) | +| Incremental reprocessing | MongoDB | Sync Streams | Included with version 4 | Included with version 4 | +| S3 object storage | Any | Sync Streams or Sync Rules | Enabled per instance by PowerSync on request | [Set up S3 object storage](#self-hosted-s3-setup) | + +Incremental reprocessing for Postgres and other source databases is planned. See the [proposal](https://github.com/orgs/powersync-ja/discussions/349) for background. It is not supported for legacy [Sync Rules](/sync/rules/overview). If you still use Sync Rules, [migrate to Sync Streams](/sync/rules/migrate-to-sync-streams). + +## Opt In + +Version 4 is not the default in PowerSync Service v1.26.0. Moving a Sync Config to version 4 runs like any other deployment: + +1. PowerSync reprocesses all data selected by your Sync Config in the background. The current version keeps serving clients, so there is no downtime. +2. When the new copy is ready, PowerSync switches to it. On PowerSync Cloud, this appears as a new deploy event in the PowerSync Dashboard. +3. Clients download their data again once, as after any deployment before version 4. On self-hosted deployments with many clients, scale out the API before the switch to absorb the re-sync. + +After this first deployment, later Sync Streams deployments use incremental reprocessing automatically when your instance meets its requirements. There is no separate setting. + +### PowerSync Cloud + +Free plan instances are upgraded automatically during the Beta. No action is needed. + +On other plans, add `storage_version: 4` to the `config` block of your Sync Config and deploy it: + +```yaml +config: + edition: 3 + storage_version: 4 + +streams: + todos: + query: SELECT * FROM todos WHERE owner_id = auth.user_id() +``` + +### Self-Hosted + + + Postgres bucket storage is not supported with version 4. + + +Add `storage_version: 4` to the `config` block of each Sync Config as shown above, then deploy or redeploy it to use version 4. + +To move a Sync Config back to version 2, set `storage_version: 2` and deploy again. This is another full reprocess. + +To also enable S3 object storage, follow the [self-hosted S3 setup instructions](#self-hosted-s3-setup) to prepare a bucket and configure the Service. + +## Incremental Reprocessing + +Incremental reprocessing is active when you use a MongoDB source database, Sync Streams, and storage version 4. + + + Self-hosted instances with Postgres bucket storage are not supported. + + +Without it, every deployment reads all data selected by the Sync Config from your source database and prepares a complete new copy. Clients then download all their data again, even if only one stream changed. + +With incremental reprocessing, PowerSync compares the new Sync Config with the current one and reprocesses only the streams you added or changed. Unchanged streams keep their data on the PowerSync Service and on clients. Deployments finish faster, your source database does less work, and clients download only the data for affected streams they subscribe to. + +- Adding a stream reads only the data that stream selects. +- Removing a stream requires no new source reads. PowerSync cleans up stored definitions when no active Sync Config still uses them. +- Renaming a stream counts as removing it and adding a new one, so its data is rebuilt. +- Changing a stream's queries may reprocess affected definitions. Changes that only affect how request parameters select existing buckets do not require reprocessing. + +For example, changing `SELECT * FROM projects WHERE user_id = auth.user_id()` to `SELECT * FROM projects WHERE user_id = auth.jwt() ->> 'owner'` reuses the existing bucket data. The data is still grouped by `user_id`; only the JWT field used to select buckets changes. + +The time saved depends on how your data is split across streams. If one stream selects most of your data, changing that stream still takes about as long as a full reprocess. + +PowerSync favors correctness over reuse. When it cannot confirm that a change leaves a stream's data unchanged, it rebuilds that stream. A deployment that reprocesses more than you expect is not an error. + +Event definitions for [Custom Write Checkpoints](/handling-writes/custom-write-checkpoints) follow the same rules. Unchanged events keep their data, and new or changed events are read again. + +### When PowerSync Reprocesses Everything + +Some changes start a full reprocess, after which clients download all their data again: + +- The first deployment on storage version 4. +- Changes to the `config` block of the Sync Config, such as `edition`, compatibility fixes, or `storage_version`. +- The **Defragment** action in the PowerSync Dashboard, which exists to rebuild all data. See [Defragmenting](/maintenance-ops/compacting-buckets#defragmenting). +- Replication failures, for example when PowerSync loses its position in the MongoDB change stream and has to start over. + +### Sync Config Versions and Replication Streams + +Each deployment has a Sync Config version. With incremental reprocessing, multiple versions can share a replication stream, the replication process and stored state. A full reprocess creates a new replication stream. + +See the [Log Reference](/debugging/log-reference#message-prefixes) for how to identify these versions and streams in your logs. + +For implementation details, see the [storage design](https://github.com/powersync-ja/powersync-service/blob/main/docs/storage/storage-v3.md). The document describes the design introduced in version 3 and carried into version 4. + +### Checking What a Deployment Reprocessed + +If a deployment takes longer or reprocesses more than you expect, see [Checking What a Deployment Reprocessed](/debugging/log-reference#checking-what-a-deployment-reprocessed) in the Log Reference for what to look for in your logs. + +## S3 Object Storage + +Your instance keeps the data it syncs to clients in its bucket storage database, alongside everything else it needs to run. With S3 object storage, larger blocks of that data move to Amazon S3 or an S3-compatible object store, and the PowerSync Service syncs them to clients directly from there. Smaller blocks, and the metadata that locates each block, stay in MongoDB. + +Reading larger blocks from S3 reduces the data MongoDB must read and transfer during sync. When those reads limit performance, offloading them can speed up initial sync and let an instance serve more concurrent clients. The benefit is most noticeable when clients sync large amounts of data or many clients connect at once. The PowerSync Service still handles every client connection, so its CPU and memory capacity also limit concurrency. + +For self-hosted instances, offloading bucket data to S3 can reduce storage and data transfer costs. Compare the reduction in database costs with the object store's storage, request, and data transfer charges for your workload. + +Clients connect only to the PowerSync Service and never to the object store, so no client changes are needed. If the object store becomes unreachable, sync is interrupted until it recovers. Clients reconnect and resume automatically. + +S3 object storage requires storage version 4 and works with Sync Streams and legacy Sync Rules. It is compatible with all PowerSync Cloud instances. + + + S3 object storage holds PowerSync's internal sync data. To store files uploaded by your app, use [Attachments](/client-sdks/advanced/attachments). + + +### PowerSync Cloud + +During the Beta, PowerSync enables S3 object storage per instance. [Contact us](/resources/contact-us) if you want it on your instance before we enable it for all instances. + +### Self-Hosted S3 Setup + + + Self-hosted instances with Postgres bucket storage are not supported. + + + + + Create a bucket. Use the same region as the PowerSync Service where possible, to keep latency low and avoid cross-region data transfer charges. Use a dedicated bucket, or a unique `prefix` per PowerSync instance, so that instances never read or delete each other's files. Give the PowerSync Service permission to list the bucket and to read, write, and delete objects under the prefix. + + Leave object versioning off, or suspend it if the bucket already has it, and leave Object Lock off. PowerSync deletes files itself once they are no longer needed, so versioning keeps charging for old versions and locked objects cannot be cleaned up. Do not add an expiration lifecycle rule: an expired object may still be referenced by MongoDB, which breaks sync for that data. + + + Add `object_storage` to the `storage` section of `service.yaml`: + + ```yaml service.yaml + storage: + type: mongodb + uri: !env PS_MONGO_STORAGE_URI + object_storage: + type: s3 + bucket: powersync-bucket-data + region: us-east-1 + prefix: production + ``` + + Without `access_key_id` and `secret_access_key`, PowerSync uses the AWS credentials available to the process, such as an IAM role. For S3-compatible providers such as MinIO or Cloudflare R2, also set `endpoint`, and set `force_path_style: true` if the provider requires path-style requests. + + Restart or redeploy the PowerSync Service to load the updated `service.yaml`. If you run replication, API, and compacting in separate containers or jobs, apply the same object storage configuration to each. + + + Deploy your Sync Configs on storage version 4 as described in [Opt In](#opt-in). Sync Configs on version 2 keep all data in MongoDB, even when `object_storage` is configured. + + Once replication reaches a healthy checkpoint, confirm that objects appear under the prefix, run a test initial sync, and run `compact` once to surface permission errors early. + + + +After enabling S3 object storage, you can raise [`max_concurrent_connections`](/configuration/powersync-service/self-hosted-instances#param-max-concurrent-connections) from its default of 200 per API process. With storage version 4 and S3 object storage, each API process can handle up to 1,000 concurrent client connections. Performance degrades if a large share of those clients run an initial sync at the same time, so scale out the API before a deployment that makes all clients download their data again. More concurrent connections also increase CPU and memory usage. + +The [S3 object storage configuration reference](/configuration/powersync-service/self-hosted-instances#param-object-storage) lists all supported settings, including timeouts, request concurrency, and the size threshold below which blocks stay in MongoDB. + +Keep the scheduled [compact](/maintenance-ops/compacting-buckets) job running. It removes files that are no longer needed. The `teardown` command deletes PowerSync's files under the prefix before it drops the storage database. The `powersync_object_storage_size_bytes` [metric](/maintenance-ops/self-hosting/monitoring) reports how much object storage PowerSync uses. diff --git a/snippets/sync-shared/types.mdx b/snippets/sync-shared/types.mdx new file mode 100644 index 000000000..e3ea89acc --- /dev/null +++ b/snippets/sync-shared/types.mdx @@ -0,0 +1,164 @@ +{/* Shared body: rendered by sync/types.mdx (Sync Streams section) and sync/rules/types.mdx (Sync Rules (Legacy) section). Keep the content valid for both engines. */} + +import BinaryType from '/snippets/binary-type.mdx'; + +The supported client-side SQLite types are: + +1. `null` +2. `integer`: a 64-bit signed integer +3. `real`: a 64-bit floating point number +4. `text`: A UTF-8 text string +5. `blob`: Binary data + + +## Postgres Type Mapping + +Postgres types are mapped to SQLite types as follows: + +| Postgres Data Type | PowerSync / SQLite Column Type | Notes | +|--------------------|--------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `text`, `varchar` | `text` | | +| `int2`, `int4`, `int8` | `integer` | | +| `numeric` / `decimal` | `text` | These types have arbitrary precision in Postgres, so can only be represented accurately as text in SQLite | +| `bool` | `integer` | `1` for true, `0` for false. There is no dedicated boolean data type in SQLite. | +| `float4`, `float8` | `real` | | +| `enum` | `text` | | +| `uuid` | `text` | | +| `timestamptz` | `text` | Format: `YYYY-MM-DD hh:mm:ss.sssZ`. This is compatible with ISO8601 and SQLite's functions. Precision matches the precision used in Postgres. `-infinity` becomes `0000-01-01 00:00:00Z` and `infinity` becomes `9999-12-31 23:59:59Z`. | +| `timestamp` | `text` | Format: `YYYY-MM-DD hh:mm:ss.sss`. In most cases, timestamptz should be used instead. `-infinity` becomes `0000-01-01 00:00:00` and `infinity` becomes `9999-12-31 23:59:59`. | +| `date`, `time` | `text` | | +| `json`, `jsonb` | `text` | `json` and `jsonb` values are treated as `text` values in their serialized representation. [JSON functions and operators](/sync/supported-sql#operators) operate directly on these `text` values. | +| `interval` | `text` | | +| `macaddr` | `text` | | +| `inet` | `text` | | +| `bytea` | `blob` | Cannot sync directly to client — convert to hex or base64 first. See [Operators & Functions](/sync/supported-sql). | +| `geometry` (PostGIS) | `text` | Hex string of the binary data. Use the [ST functions](/sync/supported-sql#functions) to convert to other formats | +| Arrays | `text` | JSON array. | +| `DOMAIN` types | `text` / depends | Depending on [compatibility options](/sync/advanced/compatibility#custom_postgres_types), inner type or raw wire representation (legacy). | +| Custom types | `text` | Depending on [compatibility options](/sync/advanced/compatibility#custom_postgres_types), JSON object or raw wire representation (legacy). | +| (Multi-)ranges | `text` | Depending on [compatibility options](/sync/advanced/compatibility#custom_postgres_types), JSON object (array for multi-ranges) or raw wire representation (legacy). | + + + + +## Convex Type Mapping + + + The Convex replicator is currently released as an [experimental feature](/resources/feature-status). APIs and + behavior may change, and we can't yet guarantee continued support or long-term stability. + + +Convex values are mapped to SQLite types as follows: + +| Convex Type | TS/JS Type | PowerSync / SQLite Column Type | Notes | +| ----------- | ---------------- | ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Id` | `string` | `text` | Convex document IDs are exposed as `_id` and can be synced as `text`. For synced client tables, use client-side ID mapping with a stable UUID column as `id` instead of relying on Convex-generated `_id` values. | +| `Null` | `null` | `null` | | +| `Int64` | `base-10 string` | `text` | Cast to `INTEGER` in Sync Streams when you want to sync the value as a SQLite integer. | +| `Float64` | `number` | `real` | | +| `Boolean` | `boolean` | `integer` | `1` for true, `0` for false. There is no dedicated boolean data type in SQLite. | +| `String` | `string` | `text` | | +| `Bytes` | `base64 string` | `text` | Decode from base64 in your app if you need binary data on the client. | +| `Array` | `Array` | `text` | Converted to a JSON string. | +| `Object` | `Object` | `text` | Converted to a JSON string. | +| `Record` | `Record` | `text` | Converted to a JSON string. | + +- Convex documents are converted to a flat list of columns, one column per top-level field. +- Nested objects and arrays are converted to JSON, and [JSON functions and operators](/sync/supported-sql#operators) can be used to query them in Sync Streams or on the client-side SQLite database. +- Cast Convex `Int64` fields to `INTEGER` in Sync Streams when you want SQLite integer values on the client, for example `CAST(an_int64_column AS INTEGER) AS an_int64_column`. + + +## MongoDB Type Mapping + +MongoDB types are mapped to SQLite types as follows: + +| BSON Type | PowerSync / SQLite Column Type | Notes | +|--------------------|--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------| +| `String` | `text` | | +| `Int`, `Long` | `integer` | | +| `Double` | `real` | | +| `Decimal128` | `text` | | +| `Object` | `text` | Converted to a JSON string | +| `Array` | `text` | Converted to a JSON string | +| `ObjectId` | `text` | Lower-case hex string | +| `UUID` | `text` | Lower-case hex string | +| `Boolean` | `integer` | `1` for true, `0` for false. There is no dedicated boolean data type in SQLite. | +| `Date` | `text` | Format: `YYYY-MM-DD hh:mm:ss.sssZ` | +| `Null` | `null` | | +| `Binary` | `blob` | Cannot sync directly to client — convert to hex or base64 first. See [Operators & Functions](/sync/supported-sql). | +| Regular Expression | `text` | JSON text in the format `{"pattern":"...","options":"..."}` | +| `Timestamp` | `integer` | Converted to a 64-bit integer | +| `Undefined` | `null` | | +| `DBPointer` | `text` | JSON text in the format `{"collection":"...","oid":"...","db":"...","fields":...}` | +| `JavaScript` | `text` | JSON text in the format `{"code": "...", "scope": ...}` | +| `Symbol` | `text` | | +| `MinKey`, `MaxKey` | `null` | | + +* Data is converted to a flat list of columns, one column per top-level field in the MongoDB document. +* Special BSON types are converted to plain SQLite alternatives. For example, `ObjectId`, `Date`, `UUID` are all converted to a plain `TEXT` column. +* Nested objects and arrays are converted to JSON, and [JSON functions and operators](/sync/supported-sql#operators) can be used to query them (in the Sync Streams / Sync Rules and/or on the client-side SQLite statements). +* Binary data nested in objects or arrays is not supported. + + + + +## MySQL Type Mapping + +MySQL support is currently in a [Beta release](/resources/feature-status). + +MySQL types are mapped to SQLite types as follows: + +| MySQL Data Type | PowerSync / SQLite Column Type | Notes | +|----------------------------------------------------|--------------------------------|-----------------------------------------------------------------------------------| +| `tinyint`, `smallint`, `mediumint`, `bigint`, `integer`, `int` | `integer` | | +| `numeric`, `decimal` | `text` | | +| `bool`, `boolean` | `integer` | `1` for true, `0` for false. There is no dedicated boolean data type in SQLite. | +| `float`, `double`, `real` | `real` | | +| `enum` | `text` | | +| `set` | `text` | Converted to JSON array | +| `char`, `varchar` | `text` | | +| `tinytext`, `text`, `mediumtext`, `longtext` | `text` | | +| `timestamp` | `text` | ISO 8601 format: `YYYY-MM-DDTHH:mm:ss.sssZ` | +| `date` | `text` | Format: `YYYY-MM-DD` | +| `time`, `datetime` | `text` | ISO 8601 format: `YYYY-MM-DDTHH:mm:ss.sssZ` | +| `year` | `text` | | +| `json` | `text` | There is no dedicated JSON type in SQLite — JSON functions operate directly on text values. | +| `bit` | `blob` | * See note below regarding syncing binary types | +| `binary`, `varbinary` | `blob` | | +| `image` | `blob` | | +| `geometry`, `geometrycollection` | `blob` | | +| `point`, `multipoint` | `blob` | | +| `linestring`, `multilinestring` | `blob` | | +| `polygon`, `multipolygon` | `blob` | | + + + + +## SQL Server Type Mapping + +SQL Server support is currently in a [Beta release](/resources/feature-status). + +SQL Server types are mapped to SQLite types as follows: + +| SQL Server Data Type | PowerSync / SQLite Column Type | Notes | +|----------------------------------------------------|--------------------------------|--------------------------------------------------------| +| `tinyint`, `smallint`, `int`, `bigint` | `integer` | | +| `numeric`, `decimal` | `text` | Numeric string | +| `float`, `real` | `real` | | +| `bit` | `integer` | | +| `money`, `smallmoney` | `text` | Numeric string | +| `xml` | `text` | | +| `char`, `nchar`, `ntext` | `text` | | +| `varchar`, `nvarchar`, `text` | `text` | | +| `uniqueidentifier` | `text` | | +| `timestamp` | `text` | ISO 8601 format: `YYYY-MM-DDTHH:mm:ss.sssZ` | +| `date` | `text` | Format: `YYYY-MM-DD` | +| `time` | `text` | Format: `HH:mm:ss.sss` | +| `datetime`, `datetime2`, `smalldatetime`, `datetimeoffset` | `text` | ISO 8601 format: `YYYY-MM-DDTHH:mm:ss.sssZ` | +| `json` | `text` | Only exists for Azure SQL Database and SQL Server 2025 | +| `geometry`, `geography` | `text` | `text` of JSON object describing the spatial data type | +| `binary`, `varbinary`, `image` | `blob` | * See note below regarding binary types | +| `rowversion`, `timestamp` | `blob` | * See note below regarding binary types | +| User Defined Types: `hiearchyid` | `blob` | * See note below regarding binary types | + + diff --git a/sync/advanced/case-sensitivity.mdx b/sync/advanced/case-sensitivity.mdx index 3591e8f74..67cf813e7 100644 --- a/sync/advanced/case-sensitivity.mdx +++ b/sync/advanced/case-sensitivity.mdx @@ -3,41 +3,8 @@ title: "Case Sensitivity" description: "Handle case-sensitive table and column names in PowerSync Sync Streams/Rules, with best practices for lowercase identifiers and quoting strategies." --- -### Case in Sync Rules +{/* Wrapper page: the content is snippets/sync-shared/case-sensitivity.mdx, which also renders at sync/rules/case-sensitivity.mdx in the Sync Rules (Legacy) section. Edit the snippet, not this file. */} -PowerSync converts all table/collection and column/field names to lower-case by default in Sync Rule queries (this is how Postgres also behaves). To preserve the case, surround the names with double quotes, for example: +import CaseSensitivity from '/snippets/sync-shared/case-sensitivity.mdx'; -```sql -SELECT "ID" as id, "Description", "ListID" FROM "TODOs" WHERE "TODOs"."ListID" = bucket.list_id -``` - -When using `SELECT *`, the original case is preserved for the returned columns/fields. - -### Client-Side Case - -On the client side, the case of table and column names in the [client-side schema](/intro/setup-guide#define-your-client-side-schema) must match the case produced by Sync Rules exactly. For the above example, use the following in Dart: - -```dart - Table('TODOs', [ - Column.text('Description'), - Column.text('ListID') - ]) -``` - -SQLite itself is case-insensitive. When querying and modifying the data on the client, any case may be used. For example, the above table may be queried using `SELECT description FROM todos WHERE listid = ?`. - -Operations (`PUT`/`PATCH`/`DELETE`) are stored in the upload queue using the case as defined in the schema above for table and column names, not the case used in queries. - -As another example, in this Sync Rule query: - -```sql -SELECT ID, todo_description as Description FROM todo_items as TODOs -``` - -Each identifier in the example is unquoted and converted to lower case. That means the client-side schema would be: - -```dart -Table('todos', [ - Column.text('description') -]) -``` + diff --git a/sync/advanced/client-id.mdx b/sync/advanced/client-id.mdx index c60869117..5a1683bca 100644 --- a/sync/advanced/client-id.mdx +++ b/sync/advanced/client-id.mdx @@ -3,66 +3,8 @@ title: "Client ID" description: "Understand PowerSync's requirement for a single text-type primary key column called id." --- -For tables where the client will create new rows: +{/* Wrapper page: the content is snippets/sync-shared/client-id.mdx, which also renders at sync/rules/client-id.mdx in the Sync Rules (Legacy) section. Edit the snippet, not this file. */} -- Postgres, MySQL and SQL Server: use a UUID for `id`. Use the `uuid()` helper to generate a random UUID (v4) on the client. -- MongoDB: use an `ObjectId` for `_id`. Generate an `ObjectId()` in your app code and store it in the client's `id` column as a string; this will map to MongoDB's `_id`. +import ClientId from '/snippets/sync-shared/client-id.mdx'; -To use a different column/field from the server-side database as the record ID on the client, use a column/field alias in your [Sync Streams](/sync/streams/overview) query (or [Sync Rules](/sync/rules/overview) data query): - -```sql -SELECT client_id as id FROM my_data -``` - - - MongoDB uses `_id` as the name of the ID field in collections. You must use `SELECT _id as id` (and include any other columns you need) in [Sync Streams](/sync/streams/overview) queries and [Sync Rules](/sync/rules/overview) data queries when using MongoDB as the backend source database. When inserting new documents from the client, prefer `ObjectId` values for `_id` (stored in the client's `id` column). - - -Custom transformations can also be used for the ID column. This is useful in certain scenarios for example when dealing with join tables, because PowerSync doesn't currently support composite primary keys. For example: - -```sql --- Concatenate multiple columns into a single id column -SELECT *, item_id || '.' || category_id as id FROM item_categories - --- the source database schema for the above example is CREATE TABLE item_categories(item_id uuid, category_id uuid, PRIMARY KEY(item_id, category_id)); -``` - - - For multiple columns with the same name (e.g. if there was an `id` column in `*`), the last column wins. Prefer writing the `*` before other columns for this reason. - - If you want to upload data to a table with a custom record ID, ensure that `uploadData()` isn't blindly using a field named `id` when handling CRUD operations. See the [Sequential ID mapping tutorial](/client-sdks/advanced/sequential-id-mapping#update-client-to-use-uuids) for an example where the record ID is aliased to `uuid` on the backend. - - -PowerSync does not perform any validation that IDs are unique. Duplicate IDs on a client could occur in any of these scenarios: - -1. A non-unique column is used for the ID. -2. Multiple table partitions are used (Postgres), with the same ID present in different partitions. -3. Multiple data queries returning the same record. This is typically not an issue if the queries return the same values (same transformations used in each query). - -We recommend using a unique index on the fields in the source database to ensure uniqueness — this will prevent (1) at least. - -If the client does sync multiple records with the same ID, only one will be present in the final database. This would typically be the one modified last, but this is subject to change — do not depend on any specific record being picked. - -### Postgres: Strategies for Auto-Incrementing IDs - -With auto-incrementing / sequential IDs (e.g. `sequence` type in Postgres), the issue is that the ID can only be generated on the server, and not on the client while offline. If this _must_ be used, there are some options, depending on the use case. - -#### Option 1: Generate ID when server receives record - -If the client does not use the ID as a reference (foreign key) elsewhere, insert any unique value on the client in the `id` field, then generate a new ID when the server receives it. - -#### Option 2: Pre-create records on the server - -For some use cases, it could work to have the server pre-create a set of e.g. 100 draft records for each user. While offline, the client can populate these records without needing to generate new IDs. This is similar to providing an employee with a paper book of blank invoices — each with an invoice number pre-printed. - -This does mean that a user has a limit on how many records can be populated while offline. - -Care must be taken if a user can populate the same records from different devices while offline — ideally each device must have a unique set of pre-created records. - -#### Option 3: Use an ID mapping - -Use UUIDs on the client, then map them to sequential IDs when performing an update on the server. This allows using a sequential primary key for each record, with a UUID as a secondary ID. - -This mapping must be performed wherever the UUIDs are referenced, including for every foreign key column. - -For more information, have a look at [Sequential ID Mapping](/client-sdks/advanced/sequential-id-mapping). \ No newline at end of file + diff --git a/sync/advanced/compatibility.mdx b/sync/advanced/compatibility.mdx index 444e6e9a0..286263be8 100644 --- a/sync/advanced/compatibility.mdx +++ b/sync/advanced/compatibility.mdx @@ -3,218 +3,8 @@ title: "Compatibility" description: "Configure compatibility editions and bucket storage format version in PowerSync's Sync Config." --- -To ensure consistency, it is important that the PowerSync Service does not interpret the same source row in different ways after updating to a new version. -At the same time, we want to fix bugs or other inaccuracies that have accumulated during the development of the Service. +{/* Wrapper page: the content is snippets/sync-shared/compatibility.mdx, which also renders at sync/rules/compatibility.mdx in the Sync Rules (Legacy) section. Edit the snippet, not this file. */} -## Overview +import Compatibility from '/snippets/sync-shared/compatibility.mdx'; -To make this trade‑off explicit, you choose whether to keep the existing behavior or turn on newer fixes that slightly change how data is processed. - -Use the `config` block in your Sync Config YAML to choose the behavior. There are two ways to turn fixes on: - -1. Set an `edition` to enable the full set of fixes for that edition. This is the recommended approach for new projects. -2. Toggle individual options for more fine‑grained control. - -For older projects, the previous behavior remains the default. New projects should enable all current fixes. - -### Configuration - -For new projects, it is recommended to enable all current fixes by setting `edition: `: - -```yaml -config: - edition: 3 # Recommended to set to the latest available edition (see 'Supported fixes' table below) - -streams: - # ... -``` - -Or, specify options individually: - -```yaml -config: - timestamps_iso8601: true - versioned_bucket_ids: true - fixed_json_extract: true - custom_postgres_types: true -``` - -## Sync Streams Requirement - -**New Sync Streams configurations should use `edition: 3`**, which enables the new compiler with an expanded SQL feature set (including `JOIN`, CTEs, multiple queries per stream, `BETWEEN`, `CASE`, and more): - -```yaml -config: - edition: 3 - -streams: - my_stream: - query: SELECT * FROM my_table WHERE user_id = auth.user_id() -``` - - -**Upgrading from alpha**: If you have existing Sync Streams using `edition: 2`, upgrade to `edition: 3` to enable the new compiler with an expanded SQL feature set (including `JOIN`, CTEs, multiple queries per stream, `BETWEEN`, `CASE`, and more). See [Supported SQL](/sync/supported-sql) for the full list of supported features. - - -## Storage Version - -A storage version tells the PowerSync Service how to organize prepared sync data in the [bucket storage database](/architecture/powersync-service#bucket-storage). - -Changing the version does not rewrite the current data in place. When you next deploy the Sync Config, PowerSync prepares a new copy using the selected version. Clients continue using the current copy until the new one is ready. This avoids taking the instance offline for a bucket storage migration. - -### Optional `config.storage_version` - -You can choose the bucket storage version in the `config` block: - -```yaml -config: - edition: 3 - storage_version: 4 - -streams: - todos: - query: SELECT * FROM todos WHERE owner_id = auth.user_id() -``` - -When you omit `storage_version`, the PowerSync Service uses its default, which is version 2 in v1.26.0. On PowerSync Cloud, PowerSync manages the default. For self-hosted deployments, set `config.storage_version` explicitly to select a different version. - -Set `storage_version` when you need to: - -- Use [storage version 4](/sync/advanced/storage-version-4), which is in Beta and enables incremental reprocessing and S3 object storage. -- Delay a storage upgrade. When the default moves to a newer version, pin `storage_version` to the version your data already uses. This keeps later Sync Config deployments on that format. Remove the pin when you are ready for the new format. -- Prepare for a Service downgrade. Select a version supported by the older Service, deploy the Sync Config, and wait for the new copy to finish before downgrading. - -### Available Versions - -All PowerSync Cloud instances use MongoDB bucket storage, so they are compatible with all available storage versions. Self-hosted instances with Postgres bucket storage can use versions 1 and 2 only. - -| Version | Bucket storage | Status | -| --- | --- | --- | -| `1` | MongoDB or Postgres | Legacy format, retained for existing deployments. | -| `2` | MongoDB or Postgres | Stable. The default in v1.26.0. | -| `3` | MongoDB | Experimental. The unstable predecessor of version 4, with the same format. Do not use it in production. Deploy with version 4 instead. | -| `4` | MongoDB | Stable. Enables [incremental reprocessing and S3 object storage](/sync/advanced/storage-version-4) (Beta). | - -Version numbers follow a pattern. Even numbers are stable formats: they stay backwards compatible and later Service versions continue to support them. Stable makes no guarantee that a format is bug-free. Odd numbers are experimental formats: their layout can change without notice and support can be removed in a later release, so use them only for testing, never in production. - -## Supported Fixes - -This table lists all fixes currently supported: - -| Name | Explanation | Added in Service version | Fixed in edition | -|----------------------------|------------------------------------|--------------|------------------| -| `timestamps_iso8601` | [Link](#timestamps_iso8601) | 1.15.0 | 2 | -| `versioned_bucket_ids` | [Link](#versioned_bucket_ids) | 1.15.0 | 2 | -| `fixed_json_extract` | [Link](#fixed_json_extract) | 1.15.0 | 2 | -| `custom_postgres_types` | [Link](#custom_postgres_types) | 1.15.3 | 2 | -| `unstable_sqlite_expression_engine` | [Link](#unstable_sqlite_expression_engine). | 1.22.0 | None (unstable) | - -### `timestamps_iso8601` - -PowerSync is supposed to encode timestamps according to the ISO-8601 standard. -Without this fix, the service encoded timestamps from MongoDB and Postgres source databases incorrectly. -To ensure time values from Postgres compare lexicographically, they're also padded to six digits of accuracy when encoded. -Since MongoDB only stores values with an accuracy of milliseconds, only three digits of accuracy are used. - -For instance, the value `2025-09-22T14:29:30` would be encoded as follows: - -- For Postgres: `2025-09-22 14:29:30` without the fix, `2025-09-22T14:29:30.000000` with the fix applied. -- For MongoDB: `2025-09-22 14:29:30.000` without the fix, `2025-09-22T14:29:30.000` with the fix applied. - -Note that MySQL has never been affected by this issue, and thus behaves the same regardless of the option used. - -#### Configurable Sub-Second Datetime Precision - -When the `timestamps_iso8601` option is enabled, PowerSync will sync date and time values with a higher -precision depending on the source database. -You can use the `timestamp_max_precision` option to configure the actual precision to use. -For instance, a Postgres timestamp value would sync as `2025-09-22T14:29:30.000000` by default. -If you don't want that level of precision, you can use the following options to make it sync as `2025-09-22T14:29:30.000`: - -```yaml sync-config.yaml -config: - edition: 3 - timestamp_max_precision: milliseconds -``` - -Valid options for `timestamp_max_precision` are `seconds`, `milliseconds`, `microseconds` and `nanoseconds`. When an explicit -value is given, all synced time values will use that precision. -If a source value has a higher precision, it will be truncated (it is not rounded). -If a source value has a lower precision, it will be padded (so setting the option to `microseconds` with a MongoDB source database -will sync values as `2025-09-22T14:29:30.123000`, with the last three sub-second digits always being set to zero). - -If no option is given, the default precision depends on the source database: - -| Source database | Default precision | Max precision | Notes | -|-----------------|-------------------|---------------|---------------------------------------------------------------------------------------------------------| -| MongoDB | Milliseconds | Milliseconds | | -| Postgres | Microseconds | Microseconds | | -| MySQL | Milliseconds | Microseconds | Defaults to milliseconds, but can be expanded with the option. | -| SQL Server | Nanoseconds | Nanoseconds | SQL Server supports 7 digits of accuracy, the sync service pads values to always use 9 for nanoseconds. | - -### `versioned_bucket_ids` - -Sync Rules define buckets, which rows to sync are then assigned to. When you run a full defragmentation or -redeploy Sync Rules, the same bucket identifiers are re-used when processing data again. - -Because the second iteration uses different checksums for the same bucket ids, clients may sync data -twice before realizing that something is off and starting from scratch. - -Applying this fix improves client-side progress estimation and is more efficient, since data would not get -downloaded twice. - -For how bucket identifiers are represented in bucket storage at the persistence layer (including automatic use of versioned bucket names with newer storage formats), see [Storage version](#storage-version). - -### `fixed_json_extract` - -This fixes the `json_extract` functions as well as the `->` and `->>` operators in Sync Rules to behave similar -to recent SQLite versions: We only split on `.` if the path starts with `$.`. - -For instance, `'json_extract({"foo.bar": "baz"}', 'foo.bar')` would evaluate to: - -1. `baz` with the option enabled. -2. `null` with the option disabled. - -### `custom_postgres_types` - -If you have custom Postgres types in your backend source database schema, older versions of the PowerSync Service -would not recognize these values and sync them with the textual wire representation used by Postgres. -This is especially noticeable when defining `DOMAIN` types with e.g. a `REAL` inner type: The wrapped -`DOMAIN` type should get synced as a real value as well, but it would actually get synced as a string. - -With this fix applied: - -- `DOMAIN TYPE`s are synced as their inner type. -- Array types of custom types get parsed correctly, and sync as a JSON array. -- Custom types get parsed and synced as a JSON object containing their members. -- Ranges sync as a JSON object corresponding to the following TypeScript definition: - ```TypeScript - export type Range = - | { - lower: T | null; - upper: T | null; - lower_exclusive: boolean; - upper_exclusive: boolean; - } - | 'empty'; - ``` -- Multi-ranges sync as an array of ranges. - -### `unstable_sqlite_expression_engine` - - -This option is experimental: When enabled, updates to the PowerSync Service might change how rows are processed -and this option may be removed in a future version of the Service. - - -Sync Streams support scalar SQL operators (like `+`, `-` and `||`) and [functions](/sync/supported-sql#functions). -SQL in Sync Streams should behave exactly as it would in SQLite, but the Service uses a custom implementation which differs -from SQLite for some edge cases. - -To perfectly align the behavior of the Service and SQLite, enabling this option makes the Service use an actual -SQLite database to evaluate Sync Streams. -Some known issues with the JavaScript evaluator that are fixed by this option are: - -- Exact null handling: `NOT NULL` evaluates to `TRUE` without this option, enabling it yields `NULL`. -- Without this option, `substr()` and `length()` operate on UTF-16 code units. Enabling it makes them operate on - Unicode code points. + diff --git a/sync/advanced/multiple-client-versions.mdx b/sync/advanced/multiple-client-versions.mdx index 1c1cb9f35..4775b1b9e 100644 --- a/sync/advanced/multiple-client-versions.mdx +++ b/sync/advanced/multiple-client-versions.mdx @@ -3,6 +3,8 @@ title: "Multiple Client Versions" description: "Handle multiple client app versions that require different output schemas from Sync Streams." --- +{/* Split page: the Sync Rules (Legacy) version of this page is sync/rules/multiple-client-versions.mdx. That page is deprecated and frozen, but when you fix an error here, check whether it needs the same fix. Do not mention Sync Rules on this page. */} + When schema changes are additive, old clients ignore the new tables and columns, and no special handling is required. More drastic changes, such as renaming tables or changing a table's structure, can break older app versions that are still in use. In these cases, define separate versions of the affected [Sync Streams](/sync/streams/overview) so that each client version receives the tables and columns it expects. ## Versioning by Stream Name @@ -33,45 +35,25 @@ Once the older app versions are no longer in use, remove the old stream from you ## Versioning with Connection Parameters -Alternatively, clients can pass their version to the PowerSync Service as a [connection parameter](/sync/streams/parameters#connection-parameters), and stream queries filter on it so each client only receives data for its version. This approach is useful when your streams are auto-subscribed: auto-subscribed streams sync to every client on connect, so clients cannot select a stream version by name. In legacy [Sync Rules](/sync/rules/overview), connection parameters are called [client parameters](/sync/rules/client-parameters). +Alternatively, clients can pass their version to the PowerSync Service as a [connection parameter](/sync/streams/parameters#connection-parameters), and stream queries filter on it so each client only receives data for its version. This approach is useful when your streams are auto-subscribed: auto-subscribed streams sync to every client on connect, so clients cannot select a stream version by name. The example below implements the same `assets` use case, with both stream versions auto-subscribed and filtered by a `schema_version` connection parameter: - - - ```yaml - # Client passes connection params: {"schema_version": } - streams: - user_assets: - auto_subscribe: true - query: SELECT * FROM assets - WHERE user_id = auth.user_id() - AND connection.parameter('schema_version') = '1' - - user_assets_v2: - auto_subscribe: true - query: SELECT * FROM assets AS assets_v2 - WHERE user_id = auth.user_id() - AND connection.parameter('schema_version') = '2' - ``` - - - ```yaml - # Client passes in: "params": {"schema_version": } - user_assets: - parameters: SELECT request.user_id() AS user_id - WHERE request.parameters() ->> 'schema_version' = '1' - data: - - SELECT * FROM assets WHERE user_id = bucket.user_id +```yaml +# Client passes connection params: {"schema_version": } +streams: + user_assets: + auto_subscribe: true + query: SELECT * FROM assets + WHERE user_id = auth.user_id() + AND connection.parameter('schema_version') = '1' - user_assets_v2: - parameters: SELECT request.user_id() AS user_id - WHERE request.parameters() ->> 'schema_version' = '2' - data: - - SELECT * FROM assets AS assets_v2 WHERE user_id = bucket.user_id - ``` - - + user_assets_v2: + auto_subscribe: true + query: SELECT * FROM assets AS assets_v2 + WHERE user_id = auth.user_id() + AND connection.parameter('schema_version') = '2' +``` Handle queries based on parameters set by the client with care. The client can send any value for these parameters, so it's not a good place to do authorization. If the parameter must be authenticated, use parameters from the JWT instead. diff --git a/sync/advanced/partitioned-tables.mdx b/sync/advanced/partitioned-tables.mdx index 251d7b2bf..0c90f8bb8 100644 --- a/sync/advanced/partitioned-tables.mdx +++ b/sync/advanced/partitioned-tables.mdx @@ -3,55 +3,34 @@ title: "Partitioned Tables (Postgres)" description: "Sync data from Postgres partitioned tables using wildcard table name matching." --- -For partitioned tables in Postgres, each individual partition is replicated and processed using [Sync Streams](/sync/streams/overview) (or legacy [Sync Rules](/sync/rules/overview)). +{/* Split page: the Sync Rules (Legacy) version of this page is sync/rules/partitioned-tables.mdx. That page is deprecated and frozen, but when you fix an error here, check whether it needs the same fix. Do not mention Sync Rules on this page. */} + +For partitioned tables in Postgres, each individual partition is replicated and processed using [Sync Streams](/sync/streams/overview). To use the same queries and same output table name for each partition, use `%` for wildcard suffix matching of the table name: - - - ```yaml - streams: - user_todos: - queries: - # Wildcard matches all user partition tables (e.g. users_2024, users_2025) - - SELECT * FROM "users_%" WHERE id = auth.user_id() - # Wildcard matches all todo partition tables (e.g. todos_2024, todos_2025) - - SELECT * FROM "todos_%" AS todos WHERE user_id = auth.user_id() - ``` - - - ```yaml - by_user: - # Use wildcard in a parameter query - parameters: SELECT id AS user_id FROM "users_%" - data: - # Use wildcard in a data query - - SELECT * FROM "todos_%" AS todos WHERE user_id = bucket.user_id - ``` - - +```yaml +streams: + user_todos: + queries: + # Wildcard matches all user partition tables (e.g. users_2024, users_2025) + - SELECT * FROM "users_%" WHERE id = auth.user_id() + # Wildcard matches all todo partition tables (e.g. todos_2024, todos_2025) + - SELECT * FROM "todos_%" AS todos WHERE user_id = auth.user_id() +``` The wildcard character can only be used as the last character in the table name. To match tables across multiple schemas instead, see [Wildcard Schemas](/sync/advanced/schemas-and-connections#wildcard-schemas-postgres). -When using wildcard table names, you can filter on the original table suffix. In Sync Streams, use the `table_suffix()` function, prefixed with the table name or alias from the `FROM` clause (requires PowerSync Service v1.24.0 or later). In legacy Sync Rules, the suffix is available as the special `_table_suffix` column instead: - - - - ```yaml - config: - edition: 3 - - streams: - active_todos: - query: SELECT * FROM "todos_%" AS todos WHERE todos.table_suffix() != 'archived' - ``` - - - ```sql - SELECT * FROM "todos_%" AS todos WHERE _table_suffix != 'archived' - ``` - - +When using wildcard table names, you can filter on the original table suffix with the `table_suffix()` function, prefixed with the table name or alias from the `FROM` clause. This requires PowerSync Service v1.24.0 or later: + +```yaml +config: + edition: 3 + +streams: + active_todos: + query: SELECT * FROM "todos_%" AS todos WHERE todos.table_suffix() != 'archived' +``` When no table alias is provided, the original table name is preserved. diff --git a/sync/advanced/reducing-bucket-count.mdx b/sync/advanced/reducing-bucket-count.mdx index 16c42f2f2..37162620d 100644 --- a/sync/advanced/reducing-bucket-count.mdx +++ b/sync/advanced/reducing-bucket-count.mdx @@ -4,241 +4,8 @@ description: "Diagnose a high bucket count, reduce the number of buckets a user sidebarTitle: "Reducing Buckets" --- -import BucketCountExampleApp from '/snippets/bucket-count-example-app.mdx'; +{/* Wrapper page: the content is snippets/sync-shared/reducing-bucket-count.mdx, which also renders at sync/rules/reducing-bucket-count.mdx in the Sync Rules (Legacy) section. Edit the snippet, not this file. */} -If a user syncs too many buckets, or you hit a `PSYNC_S2305` error, this page shows how to find the cause and bring the count down. For how buckets are counted in the first place, see [Bucket Count](/sync/streams/bucket-count). +import ReducingBucketCount from '/snippets/sync-shared/reducing-bucket-count.mdx'; -PowerSync enforces two limits per user, both with a default of 1,000. One is the number of unique buckets. The other is the number of parameter query results, counted before duplicates are removed. Exceeding either fails the sync with a `PSYNC_S2305` error. The fix is different for each, so start by finding out which one you hit from the error message. See [Limits](/sync/streams/bucket-count#limits) for the full difference. - -## Diagnosing High Bucket Count - -### Reading the Error Message First - -The `PSYNC_S2305` message tells you which limit you reached. The fix is different for each, so read it first. - -- `Too many buckets` means you reached the bucket limit. Reduce the number of unique buckets. Any strategy below helps. -- `Too many parameter query results` means you reached the parameter limit. Reduce the rows your parameter lookups return. Only some strategies help here: [Denormalizing the Scope Key](#denormalizing-the-scope-key) and [Querying the Membership Table Directly](#querying-the-membership-table-directly) cut the lookups themselves, so they lower both counts. - -```mermaid -flowchart TD - E["PSYNC_S2305 error"] --> M{"Which message?"} - M -->|"Too many buckets"| Bk["Reduce unique buckets"] - M -->|"Too many parameter query results"| Pr["Reduce parameter rows"] - Bk --> D["Denormalize the scope key,
or merge streams"] - Pr --> D -``` - -### The Contributor Breakdown - -The `PSYNC_S2305` log includes a breakdown of the streams that contribute the most. - -- For a bucket-limit error, it lists streams by bucket count, highest first. -- For a parameter-limit error, it lists the streams that returned the most rows, and then the stream that exceeded the limit. Each listed stream shows how many rows it returned. The failing stream instead shows how much budget was left when it failed. - - -For a parameter-limit error, the last stream in the breakdown is the one that ran when the limit was reached. This stream is not always the cause. PowerSync adds up parameter results across streams in order. The last stream is only the one that exceeded the limit. Check every stream in the breakdown, not just the last one. - - -### Checkpoint Logs - -Checkpoint logs record the counts for each connection. Find them in your [instance logs](/maintenance-ops/monitoring-and-alerting). For example: - -```text -New checkpoint: 800178 | write: null | buckets: 7 | param_results: 6 ["5#org_data|0[\"ef718ff3...\"]","5#org_data|1[\"1ddeddba...\"]", ...] -``` - -- `buckets` is the number of unique buckets for this connection. -- `param_results` is the total number of parameter rows for this connection. -- The array lists the bucket names. Each name already includes its parameter value. The list stops after 20 names. - -### Sync Diagnostics Client - -The [Sync Diagnostics Client](/tools/diagnostics-client) shows the buckets for one user. It does not load for a user who is over the limit, because that user's sync fails before the data loads. Use the instance logs and the error breakdown for those users. The client shows the bucket count, which may not be the limit you reached. Confirm the limit from the error message. - - - -## Reducing Bucket Count - -Start with the strategy that matches your query pattern. Most high counts come from hierarchical or many-to-many data, where denormalizing the scope key gives the biggest reduction. - -### Multiple Queries per Stream - -**Reduces:** bucket count. - -Use `queries` instead of separate streams to group related tables. All queries in a stream that filter the same way share one bucket per value. See [multiple queries per stream](/sync/streams/queries#multiple-queries-per-stream). - -**Before**: 5 separate streams, each with a direct `auth.user_id()` filter, create 5 buckets per user. - -**After**: 1 stream with 5 queries creates 1 bucket per user. - -```yaml -streams: - user_settings: # [!code --] - query: SELECT * FROM settings WHERE user_id = auth.user_id() # [!code --] - user_prefs: # [!code --] - query: SELECT * FROM preferences WHERE user_id = auth.user_id() # [!code --] - user_org_list: # [!code --] - query: SELECT * FROM org_membership WHERE user_id = auth.user_id() # [!code --] - user_region: # [!code --] - query: SELECT * FROM region_members WHERE user_id = auth.user_id() # [!code --] - user_profile: # [!code --] - query: SELECT * FROM profiles WHERE user_id = auth.user_id() # [!code --] - user_data: # [!code ++] - queries: # [!code ++] - - SELECT * FROM settings WHERE user_id = auth.user_id() # [!code ++] - - SELECT * FROM preferences WHERE user_id = auth.user_id() # [!code ++] - - SELECT * FROM org_membership WHERE user_id = auth.user_id() # [!code ++] - - SELECT * FROM region_members WHERE user_id = auth.user_id() # [!code ++] - - SELECT * FROM profiles WHERE user_id = auth.user_id() # [!code ++] -``` - -### Denormalizing the Scope Key - -**Reduces:** bucket count and parameter query results. - -This is the most effective fix for parent-child data. When chained queries through org → project → task create too many buckets, filter every table with the same top-level parameter, such as `org_id`. A bucket's key must be a column on the table you sync (see [The Partition Key Must Exist on the Row](#the-partition-key-must-exist-on-the-row) below). So this works only if the child tables have that column. If tasks only have `project_id`, add `org_id` to the tasks table. - -**Before**: chained queries create 10 + 500 = 510 buckets for 10 orgs with 50 projects each. Projects and tasks share buckets because they use the same filter. Orgs use a different filter, so they add their own buckets. - -**After**: add `org_id` to the tasks table, drop the `user_projects` CTE, and filter every table by org. This creates 10 buckets. - -```yaml -streams: - org_projects_tasks: - with: - user_orgs: SELECT org_id FROM org_membership WHERE user_id = auth.user_id() - user_projects: SELECT id FROM projects WHERE org_id IN (SELECT org_id FROM org_membership WHERE user_id = auth.user_id()) # [!code --] - queries: - - SELECT * FROM orgs WHERE id IN user_orgs - - SELECT * FROM projects WHERE id IN user_projects # [!code --] - - SELECT * FROM projects WHERE org_id IN user_orgs # [!code ++] - - SELECT * FROM tasks WHERE project_id IN user_projects # [!code --] - - SELECT * FROM tasks WHERE org_id IN user_orgs # [!code ++] -``` - -### Querying the Membership Table Directly - -**Reduces:** bucket count and parameter query results. - -When a subquery or JOIN through a membership table creates N buckets, query the membership table directly with a direct auth filter. Use no subquery and no JOIN. You often need fields from the related table, such as the org name, alongside each membership row. Denormalize those fields onto the membership table so they are available without a JOIN. - -**Before**: N org memberships create N buckets. - -**After**: 1 bucket per user, with org fields denormalized onto `org_membership`. - -```yaml -streams: - org_data: # [!code --] - query: SELECT * FROM orgs WHERE id IN (SELECT org_id FROM org_membership WHERE user_id = auth.user_id()) # [!code --] - my_org_memberships: # [!code ++] - query: SELECT * FROM org_membership WHERE user_id = auth.user_id() # [!code ++] -``` - -### Many-to-Many via a JSON Array Column - -**Reduces:** bucket count. - -A join through a link table creates one bucket per row of the table you select from. For assets linked to projects through `project_assets`, you get one bucket per asset. - -Add a denormalized `project_ids` JSON array column to `assets`, maintained with database triggers. Then use `json_each()` to traverse it. This lets PowerSync key the bucket by project ID instead of asset ID. - -**Before**: one bucket per asset. 2,000 assets create 2,000 buckets. - -**After**: key by project. 50 projects create 50 buckets. - -```yaml -streams: - assets_in_projects: - with: - user_projects: SELECT id FROM projects WHERE org_id IN (SELECT org_id FROM org_membership WHERE user_id = auth.user_id()) - query: SELECT assets.* FROM assets JOIN project_assets ON project_assets.asset_id = assets.id WHERE project_assets.project_id IN user_projects # [!code --] - query: SELECT assets.* FROM assets INNER JOIN json_each(assets.project_ids) AS p INNER JOIN user_projects ON p.value = user_projects.id # [!code ++] -``` - -The `INNER JOIN user_projects` syncs only assets that belong to at least one of the user's projects. The bucket key is the project ID, so the count matches the number of projects, not assets. - -### Subscription Parameters for On-Demand Sync - -**Reduces:** bucket count. - -Buckets are created per active subscription, not from every possible value. Use `subscription.parameter('project_id')` so the count is bounded by how many subscriptions the client has active. - -**Before**: a subquery returns all of the user's projects. 50 projects create 50 buckets. - -**After**: the client subscribes per project on demand. 3 open projects create 3 buckets. - -```yaml -streams: - project_tasks: - with: - user_projects: SELECT id FROM projects WHERE org_id IN (SELECT org_id FROM org_membership WHERE user_id = auth.user_id()) - query: SELECT * FROM tasks WHERE project_id IN user_projects # [!code --] - query: SELECT * FROM tasks WHERE project_id = subscription.parameter('project_id') AND project_id IN user_projects # [!code ++] -``` - -The client subscribes when the user opens a project and unsubscribes when they leave. This works only when the user does not need every record available offline at the same time. - -## Edge Cases and Gotchas - -### The Partition Key Must Exist on the Row - -A bucket's key must be a value that physically exists on a row of the table you sync. You cannot split a table into buckets by a column it does not have. This is why denormalizing the scope key onto child tables is the standard fix. If tasks only have `project_id`, you cannot key their buckets by `org_id` until you add `org_id` to the tasks table. - -### Subscription Parameters Choose Buckets, Not Re-Partition Them - -A subscription parameter lets the client choose which existing buckets to sync. It does not change how those buckets are defined. - -For a parameter to select a bucket, its value must match a value on the row being synced. For example, each task has a `project_id`, so you can use that column to group tasks into project buckets: - -```yaml -streams: - project_tasks: - query: SELECT * FROM tasks WHERE project_id = subscription.parameter('project_id') -``` - -Assets are different. An asset can belong to multiple projects, so the asset row does not have a single `project_id`. Passing a `project_id` as a subscription parameter therefore cannot make PowerSync group those assets by project. The asset row has no project ID to match against. - -If you want to sync assets by project, the asset row needs to contain a project reference first. For example, you could add a `project_ids` array column as described in [Reducing Bucket Count](#reducing-bucket-count). - -### Correlated Joins Behave Like Subqueries - -A correlated JOIN and an `IN (subquery)` compile to the same internal form. They create the same number of buckets. Rewriting one as the other does not reduce the count. - -### CTEs Cannot Reference Each Other - -Each CTE must be self-contained. A CTE cannot reference another CTE by name. If it does, the deploy fails. Inline the nested subquery instead. See [CTE limitations](/sync/streams/ctes#limitations). - -### Global Buckets Multiply Storage and Cost - -A stream with no filter creates one global bucket that every user syncs. Under `auto_subscribe: true`, every write to that table fans out to every user. This drives up synced data volume and cost. Scope global buckets carefully, and only mark truly shared reference data as global. - -### Bucket Storage Does Not Shrink When You Archive - -Buckets are append-only. Marking a row as archived does not remove it from bucket storage on its own. A row leaves storage only when it stops matching the data query, through a hard delete or a filter on the table's own column. Storage reclaims space during [compaction](/maintenance-ops/compacting-buckets). Filtering through a parent table does not shrink a child table's stored data. - -## Increasing the Limit - -Raise the limit only after you exhaust the reduction strategies above. - -Before you raise it, weigh the cost. Sync overhead scales roughly linearly with the number of buckets per user. Doubling the bucket count roughly doubles sync latency for a single operation. It also roughly doubles CPU and memory use on the server and the client. Many operations inside a single bucket scale much more efficiently than many buckets. The 1,000 default exists to encourage fewer, larger buckets and to protect the service from excessive counts. - -On PowerSync Cloud, you can request a higher limit on [Team and Enterprise](https://www.powersync.com/pricing) plans, up to 10,000. The limit applies per user, so your instance can still track far more buckets in total. - -For self-hosted deployments, set the limits under `api.parameters`: - -```yaml service.yaml -api: - parameters: - max_buckets_per_connection: 5000 - max_parameter_query_results: 5000 -``` - -Set both. Raising one without the other still leaves you capped by the limit you did not change. - -## Related Pages - -- [Bucket Count](/sync/streams/bucket-count) explains how buckets are counted and the two limits. -- [Writing Queries](/sync/streams/queries) covers the query syntax that determines your bucket count. -- [Common Table Expressions (CTEs)](/sync/streams/ctes) covers shared filtering logic. -- [Troubleshooting](/debugging/troubleshooting#psync_s2305-too-many-buckets-/-parameter-query-results) covers the `PSYNC_S2305` error. -- [Performance and Limits](/resources/performance-and-limits) lists the Service limits. + diff --git a/sync/advanced/schemas-and-connections.mdx b/sync/advanced/schemas-and-connections.mdx index 7c6633b3a..4a666c980 100644 --- a/sync/advanced/schemas-and-connections.mdx +++ b/sync/advanced/schemas-and-connections.mdx @@ -3,61 +3,8 @@ title: "Schemas and Connections" description: "Configure Postgres schema usage in Sync Streams/Rules queries, including wildcard schemas for schema-per-tenant setups, and connect to high-availability replicas." --- -## Schemas (Postgres) +{/* Wrapper page: the content is snippets/sync-shared/schemas-and-connections.mdx, which also renders at sync/rules/schemas-and-connections.mdx in the Sync Rules (Legacy) section. Edit the snippet, not this file. */} -When no schema is specified, the Postgres `public` schema is used for every query. A different schema can be specified as a prefix: +import SchemasAndConnections from '/snippets/sync-shared/schemas-and-connections.mdx'; -```sql --- Note: the schema must be in double quotes -SELECT * FROM "other"."assets" -``` - -## Wildcard Schemas (Postgres) - - - Wildcard schemas require [Sync Streams](/sync/streams/overview) and PowerSync Service v1.24.0 or later. They are currently only supported for Postgres connections. - - -Use `%` as a wildcard in the schema name to match tables with the same name across multiple schemas. `"%"` matches every schema, and a prefix such as `"tenant_%"` matches every schema whose name starts with `tenant_`. The wildcard can only be the last character of the schema name. Postgres system schemas (`pg_*` and `information_schema`) are never matched. - -Combine a wildcard schema with the [`schema()` function](/sync/supported-sql#functions), which returns the schema each row was replicated from, to filter rows by schema. This supports schema-per-tenant databases (a single database with one identical schema per tenant): one stream covers every tenant schema, and each client syncs only its own tenant's data, resolved from a JWT claim. - -```yaml -config: - edition: 3 - -streams: - work_orders: - query: SELECT * FROM "%".work_orders WHERE work_orders.schema() = auth.parameter('tenant_schema') -``` - -In this example, rows are grouped into a bucket per schema, and each client syncs only the bucket matching the `tenant_schema` claim in its JWT. Rows from all matched schemas sync into a single client-side table, named after the table in the query (`work_orders` here). - - - Each matched table must be part of the [PowerSync publication](/configuration/source-db/setup#3-create-powersync-publication). Tables that are not in the publication are skipped. - - -## High Availability / Replicated Databases (Postgres) - -When the source Postgres database is replicated, for example with Amazon RDS Multi-AZ deployments, specify a single connection with multiple host endpoints. Each host endpoint will be tried in sequence, with the first available primary connection being used. - -For this, each endpoint must point to the same physical database, with the same replication slots. This is the case when block-level replication is used between the databases, but not when streaming physical or logical replication is used. In those cases, replication slots are unique on each host, and all data would be re-synced in a fail-over event. - -## Multiple Separate Database Connections (Planned) - - - This feature will be available in a future release. See this [item on our roadmap](https://roadmap.powersync.com/c/84-support-for-sharding-multiple-database-connections). - - -In the future, it will be possible to configure PowerSync with multiple separate source database connections, where each connection is concurrently replicated. - -You should not add multiple connections to multiple replicas of the same database — this would cause data duplication. Only use this when the data on each connection does not overlap. - -It will be possible for each connection to be configured with a "tag", to distinguish these connections in Sync Rules. The same tag may be used for multiple connections (if the schema is the same in each). - -By default, queries will reference the "default" tag. To use a different connection or connections, assign a different tag, and specify it in the query as a schema prefix. In this case, the schema itself must also be specified. - -```sql --- Note the usage of quotes here -SELECT * FROM "secondconnection.public"."assets" -``` + diff --git a/sync/advanced/sharded-databases.mdx b/sync/advanced/sharded-databases.mdx index afde1abe3..6c5e37c45 100644 --- a/sync/advanced/sharded-databases.mdx +++ b/sync/advanced/sharded-databases.mdx @@ -3,45 +3,8 @@ title: "Sharded Databases" description: "Sync data from sharded Postgres databases with per-shard PowerSync connection configuration." --- -In the case of Postgres, PowerSync cannot replicate Postgres [foreign tables](https://www.postgresql.org/docs/current/ddl-foreign-data.html). +{/* Wrapper page: the content is snippets/sync-shared/sharded-databases.mdx, which also renders at sync/rules/sharded-databases.mdx in the Sync Rules (Legacy) section. Edit the snippet, not this file. */} -However, PowerSync does have options available to support sharded databases in general. +import ShardedDatabases from '/snippets/sync-shared/sharded-databases.mdx'; - - When using MongoDB, MySQL, or SQL Server as the backend source database, PowerSync does not currently support connecting to sharded clusters. - - -The primary options are: - -1. Use a separate PowerSync Service instance per database. -2. Add a connection for each database in the same PowerSync Service instance ([planned](https://roadmap.powersync.com/c/84-support-for-sharding-multiple-database-connections); this capability will be available in a future release). - -Where feasible, using separate PowerSync Service instances would give better performance and give more control over how changes are rolled out, especially around Sync Rule reprocessing. - -Some specific scenarios: - -#### 1\. Different Tables on Different Databases - -This is common when separate "services" use separate databases, but multiple tables across those databases need to be synced to the same users. - -Use a single PowerSync Service instance, with a separate connection for each source database ([planned](https://roadmap.powersync.com/c/84-support-for-sharding-multiple-database-connections); this capability will be available in a future release). Use a unique [connection tag](/sync/advanced/schemas-and-connections) for each source database, allowing them to be distinguished in your [Sync Streams](/sync/streams/overview) or [Sync Rules](/sync/rules/overview). - -#### 2a. All Data for a Single Customer Is Contained in a Single Shard - -This is common when sharding per customer account / organization. - -In this case, use a separate PowerSync Service instance for each database. - -#### 2b. Most Customer Data Is in a Single Shard, but Some Data Is in a Shared Database - -If the amount of shared data is small, still use a separate PowerSync Service instance for each database, but also add the shared database connection to each PowerSync Service instance using a separate connection tag ([planned](https://roadmap.powersync.com/c/84-support-for-sharding-multiple-database-connections); this capability will be available in a future release). - -#### 2c. Data for a Single Customer Is Split Across Multiple Shards - -This is more complicated than the other cases listed above. Please [reach out to us](/resources/contact-us) if this is your architecture. - -#### 3\. Only Some Tables Are Sharded - -In some cases, most tables would be on a shared server, with only a few large tables being sharded. - -For this case, use a single PowerSync Service instance. Add each shard as a new connection on this instance ([planned](https://roadmap.powersync.com/c/84-support-for-sharding-multiple-database-connections); this capability will be available in a future release) — all with the same connection tag, so that the same [Sync Streams](/sync/streams/overview) or [Sync Rules](/sync/rules/overview) applies to each. + diff --git a/sync/advanced/storage-version-4.mdx b/sync/advanced/storage-version-4.mdx index 2b6b4b9ed..e2b71e31a 100644 --- a/sync/advanced/storage-version-4.mdx +++ b/sync/advanced/storage-version-4.mdx @@ -3,171 +3,8 @@ title: "Storage Version 4" description: "Opt in to storage version 4 for faster sync, incremental reprocessing of Sync Streams changes, and S3 object storage." --- -Storage version 4 is a new version of the format the PowerSync Service uses to store the data it syncs to clients. It is in [Beta](/resources/feature-status) as of PowerSync Service v1.26.0. +{/* Wrapper page: the content is snippets/sync-shared/storage-version-4.mdx, which also renders at sync/rules/storage-version-4.mdx in the Sync Rules (Legacy) section. Edit the snippet, not this file. */} -Compared to version 2, it provides: +import StorageVersion4 from '/snippets/sync-shared/storage-version-4.mdx'; -- Faster sync and faster reprocessing after a deployment. -- [Incremental reprocessing](#incremental-reprocessing): a Sync Streams deployment reprocesses only the streams you added or changed. Clients no longer download all their data again after every deployment. -- [S3 object storage](#s3-object-storage): larger blocks of synced data move from the storage database to S3. This reduces load on the storage database when many clients sync at once or sync large amounts of data. - -## Availability - -Storage version 4 is compatible with all PowerSync Cloud instances, which already use MongoDB [bucket storage](/architecture/powersync-service#bucket-storage). Self-hosted instances must also use MongoDB bucket storage. Postgres bucket storage is not currently supported. - -The PowerSync Cloud and self-hosted columns below apply during the Beta only. Once storage version 4 is generally available, it will become the default for all supported instances. S3 object storage is then also enabled on all PowerSync Cloud instances. For self-hosted deployments, follow the [S3 setup instructions](#self-hosted-s3-setup). - -| | Source database | Sync Config | PowerSync Cloud (Beta) | Self-hosted (Beta) | -| --- | --- | --- | --- | --- | -| Storage version 4 | Any | Sync Streams or Sync Rules | Free plan: automatic. Other plans: [opt in](#opt-in). | [Opt in](#opt-in) | -| Incremental reprocessing | MongoDB | Sync Streams | Included with version 4 | Included with version 4 | -| S3 object storage | Any | Sync Streams or Sync Rules | Enabled per instance by PowerSync on request | [Set up S3 object storage](#self-hosted-s3-setup) | - -Incremental reprocessing for Postgres and other source databases is planned. See the [proposal](https://github.com/orgs/powersync-ja/discussions/349) for background. It is not supported for legacy [Sync Rules](/sync/rules/overview). If you still use Sync Rules, [migrate to Sync Streams](/sync/rules/migrate-to-sync-streams). - -## Opt In - -Version 4 is not the default in PowerSync Service v1.26.0. Moving a Sync Config to version 4 runs like any other deployment: - -1. PowerSync reprocesses all data selected by your Sync Config in the background. The current version keeps serving clients, so there is no downtime. -2. When the new copy is ready, PowerSync switches to it. On PowerSync Cloud, this appears as a new deploy event in the PowerSync Dashboard. -3. Clients download their data again once, as after any deployment before version 4. On self-hosted deployments with many clients, scale out the API before the switch to absorb the re-sync. - -After this first deployment, later Sync Streams deployments use incremental reprocessing automatically when your instance meets its requirements. There is no separate setting. - -### PowerSync Cloud - -Free plan instances are upgraded automatically during the Beta. No action is needed. - -On other plans, add `storage_version: 4` to the `config` block of your Sync Config and deploy it: - -```yaml -config: - edition: 3 - storage_version: 4 - -streams: - todos: - query: SELECT * FROM todos WHERE owner_id = auth.user_id() -``` - -### Self-Hosted - - - Postgres bucket storage is not supported with version 4. - - -Add `storage_version: 4` to the `config` block of each Sync Config as shown above, then deploy or redeploy it to use version 4. - -To move a Sync Config back to version 2, set `storage_version: 2` and deploy again. This is another full reprocess. - -To also enable S3 object storage, follow the [self-hosted S3 setup instructions](#self-hosted-s3-setup) to prepare a bucket and configure the Service. - -## Incremental Reprocessing - -Incremental reprocessing is active when you use a MongoDB source database, Sync Streams, and storage version 4. - - - Self-hosted instances with Postgres bucket storage are not supported. - - -Without it, every deployment reads all data selected by the Sync Config from your source database and prepares a complete new copy. Clients then download all their data again, even if only one stream changed. - -With incremental reprocessing, PowerSync compares the new Sync Config with the current one and reprocesses only the streams you added or changed. Unchanged streams keep their data on the PowerSync Service and on clients. Deployments finish faster, your source database does less work, and clients download only the data for affected streams they subscribe to. - -- Adding a stream reads only the data that stream selects. -- Removing a stream requires no new source reads. PowerSync cleans up stored definitions when no active Sync Config still uses them. -- Renaming a stream counts as removing it and adding a new one, so its data is rebuilt. -- Changing a stream's queries may reprocess affected definitions. Changes that only affect how request parameters select existing buckets do not require reprocessing. - -For example, changing `SELECT * FROM projects WHERE user_id = auth.user_id()` to `SELECT * FROM projects WHERE user_id = auth.jwt() ->> 'owner'` reuses the existing bucket data. The data is still grouped by `user_id`; only the JWT field used to select buckets changes. - -The time saved depends on how your data is split across streams. If one stream selects most of your data, changing that stream still takes about as long as a full reprocess. - -PowerSync favors correctness over reuse. When it cannot confirm that a change leaves a stream's data unchanged, it rebuilds that stream. A deployment that reprocesses more than you expect is not an error. - -Event definitions for [Custom Write Checkpoints](/handling-writes/custom-write-checkpoints) follow the same rules. Unchanged events keep their data, and new or changed events are read again. - -### When PowerSync Reprocesses Everything - -Some changes start a full reprocess, after which clients download all their data again: - -- The first deployment on storage version 4. -- Changes to the `config` block of the Sync Config, such as `edition`, compatibility fixes, or `storage_version`. -- The **Defragment** action in the PowerSync Dashboard, which exists to rebuild all data. See [Defragmenting](/maintenance-ops/compacting-buckets#defragmenting). -- Replication failures, for example when PowerSync loses its position in the MongoDB change stream and has to start over. - -### Sync Config Versions and Replication Streams - -Each deployment has a Sync Config version. With incremental reprocessing, multiple versions can share a replication stream, the replication process and stored state. A full reprocess creates a new replication stream. - -See the [Log Reference](/debugging/log-reference#message-prefixes) for how to identify these versions and streams in your logs. - -For implementation details, see the [storage design](https://github.com/powersync-ja/powersync-service/blob/main/docs/storage/storage-v3.md). The document describes the design introduced in version 3 and carried into version 4. - -### Checking What a Deployment Reprocessed - -If a deployment takes longer or reprocesses more than you expect, see [Checking What a Deployment Reprocessed](/debugging/log-reference#checking-what-a-deployment-reprocessed) in the Log Reference for what to look for in your logs. - -## S3 Object Storage - -Your instance keeps the data it syncs to clients in its bucket storage database, alongside everything else it needs to run. With S3 object storage, larger blocks of that data move to Amazon S3 or an S3-compatible object store, and the PowerSync Service syncs them to clients directly from there. Smaller blocks, and the metadata that locates each block, stay in MongoDB. - -Reading larger blocks from S3 reduces the data MongoDB must read and transfer during sync. When those reads limit performance, offloading them can speed up initial sync and let an instance serve more concurrent clients. The benefit is most noticeable when clients sync large amounts of data or many clients connect at once. The PowerSync Service still handles every client connection, so its CPU and memory capacity also limit concurrency. - -For self-hosted instances, offloading bucket data to S3 can reduce storage and data transfer costs. Compare the reduction in database costs with the object store's storage, request, and data transfer charges for your workload. - -Clients connect only to the PowerSync Service and never to the object store, so no client changes are needed. If the object store becomes unreachable, sync is interrupted until it recovers. Clients reconnect and resume automatically. - -S3 object storage requires storage version 4 and works with Sync Streams and legacy Sync Rules. It is compatible with all PowerSync Cloud instances. - - - S3 object storage holds PowerSync's internal sync data. To store files uploaded by your app, use [Attachments](/client-sdks/advanced/attachments). - - -### PowerSync Cloud - -During the Beta, PowerSync enables S3 object storage per instance. [Contact us](/resources/contact-us) if you want it on your instance before we enable it for all instances. - -### Self-Hosted S3 Setup - - - Self-hosted instances with Postgres bucket storage are not supported. - - - - - Create a bucket. Use the same region as the PowerSync Service where possible, to keep latency low and avoid cross-region data transfer charges. Use a dedicated bucket, or a unique `prefix` per PowerSync instance, so that instances never read or delete each other's files. Give the PowerSync Service permission to list the bucket and to read, write, and delete objects under the prefix. - - Leave object versioning off, or suspend it if the bucket already has it, and leave Object Lock off. PowerSync deletes files itself once they are no longer needed, so versioning keeps charging for old versions and locked objects cannot be cleaned up. Do not add an expiration lifecycle rule: an expired object may still be referenced by MongoDB, which breaks sync for that data. - - - Add `object_storage` to the `storage` section of `service.yaml`: - - ```yaml service.yaml - storage: - type: mongodb - uri: !env PS_MONGO_STORAGE_URI - object_storage: - type: s3 - bucket: powersync-bucket-data - region: us-east-1 - prefix: production - ``` - - Without `access_key_id` and `secret_access_key`, PowerSync uses the AWS credentials available to the process, such as an IAM role. For S3-compatible providers such as MinIO or Cloudflare R2, also set `endpoint`, and set `force_path_style: true` if the provider requires path-style requests. - - Restart or redeploy the PowerSync Service to load the updated `service.yaml`. If you run replication, API, and compacting in separate containers or jobs, apply the same object storage configuration to each. - - - Deploy your Sync Configs on storage version 4 as described in [Opt In](#opt-in). Sync Configs on version 2 keep all data in MongoDB, even when `object_storage` is configured. - - Once replication reaches a healthy checkpoint, confirm that objects appear under the prefix, run a test initial sync, and run `compact` once to surface permission errors early. - - - -After enabling S3 object storage, you can raise [`max_concurrent_connections`](/configuration/powersync-service/self-hosted-instances#param-max-concurrent-connections) from its default of 200 per API process. With storage version 4 and S3 object storage, each API process can handle up to 1,000 concurrent client connections. Performance degrades if a large share of those clients run an initial sync at the same time, so scale out the API before a deployment that makes all clients download their data again. More concurrent connections also increase CPU and memory usage. - -The [S3 object storage configuration reference](/configuration/powersync-service/self-hosted-instances#param-object-storage) lists all supported settings, including timeouts, request concurrency, and the size threshold below which blocks stay in MongoDB. - -Keep the scheduled [compact](/maintenance-ops/compacting-buckets) job running. It removes files that are no longer needed. The `teardown` command deletes PowerSync's files under the prefix before it drops the storage database. The `powersync_object_storage_size_bytes` [metric](/maintenance-ops/self-hosting/monitoring) reports how much object storage PowerSync uses. + diff --git a/sync/advanced/sync-data-by-time.mdx b/sync/advanced/sync-data-by-time.mdx index ea503e502..afd0b875c 100644 --- a/sync/advanced/sync-data-by-time.mdx +++ b/sync/advanced/sync-data-by-time.mdx @@ -1,23 +1,18 @@ --- title: "Sync Data by Time with Sync Streams" -description: "Filter and sync data based on time ranges using Sync Streams/Sync Rules, with patterns for recent-only and sliding-window queries." +description: "Filter and sync data based on time ranges using Sync Streams, with patterns for recent-only and sliding-window queries." sidebarTitle: "Sync Data by Time" --- +{/* Split page: the Sync Rules (Legacy) version of this page is sync/rules/sync-data-by-time.mdx. That page is deprecated and frozen, but when you fix an error here, check whether it needs the same fix. Do not mention Sync Rules on this page. */} + A common need in offline-first apps is syncing data based on time, for example, only syncing issues updated in the last 7 days instead of the entire dataset. You might expect to write something like: ```yaml -# Sync Streams streams: issues_after_start_date: - query: SELECT * FROM issues WHERE updated_at > subscription.parameter('start_at') - -# Sync Rules -bucket_definitions: - issues_after_start_date: - parameters: SELECT request.parameters() ->> 'start_at' as start_at - data: SELECT * FROM issues WHERE updated_at > bucket.start_date + query: SELECT * FROM issues WHERE updated_at > subscription.parameter('start_at') ``` However, this won't work. Here's why. @@ -28,8 +23,6 @@ PowerSync pre-computes and caches which rows belong to which parameters to enabl Additionally, time-based functions like `now()` aren't allowed in parameter expressions because the result changes depending on when the query runs, making pre-computation impossible. -These constraints apply to both Sync Streams and legacy Sync Rules. - This guide covers a few practical workarounds. We are working on a more elegant solution for this problem. When ready, this guide will be updated accordingly. @@ -51,73 +44,38 @@ Update it periodically using a cron job (e.g., with `pg_cron`): UPDATE issues SET updated_this_week = (updated_at > now() - interval '7 days'); ``` - - - ```yaml - config: - edition: 3 - streams: - recent_issues: - auto_subscribe: true - query: SELECT * FROM issues WHERE updated_this_week = true - ``` - - For multiple time ranges, define a stream per range and let the client subscribe to the one it needs: - - ```yaml - config: - edition: 3 - streams: - issues_1week: - query: SELECT * FROM issues WHERE updated_this_week = true - - issues_1month: - query: SELECT * FROM issues WHERE updated_this_month = true - ``` - - The client subscribes to the desired range: - - ```javascript - // Subscribe to one-week range - await db.syncStream('issues_1week').subscribe(); - // Or subscribe to one-month range - await db.syncStream('issues_1month').subscribe(); - ``` - - - ```yaml - bucket_definitions: - recent_issues: - data: - - SELECT * FROM issues WHERE updated_this_week = true - ``` - - For multiple time ranges, add multiple bucket definitions and let the client choose which bucket to sync: - - ```yaml - bucket_definitions: - issues_1week: - parameters: SELECT WHERE request.parameters() ->> 'range' = '1week' - data: - - SELECT * FROM issues WHERE updated_this_week = true - - issues_1month: - parameters: SELECT WHERE request.parameters() ->> 'range' = '1month' - data: - - SELECT * FROM issues WHERE updated_this_month = true - ``` - - The client passes the desired range as a client parameter: - - ```javascript - await db.connect(connector, { - params: { - range: '1week', - }, - }) - ``` - - +Then filter on the column in a stream query: + +```yaml +config: + edition: 3 +streams: + recent_issues: + auto_subscribe: true + query: SELECT * FROM issues WHERE updated_this_week = true +``` + +For multiple time ranges, define a stream per range and let the client subscribe to the one it needs: + +```yaml +config: + edition: 3 +streams: + issues_1week: + query: SELECT * FROM issues WHERE updated_this_week = true + + issues_1month: + query: SELECT * FROM issues WHERE updated_this_month = true +``` + +The client subscribes to the desired range: + +```javascript +// Subscribe to one-week range +await db.syncStream('issues_1week').subscribe(); +// Or subscribe to one-month range +await db.syncStream('issues_1month').subscribe(); +``` This approach works well when you have a small, fixed set of time ranges. However, it requires schema changes and a scheduled job to keep the columns updated. @@ -133,50 +91,25 @@ Instead of pre-defined ranges, create a bucket for each date and let the client Use `substring` to extract the date portion from a timestamp and match it with `=`: -For a complete working example, see the [PowerSync + Supabase: Time-Based Sync demo](https://github.com/powersync-ja/powersync-js/tree/main/demos/react-supabase-time-based-sync). - - - - ```yaml - config: - edition: 3 - streams: - issues_by_date: - query: SELECT * FROM issues WHERE substring(updated_at, 1, 10) = subscription.parameter('date') - ``` - - The client subscribes once per date it wants to sync: - - ```javascript - await db.syncStream('issues_by_date', { date: '2026-01-07' }).subscribe(); - await db.syncStream('issues_by_date', { date: '2026-01-08' }).subscribe(); - await db.syncStream('issues_by_date', { date: '2026-01-09' }).subscribe(); - ``` - - Each subscription can be managed independently — you can subscribe and unsubscribe to individual dates without affecting others. - - - ```yaml - bucket_definitions: - issues_by_update_at: - parameters: SELECT value as date FROM json_each(request.parameters() ->> 'dates') - data: - - SELECT * FROM issues WHERE substring(updated_at, 1, 10) = bucket.date - ``` - - The client passes the dates it wants as client parameters: - - ```javascript - await db.connect(connector, { - params: { - dates: ["2026-01-07", "2026-01-08", "2026-01-09"], - }, - }) - ``` - - - -This gives users full control over which dates to sync, with no schema changes or scheduled jobs required. +```yaml +config: + edition: 3 +streams: + issues_by_date: + query: SELECT * FROM issues WHERE substring(updated_at, 1, 10) = subscription.parameter('date') +``` + +The client subscribes once per date it wants to sync: + +```javascript +await db.syncStream('issues_by_date', { date: '2026-01-07' }).subscribe(); +await db.syncStream('issues_by_date', { date: '2026-01-08' }).subscribe(); +await db.syncStream('issues_by_date', { date: '2026-01-09' }).subscribe(); +``` + +Each subscription can be managed independently — you can subscribe and unsubscribe to individual dates without affecting others. + +This gives users full control over which dates to sync, with no schema changes or scheduled jobs required. For a complete working example, see the [PowerSync + Supabase: Time-Based Sync demo](https://github.com/powersync-ja/powersync-js/tree/main/demos/react-supabase-time-based-sync). The trade-off is granularity. In this example we're using daily buckets. If you need finer precision (hourly), syncing a large range means many buckets, which can degrade sync performance and approach [PowerSync's limit of 1,000 buckets per user](/resources/performance-and-limits#limits). If you use larger buckets (monthly), you lose the ability to filter accurately. @@ -190,69 +123,33 @@ You have to pick a granularity and stick with it. If that's a problem—say, you Combine multiple granularities in a single definition. This lets you use larger buckets (days) for older data and smaller buckets (hours, minutes) for recent data. - - - ```yaml - config: - edition: 3 - streams: - issues_by_partition: - queries: - # By day (e.g., "2026-01-07") - - SELECT * FROM issues WHERE substring(updated_at, 1, 10) = subscription.parameter('partition') - # By hour (e.g., "2026-01-07T14") - - SELECT * FROM issues WHERE substring(updated_at, 1, 13) = subscription.parameter('partition') - # By 10 minutes (e.g., "2026-01-07T14:3") - - SELECT * FROM issues WHERE substring(updated_at, 1, 15) = subscription.parameter('partition') - ``` - - The client subscribes once per partition, mixing granularities as needed: - - ```javascript - await db.syncStream('issues_by_partition', { partition: '2026-01-05' }).subscribe(); - await db.syncStream('issues_by_partition', { partition: '2026-01-06' }).subscribe(); - await db.syncStream('issues_by_partition', { partition: '2026-01-07T10' }).subscribe(); - await db.syncStream('issues_by_partition', { partition: '2026-01-07T11' }).subscribe(); - await db.syncStream('issues_by_partition', { partition: '2026-01-07T12:0' }).subscribe(); - await db.syncStream('issues_by_partition', { partition: '2026-01-07T12:1' }).subscribe(); - await db.syncStream('issues_by_partition', { partition: '2026-01-07T12:2' }).subscribe(); - ``` - - Each query naturally acts as a filter based on the length of the partition value — a day-format partition only matches the day query, an hour-format partition only matches the hour query, and so on. - - - ```yaml - bucket_definitions: - issues_by_time: - parameters: SELECT value as partition FROM json_each(request.parameters() ->> 'partitions') - data: - # By day (e.g., "2026-01-07") - - SELECT * FROM issues WHERE substring(updated_at, 1, 10) = bucket.partition - # By hour (e.g., "2026-01-07T14") - - SELECT * FROM issues WHERE substring(updated_at, 1, 13) = bucket.partition - # By 10 minutes (e.g., "2026-01-07T14:3") - - SELECT * FROM issues WHERE substring(updated_at, 1, 15) = bucket.partition - ``` - - The client then mixes granularities as needed: - - ```javascript - await db.connect(connector, { - params: { - partitions: [ - "2026-01-05", - "2026-01-06", - "2026-01-07T10", - "2026-01-07T11", - "2026-01-07T12:0", - "2026-01-07T12:1", - "2026-01-07T12:2" - ] - }, - }) - ``` - - +```yaml +config: + edition: 3 +streams: + issues_by_partition: + queries: + # By day (e.g., "2026-01-07") + - SELECT * FROM issues WHERE substring(updated_at, 1, 10) = subscription.parameter('partition') + # By hour (e.g., "2026-01-07T14") + - SELECT * FROM issues WHERE substring(updated_at, 1, 13) = subscription.parameter('partition') + # By 10 minutes (e.g., "2026-01-07T14:3") + - SELECT * FROM issues WHERE substring(updated_at, 1, 15) = subscription.parameter('partition') +``` + +The client subscribes once per partition, mixing granularities as needed: + +```javascript +await db.syncStream('issues_by_partition', { partition: '2026-01-05' }).subscribe(); +await db.syncStream('issues_by_partition', { partition: '2026-01-06' }).subscribe(); +await db.syncStream('issues_by_partition', { partition: '2026-01-07T10' }).subscribe(); +await db.syncStream('issues_by_partition', { partition: '2026-01-07T11' }).subscribe(); +await db.syncStream('issues_by_partition', { partition: '2026-01-07T12:0' }).subscribe(); +await db.syncStream('issues_by_partition', { partition: '2026-01-07T12:1' }).subscribe(); +await db.syncStream('issues_by_partition', { partition: '2026-01-07T12:2' }).subscribe(); +``` + +Each query naturally acts as a filter based on the length of the partition value — a day-format partition only matches the day query, an hour-format partition only matches the hour query, and so on. This syncs January 5–6 by day, the morning of January 7 by hour, and the last 30 minutes in 10-minute chunks, without creating hundreds of buckets. diff --git a/sync/grammar/sync-rules/index.mdx b/sync/grammar/sync-rules/index.mdx index 4f7c46d28..c96e0e1bc 100644 --- a/sync/grammar/sync-rules/index.mdx +++ b/sync/grammar/sync-rules/index.mdx @@ -3,9 +3,9 @@ title: "Grammar Reference (Sync Rules)" description: "Railroad diagram reference for the SQL grammar supported in legacy Sync Rules queries." --- -This page is a formal grammar reference for Sync Rules: it shows the syntax accepted for parameter queries and data queries using railroad diagrams. This page complements the [Supported SQL](/sync/supported-sql) guide, which explains in prose what you can write, with examples and restrictions. +This page is a formal grammar reference for Sync Rules: it shows the syntax accepted for parameter queries and data queries using railroad diagrams. This page complements the [Supported SQL](/sync/rules/supported-sql) guide, which explains in prose what you can write, with examples and restrictions. -**When to use this page:** If you need to check whether a construct is valid, see how parameter vs data query syntax differs, or you're used to grammar specs, use the diagrams and the "Used by" / "References" links to navigate. For most users just getting started, see [Supported SQL](/sync/supported-sql) and the [Sync Rules](/sync/rules/overview) docs. +**When to use this page:** If you need to check whether a construct is valid, see how parameter vs data query syntax differs, or you're used to grammar specs, use the diagrams and the "Used by" / "References" links to navigate. For most users just getting started, see [Supported SQL](/sync/rules/supported-sql) and the [Sync Rules](/sync/rules/overview) docs. ## ParameterQuery diff --git a/sync/rules/case-sensitivity.mdx b/sync/rules/case-sensitivity.mdx new file mode 100644 index 000000000..c4b61d188 --- /dev/null +++ b/sync/rules/case-sensitivity.mdx @@ -0,0 +1,15 @@ +--- +title: "Case Sensitivity" +description: "Handle case-sensitive table and column names in PowerSync Sync Streams/Rules, with best practices for lowercase identifiers and quoting strategies." +noindex: true +--- + +{/* Wrapper page: the content is snippets/sync-shared/case-sensitivity.mdx, which also renders at sync/advanced/case-sensitivity.mdx in the Sync Streams section. Edit the snippet, not this file. The deprecation callout stays here, outside the snippet. */} + +import CaseSensitivity from '/snippets/sync-shared/case-sensitivity.mdx'; + + +Sync Rules are deprecated. This page applies to both Sync Streams and Sync Rules unless a section says otherwise. + + + diff --git a/sync/rules/client-id.mdx b/sync/rules/client-id.mdx new file mode 100644 index 000000000..07d440ebb --- /dev/null +++ b/sync/rules/client-id.mdx @@ -0,0 +1,15 @@ +--- +title: "Client ID" +description: "Understand PowerSync's requirement for a single text-type primary key column called id." +noindex: true +--- + +{/* Wrapper page: the content is snippets/sync-shared/client-id.mdx, which also renders at sync/advanced/client-id.mdx in the Sync Streams section. Edit the snippet, not this file. The deprecation callout stays here, outside the snippet. */} + +import ClientId from '/snippets/sync-shared/client-id.mdx'; + + +Sync Rules are deprecated. This page applies to both Sync Streams and Sync Rules unless a section says otherwise. + + + diff --git a/sync/rules/compatibility.mdx b/sync/rules/compatibility.mdx new file mode 100644 index 000000000..b9fa6991e --- /dev/null +++ b/sync/rules/compatibility.mdx @@ -0,0 +1,15 @@ +--- +title: "Compatibility" +description: "Configure compatibility editions and bucket storage format version in PowerSync's Sync Config." +noindex: true +--- + +{/* Wrapper page: the content is snippets/sync-shared/compatibility.mdx, which also renders at sync/advanced/compatibility.mdx in the Sync Streams section. Edit the snippet, not this file. The deprecation callout stays here, outside the snippet. */} + +import Compatibility from '/snippets/sync-shared/compatibility.mdx'; + + +Sync Rules are deprecated. This page applies to both Sync Streams and Sync Rules unless a section says otherwise. Configuration examples use Sync Streams syntax. + + + diff --git a/sync/rules/data-queries.mdx b/sync/rules/data-queries.mdx index a9892bbe9..b654435fb 100644 --- a/sync/rules/data-queries.mdx +++ b/sync/rules/data-queries.mdx @@ -19,7 +19,7 @@ Data Queries are used to group data into buckets, so each Data Query must use ev ## Supported SQL -The supported SQL in Data Queries is based on a small subset of the SQL standard syntax. Not all SQL constructs are supported. See [Supported SQL](/sync/supported-sql) for full details. +The supported SQL in Data Queries is based on a small subset of the SQL standard syntax. Not all SQL constructs are supported. See [Supported SQL](/sync/rules/supported-sql) for full details. ## Examples diff --git a/sync/rules/migrate-to-sync-streams.mdx b/sync/rules/migrate-to-sync-streams.mdx index 69a2c4f11..e4a563424 100644 --- a/sync/rules/migrate-to-sync-streams.mdx +++ b/sync/rules/migrate-to-sync-streams.mdx @@ -18,7 +18,9 @@ If your Sync Config has a `bucket_definitions:` section, you use Sync Rules and ## Why Migrate? -Beyond matching Sync Rules, Sync Streams add: +{/* TODO: Link to the Sync Rules deprecation announcement once it is published. */} + +Sync Rules are deprecated, and PowerSync is phasing them out in favor of Sync Streams. Beyond matching Sync Rules, Sync Streams add: - **More expressive queries:** Stream queries support JOINs, [CTEs](/sync/streams/ctes), subqueries, and [multiple queries per stream](/sync/streams/queries#multiple-queries-per-stream), with syntax closer to plain SQL. You write one query instead of separate `parameters:` and `data:` blocks. - **On-demand syncing:** Define a stream once, then subscribe from your app one or more times with different parameters. Each subscription has its own lifecycle, so two screens or browser tabs can subscribe to the same stream independently. With Sync Rules, Client Parameters approximate this. You have to aggregate the parameter values yourself across screens and tabs, and remove them when they are no longer needed. diff --git a/sync/rules/multiple-client-versions.mdx b/sync/rules/multiple-client-versions.mdx new file mode 100644 index 000000000..0db532955 --- /dev/null +++ b/sync/rules/multiple-client-versions.mdx @@ -0,0 +1,39 @@ +--- +title: "Multiple Client Versions with Sync Rules" +sidebarTitle: "Multiple Client Versions" +description: "Handle multiple client app versions that require different output schemas from legacy Sync Rules." +--- + +{/* Split page: the Sync Streams version of this page is sync/advanced/multiple-client-versions.mdx. This page is deprecated and frozen: no new features or examples. When you fix an error here, check whether the Sync Streams page needs the same fix. */} + + +Sync Rules are deprecated. This page is not updated with new features. For the Sync Streams version, see [Multiple Client Versions](/sync/advanced/multiple-client-versions). + + +When schema changes are additive, old clients ignore the new tables and columns, and no special handling is required. More drastic changes, such as renaming tables or changing a table's structure, can break older app versions that are still in use. In these cases, define separate versions of the affected bucket definitions so that each client version receives the tables and columns it expects. + +## Versioning with Client Parameters + +Clients pass their version to the PowerSync Service as a [client parameter](/sync/rules/client-parameters), and parameter queries filter on it so that each client only receives the buckets for its version. + +For example, suppose a new app version changes the structure of the `assets` table in its [client-side schema](/intro/setup-guide#define-your-client-side-schema), defining it as `assets_v2`, while older app versions still define `assets`. Define a second bucket definition alongside the existing one, using an alias to map the source `assets` table to the new client-side name, and filter each on a `schema_version` client parameter: + +```yaml +# Client passes in: "params": {"schema_version": } +bucket_definitions: + user_assets: + parameters: SELECT request.user_id() AS user_id + WHERE request.parameters() ->> 'schema_version' = '1' + data: + - SELECT * FROM assets WHERE user_id = bucket.user_id + + user_assets_v2: + parameters: SELECT request.user_id() AS user_id + WHERE request.parameters() ->> 'schema_version' = '2' + data: + - SELECT * FROM assets AS assets_v2 WHERE user_id = bucket.user_id +``` + + + Handle queries based on parameters set by the client with care. The client can send any value for these parameters, so it's not a good place to do authorization. If the parameter must be authenticated, use parameters from the JWT instead. + diff --git a/sync/rules/organize-data-into-buckets.mdx b/sync/rules/organize-data-into-buckets.mdx index 130b0b560..8888b9757 100644 --- a/sync/rules/organize-data-into-buckets.mdx +++ b/sync/rules/organize-data-into-buckets.mdx @@ -44,7 +44,7 @@ bucket_definitions: - The supported SQL in _Parameter Queries_ and _Data Queries_ is based on a small subset of the SQL standard syntax. Not all SQL constructs are supported. See [Supported SQL](/sync/supported-sql). + The supported SQL in _Parameter Queries_ and _Data Queries_ is based on a small subset of the SQL standard syntax. Not all SQL constructs are supported. See [Supported SQL](/sync/rules/supported-sql). diff --git a/sync/rules/overview.mdx b/sync/rules/overview.mdx index 17921d0e2..78ae6a770 100644 --- a/sync/rules/overview.mdx +++ b/sync/rules/overview.mdx @@ -4,15 +4,17 @@ sidebarTitle: "Overview & Key Concepts" description: "Understand legacy Sync Rules for controlling which data syncs to each client." --- -Sync Rules are PowerSync's original system for partial sync, using YAML bucket definitions. They remain supported for existing projects but are considered legacy. +Sync Rules are PowerSync's original system for partial sync, using YAML bucket definitions. They are deprecated. Existing instances keep working and stay supported while you migrate to Sync Streams. - -**Sync Streams Recommended** +{/* TODO: Link to the Sync Rules deprecation announcement on releases.powersync.com once it is published. */} -[Sync Streams](/sync/streams/overview) are the recommended approach to partial sync for both new and existing projects. They support everything Sync Rules do, plus more expressive queries (including JOIN support), on-demand syncing, and a simpler developer experience (e.g. React hooks that manage subscriptions automatically). + +**Sync Rules are deprecated** -You can migrate in a few clicks. Click **Migrate to Sync Streams** in the PowerSync Dashboard, or run `powersync migrate sync-rules` in the CLI to generate a draft from your current config. See [Migrate to Sync Streams](/sync/rules/migrate-to-sync-streams) for details. - +PowerSync is phasing out Sync Rules in favor of [Sync Streams](/sync/streams/overview), which support everything Sync Rules do and add on-demand syncing, JOINs, CTEs, and subqueries. Nothing changes for your instance today: Sync Rules keep working and stay supported while you migrate. New sync config features are added to Sync Streams only. + +To migrate, click **Migrate to Sync Streams** in the PowerSync Dashboard, or run `powersync migrate sync-rules` in the CLI to convert your current config. Migrating does not change what your app syncs. See [Migrate to Sync Streams](/sync/rules/migrate-to-sync-streams). + Sync Rules are defined in a YAML file. For PowerSync Cloud, they are edited and deployed to a specific PowerSync instance in the [PowerSync Dashboard](/tools/powersync-dashboard#project-&-instance-level). For self-hosting setups, they are defined as part of your [instance configuration](/configuration/powersync-service/self-hosted-instances). @@ -52,7 +54,7 @@ The following values can be selected in Parameter Queries: - **Client Parameters** (see below) - **Values From a Table/Collection** (see below) -See [Parameter Queries](/sync/rules/parameter-queries) for more details and examples. Also see [Supported SQL](/sync/supported-sql) for limitations. +See [Parameter Queries](/sync/rules/parameter-queries) for more details and examples. Also see [Supported SQL](/sync/rules/supported-sql) for limitations. ### Authentication Parameters @@ -72,11 +74,11 @@ Clients can specify **Client Parameters** when connecting to PowerSync (i.e. whe ```yaml Example of selecting a Client Parameter in a Parameter Query parameters: SELECT (request.parameters() ->> 'current_project') as current_project ``` -The `->>` operator in the above example extracts a value from a string containing JSON (which is the format provided by ``request.parameters()``). See [Operators and Functions](/sync/supported-sql#operators) +The `->>` operator in the above example extracts a value from a string containing JSON (which is the format provided by ``request.parameters()``). See [Operators and Functions](/sync/rules/supported-sql#operators) A client can pass any value for a Client Parameter. Hence, Client Parameters should always be treated with care, and should [not be used](/sync/rules/client-parameters#security-consideration) for access control purposes. -That being said, Client Parameters can be useful for use cases such as syncing different buckets based on state in the client app, for example only syncing data for the project currently selected, or syncing different buckets based on the client version ([see here](/sync/advanced/multiple-client-versions)). +That being said, Client Parameters can be useful for use cases such as syncing different buckets based on state in the client app, for example only syncing data for the project currently selected, or syncing different buckets based on the client version ([see here](/sync/rules/multiple-client-versions)). See [Client Parameters](/sync/rules/client-parameters) and [Parameter Queries](/sync/rules/parameter-queries) for more details and examples. @@ -103,7 +105,7 @@ data: - SELECT * FROM lists WHERE owner_id = bucket.user_id ``` -See [Data Queries](/sync/rules/data-queries) for more details and examples. Also see [Supported SQL](/sync/supported-sql) for limitations. +See [Data Queries](/sync/rules/data-queries) for more details and examples. Also see [Supported SQL](/sync/rules/supported-sql) for limitations. ### Global Buckets @@ -127,6 +129,3 @@ This architecture is key to the scalability and performance of PowerSync. See th Designing your Sync Rules is basically about _organizing data into buckets_, and creating the bucket definitions accordingly. See [Organize Data Into Buckets](/sync/rules/organize-data-into-buckets). - - - diff --git a/sync/rules/parameter-queries.mdx b/sync/rules/parameter-queries.mdx index 47372f0e6..c8c3972a9 100644 --- a/sync/rules/parameter-queries.mdx +++ b/sync/rules/parameter-queries.mdx @@ -27,7 +27,7 @@ The following functions allow you to select Authentication Parameters in your Pa | `request.user_id()` | Returns the JWT subject (`sub`). Same as `request.jwt() ->> 'sub'` (see below) | | `request.jwt()` | Returns the entire (signed) JWT payload as a JSON string. If there are other _claims_ in your JWT (in addition to the user ID), you can select them from this JSON string. | -Since `request.jwt()` is a string containing JSON, use the `->>` [operator](/sync/supported-sql#operators) to select values from it: +Since `request.jwt()` is a string containing JSON, use the `->>` [operator](/sync/rules/supported-sql#operators) to select values from it: ```sql request.jwt() ->> 'sub' -- the 'subject' of the JWT - same as `request.user_id() @@ -119,7 +119,7 @@ bucket_definitions: ## Supported SQL -The supported SQL in Parameter Queries is based on a small subset of the SQL standard syntax. Not all SQL constructs are supported. See [Supported SQL](/sync/supported-sql) for full details. +The supported SQL in Parameter Queries is based on a small subset of the SQL standard syntax. Not all SQL constructs are supported. See [Supported SQL](/sync/rules/supported-sql) for full details. ## Usage Examples @@ -203,7 +203,7 @@ For more advanced details on many-to-many relationships and join tables, see [th ### Expanding JSON Array Into Multiple Parameters -Using the `json_each()` [function](/sync/supported-sql#functions) and `->` [operator](/sync/supported-sql#operators), we can expand a parameter that is a JSON array into multiple rows, thereby filtering by multiple parameter values: +Using the `json_each()` [function](/sync/rules/supported-sql#functions) and `->` [operator](/sync/rules/supported-sql#operators), we can expand a parameter that is a JSON array into multiple rows, thereby filtering by multiple parameter values: ```yaml bucket_definitions: diff --git a/sync/rules/partitioned-tables.mdx b/sync/rules/partitioned-tables.mdx new file mode 100644 index 000000000..729f31b4d --- /dev/null +++ b/sync/rules/partitioned-tables.mdx @@ -0,0 +1,37 @@ +--- +title: "Partitioned Tables (Postgres) with Sync Rules" +sidebarTitle: "Partitioned Tables (Postgres)" +description: "Sync data from Postgres partitioned tables in legacy Sync Rules using wildcard table name matching." +--- + +{/* Split page: the Sync Streams version of this page is sync/advanced/partitioned-tables.mdx. This page is deprecated and frozen: no new features or examples. When you fix an error here, check whether the Sync Streams page needs the same fix. */} + + +Sync Rules are deprecated. This page is not updated with new features. For the Sync Streams version, see [Partitioned Tables (Postgres)](/sync/advanced/partitioned-tables). + + +For partitioned tables in Postgres, each individual partition is replicated and processed using [Sync Rules](/sync/rules/overview). + +To use the same queries and same output table name for each partition, use `%` for wildcard suffix matching of the table name: + +```yaml +bucket_definitions: + by_user: + # Use wildcard in a parameter query + parameters: SELECT id AS user_id FROM "users_%" + data: + # Use wildcard in a data query + - SELECT * FROM "todos_%" AS todos WHERE user_id = bucket.user_id +``` + +The wildcard character can only be used as the last character in the table name. To match tables across multiple schemas instead, see [Wildcard Schemas](/sync/advanced/schemas-and-connections#wildcard-schemas-postgres). + +When using wildcard table names, the original table suffix is available as the special `_table_suffix` column, which you can use to filter rows: + +```sql +SELECT * FROM "todos_%" AS todos WHERE _table_suffix != 'archived' +``` + +When no table alias is provided, the original table name is preserved. + +`publish_via_partition_root` on the publication is not supported. The individual partitions must be published. diff --git a/sync/rules/prioritized-sync.mdx b/sync/rules/prioritized-sync.mdx new file mode 100644 index 000000000..e0f397f2e --- /dev/null +++ b/sync/rules/prioritized-sync.mdx @@ -0,0 +1,126 @@ +--- +title: "Prioritized Sync with Sync Rules" +sidebarTitle: "Prioritized Sync" +description: "Assign sync priorities to bucket definitions in legacy Sync Rules so that important data syncs before the rest." +--- + +{/* Split page: the Sync Streams version of this page is sync/streams/prioritized-sync.mdx. This page is deprecated and frozen: no new features or examples. When you fix an error here, check whether the Sync Streams page needs the same fix. */} + + +Sync Rules are deprecated. This page is not updated with new features. For the Sync Streams version, see [Prioritized Sync](/sync/streams/prioritized-sync). + + +## Overview + +PowerSync supports defining sync priorities, which allows you to control the sync order for different data. This is useful when certain data should be available sooner than others. + +In Sync Rules, you assign priorities to bucket definitions. The priority determines when data in that bucket syncs relative to other buckets. + + +**Availability** + +This feature was introduced in version **1.7.1** of the PowerSync Service, and in the following SDK versions: +- [Flutter v1.12.0](/client-sdks/reference/flutter) +- [React Native v1.18.1](/client-sdks/reference/react-native-and-expo) +- [JavaScript Web v1.14.2](/client-sdks/reference/javascript-web) +- [Kotlin v1.0.0-BETA26](/client-sdks/reference/kotlin) +- [Swift v1.0.0-Beta.8](/client-sdks/reference/swift) +- [.NET v0.0.6-alpha.1](/client-sdks/reference/dotnet) + + +## Why Use Sync Priorities? + +PowerSync's standard sync protocol ensures that: +- The local data view is only updated when a fully consistent checkpoint is available. +- All pending local changes must be uploaded, acknowledged, and synced back before new data is applied. + +While this guarantees consistency, it can lead to delays, especially for large datasets or continuous client-side updates. Sync priorities provide a way to speed up syncing of high-priority data while still maintaining overall integrity. + +## How It Works + +Each bucket is assigned a priority value between 0 and 3, where: + +- 0 is the highest priority and has special behavior (detailed below). +- 3 is the default and lowest priority. +- Lower numbers indicate higher priority. + +Higher-priority data syncs first, and lower-priority data syncs later. If you only use a single priority, there is no difference between priorities 1-3. The difference only comes in when you use multiple different priorities. + +## Syntax and Configuration + +Define priorities using the `priority` YAML key on a bucket definition, or with the `_priority` attribute inside a parameter query: + +```yaml +bucket_definitions: + # Using the `priority` YAML key + user_data: + priority: 1 + parameters: SELECT request.user_id() AS id WHERE ... + data: + # ... + + # Using the `_priority` attribute (useful for multiple parameter queries with different priorities) + project_data: + parameters: SELECT id AS project_id, 2 AS _priority FROM projects WHERE ... + data: + # ... +``` + + +Priorities must be static and cannot depend on row values within a parameter query. + + +## Example: Syncing Lists Before Todos + +Consider a scenario where you want to display lists immediately while loading todos in the background. This approach allows users to view and interact with lists right away without waiting for todos to sync. + +```yaml +bucket_definitions: + user_lists: + priority: 1 # Syncs first + parameters: SELECT id AS list_id FROM lists WHERE user_id = request.user_id() + data: + - SELECT * FROM lists WHERE id = bucket.list_id + + user_todos: + priority: 2 # Syncs after lists + parameters: SELECT id AS list_id FROM lists WHERE user_id = request.user_id() + data: + - SELECT * FROM todos WHERE list_id = bucket.list_id +``` + +The `user_lists` bucket syncs first (priority 1), allowing users to see and interact with their lists immediately. The `user_todos` bucket syncs afterward (priority 2), loading in the background. + +## Behavioral Considerations + +- **Interruption for Higher Priority Data:** Syncing lower-priority data _may_ be interrupted if new data for higher-priority buckets arrives. +- **Local Changes & Consistency:** If local writes fail due to validation or permission issues, they are only reverted after _all_ data has synced. +- **Deleted Data:** Deleted data may only be removed after _all_ priorities have completed syncing. +- **Data Ordering:** Lower-priority data will never appear before higher-priority data. + +## Special Case: Priority 0 + +Priority 0 buckets sync regardless of pending uploads. + +For example, in a collaborative document editing app (e.g., using Yjs), each change is stored as a separate row. Since out-of-order updates don't affect document integrity, Priority 0 can ensure immediate availability of updates. + +Caution: If misused, Priority 0 may cause flickering or inconsistencies, as updates could arrive out of order. + +## Consistency Considerations + +PowerSync's full consistency guarantees only apply once all priorities have completed syncing. + +When higher-priority data is synced, all inserts and updates at that priority level will be consistent. However, deletes are only applied when the full sync completes, so you may still have some stale data at those priority levels. + +Consider the following example: + +Imagine a task management app where users create lists and todos. Some users have millions of todos. To improve first-load speed: + +- Lists are assigned Priority 1, syncing first to allow UI rendering. +- Todos are assigned Priority 2, loading in the background. + +Now, if another user adds new todos, it's possible for the list count (synced at Priority 1) to temporarily not match the actual todos (synced at Priority 2). If real-time accuracy is required, both lists and todos should use the same priority. + +## Client-Side Considerations + +The client SDK APIs for tracking sync status per priority are the same for Sync Rules and Sync Streams: `waitForFirstSync(priority)`, `SyncStatus.priorityStatusEntries()`, and `SyncStatus.statusForPriority(priority)`. See [Client-Side Considerations](/sync/streams/prioritized-sync#client-side-considerations) on the Sync Streams page for details and a Flutter example. diff --git a/sync/rules/reducing-bucket-count.mdx b/sync/rules/reducing-bucket-count.mdx new file mode 100644 index 000000000..c20a2ec30 --- /dev/null +++ b/sync/rules/reducing-bucket-count.mdx @@ -0,0 +1,16 @@ +--- +title: "Reducing Bucket Count" +description: "Diagnose a high bucket count, reduce the number of buckets a user syncs, and raise the per-user limits when needed." +sidebarTitle: "Reducing Buckets" +noindex: true +--- + +{/* Wrapper page: the content is snippets/sync-shared/reducing-bucket-count.mdx, which also renders at sync/advanced/reducing-bucket-count.mdx in the Sync Streams section. Edit the snippet, not this file. The deprecation callout stays here, outside the snippet. */} + +import ReducingBucketCount from '/snippets/sync-shared/reducing-bucket-count.mdx'; + + +Sync Rules are deprecated. This page applies to both Sync Streams and Sync Rules unless a section says otherwise. Configuration examples use Sync Streams syntax. + + + diff --git a/sync/rules/schemas-and-connections.mdx b/sync/rules/schemas-and-connections.mdx new file mode 100644 index 000000000..f6a70c675 --- /dev/null +++ b/sync/rules/schemas-and-connections.mdx @@ -0,0 +1,15 @@ +--- +title: "Schemas and Connections" +description: "Configure Postgres schema usage in Sync Streams/Rules queries, including wildcard schemas for schema-per-tenant setups, and connect to high-availability replicas." +noindex: true +--- + +{/* Wrapper page: the content is snippets/sync-shared/schemas-and-connections.mdx, which also renders at sync/advanced/schemas-and-connections.mdx in the Sync Streams section. Edit the snippet, not this file. The deprecation callout stays here, outside the snippet. */} + +import SchemasAndConnections from '/snippets/sync-shared/schemas-and-connections.mdx'; + + +Sync Rules are deprecated. This page applies to both Sync Streams and Sync Rules unless a section says otherwise. Configuration examples use Sync Streams syntax. + + + diff --git a/sync/rules/sharded-databases.mdx b/sync/rules/sharded-databases.mdx new file mode 100644 index 000000000..8ec1b0a78 --- /dev/null +++ b/sync/rules/sharded-databases.mdx @@ -0,0 +1,15 @@ +--- +title: "Sharded Databases" +description: "Sync data from sharded Postgres databases with per-shard PowerSync connection configuration." +noindex: true +--- + +{/* Wrapper page: the content is snippets/sync-shared/sharded-databases.mdx, which also renders at sync/advanced/sharded-databases.mdx in the Sync Streams section. Edit the snippet, not this file. The deprecation callout stays here, outside the snippet. */} + +import ShardedDatabases from '/snippets/sync-shared/sharded-databases.mdx'; + + +Sync Rules are deprecated. This page applies to both Sync Streams and Sync Rules unless a section says otherwise. + + + diff --git a/sync/rules/storage-version-4.mdx b/sync/rules/storage-version-4.mdx new file mode 100644 index 000000000..e5e974de9 --- /dev/null +++ b/sync/rules/storage-version-4.mdx @@ -0,0 +1,15 @@ +--- +title: "Storage Version 4" +description: "Opt in to storage version 4 for faster sync, incremental reprocessing of Sync Streams changes, and S3 object storage." +noindex: true +--- + +{/* Wrapper page: the content is snippets/sync-shared/storage-version-4.mdx, which also renders at sync/advanced/storage-version-4.mdx in the Sync Streams section. Edit the snippet, not this file. The deprecation callout stays here, outside the snippet. */} + +import StorageVersion4 from '/snippets/sync-shared/storage-version-4.mdx'; + + +Sync Rules are deprecated. This page applies to both Sync Streams and Sync Rules unless a section says otherwise. Configuration examples use Sync Streams syntax. + + + diff --git a/sync/rules/supported-sql.mdx b/sync/rules/supported-sql.mdx new file mode 100644 index 000000000..3f98c7f1f --- /dev/null +++ b/sync/rules/supported-sql.mdx @@ -0,0 +1,157 @@ +--- +title: "Supported SQL in Sync Rules" +sidebarTitle: "Supported SQL" +description: "Reference for SQL syntax, operators, built-in functions, and type casting supported in legacy Sync Rules queries." +--- + +{/* Split page: the Sync Streams version of this page is sync/supported-sql.mdx. This page is deprecated and frozen: no new features or examples. When you fix an error here, check whether the Sync Streams page needs the same fix. */} + + +Sync Rules are deprecated. This page is not updated with new features. For the Sync Streams version, see [Supported SQL](/sync/supported-sql). + + +This guide explains the SQL supported in [Sync Rules](/sync/rules/overview) parameter queries and data queries: what you can write, with examples and restrictions. + +For the exact syntax the compiler accepts, with railroad diagrams and grammar-rule references, see the [Sync Rules grammar reference](/sync/grammar/sync-rules/index). + + + Some fundamental restrictions on the usage of SQL expressions are: + + 1. They must be deterministic: no random or time-based functions. + 2. No external state can be used. + 3. They must operate on data available within a single row/document. For example, no aggregation functions are allowed. + + For parameter-specific WHERE restrictions, see [Filtering: WHERE Clause](#filtering-where-clause). + + +## Query Syntax + +The supported SQL is based on a small subset of the standard SQL syntax: + +- Simple `SELECT` with column selection +- `WHERE` filtering on parameters (see [Filtering: WHERE Clause](#filtering-where-clause)) +- A limited set of [operators](#operators) and [functions](#functions) + +**Not supported:** subqueries, JOINs, CTEs, aggregation, sorting, or set operations (`GROUP BY`, `ORDER BY`, `LIMIT`, `UNION`, etc.). + +## Filtering: WHERE Clause + +Sync Rules queries support a subset of SQL `WHERE` syntax. Allowed operators and combinations are more restrictive than standard SQL. + +**`=` and `IS NULL`:** Compare a row column to a static value or a bucket parameter: + +```sql +-- Static value +WHERE status = 'active' +WHERE deleted_at IS NULL + +-- Bucket parameter +WHERE owner_id = bucket.user_id +``` + +**`AND`:** Supported in both Parameter Queries and Data Queries. In Parameter Queries, each condition may match a different parameter. However, you cannot combine two `IN` expressions on parameters in the same `AND`; split them into separate Parameter Queries instead. + +```sql +-- Supported: parameter condition + row-value condition +WHERE users.id = request.user_id() + AND users.is_admin = true + +-- Not supported: two IN expressions on parameters in the same AND +-- WHERE bucket.list_id IN lists.allowed_ids +-- AND bucket.org_id IN lists.allowed_org_ids +``` + +**`OR`:** Supported when both sides of the `OR` reference the exact same set of parameters. If the two sides use different parameters, use separate parameter queries instead. + +```sql +-- Supported: both sides reference the same parameter +WHERE lists.owner_id = request.user_id() + OR lists.shared_with = request.user_id() + +-- Not supported: sides reference different parameters +-- WHERE lists.owner_id = request.user_id() +-- OR lists.org_id = bucket.org_id +``` + +**`NOT`:** Supported for simple row-value conditions. Not supported on parameter-matching expressions. + +```sql +-- Supported +WHERE status != 'archived' +WHERE deleted_at IS NOT NULL +WHERE NOT users.is_admin = true + +-- Not supported in parameter queries +-- WHERE NOT users.id = request.user_id() +``` + +## Operators + +Operators can be used in `WHERE` clauses and in `SELECT` expressions. When filtering on parameters (e.g. `request.user_id()`, `bucket.user_id`), some combinations are restricted. See [Filtering: WHERE Clause](#filtering-where-clause). + + + + - **Comparison:** `=`, `!=`, `<`, `>`, `<=`, `>=` — If either side is `null`, the result is `null`. + - **Null:** `IS NULL`, `IS NOT NULL` + + + - **Logical:** `AND`, `OR`, `NOT` — See [Filtering: WHERE Clause](#filtering-where-clause) for restrictions when filtering on parameters. + - **Mathematical:** `+`, `-`, `*`, `/` + + + - `||` — Joins two text values together. + + + - `json -> 'path'` — Returns the value as a JSON string. + - `json ->> 'path'` — Returns the extracted value. + + + - `left IN right` — Returns true if `left` is in the `right` JSON array. In Data Queries, `left` must be a row column and `right` cannot be a bucket parameter. In Parameter Queries, either side may be a parameter. + + + +## Functions + +Functions can be used to transform columns/fields before being synced to a client. They operate on row data or parameters. Type names below (`text`, `integer`, `real`, `blob`, `null`) refer to [SQLite storage classes](https://www.sqlite.org/datatype3.html). + +Most functions are from [SQLite built-in functions](https://www.sqlite.org/lang_corefunc.html) and [SQLite JSON functions](https://www.sqlite.org/json1.html). + + + + - **[upper(text)](https://www.sqlite.org/lang_corefunc.html#upper)** — Convert text to upper case. + - **[lower(text)](https://www.sqlite.org/lang_corefunc.html#lower)** — Convert text to lower case. + - **[substring(text, start, length)](https://www.sqlite.org/lang_corefunc.html#substr)** — Extracts a portion of a string based on specified start index and length. Start index is 1-based. Example: `substring(created_at, 1, 10)` returns the date portion of the timestamp. + - **[instr(string, substring)](https://www.sqlite.org/lang_corefunc.html#instr)** — Finds the first occurrence of the substring within the string and returns the number of prior characters plus 1, or 0 if the substring is not found. Useful for locating a delimiter in compound strings. For example, `substring(value, 1, instr(value, '|') - 1)` extracts the portion before a `|` character. + - **[hex(data)](https://www.sqlite.org/lang_corefunc.html#hex)** — Convert blob or text data to hexadecimal text. + - **base64(data)** — Convert blob or text data to base64 text. + - **[length(data)](https://www.sqlite.org/lang_corefunc.html#length)** — For text, return the number of characters. For blob, return the number of bytes. For null, return null. For integer and real, convert to text and return the number of characters. + + + - `CAST(x AS type)` or `x :: type` — Cast to `text`, `numeric`, `integer`, `real`, or `blob`. See [Type mapping](/sync/types) and [SQLite types](https://www.sqlite.org/datatype3.html). + - **[typeof(data)](https://www.sqlite.org/lang_corefunc.html#typeof)** — Returns `text`, `integer`, `real`, `blob`, or `null`. + + + - **[json_each(data)](https://www.sqlite.org/json1.html#jeach)** — Expands a JSON array or object from a request or token parameter into a set of parameter rows. Example: `SELECT value AS project_id FROM json_each(request.jwt() -> 'project_ids')`. See [Expanding JSON Array Into Multiple Parameters](/sync/rules/parameter-queries#expanding-json-array-into-multiple-parameters). + - **[json_extract(data, path)](https://www.sqlite.org/json1.html#jex)** — Same as `->>` operator, but the path must start with `$.` + - **[json_array_length(data)](https://www.sqlite.org/json1.html#jarraylen)** — Given a JSON array (as text), returns the length of the array. If data is null, returns null. If the value is not a JSON array, returns 0. + - **[json_valid(data)](https://www.sqlite.org/json1.html#jvalid)** — Returns 1 if the data can be parsed as JSON, 0 otherwise. + - **json_keys(data)** — Returns the set of keys of a JSON object as a JSON array. Example: `SELECT * FROM items WHERE bucket.user_id IN json_keys(permissions_json)`. + + + - **[ifnull(x, y)](https://www.sqlite.org/lang_corefunc.html#ifnull)** — Returns x if non-null, otherwise returns y. + + + - **[iif(x, y, z)](https://www.sqlite.org/lang_corefunc.html#iif)** — Returns y if x is true, otherwise returns z. + + + - **[unixepoch(time-value, [modifier])](https://www.sqlite.org/lang_datefunc.html)** — Returns a time-value as Unix timestamp. If modifier is "subsec", the result is a floating point number, with milliseconds included in the fraction. The time-value argument is required. This function cannot be used to get the current time. + - **[datetime(time-value, [modifier])](https://www.sqlite.org/lang_datefunc.html)** — Returns a time-value as a date and time string, in the format YYYY-MM-DD HH:MM:SS. If the specifier is "subsec", milliseconds are also included. If the modifier is "unixepoch", the argument is interpreted as a Unix timestamp. Both modifiers can be included: `datetime(timestamp, 'unixepoch', 'subsec')`. The time-value argument is required. This function cannot be used to get the current time. + - **[uuid_blob(id)](https://sqlite.org/src/file/ext/misc/uuid.c)** — Convert a UUID string to bytes. + + + - **[ST_AsGeoJSON(geometry)](/client-sdks/advanced/gis-data-postgis)** — Convert [PostGIS](/client-sdks/advanced/gis-data-postgis) (in Postgres) geometry from WKB to GeoJSON. Combine with JSON operators to extract specific fields. + - **[ST_AsText(geometry)](/client-sdks/advanced/gis-data-postgis)** — Convert [PostGIS](/client-sdks/advanced/gis-data-postgis) (in Postgres) geometry from WKB to Well-Known Text (WKT). + - **[ST_X(point)](/client-sdks/advanced/gis-data-postgis)** — Get the X coordinate of a [PostGIS](/client-sdks/advanced/gis-data-postgis) point (in Postgres). + - **[ST_Y(point)](/client-sdks/advanced/gis-data-postgis)** — Get the Y coordinate of a [PostGIS](/client-sdks/advanced/gis-data-postgis) point (in Postgres). + + diff --git a/sync/rules/sync-data-by-time.mdx b/sync/rules/sync-data-by-time.mdx new file mode 100644 index 000000000..3860f06ab --- /dev/null +++ b/sync/rules/sync-data-by-time.mdx @@ -0,0 +1,174 @@ +--- +title: "Sync Data by Time with Sync Rules" +sidebarTitle: "Sync Data by Time" +description: "Filter and sync data based on time ranges using legacy Sync Rules, with patterns for recent-only and sliding-window queries." +--- + +{/* Split page: the Sync Streams version of this page is sync/advanced/sync-data-by-time.mdx. This page is deprecated and frozen: no new features or examples. When you fix an error here, check whether the Sync Streams page needs the same fix. */} + + +Sync Rules are deprecated. This page is not updated with new features. For the Sync Streams version, see [Sync Data by Time](/sync/advanced/sync-data-by-time). + + +A common need is syncing data based on time, for example, only syncing issues updated in the last 7 days instead of the entire dataset. You might expect to write something like: + +```yaml +bucket_definitions: + issues_after_start_date: + parameters: SELECT request.parameters() ->> 'start_at' AS start_at + data: + - SELECT * FROM issues WHERE updated_at > bucket.start_at +``` + +However, this does not work. Here's why. + +## The Problem + +PowerSync pre-computes and caches which rows belong to which bucket parameters to enable efficient streaming. This means parameter-based filtering is limited to equality checks (`=`, `IN`, `IS NULL`). Range operators like `>`, `<`, `>=`, or `<=` are not supported on parameters. + +Additionally, time-based functions like `now()` are not allowed in parameter expressions because the result changes depending on when the query runs, making pre-computation impossible. + +This guide covers a few practical workarounds. + +## Workarounds + +### 1: Pre-Defined Time Ranges + +Add a boolean column to your table that indicates whether a row falls within a specific time range. Keep this column updated in your source database using a scheduled job. + +For example, add an `updated_this_week` column: + +```sql +ALTER TABLE issues ADD COLUMN updated_this_week BOOLEAN DEFAULT false; +``` + +Update it periodically using a cron job (e.g., with `pg_cron`): + +```sql +UPDATE issues SET updated_this_week = (updated_at > now() - interval '7 days'); +``` + +Then filter on the column in a data query: + +```yaml +bucket_definitions: + recent_issues: + data: + - SELECT * FROM issues WHERE updated_this_week = true +``` + +For multiple time ranges, add multiple bucket definitions and let the client choose which bucket to sync: + +```yaml +bucket_definitions: + issues_1week: + parameters: SELECT WHERE request.parameters() ->> 'range' = '1week' + data: + - SELECT * FROM issues WHERE updated_this_week = true + + issues_1month: + parameters: SELECT WHERE request.parameters() ->> 'range' = '1month' + data: + - SELECT * FROM issues WHERE updated_this_month = true +``` + +The client passes the desired range as a [client parameter](/sync/rules/client-parameters): + +```javascript +await db.connect(connector, { + params: { + range: '1week', + }, +}) +``` + +This approach works well when you have a small, fixed set of time ranges. However, it requires schema changes and a scheduled job to keep the columns updated, and it is limited to pre-defined time ranges. + +If you need more flexibility, such as letting users pick arbitrary date ranges, see Workaround 2 below. + +### 2: Buckets Per Date + +Instead of pre-defined ranges, create a bucket for each date and let the client specify which dates to sync. + +Use `substring` to extract the date portion from a timestamp and match it with `=`: + +```yaml +bucket_definitions: + issues_by_update_at: + parameters: SELECT value AS date FROM json_each(request.parameters() ->> 'dates') + data: + - SELECT * FROM issues WHERE substring(updated_at, 1, 10) = bucket.date +``` + +The client passes the dates it wants as client parameters: + +```javascript +await db.connect(connector, { + params: { + dates: ["2026-01-07", "2026-01-08", "2026-01-09"], + }, +}) +``` + +This gives users full control over which dates to sync, with no schema changes or scheduled jobs required. + +The trade-off is granularity. In this example we're using daily buckets. If you need finer precision (hourly), syncing a large range means many buckets, which can degrade sync performance and approach [PowerSync's limit of 1,000 buckets per user](/resources/performance-and-limits#limits). If you use larger buckets (monthly), you lose the ability to filter accurately. + + +You must commit to a single granularity. Daily buckets mean too many buckets for long ranges. Monthly buckets lose precision for recent data. + + +If that is a problem, for example when you want hourly precision for recent data but do not want hundreds of buckets when syncing a full month, see Workaround 3 below. + +### 3: Multiple Granularities + +Combine multiple granularities in a single bucket definition. This lets you use larger buckets (days) for older data and smaller buckets (hours, minutes) for recent data. + +```yaml +bucket_definitions: + issues_by_time: + parameters: SELECT value AS partition FROM json_each(request.parameters() ->> 'partitions') + data: + # By day (e.g., "2026-01-07") + - SELECT * FROM issues WHERE substring(updated_at, 1, 10) = bucket.partition + # By hour (e.g., "2026-01-07T14") + - SELECT * FROM issues WHERE substring(updated_at, 1, 13) = bucket.partition + # By 10 minutes (e.g., "2026-01-07T14:3") + - SELECT * FROM issues WHERE substring(updated_at, 1, 15) = bucket.partition +``` + +The client then mixes granularities as needed: + +```javascript +await db.connect(connector, { + params: { + partitions: [ + "2026-01-05", + "2026-01-06", + "2026-01-07T10", + "2026-01-07T11", + "2026-01-07T12:0", + "2026-01-07T12:1", + "2026-01-07T12:2" + ] + }, +}) +``` + +Each data query acts as a filter based on the length of the partition value: a day-format partition only matches the day query, an hour-format partition only matches the hour query, and so on. + +This syncs January 5–6 by day, the morning of January 7 by hour, and the last 30 minutes in 10-minute chunks, without creating hundreds of buckets. + +The trade-off is complexity. The client must decide which granularity to use for each time segment, and each row belongs to multiple buckets, which increases replication overhead. + + +When using multiple time granularities (e.g., monthly, daily, hourly), rows move between buckets as time passes. Since each granularity creates a different bucket ID, the client must re-download the row from the new bucket even if it already has the data. This re-download overhead can nullify the benefits of granular filtering. For this reason, in some cases it may be better to sync entire months, avoiding the re-sync overhead, even if you sync more data initially. + + +## Conclusion + +Time-based sync is a common need, but PowerSync doesn't support range operators or time-based functions on parameters directly. To recap the workarounds: + +- **Pre-defined time ranges:** Simplest option. Use when you have a fixed set of time ranges and don't mind schema changes. +- **Buckets per date:** More flexible. Use when you need arbitrary date ranges but can live with a single granularity. +- **Multiple granularities:** Most flexible. Use when you need precision for recent data without syncing hundreds of buckets. Be mindful of the re-sync overhead. diff --git a/sync/rules/types.mdx b/sync/rules/types.mdx new file mode 100644 index 000000000..dc722ed1a --- /dev/null +++ b/sync/rules/types.mdx @@ -0,0 +1,16 @@ +--- +title: "Types" +sidebarTitle: "Type Mapping" +description: "How Postgres, MongoDB, MySQL, SQL Server and Convex types map to PowerSync's SQLite-based sync column definitions." +noindex: true +--- + +{/* Wrapper page: the content is snippets/sync-shared/types.mdx, which also renders at sync/types.mdx in the Sync Streams section. Edit the snippet, not this file. The deprecation callout stays here, outside the snippet. */} + +import TypeMapping from '/snippets/sync-shared/types.mdx'; + + +Sync Rules are deprecated. This page applies to both Sync Streams and Sync Rules unless a section says otherwise. + + + diff --git a/sync/streams/bucket-count.mdx b/sync/streams/bucket-count.mdx index 42a563bce..a7ded8057 100644 --- a/sync/streams/bucket-count.mdx +++ b/sync/streams/bucket-count.mdx @@ -318,10 +318,6 @@ This is why a checkpoint log can read `buckets: 7 | param_results: 6`. One globa The parameter limit can stop a sync while the bucket count still looks safe. A user can fail with far fewer than 1,000 buckets, because their parameter lookups returned more than 1,000 rows. Always check both numbers. - -In legacy [Sync Rules](/sync/rules/overview), these two limits were effectively one number, because each parameter-query result became one bucket. In Sync Streams they can diverge. - - ### Total Buckets vs Buckets Per User The 1,000 limit applies to each individual user, not to your whole instance. Your PowerSync Service can track millions of buckets in total, as long as each user syncs fewer than the limit. A large total bucket count is not a problem on its own. diff --git a/sync/streams/client-usage.mdx b/sync/streams/client-usage.mdx index 0765ba9c7..bd3477831 100644 --- a/sync/streams/client-usage.mdx +++ b/sync/streams/client-usage.mdx @@ -532,7 +532,7 @@ When different components subscribe to the same stream with the same parameters ## Connection Parameters -Connection parameters are a more advanced feature for values that apply to all streams in a session. They're the Sync Streams equivalent of [Client Parameters](/sync/rules/client-parameters) in legacy Sync Rules. +Connection parameters are a more advanced feature for values that apply to all streams in a session. For most use cases, **subscription parameters** (passed when subscribing) are more flexible and recommended. Use connection parameters only when you need a single global value across all streams, like an environment flag. diff --git a/sync/streams/examples.mdx b/sync/streams/examples.mdx index 172d3a2cb..991e91c78 100644 --- a/sync/streams/examples.mdx +++ b/sync/streams/examples.mdx @@ -233,7 +233,7 @@ streams: Clients auto-subscribe to all three streams when they connect. Each query joins through `board_subscriptions` to find relevant data: posts in the user's boards, comments on those posts, and other users sharing those boards. -Unlike with legacy [Sync Rules](/sync/rules/many-to-many-join-tables), you don't need to denormalize your schema or maintain array columns to express these relationships. At scale, denormalizing the scope key onto child tables can still be the best way to control how many buckets each user syncs. See [Bucket Count and Limits](/sync/streams/bucket-count). +You don't need to denormalize your schema or maintain array columns to express these relationships. At scale, denormalizing the scope key onto child tables can still be the best way to control how many buckets each user syncs. See [Bucket Count and Limits](/sync/streams/bucket-count). ## Use Case Examples diff --git a/sync/streams/overview.mdx b/sync/streams/overview.mdx index 5fbe3ba2a..4f3d808e5 100644 --- a/sync/streams/overview.mdx +++ b/sync/streams/overview.mdx @@ -8,12 +8,6 @@ With Sync Streams, you write SQL-like queries to define streams of data, and you For example, you might define a stream that syncs only the current user's to-do items, another for shared projects they have access to, and another for reference data that everyone needs. Your app subscribes to these streams on demand, and only that data syncs to the device. Apps that need all relevant data available upfront can set `auto_subscribe: true` so streams sync automatically when clients connect. - -**Are you still using Sync Rules?** Sync Streams support everything Sync Rules do, plus more expressive queries (including JOIN support), on-demand syncing, and a simpler developer experience (e.g. React hooks that manage subscriptions automatically). - -You can migrate in a few clicks. Click **Migrate to Sync Streams** in the PowerSync Dashboard, or run `powersync migrate sync-rules` in the CLI to generate a draft from your current config. See [Migrate to Sync Streams](/sync/rules/migrate-to-sync-streams) for details. - - ## How It Works Each PowerSync Service instance has a deployed Sync Streams configuration: a YAML file that defines the streams that exist. Each stream has a name and a SQL-like query that selects the tables and columns to sync, filters rows by static conditions or by parameters, and can rename or transform columns. The Service uses this configuration in two places: when it replicates data from your source database into buckets, and when it streams those buckets to clients. diff --git a/sync/streams/parameters.mdx b/sync/streams/parameters.mdx index fe3e90d41..aa88f7653 100644 --- a/sync/streams/parameters.mdx +++ b/sync/streams/parameters.mdx @@ -54,7 +54,7 @@ streams: ## Connection Parameters -Specified "globally" at the connection level, before any streams are subscribed. These are the equivalent of [Client Parameters](/sync/rules/client-parameters) in Sync Rules. Use them when you need a value that applies across all streams for the session. +Specified "globally" at the connection level, before any streams are subscribed. Use them when you need a value that applies across all streams for the session. ```yaml streams: diff --git a/sync/streams/prioritized-sync.mdx b/sync/streams/prioritized-sync.mdx index 064c1dd56..15e73407b 100644 --- a/sync/streams/prioritized-sync.mdx +++ b/sync/streams/prioritized-sync.mdx @@ -3,11 +3,13 @@ title: "Prioritized Sync" description: "Prioritize which tables sync first so users can start working immediately while remaining data continues loading in the background." --- +{/* Split page: the Sync Rules (Legacy) version of this page is sync/rules/prioritized-sync.mdx. That page is deprecated and frozen, but when you fix an error here, check whether it needs the same fix. Do not mention Sync Rules on this page. */} + ## Overview PowerSync supports defining sync priorities, which allows you to control the sync order for different data. This is particularly useful when certain data should be available sooner than others. -In Sync Streams, priorities are assigned to streams and PowerSync manages the underlying buckets internally. (In legacy Sync Rules, priorities were assigned to buckets explicitly.) +In Sync Streams, priorities are assigned to streams and PowerSync manages the underlying buckets internally. ## Why Use Sync Priorities? @@ -19,7 +21,7 @@ While this guarantees consistency, it can lead to delays, especially for large d ## How It Works -Each bucket is assigned a priority value between 0 and 3, where: +Each stream is assigned a priority value between 0 and 3, where: - 0 is the highest priority and has special behavior (detailed below). - 3 is the default and lowest priority. @@ -27,57 +29,9 @@ Each bucket is assigned a priority value between 0 and 3, where: Higher-priority data syncs first, and lower-priority data syncs later. If you only use a single priority, there is no difference between priorities 1-3. The difference only comes in when you use multiple different priorities. - - -In Sync Streams, you assign priorities directly to streams. PowerSync manages buckets internally, so you don't need to think about bucket structure. Each stream with a given priority will have its data synced at that priority level. - -```yaml -streams: - lists: - auto_subscribe: true - query: SELECT * FROM lists WHERE owner_id = auth.user_id() - priority: 1 # Syncs first - - todos: - auto_subscribe: true - query: SELECT * FROM todos WHERE list_id IN (SELECT id FROM lists WHERE owner_id = auth.user_id()) - priority: 2 # Syncs after lists -``` - -Clients can also override the priority when subscribing: - -```js -// Override the stream's default priority for this subscription -const sub = await db.syncStream('todos', { list_id: 'abc' }).subscribe({ priority: 1 }); -``` - -When different components subscribe to the same stream with the same parameters but different priorities, PowerSync uses the highest priority for syncing. That higher priority is kept until the subscription ends (or its TTL expires). Subscriptions with different parameters are independent and do not conflict. - - -In Sync Rules, you assign priorities to bucket definitions. The priority determines when data in that bucket syncs relative to other buckets. - -```yaml -bucket_definitions: - user_lists: - priority: 1 # Syncs first - parameters: SELECT id AS list_id FROM lists WHERE user_id = request.user_id() - data: - - SELECT * FROM lists WHERE id = bucket.list_id - - user_todos: - priority: 2 # Syncs after lists - parameters: SELECT id AS list_id FROM lists WHERE user_id = request.user_id() - data: - - SELECT * FROM todos WHERE list_id = bucket.list_id -``` - - - ## Syntax and Configuration - - -In Sync Streams, set the `priority` option on the stream definition: +Set the `priority` option on the stream definition: ```yaml streams: @@ -91,38 +45,24 @@ streams: query: SELECT * FROM background_table WHERE user_id = auth.user_id() priority: 2 ``` - - -In Sync Rules, priorities can be defined using the `priority` YAML key on bucket definitions, or with the `_priority` attribute inside parameter queries: -```yaml -bucket_definitions: - # Using the `priority` YAML key - user_data: - priority: 1 - parameters: SELECT request.user_id() AS id WHERE ... - data: - # ... - - # Using the `_priority` attribute (useful for multiple parameter queries with different priorities) - project_data: - parameters: SELECT id AS project_id, 2 AS _priority FROM projects WHERE ... - data: - # ... +Clients can also override the priority when subscribing: + +```js +// Override the stream's default priority for this subscription +const sub = await db.syncStream('todos', { list_id: 'abc' }).subscribe({ priority: 1 }); ``` - - + +When different components subscribe to the same stream with the same parameters but different priorities, PowerSync uses the highest priority for syncing. That higher priority is kept until the subscription ends (or its TTL expires). Subscriptions with different parameters are independent and do not conflict. -Priorities must be static and cannot depend on row values within a parameter query. +Priorities are static values that you set in the stream definition or when subscribing. They cannot depend on row values. ## Example: Syncing Lists Before Todos Consider a scenario where you want to display lists immediately while loading todos in the background. This approach allows users to view and interact with lists right away without waiting for todos to sync. - - ```yaml config: edition: 3 @@ -142,38 +82,17 @@ streams: ``` The `lists` stream syncs first (priority 1), allowing users to see and interact with their lists immediately. The `todos` stream syncs afterward (priority 2), loading in the background. - - -```yaml -bucket_definitions: - user_lists: - priority: 1 # Syncs first - parameters: SELECT id AS list_id FROM lists WHERE user_id = request.user_id() - data: - - SELECT * FROM lists WHERE id = bucket.list_id - - user_todos: - priority: 2 # Syncs after lists - parameters: SELECT id AS list_id FROM lists WHERE user_id = request.user_id() - data: - - SELECT * FROM todos WHERE list_id = bucket.list_id -``` - -The `user_lists` bucket syncs first (priority 1), allowing users to see and interact with their lists immediately. The `user_todos` bucket syncs afterward (priority 2), loading in the background. - - - ## Behavioral Considerations -- **Interruption for Higher Priority Data**: Syncing lower-priority data _may_ be interrupted if new data for higher-priority streams/buckets arrives. +- **Interruption for Higher Priority Data**: Syncing lower-priority data _may_ be interrupted if new data for higher-priority streams arrives. - **Local Changes & Consistency**: If local writes fail due to validation or permission issues, they are only reverted after _all_ data has synced. - **Deleted Data**: Deleted data may only be removed after _all_ priorities have completed syncing. Future updates may improve this behavior. - **Data Ordering**: Lower-priority data will never appear before higher-priority data. ## Special Case: Priority 0 -Priority 0 buckets sync regardless of pending uploads. +Priority 0 streams sync regardless of pending uploads. For example, in a collaborative document editing app (e.g., using Yjs), each change is stored as a separate row. Since out-of-order updates don’t affect document integrity, Priority 0 can ensure immediate availability of updates. diff --git a/sync/streams/quickstart.mdx b/sync/streams/quickstart.mdx index c8ae8cd68..36a84d050 100644 --- a/sync/streams/quickstart.mdx +++ b/sync/streams/quickstart.mdx @@ -129,7 +129,6 @@ const sub = await db.syncStream('list_todos', { list_id: 'abc123' }).subscribe() Set `auto_subscribe: true` to sync data automatically when clients connect. This is useful for: - Reference data that all users need, or that is needed in many screens in the app. - User data that should always be available offline. -- Keeping the "sync everything upfront" behavior of legacy [Sync Rules](/sync/rules/overview) when migrating to Sync Streams. ```yaml config: diff --git a/sync/supported-sql.mdx b/sync/supported-sql.mdx index 9287c428c..550e76ab6 100644 --- a/sync/supported-sql.mdx +++ b/sync/supported-sql.mdx @@ -1,11 +1,13 @@ --- title: "Supported SQL" -description: "Reference for SQL syntax, operators, built-in functions, and type casting supported in Sync Streams/Sync Rules queries." +description: "Reference for SQL syntax, operators, built-in functions, and type casting supported in Sync Streams queries." --- -This guide explains the SQL supported in [Sync Streams](/sync/streams/overview) and [Sync Rules (legacy)](/sync/rules/overview): what you can write, with examples and restrictions. +{/* Split page: the Sync Rules (Legacy) version of this page is sync/rules/supported-sql.mdx. That page is deprecated and frozen, but when you fix an error here, check whether it needs the same fix. Do not mention Sync Rules on this page. */} -For the exact syntax the compiler accepts — railroad diagrams and grammar-rule references — see the [Sync Streams](/sync/grammar/sync-streams/index) or [Sync Rules](/sync/grammar/sync-rules/index) grammar reference. +This guide explains the SQL supported in [Sync Streams](/sync/streams/overview): what you can write, with examples and restrictions. + +For the exact syntax the compiler accepts, with railroad diagrams and grammar-rule references, see the [Sync Streams grammar reference](/sync/grammar/sync-streams/index). Some fundamental restrictions on the usage of SQL expressions are: @@ -19,36 +21,22 @@ For the exact syntax the compiler accepts — railroad diagrams and grammar-rule ## Query Syntax -The supported SQL is based on a subset of the standard SQL syntax. Sync Streams support more SQL features than the legacy Sync Rules. - - - - - `SELECT` with column selection and [`WHERE` filtering](#filtering-where-clause) - - [Subqueries](/sync/streams/queries#using-subqueries) with `IN (SELECT ...)` and nested subqueries - - [`INNER JOIN`](#join-syntax) (selected columns must come from a single table) - - [Common Table Expressions (CTEs)](#cte-and-with-syntax) via the `with:` block - - Multiple queries per stream via `queries:` - - Table-valued functions such as `json_each()` for [expanding arrays](/sync/streams/parameters#expanding-json-arrays) - - `BETWEEN` and `CASE` expressions - - A limited set of [operators](#operators) and [functions](#functions) - - **Not supported**: aggregation, sorting, or set operations (`GROUP BY`, `ORDER BY`, `LIMIT`, `UNION`, etc.). See [Writing Queries](/sync/streams/queries) for details. - - - - Simple `SELECT` with column selection - - `WHERE` filtering on parameters (see [Filtering: WHERE Clause](#filtering-where-clause)) - - A limited set of [operators](#operators) and [functions](#functions) - - **Not supported**: subqueries, JOINs, CTEs, aggregation, sorting, or set operations (`GROUP BY`, `ORDER BY`, `LIMIT`, `UNION`, etc.). - - +The supported SQL is based on a subset of the standard SQL syntax: -## Filtering: WHERE Clause +- `SELECT` with column selection and [`WHERE` filtering](#filtering-where-clause) +- [Subqueries](/sync/streams/queries#using-subqueries) with `IN (SELECT ...)` and nested subqueries +- [`INNER JOIN`](#join-syntax) (selected columns must come from a single table) +- [Common Table Expressions (CTEs)](#cte-and-with-syntax) via the `with:` block +- Multiple queries per stream via `queries:` +- Table-valued functions such as `json_each()` for [expanding arrays](/sync/streams/parameters#expanding-json-arrays) +- `BETWEEN` and `CASE` expressions +- A limited set of [operators](#operators) and [functions](#functions) -Sync queries support a subset of SQL `WHERE` syntax. Allowed operators and combinations differ between Sync Streams and Sync Rules, and are more restrictive than standard SQL. +**Not supported**: aggregation, sorting, or set operations (`GROUP BY`, `ORDER BY`, `LIMIT`, `UNION`, etc.). See [Writing Queries](/sync/streams/queries) for details. - - +## Filtering: WHERE Clause + +Sync Streams queries support a subset of SQL `WHERE` syntax. Allowed operators and combinations are more restrictive than standard SQL. **`=` and `IS NULL`** — Compare a row column to a static value, a parameter, or another column: @@ -109,59 +97,6 @@ WHERE category NOT IN ROW('draft', 'hidden') -- WHERE id NOT IN subscription.parameter('excluded_ids') ``` - - - -**`=` and `IS NULL`** — Compare a row column to a static value or a bucket parameter: - -```sql --- Static value -WHERE status = 'active' -WHERE deleted_at IS NULL - --- Bucket parameter -WHERE owner_id = bucket.user_id -``` - -**`AND`** — Supported in both Parameter Queries and Data Queries. In Parameter Queries, each condition may match a different parameter. However, you cannot combine two `IN` expressions on parameters in the same `AND`; split them into separate Parameter Queries instead. - -```sql --- Supported: parameter condition + row-value condition -WHERE users.id = request.user_id() - AND users.is_admin = true - --- Not supported: two IN expressions on parameters in the same AND --- WHERE bucket.list_id IN lists.allowed_ids --- AND bucket.org_id IN lists.allowed_org_ids -``` - -**`OR`** — Supported when both sides of the `OR` reference the exact same set of parameters. If the two sides use different parameters, use separate parameter queries instead. - -```sql --- Supported: both sides reference the same parameter -WHERE lists.owner_id = request.user_id() - OR lists.shared_with = request.user_id() - --- Not supported: sides reference different parameters --- WHERE lists.owner_id = request.user_id() --- OR lists.org_id = bucket.org_id -``` - -**`NOT`** — Supported for simple row-value conditions. Not supported on parameter-matching expressions. - -```sql --- Supported -WHERE status != 'archived' -WHERE deleted_at IS NOT NULL -WHERE NOT users.is_admin = true - --- Not supported in parameter queries --- WHERE NOT users.id = request.user_id() -``` - - - - ## Operators Operators can be used in `WHERE` clauses and in `SELECT` expressions. When filtering on parameters (e.g. `auth.user_id()`, `subscription.parameter('id')`), some combinations are restricted — see [Filtering: WHERE Clause](#filtering-where-clause). @@ -183,15 +118,12 @@ Operators can be used in `WHERE` clauses and in `SELECT` expressions. When filte - `json ->> 'path'` — Returns the extracted value. - - **Sync Streams:** `left IN right` — `left` can be a row column and `right` a parameter array (e.g. `id IN subscription.parameter('ids')`), or `left` a parameter and `right` a row JSON array column. Also supports subqueries: `id IN (SELECT ...)`. - - **Sync Rules:** Returns true if `left` is in the `right` JSON array. In Data Queries, `left` must be a row column and `right` cannot be a bucket parameter. In Parameter Queries, either side may be a parameter. + - `left IN right` — `left` can be a row column and `right` a parameter array (e.g. `id IN subscription.parameter('ids')`), or `left` a parameter and `right` a row JSON array column. Also supports subqueries: `id IN (SELECT ...)`. - `x BETWEEN a AND b`, `x NOT BETWEEN a AND b` — True if `x` is in the inclusive range `[a, b]`. Usable in `WHERE` or as a `SELECT` expression. If any operand is `null`, the result is `null`. Example: `WHERE price BETWEEN 10 AND 100` - - Supported in Sync Streams only. Not available in Sync Rules. - ` && ` — True if the JSON array in `left` and the set `right` share at least one value. Use when the row stores an array (e.g. a `tagged_users` column). `left` must be a row column (JSON array); `right` must be a subquery or parameter array. @@ -199,8 +131,6 @@ Operators can be used in `WHERE` clauses and in `SELECT` expressions. When filte Example: `WHERE tagged_users && (SELECT id FROM org_members WHERE org_id = auth.parameter('org_id'))` Use `IN` when the row has a single value to check against a set; use `&&` when the row has an array and you want to match any element. - - Supported in Sync Streams only. Not available in Sync Rules. @@ -225,13 +155,11 @@ Most functions are from [SQLite built-in functions](https://www.sqlite.org/lang_ - **[typeof(data)](https://www.sqlite.org/lang_corefunc.html#typeof)** — Returns `text`, `integer`, `real`, `blob`, or `null`. - - **[json_each(data)](https://www.sqlite.org/json1.html#jeach)** — Expands a JSON array into rows. - - **Sync Streams:** Works with auth and connection parameters (e.g. `JOIN json_each(auth.parameter('ids')) AS t` or `WHERE id IN (SELECT value FROM json_each(auth.parameter('ids')))`). Can also be used with columns from joined tables in some cases (e.g. `SELECT * FROM lists WHERE id IN (SELECT lists.value FROM access_control a, json_each(a.allowed_lists) as lists WHERE a.user = auth.user_id())`). See [Expanding JSON arrays](/sync/streams/parameters#expanding-json-arrays). - - **Sync Rules:** Expands a JSON array or object from a request or token parameter into a set of parameter rows. Example: `SELECT value AS project_id FROM json_each(request.jwt() -> 'project_ids')`. + - **[json_each(data)](https://www.sqlite.org/json1.html#jeach)** — Expands a JSON array into rows. Works with auth and connection parameters (e.g. `JOIN json_each(auth.parameter('ids')) AS t` or `WHERE id IN (SELECT value FROM json_each(auth.parameter('ids')))`). Can also be used with columns from joined tables in some cases (e.g. `SELECT * FROM lists WHERE id IN (SELECT lists.value FROM access_control a, json_each(a.allowed_lists) as lists WHERE a.user = auth.user_id())`). See [Expanding JSON arrays](/sync/streams/parameters#expanding-json-arrays). - **[json_extract(data, path)](https://www.sqlite.org/json1.html#jex)** — Same as `->>` operator, but the path must start with `$.` - **[json_array_length(data)](https://www.sqlite.org/json1.html#jarraylen)** — Given a JSON array (as text), returns the length of the array. If data is null, returns null. If the value is not a JSON array, returns 0. - **[json_valid(data)](https://www.sqlite.org/json1.html#jvalid)** — Returns 1 if the data can be parsed as JSON, 0 otherwise. - - **json_keys(data)** — Returns the set of keys of a JSON object as a JSON array. Example: `SELECT * FROM items WHERE bucket.user_id IN json_keys(permissions_json)`. + - **json_keys(data)** — Returns the set of keys of a JSON object as a JSON array. Example: `SELECT id, json_keys(permissions_json) AS permission_keys FROM items`. - **[ifnull(x, y)](https://www.sqlite.org/lang_corefunc.html#ifnull)** — Returns x if non-null, otherwise returns y. @@ -251,7 +179,7 @@ Most functions are from [SQLite built-in functions](https://www.sqlite.org/lang_ - **table_name()** — Returns the name of the table the row was replicated from. This is the source table's name, not the alias or the output table name. - **table_suffix()** — Returns the part of the table name matched by the trailing `%` of a [wildcard table name](/sync/advanced/partitioned-tables). For example, with `FROM "todos_%" AS todos`, `todos.table_suffix()` returns `2024` for rows from the `todos_2024` table. On tables without a wildcard name, the result is always empty and the compiler reports a warning. - Supported in Sync Streams only, with PowerSync Service v1.24.0 or later. + Requires PowerSync Service v1.24.0 or later. - **[ST_AsGeoJSON(geometry)](/client-sdks/advanced/gis-data-postgis)** — Convert [PostGIS](/client-sdks/advanced/gis-data-postgis) (in Postgres) geometry from WKB to GeoJSON. Combine with JSON operators to extract specific fields. @@ -265,8 +193,6 @@ If you need an operator or function not listed, [contact us](/resources/contact- ## JOIN Syntax -Supported in Sync Streams only. Not available in Sync Rules. - Sync Streams support a subset of join syntax. The following rules define what is valid: - **Only inner joins:** Use `JOIN` or `INNER JOIN`. `LEFT`, `RIGHT`, and `OUTER` joins are not supported. @@ -295,8 +221,6 @@ For how to use JOINs in your stream queries (when to use them, patterns, and exa ## CTE and WITH Syntax -Supported in Sync Streams only. Not available in Sync Rules. - Common Table Expressions (CTEs) can be defined in a `with:` block **inside a stream** (stream-level, scoped to that stream) or at the **top level** of the Sync Config (global, shared across all streams). Each CTE is a name and a single `SELECT` query. The following rules apply: - **Stream-level CTEs take precedence over global CTEs.** If a stream defines a CTE with the same name as a global CTE, the stream-level definition is used within that stream. @@ -341,8 +265,6 @@ For how to use CTEs, see [Common Table Expressions (CTEs)](/sync/streams/ctes). ## CASE Expressions -Supported in Sync Streams only. Not available in Sync Rules. - `CASE` is allowed anywhere an expression is allowed — in `SELECT` columns or `WHERE` clauses. **Searched CASE** — Each `WHEN` is an independent boolean condition: diff --git a/sync/types.mdx b/sync/types.mdx index 0e69f3ce0..eae7205d2 100644 --- a/sync/types.mdx +++ b/sync/types.mdx @@ -4,165 +4,8 @@ sidebarTitle: "Type Mapping" description: "How Postgres, MongoDB, MySQL, SQL Server and Convex types map to PowerSync's SQLite-based sync column definitions." --- -import BinaryType from '/snippets/binary-type.mdx'; +{/* Wrapper page: the content is snippets/sync-shared/types.mdx, which also renders at sync/rules/types.mdx in the Sync Rules (Legacy) section. Edit the snippet, not this file. */} -The supported client-side SQLite types are: +import TypeMapping from '/snippets/sync-shared/types.mdx'; -1. `null` -2. `integer`: a 64-bit signed integer -3. `real`: a 64-bit floating point number -4. `text`: A UTF-8 text string -5. `blob`: Binary data - - -## Postgres Type Mapping - -Postgres types are mapped to SQLite types as follows: - -| Postgres Data Type | PowerSync / SQLite Column Type | Notes | -|--------------------|--------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `text`, `varchar` | `text` | | -| `int2`, `int4`, `int8` | `integer` | | -| `numeric` / `decimal` | `text` | These types have arbitrary precision in Postgres, so can only be represented accurately as text in SQLite | -| `bool` | `integer` | `1` for true, `0` for false. There is no dedicated boolean data type in SQLite. | -| `float4`, `float8` | `real` | | -| `enum` | `text` | | -| `uuid` | `text` | | -| `timestamptz` | `text` | Format: `YYYY-MM-DD hh:mm:ss.sssZ`. This is compatible with ISO8601 and SQLite's functions. Precision matches the precision used in Postgres. `-infinity` becomes `0000-01-01 00:00:00Z` and `infinity` becomes `9999-12-31 23:59:59Z`. | -| `timestamp` | `text` | Format: `YYYY-MM-DD hh:mm:ss.sss`. In most cases, timestamptz should be used instead. `-infinity` becomes `0000-01-01 00:00:00` and `infinity` becomes `9999-12-31 23:59:59`. | -| `date`, `time` | `text` | | -| `json`, `jsonb` | `text` | `json` and `jsonb` values are treated as `text` values in their serialized representation. [JSON functions and operators](/sync/supported-sql#operators) operate directly on these `text` values. | -| `interval` | `text` | | -| `macaddr` | `text` | | -| `inet` | `text` | | -| `bytea` | `blob` | Cannot sync directly to client — convert to hex or base64 first. See [Operators & Functions](/sync/supported-sql). | -| `geometry` (PostGIS) | `text` | Hex string of the binary data. Use the [ST functions](/sync/supported-sql#functions) to convert to other formats | -| Arrays | `text` | JSON array. | -| `DOMAIN` types | `text` / depends | Depending on [compatibility options](/sync/advanced/compatibility#custom_postgres_types), inner type or raw wire representation (legacy). | -| Custom types | `text` | Depending on [compatibility options](/sync/advanced/compatibility#custom_postgres_types), JSON object or raw wire representation (legacy). | -| (Multi-)ranges | `text` | Depending on [compatibility options](/sync/advanced/compatibility#custom_postgres_types), JSON object (array for multi-ranges) or raw wire representation (legacy). | - - - - -## Convex Type Mapping - - - The Convex replicator is currently released as an [experimental feature](/resources/feature-status). APIs and - behavior may change, and we can't yet guarantee continued support or long-term stability. - - -Convex values are mapped to SQLite types as follows: - -| Convex Type | TS/JS Type | PowerSync / SQLite Column Type | Notes | -| ----------- | ---------------- | ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `Id` | `string` | `text` | Convex document IDs are exposed as `_id` and can be synced as `text`. For synced client tables, use client-side ID mapping with a stable UUID column as `id` instead of relying on Convex-generated `_id` values. | -| `Null` | `null` | `null` | | -| `Int64` | `base-10 string` | `text` | Cast to `INTEGER` in Sync Streams when you want to sync the value as a SQLite integer. | -| `Float64` | `number` | `real` | | -| `Boolean` | `boolean` | `integer` | `1` for true, `0` for false. There is no dedicated boolean data type in SQLite. | -| `String` | `string` | `text` | | -| `Bytes` | `base64 string` | `text` | Decode from base64 in your app if you need binary data on the client. | -| `Array` | `Array` | `text` | Converted to a JSON string. | -| `Object` | `Object` | `text` | Converted to a JSON string. | -| `Record` | `Record` | `text` | Converted to a JSON string. | - -- Convex documents are converted to a flat list of columns, one column per top-level field. -- Nested objects and arrays are converted to JSON, and [JSON functions and operators](/sync/supported-sql#operators) can be used to query them in Sync Streams or on the client-side SQLite database. -- Cast Convex `Int64` fields to `INTEGER` in Sync Streams when you want SQLite integer values on the client, for example `CAST(an_int64_column AS INTEGER) AS an_int64_column`. - - -## MongoDB Type Mapping - -MongoDB types are mapped to SQLite types as follows: - -| BSON Type | PowerSync / SQLite Column Type | Notes | -|--------------------|--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------| -| `String` | `text` | | -| `Int`, `Long` | `integer` | | -| `Double` | `real` | | -| `Decimal128` | `text` | | -| `Object` | `text` | Converted to a JSON string | -| `Array` | `text` | Converted to a JSON string | -| `ObjectId` | `text` | Lower-case hex string | -| `UUID` | `text` | Lower-case hex string | -| `Boolean` | `integer` | `1` for true, `0` for false. There is no dedicated boolean data type in SQLite. | -| `Date` | `text` | Format: `YYYY-MM-DD hh:mm:ss.sssZ` | -| `Null` | `null` | | -| `Binary` | `blob` | Cannot sync directly to client — convert to hex or base64 first. See [Operators & Functions](/sync/supported-sql). | -| Regular Expression | `text` | JSON text in the format `{"pattern":"...","options":"..."}` | -| `Timestamp` | `integer` | Converted to a 64-bit integer | -| `Undefined` | `null` | | -| `DBPointer` | `text` | JSON text in the format `{"collection":"...","oid":"...","db":"...","fields":...}` | -| `JavaScript` | `text` | JSON text in the format `{"code": "...", "scope": ...}` | -| `Symbol` | `text` | | -| `MinKey`, `MaxKey` | `null` | | - -* Data is converted to a flat list of columns, one column per top-level field in the MongoDB document. -* Special BSON types are converted to plain SQLite alternatives. For example, `ObjectId`, `Date`, `UUID` are all converted to a plain `TEXT` column. -* Nested objects and arrays are converted to JSON, and [JSON functions and operators](/sync/supported-sql#operators) can be used to query them (in the Sync Streams / Sync Rules and/or on the client-side SQLite statements). -* Binary data nested in objects or arrays is not supported. - - - - -## MySQL Type Mapping - -MySQL support is currently in a [Beta release](/resources/feature-status). - -MySQL types are mapped to SQLite types as follows: - -| MySQL Data Type | PowerSync / SQLite Column Type | Notes | -|----------------------------------------------------|--------------------------------|-----------------------------------------------------------------------------------| -| `tinyint`, `smallint`, `mediumint`, `bigint`, `integer`, `int` | `integer` | | -| `numeric`, `decimal` | `text` | | -| `bool`, `boolean` | `integer` | `1` for true, `0` for false. There is no dedicated boolean data type in SQLite. | -| `float`, `double`, `real` | `real` | | -| `enum` | `text` | | -| `set` | `text` | Converted to JSON array | -| `char`, `varchar` | `text` | | -| `tinytext`, `text`, `mediumtext`, `longtext` | `text` | | -| `timestamp` | `text` | ISO 8601 format: `YYYY-MM-DDTHH:mm:ss.sssZ` | -| `date` | `text` | Format: `YYYY-MM-DD` | -| `time`, `datetime` | `text` | ISO 8601 format: `YYYY-MM-DDTHH:mm:ss.sssZ` | -| `year` | `text` | | -| `json` | `text` | There is no dedicated JSON type in SQLite — JSON functions operate directly on text values. | -| `bit` | `blob` | * See note below regarding syncing binary types | -| `binary`, `varbinary` | `blob` | | -| `image` | `blob` | | -| `geometry`, `geometrycollection` | `blob` | | -| `point`, `multipoint` | `blob` | | -| `linestring`, `multilinestring` | `blob` | | -| `polygon`, `multipolygon` | `blob` | | - - - - -## SQL Server Type Mapping - -SQL Server support is currently in a [Beta release](/resources/feature-status). - -SQL Server types are mapped to SQLite types as follows: - -| SQL Server Data Type | PowerSync / SQLite Column Type | Notes | -|----------------------------------------------------|--------------------------------|--------------------------------------------------------| -| `tinyint`, `smallint`, `int`, `bigint` | `integer` | | -| `numeric`, `decimal` | `text` | Numeric string | -| `float`, `real` | `real` | | -| `bit` | `integer` | | -| `money`, `smallmoney` | `text` | Numeric string | -| `xml` | `text` | | -| `char`, `nchar`, `ntext` | `text` | | -| `varchar`, `nvarchar`, `text` | `text` | | -| `uniqueidentifier` | `text` | | -| `timestamp` | `text` | ISO 8601 format: `YYYY-MM-DDTHH:mm:ss.sssZ` | -| `date` | `text` | Format: `YYYY-MM-DD` | -| `time` | `text` | Format: `HH:mm:ss.sss` | -| `datetime`, `datetime2`, `smalldatetime`, `datetimeoffset` | `text` | ISO 8601 format: `YYYY-MM-DDTHH:mm:ss.sssZ` | -| `json` | `text` | Only exists for Azure SQL Database and SQL Server 2025 | -| `geometry`, `geography` | `text` | `text` of JSON object describing the spatial data type | -| `binary`, `varbinary`, `image` | `blob` | * See note below regarding binary types | -| `rowversion`, `timestamp` | `blob` | * See note below regarding binary types | -| User Defined Types: `hiearchyid` | `blob` | * See note below regarding binary types | - - + From 7c88c482492beedd64e080e642ad1d4947020228 Mon Sep 17 00:00:00 2001 From: Benita Volkmann Date: Fri, 18 Sep 2026 18:28:11 +0200 Subject: [PATCH 2/3] Simplify the check:links architecture --- .claude/CLAUDE.md | 3 +-- .claude/commands/lint-docs.md | 8 ++++---- package.json | 3 +-- scripts/check-links.mjs | 10 ++++++++++ 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index ca29c2dab..ec4279b40 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -153,8 +153,7 @@ Sync Rules are deprecated. New documentation and updates cover Sync Streams only - Verify technical claims and run code examples before publication. Select other checks appropriate to the change. - Run `vale ` for changed MDX pages. Add new technical terms to `.github/vale/config/vocabularies/PowerSync/accept.txt`; do not add ordinary misspellings. -- After link or navigation changes, run `npx mintlify broken-links`. Mintlify requires Node 20.17–24; if needed, use `PATH="/opt/homebrew/opt/node@24/bin:$PATH" npx mintlify broken-links`. -- For anchor and snippet checks, use `pnpm check:links`. It wraps the Mintlify anchor check in `scripts/check-links.mjs` so that anchors defined in imported snippets resolve. Validate repository instruction links as file paths, since the site checker does not cover all of them. +- After link or navigation changes, run `pnpm check:links`. It runs the Mintlify path, anchor, and snippet checks through `scripts/check-links.mjs`, resolves anchors defined in imported snippets, and is the same check CI runs. It tells you if your Node version is unsupported. Validate repository instruction links as file paths, since the site checker does not cover them. - Use [the lint command](commands/lint-docs.md) for the check workflow and [the reviewer](agents/document-reviewer.md) for editorial review. Passing linters does not establish technical accuracy or style compliance. ## Git Workflow diff --git a/.claude/commands/lint-docs.md b/.claude/commands/lint-docs.md index e0ac914c1..ca9027388 100644 --- a/.claude/commands/lint-docs.md +++ b/.claude/commands/lint-docs.md @@ -1,9 +1,9 @@ --- -allowed-tools: Read, Bash(npx mintlify *), Bash(PATH=* npx mintlify *), Bash(vale *), Bash(git diff *), Bash(git status *) -description: Run Vale and Mintlify broken-link checks for documentation changes and report failures. +allowed-tools: Read, Bash(pnpm check:links), Bash(PATH=* pnpm check:links), Bash(vale *), Bash(git diff *), Bash(git status *) +description: Run Vale and the link check for documentation changes and report failures. --- -1. Read the canonical [Verification](../CLAUDE.md#verification) section for commands, supported Node versions, and vocabulary rules. +1. Read the canonical [Verification](../CLAUDE.md#verification) section for commands and vocabulary rules. 2. Use the requested file scope. Otherwise, identify changed MDX pages with `git diff main --name-only --diff-filter=ACMR -- '*.mdx'` and `git status --short`, including untracked pages. -3. Run `vale ` for each page and `npx mintlify broken-links` for the site. Use the canonical Node fallback if needed. +3. Run `vale ` for each page and `pnpm check:links` once for the site. 4. Report findings by file, suggested fixes, and totals for errors, warnings, and suggestions. Report failed or unavailable checks separately from content findings. diff --git a/package.json b/package.json index 490f97ce2..3bd14d0be 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,7 @@ "packageManager": "pnpm@11.3.0+sha512.2c403d6594527287672b1f7056343a1f7c3634036a67ffabfcc2b3d7595d843768f8787148d1b57cf7956c90606bbd192857c363af19e96d2d0ec9ec5741d215", "scripts": { "dev": "mintlify dev", - "check:links": "node scripts/check-links.mjs", - "check:links:mintlify": "mintlify broken-links --check-anchors --check-snippets" + "check:links": "node scripts/check-links.mjs" }, "devDependencies": { "mintlify": "^4.2.520" diff --git a/scripts/check-links.mjs b/scripts/check-links.mjs index a2fa6c6c8..e6f939e84 100644 --- a/scripts/check-links.mjs +++ b/scripts/check-links.mjs @@ -16,6 +16,16 @@ import { spawnSync } from 'node:child_process'; import { existsSync, readFileSync } from 'node:fs'; import path from 'node:path'; +const nodeMajor = Number(process.versions.node.split('.')[0]); +if (nodeMajor < 20 || nodeMajor > 24) { + console.error( + `The Mintlify CLI supports Node 20.17 to 24, but this is Node ${process.versions.node}.\n` + + 'Run `nvm use` (the repo pins Node 24 in .nvmrc) or prefix the command with ' + + 'PATH="/opt/homebrew/opt/node@24/bin:$PATH".', + ); + process.exit(1); +} + const root = process.cwd(); const localBin = path.join(root, 'node_modules', '.bin', 'mintlify'); const bin = existsSync(localBin) ? localBin : 'mintlify'; From c65a7e9aa635759455e9cc357b2636f2b65c8371 Mon Sep 17 00:00:00 2001 From: Benita Volkmann Date: Fri, 18 Sep 2026 18:29:28 +0200 Subject: [PATCH 3/3] Sync Rules docs aren't frozen per se --- .claude/CLAUDE.md | 4 ++-- .claude/agents/document-reviewer.md | 2 +- .claude/skills/doc-author/SKILL.md | 2 +- .claude/skills/pr-to-docs/SKILL.md | 2 +- sync/advanced/multiple-client-versions.mdx | 2 +- sync/advanced/partitioned-tables.mdx | 2 +- sync/advanced/sync-data-by-time.mdx | 2 +- sync/rules/multiple-client-versions.mdx | 4 ++-- sync/rules/partitioned-tables.mdx | 4 ++-- sync/rules/prioritized-sync.mdx | 4 ++-- sync/rules/supported-sql.mdx | 4 ++-- sync/rules/sync-data-by-time.mdx | 4 ++-- sync/streams/prioritized-sync.mdx | 2 +- sync/supported-sql.mdx | 2 +- 14 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index ec4279b40..a980780d3 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -142,9 +142,9 @@ Update `docs.json` when adding, moving, or removing pages. Add redirects for mov ## Sync Streams and Sync Rules -Sync Rules are deprecated. New documentation and updates cover Sync Streams only. Many customers still use Sync Rules, so their documentation stays online but frozen. +Sync Rules are deprecated, but many customers still run them. New documentation covers Sync Streams. Keep the Sync Rules docs accurate, and add to them when that makes them more accurate or helpful, without prioritizing that work. The goal is to avoid noisy Sync Rules references outside their own section, not to stop maintaining it. -- **Sync Rules docs are frozen.** `sync/rules/` and `sync/grammar/sync-rules/` take error fixes only: no new features, examples, or pages. Each page opens with an `` callout that starts "Sync Rules are deprecated." and links to its Sync Streams version. The sidebar group stays "Sync Rules (Legacy)". +- **Sync Rules section.** `sync/rules/` and `sync/grammar/sync-rules/` hold all Sync Rules content. Each page opens with an `` callout that starts "Sync Rules are deprecated." and links to its Sync Streams version. The sidebar group stays "Sync Rules (Legacy)". - **Keep the engines apart.** Never place Sync Streams and Sync Rules content side by side: no engine tabs, no "(or legacy Sync Rules)" asides, no pointers to a Sync Rules equivalent. Outside `sync/rules/`, mention Sync Rules only to state a behavior difference that affects Sync Rules users, and remove other mentions when you edit a page. - **Shared pages.** A page that applies to both engines keeps one body in `snippets/sync-shared/.mdx`, imported by a Sync Streams wrapper at the original path and a Sync Rules wrapper at `sync/rules/.mdx` that adds the callout and `noindex: true`. Edit the snippet, not the wrappers, and keep it valid for both engines. - **Counterpart comments.** Every split twin, wrapper, and shared snippet starts with an MDX comment naming its counterpart. Read it before editing, apply a fix to both sides where content is shared, and keep the comment when restructuring. diff --git a/.claude/agents/document-reviewer.md b/.claude/agents/document-reviewer.md index 05794244e..74464fbcf 100644 --- a/.claude/agents/document-reviewer.md +++ b/.claude/agents/document-reviewer.md @@ -17,7 +17,7 @@ Apply the canonical standards in three passes and report findings from each: 1. **Accuracy:** claims, platform scope, versions, and consistency with the surrounding page. Before reporting a claim as unverified, check the sources the PR or the user cites, such as the source PR, divergence issue, release notes, or code at the merged commit, and follow the links inside them. If nothing is cited, look up the release the text names. Report a claim as unverified only when no source covers it or a source contradicts it. Give evidence, do not invent problems, and do not approve unverified claims as correct. 2. **Necessity:** list every sentence that describes what the product prints, displays, logs, or says in an error. Treat each one as a finding to remove unless it passes the restating rule in [Content Strategy](../CLAUDE.md#content-strategy), and report it even when the sentence is accurate. Describing visible output is not a mechanism, consequence, or signal. Also flag internal mechanics, rare exceptions, and repetition. Flag missing context only when readers need it, and do not require every entry to explain a mechanism, consequence, signal, action, and trade-off. 3. **Clarity and format:** plain technical English, clear actors and actions, and suitable examples and components. -4. **Sync Rules containment:** flag any Sync Rules mention, example, or tab outside `sync/rules/` and `snippets/sync-shared/` that does not state a behavior difference, and any new Sync Rules content anywhere. Check that split twins, wrappers, and shared snippets keep their counterpart comment. +4. **Sync Rules containment:** flag any Sync Rules mention, example, or tab outside `sync/rules/` and `snippets/sync-shared/` that does not state a behavior difference. Check that split twins, wrappers, and shared snippets keep their counterpart comment. ## Default Output diff --git a/.claude/skills/doc-author/SKILL.md b/.claude/skills/doc-author/SKILL.md index 84de0db5d..c2201dbb4 100644 --- a/.claude/skills/doc-author/SKILL.md +++ b/.claude/skills/doc-author/SKILL.md @@ -18,6 +18,6 @@ Use the canonical Working Process for scope changes and unresolved decisions. 1. Identify the reader, desired outcome, and affected feature or concept. 2. Research the implementation and existing coverage. Read the most relevant related pages and `docs.json`; avoid unnecessary duplication. 3. If a plan is needed, state the proposed pages, structure, and unresolved questions before drafting. -4. Write the update under the canonical standards. Cover Sync Streams only and keep Sync Rules content out of it; Sync Rules pages take error fixes only. Keep the existing structure unless the task requires a change. +4. Write the update under the canonical standards. Write new content for Sync Streams and keep Sync Rules content inside its own section. Keep the existing structure unless the task requires a change. 5. Self-review for accuracy, reader understanding, minimum useful detail, and navigation fit. Run the canonical verification checks relevant to the change. 6. Present the result and any unresolved draft TODOs, or complete the delivery workflow already authorized by the user. diff --git a/.claude/skills/pr-to-docs/SKILL.md b/.claude/skills/pr-to-docs/SKILL.md index b74ea0ed0..b8556e124 100644 --- a/.claude/skills/pr-to-docs/SKILL.md +++ b/.claude/skills/pr-to-docs/SKILL.md @@ -33,4 +33,4 @@ Ask before expanding scope, documenting a deprecation that needs migration decis ## 4. Draft and Verify -Apply the canonical writing standards, navigation requirements, and verification checks. Document new behavior for Sync Streams only; do not add Sync Rules examples or mentions. Preserve existing structure unless restructuring is part of the approved plan. Flag unresolved facts using the canonical draft-TODO convention and report what must be resolved before publication. +Apply the canonical writing standards, navigation requirements, and verification checks. Document new behavior for Sync Streams. Update Sync Rules pages when the change affects them too, and do not add Sync Rules mentions elsewhere. Preserve existing structure unless restructuring is part of the approved plan. Flag unresolved facts using the canonical draft-TODO convention and report what must be resolved before publication. diff --git a/sync/advanced/multiple-client-versions.mdx b/sync/advanced/multiple-client-versions.mdx index 4775b1b9e..4562e6b76 100644 --- a/sync/advanced/multiple-client-versions.mdx +++ b/sync/advanced/multiple-client-versions.mdx @@ -3,7 +3,7 @@ title: "Multiple Client Versions" description: "Handle multiple client app versions that require different output schemas from Sync Streams." --- -{/* Split page: the Sync Rules (Legacy) version of this page is sync/rules/multiple-client-versions.mdx. That page is deprecated and frozen, but when you fix an error here, check whether it needs the same fix. Do not mention Sync Rules on this page. */} +{/* Split page: the Sync Rules (Legacy) version of this page is sync/rules/multiple-client-versions.mdx. When you change shared behavior or fix an error here, check whether that page needs the same change. Do not mention Sync Rules on this page. */} When schema changes are additive, old clients ignore the new tables and columns, and no special handling is required. More drastic changes, such as renaming tables or changing a table's structure, can break older app versions that are still in use. In these cases, define separate versions of the affected [Sync Streams](/sync/streams/overview) so that each client version receives the tables and columns it expects. diff --git a/sync/advanced/partitioned-tables.mdx b/sync/advanced/partitioned-tables.mdx index 0c90f8bb8..0e78e1e63 100644 --- a/sync/advanced/partitioned-tables.mdx +++ b/sync/advanced/partitioned-tables.mdx @@ -3,7 +3,7 @@ title: "Partitioned Tables (Postgres)" description: "Sync data from Postgres partitioned tables using wildcard table name matching." --- -{/* Split page: the Sync Rules (Legacy) version of this page is sync/rules/partitioned-tables.mdx. That page is deprecated and frozen, but when you fix an error here, check whether it needs the same fix. Do not mention Sync Rules on this page. */} +{/* Split page: the Sync Rules (Legacy) version of this page is sync/rules/partitioned-tables.mdx. When you change shared behavior or fix an error here, check whether that page needs the same change. Do not mention Sync Rules on this page. */} For partitioned tables in Postgres, each individual partition is replicated and processed using [Sync Streams](/sync/streams/overview). diff --git a/sync/advanced/sync-data-by-time.mdx b/sync/advanced/sync-data-by-time.mdx index afd0b875c..b0564046e 100644 --- a/sync/advanced/sync-data-by-time.mdx +++ b/sync/advanced/sync-data-by-time.mdx @@ -4,7 +4,7 @@ description: "Filter and sync data based on time ranges using Sync Streams, with sidebarTitle: "Sync Data by Time" --- -{/* Split page: the Sync Rules (Legacy) version of this page is sync/rules/sync-data-by-time.mdx. That page is deprecated and frozen, but when you fix an error here, check whether it needs the same fix. Do not mention Sync Rules on this page. */} +{/* Split page: the Sync Rules (Legacy) version of this page is sync/rules/sync-data-by-time.mdx. When you change shared behavior or fix an error here, check whether that page needs the same change. Do not mention Sync Rules on this page. */} A common need in offline-first apps is syncing data based on time, for example, only syncing issues updated in the last 7 days instead of the entire dataset. You might expect to write something like: diff --git a/sync/rules/multiple-client-versions.mdx b/sync/rules/multiple-client-versions.mdx index 0db532955..f6dd31151 100644 --- a/sync/rules/multiple-client-versions.mdx +++ b/sync/rules/multiple-client-versions.mdx @@ -4,10 +4,10 @@ sidebarTitle: "Multiple Client Versions" description: "Handle multiple client app versions that require different output schemas from legacy Sync Rules." --- -{/* Split page: the Sync Streams version of this page is sync/advanced/multiple-client-versions.mdx. This page is deprecated and frozen: no new features or examples. When you fix an error here, check whether the Sync Streams page needs the same fix. */} +{/* Split page: the Sync Streams version of this page is sync/advanced/multiple-client-versions.mdx. Sync Rules are deprecated: keep this page accurate, but do not prioritize additions. When you fix an error here, check whether the Sync Streams page needs the same fix. */} -Sync Rules are deprecated. This page is not updated with new features. For the Sync Streams version, see [Multiple Client Versions](/sync/advanced/multiple-client-versions). +Sync Rules are deprecated. For the Sync Streams version of this page, see [Multiple Client Versions](/sync/advanced/multiple-client-versions). When schema changes are additive, old clients ignore the new tables and columns, and no special handling is required. More drastic changes, such as renaming tables or changing a table's structure, can break older app versions that are still in use. In these cases, define separate versions of the affected bucket definitions so that each client version receives the tables and columns it expects. diff --git a/sync/rules/partitioned-tables.mdx b/sync/rules/partitioned-tables.mdx index 729f31b4d..c0918acfb 100644 --- a/sync/rules/partitioned-tables.mdx +++ b/sync/rules/partitioned-tables.mdx @@ -4,10 +4,10 @@ sidebarTitle: "Partitioned Tables (Postgres)" description: "Sync data from Postgres partitioned tables in legacy Sync Rules using wildcard table name matching." --- -{/* Split page: the Sync Streams version of this page is sync/advanced/partitioned-tables.mdx. This page is deprecated and frozen: no new features or examples. When you fix an error here, check whether the Sync Streams page needs the same fix. */} +{/* Split page: the Sync Streams version of this page is sync/advanced/partitioned-tables.mdx. Sync Rules are deprecated: keep this page accurate, but do not prioritize additions. When you fix an error here, check whether the Sync Streams page needs the same fix. */} -Sync Rules are deprecated. This page is not updated with new features. For the Sync Streams version, see [Partitioned Tables (Postgres)](/sync/advanced/partitioned-tables). +Sync Rules are deprecated. For the Sync Streams version of this page, see [Partitioned Tables (Postgres)](/sync/advanced/partitioned-tables). For partitioned tables in Postgres, each individual partition is replicated and processed using [Sync Rules](/sync/rules/overview). diff --git a/sync/rules/prioritized-sync.mdx b/sync/rules/prioritized-sync.mdx index e0f397f2e..52f58ff46 100644 --- a/sync/rules/prioritized-sync.mdx +++ b/sync/rules/prioritized-sync.mdx @@ -4,10 +4,10 @@ sidebarTitle: "Prioritized Sync" description: "Assign sync priorities to bucket definitions in legacy Sync Rules so that important data syncs before the rest." --- -{/* Split page: the Sync Streams version of this page is sync/streams/prioritized-sync.mdx. This page is deprecated and frozen: no new features or examples. When you fix an error here, check whether the Sync Streams page needs the same fix. */} +{/* Split page: the Sync Streams version of this page is sync/streams/prioritized-sync.mdx. Sync Rules are deprecated: keep this page accurate, but do not prioritize additions. When you fix an error here, check whether the Sync Streams page needs the same fix. */} -Sync Rules are deprecated. This page is not updated with new features. For the Sync Streams version, see [Prioritized Sync](/sync/streams/prioritized-sync). +Sync Rules are deprecated. For the Sync Streams version of this page, see [Prioritized Sync](/sync/streams/prioritized-sync). ## Overview diff --git a/sync/rules/supported-sql.mdx b/sync/rules/supported-sql.mdx index 3f98c7f1f..910971a42 100644 --- a/sync/rules/supported-sql.mdx +++ b/sync/rules/supported-sql.mdx @@ -4,10 +4,10 @@ sidebarTitle: "Supported SQL" description: "Reference for SQL syntax, operators, built-in functions, and type casting supported in legacy Sync Rules queries." --- -{/* Split page: the Sync Streams version of this page is sync/supported-sql.mdx. This page is deprecated and frozen: no new features or examples. When you fix an error here, check whether the Sync Streams page needs the same fix. */} +{/* Split page: the Sync Streams version of this page is sync/supported-sql.mdx. Sync Rules are deprecated: keep this page accurate, but do not prioritize additions. When you fix an error here, check whether the Sync Streams page needs the same fix. */} -Sync Rules are deprecated. This page is not updated with new features. For the Sync Streams version, see [Supported SQL](/sync/supported-sql). +Sync Rules are deprecated. For the Sync Streams version of this page, see [Supported SQL](/sync/supported-sql). This guide explains the SQL supported in [Sync Rules](/sync/rules/overview) parameter queries and data queries: what you can write, with examples and restrictions. diff --git a/sync/rules/sync-data-by-time.mdx b/sync/rules/sync-data-by-time.mdx index 3860f06ab..d8b32a7e2 100644 --- a/sync/rules/sync-data-by-time.mdx +++ b/sync/rules/sync-data-by-time.mdx @@ -4,10 +4,10 @@ sidebarTitle: "Sync Data by Time" description: "Filter and sync data based on time ranges using legacy Sync Rules, with patterns for recent-only and sliding-window queries." --- -{/* Split page: the Sync Streams version of this page is sync/advanced/sync-data-by-time.mdx. This page is deprecated and frozen: no new features or examples. When you fix an error here, check whether the Sync Streams page needs the same fix. */} +{/* Split page: the Sync Streams version of this page is sync/advanced/sync-data-by-time.mdx. Sync Rules are deprecated: keep this page accurate, but do not prioritize additions. When you fix an error here, check whether the Sync Streams page needs the same fix. */} -Sync Rules are deprecated. This page is not updated with new features. For the Sync Streams version, see [Sync Data by Time](/sync/advanced/sync-data-by-time). +Sync Rules are deprecated. For the Sync Streams version of this page, see [Sync Data by Time](/sync/advanced/sync-data-by-time). A common need is syncing data based on time, for example, only syncing issues updated in the last 7 days instead of the entire dataset. You might expect to write something like: diff --git a/sync/streams/prioritized-sync.mdx b/sync/streams/prioritized-sync.mdx index 15e73407b..fa25a5ccc 100644 --- a/sync/streams/prioritized-sync.mdx +++ b/sync/streams/prioritized-sync.mdx @@ -3,7 +3,7 @@ title: "Prioritized Sync" description: "Prioritize which tables sync first so users can start working immediately while remaining data continues loading in the background." --- -{/* Split page: the Sync Rules (Legacy) version of this page is sync/rules/prioritized-sync.mdx. That page is deprecated and frozen, but when you fix an error here, check whether it needs the same fix. Do not mention Sync Rules on this page. */} +{/* Split page: the Sync Rules (Legacy) version of this page is sync/rules/prioritized-sync.mdx. When you change shared behavior or fix an error here, check whether that page needs the same change. Do not mention Sync Rules on this page. */} ## Overview diff --git a/sync/supported-sql.mdx b/sync/supported-sql.mdx index 550e76ab6..b246d733f 100644 --- a/sync/supported-sql.mdx +++ b/sync/supported-sql.mdx @@ -3,7 +3,7 @@ title: "Supported SQL" description: "Reference for SQL syntax, operators, built-in functions, and type casting supported in Sync Streams queries." --- -{/* Split page: the Sync Rules (Legacy) version of this page is sync/rules/supported-sql.mdx. That page is deprecated and frozen, but when you fix an error here, check whether it needs the same fix. Do not mention Sync Rules on this page. */} +{/* Split page: the Sync Rules (Legacy) version of this page is sync/rules/supported-sql.mdx. When you change shared behavior or fix an error here, check whether that page needs the same change. Do not mention Sync Rules on this page. */} This guide explains the SQL supported in [Sync Streams](/sync/streams/overview): what you can write, with examples and restrictions.