diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts index ed4104cfb6c..e84f5cd4da6 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts @@ -138,6 +138,50 @@ describe('Connector Manual Sync API Route', () => { expect(mockDispatchSync).toHaveBeenCalledWith('conn-456', { billingAttribution, requestId: 'test-req-id', + rehydrate: false, + }) + }) + + it('dispatches a full resync when rehydrate=true is set', async () => { + const billingAttribution = { + actorUserId: 'external-admin', + workspaceId: 'ws-1', + organizationId: null, + billedAccountUserId: 'owner-1', + billingEntity: { type: 'user' as const, id: 'owner-1' }, + billingPeriod: { + start: '2026-07-01T00:00:00.000Z', + end: '2026-08-01T00:00:00.000Z', + }, + payerSubscription: null, + } + hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ + success: true, + authType: 'session', + userId: 'external-admin', + userName: 'Test', + userEmail: 'test@test.com', + }) + mockCheckWriteAccess.mockResolvedValue({ + hasAccess: true, + knowledgeBase: { workspaceId: 'ws-1', name: 'Test KB' }, + }) + mockDbChain.limit.mockResolvedValueOnce([{ id: 'conn-456', status: 'active' }]) + mockResolveBillingAttribution.mockResolvedValue(billingAttribution) + + const req = createMockRequest( + 'POST', + undefined, + {}, + 'http://localhost:3000/api/knowledge/kb-123/connectors/conn-456/sync?rehydrate=true' + ) + const response = await POST(req as never, { params: mockParams }) + + expect(response.status).toBe(200) + expect(mockDispatchSync).toHaveBeenCalledWith('conn-456', { + billingAttribution, + requestId: 'test-req-id', + rehydrate: true, }) }) }) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts index 9ef9cae6099..714f554040b 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts @@ -29,6 +29,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Route const parsed = await parseRequest(triggerKnowledgeConnectorSyncContract, request, context) if (!parsed.success) return parsed.response const { id: knowledgeBaseId, connectorId } = parsed.data.params + const { rehydrate } = parsed.data.query try { const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) @@ -81,7 +82,9 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Route workspaceId: kbWorkspaceId, }) - logger.info(`[${requestId}] Manual sync triggered for connector ${connectorId}`) + logger.info( + `[${requestId}] Manual sync${rehydrate ? ' (full rehydrate)' : ''} triggered for connector ${connectorId}` + ) captureServerEvent( auth.userId, @@ -109,12 +112,12 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Route knowledgeBaseName: writeCheck.knowledgeBase.name, connectorType: connectorRows[0].connectorType, connectorStatus: connectorRows[0].status, - syncType: 'manual', + syncType: rehydrate ? 'manual-rehydrate' : 'manual', }, request, }) - dispatchSync(connectorId, { billingAttribution, requestId }).catch((error) => { + dispatchSync(connectorId, { billingAttribution, requestId, rehydrate }).catch((error) => { logger.error( `[${requestId}] Failed to dispatch manual sync for connector ${connectorId}`, error diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx index b6e01e3c026..b7e7b4b6145 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx @@ -1,7 +1,19 @@ 'use client' import { useCallback, useEffect, useId, useMemo, useRef, useState } from 'react' -import { Badge, Button, Checkbox, ChipConfirmModal, cn, Loader, Tooltip } from '@sim/emcn' +import { + Badge, + Button, + Checkbox, + ChipConfirmModal, + cn, + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, + Loader, + Tooltip, +} from '@sim/emcn' import { createLogger } from '@sim/logger' import { format, formatDistanceToNow, isPast } from 'date-fns' import { @@ -115,14 +127,14 @@ export function ConnectorsSection({ }, []) const handleSync = useCallback( - (connectorId: string) => { + (connectorId: string, rehydrate = false) => { if (isSyncOnCooldown(connectorId)) return syncTriggeredAt.current[connectorId] = Date.now() addToSet(setSyncingIds, connectorId) triggerSync( - { knowledgeBaseId, connectorId }, + { knowledgeBaseId, connectorId, rehydrate }, { onSuccess: () => { setError(null) @@ -214,7 +226,7 @@ export function ConnectorsSection({ isSyncPending={syncingIds.has(connector.id)} isUpdating={updatingIds.has(connector.id)} syncCooldown={isSyncOnCooldown(connector.id)} - onSync={() => handleSync(connector.id)} + onSync={(rehydrate) => handleSync(connector.id, rehydrate)} onTogglePause={() => handleTogglePause(connector)} onEdit={() => setEditingConnector(connector)} onDelete={() => setDeleteTarget(connector.id)} @@ -273,7 +285,7 @@ interface ConnectorCardProps { isSyncPending: boolean isUpdating: boolean syncCooldown: boolean - onSync: () => void + onSync: (rehydrate?: boolean) => void onEdit: () => void onTogglePause: () => void onDelete: () => void @@ -327,6 +339,14 @@ function ConnectorCard({ ) const syncLogs = detail?.syncLogs ?? [] + const canFullResync = Boolean(connectorDef?.rehydrateOnFullSync) + const syncDisabled = + connector.status === 'syncing' || + connector.status === 'disabled' || + isSyncPending || + syncCooldown + const syncTooltip = syncCooldown ? 'Sync recently triggered' : canFullResync ? 'Sync' : 'Sync now' + return (
{canEdit && ( <> - - - - - - {syncCooldown ? 'Sync recently triggered' : 'Sync now'} - - + {canFullResync ? ( + + + + {/* span keeps the tooltip hoverable while the trigger button is disabled */} + + + + + + + {syncTooltip} + + + onSync(false)}>Sync now + onSync(true)}>Full resync + + + ) : ( + + + {/* span keeps the tooltip hoverable while the button is disabled */} + + + + + {syncTooltip} + + )} diff --git a/apps/sim/background/knowledge-connector-sync.test.ts b/apps/sim/background/knowledge-connector-sync.test.ts index 61f2156dddb..39a2d8f4ca6 100644 --- a/apps/sim/background/knowledge-connector-sync.test.ts +++ b/apps/sim/background/knowledge-connector-sync.test.ts @@ -72,6 +72,29 @@ describe('knowledge connector sync worker', () => { expect(mockExecuteSync).toHaveBeenCalledWith('connector-1', { billingAttribution: BILLING_ATTRIBUTION, fullSync: true, + rehydrate: undefined, + }) + }) + + it('forwards the rehydrate flag to the sync engine (async worker path)', async () => { + mockAssertConnectorSyncPayload.mockReturnValue({ + connectorId: 'connector-1', + requestId: 'request-1', + rehydrate: true, + billingAttribution: BILLING_ATTRIBUTION, + }) + + await executeConnectorSyncJob({ + connectorId: 'connector-1', + requestId: 'request-1', + rehydrate: true, + billingAttribution: BILLING_ATTRIBUTION, + }) + + expect(mockExecuteSync).toHaveBeenCalledWith('connector-1', { + billingAttribution: BILLING_ATTRIBUTION, + fullSync: undefined, + rehydrate: true, }) }) }) diff --git a/apps/sim/background/knowledge-connector-sync.ts b/apps/sim/background/knowledge-connector-sync.ts index dd29cf11108..f92c440a146 100644 --- a/apps/sim/background/knowledge-connector-sync.ts +++ b/apps/sim/background/knowledge-connector-sync.ts @@ -9,13 +9,13 @@ import { executeSync } from '@/lib/knowledge/connectors/sync-engine' const logger = createLogger('TriggerKnowledgeConnectorSync') export async function executeConnectorSyncJob(payload: unknown) { - const { connectorId, fullSync, requestId, billingAttribution } = + const { connectorId, fullSync, rehydrate, requestId, billingAttribution } = assertConnectorSyncPayload(payload) logger.info(`[${requestId}] Starting connector sync: ${connectorId}`) try { - const result = await executeSync(connectorId, { billingAttribution, fullSync }) + const result = await executeSync(connectorId, { billingAttribution, fullSync, rehydrate }) logger.info(`[${requestId}] Connector sync completed`, { connectorId, diff --git a/apps/sim/connectors/confluence/confluence.ts b/apps/sim/connectors/confluence/confluence.ts index af8e7941262..4cc5e81f705 100644 --- a/apps/sim/connectors/confluence/confluence.ts +++ b/apps/sim/connectors/confluence/confluence.ts @@ -91,6 +91,16 @@ async function fetchLabelsForPages( return labelsByPageId } +/** + * Body representation marker embedded in the contentHash. Bumping this + * invalidates every previously-synced Confluence document so a one-time + * re-hydration picks up content newly reachable by the current extraction + * (e.g. the switch from `storage` to rendered `view`, which expands Include + * Page / Excerpt macros). Without it, already-indexed pages whose version is + * unchanged classify as `unchanged` and keep their stale (empty) content. + */ +const CONTENT_REPRESENTATION = 'view' + /** * Produces a canonical metadata stub with a deterministic contentHash that * does not depend on which API surface (v1 CQL or v2) returned the page. @@ -115,7 +125,7 @@ function pageToStub( contentDeferred: true, mimeType: 'text/plain', sourceUrl: options.sourceUrl, - contentHash: `confluence:${page.id}:${versionKey}`, + contentHash: `confluence:${CONTENT_REPRESENTATION}:${page.id}:${versionKey}`, metadata: { spaceId: options.spaceId, status: page.status, @@ -233,10 +243,18 @@ export const confluenceConnector: ConnectorConfig = { if (syncContext) syncContext.cloudId = cloudId } - // Try pages first, fall back to blogposts if not found + /** + * Fetch the `view` representation rather than `storage`. Storage format only + * carries unexpanded macro references (e.g. Include Page / Excerpt Include), + * so "mirrored" content that pulls in another page's body is stripped to + * nothing by `htmlToPlainText`. The `view` representation is server-rendered + * HTML with those macros expanded inline, so included content is indexed too. + * The v2 single-item GET (`/pages/{id}`, `/blogposts/{id}`) supports + * `body-format=view`; only the bulk list endpoints are limited to storage/adf. + */ let page: Record | null = null for (const endpoint of ['pages', 'blogposts']) { - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/${endpoint}/${externalId}?body-format=storage` + const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/${endpoint}/${externalId}?body-format=view` const response = await fetchWithRetry(url, { method: 'GET', headers: { @@ -256,8 +274,8 @@ export const confluenceConnector: ConnectorConfig = { if (!page) return null const body = page.body as Record | undefined - const storage = body?.storage as Record | undefined - const rawContent = (storage?.value as string) || '' + const view = body?.view as Record | undefined + const rawContent = (view?.value as string) || '' const plainText = htmlToPlainText(rawContent) const labelMap = await fetchLabelsForPages(cloudId, accessToken, [String(page.id)]) diff --git a/apps/sim/connectors/confluence/meta.ts b/apps/sim/connectors/confluence/meta.ts index fce21b83225..83c3aa98bee 100644 --- a/apps/sim/connectors/confluence/meta.ts +++ b/apps/sim/connectors/confluence/meta.ts @@ -22,6 +22,15 @@ export const confluenceConnectorMeta: ConnectorMeta = { ], }, + /** + * Confluence pages can transclude other pages (Include Page / Excerpt macros). + * Editing an included page changes a container page's rendered `view` without + * bumping the container's version, so its version-based hash can't detect the + * change. A full resync re-hydrates and re-indexes to pick up that drift. This + * lives on the meta so the client can offer "Full resync" only where it applies. + */ + rehydrateOnFullSync: true, + configFields: [ { id: 'domain', diff --git a/apps/sim/connectors/types.ts b/apps/sim/connectors/types.ts index 2782fc588fa..71ad9ad6926 100644 --- a/apps/sim/connectors/types.ts +++ b/apps/sim/connectors/types.ts @@ -127,6 +127,20 @@ export interface ConnectorMeta { */ supportsIncrementalSync?: boolean + /** + * Whether this connector's extracted content can change without the source item's + * own change-detection hash changing — e.g. a Confluence page that transcludes + * another page via the Include Page / Excerpt macro: editing the included page + * changes the container's rendered `view` output without bumping the container's + * version, so its version-based `contentHash` stays identical. + * + * Incremental syncs remain hash-gated (cheap). On an explicit **full resync** + * (`fullSync`), the engine re-hydrates and re-indexes these connectors' documents + * even when their hash is unchanged, so transcluded/rendered-dependency changes are + * picked up. Only the deliberate full resync pays this re-index cost. + */ + rehydrateOnFullSync?: boolean + /** * Tag definitions this connector populates. Shown in the add-connector modal * as opt-out checkboxes. On connector creation, tag definitions are auto-created diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index 5e86d8549f4..a1291918bd0 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -204,11 +204,18 @@ export function useDeleteConnector() { interface TriggerSyncParams { knowledgeBaseId: string connectorId: string + /** Force re-hydration + re-index of rendered content (the "Full resync" action). */ + rehydrate?: boolean } -async function triggerSync({ knowledgeBaseId, connectorId }: TriggerSyncParams): Promise { +async function triggerSync({ + knowledgeBaseId, + connectorId, + rehydrate, +}: TriggerSyncParams): Promise { await requestJson(triggerKnowledgeConnectorSyncContract, { params: { id: knowledgeBaseId, connectorId }, + query: rehydrate ? { rehydrate: true } : {}, }) } diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.ts b/apps/sim/lib/api/contracts/knowledge/connectors.ts index eb776ee9220..ed147c741a1 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.ts @@ -4,6 +4,7 @@ import { knowledgeConnectorParamsSchema, successResponseSchema, } from '@/lib/api/contracts/knowledge/shared' +import { booleanQueryFlagSchema } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' export const createConnectorBodySchema = z.object({ @@ -150,10 +151,22 @@ export const deleteKnowledgeConnectorContract = defineRouteContract({ }, }) +export const triggerKnowledgeConnectorSyncQuerySchema = z.object({ + /** + * Force re-hydration: for connectors whose rendered content can drift without a + * hash change (e.g. Confluence transclusions), do a full listing and re-fetch + + * re-index every already-synced document rather than only hash-changed ones. The + * deletion-reconciliation safety guards stay armed. Defaults to the normal + * hash-gated sync. + */ + rehydrate: booleanQueryFlagSchema.optional().default(false), +}) + export const triggerKnowledgeConnectorSyncContract = defineRouteContract({ method: 'POST', path: '/api/knowledge/[id]/connectors/[connectorId]/sync', params: knowledgeConnectorParamsSchema, + query: triggerKnowledgeConnectorSyncQuerySchema, response: { mode: 'json', schema: z.object({ diff --git a/apps/sim/lib/knowledge/connectors/queue.test.ts b/apps/sim/lib/knowledge/connectors/queue.test.ts index 3c7d58bcf82..070ed03ea55 100644 --- a/apps/sim/lib/knowledge/connectors/queue.test.ts +++ b/apps/sim/lib/knowledge/connectors/queue.test.ts @@ -98,6 +98,7 @@ describe('connector sync queue', () => { { connectorId: 'connector-1', fullSync: true, + rehydrate: undefined, requestId: 'request-1', billingAttribution: BILLING_ATTRIBUTION, }, @@ -113,6 +114,20 @@ describe('connector sync queue', () => { ) }) + it('carries the rehydrate flag into the queued payload', async () => { + await dispatchSync('connector-1', { + billingAttribution: BILLING_ATTRIBUTION, + rehydrate: true, + requestId: 'request-1', + }) + + expect(mockTrigger).toHaveBeenCalledWith( + 'knowledge-connector-sync', + expect.objectContaining({ connectorId: 'connector-1', rehydrate: true }), + expect.anything() + ) + }) + it('rejects legacy payloads without billing attribution', () => { expect(() => assertConnectorSyncPayload({ diff --git a/apps/sim/lib/knowledge/connectors/queue.ts b/apps/sim/lib/knowledge/connectors/queue.ts index a9414c7a737..3bd73456ad5 100644 --- a/apps/sim/lib/knowledge/connectors/queue.ts +++ b/apps/sim/lib/knowledge/connectors/queue.ts @@ -19,6 +19,14 @@ const logger = createLogger('ConnectorSyncQueue') export interface ConnectorSyncPayload { connectorId: string fullSync?: boolean + /** + * Force re-hydration + re-indexing of already-synced documents for connectors + * whose rendered content can drift without a hash change (see + * `ConnectorMeta.rehydrateOnFullSync`). Forces a full (non-incremental) listing + * so every document is re-hydrated, but — unlike `fullSync` — keeps every + * deletion-reconciliation safety guard armed. + */ + rehydrate?: boolean requestId: string billingAttribution: BillingAttributionSnapshot } @@ -26,6 +34,7 @@ export interface ConnectorSyncPayload { export interface DispatchSyncOptions { billingAttribution: BillingAttributionSnapshot fullSync?: boolean + rehydrate?: boolean requestId?: string } @@ -46,6 +55,9 @@ export function assertConnectorSyncPayload(value: unknown): ConnectorSyncPayload if (value.fullSync !== undefined && typeof value.fullSync !== 'boolean') { throw new Error('Connector sync payload fullSync must be a boolean when provided') } + if (value.rehydrate !== undefined && typeof value.rehydrate !== 'boolean') { + throw new Error('Connector sync payload rehydrate must be a boolean when provided') + } if (value.billingAttribution === undefined) { throw new Error('Connector sync payload requires billing attribution') } @@ -53,6 +65,7 @@ export function assertConnectorSyncPayload(value: unknown): ConnectorSyncPayload return { connectorId: value.connectorId, fullSync: value.fullSync as boolean | undefined, + rehydrate: value.rehydrate as boolean | undefined, requestId: value.requestId, billingAttribution: assertBillingAttributionSnapshot(value.billingAttribution), } @@ -74,6 +87,7 @@ export async function dispatchSync( const payload = assertConnectorSyncPayload({ connectorId, fullSync: options?.fullSync, + rehydrate: options?.rehydrate, requestId, billingAttribution: options?.billingAttribution, }) @@ -147,6 +161,7 @@ export async function dispatchSync( executeSync(connectorId, { fullSync: payload.fullSync, + rehydrate: payload.rehydrate, billingAttribution: payload.billingAttribution, }).catch((error) => { logger.error(`Sync failed for connector ${connectorId}`, { diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts index 9cf0bcd3dc0..5d30ce45dde 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts @@ -227,6 +227,24 @@ describe('classifyExternalDoc', () => { type: 'unchanged', }) }) + + it('forces re-hydration of an unchanged deferred doc when forceRehydrate is set', async () => { + const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine') + const deferred = { ...base, content: '', contentDeferred: true } + // Same hash → normally unchanged, but forceRehydrate promotes it to update. + expect(classifyExternalDoc(deferred, { id: 'doc-1', contentHash: 'h1' }, true)).toEqual({ + type: 'update', + existingId: 'doc-1', + }) + }) + + it('does not force re-hydration of a non-deferred doc (content already final)', async () => { + const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine') + // Ready (non-deferred) content with an unchanged hash stays unchanged even under forceRehydrate. + expect(classifyExternalDoc(base, { id: 'doc-1', contentHash: 'h1' }, true)).toEqual({ + type: 'unchanged', + }) + }) }) describe('chunkOpsByByteBudget', () => { diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts index 0650202f50d..a4e07e20f9b 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.ts @@ -84,10 +84,17 @@ type DocClassification = * is already indexed is kept as-is (last-known-good) rather than downgraded. * - `drop`: empty, non-deferred content that cannot be indexed. * - `add` / `update` / `unchanged`: normal content reconciliation by content hash. + * + * `forceRehydrate` (set on a full resync of a `rehydrateOnFullSync` connector) promotes + * an otherwise-`unchanged` deferred document to `update` so its content is re-fetched — + * needed when rendered content can drift without the hash changing (e.g. Confluence + * transclusions). Non-deferred docs already carry final content from listing, so they + * are left `unchanged` (re-indexing identical content would be pointless). */ export function classifyExternalDoc( extDoc: Pick, - existing: { id: string; contentHash: string | null } | undefined + existing: { id: string; contentHash: string | null } | undefined, + forceRehydrate = false ): DocClassification { if (extDoc.skippedReason) { return existing ? { type: 'unchanged' } : { type: 'skip' } @@ -101,6 +108,9 @@ export function classifyExternalDoc( if (existing.contentHash !== extDoc.contentHash) { return { type: 'update', existingId: existing.id } } + if (forceRehydrate && extDoc.contentDeferred) { + return { type: 'update', existingId: existing.id } + } return { type: 'unchanged' } } @@ -308,7 +318,11 @@ async function resolveAccessToken( */ export async function executeSync( connectorId: string, - options: { billingAttribution: BillingAttributionSnapshot; fullSync?: boolean } + options: { + billingAttribution: BillingAttributionSnapshot + fullSync?: boolean + rehydrate?: boolean + } ): Promise { const billingAttribution = assertBillingAttributionSnapshot(options?.billingAttribution) const result: SyncResult = { @@ -418,15 +432,34 @@ export async function executeSync( let hasMore = true const syncContext: Record = { syncRunId: generateId() } - // Determine if this sync should be incremental + /** + * Determine if this sync should be incremental. A `rehydrate` request forces a + * full listing too: re-hydration must see *every* document (a container page can + * be unchanged itself yet transclude a page that changed), and an incremental + * listing would omit those unchanged containers, so they'd never be re-fetched. + */ const isIncremental = connectorConfig.supportsIncrementalSync && connector.syncMode !== 'full' && !options?.fullSync && + !options?.rehydrate && connector.lastSyncAt != null const lastSyncAt = isIncremental && connector.lastSyncAt ? new Date(connector.lastSyncAt) : undefined + /** + * Re-hydrate and re-index connectors whose rendered content can drift without a + * hash change (transclusions) — see `ConnectorMeta.rehydrateOnFullSync`. Driven + * by the dedicated `rehydrate` request (the "Full resync" action) or implied by a + * true `fullSync`. It forces a full listing (above) and re-indexes unchanged + * deferred docs, but — unlike `fullSync` — it does NOT bypass any + * deletion-reconciliation safety guard. Incremental syncs of other connectors + * stay hash-gated. + */ + const forceRehydrate = Boolean( + (options?.rehydrate || options?.fullSync) && connectorConfig.rehydrateOnFullSync + ) + for (let pageNum = 0; hasMore && pageNum < MAX_PAGES; pageNum++) { if (pageNum > 0 && connectorConfig.auth.mode === 'oauth') { accessToken = await resolveAccessToken(connector, connectorConfig, userId) @@ -551,7 +584,7 @@ export async function executeSync( } const existing = existingByExternalId.get(extDoc.externalId) - const classification = classifyExternalDoc(extDoc, existing) + const classification = classifyExternalDoc(extDoc, existing, forceRehydrate) switch (classification.type) { case 'skip': @@ -635,8 +668,15 @@ export async function executeSync( return null } const hydratedHash = fullDoc.contentHash ?? op.extDoc.contentHash + /** + * Normally an update whose hydrated hash matches the stored hash is a + * no-op (content unchanged). On a forced re-hydration the hash is + * version-based and cannot reflect the rendered-dependency change we are + * refreshing for, so re-index unconditionally instead of skipping. + */ if ( op.type === 'update' && + !forceRehydrate && existingByExternalId.get(op.extDoc.externalId)?.contentHash === hydratedHash ) { result.docsUnchanged++