From 6629d703b351c455e773226051995be9b62aad5f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 13:20:20 +0000 Subject: [PATCH 01/21] wip(service-automation): connector credential store and resolver chain Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- .../src/builtin/connector-nodes.ts | 6 +- .../src/connector-credentials.ts | 471 ++++++++++++++++ .../services/service-automation/src/engine.ts | 52 +- .../services/service-automation/src/plugin.ts | 512 +++++++++++++++--- .../src/sys-connector-credential.object.ts | 136 +++++ .../spec/src/contracts/automation-service.ts | 92 +++- 6 files changed, 1192 insertions(+), 77 deletions(-) create mode 100644 packages/services/service-automation/src/connector-credentials.ts create mode 100644 packages/services/service-automation/src/sys-connector-credential.object.ts diff --git a/packages/services/service-automation/src/builtin/connector-nodes.ts b/packages/services/service-automation/src/builtin/connector-nodes.ts index 7d548ff6324..de37e063439 100644 --- a/packages/services/service-automation/src/builtin/connector-nodes.ts +++ b/packages/services/service-automation/src/builtin/connector-nodes.ts @@ -63,14 +63,16 @@ export function registerConnectorNodes(engine: AutomationEngine, ctx: PluginCont if (!handler) { // A degraded declarative instance (#3017) is registered but has no // handlers — say WHY dispatch fails and that recovery is automatic, - // instead of the generic wiring hint below. + // instead of the generic wiring hint below. The reason names the + // cause: an unreachable upstream, or (#27237) a credential that + // is not set. const degraded = engine.getConnectorDegradedReason(cfg.connectorId); if (degraded) { return { success: false, error: `connector_action '${node.id}': connector '${cfg.connectorId}' is degraded — ${degraded}. ` - + `Dispatch is unavailable until its upstream recovers; the platform retries automatically.`, + + `Dispatch is unavailable until that is resolved; the platform retries automatically.`, }; } return { diff --git a/packages/services/service-automation/src/connector-credentials.ts b/packages/services/service-automation/src/connector-credentials.ts new file mode 100644 index 00000000000..c7ac75ebef7 --- /dev/null +++ b/packages/services/service-automation/src/connector-credentials.ts @@ -0,0 +1,471 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * # Connector credentials — the open-tier resolver chain and its store (#27237) + * + * A provider-bound declarative connector names its secret (`auth.credentialRef`, + * ADR-0097 §3). Ruling A on #27230 gives the name three places to resolve + * from, in order: + * + * 1. an environment variable of the same name — a self-hosted operator keeps + * today's path, and it wins, so a listing reports `source: 'env'` and the + * credential door refuses to shadow it; + * 2. a `sys_connector_credential` row an administrator set + * (`sys-connector-credential.object.ts`) — the `Field.secret()` channel, + * dereferenced only through the data engine's `resolveSecretField()`; + * 3. the resolver the host injected (`AutomationServicePluginOptions.credentialResolver`) + * — the enterprise vault arm, if one is wired. + * + * ⛔ A value leaves this module only as the return of {@link resolveConnectorCredential}, + * whose one caller hands it to a provider factory. Everything else here — + * fingerprints, listings, the store's rows — carries the reference, a source, + * a status and a revision, never the value. + */ + +import { defaultEnvCredentialResolver, type CredentialResolver } from './credential-resolver.js'; + +/** The object a connector credential lives in (`sys-connector-credential.object.ts`). */ +export const CONNECTOR_CREDENTIAL_OBJECT = 'sys_connector_credential'; + +/** Its `type: 'secret'` column. */ +export const CONNECTOR_CREDENTIAL_VALUE_FIELD = 'value'; + +/** The bound on a reference — the object's `ref.maxLength`, which its unique index needs. */ +export const CONNECTOR_CREDENTIAL_REF_MAX_LENGTH = 255; + +/** The bound on a value the door accepts. A PEM key or a service-account JSON fits several times over. */ +export const CONNECTOR_CREDENTIAL_VALUE_MAX_LENGTH = 16_384; + +/** Where a reference resolved from — never what it resolved to. */ +export type ConnectorCredentialSource = 'env' | 'object' | 'vault'; + +/** A stored row's status (`missing` is a listing verdict and has no row). */ +export type ConnectorCredentialRowStatus = 'set' | 'rejected'; + +/** System context — the store is plugin-owned; no caller's grants apply to it. */ +const SYSTEM_CONTEXT = { isSystem: true, positions: [], permissions: [] } as const; + +/** + * The slice of the data engine the store uses — ObjectQL's, declared + * structurally so this package keeps no build dependency on it. + */ +export interface ConnectorCredentialEngine { + find(object: string, query?: Record): Promise; + insert(object: string, data: Record, options?: Record): Promise; + update(object: string, data: Record, options?: Record): Promise; + delete(object: string, options?: Record): Promise; + /** The privileged dereference of one row's `secret`-typed field (ObjectQL ≥ #7799). */ + resolveSecretField?(object: string, recordId: string, field: string): Promise; +} + +/** One stored row, as far as anything reads it: the reference and its bookkeeping, never the value. */ +export interface ConnectorCredentialRecord { + id: string; + ref: string; + revision: number; + status: ConnectorCredentialRowStatus; + statusReason?: string; + connectorName?: string; + packageId?: string; + lastVerifiedAt?: string; + updatedAt?: string; + updatedBy?: string; +} + +/** The columns a read asks for — `value` is never among them. */ +const RECORD_FIELDS = [ + 'id', 'ref', 'revision', 'status', 'status_reason', 'connector_name', 'package_id', + 'last_verified_at', 'updated_at', 'updated_by', +]; + +/** + * ADR-0112 pair for a credential the deployment cannot handle safely right now: + * no data engine, no crypto provider, a stored row that does not decrypt. A + * standard-catalog member; `503` because the condition belongs to the + * deployment and clears when it is fixed. + */ +export const CONNECTOR_CREDENTIAL_UNAVAILABLE_CODE = 'SERVICE_UNAVAILABLE'; +export const CONNECTOR_CREDENTIAL_UNAVAILABLE_STATUS = 503; + +/** The store cannot write or read a credential safely. Its message names the reference, never the value. */ +export class ConnectorCredentialUnavailableError extends Error { + readonly code = CONNECTOR_CREDENTIAL_UNAVAILABLE_CODE; + readonly status = CONNECTOR_CREDENTIAL_UNAVAILABLE_STATUS; + readonly statusCode = CONNECTOR_CREDENTIAL_UNAVAILABLE_STATUS; + constructor(message: string) { + super(message); + this.name = 'ConnectorCredentialUnavailableError'; + } +} + +/** True when `err` is the engine's fail-closed refusal to persist a `secret` field. */ +function isSecretProtectionFailure(err: unknown): boolean { + return /Cannot persist secret field/i.test(String((err as Error)?.message ?? err ?? '')); +} + +function rowsOf(found: unknown): Record[] { + if (Array.isArray(found)) return found as Record[]; + const env = found as { data?: unknown; records?: unknown; value?: unknown } | null | undefined; + for (const candidate of [env?.data, env?.records, env?.value]) { + if (Array.isArray(candidate)) return candidate as Record[]; + } + return []; +} + +function optionalText(v: unknown): string | undefined { + if (typeof v === 'string' && v.length > 0) return v; + if (v instanceof Date) return v.toISOString(); + return undefined; +} + +function toRecord(row: Record): ConnectorCredentialRecord | undefined { + if (typeof row.ref !== 'string' || row.ref.length === 0) return undefined; + if (typeof row.id !== 'string' && typeof row.id !== 'number') return undefined; + const revision = Number(row.revision); + return { + id: String(row.id), + ref: row.ref, + revision: Number.isFinite(revision) ? revision : 0, + status: row.status === 'rejected' ? 'rejected' : 'set', + statusReason: optionalText(row.status_reason), + connectorName: optionalText(row.connector_name), + packageId: optionalText(row.package_id), + lastVerifiedAt: optionalText(row.last_verified_at), + updatedAt: optionalText(row.updated_at), + updatedBy: optionalText(row.updated_by), + }; +} + +/** + * The `sys_connector_credential` store. One per automation plugin. It never + * caches a value: {@link resolve} reads the row at the moment of use. + */ +export class ConnectorCredentialStore { + constructor(private readonly resolveEngine: () => ConnectorCredentialEngine | undefined) {} + + /** Whether a data engine is composed at all — without one nothing is stored and every row read is empty. */ + hasEngine(): boolean { + return this.resolveEngine() !== undefined; + } + + /** + * Every stored row, keyed by reference. Throws when the read fails — the + * caller decides how loud that is; a store that silently read as empty + * would degrade every connector whose credential it holds. + */ + async list(): Promise> { + const engine = this.resolveEngine(); + const out = new Map(); + if (!engine) return out; + const found = await engine.find(CONNECTOR_CREDENTIAL_OBJECT, { fields: RECORD_FIELDS, context: SYSTEM_CONTEXT }); + for (const row of rowsOf(found)) { + const record = toRecord(row); + if (record) out.set(record.ref, record); + } + return out; + } + + /** The row for one reference, or `undefined` when none is stored. */ + async read(ref: string): Promise { + const engine = this.resolveEngine(); + if (!engine) return undefined; + const found = await engine.find(CONNECTOR_CREDENTIAL_OBJECT, { + where: { ref }, + fields: RECORD_FIELDS, + context: SYSTEM_CONTEXT, + }); + for (const row of rowsOf(found)) { + const record = toRecord(row); + if (record) return record; + } + return undefined; + } + + /** + * The plaintext of one stored row, read now. Throws + * {@link ConnectorCredentialUnavailableError} when the row does not come + * back: no data engine, an engine without the privileged dereference, no + * crypto provider, or a ciphertext row that is gone. Never read as "no + * credential" — the caller degrades the instance and says why. + */ + async resolve(record: ConnectorCredentialRecord): Promise { + const engine = this.resolveEngine(); + if (!engine) { + throw new ConnectorCredentialUnavailableError( + `Connector credential '${record.ref}' is stored but cannot be read: no data engine is available to the automation service.`, + ); + } + if (typeof engine.resolveSecretField !== 'function') { + throw new ConnectorCredentialUnavailableError( + `Connector credential '${record.ref}' is stored encrypted, but this data engine cannot dereference an encrypted field.`, + ); + } + let plain: string | null; + try { + plain = await engine.resolveSecretField(CONNECTOR_CREDENTIAL_OBJECT, record.id, CONNECTOR_CREDENTIAL_VALUE_FIELD); + } catch { + // The engine's own message is not interpolated: this text reaches + // `GET /connectors` as the husk's reason, and a foreign message is + // not ours to vouch for. + throw new ConnectorCredentialUnavailableError( + `Connector credential '${record.ref}' is stored but could not be decrypted. Register the crypto provider it was ` + + 'written with on the data engine; the instance retries on its own.', + ); + } + if (typeof plain === 'string' && plain.length > 0) return plain; + throw new ConnectorCredentialUnavailableError( + `Connector credential '${record.ref}' is stored but resolved to nothing — its ciphertext row is missing. Set it again.`, + ); + } + + /** + * Store a value for `ref`, replacing what the row held and raising its + * revision. The engine encrypts it; with no crypto provider the engine + * refuses before anything is written, and so does this. + */ + async set(args: { + ref: string; + value: string; + userId?: string; + connectorNames: readonly string[]; + packageIds: readonly string[]; + }): Promise<{ created: boolean; revision: number }> { + const engine = this.resolveEngine(); + if (!engine) { + throw new ConnectorCredentialUnavailableError( + `Connector credential '${args.ref}' was not stored: no data engine is available to the automation service, ` + + 'and a connector credential is kept only in the encrypted credential store. Nothing was written.', + ); + } + const existing = await this.read(args.ref); + const context = { ...SYSTEM_CONTEXT, ...(args.userId ? { userId: args.userId } : {}) }; + const revision = (existing?.revision ?? 0) + 1; + const bookkeeping = { + revision, + status: 'set', + status_reason: null, + connector_name: args.connectorNames.length > 0 ? args.connectorNames.join(', ') : null, + package_id: args.packageIds.length > 0 ? args.packageIds.join(', ') : null, + }; + try { + if (existing) { + await engine.update( + CONNECTOR_CREDENTIAL_OBJECT, + { id: existing.id, [CONNECTOR_CREDENTIAL_VALUE_FIELD]: args.value, ...bookkeeping }, + { context }, + ); + } else { + await engine.insert( + CONNECTOR_CREDENTIAL_OBJECT, + { ref: args.ref, [CONNECTOR_CREDENTIAL_VALUE_FIELD]: args.value, ...bookkeeping }, + { context }, + ); + } + } catch (err) { + if (isSecretProtectionFailure(err)) { + throw new ConnectorCredentialUnavailableError( + `Connector credential '${args.ref}' was not stored: the encrypted credential store needs a crypto provider ` + + 'this deployment has not registered. Nothing was written. Register one on the data engine ' + + '(setCryptoProvider — LocalCryptoProvider in development, a KMS-backed provider in production) and set it again.', + ); + } + throw err; + } + return { created: !existing, revision }; + } + + /** Remove the row for `ref`. `false` when none was stored. */ + async clear(ref: string): Promise { + const engine = this.resolveEngine(); + if (!engine) return false; + const existing = await this.read(ref); + if (!existing) return false; + await engine.delete(CONNECTOR_CREDENTIAL_OBJECT, { where: { id: existing.id }, context: SYSTEM_CONTEXT }); + return true; + } + + /** + * Record that an instance materialized with revision `revision`: the + * timestamp moves, and a `rejected` row whose same value now works returns + * to `set`. A row rewritten since (another revision) is left alone. + */ + async markMaterialized(record: ConnectorCredentialRecord, at: Date = new Date()): Promise { + const engine = this.resolveEngine(); + if (!engine) return; + const current = await this.read(record.ref); + if (!current || current.revision !== record.revision) return; + await engine.update( + CONNECTOR_CREDENTIAL_OBJECT, + { id: current.id, last_verified_at: at.toISOString(), status: 'set', status_reason: null }, + { context: SYSTEM_CONTEXT }, + ); + } + + /** + * Record that an upstream refused revision `revision`. `reason` is the + * connector's message, already scrubbed of the value by the caller + * ({@link withoutCredential}). + */ + async markRejected(record: ConnectorCredentialRecord, reason: string): Promise { + const engine = this.resolveEngine(); + if (!engine) return; + const current = await this.read(record.ref); + if (!current || current.revision !== record.revision) return; + await engine.update( + CONNECTOR_CREDENTIAL_OBJECT, + { id: current.id, status: 'rejected', status_reason: reason }, + { context: SYSTEM_CONTEXT }, + ); + } +} + +/** + * `message` with every occurrence of `secret` replaced. A provider's message + * is foreign text; an upstream that echoes the credential back must not carry + * it into a log line, a husk's reason or a stored status. + */ +export function withoutCredential(message: string, secret: string | undefined): string { + if (!secret || secret.length < 4) return message; + return message.split(secret).join('[credential withheld]'); +} + +/** + * `err` with `secret` withheld from its message, keeping the structural marks + * a caller branches on (`code`, `status`, `name`). Returned unchanged when the + * secret appears nowhere in its message chain; otherwise rebuilt without its + * stack and cause, which carry the same text. + */ +export function withoutCredentialThrown(err: unknown, secret: string | undefined): unknown { + if (!secret || secret.length < 4 || !(err instanceof Error)) return err; + let leaks = false; + let cur: unknown = err; + for (let depth = 0; depth < 8 && cur instanceof Error; depth++) { + if (cur.message.includes(secret) || (cur.stack ?? '').includes(secret)) leaks = true; + cur = (cur as { cause?: unknown }).cause; + } + if (!leaks) return err; + const scrubbed = new Error(withoutCredential(err.message, secret)); + scrubbed.name = err.name; + scrubbed.stack = undefined; + const marks = err as { code?: unknown; status?: unknown }; + if (marks.code !== undefined) (scrubbed as { code?: unknown }).code = marks.code; + if (marks.status !== undefined) (scrubbed as { status?: unknown }).status = marks.status; + return scrubbed; +} + +/** The three arms of the chain, as the plugin composes them. */ +export interface ConnectorCredentialChain { + /** The environment arm. Defaults to {@link defaultEnvCredentialResolver}. */ + readonly env?: (ref: string) => string | undefined; + readonly store: ConnectorCredentialStore; + /** The host's injected resolver, if any — the vault arm. */ + readonly vault?: CredentialResolver; +} + +function envValue(chain: ConnectorCredentialChain, ref: string): string | undefined { + const read = chain.env ?? ((r: string) => defaultEnvCredentialResolver(r) as string | undefined); + const value = read(ref); + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +/** Whether an environment variable supplies `ref` — it wins, so a stored value would never be read. */ +export function isSuppliedByEnv(chain: ConnectorCredentialChain, ref: string): boolean { + return envValue(chain, ref) !== undefined; +} + +/** + * The degraded reason of an instance whose `credentialRef` no arm answers + * for: the reference, and the places it can be set. Names the reference, + * never a value — there is none. + */ +export function missingCredentialReason(ref: string, vaultWired: boolean): string { + return `credential '${ref}' is not set. Set it under Setup → Connector credentials ` + + `(PUT /api/v1/automation/connectors/credential-refs/${encodeURIComponent(ref)}) or as the environment variable ` + + `'${ref}'${vaultWired ? ", or in the host's credential resolver" : ''}; the instance materializes once it is set, ` + + 'with no restart'; +} + +/** + * A stable mark of where `ref` would resolve from right now, for the + * reconcile's signature — `env`, `object:`, `vault`, `missing`, or + * `unreadable` when the rows could not be read. Never derived from a value: a + * row's revision is what moves when its value is rewritten. + */ +export function connectorCredentialFingerprint( + chain: ConnectorCredentialChain, + ref: string, + rows: ReadonlyMap | undefined, +): string { + if (envValue(chain, ref) !== undefined) return 'env'; + if (rows === undefined) return 'unreadable'; + const row = rows.get(ref); + if (row) return `object:${row.revision}`; + return chain.vault ? 'vault' : 'missing'; +} + +/** The outcome of resolving one reference through the chain. */ +export type ConnectorCredentialResolution = + | { kind: 'resolved'; source: ConnectorCredentialSource; value: string; record?: ConnectorCredentialRecord } + | { kind: 'missing' } + | { kind: 'unavailable'; reason: string }; + +/** + * Resolve `ref` through env → stored row → vault. `rows` is the reconcile's + * one snapshot of the store (`undefined` when it could not be read). A stored + * row that does not come back is `unavailable`, never `missing`: the value + * exists, the deployment cannot read it now. A vault that throws propagates — + * the host's resolver keeps the contract it has always had. + */ +export async function resolveConnectorCredential( + chain: ConnectorCredentialChain, + ref: string, + rows: ReadonlyMap | undefined, +): Promise { + const fromEnv = envValue(chain, ref); + if (fromEnv !== undefined) return { kind: 'resolved', source: 'env', value: fromEnv }; + if (rows === undefined) { + return { + kind: 'unavailable', + reason: `the connector credential store could not be read, so credential '${ref}' cannot be resolved now; the instance retries on its own.`, + }; + } + const record = rows.get(ref); + if (record) { + try { + return { kind: 'resolved', source: 'object', value: await chain.store.resolve(record), record }; + } catch (err) { + return { kind: 'unavailable', reason: (err as Error).message }; + } + } + if (chain.vault) { + const fromVault = await chain.vault(ref); + if (typeof fromVault === 'string' && fromVault.length > 0) return { kind: 'resolved', source: 'vault', value: fromVault }; + } + return { kind: 'missing' }; +} + +/** + * Where `ref` resolves from and whether it is set, for a listing — the value + * is read only where presence cannot be told otherwise (the vault arm) and is + * dropped on the spot. + */ +export async function probeConnectorCredential( + chain: ConnectorCredentialChain, + ref: string, + rows: ReadonlyMap, +): Promise<{ source: ConnectorCredentialSource | null; status: 'set' | 'missing' | 'rejected'; record?: ConnectorCredentialRecord }> { + if (envValue(chain, ref) !== undefined) return { source: 'env', status: 'set' }; + const record = rows.get(ref); + if (record) return { source: 'object', status: record.status, record }; + if (chain.vault) { + let present = false; + try { + const v = await chain.vault(ref); + present = typeof v === 'string' && v.length > 0; + } catch { + present = false; + } + if (present) return { source: 'vault', status: 'set' }; + } + return { source: null, status: 'missing' }; +} diff --git a/packages/services/service-automation/src/engine.ts b/packages/services/service-automation/src/engine.ts index 1946c20fab9..06d5d86e395 100644 --- a/packages/services/service-automation/src/engine.ts +++ b/packages/services/service-automation/src/engine.ts @@ -10,7 +10,7 @@ import type { FlowFunctionEffect, FlowRunSummary, } from '@objectstack/spec/automation'; -import type { AutomationContext, AutomationResult, ResumeSignal, IAutomationService, RunListResult, ScreenSpec, ScreenFieldSpec, ConnectorSourcePullRequest, ConnectorSourcePullResult } from '@objectstack/spec/contracts'; +import type { AutomationContext, AutomationResult, ResumeSignal, IAutomationService, RunListResult, ScreenSpec, ScreenFieldSpec, ConnectorSourcePullRequest, ConnectorSourcePullResult, ConnectorCredentialRef, ConnectorCredentialSetRequest, ConnectorCredentialClearRequest } from '@objectstack/spec/contracts'; import type { MetadataActivationWriteOptions } from '@objectstack/core'; import type { FlowRegistration, FlowRegistrationRefusal, FlowInnerRefusal } from '@objectstack/spec/api'; import { RESUME_AUTHORITY_SERVICE } from '@objectstack/spec/contracts'; @@ -2508,6 +2508,18 @@ export interface FlowCredentialSource { resolve(flowName: string, nodeId: string, key: string): Promise; } +/** + * [#27237] The connector credential door the automation plugin attaches — the + * three `IAutomationService` credential members, served where the declared + * instances and the reconcile live. Every answer carries references and + * states, never a value. + */ +export interface ConnectorCredentialDoor { + list(): Promise; + set(request: ConnectorCredentialSetRequest): Promise; + clear(request: ConnectorCredentialClearRequest): Promise; +} + /** * [#20790] One credential a registered flow holds, by class: a literal in its * definition (a packaged flow's source), or held by the credential channel. @@ -2655,6 +2667,8 @@ export class AutomationEngine implements IAutomationService { private flowCredentialSource?: FlowCredentialSource; /** [#20281 stage ③] The connector sync executor — see {@link setConnectorPullSource}. */ private connectorPullSource?: (request: ConnectorSourcePullRequest) => Promise; + /** [#27237] The connector credential door — see {@link setConnectorCredentialDoor}. */ + private connectorCredentialDoor?: ConnectorCredentialDoor; /** [#22450] The lazy `i18n` service reader — see {@link setI18nServiceSource}. */ private i18nServiceSource?: () => RefusalI18nService | undefined; /** @@ -4734,6 +4748,42 @@ export class AutomationEngine implements IAutomationService { return this.connectorPullSource(request); } + /** + * [#27237] Attach the connector credential door — the automation plugin's, + * because the declared instances it lists and the reconcile a write runs + * are the plugin's, while the registered service is this engine. + */ + setConnectorCredentialDoor(door: ConnectorCredentialDoor | undefined): void { + this.connectorCredentialDoor = door; + } + + /** [#27237] The door, or the refusal a bare engine answers: nothing here can store a credential. */ + private requireConnectorCredentialDoor(verb: string): ConnectorCredentialDoor { + if (this.connectorCredentialDoor) return this.connectorCredentialDoor; + throw Object.assign( + new Error( + `[Automation] ${verb}: this automation engine has no connector credential door attached — it is attached by ` + + 'AutomationServicePlugin at init(); a bare engine has no declared connectors and no credential store.', + ), + { code: 'SERVICE_UNAVAILABLE', status: 503 }, + ); + } + + /** [#27237] `IAutomationService.listConnectorCredentialRefs` — references and states, never a value. */ + async listConnectorCredentialRefs(): Promise { + return this.requireConnectorCredentialDoor('listConnectorCredentialRefs()').list(); + } + + /** [#27237] `IAutomationService.setConnectorCredential` — store one value, re-materialize its instances. */ + async setConnectorCredential(request: ConnectorCredentialSetRequest): Promise { + return this.requireConnectorCredentialDoor(`setConnectorCredential('${request.ref}')`).set(request); + } + + /** [#27237] `IAutomationService.clearConnectorCredential` — remove one value, degrade its instances. */ + async clearConnectorCredential(request: ConnectorCredentialClearRequest): Promise { + return this.requireConnectorCredentialDoor(`clearConnectorCredential('${request.ref}')`).clear(request); + } + /** [#20790] Does the credential channel hold the LIVE credential at this position? */ holdsFlowCredential(flowName: string, nodeId: string, key: string): boolean { return this.flowCredentialSource?.holds(flowName, nodeId, key) ?? false; diff --git a/packages/services/service-automation/src/plugin.ts b/packages/services/service-automation/src/plugin.ts index f8d90b9c0a2..5a8395eb229 100644 --- a/packages/services/service-automation/src/plugin.ts +++ b/packages/services/service-automation/src/plugin.ts @@ -24,6 +24,27 @@ import { resolveRunDataContext } from './runtime-identity.js'; import { SysAutomationRun } from './sys-automation-run.object.js'; import { SysFlowDispatch } from './sys-flow-dispatch.object.js'; import { SysFlowCredential } from './sys-flow-credential.object.js'; +import { SysConnectorCredential } from './sys-connector-credential.object.js'; +import type { + ConnectorCredentialRef, + ConnectorCredentialSetRequest, + ConnectorCredentialClearRequest, +} from '@objectstack/spec/contracts'; +import { + CONNECTOR_CREDENTIAL_REF_MAX_LENGTH, + CONNECTOR_CREDENTIAL_VALUE_MAX_LENGTH, + ConnectorCredentialStore, + ConnectorCredentialUnavailableError, + connectorCredentialFingerprint, + isSuppliedByEnv, + missingCredentialReason, + probeConnectorCredential, + resolveConnectorCredential, + withoutCredentialThrown, + type ConnectorCredentialChain, + type ConnectorCredentialRecord, + type ConnectorCredentialResolution, +} from './connector-credentials.js'; import { FlowCredentialChannel, storedFlowStates, @@ -171,16 +192,19 @@ export interface AutomationServicePluginOptions { */ runHistoryMaxPerFlow?: number; /** - * Resolves a declarative connector instance's `auth.credentialRef` to its - * secret at boot (ADR-0097 §3). Defaults to {@link defaultEnvCredentialResolver} - * — the **open tier**: a `credentialRef` names an environment variable. An + * The host's credential resolver — the third arm of a declarative + * connector's `auth.credentialRef` chain (ADR-0097 §3, #27237): an + * environment variable of the same name wins, then the + * `sys_connector_credential` row an administrator set, then this. An * enterprise host injects a vault/KMS-backed resolver here without touching - * the materialization path. A ref that resolves to `undefined`/empty is a - * hard boot error (an app must not silently run with a dead connector). + * the materialization path. A ref no arm answers for registers the instance + * degraded, naming the ref, and it materializes once the ref is set — never + * a failed boot. A resolver that THROWS keeps its contract: fatal at boot. * * The engine reads an `api` flow's start-node `config.secretRef` through - * the same resolver, at every inbound post: a name that resolves to - * nothing refuses that post as unavailable (`503`), never verifies it. + * this resolver alone — {@link defaultEnvCredentialResolver} when absent — + * at every inbound post: a name that resolves to nothing refuses that post + * as unavailable (`503`), never verifies it. */ credentialResolver?: CredentialResolver; /** @@ -347,6 +371,51 @@ function holdsFlowDefinition(engine: AutomationEngine, name: string, held: unkno } } +/** + * [#27237] The `credentialRef` a declarative instance's `auth` names, or + * `undefined` for `none` / no `auth` — the reference the resolver chain + * (`./connector-credentials.ts`) is asked for. + */ +function credentialRefOf(auth: ConnectorInstanceAuth | undefined): string | undefined { + if (!auth || auth.type === 'none') return undefined; + return auth.credentialRef; +} + +/** + * Build the static {@link ResolvedConnectorAuth} a provider factory applies + * from a declarative instance's `auth` and the secret its `credentialRef` + * resolved to (ADR-0097 §3). `secret` is present whenever `auth` names a + * reference — an unresolved one never reaches here: the reconcile degrades + * the instance first (#27237). + */ +function instanceAuth(auth: ConnectorInstanceAuth | undefined, secret: string | undefined): ResolvedConnectorAuth | undefined { + if (!auth) return undefined; + if (auth.type === 'none') return { type: 'none' }; + const value = secret ?? ''; + switch (auth.type) { + case 'bearer': + return { type: 'bearer', token: value }; + case 'api-key': + return { + type: 'api-key', + key: value, + headerName: auth.headerName ?? 'X-API-Key', + paramName: auth.paramName, + }; + case 'basic': + return { type: 'basic', username: auth.username, password: value }; + } +} + +/** + * [#27237] A credential door refusal: an ADR-0112 standard-catalog `code` and + * its `status` on the thrown error, the shape the `/automation` domain answers + * a service rejection with. Its message names the reference, never a value. + */ +function credentialDoorRefusal(message: string, code: string, status: number): Error { + return Object.assign(new Error(message), { code, status, statusCode: status }); +} + /** * Stable signature of a provider-bound instance's materialization inputs * (ADR-0097). Drives the `metadata:reloaded` reconcile: an unchanged signature @@ -693,6 +762,12 @@ export class AutomationServicePlugin implements Plugin { * source, and the `flow` credential channel of the metadata save door. */ private credentialChannel?: FlowCredentialChannel; + /** + * [#27237] The connector credential store — the second arm of the + * `credentialRef` chain (env → this → the host's resolver) and what the + * credential door writes. See `./connector-credentials.ts`. + */ + private connectorCredentialStore?: ConnectorCredentialStore; /** [#20790] Serializes the one-time credential move — see {@link scheduleCredentialMigration}. */ private credentialMigration: Promise = Promise.resolve(); /** [#20790] The crypto-provider subscription that re-runs the move; dropped at destroy. */ @@ -775,7 +850,8 @@ export class AutomationServicePlugin implements Plugin { scope: 'system', defaultDatasource: 'cloud', namespace: 'sys', - objects: [SysAutomationRun, SysFlowDispatch, SysFlowCredential], + // [#27237] The connector credential store rides it too. + objects: [SysAutomationRun, SysFlowDispatch, SysFlowCredential, SysConnectorCredential], }); return true; } catch (err) { @@ -816,7 +892,8 @@ export class AutomationServicePlugin implements Plugin { * service — the `suspendedRunStore: 'memory'` composition, which registers * no run object: a flow credential has nowhere else to go, so the channel * needs its table on every composition. Every other composition registers - * it with the run objects ({@link registerRunObject}). + * it with the run objects ({@link registerRunObject}). [#27237] + * {@link SysConnectorCredential} rides with it, for the same reason. */ private registerCredentialObject(ctx: PluginContext): boolean { try { @@ -828,7 +905,7 @@ export class AutomationServicePlugin implements Plugin { scope: 'system', defaultDatasource: 'cloud', namespace: 'sys', - objects: [SysFlowCredential], + objects: [SysFlowCredential, SysConnectorCredential], }); return true; } catch (err) { @@ -953,6 +1030,17 @@ export class AutomationServicePlugin implements Plugin { this.credentialChannel = new FlowCredentialChannel(() => this.resolveDataEngine(ctx)); this.engine.setFlowCredentialSource(this.credentialChannel); + // [#27237] The connector credential store, and the door the + // `automation` service's credential members answer through: the + // declared instances it lists and the reconcile a write runs are this + // plugin's, while the registered service is the engine. + this.connectorCredentialStore = new ConnectorCredentialStore(() => this.resolveDataEngine(ctx)); + this.engine.setConnectorCredentialDoor({ + list: () => this.listConnectorCredentialRefs(ctx), + set: (request) => this.setConnectorCredential(ctx, request), + clear: (request) => this.clearConnectorCredential(ctx, request), + }); + // [#20281 stage ③] The connector sync executor, served on the // `automation` service's contract (`pullConnectorSource`) — the door a // job's `pull` run form binds through. Attached before the service is @@ -1545,7 +1633,13 @@ export class AutomationServicePlugin implements Plugin { | { onCryptoProviderChange?(listener: () => void): () => void } | undefined; if (!this.unsubscribeCryptoProvider && typeof dataEngine?.onCryptoProviderChange === 'function') { - this.unsubscribeCryptoProvider = dataEngine.onCryptoProviderChange(() => this.scheduleCredentialMigration(ctx)); + this.unsubscribeCryptoProvider = dataEngine.onCryptoProviderChange(() => { + this.scheduleCredentialMigration(ctx); + // [#27237] A stored connector credential decrypts only once + // the host's provider is in — which is after the boot + // reconcile — so an instance degraded on it retries now. + if (this.degradedInstances.size > 0) void this.materializeDeclaredConnectors(ctx, { fatal: false }); + }); } }); @@ -1935,6 +2029,31 @@ export class AutomationServicePlugin implements Plugin { desired.set(bound.name, { entry: bound, signature: connectorInstanceSignature(bound) }); } + // [#27237] Where each instance's `credentialRef` resolves from joins its + // signature — `env`, `object:`, `vault`, `missing` — so a + // credential write re-materializes exactly the instances naming it. One + // read of the store's bookkeeping per run, never a value; skipped when + // nothing declared names a reference. + const chain = this.connectorCredentialChain(); + let credentialRows: Map | undefined = new Map(); + if ([...desired.values()].some((d) => credentialRefOf(d.entry.auth) !== undefined)) { + try { + credentialRows = await chain.store.list(); + } catch (err) { + credentialRows = undefined; + ctx.logger.warn( + '[Automation] the connector credential store could not be read — an instance whose credential it would ' + + 'supply keeps what it has and retries.', + describeThrownForLog(err), + ); + } + } + for (const d of desired.values()) { + const ref = credentialRefOf(d.entry.auth); + if (ref === undefined) continue; + d.signature += `|credential:${connectorCredentialFingerprint(chain, ref, credentialRows)}`; + } + // 1. Remove connectors we previously materialized that are no longer // desired (deleted from the stack, or newly `enabled: false`). for (const [name, tracked] of [...this.materializedConnectors]) { @@ -1952,7 +2071,6 @@ export class AutomationServicePlugin implements Plugin { } // 2. Add new instances and re-materialize changed ones (signature diff). - const resolver = this.options.credentialResolver ?? defaultEnvCredentialResolver; let changed = 0; for (const [name, { entry, signature }] of desired) { const existing = this.materializedConnectors.get(name); @@ -1992,22 +2110,40 @@ export class AutomationServicePlugin implements Plugin { continue; } - let auth; - try { - auth = await this.resolveInstanceAuth(entry.auth, resolver, name, provider); - } catch (err) { - // #5575 — the old `fail((err as Error).message)` handed a FOREIGN - // message straight to the log message. `resolveInstanceAuth`'s own - // throw is single-line, but the resolver is host-supplied - // (`AutomationServicePluginOptions.credentialResolver` — a secrets - // service, or anything that validates with Zod), so what lands here - // is not ours to assume anything about. - fail( - `[Automation] connector instance '${name}' (provider '${provider}'): auth resolution failed (ADR-0097 §3).`, - err, - ); - continue; + // [#27237] env → the stored row → the host's resolver + // (`./connector-credentials.ts`). A reference no arm answers for, or a + // stored one this deployment cannot read now, degrades THIS instance + // and names the reference — never a failed boot: an administrator + // installs the app first and fills the credential after (ruling A on + // #27230; ADR-0097's amendment note). + const ref = credentialRefOf(entry.auth); + let credential: Extract | undefined; + if (ref !== undefined) { + let resolution: ConnectorCredentialResolution; + try { + resolution = await resolveConnectorCredential(chain, ref, credentialRows); + } catch (err) { + // #5575 — the host's resolver threw. It is host-supplied + // (`AutomationServicePluginOptions.credentialResolver` — a + // secrets service, or anything that validates with Zod), so + // what lands here is not ours to assume anything about, and its + // contract is unchanged: fatal at boot, logged and skipped on + // a reload. + fail( + `[Automation] connector instance '${name}' (provider '${provider}'): auth resolution failed (ADR-0097 §3).`, + err, + ); + continue; + } + if (resolution.kind !== 'resolved') { + await this.degradeConnectorForCredential(engine, ctx, { + name, entry, provider, signature, existing, ref, resolution, + }); + continue; + } + credential = resolution; } + const auth = instanceAuth(entry.auth, credential?.value); // The registry hands back RAW authored values (see // `DeclaredConnectorItem`), so the schema's defaults are applied @@ -2061,7 +2197,11 @@ export class AutomationServicePlugin implements Plugin { // #3017 — an *operational* fault (upstream unreachable) degrades // the one instance instead of failing the run, in BOTH modes; // only configuration faults keep the fail-loud contract below. - if (isConnectorUpstreamUnavailable(err)) { + // [#27237] A provider's message is foreign text: an upstream that + // echoes the credential back must not carry it into the husk's + // reason, a log record or the thrown boot error. + const thrown = withoutCredentialThrown(err, credential?.value); + if (isConnectorUpstreamUnavailable(thrown)) { this.degradeConnectorInstance(engine, ctx, { name, entry, @@ -2078,8 +2218,8 @@ export class AutomationServicePlugin implements Plugin { // as structured `meta` instead of interpolating it into a // message. Passing only the flattened string here is what // forced the interpolation this issue removes. - reason: (err as Error).message, - cause: err, + reason: (thrown as Error).message, + cause: thrown, }); continue; } @@ -2090,7 +2230,7 @@ export class AutomationServicePlugin implements Plugin { // to the thrown message on the fatal one. fail( `[Automation] failed to materialize connector instance '${name}' via provider '${provider}' (ADR-0097).`, - err, + thrown, ); continue; } @@ -2116,6 +2256,18 @@ export class AutomationServicePlugin implements Plugin { `[Automation] ${recovered ? 'recovered' : existing ? 're-materialized' : 'materialized'} ` + `connector instance '${name}' via provider '${provider}' (${def.actions?.length ?? 0} action(s))`, ); + // [#27237] The stored value worked: its row records when, and a + // `rejected` row whose same value now materializes returns to `set`. + if (credential?.source === 'object' && credential.record) { + try { + await chain.store.markMaterialized(credential.record); + } catch (markErr) { + ctx.logger.warn( + `[Automation] connector instance '${name}' materialized, but its credential row's last-materialized time could not be recorded.`, + describeThrownForLog(markErr), + ); + } + } } if (changed > 0) { @@ -2226,6 +2378,260 @@ export class AutomationServicePlugin implements Plugin { ); } + /** [#27237] The `credentialRef` chain this plugin resolves through: env → the store → the host's resolver. */ + private connectorCredentialChain(): ConnectorCredentialChain { + return { + store: this.connectorCredentialStore ?? new ConnectorCredentialStore(() => undefined), + vault: this.options.credentialResolver, + }; + } + + /** + * [#27237] Degrade one instance whose `credentialRef` has no value it can + * use: no arm of the chain answers (`missing`), or the stored row cannot + * be read now (`unavailable` — no crypto provider yet, an unreadable + * store). A functional degradation, so `warn`, once per cause: the husk is + * visible on `GET /connectors` and in the credential listing, and a + * `connector_action` dispatch fails naming the reference. + * + * A `missing` reference takes a live connector down with it — the value it + * was materialized with was cleared, and a connector still serving a + * cleared credential is the surprise this door must not have. An + * `unavailable` one leaves it serving: the value exists, and the + * deployment cannot read it this minute. Both retry on the #3017 backoff, + * which is also how a replica that did not take the write learns of it. + */ + private async degradeConnectorForCredential( + engine: AutomationEngine, + ctx: PluginContext, + info: { + name: string; + entry: DeclaredConnectorItem; + provider: string; + signature: string; + existing?: { close?: () => void | Promise }; + ref: string; + resolution: Exclude; + }, + ): Promise { + const reason = info.resolution.kind === 'missing' + ? missingCredentialReason(info.ref, this.options.credentialResolver !== undefined) + : info.resolution.reason; + let hasLive = info.existing !== undefined; + if (info.resolution.kind === 'missing' && info.existing) { + await this.dematerializeConnector(engine, info.name, info.existing, ctx, `its credential '${info.ref}' is no longer set`); + hasLive = false; + } + const prior = this.degradedInstances.get(info.name); + const repeat = prior !== undefined && prior.signature === info.signature && prior.reason === reason; + this.degradedInstances.set(info.name, { + attempts: repeat ? prior.attempts + 1 : 1, + signature: info.signature, + reason, + }); + // A retry that finds the same cause: the husk and the record are there. + if (repeat) return; + if (!hasLive) { + try { + engine.registerDegradedConnector(this.buildDegradedHuskDef(info.name, info.entry), reason, 'declarative'); + } catch (err) { + ctx.logger.warn( + `[Automation] could not register degraded husk for '${info.name}' — the instance stays absent from the ` + + `connector registry until a retry succeeds.`, + describeThrownForLog(err), + ); + } + } + ctx.logger.warn( + `[Automation] connector instance '${info.name}' (provider '${info.provider}') is degraded: ${reason}.` + + (hasLive ? ' The previously-materialized connector keeps serving.' : ''), + ); + } + + /** + * [#27237] Every `credentialRef` a declared provider-bound instance names, + * with the instances naming it — read from the declaration the reconcile + * reads ({@link readDeclaredConnectorItems}). An instance with + * `enabled: false` is listed too: an administrator may fill its credential + * before enabling it. + */ + private async declaredCredentialRefs(ctx: PluginContext): Promise>> { + const declared = (await this.readDeclaredConnectorItems(ctx, { post: true })) ?? []; + const out = new Map>(); + for (const raw of declared) { + const entry = raw as DeclaredConnectorItem & { _packageId?: unknown }; + if (typeof entry?.name !== 'string' || entry.name.length === 0) continue; + if (typeof entry.provider !== 'string' || entry.provider.length === 0) continue; + const ref = credentialRefOf(entry.auth); + if (ref === undefined) continue; + const packageId = typeof entry._packageId === 'string' && entry._packageId.length > 0 ? entry._packageId : undefined; + const list = out.get(ref) ?? []; + if (!list.some((c) => c.name === entry.name)) list.push({ name: entry.name, ...(packageId ? { packageId } : {}) }); + out.set(ref, list); + } + return out; + } + + /** [#27237] The stored rows, or the door's 503 when the store cannot be read — never an empty listing that hides them. */ + private async readCredentialRowsForDoor(chain: ConnectorCredentialChain): Promise> { + try { + return await chain.store.list(); + } catch { + throw new ConnectorCredentialUnavailableError( + 'The connector credential store could not be read, so which references are set cannot be told now. Nothing was changed.', + ); + } + } + + /** [#27237] One listing entry — a reference, its source and status, and the instances naming it. Never a value. */ + private async describeCredentialRef( + chain: ConnectorCredentialChain, + ref: string, + declaredBy: Array<{ name: string; packageId?: string }> | undefined, + rows: ReadonlyMap, + ): Promise { + const probe = await probeConnectorCredential(chain, ref, rows); + const record = rows.get(ref); + const descriptors = new Map((this.engine?.getConnectorDescriptors() ?? []).map((d) => [d.name, d])); + return { + ref, + status: probe.status, + source: probe.source, + writable: probe.source !== 'env', + declared: declaredBy !== undefined, + connectors: (declaredBy ?? []).map((c) => { + const d = descriptors.get(c.name); + return { + name: c.name, + ...(c.packageId ? { packageId: c.packageId } : {}), + ...(d ? { state: d.state } : {}), + ...(d?.degradedReason ? { degradedReason: d.degradedReason } : {}), + }; + }), + ...(record?.statusReason && probe.source === 'object' ? { statusReason: record.statusReason } : {}), + ...(record?.updatedAt ? { updatedAt: record.updatedAt } : {}), + ...(record?.updatedBy ? { updatedBy: record.updatedBy } : {}), + ...(record?.lastVerifiedAt ? { lastVerifiedAt: record.lastVerifiedAt } : {}), + }; + } + + /** + * [#27237] `IAutomationService.listConnectorCredentialRefs` — every + * declared reference, then every stored row no installed package declares + * any more (so it can be cleared), ordered by reference. + */ + private async listConnectorCredentialRefs(ctx: PluginContext): Promise { + const chain = this.connectorCredentialChain(); + const declared = await this.declaredCredentialRefs(ctx); + const rows = await this.readCredentialRowsForDoor(chain); + const refs = new Set([...declared.keys(), ...rows.keys()]); + const out: ConnectorCredentialRef[] = []; + for (const ref of [...refs].sort()) { + out.push(await this.describeCredentialRef(chain, ref, declared.get(ref), rows)); + } + return out; + } + + /** [#27237] The reference a door call names — present, and within the store's bound — or its `400`. */ + private credentialRefOfRequest(ref: unknown): string { + if (typeof ref !== 'string' || ref.length === 0) { + throw credentialDoorRefusal('A connector credential reference is required.', 'VALIDATION_ERROR', 400); + } + if (ref.length > CONNECTOR_CREDENTIAL_REF_MAX_LENGTH) { + throw credentialDoorRefusal( + `A connector credential reference is at most ${CONNECTOR_CREDENTIAL_REF_MAX_LENGTH} characters; this one has ${ref.length}.`, + 'VALIDATION_ERROR', + 400, + ); + } + return ref; + } + + /** + * [#27237] `IAutomationService.setConnectorCredential` — store one value + * for a declared reference, then reconcile, so every instance naming it + * re-materializes without a restart. Answers the reference's entry after + * that reconcile. Refuses, before anything is written: an undeclared + * reference (404), one an environment variable supplies (409), an empty or + * oversized value (400); the store refuses without a crypto provider (503). + */ + private async setConnectorCredential(ctx: PluginContext, request: ConnectorCredentialSetRequest): Promise { + const ref = this.credentialRefOfRequest(request?.ref); + const value: unknown = request?.value; + if (typeof value !== 'string' || value.length === 0) { + throw credentialDoorRefusal(`Credential '${ref}' needs a value: a non-empty string.`, 'VALIDATION_ERROR', 400); + } + if (value.length > CONNECTOR_CREDENTIAL_VALUE_MAX_LENGTH) { + throw credentialDoorRefusal( + `Credential '${ref}' is at most ${CONNECTOR_CREDENTIAL_VALUE_MAX_LENGTH} characters long.`, + 'VALIDATION_ERROR', + 400, + ); + } + const declared = await this.declaredCredentialRefs(ctx); + const declaredBy = declared.get(ref); + if (!declaredBy) { + throw credentialDoorRefusal( + `No installed connector declares credential '${ref}'. This door stores only a reference a declarative ` + + 'connector instance names in its auth.credentialRef; the listing at ' + + 'GET /api/v1/automation/connectors/credential-refs names them all.', + 'RESOURCE_NOT_FOUND', + 404, + ); + } + const chain = this.connectorCredentialChain(); + if (isSuppliedByEnv(chain, ref)) { + throw credentialDoorRefusal( + `Credential '${ref}' is supplied by the environment variable of the same name, which wins over a stored ` + + 'value, so a value stored here would never be read. Change it where the environment is configured, ' + + 'or unset that variable to manage it here.', + 'RESOURCE_CONFLICT', + 409, + ); + } + const userId = request.context?.userId; + const { revision } = await chain.store.set({ + ref, + value, + userId, + connectorNames: declaredBy.map((c) => c.name), + packageIds: [...new Set(declaredBy.map((c) => c.packageId).filter((p): p is string => p !== undefined))], + }); + ctx.logger.info( + `[Automation] connector credential '${ref}' set (revision ${revision})${userId ? ` by user '${userId}'` : ''}; ` + + `reconciling ${declaredBy.map((c) => `'${c.name}'`).join(', ')}.`, + ); + await this.materializeDeclaredConnectors(ctx, { fatal: false }); + return this.describeCredentialRef(chain, ref, declaredBy, await this.readCredentialRowsForDoor(chain)); + } + + /** + * [#27237] `IAutomationService.clearConnectorCredential` — remove the + * stored value for one reference, then reconcile: every instance naming it + * is torn down and registered degraded, naming the reference. `404` when + * nothing is stored for it. + */ + private async clearConnectorCredential(ctx: PluginContext, request: ConnectorCredentialClearRequest): Promise { + const ref = this.credentialRefOfRequest(request?.ref); + const chain = this.connectorCredentialChain(); + let cleared: boolean; + try { + cleared = await chain.store.clear(ref); + } catch { + throw new ConnectorCredentialUnavailableError( + `Credential '${ref}' could not be cleared: the connector credential store could not be read or written. Nothing was changed.`, + ); + } + if (!cleared) { + throw credentialDoorRefusal(`Nothing is stored for credential '${ref}'.`, 'RESOURCE_NOT_FOUND', 404); + } + const userId = request.context?.userId; + ctx.logger.info(`[Automation] connector credential '${ref}' cleared${userId ? ` by user '${userId}'` : ''}.`); + await this.materializeDeclaredConnectors(ctx, { fatal: false }); + const declared = await this.declaredCredentialRefs(ctx); + return this.describeCredentialRef(chain, ref, declared.get(ref), await this.readCredentialRowsForDoor(chain)); + } + /** The action-less def a degraded instance registers (#3017). */ private buildDegradedHuskDef(name: string, entry: DeclaredConnectorItem): Connector { return { @@ -2287,54 +2693,14 @@ export class AutomationServicePlugin implements Plugin { name: string, tracked: { close?: () => void | Promise }, ctx: PluginContext, + why = 'no longer declared', ): Promise { try { engine.unregisterConnector(name); } catch { /* ignore */ } if (tracked.close) { try { await tracked.close(); } catch { /* best-effort */ } } this.materializedConnectors.delete(name); - ctx.logger.info(`[Automation] unregistered declarative connector instance '${name}' (no longer declared)`); - } - - /** - * Resolve a declarative instance's `auth` into the static - * {@link ResolvedConnectorAuth} a provider factory applies — dereferencing - * `credentialRef` through the resolver. An `undefined`/empty resolution is a - * hard boot error (ADR-0097 §3): an app must not run with a connector whose - * credentials never loaded. - */ - private async resolveInstanceAuth( - auth: ConnectorInstanceAuth | undefined, - resolver: CredentialResolver, - connectorName: string, - provider: string, - ): Promise { - if (!auth) return undefined; - if (auth.type === 'none') return { type: 'none' }; - - const secret = await resolver(auth.credentialRef); - if (secret === undefined || secret === null || secret === '') { - throw new Error( - `[Automation] connector instance '${connectorName}' (provider '${provider}'): credentialRef ` + - `'${auth.credentialRef}' did not resolve to a value. The open tier resolves credentialRef from ` + - `environment variables — set the '${auth.credentialRef}' env var, or wire ` + - `AutomationServicePluginOptions.credentialResolver to a secrets service (ADR-0097 §3).`, - ); - } - - switch (auth.type) { - case 'bearer': - return { type: 'bearer', token: secret }; - case 'api-key': - return { - type: 'api-key', - key: secret, - headerName: auth.headerName ?? 'X-API-Key', - paramName: auth.paramName, - }; - case 'basic': - return { type: 'basic', username: auth.username, password: secret }; - } + ctx.logger.info(`[Automation] unregistered declarative connector instance '${name}' (${why})`); } /** diff --git a/packages/services/service-automation/src/sys-connector-credential.object.ts b/packages/services/service-automation/src/sys-connector-credential.object.ts new file mode 100644 index 00000000000..0151706bada --- /dev/null +++ b/packages/services/service-automation/src/sys-connector-credential.object.ts @@ -0,0 +1,136 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { ObjectSchema, Field } from '@objectstack/spec/data'; + +/** + * sys_connector_credential — the WRITE-ONLY store an administrator fills a + * declarative connector's credential into (#27237, ruling A on #27230). + * + * A provider-bound `connectors:` entry names its secret, never carries it: + * `auth: { type, credentialRef }` (ADR-0097 §3). The metadata author writes the + * name; the administrator who installs the app holds the value. Before this + * object the open tier resolved the name from `process.env` alone, so on a + * hosted environment — where the administrator has no server environment — + * every connector-bearing app installed and could never be credentialed. + * + * One row per `credentialRef` the administrator has set. The resolver chain + * reads it second: an environment variable of the same name wins (a self-hosted + * operator keeps today's path), then this row, then the host's injected + * resolver (a vault) if it wired one (`connector-credentials.ts`). + * + * `value` is `type: 'secret'` — the #7799 seam `sys_flow_credential` uses, + * unchanged: the engine encrypts it on write through the host's + * `ICryptoProvider` (fail-closed with no provider), masks it on every read + * path, and only the privileged `resolveSecretField()` dereferences it, when a + * connector instance materializes. ⛔ No second secret mechanism. + * + * Environment-level: a connector instance is environment-level (ADR-0097), so + * its credential is too, and the store carries no organization column. Per + * organization credentials on a shared environment are ruled not supported + * (S1 on #27230); the routes that write this object refuse on a walled posture. + * + * Writers: the automation plugin's credential door (set, clear) and its + * reconcile (`status`, `last_verified_at`), under a system context that + * carries the caller's user id, so the audit columns record who changed which + * reference and when. Readers: the same plugin — never the generic data door, + * which is closed below. + * + * @namespace sys + */ +export const SysConnectorCredential = ObjectSchema.create({ + name: 'sys_connector_credential', + label: 'Connector Credential', + pluralLabel: 'Connector Credentials', + icon: 'key', + isSystem: true, + managedBy: 'engine-owned', + // Environment-level (S1 on #27230): read by the connector instance that names + // it, under the system context, never by organization. + systemFields: { tenant: false }, + // ADR-0066 secure-by-default: a credential store is not covered by the + // wildcard grant. Everything reads and writes it through the automation + // plugin under a system context. + access: { default: 'private' }, + description: + "Write-only store of declarative connector credentials, one encrypted row per `credentialRef` an administrator set. Read second in the resolver chain: an environment variable of the same name wins.", + displayNameField: 'ref', + nameField: 'ref', // [ADR-0079] canonical primary-title pointer (mirrors deprecated displayNameField) + highlightFields: ['ref', 'status', 'connector_name', 'updated_at'], + + fields: { + ref: Field.text({ + label: 'Credential Reference', + required: true, + readonly: true, + // The producer is an authored `auth.credentialRef` — in practice an + // environment-variable-shaped name. The unique index needs a bound, and + // the credential door refuses a longer reference before any write. + maxLength: 255, + description: 'The `auth.credentialRef` a declarative connector instance names, exactly as authored.', + }), + + value: Field.secret({ + label: 'Value', + required: true, + description: + 'The credential, encrypted at rest by the host crypto provider and masked on every read. Resolved only server-side, when a connector instance naming this reference materializes.', + }), + + revision: Field.number({ + label: 'Revision', + required: true, + readonly: true, + defaultValue: 1, + description: + "Raised on every value write. The connector reconcile's change signal: an instance naming this reference re-materializes when it moves. Never derived from the value.", + }), + + connector_name: Field.text({ + label: 'Connectors', + readonly: true, + description: 'The declarative connector instances that named this reference when it was last set, comma-separated. Informational.', + }), + + package_id: Field.text({ + label: 'Packages', + readonly: true, + description: 'The packages that declared those instances when this reference was last set, comma-separated. Informational.', + }), + + status: Field.select(['set', 'rejected'], { + label: 'Status', + required: true, + defaultValue: 'set', + description: + "`set` when a value is stored; `rejected` when an upstream refused it while an instance materialized. A reference no source answers for is `missing` in the listing and has no row.", + }), + + status_reason: Field.textarea({ + label: 'Status Reason', + readonly: true, + description: "The connector's own message when the upstream refused the credential. Never the value.", + }), + + last_verified_at: Field.datetime({ + label: 'Last Materialized At', + readonly: true, + description: + 'When a connector instance last materialized with this value. A provider that contacts its upstream at materialization (mcp) proves the credential there; the others first use it on a dispatch.', + }), + }, + + indexes: [ + // One row per reference in the environment — the door's upsert key. + { fields: ['ref'], unique: 'global' }, + ], + + enable: { + // Engine-owned and write-only: no generic data door at all. The value is + // masked on every engine read regardless; closing the door also keeps the + // reference list out of reach of a caller without `manage_metadata`. + trackHistory: false, + searchable: false, + apiEnabled: false, + apiMethods: [], + }, +}); diff --git a/packages/spec/src/contracts/automation-service.ts b/packages/spec/src/contracts/automation-service.ts index 05cd45ba9b6..3d13447b365 100644 --- a/packages/spec/src/contracts/automation-service.ts +++ b/packages/spec/src/contracts/automation-service.ts @@ -16,7 +16,7 @@ import type { FlowParsed } from '../automation/flow.zod'; import type { ExecutionLog, ExecutionStatus, FlowRunSummary } from '../automation/execution.zod'; import type { ActionDescriptor } from '../automation/node-executor.zod'; -import type { ConnectorDescriptor } from '../integration/connector-descriptor'; +import type { ConnectorDescriptor, ConnectorState } from '../integration/connector-descriptor'; import type { ConversionNotice, ConversionTodoNotice } from '../conversions/types'; import type { ExecutionContext } from '../kernel/execution-context.zod'; import type { FlowRegistration } from '../api/protocol.zod'; @@ -953,6 +953,62 @@ export interface ConnectorSourcePullResult { summary: ConnectorSourcePullSummary; } +/** + * One declarative connector `credentialRef` (ADR-0097 §3) as the credential + * door reports it (#27237, ruling A on #27230): where it resolves from and + * whether it is set — ⛔ never its value, which no read of this door returns. + * + * The chain is env → the `sys_connector_credential` row an administrator set → + * the host's injected resolver. An environment variable of the same name wins, + * so such a reference reports `source: 'env'` and `writable: false`. + */ +export interface ConnectorCredentialRef { + /** The reference, exactly as an instance's `auth.credentialRef` names it. */ + ref: string; + /** + * `set` — a source answers for it; `missing` — none does, and every + * instance naming it is registered degraded; `rejected` — an upstream + * refused the stored value. + */ + status: 'set' | 'missing' | 'rejected'; + /** Which arm of the chain answers: `env`, `object` (the stored row) or `vault` (the host's resolver); `null` when missing. */ + source: 'env' | 'object' | 'vault' | null; + /** Whether the credential door may write it — `false` while an environment variable of the same name supplies it. */ + writable: boolean; + /** Whether an installed package still declares it. A stored row nothing declares is listed so it can be cleared. */ + declared: boolean; + /** The declarative connector instances naming it, with their registry state when registered. */ + connectors: Array<{ + name: string; + packageId?: string; + state?: ConnectorState; + degradedReason?: string; + }>; + /** The upstream's message when `status` is `rejected`. Never the value. */ + statusReason?: string; + /** When the stored row was last written, and by whom (its audit columns). */ + updatedAt?: string; + updatedBy?: string; + /** When an instance last materialized with the stored value. */ + lastVerifiedAt?: string; +} + +/** What {@link IAutomationService.setConnectorCredential} is asked to store. */ +export interface ConnectorCredentialSetRequest { + /** A reference an installed package declares. */ + ref: string; + /** The credential. Encrypted on write; never returned. */ + value: string; + /** The caller — its user id stamps the row's audit columns. */ + context?: ExecutionContext; +} + +/** What {@link IAutomationService.clearConnectorCredential} is asked to remove. */ +export interface ConnectorCredentialClearRequest { + ref: string; + context?: ExecutionContext; +} + export interface IAutomationService { /** * Execute a named flow or script @@ -1210,6 +1266,40 @@ export interface IAutomationService { */ pullConnectorSource?(request: ConnectorSourcePullRequest): Promise; + /** + * Every `credentialRef` an installed package's declarative connector + * instances name, with where it resolves from and whether it is set — + * plus any stored credential no package declares any more (#27237). ⛔ No + * value: the listing is built from the references, the stored rows' + * bookkeeping and the registry's connector states. + * + * Optional for the reason {@link getConnectorDescriptors} is: a connector + * registry is a capability of the flow-engine implementation, not of every + * automation slot. + */ + listConnectorCredentialRefs?(): Promise; + + /** + * Store the value for one declared reference in the encrypted credential + * store and re-materialize every instance naming it, without a restart; + * resolves to the reference's listing entry after that reconcile. Rejects + * with an ADR-0112 `code` + `status`, before anything is written: a + * reference no installed package declares (`404 RESOURCE_NOT_FOUND`), one + * an environment variable supplies (`409 RESOURCE_CONFLICT` — env wins, so + * a stored value would never be read), an empty or oversized reference or + * value (`400 VALIDATION_ERROR`), a deployment that cannot encrypt it + * (`503 SERVICE_UNAVAILABLE`). + */ + setConnectorCredential?(request: ConnectorCredentialSetRequest): Promise; + + /** + * Remove the stored value for one reference; every instance naming it is + * torn down and registered degraded, naming the reference. Resolves to the + * reference's listing entry after that reconcile. Rejects `404 + * RESOURCE_NOT_FOUND` when nothing is stored for it. + */ + clearConnectorCredential?(request: ConnectorCredentialClearRequest): Promise; + /** * Per-flow deployment + binding state, for operator surfaces. * From 1b2ba693d5cf93263402d7c50ebfe93f5c9f95c5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 13:30:08 +0000 Subject: [PATCH 02/21] wip(automation): credential-refs door, route ledger rows, rejected-credential arm Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- .../connector-mcp/src/mcp-provider.ts | 21 +- packages/runtime/src/dispatcher-plugin.ts | 31 +++ packages/runtime/src/domains/automation.ts | 205 ++++++++++++++++++ packages/runtime/src/route-ledger.ts | 8 + .../src/connector-credentials.ts | 58 ++++- .../services/service-automation/src/plugin.ts | 193 ++++++++++++++++- .../spec/src/api/error-code-ledger.zod.ts | 7 + .../integration/connector-provider-errors.ts | 48 ++++ 8 files changed, 549 insertions(+), 22 deletions(-) diff --git a/packages/connectors/connector-mcp/src/mcp-provider.ts b/packages/connectors/connector-mcp/src/mcp-provider.ts index 07305129abc..c2f957cb673 100644 --- a/packages/connectors/connector-mcp/src/mcp-provider.ts +++ b/packages/connectors/connector-mcp/src/mcp-provider.ts @@ -1,7 +1,7 @@ // Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. import type { ConnectorProviderFactory, ResolvedConnectorAuth } from '@objectstack/spec/integration'; -import { ConnectorUpstreamUnavailableError } from '@objectstack/spec/integration'; +import { ConnectorCredentialRejectedError, ConnectorUpstreamUnavailableError } from '@objectstack/spec/integration'; import { createMcpConnector, type McpConnectorOptions, type McpTransport } from './mcp-connector.js'; /** @@ -204,8 +204,9 @@ async function credentialRejectionStatus(err: unknown): Promise { @@ -243,15 +244,21 @@ export function createMcpProviderFactory(deps: McpProviderDeps = {}): ConnectorP // missing or under-scoped credential valid, and a degraded instance // retried with backoff reads as "server down", hiding the misconfiguration // the author has to fix. Credential-protected remote servers are the - // common case, so it is told apart by the SDK's HTTP status and thrown - // plain — fatal at boot, a skipped entry on reload. + // common case, so it is told apart by the SDK's HTTP status. [#27237] + // Thrown as `ConnectorCredentialRejectedError` (ruling on #27237, + // comment 6470087293): the materializer degrades the instance naming the + // refusal and marks a stored credential `rejected`, instead of failing + // boot — and, like the plain throw before it, does not retry it on a + // timer. The next attempt is the next change of the credential. const status = await credentialRejectionStatus(err); if (status !== undefined) { - throw new Error( + throw new ConnectorCredentialRejectedError( `connector-mcp provider: connector '${ctx.name}' was refused by its MCP server (HTTP ${status}): the ` + `credential it sent (the instance auth, or providerConfig.transport.headers) is missing, wrong or not ` + - `authorized. This is a configuration fault and is not retried — fix the credential and reload. ` + + `authorized. This is a configuration fault and is not retried — fix the credential. ` + `${(err as Error).message}`, + status, + { cause: err }, ); } // Everything else past transport validation is talking to the server diff --git a/packages/runtime/src/dispatcher-plugin.ts b/packages/runtime/src/dispatcher-plugin.ts index 2f3ebcf1702..313743268e3 100644 --- a/packages/runtime/src/dispatcher-plugin.ts +++ b/packages/runtime/src/dispatcher-plugin.ts @@ -1735,6 +1735,37 @@ export function createDispatcherPlugin(config: DispatcherPluginConfig = {}): Plu } }); + // [#27237] The connector credential door (ruling A on #27230), + // registered before `/automation/:name` for the reason above. + // The reference is re-encoded into the dispatched path, so one + // that carries a `/` stays one segment; the domain decodes it. + server!.get(`${base}/automation/connectors/credential-refs`, async (req: any, res: any) => { + try { + const result = await dispatcher.dispatch('GET', '/automation/connectors/credential-refs', undefined, req.query, { request: req }); + sendResult(result, res); + } catch (err: any) { + errorResponse(err, res); + } + }); + + server!.put(`${base}/automation/connectors/credential-refs/:ref`, async (req: any, res: any) => { + try { + const result = await dispatcher.dispatch('PUT', `/automation/connectors/credential-refs/${encodeURIComponent(req.params.ref)}`, deferRequestBody(req), req.query, { request: req }); + sendResult(result, res); + } catch (err: any) { + errorResponse(err, res); + } + }); + + server!.delete(`${base}/automation/connectors/credential-refs/:ref`, async (req: any, res: any) => { + try { + const result = await dispatcher.dispatch('DELETE', `/automation/connectors/credential-refs/${encodeURIComponent(req.params.ref)}`, undefined, req.query, { request: req }); + sendResult(result, res); + } catch (err: any) { + errorResponse(err, res); + } + }); + server!.get(`${base}/automation/_status`, async (req: any, res: any) => { try { const result = await dispatcher.dispatch('GET', '/automation/_status', undefined, req.query, { request: req }); diff --git a/packages/runtime/src/domains/automation.ts b/packages/runtime/src/domains/automation.ts index 10f891facdb..55f3a242618 100644 --- a/packages/runtime/src/domains/automation.ts +++ b/packages/runtime/src/domains/automation.ts @@ -36,6 +36,8 @@ import type { AutomationResult, IAutomationService, ISecurityService } from '@ob // [#23501] The served list read's contract — the facts a package refusal names. import type { MetadataProtocol } from '@objectstack/spec/api'; import { isServiceServeable } from '../service-serveable.js'; +// [#27237] Route rule 5 — the connector credential door's closed query set. +import { unknownQueryParamRefusalMessage } from '@objectstack/rest'; import { validationFailure, validationFailureDetails, fieldsFromZodIssues, VALIDATION_FAILED_STATUS, } from '../validation-failure.js'; @@ -701,6 +703,194 @@ async function refuseWalledFlowDefinitionWrite( }; } +/** + * [#27237] The connector credential door's segment under `/connectors`: + * `GET /connectors/credential-refs`, `PUT` and `DELETE + * /connectors/credential-refs/:ref` (ruling A on #27230). + */ +const CONNECTOR_CREDENTIAL_REFS_SEGMENT = 'credential-refs'; + +/** + * [#27237] Which routes are the connector credential door. Matched exactly as + * the route arms below match, for the reason the toggle arm of + * {@link isFlowAuthoringWrite} gives: a gate narrower than its route is a + * bypass, and a wider one over-blocks. + */ +function isConnectorCredentialDoor(parts: string[], method: string): boolean { + if (parts[0] !== 'connectors' || parts[1] !== CONNECTOR_CREDENTIAL_REFS_SEGMENT) return false; + if (method === 'GET') return parts.length === 2; + if (method === 'PUT' || method === 'DELETE') return parts.length === 3; + return false; +} + +/** + * [#27237] Route rule 5: the door's closed query-parameter set, the same for + * all three verbs — none. The listing has no filter, and a write names its + * reference in the path and its value in the body. + */ +export const CONNECTOR_CREDENTIAL_DOOR_QUERY_PARAMS: readonly string[] = Object.freeze([]); + +/** + * [#27237] The PUT body's closed key set. A misspelled key is refused naming + * the key, never answered as "a value is required" and never echoed with what + * it carried. + */ +const CONNECTOR_CREDENTIAL_PUT_BODY_KEYS: readonly string[] = Object.freeze(['value']); + +const CONNECTOR_CREDENTIAL_DENY_MESSAGE = + `Managing connector credentials requires the \`${FLOW_AUTHORING_CAPABILITY}\` capability.`; + +/** + * [#27237] The capability gate — `manage_metadata`, the capability a + * connector instance's declaration already requires on every metadata door, + * and the one the flow authoring gate above asks. Same shape, same envelope, + * and placed with those gates ahead of the service probe for their reasons: a + * refused caller learns neither whether automation is mounted here nor which + * references this environment declares. + */ +function refuseUngrantedConnectorCredentialAccess( + deps: DomainHandlerDeps, + context: HttpProtocolContext, +): HttpDispatcherResult | undefined { + const ec: any = context?.executionContext; + if (ec?.isSystem) return undefined; + if (new Set(ec?.systemPermissions ?? []).has(FLOW_AUTHORING_CAPABILITY)) return undefined; + return { + handled: true, + response: deps.error(CONNECTOR_CREDENTIAL_DENY_MESSAGE, FLOW_WRITE_DENY_STATUS, { code: FLOW_WRITE_DENY_CODE }), + }; +} + +/** [#27237] The walled refusal's sentence, naming the posture it was refused under. */ +export function connectorCredentialWalledMessage(posture: TenancyPosture): string { + return ( + `Connector credentials are environment-level, and this environment runs the '${posture}' tenancy posture, ` + + 'where several organizations share it, so this door does not store or list them here. A declarative ' + + "connector's credentialRef still resolves from a server environment variable of the same name." + ); +} + +/** + * [#27237] The walled refusal — S1 on #27230: an environment that several + * organizations live in is not supported by the credential store, so under + * `group` / `isolated` all three verbs refuse, for every caller, naming the + * posture. ⛔ Not {@link walledCatalogWriteVerdict}: that admits the platform + * operator, and the ruling admits nobody. Engine self-invocation (`isSystem`, + * never settable from the wire) is the one bypass, as at every gate here. + */ +async function refuseWalledConnectorCredentialAccess( + deps: DomainHandlerDeps, + context: HttpProtocolContext, +): Promise { + const ec: any = context?.executionContext; + if (ec?.isSystem) return undefined; + const posture = await readEffectiveTenancyPosture(deps, context); + if (posture === undefined || !postureEnforcesWall(posture)) return undefined; + return { + handled: true, + response: deps.error(connectorCredentialWalledMessage(posture), FLOW_WRITE_DENY_STATUS, { code: FLOW_WRITE_DENY_CODE }), + }; +} + +/** [#27237] The verbs the mounted service must implement, named in the door's 501. */ +const CONNECTOR_CREDENTIAL_UNSUPPORTED_MESSAGE = (member: string): string => + 'Connector credentials are not supported by the automation service this deployment mounts — it does not ' + + `implement \`${member}\`, an optional member of \`IAutomationService\`. Nothing was read or changed.`; + +/** + * [#27237] The service's own refusals — `400` / `404` / `409` / `503`, each + * carrying an ADR-0112 standard code — answered as they were raised. Anything + * else is not a refusal this door knows, and propagates to the transport's + * error exit. + */ +function connectorCredentialRefusal(deps: DomainHandlerDeps, err: unknown): HttpDispatcherResult | undefined { + const e = err as { status?: unknown; code?: unknown; message?: unknown } | null; + if (!e || typeof e.code !== 'string' || typeof e.message !== 'string') return undefined; + if (e.status !== 400 && e.status !== 404 && e.status !== 409 && e.status !== 503) return undefined; + return { handled: true, response: deps.error(e.message, e.status, { code: e.code }) }; +} + +/** + * [#27237] The connector credential door's three arms. Reached only past the + * capability gate and the walled refusal above. ⛔ The value never leaves + * here: the PUT hands it to the service and answers the reference's entry, + * which carries none. + */ +async function respondToConnectorCredentialDoor( + deps: DomainHandlerDeps, + automationService: unknown, + parts: string[], + method: string, + body: unknown, + context: HttpProtocolContext, + query: unknown, +): Promise { + const queryRefusal = unknownQueryParamRefusalMessage(query, CONNECTOR_CREDENTIAL_DOOR_QUERY_PARAMS); + if (queryRefusal !== undefined) { + return { handled: true, response: deps.error(queryRefusal, 400, { code: 'VALIDATION_ERROR' }) }; + } + const svc = automationService as Pick< + IAutomationService, + 'listConnectorCredentialRefs' | 'setConnectorCredential' | 'clearConnectorCredential' + >; + const executionContext = (context as any)?.executionContext; + try { + if (method === 'GET') { + if (typeof svc.listConnectorCredentialRefs !== 'function') { + return { handled: true, response: deps.error(CONNECTOR_CREDENTIAL_UNSUPPORTED_MESSAGE('listConnectorCredentialRefs'), 501) }; + } + const credentialRefs = await svc.listConnectorCredentialRefs(); + return { handled: true, response: deps.success({ credentialRefs, total: credentialRefs.length }) }; + } + let ref: string; + try { + ref = decodeURIComponent(parts[2]); + } catch { + return { + handled: true, + response: deps.error('The connector credential reference in the path is not a valid URL-encoded segment.', 400, { code: 'VALIDATION_ERROR' }), + }; + } + if (method === 'DELETE') { + if (typeof svc.clearConnectorCredential !== 'function') { + return { handled: true, response: deps.error(CONNECTOR_CREDENTIAL_UNSUPPORTED_MESSAGE('clearConnectorCredential'), 501) }; + } + const entry = await svc.clearConnectorCredential({ ref, context: executionContext }); + return { handled: true, response: deps.success(entry) }; + } + if (typeof body !== 'object' || body === null || Array.isArray(body)) { + return { + handled: true, + response: deps.error(`Setting credential '${ref}' needs a JSON object body: { "value": "" }.`, 400, { code: 'VALIDATION_ERROR' }), + }; + } + const unknownKeys = Object.keys(body).filter((k) => !CONNECTOR_CREDENTIAL_PUT_BODY_KEYS.includes(k)).sort(); + if (unknownKeys.length > 0) { + return { + handled: true, + response: deps.error( + `Setting credential '${ref}' takes only \`value\` in the body; unrecognised: ${unknownKeys.map((k) => `\`${k}\``).join(', ')}.`, + 400, + { code: 'VALIDATION_ERROR' }, + ), + }; + } + if (typeof svc.setConnectorCredential !== 'function') { + return { handled: true, response: deps.error(CONNECTOR_CREDENTIAL_UNSUPPORTED_MESSAGE('setConnectorCredential'), 501) }; + } + const entry = await svc.setConnectorCredential({ + ref, + value: (body as { value?: unknown }).value as string, + context: executionContext, + }); + return { handled: true, response: deps.success(entry) }; + } catch (err) { + const refusal = connectorCredentialRefusal(deps, err); + if (refusal) return refusal; + throw err; + } +} + /** [#13953] The path segment naming the cancel door (ADR-0044's operator verb). */ const RUN_CANCEL_SEGMENT = 'cancel'; /** [#13953] The path segment naming the repair door (#13909's operator verb). */ @@ -2738,6 +2928,15 @@ export async function handleAutomationRequest(deps: DomainHandlerDeps, path: str if (refusal) return refusal; } + // [#27237] CONNECTOR CREDENTIAL GATES — `manage_metadata`, then the walled + // refusal (S1 on #27230), both ahead of the service probe and of any body + // or query check, for the reasons the two gates above give. + if (isConnectorCredentialDoor(parts, m)) { + const refusal = refuseUngrantedConnectorCredentialAccess(deps, context) + ?? await refuseWalledConnectorCredentialAccess(deps, context); + if (refusal) return refusal; + } + // [ADR-0126 §5] ACTIVATION-WRITE GATE — the install-wide enable/disable // switch needs the platform operator in a walled posture. Placed directly // AFTER the authoring gate and BEFORE the service probe / body checks, for @@ -2948,6 +3147,12 @@ export async function handleAutomationRequest(deps: DomainHandlerDeps, path: str // `connector_action` connector/action/input pickers; the registry is // empty in baseline and populated by connector plugins (e.g. // @objectstack/connector-rest, @objectstack/connector-slack). + // [#27237] The connector credential door — GET the references, PUT or + // DELETE one. Ahead of `/:name` for the reason `/connectors` is. + if (isConnectorCredentialDoor(parts, m)) { + return respondToConnectorCredentialDoor(deps, automationService, parts, m, body, context, query); + } + if (parts[0] === 'connectors' && parts.length === 1 && m === 'GET') { // [#7360] The `/actions` note above applies verbatim to this filter: // `?type=rest&type=slack` arrived as an array, matched no connector, diff --git a/packages/runtime/src/route-ledger.ts b/packages/runtime/src/route-ledger.ts index 63c42ed517d..f2cb4ddc07b 100644 --- a/packages/runtime/src/route-ledger.ts +++ b/packages/runtime/src/route-ledger.ts @@ -532,6 +532,14 @@ export const ROUTE_LEDGER: readonly RouteLedgerEntry[] = [ note: "authored metadata, so `manage_metadata` gates it: a flow definition lives on the metadata plane (ADR-0106), and this door now asks the capability every other door onto that plane already asks. Fail-closed by construction — an absent executionContext, an absent `systemPermissions` or an empty one all fall through to the refusal, 403 with code `PERMISSION_DENIED` (ADR-0112); only engine self-invocation (`isSystem`, never settable from the wire) bypasses. WHICH routes is one predicate, `isFlowAuthoringWrite` in `domains/automation.ts` — this row, PUT/DELETE `/:name` below, and, since the ruling that enablement is an authoring write, `POST /:name/toggle`, with the execution doors (trigger / execute / resume) deliberately outside it. Second layer, not the first: the domain-wide anonymous floor answers an unidentified caller 401 here, not 403. Pinned in `domains/automation-write-capability-gate.test.ts`" }, { route: 'GET /automation/actions', domain: '/automation', disposition: 'sdk', client: 'automation.listActions' }, { route: 'GET /automation/connectors', domain: '/automation', disposition: 'sdk', client: 'automation.listConnectors' }, + // [#27237] The connector credential door (ruling A on #27230): an administrator + // fills the value a declarative connector's `auth.credentialRef` names. + { route: 'GET /automation/connectors/credential-refs', domain: '/automation', disposition: 'server-only', + note: "Every `credentialRef` a declared provider-bound connector instance names, plus every stored reference no installed package declares any more, each with its `status` (`set` / `missing` / `rejected`), its `source` (`env` / `object` / `vault` / null), whether this door may write it, and the instances naming it with their registry state. ⛔ Never a value: the store is `sys_connector_credential`, whose `value` is a `Field.secret` (#7799), and this listing reads only its bookkeeping columns. ⚑ `manage_metadata`, the gate the flow definition writes ask, fail-closed, only `isSystem` bypassing; then, under the `group` / `isolated` postures, refused 403 naming the posture for every caller — environment-level credentials only (S1 on #27230). Closed query set: none. NOT JS-SDK surface on this leg: the driver is the Console's connector credential page, a console surface that calls the platform API directly. Adding a client method reclassifies this row to `sdk`. Pinned in `domains/automation-connector-credential-door.test.ts`" }, + { route: 'PUT /automation/connectors/credential-refs/:ref', domain: '/automation', disposition: 'server-only', + note: "Store the value for one declared reference, then reconcile: every instance naming it re-materializes without a restart. Body is the closed `{ value }`. Answers the reference's listing entry after the reconcile, never the value. Refused, before anything is written: an undeclared reference 404 `RESOURCE_NOT_FOUND`, one an environment variable of the same name supplies 409 `RESOURCE_CONFLICT` (env wins in the chain, so a stored value would never be read), an empty or oversized value or an unknown body key 400 `VALIDATION_ERROR`, and no crypto provider 503 `SERVICE_UNAVAILABLE`. Same gates and SDK posture as the GET row above. Pinned in `domains/automation-connector-credential-door.test.ts`" }, + { route: 'DELETE /automation/connectors/credential-refs/:ref', domain: '/automation', disposition: 'server-only', + note: "Remove the stored value for one reference, then reconcile: every instance naming it is torn down and registered degraded, naming the reference. 404 `RESOURCE_NOT_FOUND` when nothing is stored for it. Same gates and SDK posture as the GET row above. Pinned in `domains/automation-connector-credential-door.test.ts`" }, { route: 'GET /automation/_status', domain: '/automation', disposition: 'sdk', client: 'automation.getRuntimeStatus' }, { route: 'POST /automation/:name/trigger', domain: '/automation', disposition: 'sdk', client: 'automation.execute' }, { route: 'POST /automation/:name/toggle', domain: '/automation', disposition: 'sdk', client: 'automation.toggle', diff --git a/packages/services/service-automation/src/connector-credentials.ts b/packages/services/service-automation/src/connector-credentials.ts index c7ac75ebef7..908e2d8ce86 100644 --- a/packages/services/service-automation/src/connector-credentials.ts +++ b/packages/services/service-automation/src/connector-credentials.ts @@ -269,18 +269,20 @@ export class ConnectorCredentialStore { + '(setCryptoProvider — LocalCryptoProvider in development, a KMS-backed provider in production) and set it again.', ); } - throw err; + // Whatever else refused the write answers without the value it was handed. + throw withoutCredentialThrown(err, args.value); } return { created: !existing, revision }; } - /** Remove the row for `ref`. `false` when none was stored. */ - async clear(ref: string): Promise { + /** Remove the row for `ref`, under the caller's user id for the audit. `false` when none was stored. */ + async clear(ref: string, userId?: string): Promise { const engine = this.resolveEngine(); if (!engine) return false; const existing = await this.read(ref); if (!existing) return false; - await engine.delete(CONNECTOR_CREDENTIAL_OBJECT, { where: { id: existing.id }, context: SYSTEM_CONTEXT }); + const context = { ...SYSTEM_CONTEXT, ...(userId ? { userId } : {}) }; + await engine.delete(CONNECTOR_CREDENTIAL_OBJECT, { where: { id: existing.id }, context }); return true; } @@ -469,3 +471,51 @@ export async function probeConnectorCredential( } return { source: null, status: 'missing' }; } + +/** + * The HTTP status of a connector call's `{ status }` result when the upstream + * refused the credential — 401 or 403 (ruling on #27237, comment 6470087293). + * The rest and openapi providers answer every response as `{ status, ok, + * body }` rather than throwing, so a refusal is read off the result. + */ +export function credentialRejectionStatusOf(result: unknown): 401 | 403 | undefined { + if (typeof result !== 'object' || result === null) return undefined; + const status = (result as { status?: unknown }).status; + return status === 401 || status === 403 ? status : undefined; +} + +/** The bound on an upstream's words kept in a reason or a row. */ +export const UPSTREAM_REFUSAL_MESSAGE_MAX_LENGTH = 500; + +/** + * The upstream's own words from a refused call's body — the body itself when + * it is text, else its usual `message` / `error_description` / `error` / + * `error.message` / `detail` / `title` key — on one line, bounded, and with + * `secret` withheld BEFORE the bound is applied, so a cut can never leave part + * of it behind. + */ +export function upstreamRefusalMessage(body: unknown, secret: string | undefined): string | undefined { + let text: unknown; + if (typeof body === 'string') { + text = body; + } else if (typeof body === 'object' && body !== null) { + const b = body as Record; + const nested = typeof b.error === 'object' && b.error !== null ? (b.error as Record).message : b.error; + text = [b.message, b.error_description, nested, b.detail, b.title].find( + (v): v is string => typeof v === 'string' && v.trim().length > 0, + ); + } + if (typeof text !== 'string') return undefined; + const flat = withoutCredential(text, secret).replace(/\s+/g, ' ').trim(); + if (flat.length === 0) return undefined; + return flat.length > UPSTREAM_REFUSAL_MESSAGE_MAX_LENGTH ? `${flat.slice(0, UPSTREAM_REFUSAL_MESSAGE_MAX_LENGTH)}…` : flat; +} + +/** The degraded reason for an instance whose upstream refused its credential on a call. */ +export function credentialRejectedOnCallReason(name: string, ref: string, status: number, upstream: string | undefined): string { + return ( + `connector '${name}' was refused by its upstream (HTTP ${status}) on a call: credential '${ref}' is missing, wrong ` + + `or not authorized${upstream ? ` (the upstream said: ${upstream})` : ''}. The instance stays degraded, without ` + + `retrying, until credential '${ref}' changes.` + ); +} diff --git a/packages/services/service-automation/src/plugin.ts b/packages/services/service-automation/src/plugin.ts index 5a8395eb229..815b4876969 100644 --- a/packages/services/service-automation/src/plugin.ts +++ b/packages/services/service-automation/src/plugin.ts @@ -9,8 +9,9 @@ import type { ConnectorInstanceAuth, ResolvedConnectorAuth, ConnectorProviderContext, + ConnectorMaterializationHandler, } from '@objectstack/spec/integration'; -import { isConnectorUpstreamUnavailable, RetryConfigSchema } from '@objectstack/spec/integration'; +import { isConnectorCredentialRejected, isConnectorUpstreamUnavailable, RetryConfigSchema } from '@objectstack/spec/integration'; import { stripReadDecorations } from '@objectstack/spec/kernel'; import type { MetadataMutationEvent } from '@objectstack/metadata-protocol'; import { AutomationEngine } from './engine.js'; @@ -36,10 +37,13 @@ import { ConnectorCredentialStore, ConnectorCredentialUnavailableError, connectorCredentialFingerprint, + credentialRejectedOnCallReason, + credentialRejectionStatusOf, isSuppliedByEnv, missingCredentialReason, probeConnectorCredential, resolveConnectorCredential, + upstreamRefusalMessage, withoutCredentialThrown, type ConnectorCredentialChain, type ConnectorCredentialRecord, @@ -724,8 +728,13 @@ export class AutomationServicePlugin implements Plugin { * Disjoint from {@link materializedConnectors} EXCEPT when a *changed* * entry's re-materialization failed — then the old live connector keeps * serving (still tracked there) while the retry is pending here. + * + * [#27237] `held` marks an instance whose upstream refused its credential: + * degraded like the others, but neither the retry timer nor a reconcile + * re-attempts it while its signature — which carries the credential's + * source and revision — stays the same. */ - private degradedInstances = new Map(); + private degradedInstances = new Map(); private declarativeRetryTimer?: ReturnType; /** * [#7742] The connector collection carried by the most recent @@ -2086,6 +2095,11 @@ export class AutomationServicePlugin implements Plugin { continue; } + // [#27237] Held: its upstream refused this very credential. The + // next attempt is the next change of the credential or the entry. + const held = this.degradedInstances.get(name); + if (held?.held && held.signature === signature) continue; + const provider = entry.provider; // §4 conflict: the name is held by a plugin-registered connector (not @@ -2201,6 +2215,24 @@ export class AutomationServicePlugin implements Plugin { // echoes the credential back must not carry it into the husk's // reason, a log record or the thrown boot error. const thrown = withoutCredentialThrown(err, credential?.value); + // [#27237] The upstream refused the credential (ruling on + // #27237, comment 6470087293): degrade the instance and mark a + // stored credential `rejected` — never a failed boot, never a + // retry loop against the same value. + if (isConnectorCredentialRejected(thrown)) { + await this.holdConnectorForRejectedCredential(engine, ctx, { + name, + entry, + provider, + signature, + live: existing, + takeDown: false, + credential, + reason: (thrown as Error).message, + cause: thrown, + }); + continue; + } if (isConnectorUpstreamUnavailable(thrown)) { this.degradeConnectorInstance(engine, ctx, { name, @@ -2246,8 +2278,16 @@ export class AutomationServicePlugin implements Plugin { // what the factory named its def, so discovery, dispatch, and the // conflict rule all agree with the metadata the author wrote. const def = { ...materialization.def, name }; - engine.registerConnector(def, materialization.handlers, 'declarative'); - this.materializedConnectors.set(name, { signature, provider, close: materialization.close }); + const tracked = { signature, provider, close: materialization.close }; + // [#27237] A credentialed instance's calls are watched for the + // upstream's 401 / 403 (rest and openapi answer them as a result). + const handlers = credential && ref !== undefined + ? this.watchForCredentialRejection(engine, ctx, { + name, entry, provider, signature, ref, credential, tracked, + }, materialization.handlers) + : materialization.handlers; + engine.registerConnector(def, handlers, 'declarative'); + this.materializedConnectors.set(name, tracked); // Success clears any pending degraded retry — including replacing a // registered husk (registerConnector above overwrote it). const recovered = this.degradedInstances.delete(name); @@ -2614,9 +2654,10 @@ export class AutomationServicePlugin implements Plugin { private async clearConnectorCredential(ctx: PluginContext, request: ConnectorCredentialClearRequest): Promise { const ref = this.credentialRefOfRequest(request?.ref); const chain = this.connectorCredentialChain(); + const userId = request?.context?.userId; let cleared: boolean; try { - cleared = await chain.store.clear(ref); + cleared = await chain.store.clear(ref, userId); } catch { throw new ConnectorCredentialUnavailableError( `Credential '${ref}' could not be cleared: the connector credential store could not be read or written. Nothing was changed.`, @@ -2625,13 +2666,142 @@ export class AutomationServicePlugin implements Plugin { if (!cleared) { throw credentialDoorRefusal(`Nothing is stored for credential '${ref}'.`, 'RESOURCE_NOT_FOUND', 404); } - const userId = request.context?.userId; ctx.logger.info(`[Automation] connector credential '${ref}' cleared${userId ? ` by user '${userId}'` : ''}.`); await this.materializeDeclaredConnectors(ctx, { fatal: false }); const declared = await this.declaredCredentialRefs(ctx); return this.describeCredentialRef(chain, ref, declared.get(ref), await this.readCredentialRowsForDoor(chain)); } + /** + * [#27237] Degrade one instance whose upstream refused its credential — + * at materialization (`ConnectorCredentialRejectedError`, the mcp + * provider) or on a call (a 401 / 403 result, rest and openapi) — and + * mark a stored credential `rejected` with the upstream's message, which + * the caller has already scrubbed of the value. Held, not retried: the + * signature carries the credential's revision, so setting it again is + * what re-attempts the instance. + * + * A refusal at materialization leaves a live connector serving, as every + * failed re-materialization does; a refusal on a call takes the connector + * that made it down (`takeDown`). + */ + private async holdConnectorForRejectedCredential( + engine: AutomationEngine, + ctx: PluginContext, + info: { + name: string; + entry: DeclaredConnectorItem; + provider: string; + signature: string; + live?: { close?: () => void | Promise }; + takeDown: boolean; + credential?: Extract; + reason: string; + cause?: unknown; + }, + ): Promise { + let hasLive = info.live !== undefined; + if (info.takeDown && info.live) { + await this.dematerializeConnector(engine, info.name, info.live, ctx, 'its upstream refused its credential'); + hasLive = false; + } + this.degradedInstances.set(info.name, { attempts: 1, signature: info.signature, reason: info.reason, held: true }); + if (!hasLive) { + try { + engine.registerDegradedConnector(this.buildDegradedHuskDef(info.name, info.entry), info.reason, 'declarative', info.cause); + } catch (err) { + ctx.logger.warn( + `[Automation] could not register degraded husk for '${info.name}' — the instance stays absent from the ` + + `connector registry until its credential changes.`, + describeThrownForLog(err), + ); + } + } + if (info.credential?.source === 'object' && info.credential.record) { + try { + await this.connectorCredentialChain().store.markRejected(info.credential.record, info.reason); + } catch (markErr) { + ctx.logger.warn( + `[Automation] connector instance '${info.name}': its credential row could not be marked rejected.`, + describeThrownForLog(markErr), + ); + } + } + // #5575 — the reason quotes the upstream, foreign text: it rides `meta`. + ctx.logger.warn( + `[Automation] connector instance '${info.name}' (provider '${info.provider}') is degraded: its upstream refused ` + + `its credential. ` + + (hasLive + ? 'The previously-materialized connector keeps serving.' + : 'Not retried until its credential or its declaration changes.'), + { reason: info.reason }, + ); + } + + /** + * [#27237] Wrap a credentialed instance's handlers so the first call its + * upstream answers 401 / 403 degrades the instance and marks its stored + * credential `rejected` (ruling on #27237, comment 6470087293). The call's + * own result is returned unchanged. Only the materialization these + * handlers belong to is taken down, once, through the reconcile queue, so + * a refusal never races a reconcile that already replaced it. + */ + private watchForCredentialRejection( + engine: AutomationEngine, + ctx: PluginContext, + info: { + name: string; + entry: DeclaredConnectorItem; + provider: string; + signature: string; + ref: string; + credential: Extract; + tracked: { signature: string; provider: string; close?: () => void | Promise }; + }, + handlers: Record, + ): Record { + let refused = false; + const watched: Record = {}; + for (const [key, handler] of Object.entries(handlers)) { + watched[key] = async (input, callCtx) => { + const result = await handler(input, callCtx); + const status = credentialRejectionStatusOf(result); + if (status === undefined || refused) return result; + refused = true; + const reason = credentialRejectedOnCallReason( + info.name, + info.ref, + status, + upstreamRefusalMessage((result as { body?: unknown }).body, info.credential.value), + ); + const run = this.reconcileQueue.then(async () => { + if (this.destroyed || this.materializedConnectors.get(info.name) !== info.tracked) return; + await this.holdConnectorForRejectedCredential(engine, ctx, { + name: info.name, + entry: info.entry, + provider: info.provider, + signature: info.signature, + live: info.tracked, + takeDown: true, + credential: info.credential, + reason, + }); + }); + this.reconcileQueue = run.catch(() => undefined); + try { + await run; + } catch (err) { + ctx.logger.warn( + `[Automation] connector instance '${info.name}' was refused its credential on a call, and could not be degraded.`, + describeThrownForLog(err), + ); + } + return result; + }; + } + return watched; + } + /** The action-less def a degraded instance registers (#3017). */ private buildDegradedHuskDef(name: string, entry: DeclaredConnectorItem): Connector { return { @@ -2671,10 +2841,11 @@ export class AutomationServicePlugin implements Plugin { */ private scheduleDeclarativeRetry(ctx: PluginContext): void { this.clearDeclarativeRetryTimer(); - if (this.destroyed || this.degradedInstances.size === 0) return; - const delay = Math.min( - ...[...this.degradedInstances.values()].map((d) => declarativeRetryDelayMs(d.attempts)), - ); + // [#27237] A held instance (its credential was refused) is not retried + // on a timer: the same value would be refused again. + const retrying = [...this.degradedInstances.values()].filter((d) => !d.held); + if (this.destroyed || retrying.length === 0) return; + const delay = Math.min(...retrying.map((d) => declarativeRetryDelayMs(d.attempts))); const timer = setTimeout(() => { this.declarativeRetryTimer = undefined; // Always soft: a background retry must never crash a running server. @@ -2683,7 +2854,7 @@ export class AutomationServicePlugin implements Plugin { (timer as unknown as { unref?: () => void }).unref?.(); this.declarativeRetryTimer = timer; ctx.logger.info( - `[Automation] ${this.degradedInstances.size} degraded connector instance(s); next retry in ${delay}ms`, + `[Automation] ${retrying.length} degraded connector instance(s); next retry in ${delay}ms`, ); } diff --git a/packages/spec/src/api/error-code-ledger.zod.ts b/packages/spec/src/api/error-code-ledger.zod.ts index 91f7a2bf848..ec2da2d0328 100644 --- a/packages/spec/src/api/error-code-ledger.zod.ts +++ b/packages/spec/src/api/error-code-ledger.zod.ts @@ -1558,6 +1558,13 @@ export const ERROR_CODE_LEDGER = { 'INVALID_REQUEST', ], '@objectstack/spec': [ + // [#27237] `door: 'none'` — `ConnectorCredentialRejectedError` + // (`integration/connector-provider-errors.ts`), thrown by a connector + // provider factory whose upstream refused the instance's credential (401 / + // 403, carried as `status`). The automation materializer catches it and + // degrades the instance; no HTTP door on this tree answers with it, the + // thrown value is the boundary. + 'CONNECTOR_CREDENTIAL_REJECTED', 'CONNECTOR_UPSTREAM_UNAVAILABLE', 'EXTERNAL_SCHEMA_MISMATCH', 'EXTERNAL_SCHEMA_MODE_VIOLATION', diff --git a/packages/spec/src/integration/connector-provider-errors.ts b/packages/spec/src/integration/connector-provider-errors.ts index 611af81a297..5b06c55d066 100644 --- a/packages/spec/src/integration/connector-provider-errors.ts +++ b/packages/spec/src/integration/connector-provider-errors.ts @@ -62,3 +62,51 @@ export function isConnectorUpstreamUnavailable(err: unknown): boolean { (err as { code?: unknown }).code === CONNECTOR_UPSTREAM_UNAVAILABLE ); } + +/** + * Marker `code` for a credential the upstream refused (HTTP 401 / 403) — #27237, + * ruling on #27237 (comment 6470087293). + * + * A third outcome beside the two above: neither an authoring fault (the + * instance is well-formed and its `credentialRef` resolved) nor an operational + * one (the upstream answered, and retrying the same value cannot change its + * answer). The materializer **degrades** the instance naming the refusal, as it + * does for an upstream outage, but does not retry it on a timer: the next + * attempt is the next change of the credential. When the value came from the + * credential store, its row is marked `rejected` with the upstream's message — + * never the value. + */ +export const CONNECTOR_CREDENTIAL_REJECTED = 'CONNECTOR_CREDENTIAL_REJECTED' as const; + +/** + * Thrown by a connector provider factory when the upstream it contacted to + * materialize an instance refused the credential it sent (HTTP 401 / 403). + * Carries {@link CONNECTOR_CREDENTIAL_REJECTED} as its `code` and the HTTP + * status. ⛔ The message must not carry the credential. + */ +export class ConnectorCredentialRejectedError extends Error { + readonly code = CONNECTOR_CREDENTIAL_REJECTED; + /** The upstream's HTTP status: 401 or 403. */ + readonly status: number; + /** The underlying failure, kept for diagnostics. */ + readonly cause?: unknown; + constructor(message: string, status: number, options?: { cause?: unknown }) { + super(message); + this.name = 'ConnectorCredentialRejectedError'; + this.status = status; + if (options && 'cause' in options) this.cause = options.cause; + } +} + +/** + * Structural check for the {@link CONNECTOR_CREDENTIAL_REJECTED} marker — reads + * `code`, never `instanceof`, for the reason {@link isConnectorUpstreamUnavailable} + * gives. + */ +export function isConnectorCredentialRejected(err: unknown): boolean { + return ( + typeof err === 'object' && + err !== null && + (err as { code?: unknown }).code === CONNECTOR_CREDENTIAL_REJECTED + ); +} From b35c8184a030521d558a8616aae4d445d6f55528 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 13:38:19 +0000 Subject: [PATCH 03/21] wip(automation): install-local credential refs, rejected-credential marker, tests Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- .../src/marketplace-install-local-plugin.ts | 86 +++- .../src/mcp-http-discovery-faults.test.ts | 14 +- .../connector-mcp/src/mcp-provider.test.ts | 19 +- ...tomation-connector-credential-door.test.ts | 232 +++++++++++ .../src/connector-credential-door.test.ts | 383 ++++++++++++++++++ .../src/connector-fail-cause.test.ts | 13 +- .../src/connector-materialization.test.ts | 21 +- .../src/credential-resolver.ts | 4 +- 8 files changed, 747 insertions(+), 25 deletions(-) create mode 100644 packages/runtime/src/domains/automation-connector-credential-door.test.ts create mode 100644 packages/services/service-automation/src/connector-credential-door.test.ts diff --git a/packages/cloud-connection/src/marketplace-install-local-plugin.ts b/packages/cloud-connection/src/marketplace-install-local-plugin.ts index f9bc5940f80..6b05aa79b5a 100644 --- a/packages/cloud-connection/src/marketplace-install-local-plugin.ts +++ b/packages/cloud-connection/src/marketplace-install-local-plugin.ts @@ -214,7 +214,9 @@ import { ConnectionCredentialStore } from './connection-credential-store.js'; import { readJsonRequestBody } from './request-json-body.js'; import { MARKETPLACE_INSTALLED_UI_BUNDLE } from './marketplace-ui.js'; import { matchSeedRows, purgeSeedRows, type PurgeSeedDataset } from './marketplace-install-local-purge.js'; -import type { AuthSessionApi, IAuthService, IHttpServer, IMetadataService, IObjectQLEngine } from '@objectstack/spec/contracts'; +import type { + AuthSessionApi, ConnectorCredentialRef, IAuthService, IAutomationService, IHttpServer, IMetadataService, IObjectQLEngine, +} from '@objectstack/spec/contracts'; import type { DeletePackageRequest, UninstallCleanupOutcome, @@ -734,6 +736,29 @@ interface InstallPrincipal { * held key is served again the moment its owner remediates (ruling C′ item * 4). `null` when the principal is not held. */ +/** + * [#27237] One connector credential an installed package names, as the install + * answer reports it. `status` is the automation service's (`set` / `missing` / + * `rejected`), or `unknown` when it could not be read. Never a value. + */ +export interface InstalledCredentialRef { + ref: string; + connectors: string[]; + status: ConnectorCredentialRef['status'] | 'unknown'; + source: ConnectorCredentialRef['source']; +} + +/** [#27237] The install answer's sentence for the credentials not set yet. */ +export function installedCredentialNote(refs: readonly string[]): string { + const list = refs.map((r) => `'${r}'`).join(', '); + return ( + `Connector credential${refs.length === 1 ? '' : 's'} ${list} ${refs.length === 1 ? 'needs' : 'need'} an ` + + "administrator — not set, refused by the upstream, or unreadable now (each entry's `status` says which). The " + + 'connector instances naming them are installed degraded and materialize once each is set: ' + + 'PUT /api/v1/automation/connectors/credential-refs/:ref, or a server environment variable of the same name.' + ); +} + function refuseHeldSession(c: any, principal: InstallPrincipal): Response | null { if (!principal.authGate) return null; const { code, message } = principal.authGate; @@ -1954,6 +1979,12 @@ export class MarketplaceInstallLocalPlugin implements Plugin { // would fire on every seeded row, which no boot does. await this.announceHotInstall(ctx, manifestId); + // 7. [#27237] Which connector credentials the package names, and which + // are not set yet — after the re-sync above, so the listing reads + // what the reconcile it triggered saw. + const credentialRefs = await this.describeInstalledCredentialRefs(ctx, converted); + const unsetRefs = (credentialRefs ?? []).filter((r) => r.status !== 'set').map((r) => r.ref); + return c.json({ success: true, data: { @@ -1969,6 +2000,10 @@ export class MarketplaceInstallLocalPlugin implements Plugin { // ONLY there: omitted, not nulled, on a hot-loaded install, so // the success answer's key set is unchanged. ...(hotLoadError !== undefined ? { hotLoadError } : {}), + // [#27237] Omitted, not emptied, for a package whose connectors + // name no credential, so that answer's key set is unchanged. + ...(credentialRefs !== undefined ? { credentialRefs } : {}), + ...(unsetRefs.length > 0 ? { credentialNote: installedCredentialNote(unsetRefs) } : {}), upgradedFrom: conflict === 'marketplace' ? 'previous-marketplace-version' : null, translationsLoaded: seededSummary.translationsLoaded, seeded: seededSummary.seeded, @@ -3464,6 +3499,55 @@ export class MarketplaceInstallLocalPlugin implements Plugin { return { jobs: collectJobs ? collectJobs(manifest) : [], hooks: collectHooks ? collectHooks(manifest) : [] }; }; + /** + * [#27237] The connector credentials an installed package's declarative + * connectors name (`connectors[].auth.credentialRef`, ruling A on #27230), + * each with its status and source as the automation service's credential + * listing answers them — never a value. `undefined` when no connector of + * the package names one. + * + * A reference that is not set does not refuse the install: the instance + * naming it installs degraded and materializes when an administrator sets + * it (`PUT /api/v1/automation/connectors/credential-refs/:ref`). With no + * automation service, or one that cannot list, the status is `unknown`. + */ + private describeInstalledCredentialRefs = async ( + ctx: PluginContext, + converted: unknown, + ): Promise => { + const connectors = (converted as { connectors?: unknown } | null)?.connectors; + const declared = new Map(); + for (const raw of Array.isArray(connectors) ? connectors : []) { + const entry = raw as { name?: unknown; provider?: unknown; auth?: { credentialRef?: unknown } } | null; + if (typeof entry?.name !== 'string' || typeof entry.provider !== 'string') continue; + const ref = entry.auth?.credentialRef; + if (typeof ref !== 'string' || ref.length === 0) continue; + declared.set(ref, [...(declared.get(ref) ?? []), entry.name]); + } + if (declared.size === 0) return undefined; + let listing: Map | undefined; + try { + const automation = ctx.getService>('automation'); + if (typeof automation?.listConnectorCredentialRefs === 'function') { + listing = new Map((await automation.listConnectorCredentialRefs()).map((e) => [e.ref, e])); + } + } catch (err: any) { + ctx.logger?.warn?.( + `[MarketplaceInstallLocal] the connector credential listing could not be read, so the install answer reports ` + + `each credential the package names as status 'unknown'. Cause: ${err?.message ?? err}`, + ); + } + return [...declared.keys()].sort().map((ref) => { + const known = listing?.get(ref); + return { + ref, + connectors: declared.get(ref)!, + status: listing === undefined ? 'unknown' : (known?.status ?? 'missing'), + source: known?.source ?? null, + }; + }); + }; + /** * [#21322] Announce a HOT install to the running kernel as * `metadata:reloaded` — the platform's one post-boot re-sync signal, which diff --git a/packages/connectors/connector-mcp/src/mcp-http-discovery-faults.test.ts b/packages/connectors/connector-mcp/src/mcp-http-discovery-faults.test.ts index d0fb33e64c1..c2e2a1dda71 100644 --- a/packages/connectors/connector-mcp/src/mcp-http-discovery-faults.test.ts +++ b/packages/connectors/connector-mcp/src/mcp-http-discovery-faults.test.ts @@ -12,7 +12,7 @@ import { createServer, type IncomingMessage, type Server, type ServerResponse } import type { AddressInfo } from 'node:net'; import { afterEach, describe, expect, it } from 'vitest'; import { LiteKernel } from '@objectstack/core'; -import { isConnectorUpstreamUnavailable } from '@objectstack/spec/integration'; +import { isConnectorCredentialRejected, isConnectorUpstreamUnavailable } from '@objectstack/spec/integration'; import { AutomationServicePlugin, type AutomationEngine } from '@objectstack/service-automation'; import { ConnectorMcpPlugin } from './connector-mcp-plugin.js'; import { createMcpConnector, MCP_TOOLS_LIST_MAX_PAGES } from './mcp-connector.js'; @@ -136,7 +136,7 @@ describe('tools/list pagination — real Streamable-HTTP server', () => { describe('credential rejection — real Streamable-HTTP server', () => { it.each([401, 403])( - 'an HTTP %i on the handshake is a plain configuration fault (not upstream-unavailable), sent once', + 'an HTTP %i on the handshake is a credential rejection (not upstream-unavailable), sent once', async (status) => { const server = await startServer({ reject: status }); const factory = createMcpProviderFactory(); @@ -153,6 +153,7 @@ describe('credential rejection — real Streamable-HTTP server', () => { ); expect(err).toBeInstanceOf(Error); expect(isConnectorUpstreamUnavailable(err)).toBe(false); + expect(isConnectorCredentialRejected(err)).toBe(true); expect((err as Error).message).toContain( `connector-mcp provider: connector 'vendor' was refused by its MCP server (HTTP ${status})`, ); @@ -161,7 +162,10 @@ describe('credential rejection — real Streamable-HTTP server', () => { }, ); - it('a declarative instance answered 401 fails the boot instead of degrading into a retry loop', async () => { + // [#27237] Ruling on #27237 (comment 6470087293): a refused credential + // degrades the instance, naming the refusal — not a failed boot — and the + // instance is held, never retried against the same credential. + it('a declarative instance answered 401 degrades, naming the refusal, and is not retried', async () => { const server = await startServer({ reject: 401 }); const declared = [ { @@ -188,9 +192,9 @@ describe('credential rejection — real Streamable-HTTP server', () => { } as never); kernel.use(new ConnectorMcpPlugin()); try { - await expect(kernel.bootstrap()).rejects.toThrow(/was refused by its MCP server \(HTTP 401\)/); + await kernel.bootstrap(); const engine = kernel.getService('automation') as AutomationEngine; - expect(engine.getConnectorDegradedReason('vendor_mcp')).toBeUndefined(); + expect(engine.getConnectorDegradedReason('vendor_mcp')).toMatch(/was refused by its MCP server \(HTTP 401\)/); // Give a (wrongly) scheduled retry the chance to fire: none does. await new Promise((r) => setTimeout(r, 300)); expect(server.requests).toEqual(['POST']); diff --git a/packages/connectors/connector-mcp/src/mcp-provider.test.ts b/packages/connectors/connector-mcp/src/mcp-provider.test.ts index 6fa5aa1f7b0..dd6eaeb5ff1 100644 --- a/packages/connectors/connector-mcp/src/mcp-provider.test.ts +++ b/packages/connectors/connector-mcp/src/mcp-provider.test.ts @@ -9,7 +9,7 @@ import { UnauthorizedError } from '@modelcontextprotocol/sdk/client/auth.js'; import { StreamableHTTPError } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; import { McpError } from '@modelcontextprotocol/sdk/types.js'; import type { ConnectorProviderContext } from '@objectstack/spec/integration'; -import { isConnectorUpstreamUnavailable } from '@objectstack/spec/integration'; +import { isConnectorCredentialRejected, isConnectorUpstreamUnavailable } from '@objectstack/spec/integration'; import type { McpClientLike, McpToolDescriptor, McpTransport } from './mcp-connector.js'; import { createMcpProviderFactory, MCP_PROVIDER_KEY } from './mcp-provider.js'; @@ -136,9 +136,10 @@ describe('mcp provider fault classification (#3017)', () => { }); }); -// A rejected credential is a configuration fault: it is thrown plain (fatal at -// boot, never retried), told apart from an operational fault by the SDK's own -// HTTP-transport error, wherever discovery raised it. +// A rejected credential is a configuration fault, told apart from an operational +// fault by the SDK's own HTTP-transport error, wherever discovery raised it. +// [#27237] Thrown as the CONNECTOR_CREDENTIAL_REJECTED marker, carrying the +// status: the materializer degrades the instance and never retries it. describe('mcp provider — a rejected credential is a configuration fault', () => { const http = { transport: { kind: 'http', url: 'https://mcp.vendor.example/mcp' } }; @@ -150,19 +151,21 @@ describe('mcp provider — a rejected credential is a configuration fault', () = ); } - it.each([401, 403])('HTTP %i on connect / handshake throws plain, naming the status', async (status) => { + it.each([401, 403])('HTTP %i on connect / handshake throws the credential-rejected marker, naming the status', async (status) => { const err = await rejectionOf(async () => { throw new StreamableHTTPError(status, 'Error POSTing to endpoint: {"error":"invalid_token"}'); }); expect(err).toBeInstanceOf(Error); expect(isConnectorUpstreamUnavailable(err)).toBe(false); + expect(isConnectorCredentialRejected(err)).toBe(true); + expect((err as { status?: unknown }).status).toBe(status); expect((err as Error).message).toContain( `connector-mcp provider: connector 'github' was refused by its MCP server (HTTP ${status})`, ); expect((err as Error).message).toContain('is not retried'); }); - it('HTTP 401 on tools/list throws plain too, and closes the client', async () => { + it('HTTP 401 on tools/list throws the marker too, and closes the client', async () => { let closed = false; const err = await rejectionOf(async () => ({ listTools: async () => { throw new StreamableHTTPError(401, 'Error POSTing to endpoint: unauthorized'); }, @@ -170,6 +173,7 @@ describe('mcp provider — a rejected credential is a configuration fault', () = close: async () => { closed = true; }, })); expect(isConnectorUpstreamUnavailable(err)).toBe(false); + expect(isConnectorCredentialRejected(err)).toBe(true); expect((err as Error).message).toContain('(HTTP 401)'); expect(closed).toBe(true); }); @@ -177,6 +181,7 @@ describe('mcp provider — a rejected credential is a configuration fault', () = it("the SDK's UnauthorizedError, or a 401 in the cause chain, is the same fault", async () => { const direct = await rejectionOf(async () => { throw new UnauthorizedError(); }); expect(isConnectorUpstreamUnavailable(direct)).toBe(false); + expect(isConnectorCredentialRejected(direct)).toBe(true); expect((direct as Error).message).toContain('(HTTP 401)'); const wrapped = Object.assign(new Error('handshake failed'), { cause: new StreamableHTTPError(403, 'forbidden') }); const nested = await rejectionOf(async () => { throw wrapped; }); @@ -187,8 +192,10 @@ describe('mcp provider — a rejected credential is a configuration fault', () = it('control: other HTTP statuses and a JSON-RPC error with code 401 stay upstream-unavailable', async () => { const status503 = await rejectionOf(async () => { throw new StreamableHTTPError(503, 'unavailable'); }); expect(isConnectorUpstreamUnavailable(status503)).toBe(true); + expect(isConnectorCredentialRejected(status503)).toBe(false); const jsonRpc401 = await rejectionOf(async () => { throw new McpError(401, 'tool-level error code'); }); expect(isConnectorUpstreamUnavailable(jsonRpc401)).toBe(true); + expect(isConnectorCredentialRejected(jsonRpc401)).toBe(false); }); }); diff --git a/packages/runtime/src/domains/automation-connector-credential-door.test.ts b/packages/runtime/src/domains/automation-connector-credential-door.test.ts new file mode 100644 index 00000000000..7e68f428504 --- /dev/null +++ b/packages/runtime/src/domains/automation-connector-credential-door.test.ts @@ -0,0 +1,232 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// [#27237] The connector credential door — `GET /automation/connectors/ +// credential-refs`, `PUT` / `DELETE …/credential-refs/:ref` (ruling A on +// #27230). +// +// What is pinned here is the DOOR: who may reach it (`manage_metadata`, then +// no walled posture at all — S1 on #27230), what it accepts (no query +// parameter, a closed `{ value }` body), how the service's refusals reach the +// wire, and that the value it carries in never travels back out — not in a +// success body and not in an error body. The store, the resolver chain and the +// reconcile behind it are pinned in service-automation +// (`connector-credential-door.test.ts`). + +import { describe, it, expect, vi } from 'vitest'; + +import { HttpDispatcher } from '../http-dispatcher.js'; +import type { HttpProtocolContext } from '../http-dispatcher.js'; +import { connectorCredentialWalledMessage } from './automation.js'; + +/** An obvious dummy; never a real credential. */ +const DUMMY_VALUE = 'dummy-credential-value-0001'; +const REF = 'BILLING_TOKEN'; + +const ENTRY = { + ref: REF, + status: 'set', + source: 'object', + writable: true, + declared: true, + connectors: [{ name: 'billing', state: 'ready' }], +}; + +interface Harness { + dispatcher: HttpDispatcher; + list: ReturnType; + set: ReturnType; + clear: ReturnType; +} + +function boot(opts: { + posture?: 'single' | 'group' | 'isolated' | null; + service?: Record; + set?: (req: any) => Promise; +} = {}): Harness { + const list = vi.fn(async () => [ENTRY]); + const set = vi.fn(opts.set ?? (async () => ENTRY)); + const clear = vi.fn(async () => ({ ...ENTRY, status: 'missing', source: null })); + const services: Record = { + automation: opts.service ?? { + handlerReady: true, + listConnectorCredentialRefs: list, + setConnectorCredential: set, + clearConnectorCredential: clear, + }, + }; + const posture = opts.posture === undefined ? 'single' : opts.posture; + if (posture) services.tenancy = { posture }; + const resolve = (name: string): unknown => services[name]; + const kernel = { + getService: resolve, + getServiceAsync: async (name: string) => resolve(name), + context: { getService: resolve }, + }; + return { dispatcher: new HttpDispatcher(kernel as never), list, set, clear }; +} + +const ADMIN = (): HttpProtocolContext => ({ + request: {}, + executionContext: { userId: 'u_admin', positions: [], permissions: [], systemPermissions: ['manage_metadata'] }, +} as HttpProtocolContext); + +const MEMBER = (): HttpProtocolContext => ({ + request: {}, + executionContext: { userId: 'u_member', positions: [], permissions: [], systemPermissions: [] }, +} as HttpProtocolContext); + +/** The platform operator — refused under a wall all the same: the ruling admits nobody there. */ +const OPERATOR = (): HttpProtocolContext => ({ + request: {}, + executionContext: { + userId: 'u_operator', + positions: ['platform_admin'], + permissions: ['admin_full_access'], + systemPermissions: ['manage_metadata'], + posture: 'PLATFORM_ADMIN', + }, +} as HttpProtocolContext); + +const ANONYMOUS = (): HttpProtocolContext => ({ request: {}, executionContext: {} } as HttpProtocolContext); + +const statusOf = (r: unknown): unknown => (r as any)?.status; +const codeOf = (r: unknown): unknown => (r as any)?.body?.error?.code; +const messageOf = (r: unknown): string => String((r as any)?.body?.error?.message ?? ''); + +const get = (h: Harness, ctx: HttpProtocolContext, query?: unknown) => + h.dispatcher.handleAutomation('/connectors/credential-refs', 'GET', undefined, ctx, query); +const put = (h: Harness, ctx: HttpProtocolContext, body: unknown, ref = REF, query?: unknown) => + h.dispatcher.handleAutomation(`/connectors/credential-refs/${encodeURIComponent(ref)}`, 'PUT', body, ctx, query); +const del = (h: Harness, ctx: HttpProtocolContext, ref = REF) => + h.dispatcher.handleAutomation(`/connectors/credential-refs/${encodeURIComponent(ref)}`, 'DELETE', undefined, ctx, undefined); + +describe('[#27237] connector credential door — who may reach it', () => { + it('answers an anonymous caller 401 before anything else', async () => { + const h = boot(); + const { response } = await get(h, ANONYMOUS()); + expect(statusOf(response)).toBe(401); + expect(h.list).not.toHaveBeenCalled(); + }); + + it.each([ + ['GET', (h: Harness) => get(h, MEMBER())], + ['PUT', (h: Harness) => put(h, MEMBER(), { value: DUMMY_VALUE })], + ['DELETE', (h: Harness) => del(h, MEMBER())], + ] as const)('%s without `manage_metadata` is refused 403 and never reaches the service', async (_verb, call) => { + const h = boot(); + const { response } = await call(h); + expect(statusOf(response)).toBe(403); + expect(codeOf(response)).toBe('PERMISSION_DENIED'); + expect(messageOf(response)).toContain('`manage_metadata`'); + expect(h.list).not.toHaveBeenCalled(); + expect(h.set).not.toHaveBeenCalled(); + expect(h.clear).not.toHaveBeenCalled(); + }); + + for (const posture of ['group', 'isolated'] as const) { + it.each([ + ['GET', (h: Harness, ctx: HttpProtocolContext) => get(h, ctx)], + ['PUT', (h: Harness, ctx: HttpProtocolContext) => put(h, ctx, { value: DUMMY_VALUE })], + ['DELETE', (h: Harness, ctx: HttpProtocolContext) => del(h, ctx)], + ] as const)(`\`${posture}\`: %s is refused for every caller, naming the posture`, async (_verb, call) => { + for (const ctx of [ADMIN(), OPERATOR()]) { + const h = boot({ posture }); + const { response } = await call(h, ctx); + expect(statusOf(response)).toBe(403); + expect(codeOf(response)).toBe('PERMISSION_DENIED'); + expect(messageOf(response)).toBe(connectorCredentialWalledMessage(posture)); + expect(messageOf(response)).toContain(`'${posture}' tenancy posture`); + expect(h.list).not.toHaveBeenCalled(); + expect(h.set).not.toHaveBeenCalled(); + expect(h.clear).not.toHaveBeenCalled(); + } + }); + } + + it('`single` and no tenancy service at all admit the `manage_metadata` holder', async () => { + for (const posture of ['single', null] as const) { + const h = boot({ posture }); + const { response } = await get(h, ADMIN()); + expect(statusOf(response)).toBe(200); + expect((response as any).body.data).toEqual({ credentialRefs: [ENTRY], total: 1 }); + } + }); +}); + +describe('[#27237] connector credential door — what it accepts', () => { + it('route rule 5: any query parameter is refused 400, naming it', async () => { + const h = boot(); + const { response } = await get(h, ADMIN(), { status: 'missing' }); + expect(statusOf(response)).toBe(400); + expect(codeOf(response)).toBe('VALIDATION_ERROR'); + expect(messageOf(response)).toContain('status'); + expect(h.list).not.toHaveBeenCalled(); + const { response: putResponse } = await put(h, ADMIN(), { value: DUMMY_VALUE }, REF, { force: 'true' }); + expect(statusOf(putResponse)).toBe(400); + expect(h.set).not.toHaveBeenCalled(); + }); + + it('PUT hands the decoded reference, the value and the caller to the service, and answers the entry without the value', async () => { + const h = boot(); + const { response } = await put(h, ADMIN(), { value: DUMMY_VALUE }, 'VENDOR/KEY'); + expect(statusOf(response)).toBe(200); + expect(h.set).toHaveBeenCalledWith(expect.objectContaining({ + ref: 'VENDOR/KEY', + value: DUMMY_VALUE, + context: expect.objectContaining({ userId: 'u_admin' }), + })); + expect((response as any).body.data).toEqual(ENTRY); + expect(JSON.stringify(response)).not.toContain(DUMMY_VALUE); + }); + + it('PUT refuses a body that is not an object, and an unknown key by NAME, never echoing what it carried', async () => { + const h = boot(); + const notObject = await put(h, ADMIN(), DUMMY_VALUE); + expect(statusOf(notObject.response)).toBe(400); + expect(JSON.stringify(notObject.response)).not.toContain(DUMMY_VALUE); + + const misspelled = await put(h, ADMIN(), { secret: DUMMY_VALUE }); + expect(statusOf(misspelled.response)).toBe(400); + expect(codeOf(misspelled.response)).toBe('VALIDATION_ERROR'); + expect(messageOf(misspelled.response)).toContain('`secret`'); + expect(JSON.stringify(misspelled.response)).not.toContain(DUMMY_VALUE); + expect(h.set).not.toHaveBeenCalled(); + }); + + it("answers the service's own refusals with their status and code", async () => { + const refusal = (message: string, code: string, status: number) => + Object.assign(new Error(message), { code, status }); + for (const [code, status] of [ + ['VALIDATION_ERROR', 400], + ['RESOURCE_NOT_FOUND', 404], + ['RESOURCE_CONFLICT', 409], + ['SERVICE_UNAVAILABLE', 503], + ] as const) { + const h = boot({ set: async () => { throw refusal(`refused ${code}`, code, status); } }); + const { response } = await put(h, ADMIN(), { value: DUMMY_VALUE }); + expect(statusOf(response)).toBe(status); + expect(codeOf(response)).toBe(code); + expect(messageOf(response)).toBe(`refused ${code}`); + expect(JSON.stringify(response)).not.toContain(DUMMY_VALUE); + } + }); + + it('DELETE hands the reference and the caller to the service', async () => { + const h = boot(); + const { response } = await del(h, ADMIN()); + expect(statusOf(response)).toBe(200); + expect(h.clear).toHaveBeenCalledWith(expect.objectContaining({ + ref: REF, + context: expect.objectContaining({ userId: 'u_admin' }), + })); + }); + + it('a service without the optional members answers 501, never a 200 or a fall-through', async () => { + const h = boot({ service: { handlerReady: true } }); + for (const call of [() => get(h, ADMIN()), () => put(h, ADMIN(), { value: DUMMY_VALUE }), () => del(h, ADMIN())]) { + const { response } = await call(); + expect(statusOf(response)).toBe(501); + expect(codeOf(response)).toBe('NOT_IMPLEMENTED'); + } + }); +}); diff --git a/packages/services/service-automation/src/connector-credential-door.test.ts b/packages/services/service-automation/src/connector-credential-door.test.ts new file mode 100644 index 00000000000..3b3cbed90b0 --- /dev/null +++ b/packages/services/service-automation/src/connector-credential-door.test.ts @@ -0,0 +1,383 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#27237] The connector credential store, the resolver chain and the + * reconcile a credential write runs (ruling A on #27230) — on a real ObjectQL + * over SQLite, so the `Field.secret` seam (#7799) is the engine's, not a + * double's. + * + * The acceptance list, one block each: + * + * - a missing reference degrades the instance naming it; a set → ready, with + * no restart; a clear → degraded again; + * - the value appears nowhere: not in a log record, the listing, `GET + * /connectors`' descriptors, an engine read of the object, or the audit's + * copy of the row; + * - an environment variable of the same name wins, and the listing says + * `source: env`; + * - a wrong value → the instance degraded and the row `rejected` (ruling on + * #27237, comment 6470087293): at materialization (the mcp shape) and on + * the first call answering 401 / 403 (the rest / openapi shape). + * + * The values below are obvious dummies. + */ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { LiteKernel } from '@objectstack/core'; +import { ObjectQL } from '@objectstack/objectql'; +import { collectMaskedReadFields } from '@objectstack/objectql/core'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { SysSecret } from '@objectstack/platform-objects'; +import type { CryptoContext, CryptoHandle, ICryptoProvider } from '@objectstack/spec/contracts'; +import { + ConnectorCredentialRejectedError, + type Connector, + type ConnectorProviderContext, + type ConnectorProviderFactory, +} from '@objectstack/spec/integration'; + +import { AutomationServicePlugin, type CredentialResolver } from './plugin.js'; +import type { AutomationEngine } from './engine.js'; +import { SysConnectorCredential } from './sys-connector-credential.object.js'; +import { SysFlowCredential } from './sys-flow-credential.object.js'; +import { CONNECTOR_CREDENTIAL_OBJECT } from './connector-credentials.js'; + +const REF = 'DUMMY_BILLING_TOKEN_27237'; +const DUMMY_VALUE = 'dummy-value-aaaa-0001'; +const DUMMY_VALUE_2 = 'dummy-value-bbbb-0002'; +const DUMMY_ENV_VALUE = 'dummy-env-value-cccc-0003'; +const SYSTEM = { isSystem: true } as const; + +const flush = () => new Promise((r) => setTimeout(r, 0)); + +/** Reversible stand-in cipher — the seam's behaviour is the engine's, not the cipher's. */ +function fakeCrypto(): ICryptoProvider { + let n = 0; + return { + async encrypt(plain: string, _ctx: CryptoContext): Promise { + n += 1; + return { id: `sec_${n}`, kmsKeyId: 'test', alg: 'test-rev', version: 1, ciphertext: [...plain].reverse().join('') }; + }, + async decrypt(handle: CryptoHandle): Promise { + return [...handle.ciphertext].reverse().join(''); + }, + async rotateKey(handle: CryptoHandle): Promise { + return { ...handle, version: handle.version + 1 }; + }, + digest: (plain: string) => `d:${plain.length}`, + keyedDigest: async (plain: string) => `k:${plain.length}`, + }; +} + +/** A provider-bound instance naming {@link REF}. */ +function billing(name = 'billing') { + return { + name, + label: name, + type: 'api', + provider: 'fake', + providerConfig: {}, + auth: { type: 'bearer', credentialRef: REF }, + _packageId: 'com.example.billing', + }; +} + +/** + * A fake provider in the shape of the real ones: `request` answers `{ status, + * ok, body }` like rest / openapi, and `rejectAtMaterialization` throws the + * marker like mcp. The upstream ECHOES the token it was sent, so a scrub that + * stopped working shows up. + */ +function fakeProvider() { + const state = { + materialized: [] as ConnectorProviderContext[], + rejectAtMaterialization: false, + callStatus: 200, + }; + const tokenOf = (ctx: ConnectorProviderContext) => (ctx.auth?.type === 'bearer' ? ctx.auth.token : undefined); + const factory: ConnectorProviderFactory = (ctx) => { + state.materialized.push(ctx); + if (state.rejectAtMaterialization) { + throw new ConnectorCredentialRejectedError( + `fake provider: connector '${ctx.name}' was refused by its server (HTTP 401): invalid token ${tokenOf(ctx)}`, + 401, + ); + } + return { + def: { + name: ctx.name, + label: ctx.label, + type: 'api', + authentication: { type: 'none' }, + actions: [{ key: 'request', label: 'Request' }], + } as unknown as Connector, + handlers: { + request: async () => (state.callStatus === 200 + ? { status: 200, ok: true, body: { invoices: [] } } + : { status: state.callStatus, ok: false, body: { message: `token ${tokenOf(ctx)} is not valid` } }), + }, + }; + }; + return { factory, state, tokenOf }; +} + +const engines: ObjectQL[] = []; +const kernels: LiteKernel[] = []; +const envKeys: string[] = []; +afterEach(async () => { + for (const k of kernels.splice(0)) await k.shutdown().catch(() => {}); + for (const e of engines.splice(0)) { + try { await (e as any).destroy?.(); } catch { /* noop */ } + } + for (const key of envKeys.splice(0)) delete process.env[key]; + vi.restoreAllMocks(); +}); + +/** + * Everything the process prints while a test runs — the kernel logger at + * `debug` in JSON (stdout and stderr) and any `console` call. + */ +function captureOutput(): { text: () => string } { + const chunks: string[] = []; + const keep = ((c: string | Uint8Array) => { chunks.push(String(c)); return true; }) as never; + vi.spyOn(process.stdout, 'write').mockImplementation(keep); + vi.spyOn(process.stderr, 'write').mockImplementation(keep); + for (const m of ['log', 'info', 'warn', 'error', 'debug'] as const) { + vi.spyOn(console, m).mockImplementation((...args: unknown[]) => { chunks.push(args.map(String).join(' ')); }); + } + return { text: () => chunks.join('\n') }; +} + +async function dataEngine(): Promise { + const ql = new ObjectQL(); + engines.push(ql); + const driver = new SqlDriver({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }); + await driver.connect(); + ql.registerDriver(driver, true); + await ql.init(); + (ql as any).registry.registerObject(SysSecret as never, 'test'); + (ql as any).registry.registerObject(SysConnectorCredential as never, 'test'); + (ql as any).registry.registerObject(SysFlowCredential as never, 'test'); + await ql.syncSchemas(); + ql.setCryptoProvider(fakeCrypto()); + return ql; +} + +async function boot(declared: unknown[], opts: { credentialResolver?: CredentialResolver } = {}) { + const ql = await dataEngine(); + const provider = fakeProvider(); + const kernel = new LiteKernel({ logger: { level: 'debug', format: 'json' } } as never); + kernels.push(kernel); + kernel.use(new AutomationServicePlugin({ suspendedRunStore: 'memory', credentialResolver: opts.credentialResolver ?? (() => undefined) })); + let captured: any; + kernel.use({ + name: 'test.harness', + type: 'standard' as const, + version: '1.0.0', + dependencies: ['com.objectstack.service-automation'], + async init(ctx: any) { + captured = ctx; + // The declared connectors, as the registry serves them; the data + // engine under `data`, where the plugin's resolution finds it. + ctx.registerService('objectql', { registry: { listItems: (t: string) => (t === 'connector' ? declared : []) } }); + ctx.registerService('data', ql); + ctx.getService('automation').registerConnectorProvider('fake', provider.factory); + }, + async start() {}, + } as never); + await kernel.bootstrap(); + await flush(); + const engine = kernel.getService('automation') as AutomationEngine; + const call = async (name = 'billing') => engine.resolveConnectorAction(name, 'request')!({}, {} as never); + /** A soft reconcile with nothing changed — the `metadata:reloaded` path. */ + const reload = async () => { + await captured.trigger('metadata:reloaded'); + await flush(); + }; + return { ql, engine, provider, call, reload }; +} + +const ADMIN = { userId: 'usr_admin_27237' } as never; + +describe('[#27237] a missing credential degrades its instance; setting it materializes it, no restart', () => { + it('missing → degraded naming the reference; set → ready; clear → degraded again', async () => { + const { engine, provider, ql } = await boot([billing()]); + + // Missing: no arm of the chain answers — degraded, named, factory never asked. + expect(provider.state.materialized).toHaveLength(0); + expect(engine.getConnectorDegradedReason('billing')).toContain(`credential '${REF}'`); + expect(engine.getConnectorDegradedReason('billing')).toContain(`/api/v1/automation/connectors/credential-refs/${REF}`); + const [missing] = await engine.listConnectorCredentialRefs(); + expect(missing).toMatchObject({ ref: REF, status: 'missing', source: null, writable: true, declared: true }); + expect(missing.connectors).toEqual([ + expect.objectContaining({ name: 'billing', packageId: 'com.example.billing', state: 'degraded' }), + ]); + + // Set: the same kernel, the instance materializes with the value. The + // write carries the caller's user id, which the audit stamps record. + const insert = vi.spyOn(ql, 'insert'); + const entry = await engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE, context: ADMIN }); + expect(insert).toHaveBeenCalledWith( + CONNECTOR_CREDENTIAL_OBJECT, + expect.anything(), + { context: expect.objectContaining({ isSystem: true, userId: 'usr_admin_27237' }) }, + ); + expect(engine.getConnectorDegradedReason('billing')).toBeUndefined(); + expect(engine.getRegisteredConnectors()).toContain('billing'); + expect(provider.state.materialized).toHaveLength(1); + expect(provider.tokenOf(provider.state.materialized[0])).toBe(DUMMY_VALUE); + expect(entry).toMatchObject({ ref: REF, status: 'set', source: 'object' }); + expect(entry.connectors[0]).toMatchObject({ name: 'billing', state: 'ready' }); + expect(entry.lastVerifiedAt).toBeDefined(); + + // Set again: a new revision re-materializes with the new value. + await engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE_2, context: ADMIN }); + expect(provider.state.materialized).toHaveLength(2); + expect(provider.tokenOf(provider.state.materialized[1])).toBe(DUMMY_VALUE_2); + + // Clear: torn down and degraded again, naming the reference. + const cleared = await engine.clearConnectorCredential({ ref: REF, context: ADMIN }); + expect(cleared).toMatchObject({ ref: REF, status: 'missing', source: null }); + expect(engine.getConnectorDegradedReason('billing')).toContain(`credential '${REF}'`); + }); + + it('refuses an undeclared reference (404), an empty value (400), and a clear of nothing stored (404) — writing nothing', async () => { + const { engine, ql } = await boot([billing()]); + await expect(engine.setConnectorCredential({ ref: 'NOT_DECLARED_ANYWHERE', value: DUMMY_VALUE, context: ADMIN })) + .rejects.toMatchObject({ status: 404, code: 'RESOURCE_NOT_FOUND' }); + await expect(engine.setConnectorCredential({ ref: REF, value: '', context: ADMIN })) + .rejects.toMatchObject({ status: 400, code: 'VALIDATION_ERROR' }); + await expect(engine.clearConnectorCredential({ ref: REF, context: ADMIN })) + .rejects.toMatchObject({ status: 404, code: 'RESOURCE_NOT_FOUND' }); + expect(await ql.find(CONNECTOR_CREDENTIAL_OBJECT, { context: SYSTEM } as never)).toHaveLength(0); + }); +}); + +describe('[#27237] the value appears nowhere it is not used', () => { + it('not in a log record, the listing, the connector descriptors or an engine read of the row', async () => { + const out = captureOutput(); + const { engine, ql } = await boot([billing()]); + await engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE, context: ADMIN }); + await engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE_2, context: ADMIN }); + + const listing = JSON.stringify(await engine.listConnectorCredentialRefs()); + const descriptors = JSON.stringify(engine.getConnectorDescriptors()); + const rows = (await ql.find(CONNECTOR_CREDENTIAL_OBJECT, { context: SYSTEM } as never)) as any[]; + await engine.clearConnectorCredential({ ref: REF, context: ADMIN }); + + expect(rows).toHaveLength(1); + expect(rows[0].value).not.toBe(DUMMY_VALUE_2); // masked on the engine's read + for (const [where, text] of [ + ['listing', listing], + ['descriptors', descriptors], + ['engine read', JSON.stringify(rows)], + ['logs', out.text()], + ] as const) { + expect(text.includes(DUMMY_VALUE), where).toBe(false); + expect(text.includes(DUMMY_VALUE_2), where).toBe(false); + } + // The logs did record the writes — the pin is not passing on silence. + expect(out.text()).toContain(`connector credential '${REF}' set (revision 2) by user 'usr_admin_27237'`); + }); + + it("the audit copy masks it: `value` is among the fields plugin-audit's writers mask", () => { + // plugin-audit masks exactly `collectMaskedReadFields(objectDef)` on + // every side of a row it records (`audit-writers.ts`, #6656). + expect(collectMaskedReadFields(SysConnectorCredential as never)).toEqual(['value']); + }); +}); + +describe('[#27237] an environment variable of the same name wins', () => { + it('the listing says `source: env`, the instance gets the env value, and the door will not store a shadowed value', async () => { + process.env[REF] = DUMMY_ENV_VALUE; + envKeys.push(REF); + const { engine, provider } = await boot([billing()]); + expect(provider.tokenOf(provider.state.materialized[0])).toBe(DUMMY_ENV_VALUE); + const [entry] = await engine.listConnectorCredentialRefs(); + expect(entry).toMatchObject({ ref: REF, status: 'set', source: 'env', writable: false }); + await expect(engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE, context: ADMIN })) + .rejects.toMatchObject({ status: 409, code: 'RESOURCE_CONFLICT' }); + }); + + it('order: env, then the stored row, then the host resolver', async () => { + const vault = vi.fn(async (ref: string) => (ref === REF ? 'dummy-vault-value-dddd-0004' : undefined)); + const { engine, provider } = await boot([billing()], { credentialResolver: vault }); + // Only the vault answers: it is the source. + expect(provider.tokenOf(provider.state.materialized[0])).toBe('dummy-vault-value-dddd-0004'); + expect((await engine.listConnectorCredentialRefs())[0]).toMatchObject({ source: 'vault', status: 'set' }); + // A stored row is read before the vault. + await engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE, context: ADMIN }); + expect(provider.tokenOf(provider.state.materialized[provider.state.materialized.length - 1])).toBe(DUMMY_VALUE); + expect((await engine.listConnectorCredentialRefs())[0]).toMatchObject({ source: 'object', status: 'set' }); + }); +}); + +describe('[#27237] a wrong value degrades the instance and marks the row rejected', () => { + it('refused at materialization (the mcp shape): degraded, `rejected`, the reason scrubbed, never retried', async () => { + const out = captureOutput(); + const { engine, provider, reload } = await boot([billing()]); + provider.state.rejectAtMaterialization = true; + await engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE, context: ADMIN }); + + const reason = engine.getConnectorDegradedReason('billing'); + expect(reason).toContain('(HTTP 401)'); + expect(reason).toContain('[credential withheld]'); + const [entry] = await engine.listConnectorCredentialRefs(); + expect(entry).toMatchObject({ ref: REF, status: 'rejected', source: 'object' }); + expect(entry.statusReason).toContain('(HTTP 401)'); + + // Held: a reconcile with nothing changed does not ask the upstream again. + const attempts = provider.state.materialized.length; + await reload(); + expect(provider.state.materialized).toHaveLength(attempts); + provider.state.rejectAtMaterialization = false; + + // Setting it again is what re-attempts it. + await engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE_2, context: ADMIN }); + expect(engine.getConnectorDegradedReason('billing')).toBeUndefined(); + expect((await engine.listConnectorCredentialRefs())[0]).toMatchObject({ status: 'set' }); + + for (const text of [JSON.stringify(entry), reason ?? '', out.text()]) { + expect(text.includes(DUMMY_VALUE)).toBe(false); + } + }); + + it.each([401, 403])( + 'refused on the first call (the rest / openapi shape, HTTP %i): degraded, `rejected`, the call answered as it was', + async (status) => { + const out = captureOutput(); + const { engine, provider, call, reload } = await boot([billing()]); + await engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE, context: ADMIN }); + expect(await call()).toMatchObject({ status: 200 }); + expect(engine.getConnectorDegradedReason('billing')).toBeUndefined(); + + provider.state.callStatus = status; + const answered = await call(); + expect(answered).toMatchObject({ status, ok: false }); + + const reason = engine.getConnectorDegradedReason('billing'); + expect(reason).toContain(`(HTTP ${status}) on a call`); + expect(reason).toContain(`credential '${REF}'`); + expect(reason).toContain('[credential withheld]'); + expect(engine.resolveConnectorAction('billing', 'request')).toBeUndefined(); + const [entry] = await engine.listConnectorCredentialRefs(); + expect(entry).toMatchObject({ status: 'rejected', source: 'object' }); + expect(entry.statusReason).toBe(reason); + + // Held: a reconcile with nothing changed leaves it degraded. + const attempts = provider.state.materialized.length; + await reload(); + expect(provider.state.materialized).toHaveLength(attempts); + expect(engine.getConnectorDegradedReason('billing')).toBe(reason); + + // Setting it again re-materializes it. + provider.state.callStatus = 200; + await engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE_2, context: ADMIN }); + expect(await call()).toMatchObject({ status: 200 }); + expect((await engine.listConnectorCredentialRefs())[0]).toMatchObject({ status: 'set' }); + + for (const text of [JSON.stringify(entry), reason ?? '', out.text()]) { + expect(text.includes(DUMMY_VALUE)).toBe(false); + } + }, + ); +}); diff --git a/packages/services/service-automation/src/connector-fail-cause.test.ts b/packages/services/service-automation/src/connector-fail-cause.test.ts index 0af88f0b682..93573609a18 100644 --- a/packages/services/service-automation/src/connector-fail-cause.test.ts +++ b/packages/services/service-automation/src/connector-fail-cause.test.ts @@ -360,13 +360,18 @@ describe('#5575 — boot (fatal) still carries the cause text, not structured me ); }); - it('appends the auth failure, whose own text still names the credentialRef', async () => { + // [#27237] An unresolved reference degrades its instance now; what stays + // fatal at boot is the host's resolver THROWING — foreign code whose + // failure is not "the credential is not set". + it("appends the host resolver's failure", async () => { await expect( - bootFatal([providerConnector('billing', { auth: { type: 'bearer', credentialRef: 'MISSING' } })], { + bootFatal([providerConnector('billing', { auth: { type: 'bearer', credentialRef: 'VAULT_REF' } })], { factory: okFactory, - credentialResolver: () => undefined, + credentialResolver: () => { + throw new Error('vault session expired'); + }, }), - ).rejects.toThrow(/auth resolution failed \(ADR-0097 §3\)\. cause: .*credentialRef 'MISSING' did not resolve/s); + ).rejects.toThrow(/auth resolution failed \(ADR-0097 §3\)\. cause: vault session expired/); }); it('keeps a multi-line ZodError dump intact on the fatal path', async () => { diff --git a/packages/services/service-automation/src/connector-materialization.test.ts b/packages/services/service-automation/src/connector-materialization.test.ts index 6a8e9b83ad1..e6c5ad18136 100644 --- a/packages/services/service-automation/src/connector-materialization.test.ts +++ b/packages/services/service-automation/src/connector-materialization.test.ts @@ -328,14 +328,19 @@ describe('ADR-0097 — declarative connector materialization', () => { ).rejects.toThrow(/provider 'openapi'.*no provider factory/s); }); - it('fails boot loudly when credentialRef does not resolve', async () => { - const { factory } = makeFakeProvider(); - await expect( - boot([providerConnector('billing', { auth: { type: 'bearer', credentialRef: 'MISSING' } })], { - providerFactory: factory, - credentialResolver: () => undefined, - }), - ).rejects.toThrow(/credentialRef 'MISSING' did not resolve/); + // [#27237] Ruling A on #27230: an administrator installs the app first and + // fills the credential after, so a reference no arm of the chain answers + // for degrades its instance — named — instead of failing the boot. + it('degrades the instance, naming the reference, when credentialRef does not resolve', async () => { + const { factory, calls } = makeFakeProvider(); + const kernel = await boot([providerConnector('billing', { auth: { type: 'bearer', credentialRef: 'MISSING' } })], { + providerFactory: factory, + credentialResolver: () => undefined, + }); + const engine = automationOf(kernel); + expect(calls).toHaveLength(0); + expect(engine.getConnectorDegradedReason('billing')).toMatch(/credential 'MISSING'/); + await kernel.shutdown(); }); it('fails boot loudly when a provider factory throws (invalid providerConfig)', async () => { diff --git a/packages/services/service-automation/src/credential-resolver.ts b/packages/services/service-automation/src/credential-resolver.ts index b252e5eb073..c1e6a7480c4 100644 --- a/packages/services/service-automation/src/credential-resolver.ts +++ b/packages/services/service-automation/src/credential-resolver.ts @@ -3,7 +3,9 @@ /** * Resolves a credential REFERENCE to its plaintext secret, or `undefined`/empty * when unknown. Two references go through it: a declarative connector's - * `auth.credentialRef` (at boot — an unresolved one is a hard boot error), and + * `auth.credentialRef` (third in its chain, after the environment and the + * `sys_connector_credential` store — #27237; an unresolved one degrades the + * instance naming it), and * an `api` flow's start-node `config.secretRef` (at every inbound post — an * unresolved one refuses that post as unavailable). May be async so an * enterprise resolver can hit a vault. The open-tier default reads from From a57b6b46343e608f18358c1485e4a27f1971b4e1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 13:50:03 +0000 Subject: [PATCH 04/21] wip(automation): docs, ADR-0097 amendment notes, changesets, ledger rows, rest rejection pin Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- .../27237-connector-credential-refs-routes.md | 12 ++ ...37-connector-credential-rejected-marker.md | 15 ++ .../27237-connector-credential-store.md | 19 +++ .../27237-install-local-credential-refs.md | 10 ++ .changeset/27237-mcp-credential-rejected.md | 10 ++ content/docs/automation/connectors.mdx | 87 +++++++++-- .../0097-declarative-connector-instances.md | 20 ++- ...lace-install-local-credential-refs.test.ts | 137 ++++++++++++++++++ .../src/connector-rest-plugin.test.ts | 59 ++++++++ .../services/service-automation/README.md | 2 +- .../activation-ledger-registration.test.ts | 11 +- .../src/connector-credential-door.test.ts | 4 +- .../src/plugin-suspended-run-wiring.test.ts | 5 +- .../system/constants/platform-object-names.ts | 4 +- scripts/platform-object-tenancy-census.json | 15 +- 15 files changed, 384 insertions(+), 26 deletions(-) create mode 100644 .changeset/27237-connector-credential-refs-routes.md create mode 100644 .changeset/27237-connector-credential-rejected-marker.md create mode 100644 .changeset/27237-connector-credential-store.md create mode 100644 .changeset/27237-install-local-credential-refs.md create mode 100644 .changeset/27237-mcp-credential-rejected.md create mode 100644 packages/cloud-connection/src/marketplace-install-local-credential-refs.test.ts diff --git a/.changeset/27237-connector-credential-refs-routes.md b/.changeset/27237-connector-credential-refs-routes.md new file mode 100644 index 00000000000..1b6f2907d7d --- /dev/null +++ b/.changeset/27237-connector-credential-refs-routes.md @@ -0,0 +1,12 @@ +--- +'@objectstack/runtime': patch +--- + +feat(runtime): `GET`/`PUT`/`DELETE /api/v1/automation/connectors/credential-refs` — list the connector credentials an environment's connectors name, and set or clear one + +Clause-②: yes (widening) +Metadata-protocol: none + +Three new routes serve the automation service's connector credential store (#27237). `GET …/credential-refs` lists every `credentialRef` an installed connector names, plus stored references nothing names any more. Each entry has its `status` (`set` / `missing` / `rejected`), its `source` (`env` / `object` / `vault` / null) and the instances naming it, and never a value. `PUT …/credential-refs/:ref` with the closed body `{ "value": "…" }` stores a value, and the instances naming the reference re-materialize without a restart. `DELETE …/credential-refs/:ref` removes it. + +All three require the `manage_metadata` capability. Under the `group` and `isolated` tenancy postures they refuse every caller with a 403 that names the posture, because connector credentials are environment-level. They take no query parameters. The service's refusals keep their standard codes: 400 `VALIDATION_ERROR`, 404 `RESOURCE_NOT_FOUND` for an undeclared reference, 409 `RESOURCE_CONFLICT` for one an environment variable supplies, and 503 `SERVICE_UNAVAILABLE` with no crypto provider. A mounted service without these optional members answers 501. diff --git a/.changeset/27237-connector-credential-rejected-marker.md b/.changeset/27237-connector-credential-rejected-marker.md new file mode 100644 index 00000000000..b35255e7212 --- /dev/null +++ b/.changeset/27237-connector-credential-rejected-marker.md @@ -0,0 +1,15 @@ +--- +'@objectstack/spec': patch +--- + +feat(spec): `ConnectorCredentialRejectedError` and the connector credential members of `IAutomationService` + +Clause-②: yes (widening) +Metadata-protocol: none + +Two additions for the connector credential store (#27237): + +- `@objectstack/spec/integration` exports `CONNECTOR_CREDENTIAL_REJECTED`, `ConnectorCredentialRejectedError` (with `status`: 401 or 403) and the structural check `isConnectorCredentialRejected`. They sit beside the upstream-unavailable marker. A connector provider factory throws it when the upstream refuses the credential it sent, and the materializer then degrades the instance instead of failing the boot. The code is registered in the error-code ledger under `@objectstack/spec`. +- `IAutomationService` gains three optional members, `listConnectorCredentialRefs`, `setConnectorCredential` and `clearConnectorCredential`, with their types `ConnectorCredentialRef`, `ConnectorCredentialSetRequest` and `ConnectorCredentialClearRequest`. A listing entry never carries a value. + +`PLATFORM_OBJECT_NAMES['service-automation']` lists the new `sys_connector_credential` object. diff --git a/.changeset/27237-connector-credential-store.md b/.changeset/27237-connector-credential-store.md new file mode 100644 index 00000000000..b24cd8dc431 --- /dev/null +++ b/.changeset/27237-connector-credential-store.md @@ -0,0 +1,19 @@ +--- +'@objectstack/service-automation': patch +--- + +feat(service-automation): an administrator sets a declarative connector's credential in the environment's credential store, and a missing one degrades its instance instead of failing the boot + +Clause-②: yes (widening) +Metadata-protocol: none + +A provider-bound `connectors:` entry names its secret with `auth.credentialRef`. Before, the open tier resolved that name from the server's environment variables only, and a name with no value failed the boot. On a hosted environment, where the administrator who installs an app has no server environment, a connector-bearing app could never be credentialed. + +The reference now resolves through three sources, in order: an environment variable of the same name (it always wins), then the new `sys_connector_credential` store, then the host's `credentialResolver` if one is injected. The store keeps one row per reference with its value in a `Field.secret` column: encrypted at rest by the host crypto provider and masked on every read. It is written through `setConnectorCredential` / `clearConnectorCredential` and listed by `listConnectorCredentialRefs` on the `automation` service, which the REST routes `GET`/`PUT`/`DELETE /api/v1/automation/connectors/credential-refs` serve. + +- A reference no source answers for degrades the instance naming it, with a reason that names the reference and how to set it. Before, it failed the boot. A `credentialResolver` that throws is still fatal at boot. +- Setting or clearing a credential reconciles at once: every instance naming the reference re-materializes, or is torn down and degraded, with no restart. +- A credential the upstream refuses (HTTP 401 / 403) degrades the instance and marks a stored credential `rejected` with the upstream's message, never the value. An `mcp` instance is refused when it connects; a `rest` or `openapi` instance on the first call that answers 401 / 403, which still returns its answer. A refused instance is not retried on a timer until its credential or declaration changes. +- The value never appears in a log record, the listing, `GET /connectors`, an error body or a read of the object. A provider message that echoes it carries `[credential withheld]` instead. + +A deployment that relied on the boot failing for an unset reference now boots, with that instance degraded and named on `GET /api/v1/automation/connectors`. diff --git a/.changeset/27237-install-local-credential-refs.md b/.changeset/27237-install-local-credential-refs.md new file mode 100644 index 00000000000..d7f24a02125 --- /dev/null +++ b/.changeset/27237-install-local-credential-refs.md @@ -0,0 +1,10 @@ +--- +'@objectstack/cloud-connection': patch +--- + +feat(cloud-connection): the install-local answer lists the connector credentials the installed package names, and which are not set yet + +Clause-②: yes (widening) +Metadata-protocol: none + +`POST /api/v1/marketplace/install-local` answers with `credentialRefs` when the package's connectors name any `auth.credentialRef`. Each entry has the reference, the instances naming it, its `status` from the automation service's credential listing (`set` / `missing` / `rejected`, or `unknown` when that listing cannot be read) and its `source`, never a value. When any of them is not set, a `credentialNote` names them and says how to set them. An unset reference never refuses the install: the instance naming it installs degraded and materializes once the credential is set (#27237). The answer's keys are unchanged for a package whose connectors name no credential. diff --git a/.changeset/27237-mcp-credential-rejected.md b/.changeset/27237-mcp-credential-rejected.md new file mode 100644 index 00000000000..6c435130e12 --- /dev/null +++ b/.changeset/27237-mcp-credential-rejected.md @@ -0,0 +1,10 @@ +--- +'@objectstack/connector-mcp': patch +--- + +fix(connector-mcp): a credential the MCP server rejects is thrown as `ConnectorCredentialRejectedError`, so the instance is degraded instead of failing the boot + +Clause-②: yes (widening) +Metadata-protocol: none + +An HTTP 401 / 403 on connect, handshake or `tools/list` used to throw a plain `Error`, which failed the boot. The provider now throws `ConnectorCredentialRejectedError` from `@objectstack/spec/integration`, with the same message, the status, and the SDK error as `cause`. The automation service degrades the instance naming the refusal and marks a stored credential `rejected`, and it does not retry until the credential changes (#27237). Every other connect or `tools/list` failure is unchanged. diff --git a/content/docs/automation/connectors.mdx b/content/docs/automation/connectors.mdx index d5e27d1bced..b835357f36b 100644 --- a/content/docs/automation/connectors.mdx +++ b/content/docs/automation/connectors.mdx @@ -76,7 +76,10 @@ At boot, the automation service resolves each provider-bound entry: three in `plugins:` by default, paired with the `automation` capability that performs the materialization. 2. **The factory validates `providerConfig`** and the automation service - resolves `auth.credentialRef` to its secret. + resolves `auth.credentialRef` to its secret (see + [What `credentialRef` resolves through](#what-credentialref-resolves-through)). + A reference that is not set yet does not stop the boot: that one instance + registers degraded until an administrator sets it. 3. **The result is registered** on the connector registry — indistinguishable from a hand-written connector. Its **actions are derived from the upstream** (the OpenAPI document's operations, the MCP server's @@ -238,18 +241,77 @@ Variant notes: - **`basic`** — the `username` stays in metadata (it is not a secret); only the password resolves through `credentialRef`. -**What `credentialRef` resolves through.** In the open tier the ref is the -name of an **environment variable** (`BILLING_API_TOKEN` above means -`process.env.BILLING_API_TOKEN` on the server). A ref that resolves to nothing -is a **hard boot error** — an app must not silently run with a dead connector. -The enterprise tier swaps in a vault/KMS-backed resolver -(`AutomationServicePluginOptions.credentialResolver`) without changing what -you author: the entry still just names a ref. +### What `credentialRef` resolves through + +The metadata author names a credential; the administrator who installs the app +holds it. The automation service asks three sources, in order, and the first +that answers wins: + +1. **An environment variable of the same name** on the server + (`BILLING_API_TOKEN` above means `process.env.BILLING_API_TOKEN`). A + self-hosted operator keeps this path, and it always wins. +2. **The connector credential store**, `sys_connector_credential`: one row per + reference, its value encrypted at rest by the host's crypto provider and + masked on every read. An administrator fills it through the API below, with + no access to the server environment, which is the hosted case. +3. **The host's resolver**, when it injects one + (`AutomationServicePluginOptions.credentialResolver`, such as a vault or KMS + client). You author the same entry either way: it still just names a ref. + +A reference no source answers for **does not fail the boot**. An app installs +first and gets its credentials after, so that one instance registers +**degraded**: it is listed on `GET /api/v1/automation/connectors` with a reason +that names the reference, and a dispatch to it fails with the same reason. + +#### Setting a credential + +Three routes, each requiring the `manage_metadata` capability: + +```http +GET /api/v1/automation/connectors/credential-refs +PUT /api/v1/automation/connectors/credential-refs/BILLING_API_TOKEN { "value": "…" } +DELETE /api/v1/automation/connectors/credential-refs/BILLING_API_TOKEN +``` + +- **`GET`** lists every reference an installed connector names, plus every + stored one that nothing names any more. Each entry has a `status` (`set`, + `missing` or `rejected`), a `source` (`env`, `object`, `vault` or `null`), + whether this door can write it, and the instances naming it with their state. + It never includes a value. +- **`PUT`** stores the value, then reconciles. Every instance naming the + reference re-materializes with it **without a restart**, and the answer is + the reference's entry after that. The route refuses, before writing anything: + a reference no installed connector declares (`404`), one an environment + variable already supplies (`409`, because the variable wins and a stored value + would never be read), and an empty value (`400`). With no crypto provider it + answers `503`, since a credential is only ever stored encrypted. +- **`DELETE`** removes the stored value. The instances naming the reference + are torn down and registered degraded again. + +Installing a package reports the same facts for it: the install-local answer +(`POST /api/v1/marketplace/install-local`, which `os package install` calls) +lists the `credentialRefs` the package's connectors name, each with its status, +plus a `credentialNote` naming the ones still to set. + +**A credential the upstream refuses** (HTTP `401` / `403`) degrades the +instance, and a stored credential is marked `rejected` with the upstream's +message, never the value. An `mcp` instance finds this out when it connects. A +`rest` or `openapi` instance finds out on the first call answering `401` / +`403`; that call returns its answer as usual, and the instance is degraded +after it. A refused instance is not retried on a timer. Setting the credential +again is what re-attempts it. + +Credentials are **environment-level**: one value per reference for the whole +environment, read by whichever instance names it. On a multi-organization +environment (the `group` or `isolated` tenancy posture) the three routes refuse +with a message naming the posture. There, a reference resolves from the +environment variable or the host's resolver only. The same resolver serves an `api` flow's start-node `config.secretRef`, the -name of its inbound hook secret. That one is read at every post rather than at -boot, so a name with no value refuses each post with `503` instead of failing -the boot ([Flows](/docs/automation/flows)). +name of its inbound hook secret. That one goes through the injected resolver +alone (environment variables by default) and is read at every post rather than +at boot, so a name with no value refuses each post with `503` +([Flows](/docs/automation/flows)). Inlining a secret is rejected when the stack is validated. Writing the runtime shape on a provider-bound instance — @@ -322,7 +384,8 @@ retry): |:---|:---| | `provider` names no installed factory | **Hard boot error** naming the entry, the provider, and the plugin that supplies it — add the executor to `plugins:` | | Invalid `providerConfig` (missing `baseUrl` / `spec` / `transport`, wrong shape) | **Hard boot error** from the provider factory | -| `credentialRef` resolves to nothing | **Hard boot error** — set the env var (open tier) or fix the vault ref | +| `credentialRef` resolves to nothing | **Not fatal**: the instance registers **degraded**, naming the reference — set it (`PUT …/credential-refs/:ref`), and it materializes without a restart | +| The upstream refuses the credential (`401` / `403`) | **Not fatal**: the instance registers **degraded** and a stored credential is marked `rejected` — not retried until the credential changes | | Instance `name` collides with a plugin-registered connector | **Hard boot error** — there is no silent precedence between the two worlds | | stdio transport without host opt-in | **Hard boot error** with the `declarativeStdio` fix inline — a security rejection is never retried | | MCP server unreachable / `tools/list` fails; remote spec URL unreachable or transiently failing (408/429/5xx) | **Not fatal**: the instance registers **degraded** (visible on `GET /api/v1/automation/connectors` with the reason), dispatches fail with that reason, and the platform retries on a 5s→5min backoff — recovery is automatic | diff --git a/docs/adr/0097-declarative-connector-instances.md b/docs/adr/0097-declarative-connector-instances.md index d5daf48f4c1..6203bafb0e0 100644 --- a/docs/adr/0097-declarative-connector-instances.md +++ b/docs/adr/0097-declarative-connector-instances.md @@ -5,7 +5,7 @@ **Builds on**: [ADR-0015](./0015-external-datasource-federation.md) (open mechanism / enterprise lifecycle split), [ADR-0018](./0018-unified-node-action-registry.md) (`connector_action` baseline dispatch + `engine.registerConnector()`), [ADR-0022](./0022-connectors-vs-messaging-channels.md) (Connector = transport/integration mechanism), [ADR-0023](./0023-openapi-to-connector-generator.md) (OpenAPI → Connector generator), [ADR-0024](./0024-mcp-connectors.md) (MCP servers as connectors) **Tracking**: framework#2977 (supersedes the interim descriptor-only contract from framework#2612) **Consumers**: `@objectstack/spec` (`stack.zod.ts`, `integration/connector.zod.ts`), `@objectstack/service-automation` (boot audit → provider binding), `@objectstack/connector-openapi`, `@objectstack/connector-mcp`, `@objectstack/connector-rest`, the Studio flow palette, AI authoring (stack generation) -**Amended by**: [ADR-0143](./0143-outbound-connector-auth-is-open-tier.md) (2026-10-27) — OAuth2, the vendor app-credential exchange and request signing join `ConnectorInstanceAuthSchema` in the open tier, and those kinds resolve per dispatch; see the notes under §3 and under "Open / enterprise line". PR #26798 (2026-10-27, #26709 item 3(b)) — a credential the MCP server rejects (HTTP `401` / `403`) is a fatal configuration fault, not upstream-unavailable; see the note under "Upstream availability". +**Amended by**: [ADR-0143](./0143-outbound-connector-auth-is-open-tier.md) (2026-10-27) — OAuth2, the vendor app-credential exchange and request signing join `ConnectorInstanceAuthSchema` in the open tier, and those kinds resolve per dispatch; see the notes under §3 and under "Open / enterprise line". PR #26798 (2026-10-27, #26709 item 3(b)) — a credential the MCP server rejects (HTTP `401` / `403`) is a fatal configuration fault, not upstream-unavailable; see the note under "Upstream availability". #27237 (2026-10-28, ruling A on #27230 and the ruling on #27237) — an unresolved `credentialRef` and a rejected credential degrade the instance instead of failing the boot, and the open tier reads a `sys_connector_credential` store between the environment and the host's resolver; see the notes under §6 and under "Upstream availability". --- @@ -95,6 +95,16 @@ If a provider-bound instance and a plugin-registered connector share a `name`, b - The showcase demonstrates the declarative path live (upgrading the catalog-descriptor demo shipped with #2612) and `GET /connectors` lists the materialized instance. - Boot fails loudly for: unknown provider, invalid providerConfig, unresolvable credentialRef, name conflict with a plugin-registered connector. +> **Amendment note (2026-10-28, #27237 — ruling A on #27230, comment 6468327468).** An +> unresolvable `credentialRef` leaves this list. An administrator installs an app first and fills +> its credentials after, so a reference no source answers for degrades the instance naming it (a +> husk on `GET /connectors` whose reason names the reference), and the instance materializes, +> without a restart, once the credential is set. The open-tier sources are, in order: an +> environment variable of the same name, the `sys_connector_credential` store (a `Field.secret` +> row written through `PUT /api/v1/automation/connectors/credential-refs/:ref`), then the host's +> injected resolver. Environment-level only (S1 on #27230). A host resolver that throws stays +> fatal at boot. + --- ## Consequences @@ -209,6 +219,14 @@ exactly the plausible-but-dead shape this ADR exists to kill. > Streamable-HTTP server in > `packages/connectors/connector-mcp/src/mcp-http-discovery-faults.test.ts`. +> **Amendment note (2026-10-28, #27237 — ruling on #27237, comment 6470087293).** The note above +> changes on one point: a rejected credential no longer fails the boot. The `mcp` provider throws +> `ConnectorCredentialRejectedError` (`CONNECTOR_CREDENTIAL_REJECTED`) at materialization; a +> `rest` or `openapi` instance reports it on the first call answering `401` / `403`. Either way +> the instance is degraded with the upstream's message, never the value, and a stored credential +> is marked `rejected`. It is still never retried on a timer: the next attempt is the next change +> of the credential or of the declaration. + The `openapi` provider applies the same split to its **remote-URL spec fetch**: an unreachable endpoint (network error) or a transient HTTP status (`408` / `429` / `5xx`, mirroring the `retryableStatusCodes` convention) is diff --git a/packages/cloud-connection/src/marketplace-install-local-credential-refs.test.ts b/packages/cloud-connection/src/marketplace-install-local-credential-refs.test.ts new file mode 100644 index 00000000000..5b2513d9126 --- /dev/null +++ b/packages/cloud-connection/src/marketplace-install-local-credential-refs.test.ts @@ -0,0 +1,137 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#27237] The install answer says which connector credentials the package + * names and which are not set (ruling A on #27230) — and a reference that is + * not set never refuses the install: the instance naming it installs degraded, + * and the answer says how to set it. + * + * Statuses come from the automation service's credential listing; the answer + * carries a reference, its status and source and the instances naming it, + * never a value. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +// The first load of the runtime's dist paid at module top, never inside a +// clocked `it` (`scripts/check-test-source-alias.mjs`). +import '@objectstack/runtime'; +import { MarketplaceInstallLocalPlugin, installedCredentialNote } from './marketplace-install-local-plugin.js'; +import { installerAuthService, withInstallerGrants } from './install-local-principal.fixtures.js'; + +const APP_ID = 'com.example.billingapp'; + +function artifact(connectors: unknown[]) { + return { + manifest: { id: APP_ID, namespace: 'billing_app', version: '0.1.0', type: 'app', name: 'Billing App' }, + objects: [{ name: 'billing_app_invoice', label: 'Invoice', fields: { name: { type: 'text', label: 'Name' } } }], + connectors, + }; +} + +const bearer = (name: string, credentialRef: string) => ({ + name, + label: name, + type: 'api', + provider: 'rest', + providerConfig: { baseUrl: 'https://billing.example.test' }, + auth: { type: 'bearer', credentialRef }, +}); + +type Handler = (c: any) => Promise; + +function makeC(body: any) { + const json = vi.fn((payload: any, status?: number) => ({ payload, status: status ?? 200 })); + return { + req: { + url: 'http://localhost:3000/api/v1/marketplace/install-local', + raw: new Request('http://localhost:3000/x'), + json: async () => body, + param: () => undefined, + }, + json, + }; +} + +let dir: string; +beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'mil-credrefs-')); }); +afterEach(() => { rmSync(dir, { recursive: true, force: true }); vi.restoreAllMocks(); }); + +async function bootPlugin(automation?: Record) { + const routes = new Map(); + const rawApp = { + get: (p: string, h: Handler) => routes.set(`GET ${p}`, h), + post: (p: string, h: Handler) => routes.set(`POST ${p}`, h), + delete: (p: string, h: Handler) => routes.set(`DELETE ${p}`, h), + }; + const hooks = new Map(); + const services: Record = { + manifest: { register: vi.fn() }, + auth: installerAuthService(), + objectql: withInstallerGrants({ syncSchemas: async () => undefined }), + ...(automation ? { automation } : {}), + }; + const ctx: any = { + hook: (e: string, h: any) => hooks.set(e, h), + getService: (name: string) => { + if (name === 'http-server') return { getRawApp: () => rawApp }; + const svc = services[name]; + if (svc === undefined) throw new Error(`no ${name}`); + return svc; + }, + trigger: async () => undefined, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, + }; + const plugin = new MarketplaceInstallLocalPlugin({ controlPlaneUrl: 'off', storageDir: dir }); + await plugin.start(ctx); + await hooks.get('kernel:ready')?.(); + return async (connectors: unknown[]) => + routes.get('POST /api/v1/marketplace/install-local')!(makeC({ manifest: artifact(connectors) })); +} + +describe('[#27237] the install answer names the connector credentials the package declares', () => { + it('reports each reference with its status, and a missing one installs — degraded — with the note saying how to set it', async () => { + const listConnectorCredentialRefs = vi.fn(async () => [ + { ref: 'BILLING_SET_REF', status: 'set', source: 'object', writable: true, declared: true, connectors: [] }, + { ref: 'BILLING_MISSING_REF', status: 'missing', source: null, writable: true, declared: true, connectors: [] }, + ]); + const install = await bootPlugin({ listConnectorCredentialRefs }); + + const res = await install([ + bearer('billing', 'BILLING_SET_REF'), + bearer('billing_eu', 'BILLING_MISSING_REF'), + bearer('billing_us', 'BILLING_MISSING_REF'), + { name: 'catalog_only', label: 'Catalog', type: 'api' }, + ]); + + expect(res.status).toBe(200); + expect(res.payload?.success, JSON.stringify(res.payload)).toBe(true); + expect(res.payload.data.credentialRefs).toEqual([ + { ref: 'BILLING_MISSING_REF', connectors: ['billing_eu', 'billing_us'], status: 'missing', source: null }, + { ref: 'BILLING_SET_REF', connectors: ['billing'], status: 'set', source: 'object' }, + ]); + expect(res.payload.data.credentialNote).toBe(installedCredentialNote(['BILLING_MISSING_REF'])); + expect(res.payload.data.credentialNote).toContain('/api/v1/automation/connectors/credential-refs/:ref'); + expect(res.payload.data.credentialNote).not.toContain('BILLING_SET_REF'); + }); + + it('with no automation service to ask, each reference is `unknown`, never assumed set', async () => { + const install = await bootPlugin(); + const res = await install([bearer('billing', 'BILLING_SET_REF')]); + expect(res.payload?.success).toBe(true); + expect(res.payload.data.credentialRefs).toEqual([ + { ref: 'BILLING_SET_REF', connectors: ['billing'], status: 'unknown', source: null }, + ]); + expect(res.payload.data.credentialNote).toBe(installedCredentialNote(['BILLING_SET_REF'])); + }); + + it('a package whose connectors name no credential keeps the answer key set unchanged', async () => { + const install = await bootPlugin({ listConnectorCredentialRefs: async () => [] }); + const res = await install([{ name: 'catalog_only', label: 'Catalog', type: 'api' }]); + expect(res.payload?.success).toBe(true); + expect('credentialRefs' in res.payload.data).toBe(false); + expect('credentialNote' in res.payload.data).toBe(false); + }); +}); diff --git a/packages/connectors/connector-rest/src/connector-rest-plugin.test.ts b/packages/connectors/connector-rest/src/connector-rest-plugin.test.ts index 284876511f1..7d6fb4d212f 100644 --- a/packages/connectors/connector-rest/src/connector-rest-plugin.test.ts +++ b/packages/connectors/connector-rest/src/connector-rest-plugin.test.ts @@ -152,3 +152,62 @@ describe('ConnectorRestPlugin — end to end with the automation engine', () => await kernel.shutdown(); }); }); + +// [#27237] The ruling on #27237 (comment 6470087293), on the REAL rest +// provider: its handler answers an upstream 401 as `{ status, ok, body }`, so +// the automation service reads the refusal off the first credentialed call — +// the call still answers as it was, and the instance is degraded after it, +// naming the reference, with the echoed credential withheld. +describe('ConnectorRestPlugin — a credential the upstream refuses (#27237)', () => { + it('degrades the declarative instance on the first call answering 401', async () => { + const DUMMY = 'dummy-vault-token-0005'; + const calls: string[] = []; + const impl = (async (url: string) => { + calls.push(url); + return { + status: 401, + ok: false, + headers: { get: (h: string) => (h.toLowerCase() === 'content-type' ? 'application/json' : null) }, + json: async () => ({ message: `bad credentials: ${DUMMY}` }), + text: async () => JSON.stringify({ message: `bad credentials: ${DUMMY}` }), + }; + }) as unknown as typeof fetch; + const declared = [{ + name: 'billing', + label: 'Billing API', + type: 'api', + provider: 'rest', + providerConfig: { baseUrl: 'https://billing.example.com' }, + auth: { type: 'bearer', credentialRef: 'DUMMY_BILLING_TOKEN_27237' }, + }]; + const kernel = new LiteKernel(); + kernel.use(new AutomationServicePlugin({ credentialResolver: (r) => (r === 'DUMMY_BILLING_TOKEN_27237' ? DUMMY : undefined) })); + kernel.use(new ConnectorRestPlugin({ fetchImpl: impl })); + kernel.use({ + name: 'test.metadata', + type: 'standard', + version: '1.0.0', + dependencies: [], + async init(ctx: any) { + ctx.registerService('objectql', { registry: { listItems: (t: string) => (t === 'connector' ? declared : []) } }); + }, + async start() {}, + } as never); + await kernel.bootstrap(); + const engine = kernel.getService('automation'); + expect(engine.getConnectorDegradedReason('billing')).toBeUndefined(); + + const answered = await engine.resolveConnectorAction('billing', 'request')!({ path: '/invoices' }, {} as never); + expect(answered).toMatchObject({ status: 401, ok: false }); + expect(calls).toEqual(['https://billing.example.com/invoices']); + + const reason = engine.getConnectorDegradedReason('billing') ?? ''; + expect(reason).toContain('(HTTP 401) on a call'); + expect(reason).toContain("credential 'DUMMY_BILLING_TOKEN_27237'"); + expect(reason).toContain('[credential withheld]'); + expect(reason.includes(DUMMY)).toBe(false); + expect(engine.resolveConnectorAction('billing', 'request')).toBeUndefined(); + + await kernel.shutdown(); + }); +}); diff --git a/packages/services/service-automation/README.md b/packages/services/service-automation/README.md index 1a1cbd68396..bbc16472be1 100644 --- a/packages/services/service-automation/README.md +++ b/packages/services/service-automation/README.md @@ -46,7 +46,7 @@ Every field of `AutomationServicePluginOptions` is optional. | `maxLogSize` | `number` | `DEFAULT_MAX_EXECUTION_LOG_SIZE` (1000) | In-memory execution-log ring buffer size. | | `runSummaryLog` | `RunSummaryLogLevel` | `'info'` | Level for the one-line-per-terminal-run summary. Turning it down changes narration only — the summary is still computed, returned and persisted. | | `runHistoryMaxPerFlow` | `number` | `DEFAULT_MAX_TERMINAL_RUNS_PER_FLOW` (100) | Per-flow cap on terminal run-history rows; `0` disables the cap. | -| `credentialResolver` | `CredentialResolver` | env-var resolver | Resolves a declarative connector's `auth.credentialRef` at boot. | +| `credentialResolver` | `CredentialResolver` | none for connectors; env-var resolver for `secretRef` | The host's resolver (a vault): third in a declarative connector's `auth.credentialRef` chain, after an environment variable of the same name and the `sys_connector_credential` store (`GET`/`PUT`/`DELETE /api/v1/automation/connectors/credential-refs`). A reference none of them answers degrades its instance; a resolver that throws is fatal at boot. An `api` flow's `secretRef` resolves through this resolver alone. | | `packageRoot` | `string` | `process.cwd()` | Root that relative file refs in connector entries resolve against; reads are confined to it. | The AGE half of run retention is declarative, not an option here: `sys_automation_run` diff --git a/packages/services/service-automation/src/activation-ledger-registration.test.ts b/packages/services/service-automation/src/activation-ledger-registration.test.ts index 445b08f1e4c..cc6404ef075 100644 --- a/packages/services/service-automation/src/activation-ledger-registration.test.ts +++ b/packages/services/service-automation/src/activation-ledger-registration.test.ts @@ -60,13 +60,16 @@ async function registeredObjectNames(): Promise { } describe('#12359 — the automation service is NOT the activation ledger\'s registrant', () => { - it('registers its own three objects and nothing else', async () => { + it('registers its own four objects and nothing else', async () => { // Equality, not `not.toContain`: an assertion that only names the - // absent object would stay green while this manifest grew a FOURTH + // absent object would stay green while this manifest grew a FIFTH // registration nobody reviewed, which is the same class of drift. // [#20790] The third is reviewed: `sys_flow_credential`, the write-only - // flow credential channel this service owns. - expect(await registeredObjectNames()).toEqual(['sys_automation_run', 'sys_flow_dispatch', 'sys_flow_credential']); + // flow credential channel this service owns. [#27237] So is the fourth: + // `sys_connector_credential`, the connector credential store. + expect(await registeredObjectNames()).toEqual([ + 'sys_automation_run', 'sys_flow_dispatch', 'sys_flow_credential', 'sys_connector_credential', + ]); }); it('does not name sys_metadata_activation in any manifest it registers', async () => { diff --git a/packages/services/service-automation/src/connector-credential-door.test.ts b/packages/services/service-automation/src/connector-credential-door.test.ts index 3b3cbed90b0..2bf19abaec7 100644 --- a/packages/services/service-automation/src/connector-credential-door.test.ts +++ b/packages/services/service-automation/src/connector-credential-door.test.ts @@ -107,7 +107,9 @@ function fakeProvider() { name: ctx.name, label: ctx.label, type: 'api', - authentication: { type: 'none' }, + // As the rest provider does: the runtime def carries the + // resolved auth, token included — what a descriptor must not leak. + authentication: ctx.auth ?? { type: 'none' }, actions: [{ key: 'request', label: 'Request' }], } as unknown as Connector, handlers: { diff --git a/packages/services/service-automation/src/plugin-suspended-run-wiring.test.ts b/packages/services/service-automation/src/plugin-suspended-run-wiring.test.ts index 11d67d80df1..bb44adb5a7a 100644 --- a/packages/services/service-automation/src/plugin-suspended-run-wiring.test.ts +++ b/packages/services/service-automation/src/plugin-suspended-run-wiring.test.ts @@ -255,8 +255,9 @@ describe('durable suspended-run wiring (#4420)', () => { // An explicitly ephemeral engine is a legitimate mode, not a // degradation — it must not register the run objects nor complain. // [#20790] The one object it does register is the write-only flow - // credential channel's, which every composition needs. - expect(h.registeredObjects()).toEqual(['sys_flow_credential']); + // credential channel's, which every composition needs — [#27237] with + // the connector credential store beside it, for the same reason. + expect(h.registeredObjects()).toEqual(['sys_flow_credential', 'sys_connector_credential']); expect((h.engine as any).store).toBeUndefined(); expect(h.errors()).toBe(''); }); diff --git a/packages/spec/src/system/constants/platform-object-names.ts b/packages/spec/src/system/constants/platform-object-names.ts index eb1ab60ebf5..0fa4a9c7323 100644 --- a/packages/spec/src/system/constants/platform-object-names.ts +++ b/packages/spec/src/system/constants/platform-object-names.ts @@ -121,8 +121,8 @@ export const PLATFORM_OBJECTS_BY_PACKAGE: Readonly Date: Wed, 28 Oct 2026 13:53:58 +0000 Subject: [PATCH 05/21] chore(spec): regenerate api-surface, export-origins and reference docs for the connector credential exports Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- content/docs/references/api/automation-api.mdx | 3 ++- content/docs/references/api/contract.mdx | 3 ++- content/docs/references/api/error-code-ledger.mdx | 1 + packages/spec/api-surface/contracts.json | 3 +++ packages/spec/api-surface/integration.json | 3 +++ packages/spec/export-origins/contracts.json | 3 +++ packages/spec/export-origins/integration.json | 3 +++ 7 files changed, 17 insertions(+), 2 deletions(-) diff --git a/content/docs/references/api/automation-api.mdx b/content/docs/references/api/automation-api.mdx index 46ead571b7f..7b0038272ef 100644 --- a/content/docs/references/api/automation-api.mdx +++ b/content/docs/references/api/automation-api.mdx @@ -347,7 +347,7 @@ const result = AutomationApiErrorCode.parse(data); | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | -| **code** | `Enum<'VALIDATION_ERROR' \| 'INVALID_FIELD' \| 'MISSING_REQUIRED_FIELD' \| 'INVALID_FORMAT' \| 'VALUE_TOO_LONG' \| 'VALUE_TOO_SHORT' \| 'VALUE_OUT_OF_RANGE' \| … +347 more>` | ✅ | The inner refusal's code in its WIRE spelling - the ADR-0112 member a record save of the same write answers in `error.code` (e.g. `UNIQUE_VIOLATION`); the closed set `ApiErrorSchema.code` parses against | +| **code** | `Enum<'VALIDATION_ERROR' \| 'INVALID_FIELD' \| 'MISSING_REQUIRED_FIELD' \| 'INVALID_FORMAT' \| 'VALUE_TOO_LONG' \| 'VALUE_TOO_SHORT' \| 'VALUE_OUT_OF_RANGE' \| … +348 more>` | ✅ | The inner refusal's code in its WIRE spelling - the ADR-0112 member a record save of the same write answers in `error.code` (e.g. `UNIQUE_VIOLATION`); the closed set `ApiErrorSchema.code` parses against | | **message** | `string` | ✅ | The inner refusal's own diagnostic text, verbatim, without the node and operation prefixes the run's failure text adds | ### Allowed Values: `FlowInnerRefusal.code` @@ -446,6 +446,7 @@ const result = AutomationApiErrorCode.parse(data); * `CONCURRENT_UPDATE` * `CONFLICT` * `CONFLICTING_MAPPING` +* `CONNECTOR_CREDENTIAL_REJECTED` * `CONNECTOR_UPSTREAM_UNAVAILABLE` * `CREATE_FAILED` * `CUBE_NOT_FOUND` diff --git a/content/docs/references/api/contract.mdx b/content/docs/references/api/contract.mdx index a8b16daab90..e7ded07f5c2 100644 --- a/content/docs/references/api/contract.mdx +++ b/content/docs/references/api/contract.mdx @@ -28,7 +28,7 @@ const result = ApiErrorSchema.parse(data); | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | -| **code** | `Enum<'VALIDATION_ERROR' \| 'INVALID_FIELD' \| 'MISSING_REQUIRED_FIELD' \| 'INVALID_FORMAT' \| 'VALUE_TOO_LONG' \| 'VALUE_TOO_SHORT' \| 'VALUE_OUT_OF_RANGE' \| … +347 more>` | ✅ | Error code (e.g. VALIDATION_ERROR; StandardErrorCode ∪ the ledger the serving side registers — ERROR_CODE_LEDGER for framework packages) | +| **code** | `Enum<'VALIDATION_ERROR' \| 'INVALID_FIELD' \| 'MISSING_REQUIRED_FIELD' \| 'INVALID_FORMAT' \| 'VALUE_TOO_LONG' \| 'VALUE_TOO_SHORT' \| 'VALUE_OUT_OF_RANGE' \| … +348 more>` | ✅ | Error code (e.g. VALIDATION_ERROR; StandardErrorCode ∪ the ledger the serving side registers — ERROR_CODE_LEDGER for framework packages) | | **declaredCode** | `string` | optional | The producer-declared code, verbatim, when it is not a member of the closed `code` vocabulary — the open, author-authored channel (app-specific spellings; ADR-0112) | | **message** | `string` | ✅ | Readable error message | | **userMessage** | `string` | optional | Producer-marked user-facing refusal text, verbatim. Present exactly when the producer opted in at throw time; consumers render it to end users and keep their generic substitution for anything unmarked. Status-agnostic; never replaces `message`. | @@ -134,6 +134,7 @@ const result = ApiErrorSchema.parse(data); * `CONCURRENT_UPDATE` * `CONFLICT` * `CONFLICTING_MAPPING` +* `CONNECTOR_CREDENTIAL_REJECTED` * `CONNECTOR_UPSTREAM_UNAVAILABLE` * `CREATE_FAILED` * `CUBE_NOT_FOUND` diff --git a/content/docs/references/api/error-code-ledger.mdx b/content/docs/references/api/error-code-ledger.mdx index 45028637f64..62c5e18f373 100644 --- a/content/docs/references/api/error-code-ledger.mdx +++ b/content/docs/references/api/error-code-ledger.mdx @@ -301,6 +301,7 @@ const result = ErrorCode.parse(data); * `CONCURRENT_UPDATE` * `CONFLICT` * `CONFLICTING_MAPPING` +* `CONNECTOR_CREDENTIAL_REJECTED` * `CONNECTOR_UPSTREAM_UNAVAILABLE` * `CREATE_FAILED` * `CUBE_NOT_FOUND` diff --git a/packages/spec/api-surface/contracts.json b/packages/spec/api-surface/contracts.json index 8f1328d110e..a96a7ac4549 100644 --- a/packages/spec/api-surface/contracts.json +++ b/packages/spec/api-surface/contracts.json @@ -64,6 +64,9 @@ "CheckNamespaceInput (interface)", "CheckNamespaceResult (interface)", "ClusterCallContext (interface)", + "ConnectorCredentialClearRequest (interface)", + "ConnectorCredentialRef (interface)", + "ConnectorCredentialSetRequest (interface)", "ConnectorSourcePullRequest (interface)", "ConnectorSourcePullResult (interface)", "ConnectorSourcePullSummary (interface)", diff --git a/packages/spec/api-surface/integration.json b/packages/spec/api-surface/integration.json index 72c2a5a112b..d85aba30d29 100644 --- a/packages/spec/api-surface/integration.json +++ b/packages/spec/api-surface/integration.json @@ -2,6 +2,7 @@ "description": "Every exported `name (kind)` of one published entry point of @objectstack/spec — the breadth half of the ADR-0059 backward-compatibility gate. Sharded by entry point (#5837) so two PRs touching different entry points never share a file. Reads the BUILT dist/*.d.ts: regenerate with `pnpm --filter @objectstack/spec gen:api-surface` after a real build.", "entry": "./integration", "exports": [ + "CONNECTOR_CREDENTIAL_REJECTED (const)", "CONNECTOR_UPSTREAM_UNAVAILABLE (const)", "Connector (type)", "ConnectorAction (type)", @@ -9,6 +10,7 @@ "ConnectorActionEffect (type)", "ConnectorActionEffectSchema (const)", "ConnectorActionSchema (const)", + "ConnectorCredentialRejectedError (class)", "ConnectorDescriptor (interface)", "ConnectorFetchPolicy (interface)", "ConnectorInstanceAPIKeyAuth (type)", @@ -44,6 +46,7 @@ "RetryConfigSchema (const)", "connectorFetchOptions (function)", "defineConnector (function)", + "isConnectorCredentialRejected (function)", "isConnectorUpstreamUnavailable (function)" ] } diff --git a/packages/spec/export-origins/contracts.json b/packages/spec/export-origins/contracts.json index 0b87030569b..38981c5802f 100644 --- a/packages/spec/export-origins/contracts.json +++ b/packages/spec/export-origins/contracts.json @@ -64,6 +64,9 @@ "CheckNamespaceInput": "src/contracts/package-service.ts#CheckNamespaceInput (interface)", "CheckNamespaceResult": "src/contracts/package-service.ts#CheckNamespaceResult (interface)", "ClusterCallContext": "src/contracts/cluster-service.ts#ClusterCallContext (interface)", + "ConnectorCredentialClearRequest": "src/contracts/automation-service.ts#ConnectorCredentialClearRequest (interface)", + "ConnectorCredentialRef": "src/contracts/automation-service.ts#ConnectorCredentialRef (interface)", + "ConnectorCredentialSetRequest": "src/contracts/automation-service.ts#ConnectorCredentialSetRequest (interface)", "ConnectorSourcePullRequest": "src/contracts/automation-service.ts#ConnectorSourcePullRequest (interface)", "ConnectorSourcePullResult": "src/contracts/automation-service.ts#ConnectorSourcePullResult (interface)", "ConnectorSourcePullSummary": "src/contracts/automation-service.ts#ConnectorSourcePullSummary (interface)", diff --git a/packages/spec/export-origins/integration.json b/packages/spec/export-origins/integration.json index 9eceeaf1bdd..48948f79eee 100644 --- a/packages/spec/export-origins/integration.json +++ b/packages/spec/export-origins/integration.json @@ -2,6 +2,7 @@ "description": "Which SOURCE DECLARATION each name exported by one public entry point of @objectstack/spec resolves to, after its alias chain is unwound: `# ()`. Two exports share an origin string iff they are the same declaration — so equal origins across two entries are a harmless re-export, and different origins under one name are the #4411 dual-source trap. Generated from src/ (no build needed) and read by the export-surface pin tests, which compare against it instead of each building their own ts.createProgram — that was ~55s of compilation per CI lap and a non-deterministic timeout that ejected unrelated PRs from the merge queue (#4796). Sharded by entry point (#5837) so two retirement PRs never share a file. Carries NO line numbers: the pins asserted the line as `\\d+`, and recording it would rewrite this artifact on every edit that shifts a line in any .zod.ts. Regenerate with `pnpm --filter @objectstack/spec gen:export-origins` and read the diff.", "entry": "./integration", "exports": { + "CONNECTOR_CREDENTIAL_REJECTED": "src/integration/connector-provider-errors.ts#CONNECTOR_CREDENTIAL_REJECTED (const)", "CONNECTOR_UPSTREAM_UNAVAILABLE": "src/integration/connector-provider-errors.ts#CONNECTOR_UPSTREAM_UNAVAILABLE (const)", "Connector": "src/integration/connector.zod.ts#Connector (type)", "ConnectorAction": "src/integration/connector.zod.ts#ConnectorAction (type)", @@ -9,6 +10,7 @@ "ConnectorActionEffect": "src/integration/connector.zod.ts#ConnectorActionEffect (type)", "ConnectorActionEffectSchema": "src/integration/connector.zod.ts#ConnectorActionEffectSchema (const)", "ConnectorActionSchema": "src/integration/connector.zod.ts#ConnectorActionSchema (const)", + "ConnectorCredentialRejectedError": "src/integration/connector-provider-errors.ts#ConnectorCredentialRejectedError (class)", "ConnectorDescriptor": "src/integration/connector-descriptor.ts#ConnectorDescriptor (interface)", "ConnectorFetchPolicy": "src/integration/connector-fetch-policy.ts#ConnectorFetchPolicy (interface)", "ConnectorInstanceAPIKeyAuth": "src/shared/connector-auth.zod.ts#ConnectorInstanceAPIKeyAuth (type)", @@ -44,6 +46,7 @@ "RetryConfigSchema": "src/integration/connector.zod.ts#RetryConfigSchema (const)", "connectorFetchOptions": "src/integration/connector-fetch-policy.ts#connectorFetchOptions (function)", "defineConnector": "src/integration/connector.zod.ts#defineConnector (function)", + "isConnectorCredentialRejected": "src/integration/connector-provider-errors.ts#isConnectorCredentialRejected (function)", "isConnectorUpstreamUnavailable": "src/integration/connector-provider-errors.ts#isConnectorUpstreamUnavailable (function)" } } From bfda0a2f71187e05a54a13d8a51dbc46cc9395d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 14:17:54 +0000 Subject: [PATCH 06/21] =?UTF-8?q?WIP(#27237):=20gate=20fixes=20=E2=80=94?= =?UTF-8?q?=20keep=20the=20ADR-0143=20anchor,=20recount=20the=20tenant=20a?= =?UTF-8?q?udit=20census?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename the auth builder back to `resolveInstanceAuth` so ADR-0143's `plugin.ts#resolveInstanceAuth` anchor still resolves. - Regenerate the tenant-audit census (five new write call sites in connector-credentials.ts) and restate the prose figures it holds. Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- .../docs/permissions/tenant-audit-census.mdx | 44 +++++++++---------- ...08-tenant-audit-write-call-sites.counts.md | 26 ++++++----- .../services/service-automation/src/plugin.ts | 4 +- 3 files changed, 39 insertions(+), 35 deletions(-) diff --git a/content/docs/permissions/tenant-audit-census.mdx b/content/docs/permissions/tenant-audit-census.mdx index 15fd109ceef..44d4fd08d14 100644 --- a/content/docs/permissions/tenant-audit-census.mdx +++ b/content/docs/permissions/tenant-audit-census.mdx @@ -93,7 +93,7 @@ what moved this page's population from 225 to 227; nothing about the two sites changed, only whether this instrument could see them. **The expensive failure direction is a keyword.** Sites whose receiver the author -typed `any` have no type to read, and there are 55 of them — just over a fifth +typed `any` have no type to read, and there are 60 of them — just over a fifth of the population, concentrated in exactly the seed and bootstrap paths this control exists for. Scoring an unreadable receiver as "not an engine" would have dropped every one of them silently, with a clean exit and a smaller number that @@ -132,7 +132,7 @@ are reported as `undecidable` rather than assumed either way. The same holds twice over for the context. An options argument spelled as a literal can be read; one spelled `options`, `{ ...opts }`, or handed through a -forwarding shim cannot, and **51 of the 244 sites are spelled that way**. A +forwarding shim cannot, and **51 of the 249 sites are spelled that way**. A context resolved from an inline literal or a local `const` can be tested for `isSystem`; one arriving from a helper call cannot. @@ -197,10 +197,10 @@ reproduce them. Where it disagrees, it disagrees on the page: | carried figure | where it survives | this census | | :--- | :--- | ---: | -| 175 write call sites | quoted in the merged changeset | **244** | +| 175 write call sites | quoted in the merged changeset | **249** | | 24 carrying no tenant context | quoted in the merged changeset | **2** provable and tenancy-enabled; **29** more whose options argument is unreadable | -| 127 of 175 statically decidable, 48 runtime-parameter-name sites | restated on the `isSystem`-scoping card | **166 of 244** decidable, **78** undecidable | -| 135 (77%) silenced by the `isSystem` guard before the posture gate | the lost issue body — **no surviving corroboration** | **not reproduced**: 141 decidably elevated, 0 decidably not, 95 undecidable | +| 127 of 175 statically decidable, 48 runtime-parameter-name sites | restated on the `isSystem`-scoping card | **171 of 249** decidable, **78** undecidable | +| 135 (77%) silenced by the `isSystem` guard before the posture gate | the lost issue body — **no surviving corroboration** | **not reproduced**: 143 decidably elevated, 0 decidably not, 98 undecidable | | 141 and 132, two independent re-derivations | the card that filed this work | — | **The differences are not reconciled, and deliberately so.** The old census's @@ -210,18 +210,18 @@ be stated is what this instrument counts, which is written above and re-runnable at any commit. Two structural facts do plausibly widen this reading against any hand or regex -one, and both are counted in the generated tables below: the 55 sites reached -through an erased (`any`) receiver, and the 51 that name their object through a +one, and both are counted in the generated tables below: the 60 sites reached +through an erased (`any`) receiver, and the 56 that name their object through a `const` rather than inline. An instrument that read either the way a person does would report a smaller number and would not say so. The fourth row is the one worth flagging to anyone citing it. **The 135 / 77% figure has no surviving corroboration anywhere in the tree.** This census reads -141 of 244 (58%) as decidably elevated, with 95 more whose elevation is a +143 of 249 (57%) as decidably elevated, with 98 more whose elevation is a run-time fact — so the claim is neither confirmed nor refuted, and the honest answer is that a static reading cannot settle it. -⇒ **Cite `2 / 244`, and say what it is**: the sites whose options argument was +⇒ **Cite `2 / 249`, and say what it is**: the sites whose options argument was READ and holds no tenant context, against a decidably tenancy-enabled object. That is the control's provable yield surface. ⛔ Do not cite it as "the sites without tenant context" — **29 further sites** have an options argument this @@ -233,29 +233,29 @@ cannot read, and they are neither in nor out. | what | count | | :--- | ---: | -| write call sites on the application surface | **244** | -| …whose object name is statically decidable | 166 | +| write call sites on the application surface | **249** | +| …whose object name is statically decidable | 171 | | …whose object name is chosen at run time | 78 | -| …against an object with tenancy ENABLED | 165 | +| …against an object with tenancy ENABLED | 170 | | …against an object that declares tenancy off | 1 | -| threading a tenant context | 185 | +| threading a tenant context | 190 | | PROVABLY carrying none (options read, no context key) | **8** | | …of those, against a decidably tenancy-enabled object | **2** | | options argument UNREADABLE — may or may not carry one | 51 | | …of those, against a decidably tenancy-enabled object | 29 | -| threading a decidably ELEVATED (`isSystem`) context | 141 | +| threading a decidably ELEVATED (`isSystem`) context | 143 | | threading a context that is decidably NOT elevated | 0 | -| threading a context whose elevation is a run-time fact | 95 | +| threading a context whose elevation is a run-time fact | 98 | | how the instrument reached the site | count | | :--- | ---: | | receiver carried a readable engine type | 189 | -| receiver erased, placed by the object NAME | 45 | +| receiver erased, placed by the object NAME | 50 | | receiver erased, placed by an `object: string` PARAMETER | 5 | | receiver erased, placed by an `UNTYPED_RECEIVERS` row | 5 | | object name spelled inline | 115 | -| object name spelled through a `const` | 51 | +| object name spelled through a `const` | 56 | | object name is an `object: string` parameter | 7 | | object name is some other run-time expression | 71 | @@ -307,13 +307,13 @@ holds still. They are required to be HERE and to say WHEN they were true; their values are not compared. The reasoning, and the measurement behind it, are in `scripts/check-tenant-audit-census.mjs`. -Measured on 2026-10-26 at `700904cd5`. +Measured on 2026-10-28 at `28c124d6d`. | corpus scale (not enforced) | count | | :--- | ---: | -| tracked non-test sources scanned | 779 | -| engine-shaped types recognised | 89 | -| declared objects in the registry | 117 | -| same-named calls subtracted as non-engine | 177 | +| tracked non-test sources scanned | 800 | +| engine-shaped types recognised | 94 | +| declared objects in the registry | 121 | +| same-named calls subtracted as non-engine | 181 | {/* END GENERATED: tenant-audit-census */} diff --git a/docs/audits/2026-08-tenant-audit-write-call-sites.counts.md b/docs/audits/2026-08-tenant-audit-write-call-sites.counts.md index 5f50a57161c..02c2db29809 100644 --- a/docs/audits/2026-08-tenant-audit-write-call-sites.counts.md +++ b/docs/audits/2026-08-tenant-audit-write-call-sites.counts.md @@ -33,19 +33,19 @@ silent, and `node scripts/tenant-audit-census.mjs --write` is the resolution. | Measure | Value | |---|---:| -| Write call sites | 244 | -| Object name statically decidable | 166 | +| Write call sites | 249 | +| Object name statically decidable | 171 | | Object name chosen at run time | 78 | -| Against a tenancy-enabled object | 165 | +| Against a tenancy-enabled object | 170 | | Against an object declaring tenancy off | 1 | -| Threading a tenant context | 185 | +| Threading a tenant context | 190 | | Provably carrying none | 8 | | …and decidably tenancy-enabled | 2 | | Options argument unreadable | 51 | | …and decidably tenancy-enabled | 29 | -| Threading a decidably elevated context | 141 | +| Threading a decidably elevated context | 143 | | Threading a decidably non-elevated context | 0 | -| Threading a context of undecidable elevation | 95 | +| Threading a context of undecidable elevation | 98 | ## Subtractions the census could NOT defend — enforced @@ -90,14 +90,14 @@ holds still. They are required to be HERE and to say WHEN they were true; their values are not compared. The reasoning, and the measurement behind it, are in `scripts/check-tenant-audit-census.mjs`. -Measured on 2026-10-26 at `700904cd5`. +Measured on 2026-10-28 at `28c124d6d`. | corpus scale (not enforced) | count | | :--- | ---: | -| tracked non-test sources scanned | 779 | -| engine-shaped types recognised | 89 | -| declared objects in the registry | 117 | -| same-named calls subtracted as non-engine | 177 | +| tracked non-test sources scanned | 800 | +| engine-shaped types recognised | 94 | +| declared objects in the registry | 121 | +| same-named calls subtracted as non-engine | 181 | ## Every site @@ -201,6 +201,10 @@ Measured on 2026-10-26 at `700904cd5`. | `packages/services/service-automation/src/builtin/crud-nodes.ts` | `delete` | `objectName` | undecidable | context, elevation undecidable | 1 | | `packages/services/service-automation/src/builtin/crud-nodes.ts` | `insert` | `objectName` | undecidable | context, elevation undecidable | 1 | | `packages/services/service-automation/src/builtin/crud-nodes.ts` | `update` | `objectName` | undecidable | context, elevation undecidable | 1 | +| `packages/services/service-automation/src/connector-credentials.ts` | `delete` | `sys_connector_credential` | enabled | context, elevation undecidable | 1 | +| `packages/services/service-automation/src/connector-credentials.ts` | `insert` | `sys_connector_credential` | enabled | context, elevation undecidable | 1 | +| `packages/services/service-automation/src/connector-credentials.ts` | `update` | `sys_connector_credential` | enabled | context, elevation undecidable | 1 | +| `packages/services/service-automation/src/connector-credentials.ts` | `update` | `sys_connector_credential` | enabled | elevated | 2 | | `packages/services/service-automation/src/flow-credential-channel.ts` | `delete` | `sys_flow_credential` | enabled | elevated | 5 | | `packages/services/service-automation/src/flow-credential-channel.ts` | `insert` | `sys_flow_credential` | enabled | elevated | 1 | | `packages/services/service-automation/src/flow-credential-channel.ts` | `update` | `sys_flow_credential` | enabled | elevated | 2 | diff --git a/packages/services/service-automation/src/plugin.ts b/packages/services/service-automation/src/plugin.ts index 815b4876969..6f1aaedbedf 100644 --- a/packages/services/service-automation/src/plugin.ts +++ b/packages/services/service-automation/src/plugin.ts @@ -392,7 +392,7 @@ function credentialRefOf(auth: ConnectorInstanceAuth | undefined): string | unde * reference — an unresolved one never reaches here: the reconcile degrades * the instance first (#27237). */ -function instanceAuth(auth: ConnectorInstanceAuth | undefined, secret: string | undefined): ResolvedConnectorAuth | undefined { +function resolveInstanceAuth(auth: ConnectorInstanceAuth | undefined, secret: string | undefined): ResolvedConnectorAuth | undefined { if (!auth) return undefined; if (auth.type === 'none') return { type: 'none' }; const value = secret ?? ''; @@ -2157,7 +2157,7 @@ export class AutomationServicePlugin implements Plugin { } credential = resolution; } - const auth = instanceAuth(entry.auth, credential?.value); + const auth = resolveInstanceAuth(entry.auth, credential?.value); // The registry hands back RAW authored values (see // `DeclaredConnectorItem`), so the schema's defaults are applied From 9c8c99233e07a32edef33b997080af6b17cc0703 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 20:18:55 +0000 Subject: [PATCH 07/21] WIP(#27237): keep tracker ids out of the credential-door ledger notes check:doc-authoring refuses an internal issue id in runtime string prose; the ids move to the adjacent comment. Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- packages/runtime/src/route-ledger.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/runtime/src/route-ledger.ts b/packages/runtime/src/route-ledger.ts index f2cb4ddc07b..dde521ec7b6 100644 --- a/packages/runtime/src/route-ledger.ts +++ b/packages/runtime/src/route-ledger.ts @@ -533,9 +533,10 @@ export const ROUTE_LEDGER: readonly RouteLedgerEntry[] = [ { route: 'GET /automation/actions', domain: '/automation', disposition: 'sdk', client: 'automation.listActions' }, { route: 'GET /automation/connectors', domain: '/automation', disposition: 'sdk', client: 'automation.listConnectors' }, // [#27237] The connector credential door (ruling A on #27230): an administrator - // fills the value a declarative connector's `auth.credentialRef` names. + // fills the value a declarative connector's `auth.credentialRef` names. The + // store's `value` is a #7799 `Field.secret`; the posture refusal is S1 on #27230. { route: 'GET /automation/connectors/credential-refs', domain: '/automation', disposition: 'server-only', - note: "Every `credentialRef` a declared provider-bound connector instance names, plus every stored reference no installed package declares any more, each with its `status` (`set` / `missing` / `rejected`), its `source` (`env` / `object` / `vault` / null), whether this door may write it, and the instances naming it with their registry state. ⛔ Never a value: the store is `sys_connector_credential`, whose `value` is a `Field.secret` (#7799), and this listing reads only its bookkeeping columns. ⚑ `manage_metadata`, the gate the flow definition writes ask, fail-closed, only `isSystem` bypassing; then, under the `group` / `isolated` postures, refused 403 naming the posture for every caller — environment-level credentials only (S1 on #27230). Closed query set: none. NOT JS-SDK surface on this leg: the driver is the Console's connector credential page, a console surface that calls the platform API directly. Adding a client method reclassifies this row to `sdk`. Pinned in `domains/automation-connector-credential-door.test.ts`" }, + note: "Every `credentialRef` a declared provider-bound connector instance names, plus every stored reference no installed package declares any more, each with its `status` (`set` / `missing` / `rejected`), its `source` (`env` / `object` / `vault` / null), whether this door may write it, and the instances naming it with their registry state. ⛔ Never a value: the store is `sys_connector_credential`, whose `value` is a `Field.secret`, and this listing reads only its bookkeeping columns. ⚑ `manage_metadata`, the gate the flow definition writes ask, fail-closed, only `isSystem` bypassing; then, under the `group` / `isolated` postures, refused 403 naming the posture for every caller — environment-level credentials only. Closed query set: none. NOT JS-SDK surface on this leg: the driver is the Console's connector credential page, a console surface that calls the platform API directly. Adding a client method reclassifies this row to `sdk`. Pinned in `domains/automation-connector-credential-door.test.ts`" }, { route: 'PUT /automation/connectors/credential-refs/:ref', domain: '/automation', disposition: 'server-only', note: "Store the value for one declared reference, then reconcile: every instance naming it re-materializes without a restart. Body is the closed `{ value }`. Answers the reference's listing entry after the reconcile, never the value. Refused, before anything is written: an undeclared reference 404 `RESOURCE_NOT_FOUND`, one an environment variable of the same name supplies 409 `RESOURCE_CONFLICT` (env wins in the chain, so a stored value would never be read), an empty or oversized value or an unknown body key 400 `VALIDATION_ERROR`, and no crypto provider 503 `SERVICE_UNAVAILABLE`. Same gates and SDK posture as the GET row above. Pinned in `domains/automation-connector-credential-door.test.ts`" }, { route: 'DELETE /automation/connectors/credential-refs/:ref', domain: '/automation', disposition: 'server-only', From 702e17dedda1b9c6ec306ffa31fedd7ac8c948cc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 20:21:44 +0000 Subject: [PATCH 08/21] WIP(#27237): route ledger census sentence counts the three credential rows Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- packages/runtime/src/route-ledger.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/runtime/src/route-ledger.ts b/packages/runtime/src/route-ledger.ts index dde521ec7b6..5cdc6fc6d08 100644 --- a/packages/runtime/src/route-ledger.ts +++ b/packages/runtime/src/route-ledger.ts @@ -279,7 +279,7 @@ export const NON_DISPATCH_MOUNT_PREFIXES = [ /** * The ledger. * - * CENSUS (generated): this list holds 105 rows. + * CENSUS (generated): this list holds 108 rows. * * ⛔ THAT NUMBER IS WRITTEN BY A TOOL — never by hand. * `pnpm check:route-ledger-census` counts the rows below and fails when the two From bf5928c9e4668e6d1d68d7057d77fe89327c77a4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 21:11:31 +0000 Subject: [PATCH 09/21] WIP(#27237): system-context census row for the credential door's isSystem bypass Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- content/docs/permissions/system-context.mdx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/content/docs/permissions/system-context.mdx b/content/docs/permissions/system-context.mdx index 54eec363455..3e93ab06f77 100644 --- a/content/docs/permissions/system-context.mdx +++ b/content/docs/permissions/system-context.mdx @@ -198,6 +198,7 @@ One of the largest consumers of the flag; | 56 | Activation write / authoring refusals do not fire | runtime | Get: activation artifacts writable and authorable without the activation-authoring capability | `refuseUngrantedActivationWrite`, `refuseUngrantedActivationAuthoring` | | 55b | Walled environment-metadata refusal does not fire | metadata-core, rest, runtime | Get: under `isolated` / `group`, a write of the environment's metadata, of any type, at the `/meta` write doors (save, reset, publish, rollback) and `POST /meta/_migrate-stored` on both transports, at every `/packages` write door (create, the manifest `PATCH`, enable, disable, publish, `publish-drafts`, `discard-drafts`, `revert`, `rollback`, `commits/:commitId/revert`, `duplicate`, `adopt-orphans`, delete), at the `/automation` definition writes (create, update, delete, clone), and at the permission-set clone door. Every other caller below the platform-operator rung is refused `403` there (ADR-0131 D3, D12 items 7–8, D11), because one organization's write would change what every organization resolves. The seeders, the package door and migrations write the environment's metadata this way. Under `single` the refusal is inert for every caller | `walledCatalogWriteVerdict`, `walledCatalogRefusal`, `refuseWalledCatalogPackageWrite`, `refuseWalledFlowDefinitionWrite` | | 57 | Automation run-state read, flow-authoring write, the paused-run caller gate (screen read and resume), the two operator run-lifecycle writes and the elevated self-triggered start at every door that starts a flow by name all pass | runtime | Get: run state, flow writes, a paused run's screen and its resume with no grant and without being the run's starter — and cancelling or restoring a suspension without the platform-operator rung. The screen read and the resume ask one predicate, so this is one bypass for both. The lifecycle bypass is the in-process owner's door: plugin-approvals' revise-window recall (ADR-0044) cancels on behalf of a decision it already authorized and recorded. And starting a flow declared `runAs: 'system'` whose type is self-triggered (`autolaunched`, `record_change`, `schedule`) through any door that starts a flow by name — the trigger route, an action of `type: 'flow'` (REST `/actions` and the MCP `run_action` bridge) and a declared endpoint of `type: 'flow'` — which every other caller, the guest principal included, is refused `403 PERMISSION_DENIED` (the maintainer's rulings, letters B and A) — and starting, through the same doors, a flow whose `requiredPermissions` the caller does not hold (ruling record 6377848497, letter A). The three doors ask one check, so this is one bypass for all of them, and it is the in-process caller's, such as a job: inbound HTTP never carries the flag | `mayReadRunState`, `refuseUngrantedFlowWrite`, `isRunStarterOrRunStateReader`, `refuseUngrantedRunLifecycleWrite`, `flowStartRefusal` | +| 57b | Connector credential door's capability and posture gates pass | runtime | Get: listing, setting and clearing the environment's stored connector credentials (`/automation/connectors/credential-refs`) without `manage_metadata`, and under the `group` / `isolated` postures, where the door refuses every other caller `403 PERMISSION_DENIED` naming the posture, the platform operator included, because the store is environment-level. The bypass is the in-process caller's: inbound HTTP never carries the flag. Get nothing more about the value: no answer of this door carries it | `refuseUngrantedConnectorCredentialAccess`, `refuseWalledConnectorCredentialAccess` | | 58 | Audience-binding suggestion authority gate passes | plugin-security | Get: a system caller lists, accepts and dismisses suggestions without `manage_metadata` and, under a wall, without the platform-operator rung | `assertDeploymentAuthority` | | 59 | Email-template organization door passes; webhook provenance stamp skipped | plugin-email, plugin-webhooks | Get (`sys_email_template`): a create or update of a template row. The door refuses every other write that names a caller, `403 PERMISSION_DENIED` (ADR-0131 D6); the plugin itself no longer writes the table (templates resolve from the registry), so the v18 migration ceremony's promotion is the one remaining writer. Lose (`sys_webhook`): the row is not marked as an admin customization, and the declared-name guard does not run (a new row or a rename taking a declared webhook's name, 409) — the system writers are the platform's machinery (the legacy-secret sweep, the ownership ceremony, migrations) | `isOrganizationWrite`, `bindWebhookProvenanceStamp`, `bindDeclaredWebhookNameGuard` | | 60 | **Automation flow data nodes re-add the `owner_id` stamp** (the one place row 2's gap is compensated inline) | service-automation | Get: a flow-authored INSERT under system elevation still lands owned, when the run resolved a user. Fill-only — flow-authored values win | `stampSystemInsertOwner`, called from `packages/services/service-automation/src/builtin/crud-nodes.ts#registerCrudNodes` | @@ -622,8 +623,10 @@ and CI refuses one that differs from what the tree produces. | `packages/runtime/src/domains/automation.ts#handleAutomationRequest` | 1 | 52 | | `packages/runtime/src/domains/automation.ts#isRunStarterOrRunStateReader` | 1 | 57 | | `packages/runtime/src/domains/automation.ts#mayReadRunState` | 1 | 57 | +| `packages/runtime/src/domains/automation.ts#refuseUngrantedConnectorCredentialAccess` | 1 | 57b | | `packages/runtime/src/domains/automation.ts#refuseUngrantedFlowWrite` | 1 | 57 | | `packages/runtime/src/domains/automation.ts#refuseUngrantedRunLifecycleWrite` | 1 | 57 | +| `packages/runtime/src/domains/automation.ts#refuseWalledConnectorCredentialAccess` | 1 | 57b | | `packages/runtime/src/domains/automation.ts#refuseWalledFlowDefinitionWrite` | 1 | 55b | | `packages/runtime/src/domains/i18n.ts#handleI18nRequest` | 1 | 52 | | `packages/runtime/src/domains/mcp.ts#handleMcpRequest` | 1 | 53 | From 70a67e293faf8622509700b356c92f14aab66535 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 21:27:48 +0000 Subject: [PATCH 10/21] WIP(#27237): integration recipes say how an unset credentialRef behaves now Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- .../references/integration-recipes.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/skills/objectstack-automation/references/integration-recipes.md b/skills/objectstack-automation/references/integration-recipes.md index 4d530a00ff0..63f72b9a1cf 100644 --- a/skills/objectstack-automation/references/integration-recipes.md +++ b/skills/objectstack-automation/references/integration-recipes.md @@ -4,9 +4,9 @@ The rule is `SKILL.md` § Integrations are declared, not coded; this file is the shapes. Each recipe is what the **app author** declares (a `connectors:` entry, the fields the answer lands in, a flow dispatching `connector_action`) and the one thing the **customer's administrator** supplies: the value behind -`auth.credentialRef` — today an environment variable of that name on the -deployment (a credential object filled in the running app is a platform card in -flight); an unset ref is a hard boot error naming the entry. A spec is not a +`auth.credentialRef` — an environment variable of that name (wins), or a +value set in the running app at `PUT /api/v1/automation/connectors/credential-refs/:ref`; +an unset ref leaves its instance degraded until set. A spec is not a secret and ships in the package; an `api` flow's hook secret goes to the deployment and to the sender. Scaffolded apps list the three executors in `plugins:`; keep `automation` in `requires` whenever a connector is declared. Full guide: https://objectstack.ai/docs/automation/connectors. From f227c26b47c41854b22c246b13867c49e1631e5e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 22:33:21 +0000 Subject: [PATCH 11/21] fix(cli): ownership inventory row for sys_connector_credential (#27237) The ADR-0131 D10 inventory pin reds a registered platform object without a row; the new environment-level credential store takes the flow credential store's `column-drop` fate. Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- ...237-cli-ownership-inventory-connector-credential.md | 10 ++++++++++ .../cli/src/utils/organization-ownership-inventory.ts | 8 ++++++++ 2 files changed, 18 insertions(+) create mode 100644 .changeset/27237-cli-ownership-inventory-connector-credential.md diff --git a/.changeset/27237-cli-ownership-inventory-connector-credential.md b/.changeset/27237-cli-ownership-inventory-connector-credential.md new file mode 100644 index 00000000000..7860396d4d2 --- /dev/null +++ b/.changeset/27237-cli-ownership-inventory-connector-credential.md @@ -0,0 +1,10 @@ +--- +'@objectstack/cli': patch +--- + +fix(cli): the organization-ownership inventory has a row for `sys_connector_credential` + +Clause-②: no +Metadata-protocol: none + +The ADR-0131 D10 inventory now has a row for the new environment-level connector credential store (`systemFields.tenant: false`). Its fate is `column-drop`, the same fate the flow credential store has: the upgrade plan treats the table as deployment-level state with no organization column, so a database carrying it is not refused as an uninventoried platform table. diff --git a/packages/cli/src/utils/organization-ownership-inventory.ts b/packages/cli/src/utils/organization-ownership-inventory.ts index 354ee2064f7..7e77436e649 100644 --- a/packages/cli/src/utils/organization-ownership-inventory.ts +++ b/packages/cli/src/utils/organization-ownership-inventory.ts @@ -752,6 +752,14 @@ export const ORGANIZATION_OWNERSHIP_INVENTORY: readonly InventoryEntry[] = [ `${D7}: a flow's encrypted credential is read by its flow, never by organization; systemFields.tenant: false ` + 'with the open ceremony (18.x step 6, line fact F2)', }, + { + object: 'sys_connector_credential', + census: PMAIN, + fate: 'column-drop', + citation: + `${CENSUS_MAIN}: systemFields.tenant: false; ${D7}: a declarative connector's stored credential is environment-level, ` + + 'read by the reference its instances name, never by organization', + }, // Inbound mail (`@objectstack/service-mailbox`): every row is written by the // system with the mailbox's organization stamped (ADR-0131 D9), so a NULL // row has no producer today; the parents below are where one would take it. From d68bc8d9982644c8896293f2ceced164c001802e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 00:59:23 +0000 Subject: [PATCH 12/21] =?UTF-8?q?WIP(#27237):=20seat=20review=20r1=20?= =?UTF-8?q?=E2=80=94=20hold=20only=20stored=20credentials,=20proof-based?= =?UTF-8?q?=20reset,=20serialized=20door=20writes,=20walled=20chain?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- .../src/mcp-http-discovery-faults.test.ts | 2 +- .../connector-mcp/src/mcp-provider.test.ts | 2 +- .../connector-mcp/src/mcp-provider.ts | 29 +- .../src/connector-credentials.ts | 127 +++-- .../services/service-automation/src/plugin.ts | 462 +++++++++++++----- .../src/sys-connector-credential.object.ts | 35 +- .../src/integration/connector-provider.ts | 10 + 7 files changed, 504 insertions(+), 163 deletions(-) diff --git a/packages/connectors/connector-mcp/src/mcp-http-discovery-faults.test.ts b/packages/connectors/connector-mcp/src/mcp-http-discovery-faults.test.ts index c2e2a1dda71..0ce6924e583 100644 --- a/packages/connectors/connector-mcp/src/mcp-http-discovery-faults.test.ts +++ b/packages/connectors/connector-mcp/src/mcp-http-discovery-faults.test.ts @@ -157,7 +157,7 @@ describe('credential rejection — real Streamable-HTTP server', () => { expect((err as Error).message).toContain( `connector-mcp provider: connector 'vendor' was refused by its MCP server (HTTP ${status})`, ); - expect((err as Error).message).toContain('is not retried'); + expect((err as Error).message).toContain('This is a configuration fault — fix the credential.'); expect(server.requests).toEqual(['POST']); }, ); diff --git a/packages/connectors/connector-mcp/src/mcp-provider.test.ts b/packages/connectors/connector-mcp/src/mcp-provider.test.ts index dd6eaeb5ff1..a1995174943 100644 --- a/packages/connectors/connector-mcp/src/mcp-provider.test.ts +++ b/packages/connectors/connector-mcp/src/mcp-provider.test.ts @@ -162,7 +162,7 @@ describe('mcp provider — a rejected credential is a configuration fault', () = expect((err as Error).message).toContain( `connector-mcp provider: connector 'github' was refused by its MCP server (HTTP ${status})`, ); - expect((err as Error).message).toContain('is not retried'); + expect((err as Error).message).toContain('This is a configuration fault — fix the credential.'); }); it('HTTP 401 on tools/list throws the marker too, and closes the client', async () => { diff --git a/packages/connectors/connector-mcp/src/mcp-provider.ts b/packages/connectors/connector-mcp/src/mcp-provider.ts index c2f957cb673..999154b7fd3 100644 --- a/packages/connectors/connector-mcp/src/mcp-provider.ts +++ b/packages/connectors/connector-mcp/src/mcp-provider.ts @@ -205,8 +205,9 @@ async function credentialRejectionStatus(err: unknown): Promise { @@ -248,14 +249,15 @@ export function createMcpProviderFactory(deps: McpProviderDeps = {}): ConnectorP // Thrown as `ConnectorCredentialRejectedError` (ruling on #27237, // comment 6470087293): the materializer degrades the instance naming the // refusal and marks a stored credential `rejected`, instead of failing - // boot — and, like the plain throw before it, does not retry it on a - // timer. The next attempt is the next change of the credential. + // boot. Whether it retries is the materializer's call, by where the + // credential came from: a stored or declared one waits for its next + // change, an environment or vault one retries with backoff. const status = await credentialRejectionStatus(err); if (status !== undefined) { throw new ConnectorCredentialRejectedError( `connector-mcp provider: connector '${ctx.name}' was refused by its MCP server (HTTP ${status}): the ` + `credential it sent (the instance auth, or providerConfig.transport.headers) is missing, wrong or not ` + - `authorized. This is a configuration fault and is not retried — fix the credential. ` + + `authorized. This is a configuration fault — fix the credential. ` + `${(err as Error).message}`, status, { cause: err }, @@ -268,6 +270,21 @@ export function createMcpProviderFactory(deps: McpProviderDeps = {}): ConnectorP { cause: err }, ); } - return { def: bundle.def, handlers: bundle.handlers, close: bundle.close }; + // [#27237] The handshake carried the instance's credential only over + // http, and only as a header (an api-key `paramName` is not sent): then + // the server's answer proved it. + const credentialVerified = transport.kind === 'http' && sendsInstanceAuth(ctx.auth); + return { + def: bundle.def, + handlers: bundle.handlers, + close: bundle.close, + ...(credentialVerified ? { credentialVerified } : {}), + }; }; } + +/** Whether {@link applyAuthToHeaders} puts the instance's credential on the http transport. */ +function sendsInstanceAuth(auth: ResolvedConnectorAuth | undefined): boolean { + if (!auth || auth.type === 'none') return false; + return !(auth.type === 'api-key' && auth.paramName); +} diff --git a/packages/services/service-automation/src/connector-credentials.ts b/packages/services/service-automation/src/connector-credentials.ts index 908e2d8ce86..390aabda83d 100644 --- a/packages/services/service-automation/src/connector-credentials.ts +++ b/packages/services/service-automation/src/connector-credentials.ts @@ -16,6 +16,11 @@ * 3. the resolver the host injected (`AutomationServicePluginOptions.credentialResolver`) * — the enterprise vault arm, if one is wired. * + * Under a walled tenancy posture (`group` / `isolated`) the stored arm is not + * consulted at all (S1 on #27230): the plugin composes the chain without a + * store, so a reference resolves from the environment or the host's resolver + * only, as the credential door's refusal there says. + * * ⛔ A value leaves this module only as the return of {@link resolveConnectorCredential}, * whose one caller hands it to a provider factory. Everything else here — * fingerprints, listings, the store's rows — carries the reference, a source, @@ -68,6 +73,7 @@ export interface ConnectorCredentialRecord { connectorName?: string; packageId?: string; lastVerifiedAt?: string; + /** When the value was last set through the door, and by whom — not moved by a status change. */ updatedAt?: string; updatedBy?: string; } @@ -75,9 +81,26 @@ export interface ConnectorCredentialRecord { /** The columns a read asks for — `value` is never among them. */ const RECORD_FIELDS = [ 'id', 'ref', 'revision', 'status', 'status_reason', 'connector_name', 'package_id', - 'last_verified_at', 'updated_at', 'updated_by', + 'last_verified_at', 'set_at', 'set_by', ]; +/** + * The prefix the data engine reads as a handle to a secret it already holds + * (ObjectQL's `SECRET_REF_PREFIX`): a `secret`-typed value spelled this way is + * stored as given, unencrypted, and dereferenced as another `sys_secret` row. + * The credential door refuses a value that starts with it. + */ +export const SECRET_HANDLE_PREFIX = 'secret:'; + +/** + * A credential door refusal: an ADR-0112 standard-catalog `code` and its + * `status` on the thrown error, the shape the `/automation` domain answers a + * service rejection with. Its message names the reference, never a value. + */ +export function credentialDoorRefusal(message: string, code: string, status: number): Error { + return Object.assign(new Error(message), { code, status, statusCode: status }); +} + /** * ADR-0112 pair for a credential the deployment cannot handle safely right now: * no data engine, no crypto provider, a stored row that does not decrypt. A @@ -131,8 +154,8 @@ function toRecord(row: Record): ConnectorCredentialRecord | und connectorName: optionalText(row.connector_name), packageId: optionalText(row.package_id), lastVerifiedAt: optionalText(row.last_verified_at), - updatedAt: optionalText(row.updated_at), - updatedBy: optionalText(row.updated_by), + updatedAt: optionalText(row.set_at), + updatedBy: optionalText(row.set_by), }; } @@ -222,6 +245,11 @@ export class ConnectorCredentialStore { * Store a value for `ref`, replacing what the row held and raising its * revision. The engine encrypts it; with no crypto provider the engine * refuses before anything is written, and so does this. + * + * Read-then-write: the plugin runs every door write on its reconcile + * queue, so two writes in one process never interleave. Two replicas + * inserting the first row for one reference at once meet the unique index + * on `ref`, and the loser answers `409`. */ async set(args: { ref: string; @@ -244,6 +272,9 @@ export class ConnectorCredentialStore { revision, status: 'set', status_reason: null, + last_verified_at: null, + set_at: new Date().toISOString(), + set_by: args.userId ?? null, connector_name: args.connectorNames.length > 0 ? args.connectorNames.join(', ') : null, package_id: args.packageIds.length > 0 ? args.packageIds.join(', ') : null, }; @@ -269,6 +300,14 @@ export class ConnectorCredentialStore { + '(setCryptoProvider — LocalCryptoProvider in development, a KMS-backed provider in production) and set it again.', ); } + if ((err as { code?: unknown })?.code === 'DUPLICATE_RECORD') { + throw credentialDoorRefusal( + `Credential '${args.ref}' was stored by another writer at the same moment. Nothing was written by this ` + + 'request; read the listing and set it again if it still needs this value.', + 'RESOURCE_CONFLICT', + 409, + ); + } // Whatever else refused the write answers without the value it was handed. throw withoutCredentialThrown(err, args.value); } @@ -287,11 +326,14 @@ export class ConnectorCredentialStore { } /** - * Record that an instance materialized with revision `revision`: the - * timestamp moves, and a `rejected` row whose same value now works returns - * to `set`. A row rewritten since (another revision) is left alone. + * Record that an upstream ACCEPTED revision `revision`: a provider that + * proved it at materialization (`credentialVerified`, the mcp provider), or + * a call its upstream answered `2xx`. The timestamp moves, and a `rejected` + * row whose same value now works returns to `set`. A materialization alone + * proves nothing (rest and openapi first send the value on a call), so it + * never calls this. A row rewritten since (another revision) is left alone. */ - async markMaterialized(record: ConnectorCredentialRecord, at: Date = new Date()): Promise { + async markVerified(record: ConnectorCredentialRecord, at: Date = new Date()): Promise { const engine = this.resolveEngine(); if (!engine) return; const current = await this.read(record.ref); @@ -359,7 +401,8 @@ export function withoutCredentialThrown(err: unknown, secret: string | undefined export interface ConnectorCredentialChain { /** The environment arm. Defaults to {@link defaultEnvCredentialResolver}. */ readonly env?: (ref: string) => string | undefined; - readonly store: ConnectorCredentialStore; + /** The stored arm. Absent under a walled tenancy posture, where it is not consulted. */ + readonly store?: ConnectorCredentialStore; /** The host's injected resolver, if any — the vault arm. */ readonly vault?: CredentialResolver; } @@ -399,9 +442,11 @@ export function connectorCredentialFingerprint( rows: ReadonlyMap | undefined, ): string { if (envValue(chain, ref) !== undefined) return 'env'; - if (rows === undefined) return 'unreadable'; - const row = rows.get(ref); - if (row) return `object:${row.revision}`; + if (chain.store) { + if (rows === undefined) return 'unreadable'; + const row = rows.get(ref); + if (row) return `object:${row.revision}`; + } return chain.vault ? 'vault' : 'missing'; } @@ -425,18 +470,20 @@ export async function resolveConnectorCredential( ): Promise { const fromEnv = envValue(chain, ref); if (fromEnv !== undefined) return { kind: 'resolved', source: 'env', value: fromEnv }; - if (rows === undefined) { - return { - kind: 'unavailable', - reason: `the connector credential store could not be read, so credential '${ref}' cannot be resolved now; the instance retries on its own.`, - }; - } - const record = rows.get(ref); - if (record) { - try { - return { kind: 'resolved', source: 'object', value: await chain.store.resolve(record), record }; - } catch (err) { - return { kind: 'unavailable', reason: (err as Error).message }; + if (chain.store) { + if (rows === undefined) { + return { + kind: 'unavailable', + reason: `the connector credential store could not be read, so credential '${ref}' cannot be resolved now; the instance retries on its own.`, + }; + } + const record = rows.get(ref); + if (record) { + try { + return { kind: 'resolved', source: 'object', value: await chain.store.resolve(record), record }; + } catch (err) { + return { kind: 'unavailable', reason: (err as Error).message }; + } } } if (chain.vault) { @@ -457,7 +504,7 @@ export async function probeConnectorCredential( rows: ReadonlyMap, ): Promise<{ source: ConnectorCredentialSource | null; status: 'set' | 'missing' | 'rejected'; record?: ConnectorCredentialRecord }> { if (envValue(chain, ref) !== undefined) return { source: 'env', status: 'set' }; - const record = rows.get(ref); + const record = chain.store ? rows.get(ref) : undefined; if (record) return { source: 'object', status: record.status, record }; if (chain.vault) { let present = false; @@ -511,11 +558,35 @@ export function upstreamRefusalMessage(body: unknown, secret: string | undefined return flat.length > UPSTREAM_REFUSAL_MESSAGE_MAX_LENGTH ? `${flat.slice(0, UPSTREAM_REFUSAL_MESSAGE_MAX_LENGTH)}…` : flat; } -/** The degraded reason for an instance whose upstream refused its credential on a call. */ -export function credentialRejectedOnCallReason(name: string, ref: string, status: number, upstream: string | undefined): string { +/** + * The degraded reason for an instance whose upstream refused its credential + * on a call. `held`: the value is a stored one, so the instance waits for it + * to be set again (or a restart); otherwise (an environment variable, the + * host's resolver) it is retried on the backoff, since the value can change + * where this door cannot see it. + */ +export function credentialRejectedOnCallReason( + name: string, + ref: string, + status: number, + upstream: string | undefined, + held: boolean, +): string { return ( `connector '${name}' was refused by its upstream (HTTP ${status}) on a call: credential '${ref}' is missing, wrong ` + - `or not authorized${upstream ? ` (the upstream said: ${upstream})` : ''}. The instance stays degraded, without ` + - `retrying, until credential '${ref}' changes.` + `or not authorized${upstream ? ` (the upstream said: ${upstream})` : ''}. ` + + (held + ? `The instance stays degraded, without retrying, until credential '${ref}' is set again or the server restarts.` + : 'The instance is retried with backoff.') ); } + +/** + * A `2xx` result of a connector call — the upstream accepted the credential. + * The rest and openapi providers answer `{ status, ok, body }`. + */ +export function isAcceptedCallResult(result: unknown): boolean { + if (typeof result !== 'object' || result === null) return false; + const status = (result as { status?: unknown }).status; + return typeof status === 'number' && status >= 200 && status < 300; +} diff --git a/packages/services/service-automation/src/plugin.ts b/packages/services/service-automation/src/plugin.ts index d22a1d4fbf7..b337babbd9b 100644 --- a/packages/services/service-automation/src/plugin.ts +++ b/packages/services/service-automation/src/plugin.ts @@ -1,7 +1,7 @@ // Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. import type { Plugin, PluginContext } from '@objectstack/core'; -import { resolveLocalizationContext, resolveUserAuthzGrants } from '@objectstack/core'; +import { effectiveTenancyPosture, isServiceNotRegisteredError, resolveLocalizationContext, resolveUserAuthzGrants } from '@objectstack/core'; import type { IJobService } from '@objectstack/spec/contracts'; import type { FlowFunctionEffect } from '@objectstack/spec/automation'; import type { @@ -13,6 +13,7 @@ import type { } from '@objectstack/spec/integration'; import { isConnectorCredentialRejected, isConnectorUpstreamUnavailable, RetryConfigSchema } from '@objectstack/spec/integration'; import { stripReadDecorations } from '@objectstack/spec/kernel'; +import { postureEnforcesWall, type TenancyPosture } from '@objectstack/spec/security'; import type { MetadataMutationEvent } from '@objectstack/metadata-protocol'; import { AutomationEngine } from './engine.js'; import type { AutomationEngineOptions, RefusalI18nService, RunSummaryLogLevel } from './engine.js'; @@ -36,9 +37,12 @@ import { CONNECTOR_CREDENTIAL_VALUE_MAX_LENGTH, ConnectorCredentialStore, ConnectorCredentialUnavailableError, + SECRET_HANDLE_PREFIX, connectorCredentialFingerprint, + credentialDoorRefusal, credentialRejectedOnCallReason, credentialRejectionStatusOf, + isAcceptedCallResult, isSuppliedByEnv, missingCredentialReason, probeConnectorCredential, @@ -411,15 +415,6 @@ function resolveInstanceAuth(auth: ConnectorInstanceAuth | undefined, secret: st } } -/** - * [#27237] A credential door refusal: an ADR-0112 standard-catalog `code` and - * its `status` on the thrown error, the shape the `/automation` domain answers - * a service rejection with. Its message names the reference, never a value. - */ -function credentialDoorRefusal(message: string, code: string, status: number): Error { - return Object.assign(new Error(message), { code, status, statusCode: status }); -} - /** * Stable signature of a provider-bound instance's materialization inputs * (ADR-0097). Drives the `metadata:reloaded` reconcile: an unchanged signature @@ -729,12 +724,33 @@ export class AutomationServicePlugin implements Plugin { * entry's re-materialization failed — then the old live connector keeps * serving (still tracked there) while the retry is pending here. * - * [#27237] `held` marks an instance whose upstream refused its credential: - * degraded like the others, but neither the retry timer nor a reconcile - * re-attempts it while its signature — which carries the credential's - * source and revision — stays the same. + * [#27237] `held` marks an instance whose upstream refused a credential + * stored through the credential door: degraded like the others, but + * neither the retry timer nor a reconcile re-attempts it while its + * signature — which carries the row's revision — stays the same. Setting + * the credential again, a change of the declaration, or a restart lifts + * it. A refused value the door does not hold (an environment variable, + * the host's resolver) is never held: it can change where this process + * cannot see it, so its instance retries on the backoff. */ private degradedInstances = new Map(); + /** + * [#27237] References whose upstream refused a value the credential door + * does not hold — an environment variable or the host's resolver — keyed + * by reference, with the source that was refused and the reason (already + * scrubbed of the value). There is no row to mark, so this is what the + * listing reports as `rejected` until an upstream accepts the reference + * again or it stops resolving from that source. + */ + private credentialRefusals = new Map(); + /** + * [#27237] The backoff count of an instance refused on such a value, by + * instance name. Kept across a re-materialization — a rest or openapi + * instance materializes without sending its credential, so the count would + * otherwise restart at one on every retry — and dropped once an upstream + * accepts the credential, or the instance or its credential's source changes. + */ + private credentialRefusalAttempts = new Map(); private declarativeRetryTimer?: ReturnType; /** * [#7742] The connector collection carried by the most recent @@ -1561,14 +1577,17 @@ export class AutomationServicePlugin implements Plugin { // registry is fully populated with declared `connectors:` entries. Turn each // provider-bound entry into a live, dispatchable connector now. Deliberately // NOT wrapped in a swallowing try/catch: a misconfiguration (unknown provider, - // invalid providerConfig, unresolvable credentialRef, name conflict) MUST fail - // boot loudly — a metadata platform shipping a plausible-but-dead connector is - // the worst failure mode. A start()-phase throw is fatal under both LiteKernel - // and ObjectKernel (rollbackOnFailure defaults on), so the operator sees it. - // One carve-out (#3017): a factory signalling its UPSTREAM is temporarily - // unreachable (CONNECTOR_UPSTREAM_UNAVAILABLE) degrades that one instance — - // registered action-less + retried with backoff — instead of failing boot; - // an operational blip in one integration must not take the whole app down. + // invalid providerConfig, a host credential resolver that throws, name + // conflict) MUST fail boot loudly — a metadata platform shipping a + // plausible-but-dead connector is the worst failure mode. A start()-phase + // throw is fatal under both LiteKernel and ObjectKernel (rollbackOnFailure + // defaults on), so the operator sees it. + // Carve-outs degrade that one instance (registered action-less) instead of + // failing boot: a factory signalling its UPSTREAM is temporarily unreachable + // (#3017, CONNECTOR_UPSTREAM_UNAVAILABLE — retried with backoff), and, since + // #27237, a `credentialRef` no source answers for (it materializes once the + // credential is set) or one its upstream refuses (CONNECTOR_CREDENTIAL_REJECTED). + // An operational blip in one integration must not take the whole app down. await this.materializeDeclaredConnectors(ctx, { fatal: true }); // ── Runtime re-bind: re-sync flow triggers on 'metadata:reloaded' ────── @@ -1985,8 +2004,18 @@ export class AutomationServicePlugin implements Plugin { // registry and the tracked maps. Each caller still observes its own // run's failure (a fatal boot error propagates), while the chain itself // is never left poisoned for the next caller. - const run = this.reconcileQueue.then(() => this.reconcileDeclaredConnectors(ctx, opts)); - this.reconcileQueue = run.catch(() => undefined); + return this.onReconcileQueue(() => this.reconcileDeclaredConnectors(ctx, opts)); + } + + /** + * Run `fn` after every reconcile already queued, and before any queued + * after it. The credential door's writes run here too (#27237), so a write + * and the reconcile it triggers never interleave with another write's, and + * the instance always ends on the value the store ends on. + */ + private onReconcileQueue(fn: () => Promise): Promise { + const run = this.reconcileQueue.then(fn); + this.reconcileQueue = run.then(() => undefined, () => undefined); return run; } @@ -2054,25 +2083,39 @@ export class AutomationServicePlugin implements Plugin { // signature — `env`, `object:`, `vault`, `missing` — so a // credential write re-materializes exactly the instances naming it. One // read of the store's bookkeeping per run, never a value; skipped when - // nothing declared names a reference. - const chain = this.connectorCredentialChain(); + // nothing declared names a reference. Under a walled tenancy posture + // the chain has no stored arm (S1 on #27230); a posture that cannot be + // read is not one that is absent, so the rows count as unreadable then. + let chain: ConnectorCredentialChain = { vault: this.options.credentialResolver }; let credentialRows: Map | undefined = new Map(); if ([...desired.values()].some((d) => credentialRefOf(d.entry.auth) !== undefined)) { try { - credentialRows = await chain.store.list(); + chain = await this.connectorCredentialChain(ctx); + if (chain.store) credentialRows = await chain.store.list(); } catch (err) { + chain = { store: this.connectorCredentialStoreOrNone(), vault: this.options.credentialResolver }; credentialRows = undefined; ctx.logger.warn( - '[Automation] the connector credential store could not be read — an instance whose credential it would ' - + 'supply keeps what it has and retries.', + '[Automation] the connector credential store, or the tenancy posture that decides whether it is read, ' + + 'could not be read — an instance whose credential it would supply keeps what it has and retries.', describeThrownForLog(err), ); } } + const credentialSources = new Map(); for (const d of desired.values()) { const ref = credentialRefOf(d.entry.auth); if (ref === undefined) continue; - d.signature += `|credential:${connectorCredentialFingerprint(chain, ref, credentialRows)}`; + const fingerprint = connectorCredentialFingerprint(chain, ref, credentialRows); + credentialSources.set(ref, fingerprint); + d.signature += `|credential:${fingerprint}`; + } + // A refusal of a value the door does not hold stands until an upstream + // accepts the reference — or the reference stops resolving from the + // source that was refused (`env` / `vault` are fingerprints as well). + for (const [ref, refusal] of [...this.credentialRefusals]) { + const fingerprint = credentialSources.get(ref); + if (fingerprint !== 'unreadable' && fingerprint !== refusal.source) this.credentialRefusals.delete(ref); } // 1. Remove connectors we previously materialized that are no longer @@ -2090,6 +2133,9 @@ export class AutomationServicePlugin implements Plugin { try { engine.unregisterConnector(name); } catch { /* ignore */ } ctx.logger.info(`[Automation] dropped degraded connector instance '${name}' (no longer declared)`); } + for (const name of [...this.credentialRefusalAttempts.keys()]) { + if (!desired.has(name)) this.credentialRefusalAttempts.delete(name); + } // 2. Add new instances and re-materialize changed ones (signature diff). let changed = 0; @@ -2107,8 +2153,9 @@ export class AutomationServicePlugin implements Plugin { continue; } - // [#27237] Held: its upstream refused this very credential. The - // next attempt is the next change of the credential or the entry. + // [#27237] Held: its upstream refused this very stored value. The + // next attempt is the next change of the credential or the entry, + // or a restart. const held = this.degradedInstances.get(name); if (held?.held && held.signature === signature) continue; @@ -2237,6 +2284,7 @@ export class AutomationServicePlugin implements Plugin { entry, provider, signature, + ref, live: existing, takeDown: false, credential, @@ -2292,10 +2340,13 @@ export class AutomationServicePlugin implements Plugin { const def = { ...materialization.def, name }; const tracked = { signature, provider, close: materialization.close }; // [#27237] A credentialed instance's calls are watched for the - // upstream's 401 / 403 (rest and openapi answer them as a result). + // upstream's 401 / 403 (rest and openapi answer them as a result), + // and — unless the provider proved the credential while + // materializing — for the first one it accepts. + const proven = materialization.credentialVerified === true; const handlers = credential && ref !== undefined ? this.watchForCredentialRejection(engine, ctx, { - name, entry, provider, signature, ref, credential, tracked, + name, entry, provider, signature, ref, credential, tracked, proven, }, materialization.handlers) : materialization.handlers; engine.registerConnector(def, handlers, 'declarative'); @@ -2308,17 +2359,13 @@ export class AutomationServicePlugin implements Plugin { `[Automation] ${recovered ? 'recovered' : existing ? 're-materialized' : 'materialized'} ` + `connector instance '${name}' via provider '${provider}' (${def.actions?.length ?? 0} action(s))`, ); - // [#27237] The stored value worked: its row records when, and a - // `rejected` row whose same value now materializes returns to `set`. - if (credential?.source === 'object' && credential.record) { - try { - await chain.store.markMaterialized(credential.record); - } catch (markErr) { - ctx.logger.warn( - `[Automation] connector instance '${name}' materialized, but its credential row's last-materialized time could not be recorded.`, - describeThrownForLog(markErr), - ); - } + // [#27237] Only a provider that proved the credential while + // materializing (`credentialVerified` — mcp's handshake) says the + // upstream accepted it. rest and openapi first send it on a call, + // where the watcher records the first accepted one; a + // materialization alone never turns a `rejected` row back to `set`. + if (credential && ref !== undefined && proven) { + await this.recordCredentialAccepted(ctx, name, ref, credential); } } @@ -2430,12 +2477,87 @@ export class AutomationServicePlugin implements Plugin { ); } - /** [#27237] The `credentialRef` chain this plugin resolves through: env → the store → the host's resolver. */ - private connectorCredentialChain(): ConnectorCredentialChain { - return { - store: this.connectorCredentialStore ?? new ConnectorCredentialStore(() => undefined), - vault: this.options.credentialResolver, - }; + /** + * [#27237] The `credentialRef` chain this plugin resolves through: env → + * the store → the host's resolver. Under a walled tenancy posture + * (`group` / `isolated`) the store is left out — S1 on #27230: a stored + * connector credential is environment-level, and an environment several + * organizations share is not supported by the store — so a reference + * resolves from the environment or the host's resolver only. Throws when + * the posture cannot be read. + */ + private async connectorCredentialChain(ctx: PluginContext): Promise { + const vault = this.options.credentialResolver; + const posture = await this.readTenancyPosture(ctx); + if (posture !== undefined && postureEnforcesWall(posture)) return { vault }; + return { store: this.connectorCredentialStoreOrNone(), vault }; + } + + /** [#27237] The store, or one with no data engine — which stores nothing and reads every row as absent. */ + private connectorCredentialStoreOrNone(): ConnectorCredentialStore { + return this.connectorCredentialStore ?? new ConnectorCredentialStore(() => undefined); + } + + /** + * [#27237] The deployment's effective tenancy posture, from the `tenancy` + * service; `undefined` when none is registered (a single-organization + * composition). Only "not registered" is absorbed: a registered service + * that fails to resolve throws, since a posture that could not be read is + * not one that is absent — the rest plugin's `tenancyServiceProvider` + * (#13906) for the same reason. + */ + private async readTenancyPosture(ctx: PluginContext): Promise { + const kernel = typeof ctx.getKernel === 'function' ? ctx.getKernel() : undefined; + let tenancy: unknown; + if (kernel && typeof kernel.getServiceAsync === 'function') { + try { + tenancy = await kernel.getServiceAsync('tenancy'); + } catch (err) { + if (!isServiceNotRegisteredError(err)) throw err; + } + } else { + // A `KernelBase`-shaped host (`LiteKernel`) has no service + // factories, so "not registered" is the only fault its accessor reports. + try { tenancy = ctx.getService('tenancy'); } catch { tenancy = undefined; } + } + return effectiveTenancyPosture(tenancy as Parameters[0]); + } + + /** + * [#27237] The chain the credential door reads and writes through, with + * the posture it was composed under. A posture that cannot be read is the + * door's `503`, never a guess. + */ + private async doorCredentialChain( + ctx: PluginContext, + ): Promise< + | { chain: ConnectorCredentialChain & { store: ConnectorCredentialStore }; walled?: undefined } + | { chain: ConnectorCredentialChain; walled: TenancyPosture } + > { + let posture: TenancyPosture | undefined; + try { + posture = await this.readTenancyPosture(ctx); + } catch { + throw new ConnectorCredentialUnavailableError( + 'The tenancy posture could not be read, so whether connector credentials are stored in this environment ' + + 'cannot be told now. Nothing was read or changed.', + ); + } + const vault = this.options.credentialResolver; + if (posture !== undefined && postureEnforcesWall(posture)) return { chain: { vault }, walled: posture }; + return { chain: { store: this.connectorCredentialStoreOrNone(), vault } }; + } + + /** [#27237] The walled refusal of a credential write — S1 on #27230 — naming the posture. */ + private walledCredentialRefusal(posture: TenancyPosture): Error { + return credentialDoorRefusal( + `Connector credentials are environment-level, and this environment runs the '${posture}' tenancy posture, ` + + "where several organizations share it, so none is stored here. A declarative connector's credentialRef " + + "still resolves from a server environment variable of the same name, or the host's credential resolver. " + + 'Nothing was changed.', + 'PERMISSION_DENIED', + 403, + ); } /** @@ -2526,6 +2648,7 @@ export class AutomationServicePlugin implements Plugin { /** [#27237] The stored rows, or the door's 503 when the store cannot be read — never an empty listing that hides them. */ private async readCredentialRowsForDoor(chain: ConnectorCredentialChain): Promise> { + if (!chain.store) return new Map(); try { return await chain.store.list(); } catch { @@ -2543,13 +2666,18 @@ export class AutomationServicePlugin implements Plugin { rows: ReadonlyMap, ): Promise { const probe = await probeConnectorCredential(chain, ref, rows); - const record = rows.get(ref); + const record = chain.store ? rows.get(ref) : undefined; const descriptors = new Map((this.engine?.getConnectorDescriptors() ?? []).map((d) => [d.name, d])); + // A refused value the door does not hold has no row: its refusal is + // kept in memory, and reported while the reference still resolves from there. + const refusal = this.credentialRefusals.get(ref); + const refused = refusal !== undefined && refusal.source === probe.source ? refusal : undefined; + const statusReason = refused ? refused.reason : probe.source === 'object' ? record?.statusReason : undefined; return { ref, - status: probe.status, + status: refused ? 'rejected' : probe.status, source: probe.source, - writable: probe.source !== 'env', + writable: probe.source !== 'env' && chain.store !== undefined, declared: declaredBy !== undefined, connectors: (declaredBy ?? []).map((c) => { const d = descriptors.get(c.name); @@ -2560,7 +2688,7 @@ export class AutomationServicePlugin implements Plugin { ...(d?.degradedReason ? { degradedReason: d.degradedReason } : {}), }; }), - ...(record?.statusReason && probe.source === 'object' ? { statusReason: record.statusReason } : {}), + ...(statusReason ? { statusReason } : {}), ...(record?.updatedAt ? { updatedAt: record.updatedAt } : {}), ...(record?.updatedBy ? { updatedBy: record.updatedBy } : {}), ...(record?.lastVerifiedAt ? { lastVerifiedAt: record.lastVerifiedAt } : {}), @@ -2573,7 +2701,7 @@ export class AutomationServicePlugin implements Plugin { * any more (so it can be cleared), ordered by reference. */ private async listConnectorCredentialRefs(ctx: PluginContext): Promise { - const chain = this.connectorCredentialChain(); + const { chain } = await this.doorCredentialChain(ctx); const declared = await this.declaredCredentialRefs(ctx); const rows = await this.readCredentialRowsForDoor(chain); const refs = new Set([...declared.keys(), ...rows.keys()]); @@ -2603,12 +2731,23 @@ export class AutomationServicePlugin implements Plugin { * [#27237] `IAutomationService.setConnectorCredential` — store one value * for a declared reference, then reconcile, so every instance naming it * re-materializes without a restart. Answers the reference's entry after - * that reconcile. Refuses, before anything is written: an undeclared - * reference (404), one an environment variable supplies (409), an empty or - * oversized value (400); the store refuses without a crypto provider (503). + * that reconcile. Refuses, before anything is written: a walled tenancy + * posture (403, S1 on #27230), an empty or oversized value or one spelled + * as a secret handle (400), an undeclared reference (404), one an + * environment variable supplies (409); the store refuses without a crypto + * provider (503). + * + * The write and its reconcile run on the reconcile queue, so two writes + * in one process never interleave: the instance ends on the value the + * store ends on. Two replicas inserting the first row for one reference + * at once meet the unique index, and the second answers `409`. */ private async setConnectorCredential(ctx: PluginContext, request: ConnectorCredentialSetRequest): Promise { const ref = this.credentialRefOfRequest(request?.ref); + const door = await this.doorCredentialChain(ctx); + if (door.walled !== undefined) throw this.walledCredentialRefusal(door.walled); + const { chain } = door; + const store = chain.store; const value: unknown = request?.value; if (typeof value !== 'string' || value.length === 0) { throw credentialDoorRefusal(`Credential '${ref}' needs a value: a non-empty string.`, 'VALIDATION_ERROR', 400); @@ -2620,6 +2759,17 @@ export class AutomationServicePlugin implements Plugin { 400, ); } + // The data engine stores a `secret` value spelled as a handle as + // given, and dereferences it as a secret it already holds — which + // would point this connector at another row's secret. + if (value.startsWith(SECRET_HANDLE_PREFIX)) { + throw credentialDoorRefusal( + `Credential '${ref}' cannot start with '${SECRET_HANDLE_PREFIX}': the credential store reads a value spelled that way ` + + 'as a handle to a secret it already holds, not as a credential. Nothing was written.', + 'VALIDATION_ERROR', + 400, + ); + } const declared = await this.declaredCredentialRefs(ctx); const declaredBy = declared.get(ref); if (!declaredBy) { @@ -2631,7 +2781,6 @@ export class AutomationServicePlugin implements Plugin { 404, ); } - const chain = this.connectorCredentialChain(); if (isSuppliedByEnv(chain, ref)) { throw credentialDoorRefusal( `Credential '${ref}' is supplied by the environment variable of the same name, which wins over a stored ` + @@ -2642,18 +2791,20 @@ export class AutomationServicePlugin implements Plugin { ); } const userId = request.context?.userId; - const { revision } = await chain.store.set({ - ref, - value, - userId, - connectorNames: declaredBy.map((c) => c.name), - packageIds: [...new Set(declaredBy.map((c) => c.packageId).filter((p): p is string => p !== undefined))], + await this.onReconcileQueue(async () => { + const { revision } = await store.set({ + ref, + value, + userId, + connectorNames: declaredBy.map((c) => c.name), + packageIds: [...new Set(declaredBy.map((c) => c.packageId).filter((p): p is string => p !== undefined))], + }); + ctx.logger.info( + `[Automation] connector credential '${ref}' set (revision ${revision})${userId ? ` by user '${userId}'` : ''}; ` + + `reconciling ${declaredBy.map((c) => `'${c.name}'`).join(', ')}.`, + ); + await this.reconcileDeclaredConnectors(ctx, { fatal: false }); }); - ctx.logger.info( - `[Automation] connector credential '${ref}' set (revision ${revision})${userId ? ` by user '${userId}'` : ''}; ` + - `reconciling ${declaredBy.map((c) => `'${c.name}'`).join(', ')}.`, - ); - await this.materializeDeclaredConnectors(ctx, { fatal: false }); return this.describeCredentialRef(chain, ref, declaredBy, await this.readCredentialRowsForDoor(chain)); } @@ -2661,25 +2812,31 @@ export class AutomationServicePlugin implements Plugin { * [#27237] `IAutomationService.clearConnectorCredential` — remove the * stored value for one reference, then reconcile: every instance naming it * is torn down and registered degraded, naming the reference. `404` when - * nothing is stored for it. + * nothing is stored for it; `403` under a walled tenancy posture. Runs on + * the reconcile queue, as a write does. */ private async clearConnectorCredential(ctx: PluginContext, request: ConnectorCredentialClearRequest): Promise { const ref = this.credentialRefOfRequest(request?.ref); - const chain = this.connectorCredentialChain(); + const door = await this.doorCredentialChain(ctx); + if (door.walled !== undefined) throw this.walledCredentialRefusal(door.walled); + const { chain } = door; + const store = chain.store; const userId = request?.context?.userId; - let cleared: boolean; - try { - cleared = await chain.store.clear(ref, userId); - } catch { - throw new ConnectorCredentialUnavailableError( - `Credential '${ref}' could not be cleared: the connector credential store could not be read or written. Nothing was changed.`, - ); - } - if (!cleared) { - throw credentialDoorRefusal(`Nothing is stored for credential '${ref}'.`, 'RESOURCE_NOT_FOUND', 404); - } - ctx.logger.info(`[Automation] connector credential '${ref}' cleared${userId ? ` by user '${userId}'` : ''}.`); - await this.materializeDeclaredConnectors(ctx, { fatal: false }); + await this.onReconcileQueue(async () => { + let cleared: boolean; + try { + cleared = await store.clear(ref, userId); + } catch { + throw new ConnectorCredentialUnavailableError( + `Credential '${ref}' could not be cleared: the connector credential store could not be read or written. Nothing was changed.`, + ); + } + if (!cleared) { + throw credentialDoorRefusal(`Nothing is stored for credential '${ref}'.`, 'RESOURCE_NOT_FOUND', 404); + } + ctx.logger.info(`[Automation] connector credential '${ref}' cleared${userId ? ` by user '${userId}'` : ''}.`); + await this.reconcileDeclaredConnectors(ctx, { fatal: false }); + }); const declared = await this.declaredCredentialRefs(ctx); return this.describeCredentialRef(chain, ref, declared.get(ref), await this.readCredentialRowsForDoor(chain)); } @@ -2687,11 +2844,20 @@ export class AutomationServicePlugin implements Plugin { /** * [#27237] Degrade one instance whose upstream refused its credential — * at materialization (`ConnectorCredentialRejectedError`, the mcp - * provider) or on a call (a 401 / 403 result, rest and openapi) — and - * mark a stored credential `rejected` with the upstream's message, which - * the caller has already scrubbed of the value. Held, not retried: the - * signature carries the credential's revision, so setting it again is - * what re-attempts the instance. + * provider) or on a call (a 401 / 403 result, rest and openapi). The + * reason quotes the upstream, already scrubbed of the value by the caller. + * + * A value stored through the door is marked `rejected` on its row and + * the instance is held: no timer retry, since its signature carries the + * row's revision and the same value would be refused again; setting it + * again, a change of the declaration, or a restart re-attempts it. So is + * an instance with no `credentialRef` (its credential is written into the + * declaration, e.g. an MCP transport header): only a change of the + * declaration changes what it sends. A value the door does not hold — an + * environment variable, the host's resolver — can change where this + * process cannot see it (a rotation, a rate limit that lifts), so its + * instance retries on the #3017 backoff, and the listing reports the + * refusal from {@link credentialRefusals}. * * A refusal at materialization leaves a live connector serving, as every * failed re-materialization does; a refusal on a call takes the connector @@ -2705,6 +2871,7 @@ export class AutomationServicePlugin implements Plugin { entry: DeclaredConnectorItem; provider: string; signature: string; + ref?: string; live?: { close?: () => void | Promise }; takeDown: boolean; credential?: Extract; @@ -2717,21 +2884,35 @@ export class AutomationServicePlugin implements Plugin { await this.dematerializeConnector(engine, info.name, info.live, ctx, 'its upstream refused its credential'); hasLive = false; } - this.degradedInstances.set(info.name, { attempts: 1, signature: info.signature, reason: info.reason, held: true }); + const stored = info.credential?.source === 'object' ? info.credential.record : undefined; + const held = info.credential === undefined || stored !== undefined; + let attempts = 1; + if (held) { + this.degradedInstances.set(info.name, { attempts, signature: info.signature, reason: info.reason, held: true }); + } else { + const prior = this.credentialRefusalAttempts.get(info.name); + attempts = prior && prior.signature === info.signature ? prior.attempts + 1 : 1; + this.credentialRefusalAttempts.set(info.name, { signature: info.signature, attempts }); + this.degradedInstances.set(info.name, { attempts, signature: info.signature, reason: info.reason }); + const source = info.credential?.source; + if (info.ref !== undefined && (source === 'env' || source === 'vault')) { + this.credentialRefusals.set(info.ref, { source, reason: info.reason }); + } + } if (!hasLive) { try { engine.registerDegradedConnector(this.buildDegradedHuskDef(info.name, info.entry), info.reason, 'declarative', info.cause); } catch (err) { ctx.logger.warn( `[Automation] could not register degraded husk for '${info.name}' — the instance stays absent from the ` + - `connector registry until its credential changes.`, + `connector registry until ${held ? 'its credential changes' : 'a retry succeeds'}.`, describeThrownForLog(err), ); } } - if (info.credential?.source === 'object' && info.credential.record) { + if (stored) { try { - await this.connectorCredentialChain().store.markRejected(info.credential.record, info.reason); + await this.connectorCredentialStoreOrNone().markRejected(stored, info.reason); } catch (markErr) { ctx.logger.warn( `[Automation] connector instance '${info.name}': its credential row could not be marked rejected.`, @@ -2744,19 +2925,54 @@ export class AutomationServicePlugin implements Plugin { `[Automation] connector instance '${info.name}' (provider '${info.provider}') is degraded: its upstream refused ` + `its credential. ` + (hasLive - ? 'The previously-materialized connector keeps serving.' - : 'Not retried until its credential or its declaration changes.'), + ? 'The previously-materialized connector keeps serving' + (held ? '.' : `; retrying with backoff, attempt ${attempts}.`) + : held + ? 'Not retried until its credential is set again, its declaration changes, or the server restarts.' + : `Retrying with backoff, attempt ${attempts}.`), { reason: info.reason }, ); } + /** + * [#27237] An upstream accepted an instance's credential — the provider + * proved it while materializing, or a call was answered `2xx`. A stored + * value's row records when, and a `rejected` row of that same revision + * returns to `set`; a refusal of a value the door does not hold is + * forgotten, and so is the instance's backoff count. + */ + private async recordCredentialAccepted( + ctx: PluginContext, + name: string, + ref: string, + credential: Extract, + ): Promise { + this.credentialRefusalAttempts.delete(name); + if (credential.source !== 'object') { + if (this.credentialRefusals.get(ref)?.source === credential.source) this.credentialRefusals.delete(ref); + return; + } + if (!credential.record) return; + try { + await this.connectorCredentialStoreOrNone().markVerified(credential.record); + } catch (markErr) { + ctx.logger.warn( + `[Automation] connector instance '${name}': an upstream accepted its credential, but the credential row could not record it.`, + describeThrownForLog(markErr), + ); + } + } + /** * [#27237] Wrap a credentialed instance's handlers so the first call its - * upstream answers 401 / 403 degrades the instance and marks its stored - * credential `rejected` (ruling on #27237, comment 6470087293). The call's - * own result is returned unchanged. Only the materialization these - * handlers belong to is taken down, once, through the reconcile queue, so - * a refusal never races a reconcile that already replaced it. + * upstream answers 401 / 403 degrades the instance (ruling on #27237, + * comment 6470087293), and — unless the provider already `proven` the + * credential at materialization — the first call it answers `2xx` records + * the credential as accepted. A call's own result is returned unchanged. + * Both run on the reconcile queue, and only against the materialization + * these handlers belong to, so neither races a reconcile or a credential + * write that already replaced it. A refusal is awaited (the caller sees + * the instance degraded once its call returns); an acceptance is not, + * so a call never waits on a reconcile. */ private watchForCredentialRejection( engine: AutomationEngine, @@ -2769,14 +2985,24 @@ export class AutomationServicePlugin implements Plugin { ref: string; credential: Extract; tracked: { signature: string; provider: string; close?: () => void | Promise }; + proven: boolean; }, handlers: Record, ): Record { let refused = false; + let accepted = info.proven; + const current = () => !this.destroyed && this.materializedConnectors.get(info.name) === info.tracked; const watched: Record = {}; for (const [key, handler] of Object.entries(handlers)) { watched[key] = async (input, callCtx) => { const result = await handler(input, callCtx); + if (!accepted && !refused && isAcceptedCallResult(result)) { + accepted = true; + void this.onReconcileQueue(async () => { + if (current()) await this.recordCredentialAccepted(ctx, info.name, info.ref, info.credential); + }); + return result; + } const status = credentialRejectionStatusOf(result); if (status === undefined || refused) return result; refused = true; @@ -2785,23 +3011,25 @@ export class AutomationServicePlugin implements Plugin { info.ref, status, upstreamRefusalMessage((result as { body?: unknown }).body, info.credential.value), + info.credential.source === 'object', ); - const run = this.reconcileQueue.then(async () => { - if (this.destroyed || this.materializedConnectors.get(info.name) !== info.tracked) return; - await this.holdConnectorForRejectedCredential(engine, ctx, { - name: info.name, - entry: info.entry, - provider: info.provider, - signature: info.signature, - live: info.tracked, - takeDown: true, - credential: info.credential, - reason, - }); - }); - this.reconcileQueue = run.catch(() => undefined); try { - await run; + await this.onReconcileQueue(async () => { + if (!current()) return; + await this.holdConnectorForRejectedCredential(engine, ctx, { + name: info.name, + entry: info.entry, + provider: info.provider, + signature: info.signature, + ref: info.ref, + live: info.tracked, + takeDown: true, + credential: info.credential, + reason, + }); + // Retry on the backoff when the refused value is not a stored one. + this.scheduleDeclarativeRetry(ctx); + }); } catch (err) { ctx.logger.warn( `[Automation] connector instance '${info.name}' was refused its credential on a call, and could not be degraded.`, diff --git a/packages/services/service-automation/src/sys-connector-credential.object.ts b/packages/services/service-automation/src/sys-connector-credential.object.ts index 0151706bada..e417b82f7ff 100644 --- a/packages/services/service-automation/src/sys-connector-credential.object.ts +++ b/packages/services/service-automation/src/sys-connector-credential.object.ts @@ -27,13 +27,15 @@ import { ObjectSchema, Field } from '@objectstack/spec/data'; * Environment-level: a connector instance is environment-level (ADR-0097), so * its credential is too, and the store carries no organization column. Per * organization credentials on a shared environment are ruled not supported - * (S1 on #27230); the routes that write this object refuse on a walled posture. + * (S1 on #27230): on a walled posture the routes that write this object refuse, + * and the resolver chain does not read it. * - * Writers: the automation plugin's credential door (set, clear) and its - * reconcile (`status`, `last_verified_at`), under a system context that - * carries the caller's user id, so the audit columns record who changed which - * reference and when. Readers: the same plugin — never the generic data door, - * which is closed below. + * Writers: the automation plugin's credential door (set, clear), which stamps + * `set_at` / `set_by` and runs under a system context carrying the caller's + * user id, and its reconcile (`status`, `status_reason`, `last_verified_at`), + * which leaves `set_at` / `set_by` alone so the listing keeps saying when the + * value was set and by whom. Readers: the same plugin — never the generic data + * door, which is closed below. * * @namespace sys */ @@ -55,7 +57,7 @@ export const SysConnectorCredential = ObjectSchema.create({ "Write-only store of declarative connector credentials, one encrypted row per `credentialRef` an administrator set. Read second in the resolver chain: an environment variable of the same name wins.", displayNameField: 'ref', nameField: 'ref', // [ADR-0079] canonical primary-title pointer (mirrors deprecated displayNameField) - highlightFields: ['ref', 'status', 'connector_name', 'updated_at'], + highlightFields: ['ref', 'status', 'connector_name', 'set_at'], fields: { ref: Field.text({ @@ -102,7 +104,7 @@ export const SysConnectorCredential = ObjectSchema.create({ required: true, defaultValue: 'set', description: - "`set` when a value is stored; `rejected` when an upstream refused it while an instance materialized. A reference no source answers for is `missing` in the listing and has no row.", + "`set` when a value is stored; `rejected` when an upstream refused it (an instance's materialization or call answered 401 / 403). Back to `set` only when the value is set again or an upstream accepts it. A reference no source answers for is `missing` in the listing and has no row.", }), status_reason: Field.textarea({ @@ -112,10 +114,23 @@ export const SysConnectorCredential = ObjectSchema.create({ }), last_verified_at: Field.datetime({ - label: 'Last Materialized At', + label: 'Last Verified At', readonly: true, description: - 'When a connector instance last materialized with this value. A provider that contacts its upstream at materialization (mcp) proves the credential there; the others first use it on a dispatch.', + 'When an upstream last accepted this value: a provider that proves the credential at materialization (mcp), or the first call answered 2xx after an instance materialized (rest, openapi). Cleared when the value is set again.', + }), + + set_at: Field.datetime({ + label: 'Set At', + readonly: true, + description: 'When the value was last set through the credential door. A status change does not move it.', + }), + + set_by: Field.text({ + label: 'Set By', + readonly: true, + maxLength: 255, + description: 'The user id that last set the value through the credential door.', }), }, diff --git a/packages/spec/src/integration/connector-provider.ts b/packages/spec/src/integration/connector-provider.ts index d36e0852921..683703c43ff 100644 --- a/packages/spec/src/integration/connector-provider.ts +++ b/packages/spec/src/integration/connector-provider.ts @@ -43,6 +43,16 @@ export interface ConnectorMaterialization { def: Connector; handlers: Record; close?: () => void | Promise; + /** + * `true` when the upstream accepted the instance's resolved `auth` while + * materializing — the credential travelled with a request the upstream + * answered (an MCP handshake over http). The automation service then records + * the credential as verified, and a stored one marked `rejected` returns to + * `set`. Leave it unset when materializing sends no credential (a provider + * that first presents it on a call): the service waits for a call the + * upstream accepts instead. + */ + credentialVerified?: boolean; } /** From 5671ee2dff2421322665f4a0bd75129075aeabf1 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 01:09:34 +0000 Subject: [PATCH 13/21] =?UTF-8?q?WIP(#27237):=20seat=20review=20r1=20?= =?UTF-8?q?=E2=80=94=20tests,=20ablations,=20docs,=20ADR=20note,=20changes?= =?UTF-8?q?ets,=20DELETE=20query=20pin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- .../27237-connector-credential-refs-routes.md | 2 +- ...37-connector-credential-rejected-marker.md | 3 +- .../27237-connector-credential-store.md | 4 +- .changeset/27237-mcp-credential-rejected.md | 4 +- content/docs/automation/connectors.mdx | 46 ++-- .../0097-declarative-connector-instances.md | 29 ++- ...tomation-connector-credential-door.test.ts | 10 +- packages/runtime/src/domains/automation.ts | 11 +- packages/runtime/src/route-ledger.ts | 2 +- .../src/connector-credential-door.test.ts | 246 +++++++++++++++++- .../spec/src/contracts/automation-service.ts | 24 +- 11 files changed, 329 insertions(+), 52 deletions(-) diff --git a/.changeset/27237-connector-credential-refs-routes.md b/.changeset/27237-connector-credential-refs-routes.md index 1b6f2907d7d..9ec302044fd 100644 --- a/.changeset/27237-connector-credential-refs-routes.md +++ b/.changeset/27237-connector-credential-refs-routes.md @@ -9,4 +9,4 @@ Metadata-protocol: none Three new routes serve the automation service's connector credential store (#27237). `GET …/credential-refs` lists every `credentialRef` an installed connector names, plus stored references nothing names any more. Each entry has its `status` (`set` / `missing` / `rejected`), its `source` (`env` / `object` / `vault` / null) and the instances naming it, and never a value. `PUT …/credential-refs/:ref` with the closed body `{ "value": "…" }` stores a value, and the instances naming the reference re-materialize without a restart. `DELETE …/credential-refs/:ref` removes it. -All three require the `manage_metadata` capability. Under the `group` and `isolated` tenancy postures they refuse every caller with a 403 that names the posture, because connector credentials are environment-level. They take no query parameters. The service's refusals keep their standard codes: 400 `VALIDATION_ERROR`, 404 `RESOURCE_NOT_FOUND` for an undeclared reference, 409 `RESOURCE_CONFLICT` for one an environment variable supplies, and 503 `SERVICE_UNAVAILABLE` with no crypto provider. A mounted service without these optional members answers 501. +All three require the `manage_metadata` capability. Under the `group` and `isolated` tenancy postures they refuse every caller with a 403 that names the posture, because connector credentials are environment-level. They take no query parameters. The service's refusals keep their standard codes: 400 `VALIDATION_ERROR` (an empty value, or one starting with `secret:`), 403 `PERMISSION_DENIED` (the service's own walled refusal), 404 `RESOURCE_NOT_FOUND` for an undeclared reference, 409 `RESOURCE_CONFLICT` for one an environment variable supplies or a first write another replica made at the same moment, and 503 `SERVICE_UNAVAILABLE` with no crypto provider. A mounted service without these optional members answers 501. diff --git a/.changeset/27237-connector-credential-rejected-marker.md b/.changeset/27237-connector-credential-rejected-marker.md index b35255e7212..90c68a293bd 100644 --- a/.changeset/27237-connector-credential-rejected-marker.md +++ b/.changeset/27237-connector-credential-rejected-marker.md @@ -7,9 +7,10 @@ feat(spec): `ConnectorCredentialRejectedError` and the connector credential memb Clause-②: yes (widening) Metadata-protocol: none -Two additions for the connector credential store (#27237): +Three additions for the connector credential store (#27237): - `@objectstack/spec/integration` exports `CONNECTOR_CREDENTIAL_REJECTED`, `ConnectorCredentialRejectedError` (with `status`: 401 or 403) and the structural check `isConnectorCredentialRejected`. They sit beside the upstream-unavailable marker. A connector provider factory throws it when the upstream refuses the credential it sent, and the materializer then degrades the instance instead of failing the boot. The code is registered in the error-code ledger under `@objectstack/spec`. +- `ConnectorMaterialization` gains the optional `credentialVerified`. A provider factory sets it when the upstream accepted the instance's credential while materializing (the MCP handshake over http); the automation service then records the credential as verified, and a stored one marked `rejected` returns to `set`. A provider that first sends the credential on a call leaves it unset. - `IAutomationService` gains three optional members, `listConnectorCredentialRefs`, `setConnectorCredential` and `clearConnectorCredential`, with their types `ConnectorCredentialRef`, `ConnectorCredentialSetRequest` and `ConnectorCredentialClearRequest`. A listing entry never carries a value. `PLATFORM_OBJECT_NAMES['service-automation']` lists the new `sys_connector_credential` object. diff --git a/.changeset/27237-connector-credential-store.md b/.changeset/27237-connector-credential-store.md index b24cd8dc431..d3e7bfdfe4b 100644 --- a/.changeset/27237-connector-credential-store.md +++ b/.changeset/27237-connector-credential-store.md @@ -13,7 +13,9 @@ The reference now resolves through three sources, in order: an environment varia - A reference no source answers for degrades the instance naming it, with a reason that names the reference and how to set it. Before, it failed the boot. A `credentialResolver` that throws is still fatal at boot. - Setting or clearing a credential reconciles at once: every instance naming the reference re-materializes, or is torn down and degraded, with no restart. -- A credential the upstream refuses (HTTP 401 / 403) degrades the instance and marks a stored credential `rejected` with the upstream's message, never the value. An `mcp` instance is refused when it connects; a `rest` or `openapi` instance on the first call that answers 401 / 403, which still returns its answer. A refused instance is not retried on a timer until its credential or declaration changes. +- A credential the upstream refuses (HTTP 401 / 403) degrades the instance with the upstream's message, never the value. An `mcp` instance is refused when it connects; a `rest` or `openapi` instance on the first call that answers 401 / 403, which still returns its answer. A stored credential is marked `rejected`, and its instance is not retried until the credential is set again, the declaration changes, or the server restarts; the mark itself stays until an upstream accepts the value (an `mcp` handshake, or a call answered 2xx). A credential from an environment variable or the host's resolver is retried on the usual backoff instead, since it can change where the service cannot see it, and the listing reports it `rejected` until it is accepted. +- Under the `group` and `isolated` tenancy postures the store is not read, and writes to it are refused with a 403 naming the posture. +- A value starting with `secret:` is refused with a 400: the data engine would store it as a handle to another stored secret. Writes run one at a time with their reconcile, so the instances end on the value the store ends on; a first write another replica makes at the same moment answers 409. - The value never appears in a log record, the listing, `GET /connectors`, an error body or a read of the object. A provider message that echoes it carries `[credential withheld]` instead. A deployment that relied on the boot failing for an unset reference now boots, with that instance degraded and named on `GET /api/v1/automation/connectors`. diff --git a/.changeset/27237-mcp-credential-rejected.md b/.changeset/27237-mcp-credential-rejected.md index 6c435130e12..f24724f8f49 100644 --- a/.changeset/27237-mcp-credential-rejected.md +++ b/.changeset/27237-mcp-credential-rejected.md @@ -7,4 +7,6 @@ fix(connector-mcp): a credential the MCP server rejects is thrown as `ConnectorC Clause-②: yes (widening) Metadata-protocol: none -An HTTP 401 / 403 on connect, handshake or `tools/list` used to throw a plain `Error`, which failed the boot. The provider now throws `ConnectorCredentialRejectedError` from `@objectstack/spec/integration`, with the same message, the status, and the SDK error as `cause`. The automation service degrades the instance naming the refusal and marks a stored credential `rejected`, and it does not retry until the credential changes (#27237). Every other connect or `tools/list` failure is unchanged. +An HTTP 401 / 403 on connect, handshake or `tools/list` used to throw a plain `Error`, which failed the boot. The provider now throws `ConnectorCredentialRejectedError` from `@objectstack/spec/integration`, with the status and the SDK error as `cause`; its message no longer says the fault "is not retried", since whether it is now depends on where the credential came from. The automation service degrades the instance naming the refusal and marks a stored credential `rejected` (#27237). Every other connect or `tools/list` failure is unchanged. + +An http instance whose handshake carried its credential (any auth but `none`, and not an api-key sent as a query parameter) now answers `credentialVerified: true`, so a stored credential the server accepted is recorded as verified. diff --git a/content/docs/automation/connectors.mdx b/content/docs/automation/connectors.mdx index b835357f36b..e7342bc4b1f 100644 --- a/content/docs/automation/connectors.mdx +++ b/content/docs/automation/connectors.mdx @@ -190,8 +190,9 @@ paginated tool list is followed through every page (`nextCursor`), up to 100 pages; a server still paging after that fails discovery instead of yielding a truncated tool list. A server that is unreachable or failing degrades the instance, which is retried. A credential the server rejects (HTTP 401 / 403 on -connect, handshake or `tools/list`) is a configuration fault instead: it fails -the boot and is not retried — fix the credential and reload. +connect, handshake or `tools/list`) is a configuration fault instead: the +instance is degraded naming the refusal, as described under *A credential the +upstream refuses* below. **Declarative stdio transports are denied by default.** Materializing one @@ -277,14 +278,19 @@ DELETE /api/v1/automation/connectors/credential-refs/BILLING_API_TOKEN stored one that nothing names any more. Each entry has a `status` (`set`, `missing` or `rejected`), a `source` (`env`, `object`, `vault` or `null`), whether this door can write it, and the instances naming it with their state. - It never includes a value. + A stored one also says when and by whom it was last set (`updatedAt`, + `updatedBy`) and when an upstream last accepted it (`lastVerifiedAt`). It + never includes a value. - **`PUT`** stores the value, then reconciles. Every instance naming the reference re-materializes with it **without a restart**, and the answer is the reference's entry after that. The route refuses, before writing anything: a reference no installed connector declares (`404`), one an environment variable already supplies (`409`, because the variable wins and a stored value - would never be read), and an empty value (`400`). With no crypto provider it - answers `503`, since a credential is only ever stored encrypted. + would never be read), an empty value, and a value starting with `secret:` + (`400`: the store would read it as a handle to another stored secret). With no + crypto provider it answers `503`, since a credential is only ever stored + encrypted. Writes on one server run one at a time, each with its reconcile, so + the instances always end on the value the store ends on. - **`DELETE`** removes the stored value. The instances naming the reference are torn down and registered degraded again. @@ -294,18 +300,28 @@ lists the `credentialRefs` the package's connectors name, each with its status, plus a `credentialNote` naming the ones still to set. **A credential the upstream refuses** (HTTP `401` / `403`) degrades the -instance, and a stored credential is marked `rejected` with the upstream's -message, never the value. An `mcp` instance finds this out when it connects. A -`rest` or `openapi` instance finds out on the first call answering `401` / -`403`; that call returns its answer as usual, and the instance is degraded -after it. A refused instance is not retried on a timer. Setting the credential -again is what re-attempts it. +instance, with a reason quoting the upstream's message, never the value. An +`mcp` instance finds this out when it connects. A `rest` or `openapi` instance +finds out on the first call answering `401` / `403`; that call returns its +answer as usual, and the instance is degraded after it. What happens next +depends on where the value came from: + +- **A value stored here** is marked `rejected`, and its instance is not + retried until the credential is set again, its declaration changes, or the + server restarts. The mark outlives a restart: it clears only when an upstream + accepts the value, either an `mcp` handshake that carries it or a `rest` / + `openapi` call answered `2xx`. +- **A value from an environment variable or the host's resolver** can change + where this door cannot see it (a rotation, or a `403` rate limit that lifts), + so its instance is retried on the same 5s→5min backoff as an unreachable + upstream. The listing reports it `rejected` until an upstream accepts it. Credentials are **environment-level**: one value per reference for the whole environment, read by whichever instance names it. On a multi-organization -environment (the `group` or `isolated` tenancy posture) the three routes refuse -with a message naming the posture. There, a reference resolves from the -environment variable or the host's resolver only. +environment (the `group` or `isolated` tenancy posture) the store is not used +at all. The three routes refuse with a message naming the posture, and a +reference resolves from the environment variable or the host's resolver only, +so a row stored before the posture changed is not read. The same resolver serves an `api` flow's start-node `config.secretRef`, the name of its inbound hook secret. That one goes through the injected resolver @@ -385,7 +401,7 @@ retry): | `provider` names no installed factory | **Hard boot error** naming the entry, the provider, and the plugin that supplies it — add the executor to `plugins:` | | Invalid `providerConfig` (missing `baseUrl` / `spec` / `transport`, wrong shape) | **Hard boot error** from the provider factory | | `credentialRef` resolves to nothing | **Not fatal**: the instance registers **degraded**, naming the reference — set it (`PUT …/credential-refs/:ref`), and it materializes without a restart | -| The upstream refuses the credential (`401` / `403`) | **Not fatal**: the instance registers **degraded** and a stored credential is marked `rejected` — not retried until the credential changes | +| The upstream refuses the credential (`401` / `403`) | **Not fatal**: the instance registers **degraded**. A stored credential is marked `rejected` and waits until it is set again (or a restart); an environment or vault credential is retried on the backoff | | Instance `name` collides with a plugin-registered connector | **Hard boot error** — there is no silent precedence between the two worlds | | stdio transport without host opt-in | **Hard boot error** with the `declarativeStdio` fix inline — a security rejection is never retried | | MCP server unreachable / `tools/list` fails; remote spec URL unreachable or transiently failing (408/429/5xx) | **Not fatal**: the instance registers **degraded** (visible on `GET /api/v1/automation/connectors` with the reason), dispatches fail with that reason, and the platform retries on a 5s→5min backoff — recovery is automatic | diff --git a/docs/adr/0097-declarative-connector-instances.md b/docs/adr/0097-declarative-connector-instances.md index 6203bafb0e0..50c9f4be377 100644 --- a/docs/adr/0097-declarative-connector-instances.md +++ b/docs/adr/0097-declarative-connector-instances.md @@ -5,7 +5,7 @@ **Builds on**: [ADR-0015](./0015-external-datasource-federation.md) (open mechanism / enterprise lifecycle split), [ADR-0018](./0018-unified-node-action-registry.md) (`connector_action` baseline dispatch + `engine.registerConnector()`), [ADR-0022](./0022-connectors-vs-messaging-channels.md) (Connector = transport/integration mechanism), [ADR-0023](./0023-openapi-to-connector-generator.md) (OpenAPI → Connector generator), [ADR-0024](./0024-mcp-connectors.md) (MCP servers as connectors) **Tracking**: framework#2977 (supersedes the interim descriptor-only contract from framework#2612) **Consumers**: `@objectstack/spec` (`stack.zod.ts`, `integration/connector.zod.ts`), `@objectstack/service-automation` (boot audit → provider binding), `@objectstack/connector-openapi`, `@objectstack/connector-mcp`, `@objectstack/connector-rest`, the Studio flow palette, AI authoring (stack generation) -**Amended by**: [ADR-0143](./0143-outbound-connector-auth-is-open-tier.md) (2026-10-27) — OAuth2, the vendor app-credential exchange and request signing join `ConnectorInstanceAuthSchema` in the open tier, and those kinds resolve per dispatch; see the notes under §3 and under "Open / enterprise line". PR #26798 (2026-10-27, #26709 item 3(b)) — a credential the MCP server rejects (HTTP `401` / `403`) is a fatal configuration fault, not upstream-unavailable; see the note under "Upstream availability". #27237 (2026-10-28, ruling A on #27230 and the ruling on #27237) — an unresolved `credentialRef` and a rejected credential degrade the instance instead of failing the boot, and the open tier reads a `sys_connector_credential` store between the environment and the host's resolver; see the notes under §6 and under "Upstream availability". +**Amended by**: [ADR-0143](./0143-outbound-connector-auth-is-open-tier.md) (2026-10-27) — OAuth2, the vendor app-credential exchange and request signing join `ConnectorInstanceAuthSchema` in the open tier, and those kinds resolve per dispatch; see the notes under §3 and under "Open / enterprise line". PR #26798 (2026-10-27, #26709 item 3(b)) — a credential the MCP server rejects (HTTP `401` / `403`) is a fatal configuration fault, not upstream-unavailable; see the note under "Upstream availability". #27237 (2026-10-28, ruling A on #27230 and the ruling on #27237) — an unresolved `credentialRef` and a rejected credential degrade the instance instead of failing the boot (a refused stored credential holds its instance until it is set again or a restart; an environment or vault one is retried on the backoff), and the open tier reads a `sys_connector_credential` store between the environment and the host's resolver, except under a walled tenancy posture; see the notes under §6 and under "Upstream availability". --- @@ -102,8 +102,10 @@ If a provider-bound instance and a plugin-registered connector share a `name`, b > without a restart, once the credential is set. The open-tier sources are, in order: an > environment variable of the same name, the `sys_connector_credential` store (a `Field.secret` > row written through `PUT /api/v1/automation/connectors/credential-refs/:ref`), then the host's -> injected resolver. Environment-level only (S1 on #27230). A host resolver that throws stays -> fatal at boot. +> injected resolver. Environment-level only (S1 on #27230): under a walled tenancy posture +> (`group` / `isolated`) the store is not read at all and its door refuses, naming the posture, +> so a reference resolves from the environment or the host's resolver only. A host resolver that +> throws stays fatal at boot. --- @@ -220,12 +222,21 @@ exactly the plausible-but-dead shape this ADR exists to kill. > `packages/connectors/connector-mcp/src/mcp-http-discovery-faults.test.ts`. > **Amendment note (2026-10-28, #27237 — ruling on #27237, comment 6470087293).** The note above -> changes on one point: a rejected credential no longer fails the boot. The `mcp` provider throws -> `ConnectorCredentialRejectedError` (`CONNECTOR_CREDENTIAL_REJECTED`) at materialization; a -> `rest` or `openapi` instance reports it on the first call answering `401` / `403`. Either way -> the instance is degraded with the upstream's message, never the value, and a stored credential -> is marked `rejected`. It is still never retried on a timer: the next attempt is the next change -> of the credential or of the declaration. +> changes on two points. First, a rejected credential no longer fails the boot. The `mcp` +> provider throws `ConnectorCredentialRejectedError` (`CONNECTOR_CREDENTIAL_REJECTED`) at +> materialization; a `rest` or `openapi` instance reports it on the first call answering `401` / +> `403`. Either way the instance is degraded with the upstream's message, never the value. +> Second, whether it is retried follows where the value came from. A value stored in +> `sys_connector_credential` is marked `rejected`, and its instance is held, with no timer retry, +> until the credential is set again, the declaration changes, or the process restarts (a restart +> lifts the hold). An instance with no `credentialRef`, whose credential is written in the +> declaration, is held the same way. The `rejected` mark survives a re-materialization, since a +> `rest` or `openapi` instance materializes without sending the value. Only an upstream accepting +> the value clears it: a provider answering `credentialVerified` at materialization (the `mcp` +> handshake over http), or a call answered `2xx`. A value from an environment variable or the +> host's resolver can change where the process cannot see it (a rotation, or a `403` rate limit +> that lifts), so its instance is retried on the backoff above instead, and the credential +> listing reports the refusal until an upstream accepts it. The `openapi` provider applies the same split to its **remote-URL spec fetch**: an unreachable endpoint (network error) or a transient HTTP status diff --git a/packages/runtime/src/domains/automation-connector-credential-door.test.ts b/packages/runtime/src/domains/automation-connector-credential-door.test.ts index 7e68f428504..aed14537982 100644 --- a/packages/runtime/src/domains/automation-connector-credential-door.test.ts +++ b/packages/runtime/src/domains/automation-connector-credential-door.test.ts @@ -97,8 +97,8 @@ const get = (h: Harness, ctx: HttpProtocolContext, query?: unknown) => h.dispatcher.handleAutomation('/connectors/credential-refs', 'GET', undefined, ctx, query); const put = (h: Harness, ctx: HttpProtocolContext, body: unknown, ref = REF, query?: unknown) => h.dispatcher.handleAutomation(`/connectors/credential-refs/${encodeURIComponent(ref)}`, 'PUT', body, ctx, query); -const del = (h: Harness, ctx: HttpProtocolContext, ref = REF) => - h.dispatcher.handleAutomation(`/connectors/credential-refs/${encodeURIComponent(ref)}`, 'DELETE', undefined, ctx, undefined); +const del = (h: Harness, ctx: HttpProtocolContext, ref = REF, query?: unknown) => + h.dispatcher.handleAutomation(`/connectors/credential-refs/${encodeURIComponent(ref)}`, 'DELETE', undefined, ctx, query); describe('[#27237] connector credential door — who may reach it', () => { it('answers an anonymous caller 401 before anything else', async () => { @@ -164,6 +164,11 @@ describe('[#27237] connector credential door — what it accepts', () => { const { response: putResponse } = await put(h, ADMIN(), { value: DUMMY_VALUE }, REF, { force: 'true' }); expect(statusOf(putResponse)).toBe(400); expect(h.set).not.toHaveBeenCalled(); + const { response: delResponse } = await del(h, ADMIN(), REF, { cascade: 'true' }); + expect(statusOf(delResponse)).toBe(400); + expect(codeOf(delResponse)).toBe('VALIDATION_ERROR'); + expect(messageOf(delResponse)).toContain('cascade'); + expect(h.clear).not.toHaveBeenCalled(); }); it('PUT hands the decoded reference, the value and the caller to the service, and answers the entry without the value', async () => { @@ -198,6 +203,7 @@ describe('[#27237] connector credential door — what it accepts', () => { Object.assign(new Error(message), { code, status }); for (const [code, status] of [ ['VALIDATION_ERROR', 400], + ['PERMISSION_DENIED', 403], ['RESOURCE_NOT_FOUND', 404], ['RESOURCE_CONFLICT', 409], ['SERVICE_UNAVAILABLE', 503], diff --git a/packages/runtime/src/domains/automation.ts b/packages/runtime/src/domains/automation.ts index 55f3a242618..716abc899aa 100644 --- a/packages/runtime/src/domains/automation.ts +++ b/packages/runtime/src/domains/automation.ts @@ -798,15 +798,16 @@ const CONNECTOR_CREDENTIAL_UNSUPPORTED_MESSAGE = (member: string): string => + `implement \`${member}\`, an optional member of \`IAutomationService\`. Nothing was read or changed.`; /** - * [#27237] The service's own refusals — `400` / `404` / `409` / `503`, each - * carrying an ADR-0112 standard code — answered as they were raised. Anything - * else is not a refusal this door knows, and propagates to the transport's - * error exit. + * [#27237] The service's own refusals — `400` / `403` / `404` / `409` / `503`, + * each carrying an ADR-0112 standard code — answered as they were raised. The + * `403` is the service's own walled refusal, which the gate above normally + * answers first. Anything else is not a refusal this door knows, and + * propagates to the transport's error exit. */ function connectorCredentialRefusal(deps: DomainHandlerDeps, err: unknown): HttpDispatcherResult | undefined { const e = err as { status?: unknown; code?: unknown; message?: unknown } | null; if (!e || typeof e.code !== 'string' || typeof e.message !== 'string') return undefined; - if (e.status !== 400 && e.status !== 404 && e.status !== 409 && e.status !== 503) return undefined; + if (e.status !== 400 && e.status !== 403 && e.status !== 404 && e.status !== 409 && e.status !== 503) return undefined; return { handled: true, response: deps.error(e.message, e.status, { code: e.code }) }; } diff --git a/packages/runtime/src/route-ledger.ts b/packages/runtime/src/route-ledger.ts index 5cdc6fc6d08..87514bf6467 100644 --- a/packages/runtime/src/route-ledger.ts +++ b/packages/runtime/src/route-ledger.ts @@ -538,7 +538,7 @@ export const ROUTE_LEDGER: readonly RouteLedgerEntry[] = [ { route: 'GET /automation/connectors/credential-refs', domain: '/automation', disposition: 'server-only', note: "Every `credentialRef` a declared provider-bound connector instance names, plus every stored reference no installed package declares any more, each with its `status` (`set` / `missing` / `rejected`), its `source` (`env` / `object` / `vault` / null), whether this door may write it, and the instances naming it with their registry state. ⛔ Never a value: the store is `sys_connector_credential`, whose `value` is a `Field.secret`, and this listing reads only its bookkeeping columns. ⚑ `manage_metadata`, the gate the flow definition writes ask, fail-closed, only `isSystem` bypassing; then, under the `group` / `isolated` postures, refused 403 naming the posture for every caller — environment-level credentials only. Closed query set: none. NOT JS-SDK surface on this leg: the driver is the Console's connector credential page, a console surface that calls the platform API directly. Adding a client method reclassifies this row to `sdk`. Pinned in `domains/automation-connector-credential-door.test.ts`" }, { route: 'PUT /automation/connectors/credential-refs/:ref', domain: '/automation', disposition: 'server-only', - note: "Store the value for one declared reference, then reconcile: every instance naming it re-materializes without a restart. Body is the closed `{ value }`. Answers the reference's listing entry after the reconcile, never the value. Refused, before anything is written: an undeclared reference 404 `RESOURCE_NOT_FOUND`, one an environment variable of the same name supplies 409 `RESOURCE_CONFLICT` (env wins in the chain, so a stored value would never be read), an empty or oversized value or an unknown body key 400 `VALIDATION_ERROR`, and no crypto provider 503 `SERVICE_UNAVAILABLE`. Same gates and SDK posture as the GET row above. Pinned in `domains/automation-connector-credential-door.test.ts`" }, + note: "Store the value for one declared reference, then reconcile: every instance naming it re-materializes without a restart. Body is the closed `{ value }`. Answers the reference's listing entry after the reconcile, never the value. Refused, before anything is written: an undeclared reference 404 `RESOURCE_NOT_FOUND`, one an environment variable of the same name supplies 409 `RESOURCE_CONFLICT` (env wins in the chain, so a stored value would never be read), an empty or oversized value, one starting with `secret:` (the data engine would store it as a handle to another secret) or an unknown body key 400 `VALIDATION_ERROR`, and no crypto provider 503 `SERVICE_UNAVAILABLE`. Writes in one process run one at a time, with their reconcile; a concurrent first write from another replica answers 409 `RESOURCE_CONFLICT`. Same gates and SDK posture as the GET row above. Pinned in `domains/automation-connector-credential-door.test.ts`" }, { route: 'DELETE /automation/connectors/credential-refs/:ref', domain: '/automation', disposition: 'server-only', note: "Remove the stored value for one reference, then reconcile: every instance naming it is torn down and registered degraded, naming the reference. 404 `RESOURCE_NOT_FOUND` when nothing is stored for it. Same gates and SDK posture as the GET row above. Pinned in `domains/automation-connector-credential-door.test.ts`" }, { route: 'GET /automation/_status', domain: '/automation', disposition: 'sdk', client: 'automation.getRuntimeStatus' }, diff --git a/packages/services/service-automation/src/connector-credential-door.test.ts b/packages/services/service-automation/src/connector-credential-door.test.ts index 2bf19abaec7..b7033a1f60d 100644 --- a/packages/services/service-automation/src/connector-credential-door.test.ts +++ b/packages/services/service-automation/src/connector-credential-door.test.ts @@ -17,7 +17,13 @@ * `source: env`; * - a wrong value → the instance degraded and the row `rejected` (ruling on * #27237, comment 6470087293): at materialization (the mcp shape) and on - * the first call answering 401 / 403 (the rest / openapi shape). + * the first call answering 401 / 403 (the rest / openapi shape); + * - the seat review's round (#27344): only a stored value holds its instance + * (an environment or vault one retries on the backoff, and recovers), + * `rejected` survives a re-materialization and a restart until an upstream + * accepts the value, a value spelled as a secret handle is refused, door + * writes serialize, the no-crypto refusal writes nothing, and a walled + * posture leaves the store out of the chain. * * The values below are obvious dummies. */ @@ -39,7 +45,7 @@ import { AutomationServicePlugin, type CredentialResolver } from './plugin.js'; import type { AutomationEngine } from './engine.js'; import { SysConnectorCredential } from './sys-connector-credential.object.js'; import { SysFlowCredential } from './sys-flow-credential.object.js'; -import { CONNECTOR_CREDENTIAL_OBJECT } from './connector-credentials.js'; +import { CONNECTOR_CREDENTIAL_OBJECT, ConnectorCredentialStore } from './connector-credentials.js'; const REF = 'DUMMY_BILLING_TOKEN_27237'; const DUMMY_VALUE = 'dummy-value-aaaa-0001'; @@ -91,6 +97,8 @@ function fakeProvider() { const state = { materialized: [] as ConnectorProviderContext[], rejectAtMaterialization: false, + /** Answer `credentialVerified`, as mcp does when its handshake carried the credential. */ + verifyAtMaterialization: false, callStatus: 200, }; const tokenOf = (ctx: ConnectorProviderContext) => (ctx.auth?.type === 'bearer' ? ctx.auth.token : undefined); @@ -117,6 +125,7 @@ function fakeProvider() { ? { status: 200, ok: true, body: { invoices: [] } } : { status: state.callStatus, ok: false, body: { message: `token ${tokenOf(ctx)} is not valid` } }), }, + ...(state.verifyAtMaterialization ? { credentialVerified: true } : {}), }; }; return { factory, state, tokenOf }; @@ -149,7 +158,7 @@ function captureOutput(): { text: () => string } { return { text: () => chunks.join('\n') }; } -async function dataEngine(): Promise { +async function dataEngine(opts: { crypto?: boolean } = {}): Promise { const ql = new ObjectQL(); engines.push(ql); const driver = new SqlDriver({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }); @@ -160,13 +169,24 @@ async function dataEngine(): Promise { (ql as any).registry.registerObject(SysConnectorCredential as never, 'test'); (ql as any).registry.registerObject(SysFlowCredential as never, 'test'); await ql.syncSchemas(); - ql.setCryptoProvider(fakeCrypto()); + if (opts.crypto !== false) ql.setCryptoProvider(fakeCrypto()); return ql; } -async function boot(declared: unknown[], opts: { credentialResolver?: CredentialResolver } = {}) { - const ql = await dataEngine(); - const provider = fakeProvider(); +async function boot( + declared: unknown[], + opts: { + credentialResolver?: CredentialResolver; + /** Reuse a data engine — a restart over the same database. */ + ql?: ObjectQL; + crypto?: boolean; + /** Register a `tenancy` service with this posture. */ + posture?: string; + provider?: ReturnType; + } = {}, +) { + const ql = opts.ql ?? await dataEngine({ crypto: opts.crypto }); + const provider = opts.provider ?? fakeProvider(); const kernel = new LiteKernel({ logger: { level: 'debug', format: 'json' } } as never); kernels.push(kernel); kernel.use(new AutomationServicePlugin({ suspendedRunStore: 'memory', credentialResolver: opts.credentialResolver ?? (() => undefined) })); @@ -182,6 +202,7 @@ async function boot(declared: unknown[], opts: { credentialResolver?: Credential // engine under `data`, where the plugin's resolution finds it. ctx.registerService('objectql', { registry: { listItems: (t: string) => (t === 'connector' ? declared : []) } }); ctx.registerService('data', ql); + if (opts.posture !== undefined) ctx.registerService('tenancy', { posture: opts.posture }); ctx.getService('automation').registerConnectorProvider('fake', provider.factory); }, async start() {}, @@ -195,14 +216,22 @@ async function boot(declared: unknown[], opts: { credentialResolver?: Credential await captured.trigger('metadata:reloaded'); await flush(); }; - return { ql, engine, provider, call, reload }; + return { ql, engine, provider, call, reload, kernel }; +} + +/** Wait until the listing entry for {@link REF} matches — an acceptance is recorded off the call's path. */ +async function listingSettles(engine: AutomationEngine, expected: Record) { + await vi.waitFor(async () => { + expect((await engine.listConnectorCredentialRefs()).find((e) => e.ref === REF)).toMatchObject(expected); + }); + return (await engine.listConnectorCredentialRefs()).find((e) => e.ref === REF)!; } const ADMIN = { userId: 'usr_admin_27237' } as never; describe('[#27237] a missing credential degrades its instance; setting it materializes it, no restart', () => { it('missing → degraded naming the reference; set → ready; clear → degraded again', async () => { - const { engine, provider, ql } = await boot([billing()]); + const { engine, provider, ql, call } = await boot([billing()]); // Missing: no arm of the chain answers — degraded, named, factory never asked. expect(provider.state.materialized).toHaveLength(0); @@ -227,9 +256,14 @@ describe('[#27237] a missing credential degrades its instance; setting it materi expect(engine.getRegisteredConnectors()).toContain('billing'); expect(provider.state.materialized).toHaveLength(1); expect(provider.tokenOf(provider.state.materialized[0])).toBe(DUMMY_VALUE); - expect(entry).toMatchObject({ ref: REF, status: 'set', source: 'object' }); + expect(entry).toMatchObject({ ref: REF, status: 'set', source: 'object', updatedBy: 'usr_admin_27237' }); + expect(entry.updatedAt).toBeDefined(); expect(entry.connectors[0]).toMatchObject({ name: 'billing', state: 'ready' }); - expect(entry.lastVerifiedAt).toBeDefined(); + // Materializing sent nothing upstream (the rest shape), so nothing is verified yet… + expect(entry.lastVerifiedAt).toBeUndefined(); + // …until a call the upstream accepts. + expect(await call()).toMatchObject({ status: 200 }); + await vi.waitFor(async () => expect((await engine.listConnectorCredentialRefs())[0].lastVerifiedAt).toBeDefined()); // Set again: a new revision re-materializes with the new value. await engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE_2, context: ADMIN }); @@ -383,3 +417,193 @@ describe('[#27237] a wrong value degrades the instance and marks the row rejecte }, ); }); + +describe('[#27237] seat review r1: only a stored value holds its instance', () => { + it('an environment value refused on a call retries on the backoff, says so, and recovers once accepted', async () => { + process.env[REF] = DUMMY_ENV_VALUE; + envKeys.push(REF); + const out = captureOutput(); + const { engine, provider, call, reload } = await boot([billing()]); + expect(await call()).toMatchObject({ status: 200 }); + + provider.state.callStatus = 403; + expect(await call()).toMatchObject({ status: 403 }); + const reason = engine.getConnectorDegradedReason('billing'); + expect(reason).toContain('(HTTP 403) on a call'); + expect(reason).toContain('The instance is retried with backoff.'); + // The listing reports the refusal, though there is no row to mark. + const refused = await engine.listConnectorCredentialRefs(); + expect(refused[0]).toMatchObject({ ref: REF, source: 'env', status: 'rejected', statusReason: reason, writable: false }); + // On the backoff: the retry timer is armed for it (a held instance arms none). + expect(out.text()).toContain('1 degraded connector instance(s); next retry in 5000ms'); + + // The retry — the timer's own soft reconcile — re-materializes it. + provider.state.callStatus = 200; + const before = provider.state.materialized.length; + await reload(); + expect(provider.state.materialized).toHaveLength(before + 1); + expect(engine.getConnectorDegradedReason('billing')).toBeUndefined(); + // Still reported refused until an upstream accepts it… + expect((await engine.listConnectorCredentialRefs())[0]).toMatchObject({ status: 'rejected' }); + // …which the first accepted call does. + expect(await call()).toMatchObject({ status: 200 }); + const recovered = await listingSettles(engine, { source: 'env', status: 'set' }); + expect(recovered.statusReason).toBeUndefined(); + }); + + it("a vault value refused at materialization retries, and a rotation the provider proves recovers it", async () => { + let vaultValue = 'dummy-vault-value-eeee-0005'; + const vault = vi.fn(async (ref: string) => (ref === REF ? vaultValue : undefined)); + const provider = fakeProvider(); + provider.state.rejectAtMaterialization = true; + const { engine, reload } = await boot([billing()], { credentialResolver: vault, provider }); + expect(engine.getConnectorDegradedReason('billing')).toContain('(HTTP 401)'); + expect((await engine.listConnectorCredentialRefs())[0]).toMatchObject({ source: 'vault', status: 'rejected' }); + + // Rotated in the vault, where this process cannot see it: the retry picks it up. + vaultValue = 'dummy-vault-value-ffff-0006'; + provider.state.rejectAtMaterialization = false; + provider.state.verifyAtMaterialization = true; + await reload(); + expect(provider.tokenOf(provider.state.materialized.at(-1)!)).toBe('dummy-vault-value-ffff-0006'); + expect(engine.getConnectorDegradedReason('billing')).toBeUndefined(); + // The provider proved it while materializing: no call needed. + expect((await engine.listConnectorCredentialRefs())[0]).toMatchObject({ source: 'vault', status: 'set' }); + }); + + it('a stored value refused on a call holds its instance; a restart lifts the hold, the row stays `rejected` until a call is accepted', async () => { + const first = await boot([billing()]); + await first.engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE, context: ADMIN }); + const set = (await first.engine.listConnectorCredentialRefs())[0]; + first.provider.state.callStatus = 401; + await first.call(); + const reason = first.engine.getConnectorDegradedReason('billing'); + expect(reason).toContain(`until credential '${REF}' is set again or the server restarts`); + const rejected = (await first.engine.listConnectorCredentialRefs())[0]; + expect(rejected).toMatchObject({ status: 'rejected', source: 'object', statusReason: reason }); + // Marking it moved neither who set it nor when. + expect(rejected.updatedAt).toBe(set.updatedAt); + expect(rejected.updatedBy).toBe('usr_admin_27237'); + + // Held: the reconcile a retry would run leaves it alone. + const attempts = first.provider.state.materialized.length; + await first.reload(); + expect(first.provider.state.materialized).toHaveLength(attempts); + await first.kernel.shutdown(); + + // Restart over the same database: the instance materializes again (rest + // sends nothing while materializing), and the row is still `rejected`. + const second = await boot([billing()], { ql: first.ql }); + expect(second.provider.state.materialized).toHaveLength(1); + expect(second.engine.getConnectorDegradedReason('billing')).toBeUndefined(); + expect((await second.engine.listConnectorCredentialRefs())[0]).toMatchObject({ status: 'rejected', statusReason: reason }); + // A reconcile that re-materializes it changes nothing either. + await second.reload(); + expect((await second.engine.listConnectorCredentialRefs())[0]).toMatchObject({ status: 'rejected' }); + + // The upstream accepting the same value is what clears it. + expect(await second.call()).toMatchObject({ status: 200 }); + const accepted = await listingSettles(second.engine, { status: 'set', source: 'object' }); + expect(accepted.statusReason).toBeUndefined(); + expect(accepted.lastVerifiedAt).toBeDefined(); + }); + + it('a provider that proves the credential while materializing (mcp) returns a `rejected` row to `set` on restart', async () => { + const first = await boot([billing()]); + await first.engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE, context: ADMIN }); + first.provider.state.callStatus = 401; + await first.call(); + expect((await first.engine.listConnectorCredentialRefs())[0]).toMatchObject({ status: 'rejected' }); + await first.kernel.shutdown(); + + const provider = fakeProvider(); + provider.state.verifyAtMaterialization = true; + const second = await boot([billing()], { ql: first.ql, provider }); + const entry = (await second.engine.listConnectorCredentialRefs())[0]; + expect(entry).toMatchObject({ status: 'set', source: 'object' }); + expect(entry.lastVerifiedAt).toBeDefined(); + }); +}); + +describe('[#27237] seat review r1: the door refuses what it must, and serializes what it writes', () => { + it("refuses a value spelled as a secret handle (400), writing nothing and not echoing it", async () => { + const { engine, ql } = await boot([billing()]); + const err = await engine.setConnectorCredential({ ref: REF, value: 'secret:sec_dummy_0007', context: ADMIN }) + .then(() => undefined, (e: unknown) => e as Error & { status?: number; code?: string }); + expect(err).toMatchObject({ status: 400, code: 'VALIDATION_ERROR' }); + expect(err?.message).toContain("cannot start with 'secret:'"); + expect(err?.message.includes('sec_dummy_0007')).toBe(false); + expect(await ql.find(CONNECTOR_CREDENTIAL_OBJECT, { context: SYSTEM } as never)).toHaveLength(0); + }); + + it('two concurrent PUTs serialize: the instance ends on the value the store ends on', async () => { + const { engine, provider, ql } = await boot([billing()]); + await Promise.all([ + engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE, context: ADMIN }), + engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE_2, context: ADMIN }), + ]); + const rows = (await ql.find(CONNECTOR_CREDENTIAL_OBJECT, { context: SYSTEM } as never)) as Array<{ id: string; revision: number }>; + expect(rows).toHaveLength(1); + expect(Number(rows[0].revision)).toBe(2); + const stored = await (ql as any).resolveSecretField(CONNECTOR_CREDENTIAL_OBJECT, rows[0].id, 'value'); + expect(stored).toBe(DUMMY_VALUE_2); + expect(provider.state.materialized.map(provider.tokenOf)).toEqual([DUMMY_VALUE, DUMMY_VALUE_2]); + }); + + it('two writers inserting the first row at once (two replicas): one stores it, the other answers 409', async () => { + const ql = await dataEngine(); + const replicas = [new ConnectorCredentialStore(() => ql as never), new ConnectorCredentialStore(() => ql as never)]; + const outcomes = await Promise.allSettled( + replicas.map((store, i) => store.set({ ref: REF, value: i === 0 ? DUMMY_VALUE : DUMMY_VALUE_2, connectorNames: ['billing'], packageIds: [] })), + ); + expect(outcomes.filter((o) => o.status === 'fulfilled')).toHaveLength(1); + const refused = outcomes.find((o): o is PromiseRejectedResult => o.status === 'rejected'); + expect(refused?.reason).toMatchObject({ status: 409, code: 'RESOURCE_CONFLICT' }); + for (const v of [DUMMY_VALUE, DUMMY_VALUE_2]) expect(String(refused?.reason?.message).includes(v)).toBe(false); + expect(await ql.find(CONNECTOR_CREDENTIAL_OBJECT, { context: SYSTEM } as never)).toHaveLength(1); + }); + + it('with no crypto provider registered: 503 SERVICE_UNAVAILABLE, no row written, the instance still degraded', async () => { + const { engine, ql } = await boot([billing()], { crypto: false }); + const err = await engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE, context: ADMIN }) + .then(() => undefined, (e: unknown) => e as Error); + expect(err).toMatchObject({ status: 503, code: 'SERVICE_UNAVAILABLE' }); + expect(err?.message).toContain('needs a crypto provider'); + expect(err?.message.includes(DUMMY_VALUE)).toBe(false); + expect(await ql.find(CONNECTOR_CREDENTIAL_OBJECT, { context: SYSTEM } as never)).toHaveLength(0); + expect(engine.getConnectorDegradedReason('billing')).toContain(`credential '${REF}' is not set`); + }); +}); + +describe('[#27237] seat review r1: under a walled posture the store is not in the chain', () => { + it.each(['group', 'isolated'])('%s: a stored row is not read, and the door refuses to write, naming the posture', async (posture) => { + const single = await boot([billing()]); + await single.engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE, context: ADMIN }); + await single.kernel.shutdown(); + + const walled = await boot([billing()], { ql: single.ql, posture }); + // The row is there, and nothing resolves from it. + expect(walled.provider.state.materialized).toHaveLength(0); + expect(walled.engine.getConnectorDegradedReason('billing')).toContain(`credential '${REF}' is not set`); + const [entry] = await walled.engine.listConnectorCredentialRefs(); + expect(entry).toMatchObject({ ref: REF, status: 'missing', source: null, writable: false }); + for (const write of [ + () => walled.engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE_2, context: ADMIN }), + () => walled.engine.clearConnectorCredential({ ref: REF, context: ADMIN }), + ]) { + const err = await write().then(() => undefined, (e: unknown) => e as Error); + expect(err).toMatchObject({ status: 403, code: 'PERMISSION_DENIED' }); + expect(err?.message).toContain(`'${posture}' tenancy posture`); + } + const rows = (await single.ql.find(CONNECTOR_CREDENTIAL_OBJECT, { context: SYSTEM } as never)) as Array<{ revision: number }>; + expect(rows.map((r) => Number(r.revision))).toEqual([1]); + }); + + it('an environment variable still resolves under a walled posture', async () => { + process.env[REF] = DUMMY_ENV_VALUE; + envKeys.push(REF); + const { engine, provider } = await boot([billing()], { posture: 'isolated' }); + expect(provider.tokenOf(provider.state.materialized[0])).toBe(DUMMY_ENV_VALUE); + expect((await engine.listConnectorCredentialRefs())[0]).toMatchObject({ source: 'env', status: 'set', writable: false }); + }); +}); diff --git a/packages/spec/src/contracts/automation-service.ts b/packages/spec/src/contracts/automation-service.ts index bd44a9987df..8ee60e25cc5 100644 --- a/packages/spec/src/contracts/automation-service.ts +++ b/packages/spec/src/contracts/automation-service.ts @@ -977,7 +977,9 @@ export interface ConnectorSourcePullResult { * * The chain is env → the `sys_connector_credential` row an administrator set → * the host's injected resolver. An environment variable of the same name wins, - * so such a reference reports `source: 'env'` and `writable: false`. + * so such a reference reports `source: 'env'` and `writable: false`. Under a + * walled tenancy posture (`group` / `isolated`) the stored arm is not read, and + * nothing is `writable`. */ export interface ConnectorCredentialRef { /** The reference, exactly as an instance's `auth.credentialRef` names it. */ @@ -985,12 +987,17 @@ export interface ConnectorCredentialRef { /** * `set` — a source answers for it; `missing` — none does, and every * instance naming it is registered degraded; `rejected` — an upstream - * refused the stored value. + * refused the value `source` answers with, and none has accepted it since. + * A stored value stays `rejected` across a restart until an upstream + * accepts it or it is set again. */ status: 'set' | 'missing' | 'rejected'; /** Which arm of the chain answers: `env`, `object` (the stored row) or `vault` (the host's resolver); `null` when missing. */ source: 'env' | 'object' | 'vault' | null; - /** Whether the credential door may write it — `false` while an environment variable of the same name supplies it. */ + /** + * Whether the credential door may write it — `false` while an environment + * variable of the same name supplies it, and under a walled tenancy posture. + */ writable: boolean; /** Whether an installed package still declares it. A stored row nothing declares is listed so it can be cleared. */ declared: boolean; @@ -1003,10 +1010,17 @@ export interface ConnectorCredentialRef { }>; /** The upstream's message when `status` is `rejected`. Never the value. */ statusReason?: string; - /** When the stored row was last written, and by whom (its audit columns). */ + /** + * When the stored value was last set through the credential door, and by + * whom. A status change (`rejected`, verified) moves neither. + */ updatedAt?: string; updatedBy?: string; - /** When an instance last materialized with the stored value. */ + /** + * When an upstream last accepted the stored value: a provider that proved + * it while materializing, or a call answered `2xx`. Cleared when the value + * is set again. + */ lastVerifiedAt?: string; } From d9df7ad61e1c449724d258940433fa9e658718ce Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 01:16:33 +0000 Subject: [PATCH 14/21] WIP(#27237): door test reads the last materialization without Array.prototype.at Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- .../service-automation/src/connector-credential-door.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/services/service-automation/src/connector-credential-door.test.ts b/packages/services/service-automation/src/connector-credential-door.test.ts index b7033a1f60d..5502efcd235 100644 --- a/packages/services/service-automation/src/connector-credential-door.test.ts +++ b/packages/services/service-automation/src/connector-credential-door.test.ts @@ -465,7 +465,7 @@ describe('[#27237] seat review r1: only a stored value holds its instance', () = provider.state.rejectAtMaterialization = false; provider.state.verifyAtMaterialization = true; await reload(); - expect(provider.tokenOf(provider.state.materialized.at(-1)!)).toBe('dummy-vault-value-ffff-0006'); + expect(provider.tokenOf(provider.state.materialized[provider.state.materialized.length - 1])).toBe('dummy-vault-value-ffff-0006'); expect(engine.getConnectorDegradedReason('billing')).toBeUndefined(); // The provider proved it while materializing: no call needed. expect((await engine.listConnectorCredentialRefs())[0]).toMatchObject({ source: 'vault', status: 'set' }); From 7b75d460eaa30582087c86accf6270c8ed817787 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 02:34:27 +0000 Subject: [PATCH 15/21] =?UTF-8?q?WIP(#27237):=20seat=20review=20r2=20?= =?UTF-8?q?=E2=80=94=20a=202xx=20call=20stamps=20a=20set=20row's=20time=20?= =?UTF-8?q?and=20never=20clears=20rejected?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- ...37-connector-credential-rejected-marker.md | 2 +- .../27237-connector-credential-store.md | 2 +- content/docs/automation/connectors.mdx | 14 +++-- .../0097-declarative-connector-instances.md | 15 +++-- .../src/connector-credential-door.test.ts | 47 ++++++++------- .../src/connector-credentials.ts | 38 +++++++++--- .../services/service-automation/src/plugin.ts | 60 +++++++++++++------ .../src/sys-connector-credential.object.ts | 4 +- .../spec/src/contracts/automation-service.ts | 13 ++-- .../src/integration/connector-provider.ts | 5 +- 10 files changed, 129 insertions(+), 71 deletions(-) diff --git a/.changeset/27237-connector-credential-rejected-marker.md b/.changeset/27237-connector-credential-rejected-marker.md index 90c68a293bd..0242417762a 100644 --- a/.changeset/27237-connector-credential-rejected-marker.md +++ b/.changeset/27237-connector-credential-rejected-marker.md @@ -10,7 +10,7 @@ Metadata-protocol: none Three additions for the connector credential store (#27237): - `@objectstack/spec/integration` exports `CONNECTOR_CREDENTIAL_REJECTED`, `ConnectorCredentialRejectedError` (with `status`: 401 or 403) and the structural check `isConnectorCredentialRejected`. They sit beside the upstream-unavailable marker. A connector provider factory throws it when the upstream refuses the credential it sent, and the materializer then degrades the instance instead of failing the boot. The code is registered in the error-code ledger under `@objectstack/spec`. -- `ConnectorMaterialization` gains the optional `credentialVerified`. A provider factory sets it when the upstream accepted the instance's credential while materializing (the MCP handshake over http); the automation service then records the credential as verified, and a stored one marked `rejected` returns to `set`. A provider that first sends the credential on a call leaves it unset. +- `ConnectorMaterialization` gains the optional `credentialVerified`. A provider factory sets it when the upstream accepted the instance's credential while materializing (the MCP handshake over http); the automation service then records the credential as verified, and a stored one marked `rejected` returns to `set`. A provider that first sends the credential on a call leaves it unset; a call answered 2xx is not taken as proof. - `IAutomationService` gains three optional members, `listConnectorCredentialRefs`, `setConnectorCredential` and `clearConnectorCredential`, with their types `ConnectorCredentialRef`, `ConnectorCredentialSetRequest` and `ConnectorCredentialClearRequest`. A listing entry never carries a value. `PLATFORM_OBJECT_NAMES['service-automation']` lists the new `sys_connector_credential` object. diff --git a/.changeset/27237-connector-credential-store.md b/.changeset/27237-connector-credential-store.md index d3e7bfdfe4b..a057ebab270 100644 --- a/.changeset/27237-connector-credential-store.md +++ b/.changeset/27237-connector-credential-store.md @@ -13,7 +13,7 @@ The reference now resolves through three sources, in order: an environment varia - A reference no source answers for degrades the instance naming it, with a reason that names the reference and how to set it. Before, it failed the boot. A `credentialResolver` that throws is still fatal at boot. - Setting or clearing a credential reconciles at once: every instance naming the reference re-materializes, or is torn down and degraded, with no restart. -- A credential the upstream refuses (HTTP 401 / 403) degrades the instance with the upstream's message, never the value. An `mcp` instance is refused when it connects; a `rest` or `openapi` instance on the first call that answers 401 / 403, which still returns its answer. A stored credential is marked `rejected`, and its instance is not retried until the credential is set again, the declaration changes, or the server restarts; the mark itself stays until an upstream accepts the value (an `mcp` handshake, or a call answered 2xx). A credential from an environment variable or the host's resolver is retried on the usual backoff instead, since it can change where the service cannot see it, and the listing reports it `rejected` until it is accepted. +- A credential the upstream refuses (HTTP 401 / 403) degrades the instance with the upstream's message, never the value. An `mcp` instance is refused when it connects; a `rest` or `openapi` instance on the first call that answers 401 / 403, which still returns its answer. A stored credential is marked `rejected`, and its instance is not retried until the credential is set again, the declaration changes, or the server restarts; the mark itself stays until the value is set again or an `mcp` handshake proves it. A call answered 2xx does not clear it, since the endpoint may not check the credential; it only stamps the last-verified time of a credential still `set`. A credential from an environment variable or the host's resolver is retried on the usual backoff instead, since it can change where the service cannot see it, and the listing reports it `rejected` until a provider proves it or the server restarts. - Under the `group` and `isolated` tenancy postures the store is not read, and writes to it are refused with a 403 naming the posture. - A value starting with `secret:` is refused with a 400: the data engine would store it as a handle to another stored secret. Writes run one at a time with their reconcile, so the instances end on the value the store ends on; a first write another replica makes at the same moment answers 409. - The value never appears in a log record, the listing, `GET /connectors`, an error body or a read of the object. A provider message that echoes it carries `[credential withheld]` instead. diff --git a/content/docs/automation/connectors.mdx b/content/docs/automation/connectors.mdx index e7342bc4b1f..fd12b1a78b7 100644 --- a/content/docs/automation/connectors.mdx +++ b/content/docs/automation/connectors.mdx @@ -279,8 +279,8 @@ DELETE /api/v1/automation/connectors/credential-refs/BILLING_API_TOKEN `missing` or `rejected`), a `source` (`env`, `object`, `vault` or `null`), whether this door can write it, and the instances naming it with their state. A stored one also says when and by whom it was last set (`updatedAt`, - `updatedBy`) and when an upstream last accepted it (`lastVerifiedAt`). It - never includes a value. + `updatedBy`) and when it was last seen working (`lastVerifiedAt`). It never + includes a value. - **`PUT`** stores the value, then reconciles. Every instance naming the reference re-materializes with it **without a restart**, and the answer is the reference's entry after that. The route refuses, before writing anything: @@ -308,13 +308,15 @@ depends on where the value came from: - **A value stored here** is marked `rejected`, and its instance is not retried until the credential is set again, its declaration changes, or the - server restarts. The mark outlives a restart: it clears only when an upstream - accepts the value, either an `mcp` handshake that carries it or a `rest` / - `openapi` call answered `2xx`. + server restarts. The mark outlives a restart: it clears only when the value + is set again or an `mcp` handshake that carries it succeeds. A `rest` / + `openapi` call answered `2xx` does not clear it, because the endpoint may not + check the credential at all. - **A value from an environment variable or the host's resolver** can change where this door cannot see it (a rotation, or a `403` rate limit that lifts), so its instance is retried on the same 5s→5min backoff as an unreachable - upstream. The listing reports it `rejected` until an upstream accepts it. + upstream. The listing reports it `rejected` until an `mcp` handshake proves + it or the server restarts, even while the instance is serving again. Credentials are **environment-level**: one value per reference for the whole environment, read by whichever instance names it. On a multi-organization diff --git a/docs/adr/0097-declarative-connector-instances.md b/docs/adr/0097-declarative-connector-instances.md index 50c9f4be377..556e1b1dd62 100644 --- a/docs/adr/0097-declarative-connector-instances.md +++ b/docs/adr/0097-declarative-connector-instances.md @@ -231,12 +231,15 @@ exactly the plausible-but-dead shape this ADR exists to kill. > until the credential is set again, the declaration changes, or the process restarts (a restart > lifts the hold). An instance with no `credentialRef`, whose credential is written in the > declaration, is held the same way. The `rejected` mark survives a re-materialization, since a -> `rest` or `openapi` instance materializes without sending the value. Only an upstream accepting -> the value clears it: a provider answering `credentialVerified` at materialization (the `mcp` -> handshake over http), or a call answered `2xx`. A value from an environment variable or the -> host's resolver can change where the process cannot see it (a rotation, or a `403` rate limit -> that lifts), so its instance is retried on the backoff above instead, and the credential -> listing reports the refusal until an upstream accepts it. +> `rest` or `openapi` instance materializes without sending the value. Only proof clears it: a +> provider answering `credentialVerified` at materialization (the `mcp` handshake over http), or +> setting the value again. A call answered `2xx` is not proof, since the endpoint may not check +> the credential (a public path, an operation with `security: []`); it only stamps the +> last-verified time of a row still at `set`. A value from an environment variable or the host's +> resolver can change where the process cannot see it (a rotation, or a `403` rate limit that +> lifts), so its instance is retried on the backoff above instead, and the credential listing +> reports the refusal until a provider proves the value, it stops resolving from that source, or +> the process restarts. The `openapi` provider applies the same split to its **remote-URL spec fetch**: an unreachable endpoint (network error) or a transient HTTP status diff --git a/packages/services/service-automation/src/connector-credential-door.test.ts b/packages/services/service-automation/src/connector-credential-door.test.ts index 5502efcd235..37141ba1811 100644 --- a/packages/services/service-automation/src/connector-credential-door.test.ts +++ b/packages/services/service-automation/src/connector-credential-door.test.ts @@ -219,19 +219,12 @@ async function boot( return { ql, engine, provider, call, reload, kernel }; } -/** Wait until the listing entry for {@link REF} matches — an acceptance is recorded off the call's path. */ -async function listingSettles(engine: AutomationEngine, expected: Record) { - await vi.waitFor(async () => { - expect((await engine.listConnectorCredentialRefs()).find((e) => e.ref === REF)).toMatchObject(expected); - }); - return (await engine.listConnectorCredentialRefs()).find((e) => e.ref === REF)!; -} const ADMIN = { userId: 'usr_admin_27237' } as never; describe('[#27237] a missing credential degrades its instance; setting it materializes it, no restart', () => { it('missing → degraded naming the reference; set → ready; clear → degraded again', async () => { - const { engine, provider, ql, call } = await boot([billing()]); + const { engine, provider, ql, call, reload } = await boot([billing()]); // Missing: no arm of the chain answers — degraded, named, factory never asked. expect(provider.state.materialized).toHaveLength(0); @@ -261,9 +254,10 @@ describe('[#27237] a missing credential degrades its instance; setting it materi expect(entry.connectors[0]).toMatchObject({ name: 'billing', state: 'ready' }); // Materializing sent nothing upstream (the rest shape), so nothing is verified yet… expect(entry.lastVerifiedAt).toBeUndefined(); - // …until a call the upstream accepts. + // …until a call answered 2xx stamps the time on the row, which is `set`. expect(await call()).toMatchObject({ status: 200 }); - await vi.waitFor(async () => expect((await engine.listConnectorCredentialRefs())[0].lastVerifiedAt).toBeDefined()); + await reload(); // the reconcile queue has run the call's bookkeeping by now + expect((await engine.listConnectorCredentialRefs())[0]).toMatchObject({ status: 'set', lastVerifiedAt: expect.any(String) }); // Set again: a new revision re-materializes with the new value. await engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE_2, context: ADMIN }); @@ -419,7 +413,7 @@ describe('[#27237] a wrong value degrades the instance and marks the row rejecte }); describe('[#27237] seat review r1: only a stored value holds its instance', () => { - it('an environment value refused on a call retries on the backoff, says so, and recovers once accepted', async () => { + it('an environment value refused on a call retries on the backoff, says so, and recovers', async () => { process.env[REF] = DUMMY_ENV_VALUE; envKeys.push(REF); const out = captureOutput(); @@ -437,18 +431,18 @@ describe('[#27237] seat review r1: only a stored value holds its instance', () = // On the backoff: the retry timer is armed for it (a held instance arms none). expect(out.text()).toContain('1 degraded connector instance(s); next retry in 5000ms'); - // The retry — the timer's own soft reconcile — re-materializes it. + // The retry — the timer's own soft reconcile — re-materializes it, and it serves again. provider.state.callStatus = 200; const before = provider.state.materialized.length; await reload(); expect(provider.state.materialized).toHaveLength(before + 1); expect(engine.getConnectorDegradedReason('billing')).toBeUndefined(); - // Still reported refused until an upstream accepts it… - expect((await engine.listConnectorCredentialRefs())[0]).toMatchObject({ status: 'rejected' }); - // …which the first accepted call does. expect(await call()).toMatchObject({ status: 200 }); - const recovered = await listingSettles(engine, { source: 'env', status: 'set' }); - expect(recovered.statusReason).toBeUndefined(); + await reload(); + // Still reported refused: a 2xx is no proof (the endpoint may not check the credential). + const after = (await engine.listConnectorCredentialRefs())[0]; + expect(after).toMatchObject({ source: 'env', status: 'rejected' }); + expect(after.connectors[0]).toMatchObject({ state: 'ready' }); }); it("a vault value refused at materialization retries, and a rotation the provider proves recovers it", async () => { @@ -471,7 +465,7 @@ describe('[#27237] seat review r1: only a stored value holds its instance', () = expect((await engine.listConnectorCredentialRefs())[0]).toMatchObject({ source: 'vault', status: 'set' }); }); - it('a stored value refused on a call holds its instance; a restart lifts the hold, the row stays `rejected` until a call is accepted', async () => { + it('a stored value refused on a call holds its instance; a restart lifts the hold, and the row stays `rejected` — a 2xx call does not clear it', async () => { const first = await boot([billing()]); await first.engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE, context: ADMIN }); const set = (await first.engine.listConnectorCredentialRefs())[0]; @@ -501,11 +495,20 @@ describe('[#27237] seat review r1: only a stored value holds its instance', () = await second.reload(); expect((await second.engine.listConnectorCredentialRefs())[0]).toMatchObject({ status: 'rejected' }); - // The upstream accepting the same value is what clears it. + // A call answered 2xx is no proof — a public path or an operation with + // `security: []` never checks the credential — so the row stays + // `rejected`, its reason and its unverified time unmoved. expect(await second.call()).toMatchObject({ status: 200 }); - const accepted = await listingSettles(second.engine, { status: 'set', source: 'object' }); - expect(accepted.statusReason).toBeUndefined(); - expect(accepted.lastVerifiedAt).toBeDefined(); + await second.reload(); // the reconcile queue has run the call's bookkeeping by now + const afterCall = (await second.engine.listConnectorCredentialRefs())[0]; + expect(afterCall).toMatchObject({ status: 'rejected', source: 'object', statusReason: reason }); + expect(afterCall.lastVerifiedAt).toBeUndefined(); + + // Setting it again is what clears it. + await second.engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE_2, context: ADMIN }); + const reset = (await second.engine.listConnectorCredentialRefs())[0]; + expect(reset).toMatchObject({ status: 'set', source: 'object' }); + expect(reset.statusReason).toBeUndefined(); }); it('a provider that proves the credential while materializing (mcp) returns a `rejected` row to `set` on restart', async () => { diff --git a/packages/services/service-automation/src/connector-credentials.ts b/packages/services/service-automation/src/connector-credentials.ts index 390aabda83d..57b5f148c08 100644 --- a/packages/services/service-automation/src/connector-credentials.ts +++ b/packages/services/service-automation/src/connector-credentials.ts @@ -326,12 +326,13 @@ export class ConnectorCredentialStore { } /** - * Record that an upstream ACCEPTED revision `revision`: a provider that - * proved it at materialization (`credentialVerified`, the mcp provider), or - * a call its upstream answered `2xx`. The timestamp moves, and a `rejected` - * row whose same value now works returns to `set`. A materialization alone - * proves nothing (rest and openapi first send the value on a call), so it - * never calls this. A row rewritten since (another revision) is left alone. + * Record that a provider PROVED revision `revision` while materializing + * (`credentialVerified` — the mcp handshake carried it and the upstream + * answered). The timestamp moves, and a `rejected` row whose same value now + * works returns to `set`: this and setting the value again are the only + * ways back. A materialization alone proves nothing (rest and openapi + * first send the value on a call), so it never calls this. A row rewritten + * since (another revision) is left alone. */ async markVerified(record: ConnectorCredentialRecord, at: Date = new Date()): Promise { const engine = this.resolveEngine(); @@ -345,6 +346,26 @@ export class ConnectorCredentialStore { ); } + /** + * Record that a call sent with revision `revision` was answered `2xx`: + * the timestamp moves on a row still at `set`, and nothing else. A `2xx` + * proves nothing about a credential the endpoint may not have checked (a + * public path, an openapi operation with `security: []`), so it never + * returns a `rejected` row to `set` — {@link markVerified} or a new value + * does that. + */ + async markCallAccepted(record: ConnectorCredentialRecord, at: Date = new Date()): Promise { + const engine = this.resolveEngine(); + if (!engine) return; + const current = await this.read(record.ref); + if (!current || current.revision !== record.revision || current.status !== 'set') return; + await engine.update( + CONNECTOR_CREDENTIAL_OBJECT, + { id: current.id, last_verified_at: at.toISOString() }, + { context: SYSTEM_CONTEXT }, + ); + } + /** * Record that an upstream refused revision `revision`. `reason` is the * connector's message, already scrubbed of the value by the caller @@ -582,8 +603,9 @@ export function credentialRejectedOnCallReason( } /** - * A `2xx` result of a connector call — the upstream accepted the credential. - * The rest and openapi providers answer `{ status, ok, body }`. + * A `2xx` result of a connector call. The rest and openapi providers answer + * `{ status, ok, body }`. Not a proof the credential is valid: the endpoint + * may not have checked it. */ export function isAcceptedCallResult(result: unknown): boolean { if (typeof result !== 'object' || result === null) return false; diff --git a/packages/services/service-automation/src/plugin.ts b/packages/services/service-automation/src/plugin.ts index b337babbd9b..ea7be26b11a 100644 --- a/packages/services/service-automation/src/plugin.ts +++ b/packages/services/service-automation/src/plugin.ts @@ -739,16 +739,18 @@ export class AutomationServicePlugin implements Plugin { * does not hold — an environment variable or the host's resolver — keyed * by reference, with the source that was refused and the reason (already * scrubbed of the value). There is no row to mark, so this is what the - * listing reports as `rejected` until an upstream accepts the reference - * again or it stops resolving from that source. + * listing reports as `rejected` until a provider proves the reference while + * materializing, it stops resolving from that source, or the process + * restarts. A call answered `2xx` does not clear it, as it does not clear + * a row. */ private credentialRefusals = new Map(); /** * [#27237] The backoff count of an instance refused on such a value, by * instance name. Kept across a re-materialization — a rest or openapi * instance materializes without sending its credential, so the count would - * otherwise restart at one on every retry — and dropped once an upstream - * accepts the credential, or the instance or its credential's source changes. + * otherwise restart at one on every retry — and dropped once a provider + * proves the credential, or the instance or its credential's source changes. */ private credentialRefusalAttempts = new Map(); private declarativeRetryTimer?: ReturnType; @@ -2362,10 +2364,10 @@ export class AutomationServicePlugin implements Plugin { // [#27237] Only a provider that proved the credential while // materializing (`credentialVerified` — mcp's handshake) says the // upstream accepted it. rest and openapi first send it on a call, - // where the watcher records the first accepted one; a - // materialization alone never turns a `rejected` row back to `set`. + // where a `2xx` only stamps a `set` row's time; neither that nor a + // materialization alone turns a `rejected` row back to `set`. if (credential && ref !== undefined && proven) { - await this.recordCredentialAccepted(ctx, name, ref, credential); + await this.recordCredentialProven(ctx, name, ref, credential); } } @@ -2934,13 +2936,13 @@ export class AutomationServicePlugin implements Plugin { } /** - * [#27237] An upstream accepted an instance's credential — the provider - * proved it while materializing, or a call was answered `2xx`. A stored - * value's row records when, and a `rejected` row of that same revision - * returns to `set`; a refusal of a value the door does not hold is - * forgotten, and so is the instance's backoff count. + * [#27237] A provider proved an instance's credential while materializing + * (`credentialVerified`). A stored value's row records when, and a + * `rejected` row of that same revision returns to `set`; a refusal of a + * value the door does not hold is forgotten, and so is the instance's + * backoff count. */ - private async recordCredentialAccepted( + private async recordCredentialProven( ctx: PluginContext, name: string, ref: string, @@ -2956,7 +2958,30 @@ export class AutomationServicePlugin implements Plugin { await this.connectorCredentialStoreOrNone().markVerified(credential.record); } catch (markErr) { ctx.logger.warn( - `[Automation] connector instance '${name}': an upstream accepted its credential, but the credential row could not record it.`, + `[Automation] connector instance '${name}': its provider proved its credential, but the credential row could not record it.`, + describeThrownForLog(markErr), + ); + } + } + + /** + * [#27237] A call with a stored credential was answered `2xx`: the row's + * last-verified time moves if it is still `set`. Nothing else — a `2xx` + * from an endpoint that never checked the credential is no proof, so a + * `rejected` row, a refusal of a value the door does not hold and the + * backoff count all stay as they are. + */ + private async recordCredentialCallAccepted( + ctx: PluginContext, + name: string, + credential: Extract, + ): Promise { + if (credential.source !== 'object' || !credential.record) return; + try { + await this.connectorCredentialStoreOrNone().markCallAccepted(credential.record); + } catch (markErr) { + ctx.logger.warn( + `[Automation] connector instance '${name}': a call with its credential was answered 2xx, but the credential row could not record it.`, describeThrownForLog(markErr), ); } @@ -2966,8 +2991,9 @@ export class AutomationServicePlugin implements Plugin { * [#27237] Wrap a credentialed instance's handlers so the first call its * upstream answers 401 / 403 degrades the instance (ruling on #27237, * comment 6470087293), and — unless the provider already `proven` the - * credential at materialization — the first call it answers `2xx` records - * the credential as accepted. A call's own result is returned unchanged. + * credential at materialization — the first call it answers `2xx` stamps a + * stored credential's last-verified time while its row is `set` (never a + * proof that clears `rejected`). A call's own result is returned unchanged. * Both run on the reconcile queue, and only against the materialization * these handlers belong to, so neither races a reconcile or a credential * write that already replaced it. A refusal is awaited (the caller sees @@ -2999,7 +3025,7 @@ export class AutomationServicePlugin implements Plugin { if (!accepted && !refused && isAcceptedCallResult(result)) { accepted = true; void this.onReconcileQueue(async () => { - if (current()) await this.recordCredentialAccepted(ctx, info.name, info.ref, info.credential); + if (current()) await this.recordCredentialCallAccepted(ctx, info.name, info.credential); }); return result; } diff --git a/packages/services/service-automation/src/sys-connector-credential.object.ts b/packages/services/service-automation/src/sys-connector-credential.object.ts index e417b82f7ff..1718dcdc6ac 100644 --- a/packages/services/service-automation/src/sys-connector-credential.object.ts +++ b/packages/services/service-automation/src/sys-connector-credential.object.ts @@ -104,7 +104,7 @@ export const SysConnectorCredential = ObjectSchema.create({ required: true, defaultValue: 'set', description: - "`set` when a value is stored; `rejected` when an upstream refused it (an instance's materialization or call answered 401 / 403). Back to `set` only when the value is set again or an upstream accepts it. A reference no source answers for is `missing` in the listing and has no row.", + "`set` when a value is stored; `rejected` when an upstream refused it (an instance's materialization or call answered 401 / 403). Back to `set` only when the value is set again or a provider proves it while materializing (mcp); a call answered 2xx does not, since the endpoint may not check the credential. A reference no source answers for is `missing` in the listing and has no row.", }), status_reason: Field.textarea({ @@ -117,7 +117,7 @@ export const SysConnectorCredential = ObjectSchema.create({ label: 'Last Verified At', readonly: true, description: - 'When an upstream last accepted this value: a provider that proves the credential at materialization (mcp), or the first call answered 2xx after an instance materialized (rest, openapi). Cleared when the value is set again.', + 'When this value was last seen working: a provider proved it while materializing (mcp), or, while the row is `set`, the first call answered 2xx after an instance materialized (rest, openapi). Cleared when the value is set again.', }), set_at: Field.datetime({ diff --git a/packages/spec/src/contracts/automation-service.ts b/packages/spec/src/contracts/automation-service.ts index 8ee60e25cc5..e6e561a9d2d 100644 --- a/packages/spec/src/contracts/automation-service.ts +++ b/packages/spec/src/contracts/automation-service.ts @@ -987,9 +987,10 @@ export interface ConnectorCredentialRef { /** * `set` — a source answers for it; `missing` — none does, and every * instance naming it is registered degraded; `rejected` — an upstream - * refused the value `source` answers with, and none has accepted it since. - * A stored value stays `rejected` across a restart until an upstream - * accepts it or it is set again. + * refused the value `source` answers with, and no provider has proved it + * since. A stored value stays `rejected` across a restart until a provider + * proves it while materializing or it is set again; a call answered `2xx` + * does not clear it, since the endpoint may not check the credential. */ status: 'set' | 'missing' | 'rejected'; /** Which arm of the chain answers: `env`, `object` (the stored row) or `vault` (the host's resolver); `null` when missing. */ @@ -1017,9 +1018,9 @@ export interface ConnectorCredentialRef { updatedAt?: string; updatedBy?: string; /** - * When an upstream last accepted the stored value: a provider that proved - * it while materializing, or a call answered `2xx`. Cleared when the value - * is set again. + * When the stored value was last seen working: a provider proved it while + * materializing, or, while `status` is `set`, a call with it was answered + * `2xx`. Cleared when the value is set again. */ lastVerifiedAt?: string; } diff --git a/packages/spec/src/integration/connector-provider.ts b/packages/spec/src/integration/connector-provider.ts index 683703c43ff..385dfcde14f 100644 --- a/packages/spec/src/integration/connector-provider.ts +++ b/packages/spec/src/integration/connector-provider.ts @@ -49,8 +49,9 @@ export interface ConnectorMaterialization { * answered (an MCP handshake over http). The automation service then records * the credential as verified, and a stored one marked `rejected` returns to * `set`. Leave it unset when materializing sends no credential (a provider - * that first presents it on a call): the service waits for a call the - * upstream accepts instead. + * that first presents it on a call): a call answered `2xx` is not proof, + * since the endpoint may not check the credential, so such a provider's + * `rejected` credential stays so until it is set again. */ credentialVerified?: boolean; } From bda699c29b99f3235aa539f34d188bb81a32e949 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 02:42:12 +0000 Subject: [PATCH 16/21] WIP(#27237): regenerate the tenant-audit and platform-object tenancy censuses after merging main Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- .../docs/permissions/tenant-audit-census.mdx | 42 +++++++++---------- ...08-tenant-audit-write-call-sites.counts.md | 22 +++++----- scripts/platform-object-tenancy-census.json | 15 +++++-- 3 files changed, 44 insertions(+), 35 deletions(-) diff --git a/content/docs/permissions/tenant-audit-census.mdx b/content/docs/permissions/tenant-audit-census.mdx index 44d4fd08d14..bea7f8fd959 100644 --- a/content/docs/permissions/tenant-audit-census.mdx +++ b/content/docs/permissions/tenant-audit-census.mdx @@ -93,7 +93,7 @@ what moved this page's population from 225 to 227; nothing about the two sites changed, only whether this instrument could see them. **The expensive failure direction is a keyword.** Sites whose receiver the author -typed `any` have no type to read, and there are 60 of them — just over a fifth +typed `any` have no type to read, and there are 61 of them — just over a fifth of the population, concentrated in exactly the seed and bootstrap paths this control exists for. Scoring an unreadable receiver as "not an engine" would have dropped every one of them silently, with a clean exit and a smaller number that @@ -132,7 +132,7 @@ are reported as `undecidable` rather than assumed either way. The same holds twice over for the context. An options argument spelled as a literal can be read; one spelled `options`, `{ ...opts }`, or handed through a -forwarding shim cannot, and **51 of the 249 sites are spelled that way**. A +forwarding shim cannot, and **51 of the 250 sites are spelled that way**. A context resolved from an inline literal or a local `const` can be tested for `isSystem`; one arriving from a helper call cannot. @@ -197,10 +197,10 @@ reproduce them. Where it disagrees, it disagrees on the page: | carried figure | where it survives | this census | | :--- | :--- | ---: | -| 175 write call sites | quoted in the merged changeset | **249** | +| 175 write call sites | quoted in the merged changeset | **250** | | 24 carrying no tenant context | quoted in the merged changeset | **2** provable and tenancy-enabled; **29** more whose options argument is unreadable | -| 127 of 175 statically decidable, 48 runtime-parameter-name sites | restated on the `isSystem`-scoping card | **171 of 249** decidable, **78** undecidable | -| 135 (77%) silenced by the `isSystem` guard before the posture gate | the lost issue body — **no surviving corroboration** | **not reproduced**: 143 decidably elevated, 0 decidably not, 98 undecidable | +| 127 of 175 statically decidable, 48 runtime-parameter-name sites | restated on the `isSystem`-scoping card | **172 of 250** decidable, **78** undecidable | +| 135 (77%) silenced by the `isSystem` guard before the posture gate | the lost issue body — **no surviving corroboration** | **not reproduced**: 144 decidably elevated, 0 decidably not, 98 undecidable | | 141 and 132, two independent re-derivations | the card that filed this work | — | **The differences are not reconciled, and deliberately so.** The old census's @@ -210,18 +210,18 @@ be stated is what this instrument counts, which is written above and re-runnable at any commit. Two structural facts do plausibly widen this reading against any hand or regex -one, and both are counted in the generated tables below: the 60 sites reached -through an erased (`any`) receiver, and the 56 that name their object through a +one, and both are counted in the generated tables below: the 61 sites reached +through an erased (`any`) receiver, and the 57 that name their object through a `const` rather than inline. An instrument that read either the way a person does would report a smaller number and would not say so. The fourth row is the one worth flagging to anyone citing it. **The 135 / 77% figure has no surviving corroboration anywhere in the tree.** This census reads -143 of 249 (57%) as decidably elevated, with 98 more whose elevation is a +144 of 250 (58%) as decidably elevated, with 98 more whose elevation is a run-time fact — so the claim is neither confirmed nor refuted, and the honest answer is that a static reading cannot settle it. -⇒ **Cite `2 / 249`, and say what it is**: the sites whose options argument was +⇒ **Cite `2 / 250`, and say what it is**: the sites whose options argument was READ and holds no tenant context, against a decidably tenancy-enabled object. That is the control's provable yield surface. ⛔ Do not cite it as "the sites without tenant context" — **29 further sites** have an options argument this @@ -233,29 +233,29 @@ cannot read, and they are neither in nor out. | what | count | | :--- | ---: | -| write call sites on the application surface | **249** | -| …whose object name is statically decidable | 171 | +| write call sites on the application surface | **250** | +| …whose object name is statically decidable | 172 | | …whose object name is chosen at run time | 78 | -| …against an object with tenancy ENABLED | 170 | +| …against an object with tenancy ENABLED | 171 | | …against an object that declares tenancy off | 1 | -| threading a tenant context | 190 | +| threading a tenant context | 191 | | PROVABLY carrying none (options read, no context key) | **8** | | …of those, against a decidably tenancy-enabled object | **2** | | options argument UNREADABLE — may or may not carry one | 51 | | …of those, against a decidably tenancy-enabled object | 29 | -| threading a decidably ELEVATED (`isSystem`) context | 143 | +| threading a decidably ELEVATED (`isSystem`) context | 144 | | threading a context that is decidably NOT elevated | 0 | | threading a context whose elevation is a run-time fact | 98 | | how the instrument reached the site | count | | :--- | ---: | | receiver carried a readable engine type | 189 | -| receiver erased, placed by the object NAME | 50 | +| receiver erased, placed by the object NAME | 51 | | receiver erased, placed by an `object: string` PARAMETER | 5 | | receiver erased, placed by an `UNTYPED_RECEIVERS` row | 5 | | object name spelled inline | 115 | -| object name spelled through a `const` | 56 | +| object name spelled through a `const` | 57 | | object name is an `object: string` parameter | 7 | | object name is some other run-time expression | 71 | @@ -307,13 +307,13 @@ holds still. They are required to be HERE and to say WHEN they were true; their values are not compared. The reasoning, and the measurement behind it, are in `scripts/check-tenant-audit-census.mjs`. -Measured on 2026-10-28 at `28c124d6d`. +Measured on 2026-10-29 at `82760e4ee`. | corpus scale (not enforced) | count | | :--- | ---: | -| tracked non-test sources scanned | 800 | -| engine-shaped types recognised | 94 | -| declared objects in the registry | 121 | -| same-named calls subtracted as non-engine | 181 | +| tracked non-test sources scanned | 801 | +| engine-shaped types recognised | 95 | +| declared objects in the registry | 122 | +| same-named calls subtracted as non-engine | 185 | {/* END GENERATED: tenant-audit-census */} diff --git a/docs/audits/2026-08-tenant-audit-write-call-sites.counts.md b/docs/audits/2026-08-tenant-audit-write-call-sites.counts.md index 02c2db29809..80b3ef529aa 100644 --- a/docs/audits/2026-08-tenant-audit-write-call-sites.counts.md +++ b/docs/audits/2026-08-tenant-audit-write-call-sites.counts.md @@ -33,17 +33,17 @@ silent, and `node scripts/tenant-audit-census.mjs --write` is the resolution. | Measure | Value | |---|---:| -| Write call sites | 249 | -| Object name statically decidable | 171 | +| Write call sites | 250 | +| Object name statically decidable | 172 | | Object name chosen at run time | 78 | -| Against a tenancy-enabled object | 170 | +| Against a tenancy-enabled object | 171 | | Against an object declaring tenancy off | 1 | -| Threading a tenant context | 190 | +| Threading a tenant context | 191 | | Provably carrying none | 8 | | …and decidably tenancy-enabled | 2 | | Options argument unreadable | 51 | | …and decidably tenancy-enabled | 29 | -| Threading a decidably elevated context | 143 | +| Threading a decidably elevated context | 144 | | Threading a decidably non-elevated context | 0 | | Threading a context of undecidable elevation | 98 | @@ -90,14 +90,14 @@ holds still. They are required to be HERE and to say WHEN they were true; their values are not compared. The reasoning, and the measurement behind it, are in `scripts/check-tenant-audit-census.mjs`. -Measured on 2026-10-28 at `28c124d6d`. +Measured on 2026-10-29 at `82760e4ee`. | corpus scale (not enforced) | count | | :--- | ---: | -| tracked non-test sources scanned | 800 | -| engine-shaped types recognised | 94 | -| declared objects in the registry | 121 | -| same-named calls subtracted as non-engine | 181 | +| tracked non-test sources scanned | 801 | +| engine-shaped types recognised | 95 | +| declared objects in the registry | 122 | +| same-named calls subtracted as non-engine | 185 | ## Every site @@ -204,7 +204,7 @@ Measured on 2026-10-28 at `28c124d6d`. | `packages/services/service-automation/src/connector-credentials.ts` | `delete` | `sys_connector_credential` | enabled | context, elevation undecidable | 1 | | `packages/services/service-automation/src/connector-credentials.ts` | `insert` | `sys_connector_credential` | enabled | context, elevation undecidable | 1 | | `packages/services/service-automation/src/connector-credentials.ts` | `update` | `sys_connector_credential` | enabled | context, elevation undecidable | 1 | -| `packages/services/service-automation/src/connector-credentials.ts` | `update` | `sys_connector_credential` | enabled | elevated | 2 | +| `packages/services/service-automation/src/connector-credentials.ts` | `update` | `sys_connector_credential` | enabled | elevated | 3 | | `packages/services/service-automation/src/flow-credential-channel.ts` | `delete` | `sys_flow_credential` | enabled | elevated | 5 | | `packages/services/service-automation/src/flow-credential-channel.ts` | `insert` | `sys_flow_credential` | enabled | elevated | 1 | | `packages/services/service-automation/src/flow-credential-channel.ts` | `update` | `sys_flow_credential` | enabled | elevated | 2 | diff --git a/scripts/platform-object-tenancy-census.json b/scripts/platform-object-tenancy-census.json index cabb74df3d1..bb76672a351 100644 --- a/scripts/platform-object-tenancy-census.json +++ b/scripts/platform-object-tenancy-census.json @@ -26,13 +26,13 @@ "predicate": "resolveTenantFieldName(registered schema) !== null, where the registered schema is the authored schema plus the columns resolveInjectedSystemColumns says the registration injects (the applySystemFields pass). Both functions are loaded from source and executed; neither is re-spelled here.", "population": "every object registered by a tracked packages/**/*.object.ts module whose name carries a platform prefix (sys_ / cloud_ / ai_). The cloud repository's own cloud_ objects are not in this tree and so not in this census.", "totals": { - "registered": 88, + "registered": 89, "inReach": 44, - "outOfReach": 44 + "outOfReach": 45 }, "reasonTotals": { "managedBy: 'better-auth'": 25, - "systemFields.tenant: false": 19, + "systemFields.tenant: false": 20, "tenancy.enabled: false": 2 }, "unexplained": [], @@ -687,6 +687,15 @@ "managedBy: 'better-auth'" ] }, + { + "name": "sys_trusted_key", + "file": "packages/platform-objects/src/system/sys-trusted-key.object.ts", + "reach": "out", + "tenantField": null, + "reasons": [ + "systemFields.tenant: false" + ] + }, { "name": "sys_two_factor", "file": "packages/platform-objects/src/identity/sys-two-factor.object.ts", From a0a6437c8203881d4cdd98885b8ec95708c4af23 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 04:18:01 +0000 Subject: [PATCH 17/21] =?UTF-8?q?WIP(#27237):=20contract=20review=20r3=20?= =?UTF-8?q?=E2=80=94=20degrade=20wording,=20encoded-form=20scrub,=20walled?= =?UTF-8?q?=20install=20note?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- ...lace-install-local-credential-refs.test.ts | 12 +++ .../src/marketplace-install-local-plugin.ts | 33 ++++++-- .../src/connector-credential-door.test.ts | 75 ++++++++++++++++++- .../src/connector-credentials.ts | 70 +++++++++++++---- .../services/service-automation/src/plugin.ts | 13 +++- .../spec/src/contracts/automation-service.ts | 15 ++-- .../integration/connector-provider-errors.ts | 11 ++- .../spec/src/integration/connector.zod.ts | 2 +- .../spec/src/shared/connector-auth.zod.ts | 6 +- packages/spec/src/stack.zod.ts | 10 ++- .../evals/integrations-declared.json | 2 +- 11 files changed, 207 insertions(+), 42 deletions(-) diff --git a/packages/cloud-connection/src/marketplace-install-local-credential-refs.test.ts b/packages/cloud-connection/src/marketplace-install-local-credential-refs.test.ts index 5b2513d9126..7c8d3448d8e 100644 --- a/packages/cloud-connection/src/marketplace-install-local-credential-refs.test.ts +++ b/packages/cloud-connection/src/marketplace-install-local-credential-refs.test.ts @@ -117,6 +117,18 @@ describe('[#27237] the install answer names the connector credentials the packag expect(res.payload.data.credentialNote).not.toContain('BILLING_SET_REF'); }); + it('under a walled posture (the listing says nothing is writable) the note does not send anyone to the PUT route', async () => { + const listConnectorCredentialRefs = vi.fn(async () => [ + { ref: 'BILLING_MISSING_REF', status: 'missing', source: null, writable: false, declared: true, connectors: [] }, + ]); + const install = await bootPlugin({ listConnectorCredentialRefs }); + const res = await install([bearer('billing', 'BILLING_MISSING_REF')]); + expect(res.payload?.success, JSON.stringify(res.payload)).toBe(true); + expect(res.payload.data.credentialNote).toBe(installedCredentialNote(['BILLING_MISSING_REF'], { door: false })); + expect(res.payload.data.credentialNote).not.toContain('credential-refs'); + expect(res.payload.data.credentialNote).toContain('server environment variable of the same name'); + }); + it('with no automation service to ask, each reference is `unknown`, never assumed set', async () => { const install = await bootPlugin(); const res = await install([bearer('billing', 'BILLING_SET_REF')]); diff --git a/packages/cloud-connection/src/marketplace-install-local-plugin.ts b/packages/cloud-connection/src/marketplace-install-local-plugin.ts index 6b05aa79b5a..28d3ad5d066 100644 --- a/packages/cloud-connection/src/marketplace-install-local-plugin.ts +++ b/packages/cloud-connection/src/marketplace-install-local-plugin.ts @@ -748,14 +748,23 @@ export interface InstalledCredentialRef { source: ConnectorCredentialRef['source']; } -/** [#27237] The install answer's sentence for the credentials not set yet. */ -export function installedCredentialNote(refs: readonly string[]): string { +/** + * [#27237] The install answer's sentence for the credentials not set yet. + * `door: false` when the credential door cannot store any of them — the + * automation service's listing says none is `writable`, as under the `group` / + * `isolated` postures, where the door refuses — so the sentence does not send + * an administrator to a route that will refuse. + */ +export function installedCredentialNote(refs: readonly string[], opts: { door?: boolean } = {}): string { const list = refs.map((r) => `'${r}'`).join(', '); return ( `Connector credential${refs.length === 1 ? '' : 's'} ${list} ${refs.length === 1 ? 'needs' : 'need'} an ` + "administrator — not set, refused by the upstream, or unreadable now (each entry's `status` says which). The " + 'connector instances naming them are installed degraded and materialize once each is set: ' - + 'PUT /api/v1/automation/connectors/credential-refs/:ref, or a server environment variable of the same name.' + + (opts.door === false + ? "as a server environment variable of the same name, or in the host's credential resolver. This environment " + + 'does not store connector credentials through the credential door.' + : 'PUT /api/v1/automation/connectors/credential-refs/:ref, or a server environment variable of the same name.') ); } @@ -1982,8 +1991,12 @@ export class MarketplaceInstallLocalPlugin implements Plugin { // 7. [#27237] Which connector credentials the package names, and which // are not set yet — after the re-sync above, so the listing reads // what the reconcile it triggered saw. - const credentialRefs = await this.describeInstalledCredentialRefs(ctx, converted); + const described = await this.describeInstalledCredentialRefs(ctx, converted); + const credentialRefs = described?.entries; const unsetRefs = (credentialRefs ?? []).filter((r) => r.status !== 'set').map((r) => r.ref); + // The door is named only when it can store one of them: a listing that + // says none is writable (a walled posture) gets the other places. + const door = described?.writable === undefined || unsetRefs.some((r) => described.writable!.has(r)); return c.json({ success: true, @@ -2003,7 +2016,7 @@ export class MarketplaceInstallLocalPlugin implements Plugin { // [#27237] Omitted, not emptied, for a package whose connectors // name no credential, so that answer's key set is unchanged. ...(credentialRefs !== undefined ? { credentialRefs } : {}), - ...(unsetRefs.length > 0 ? { credentialNote: installedCredentialNote(unsetRefs) } : {}), + ...(unsetRefs.length > 0 ? { credentialNote: installedCredentialNote(unsetRefs, { door }) } : {}), upgradedFrom: conflict === 'marketplace' ? 'previous-marketplace-version' : null, translationsLoaded: seededSummary.translationsLoaded, seeded: seededSummary.seeded, @@ -3510,11 +3523,13 @@ export class MarketplaceInstallLocalPlugin implements Plugin { * naming it installs degraded and materializes when an administrator sets * it (`PUT /api/v1/automation/connectors/credential-refs/:ref`). With no * automation service, or one that cannot list, the status is `unknown`. + * `writable` is the references the credential door can store, as the + * listing says; `undefined` when there is no listing to say it. */ private describeInstalledCredentialRefs = async ( ctx: PluginContext, converted: unknown, - ): Promise => { + ): Promise<{ entries: InstalledCredentialRef[]; writable?: ReadonlySet } | undefined> => { const connectors = (converted as { connectors?: unknown } | null)?.connectors; const declared = new Map(); for (const raw of Array.isArray(connectors) ? connectors : []) { @@ -3537,7 +3552,7 @@ export class MarketplaceInstallLocalPlugin implements Plugin { + `each credential the package names as status 'unknown'. Cause: ${err?.message ?? err}`, ); } - return [...declared.keys()].sort().map((ref) => { + const entries = [...declared.keys()].sort().map((ref): InstalledCredentialRef => { const known = listing?.get(ref); return { ref, @@ -3546,6 +3561,10 @@ export class MarketplaceInstallLocalPlugin implements Plugin { source: known?.source ?? null, }; }); + const writable = listing === undefined + ? undefined + : new Set([...declared.keys()].filter((ref) => listing!.get(ref)?.writable !== false)); + return { entries, ...(writable ? { writable } : {}) }; }; /** diff --git a/packages/services/service-automation/src/connector-credential-door.test.ts b/packages/services/service-automation/src/connector-credential-door.test.ts index 37141ba1811..1d5a2ca5da2 100644 --- a/packages/services/service-automation/src/connector-credential-door.test.ts +++ b/packages/services/service-automation/src/connector-credential-door.test.ts @@ -45,7 +45,14 @@ import { AutomationServicePlugin, type CredentialResolver } from './plugin.js'; import type { AutomationEngine } from './engine.js'; import { SysConnectorCredential } from './sys-connector-credential.object.js'; import { SysFlowCredential } from './sys-flow-credential.object.js'; -import { CONNECTOR_CREDENTIAL_OBJECT, ConnectorCredentialStore } from './connector-credentials.js'; +import { + CONNECTOR_CREDENTIAL_OBJECT, + ConnectorCredentialStore, + credentialForms, + upstreamRefusalMessage, + withoutCredential, + withoutCredentialThrown, +} from './connector-credentials.js'; const REF = 'DUMMY_BILLING_TOKEN_27237'; const DUMMY_VALUE = 'dummy-value-aaaa-0001'; @@ -610,3 +617,69 @@ describe('[#27237] seat review r1: under a walled posture the store is not in th expect((await engine.listConnectorCredentialRefs())[0]).toMatchObject({ source: 'env', status: 'set', writable: false }); }); }); + +describe('[#27237] contract review: an encoded spelling of the value is withheld too', () => { + const USERNAME = 'svc_dummy_27237'; + // Characters that URL-encode, so the encoded spelling differs from the raw one. + const ENCODABLE = 'dummy/value+eeee 0010'; + const b64 = (text: string) => Buffer.from(text, 'utf8').toString('base64'); + + it('lists the raw, URL-encoded and base64 spellings, and base64(`user:value`) for basic auth', () => { + const forms = credentialForms(ENCODABLE, USERNAME); + expect(forms).toContain(ENCODABLE); + expect(forms).toContain(encodeURIComponent(ENCODABLE)); + expect(forms).toContain(b64(ENCODABLE).replace(/=+$/, '')); + expect(forms).toContain(b64(`${USERNAME}:${ENCODABLE}`).replace(/=+$/, '')); + const echoed = `Authorization: Basic ${b64(`${USERNAME}:${ENCODABLE}`)} url ?key=${encodeURIComponent(ENCODABLE)}`; + const scrubbed = withoutCredential(echoed, forms); + for (const form of forms) expect(scrubbed.includes(form), form.length.toString()).toBe(false); + expect(scrubbed).toContain('[credential withheld]'); + const thrown = withoutCredentialThrown(Object.assign(new Error('outer'), { cause: new Error(echoed) }), forms) as Error; + expect(String(thrown.message)).toBe('outer'); + expect((thrown as { cause?: unknown }).cause).toBeUndefined(); + expect(upstreamRefusalMessage({ message: echoed }, forms)).not.toContain(encodeURIComponent(ENCODABLE)); + }); + + it('a basic-auth instance whose upstream echoes the encoded header and URL: no spelling reaches the reason, the row or a log', async () => { + const out = captureOutput(); + const header = b64(`${USERNAME}:${ENCODABLE}`); + const provider = fakeProvider(); + const factory: ConnectorProviderFactory = (ctx) => { + provider.state.materialized.push(ctx); + const password = ctx.auth?.type === 'basic' ? ctx.auth.password : ''; + const echo = `sent Authorization: Basic ${b64(`${USERNAME}:${password}`)} to https://erp.example/?key=${encodeURIComponent(password)}`; + if (provider.state.rejectAtMaterialization) throw new ConnectorCredentialRejectedError(`refused (HTTP 401): ${echo}`, 401); + return { + def: { name: ctx.name, label: ctx.label, type: 'api', actions: [{ key: 'request', label: 'Request' }] } as unknown as Connector, + handlers: { request: async () => ({ status: 401, ok: false, body: { message: echo } }) }, + }; + }; + const erp = { ...billing('erp'), auth: { type: 'basic', username: USERNAME, credentialRef: REF } }; + const { engine, call } = await boot([erp], { provider: { ...provider, factory } }); + + // On a call (the rest / openapi shape). + await engine.setConnectorCredential({ ref: REF, value: ENCODABLE, context: ADMIN }); + expect(await call('erp')).toMatchObject({ status: 401 }); + const onCall = engine.getConnectorDegradedReason('erp') ?? ''; + expect(onCall).toContain('[credential withheld]'); + const listedOnCall = JSON.stringify(await engine.listConnectorCredentialRefs()); + + // At materialization (the mcp shape). + provider.state.rejectAtMaterialization = true; + await engine.setConnectorCredential({ ref: REF, value: DUMMY_VALUE_2, context: ADMIN }); + const atMaterialization = engine.getConnectorDegradedReason('erp') ?? ''; + expect(atMaterialization).toContain('[credential withheld]'); + + for (const [where, text] of [ + ['reason on a call', onCall], + ['listing after the call', listedOnCall], + ['reason at materialization', atMaterialization], + ['listing after materialization', JSON.stringify(await engine.listConnectorCredentialRefs())], + ['logs', out.text()], + ] as const) { + for (const form of [header, encodeURIComponent(ENCODABLE), ENCODABLE, b64(`${USERNAME}:${DUMMY_VALUE_2}`).replace(/=+$/, '')]) { + expect(text.includes(form), `${where}: a spelling of the value`).toBe(false); + } + } + }); +}); diff --git a/packages/services/service-automation/src/connector-credentials.ts b/packages/services/service-automation/src/connector-credentials.ts index 57b5f148c08..f536fcd8706 100644 --- a/packages/services/service-automation/src/connector-credentials.ts +++ b/packages/services/service-automation/src/connector-credentials.ts @@ -309,7 +309,7 @@ export class ConnectorCredentialStore { ); } // Whatever else refused the write answers without the value it was handed. - throw withoutCredentialThrown(err, args.value); + throw withoutCredentialThrown(err, credentialForms(args.value)); } return { created: !existing, revision }; } @@ -384,32 +384,72 @@ export class ConnectorCredentialStore { } } +/** What a scrub withholds: one value, or every spelling of it ({@link credentialForms}). */ +export type WithheldCredential = string | readonly string[] | undefined; + +/** The shortest spelling a scrub withholds — a shorter one would blank ordinary words. */ +const MIN_WITHHELD_LENGTH = 4; + +function base64Of(text: string): string { + const bytes = new TextEncoder().encode(text); + let binary = ''; + for (const byte of bytes) binary += String.fromCharCode(byte); + return btoa(binary); +} + +/** + * Every spelling of `value` an upstream or a client library may echo back: + * the value itself; URL-encoded, as an api-key sent as a query parameter + * travels in the request URL; base64; and, given the `basic` `username`, + * base64(`username:value`), the `Authorization: Basic` header. A base64 form + * is kept without its `=` padding, so an echo that drops the padding is + * withheld too. Longest first, so a longer form is replaced before a shorter + * one inside it. + */ +export function credentialForms(value: string | undefined, username?: string): string[] { + if (!value) return []; + const forms = new Set([value, encodeURIComponent(value), base64Of(value).replace(/=+$/, '')]); + if (username !== undefined) forms.add(base64Of(`${username}:${value}`).replace(/=+$/, '')); + return [...forms].filter((f) => f.length >= MIN_WITHHELD_LENGTH).sort((a, b) => b.length - a.length); +} + +function withheldForms(secret: WithheldCredential): readonly string[] { + if (secret === undefined) return []; + if (typeof secret === 'string') return secret.length >= MIN_WITHHELD_LENGTH ? [secret] : []; + return secret.filter((f) => f.length >= MIN_WITHHELD_LENGTH); +} + /** - * `message` with every occurrence of `secret` replaced. A provider's message - * is foreign text; an upstream that echoes the credential back must not carry - * it into a log line, a husk's reason or a stored status. + * `message` with every occurrence of `secret` replaced — one value, or every + * spelling {@link credentialForms} lists. A provider's message is foreign + * text; an upstream that echoes the credential back, in any of those + * spellings, must not carry it into a log line, a husk's reason or a stored + * status. */ -export function withoutCredential(message: string, secret: string | undefined): string { - if (!secret || secret.length < 4) return message; - return message.split(secret).join('[credential withheld]'); +export function withoutCredential(message: string, secret: WithheldCredential): string { + let out = message; + for (const form of withheldForms(secret)) out = out.split(form).join('[credential withheld]'); + return out; } /** * `err` with `secret` withheld from its message, keeping the structural marks - * a caller branches on (`code`, `status`, `name`). Returned unchanged when the - * secret appears nowhere in its message chain; otherwise rebuilt without its - * stack and cause, which carry the same text. + * a caller branches on (`code`, `status`, `name`). Returned unchanged when no + * spelling of the secret appears in its message chain; otherwise rebuilt + * without its stack and cause, which carry the same text. */ -export function withoutCredentialThrown(err: unknown, secret: string | undefined): unknown { - if (!secret || secret.length < 4 || !(err instanceof Error)) return err; +export function withoutCredentialThrown(err: unknown, secret: WithheldCredential): unknown { + const forms = withheldForms(secret); + if (forms.length === 0 || !(err instanceof Error)) return err; let leaks = false; let cur: unknown = err; for (let depth = 0; depth < 8 && cur instanceof Error; depth++) { - if (cur.message.includes(secret) || (cur.stack ?? '').includes(secret)) leaks = true; + const text = `${cur.message}\n${cur.stack ?? ''}`; + if (forms.some((f) => text.includes(f))) leaks = true; cur = (cur as { cause?: unknown }).cause; } if (!leaks) return err; - const scrubbed = new Error(withoutCredential(err.message, secret)); + const scrubbed = new Error(withoutCredential(err.message, forms)); scrubbed.name = err.name; scrubbed.stack = undefined; const marks = err as { code?: unknown; status?: unknown }; @@ -562,7 +602,7 @@ export const UPSTREAM_REFUSAL_MESSAGE_MAX_LENGTH = 500; * `secret` withheld BEFORE the bound is applied, so a cut can never leave part * of it behind. */ -export function upstreamRefusalMessage(body: unknown, secret: string | undefined): string | undefined { +export function upstreamRefusalMessage(body: unknown, secret: WithheldCredential): string | undefined { let text: unknown; if (typeof body === 'string') { text = body; diff --git a/packages/services/service-automation/src/plugin.ts b/packages/services/service-automation/src/plugin.ts index ea7be26b11a..a827d780254 100644 --- a/packages/services/service-automation/src/plugin.ts +++ b/packages/services/service-automation/src/plugin.ts @@ -40,6 +40,7 @@ import { SECRET_HANDLE_PREFIX, connectorCredentialFingerprint, credentialDoorRefusal, + credentialForms, credentialRejectedOnCallReason, credentialRejectionStatusOf, isAcceptedCallResult, @@ -389,6 +390,11 @@ function credentialRefOf(auth: ConnectorInstanceAuth | undefined): string | unde return auth.credentialRef; } +/** The `basic` username an instance sends beside its secret — what its `Authorization` header encodes with it. */ +function basicUsernameOf(auth: ConnectorInstanceAuth | undefined): string | undefined { + return auth?.type === 'basic' ? auth.username : undefined; +} + /** * Build the static {@link ResolvedConnectorAuth} a provider factory applies * from a declarative instance's `auth` and the secret its `credentialRef` @@ -2275,7 +2281,7 @@ export class AutomationServicePlugin implements Plugin { // [#27237] A provider's message is foreign text: an upstream that // echoes the credential back must not carry it into the husk's // reason, a log record or the thrown boot error. - const thrown = withoutCredentialThrown(err, credential?.value); + const thrown = withoutCredentialThrown(err, credentialForms(credential?.value, basicUsernameOf(entry.auth))); // [#27237] The upstream refused the credential (ruling on // #27237, comment 6470087293): degrade the instance and mark a // stored credential `rejected` — never a failed boot, never a @@ -3036,7 +3042,10 @@ export class AutomationServicePlugin implements Plugin { info.name, info.ref, status, - upstreamRefusalMessage((result as { body?: unknown }).body, info.credential.value), + upstreamRefusalMessage( + (result as { body?: unknown }).body, + credentialForms(info.credential.value, basicUsernameOf(info.entry.auth)), + ), info.credential.source === 'object', ); try { diff --git a/packages/spec/src/contracts/automation-service.ts b/packages/spec/src/contracts/automation-service.ts index e6e561a9d2d..9ce81313967 100644 --- a/packages/spec/src/contracts/automation-service.ts +++ b/packages/spec/src/contracts/automation-service.ts @@ -1316,10 +1316,14 @@ export interface IAutomationService { * store and re-materialize every instance naming it, without a restart; * resolves to the reference's listing entry after that reconcile. Rejects * with an ADR-0112 `code` + `status`, before anything is written: a - * reference no installed package declares (`404 RESOURCE_NOT_FOUND`), one - * an environment variable supplies (`409 RESOURCE_CONFLICT` — env wins, so - * a stored value would never be read), an empty or oversized reference or - * value (`400 VALIDATION_ERROR`), a deployment that cannot encrypt it + * walled tenancy posture, `group` or `isolated`, where the store is not + * used (`403 PERMISSION_DENIED`, naming the posture); a reference no + * installed package declares (`404 RESOURCE_NOT_FOUND`); one an environment + * variable supplies (`409 RESOURCE_CONFLICT` — env wins, so a stored value + * would never be read), or a first write another replica made at the same + * moment (`409`); an empty or oversized reference or value, or a value + * starting with `secret:` (`400 VALIDATION_ERROR`); a deployment that + * cannot encrypt it, or whose tenancy posture cannot be read * (`503 SERVICE_UNAVAILABLE`). */ setConnectorCredential?(request: ConnectorCredentialSetRequest): Promise; @@ -1328,7 +1332,8 @@ export interface IAutomationService { * Remove the stored value for one reference; every instance naming it is * torn down and registered degraded, naming the reference. Resolves to the * reference's listing entry after that reconcile. Rejects `404 - * RESOURCE_NOT_FOUND` when nothing is stored for it. + * RESOURCE_NOT_FOUND` when nothing is stored for it, and `403 + * PERMISSION_DENIED` under a walled tenancy posture, as a set does. */ clearConnectorCredential?(request: ConnectorCredentialClearRequest): Promise; diff --git a/packages/spec/src/integration/connector-provider-errors.ts b/packages/spec/src/integration/connector-provider-errors.ts index 5b06c55d066..ca39a09c5b0 100644 --- a/packages/spec/src/integration/connector-provider-errors.ts +++ b/packages/spec/src/integration/connector-provider-errors.ts @@ -71,10 +71,13 @@ export function isConnectorUpstreamUnavailable(err: unknown): boolean { * instance is well-formed and its `credentialRef` resolved) nor an operational * one (the upstream answered, and retrying the same value cannot change its * answer). The materializer **degrades** the instance naming the refusal, as it - * does for an upstream outage, but does not retry it on a timer: the next - * attempt is the next change of the credential. When the value came from the - * credential store, its row is marked `rejected` with the upstream's message — - * never the value. + * does for an upstream outage. When the value is a stored one (the credential + * store) or written in the declaration, it does not retry it on a timer: the + * next attempt is the next change of the credential or the declaration, or a + * restart, and a stored value's row is marked `rejected` with the upstream's + * message — never the value. A value from an environment variable or the + * host's resolver can change where the process cannot see it, so its instance + * is retried on the backoff. */ export const CONNECTOR_CREDENTIAL_REJECTED = 'CONNECTOR_CREDENTIAL_REJECTED' as const; diff --git a/packages/spec/src/integration/connector.zod.ts b/packages/spec/src/integration/connector.zod.ts index 43663c9e629..2cff38efb59 100644 --- a/packages/spec/src/integration/connector.zod.ts +++ b/packages/spec/src/integration/connector.zod.ts @@ -904,7 +904,7 @@ const ConnectorBaseSchema = lazySchema(() => z.object({ * secret inline. Requires `provider`. */ auth: ConnectorInstanceAuthSchema.optional().describe( - 'Declarative instance auth — references credentials via `credentialRef` (resolved at boot), never inline secrets. Requires `provider` (ADR-0097).', + 'Declarative instance auth — references credentials via `credentialRef` (resolved when the instance materializes), never inline secrets. Requires `provider` (ADR-0097).', ), /** Zapier-style Capabilities */ diff --git a/packages/spec/src/shared/connector-auth.zod.ts b/packages/spec/src/shared/connector-auth.zod.ts index 747bb855f3e..349e3d60612 100644 --- a/packages/spec/src/shared/connector-auth.zod.ts +++ b/packages/spec/src/shared/connector-auth.zod.ts @@ -120,7 +120,7 @@ export const ConnectorInstanceNoAuthSchema = lazySchema(() => z.object({ })); export type ConnectorInstanceNoAuth = z.input; -/** Bearer-token auth; the token is resolved from `credentialRef` at boot. */ +/** Bearer-token auth; the token is resolved from `credentialRef` when the instance materializes. */ export const ConnectorInstanceBearerAuthSchema = lazySchema(() => z.object({ type: z.literal('bearer'), credentialRef: z.string().min(1).describe( @@ -129,7 +129,7 @@ export const ConnectorInstanceBearerAuthSchema = lazySchema(() => z.object({ })); export type ConnectorInstanceBearerAuth = z.input; -/** API-key auth; the key is resolved from `credentialRef` at boot. */ +/** API-key auth; the key is resolved from `credentialRef` when the instance materializes. */ export const ConnectorInstanceAPIKeyAuthSchema = lazySchema(() => z.object({ type: z.literal('api-key'), credentialRef: z.string().min(1).describe( @@ -140,7 +140,7 @@ export const ConnectorInstanceAPIKeyAuthSchema = lazySchema(() => z.object({ })); export type ConnectorInstanceAPIKeyAuth = z.input; -/** Basic auth; the password is resolved from `credentialRef` at boot. */ +/** Basic auth; the password is resolved from `credentialRef` when the instance materializes. */ export const ConnectorInstanceBasicAuthSchema = lazySchema(() => z.object({ type: z.literal('basic'), username: z.string().describe('Username (not a secret; safe to keep in metadata).'), diff --git a/packages/spec/src/stack.zod.ts b/packages/spec/src/stack.zod.ts index 941bcb193e5..36d5974b451 100644 --- a/packages/spec/src/stack.zod.ts +++ b/packages/spec/src/stack.zod.ts @@ -1103,11 +1103,14 @@ const STACK_DEFINITION_COLLECTIONS_SHAPE = { * connector authored as pure metadata. At boot the automation service looks * up the installed generic executor named by `provider` (`openapi` / `mcp` / * `rest`, contributed by the matching plugin in `plugins:`), resolves - * `auth.credentialRef` through the secrets/env layer, and registers the + * `auth.credentialRef` (an environment variable, the environment's + * connector credential store, or the host's resolver), and registers the * materialized `{ def, handlers }` on the connector registry — so * `connector_action` dispatches it and `GET /connectors` lists it, exactly * like a hand-written connector. A declared `provider` with no installed - * factory is a **hard boot error**. + * factory is a **hard boot error**. A `credentialRef` nothing resolves, or + * one the upstream refuses, degrades that one instance until the + * credential is set; the boot goes on. * * 2. **Catalog descriptor** (no `provider`, the #2612 interim contract): an * inert metadata entry for discovery / documentation / marketplace listing. @@ -1125,7 +1128,8 @@ const STACK_DEFINITION_COLLECTIONS_SHAPE = { 'External System Connectors. A provider-bound entry (has `provider`: openapi/mcp/rest) is materialized into a ' + 'live, dispatchable connector at boot and referenced by flows via `connector_action`; credentials are `auth.credentialRef` ' + 'references, never inline secrets. An entry with no `provider` is a catalog descriptor only (NOT dispatchable) — set ' + - '`enabled: false` on deliberate descriptors. Unknown provider / unresolvable credentialRef / name conflict ⇒ hard boot error (ADR-0097).', + '`enabled: false` on deliberate descriptors. Unknown provider / name conflict ⇒ hard boot error; a credentialRef nothing resolves, or one ' + + 'the upstream refuses, degrades that instance until the credential is set (ADR-0097).', ), /** diff --git a/skills/objectstack-automation/evals/integrations-declared.json b/skills/objectstack-automation/evals/integrations-declared.json index cfdbf2704a9..167e8c3ff41 100644 --- a/skills/objectstack-automation/evals/integrations-declared.json +++ b/skills/objectstack-automation/evals/integrations-declared.json @@ -14,7 +14,7 @@ { "id": 2, "prompt": "Every night, read our Stripe account balance and write the available amount and the sync time onto our own company's `crm_account` row (it is flagged `is_house_account`). The Stripe secret key must not appear in metadata.", - "expected_output": "A `connectors:` entry `defineConnector({ provider: 'openapi', providerConfig: { spec: './…/stripe.trimmed.json' }, auth: { type: 'bearer', credentialRef: 'STRIPE_SECRET_KEY' } })` over a trimmed, packaged OpenAPI 3.x document holding `GetBalance`; the administrator sets `STRIPE_SECRET_KEY` on the deployment, an unset ref fails boot. A `type: 'schedule'` flow with the cadence on the start node's `config.schedule`, `runAs: 'system'`, a `connector_action` with `actionId: 'GetBalance'` and a literal `input: {}`, then an `update_record` on `crm_account` with `filter: { is_house_account: true }` and `fields` read from the node's answer via CEL (`bal.body.available[0].amount / 100.0`, `isoDatetime(now())`). `requires` lists `automation`, `triggers`, `job`. No `http` node with the key in a header, no plugin.", + "expected_output": "A `connectors:` entry `defineConnector({ provider: 'openapi', providerConfig: { spec: './…/stripe.trimmed.json' }, auth: { type: 'bearer', credentialRef: 'STRIPE_SECRET_KEY' } })` over a trimmed, packaged OpenAPI 3.x document holding `GetBalance`; the administrator sets `STRIPE_SECRET_KEY` (an environment variable of that name, or `PUT /api/v1/automation/connectors/credential-refs/STRIPE_SECRET_KEY`); until it is set the instance is degraded, and the app still boots. A `type: 'schedule'` flow with the cadence on the start node's `config.schedule`, `runAs: 'system'`, a `connector_action` with `actionId: 'GetBalance'` and a literal `input: {}`, then an `update_record` on `crm_account` with `filter: { is_house_account: true }` and `fields` read from the node's answer via CEL (`bal.body.available[0].amount / 100.0`, `isoDatetime(now())`). `requires` lists `automation`, `triggers`, `job`. No `http` node with the key in a header, no plugin.", "files": [], "assertions": { "must_contain": ["provider: 'openapi'", "credentialRef", "type: 'schedule'", "runAs: 'system'", "connector_action", "update_record", "dialect: 'cel'"], From aff9d4f4137604ff0449b6a687e7e25427f52ed0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 04:21:44 +0000 Subject: [PATCH 18/21] =?UTF-8?q?WIP(#27237):=20contract=20review=20r3=20?= =?UTF-8?q?=E2=80=94=20regenerate=20the=20spec=20reference=20pages,=20chan?= =?UTF-8?q?gesets,=20stale=20test=20header?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- .changeset/27237-connector-credential-rejected-marker.md | 2 ++ .changeset/27237-connector-credential-store.md | 2 +- .changeset/27237-install-local-credential-refs.md | 2 +- content/docs/references/api/package-api-assembled.mdx | 4 ++-- content/docs/references/integration/connector.mdx | 4 ++-- .../service-automation/src/connector-materialization.test.ts | 4 ++-- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.changeset/27237-connector-credential-rejected-marker.md b/.changeset/27237-connector-credential-rejected-marker.md index 0242417762a..70a40fc67ef 100644 --- a/.changeset/27237-connector-credential-rejected-marker.md +++ b/.changeset/27237-connector-credential-rejected-marker.md @@ -14,3 +14,5 @@ Three additions for the connector credential store (#27237): - `IAutomationService` gains three optional members, `listConnectorCredentialRefs`, `setConnectorCredential` and `clearConnectorCredential`, with their types `ConnectorCredentialRef`, `ConnectorCredentialSetRequest` and `ConnectorCredentialClearRequest`. A listing entry never carries a value. `PLATFORM_OBJECT_NAMES['service-automation']` lists the new `sys_connector_credential` object. + +The descriptions now match the new behaviour. The stack's `connectors` field says a `credentialRef` nothing resolves, or one the upstream refuses, degrades that instance until the credential is set, where it said such a reference failed the boot. The auth and connector schemas say a credential is resolved when the instance materializes, not at boot. diff --git a/.changeset/27237-connector-credential-store.md b/.changeset/27237-connector-credential-store.md index a057ebab270..14cbbe4736c 100644 --- a/.changeset/27237-connector-credential-store.md +++ b/.changeset/27237-connector-credential-store.md @@ -16,6 +16,6 @@ The reference now resolves through three sources, in order: an environment varia - A credential the upstream refuses (HTTP 401 / 403) degrades the instance with the upstream's message, never the value. An `mcp` instance is refused when it connects; a `rest` or `openapi` instance on the first call that answers 401 / 403, which still returns its answer. A stored credential is marked `rejected`, and its instance is not retried until the credential is set again, the declaration changes, or the server restarts; the mark itself stays until the value is set again or an `mcp` handshake proves it. A call answered 2xx does not clear it, since the endpoint may not check the credential; it only stamps the last-verified time of a credential still `set`. A credential from an environment variable or the host's resolver is retried on the usual backoff instead, since it can change where the service cannot see it, and the listing reports it `rejected` until a provider proves it or the server restarts. - Under the `group` and `isolated` tenancy postures the store is not read, and writes to it are refused with a 403 naming the posture. - A value starting with `secret:` is refused with a 400: the data engine would store it as a handle to another stored secret. Writes run one at a time with their reconcile, so the instances end on the value the store ends on; a first write another replica makes at the same moment answers 409. -- The value never appears in a log record, the listing, `GET /connectors`, an error body or a read of the object. A provider message that echoes it carries `[credential withheld]` instead. +- The value never appears in a log record, the listing, `GET /connectors`, an error body or a read of the object. A provider message that echoes it carries `[credential withheld]` instead, and so does one that echoes it percent-encoded or base64-encoded, including the base64 `user:value` pair of a basic credential. A deployment that relied on the boot failing for an unset reference now boots, with that instance degraded and named on `GET /api/v1/automation/connectors`. diff --git a/.changeset/27237-install-local-credential-refs.md b/.changeset/27237-install-local-credential-refs.md index d7f24a02125..af52ef77f42 100644 --- a/.changeset/27237-install-local-credential-refs.md +++ b/.changeset/27237-install-local-credential-refs.md @@ -7,4 +7,4 @@ feat(cloud-connection): the install-local answer lists the connector credentials Clause-②: yes (widening) Metadata-protocol: none -`POST /api/v1/marketplace/install-local` answers with `credentialRefs` when the package's connectors name any `auth.credentialRef`. Each entry has the reference, the instances naming it, its `status` from the automation service's credential listing (`set` / `missing` / `rejected`, or `unknown` when that listing cannot be read) and its `source`, never a value. When any of them is not set, a `credentialNote` names them and says how to set them. An unset reference never refuses the install: the instance naming it installs degraded and materializes once the credential is set (#27237). The answer's keys are unchanged for a package whose connectors name no credential. +`POST /api/v1/marketplace/install-local` answers with `credentialRefs` when the package's connectors name any `auth.credentialRef`. Each entry has the reference, the instances naming it, its `status` from the automation service's credential listing (`set` / `missing` / `rejected`, or `unknown` when that listing cannot be read) and its `source`, never a value. When any of them is not set, a `credentialNote` names them and says how to set them. An unset reference never refuses the install: the instance naming it installs degraded and materializes once the credential is set (#27237). Under the `group` and `isolated` tenancy postures, where the credential routes refuse writes, the note names the environment variable or the host's resolver instead of the `PUT` route. The answer's keys are unchanged for a package whose connectors name no credential. diff --git a/content/docs/references/api/package-api-assembled.mdx b/content/docs/references/api/package-api-assembled.mdx index 7b9fbe81d3d..44cc0a0c4e4 100644 --- a/content/docs/references/api/package-api-assembled.mdx +++ b/content/docs/references/api/package-api-assembled.mdx @@ -140,7 +140,7 @@ Installed package row whose manifest is the assembled package body | **hooks** | `{ name: string; label?: string; object: string \| string[]; events: Enum<'beforeFind' \| 'afterFind' \| 'beforeInsert' \| 'afterInsert' \| 'beforeUpdate' \| …>[]; … }[]` | optional | Object Lifecycle Hooks, as a JSON document carries them | | **mappings** | `{ name: string; label?: string; sourceFormat?: Enum<'csv' \| 'json' \| 'xml' \| 'sql'>; targetObject: string; … }[]` | optional | Data Import/Export Mappings | | **analyticsCubes** | `{ name: string; title?: string; description?: string; sql: string; … }[]` | optional | Analytics Semantic Layer Cubes | -| **connectors** | `{ name: string; label: string; type: Enum<'saas' \| 'database' \| 'file_storage' \| 'message_queue' \| 'api' \| 'custom'>; description?: string; … }[]` | optional | External System Connectors. A provider-bound entry (has `provider`: openapi/mcp/rest) is materialized into a live, dispatchable connector at boot and referenced by flows via `connector_action`; credentials are `auth.credentialRef` references, never inline secrets. An entry with no `provider` is a catalog descriptor only (NOT dispatchable) — set `enabled: false` on deliberate descriptors. Unknown provider / unresolvable credentialRef / name conflict ⇒ hard boot error (ADR-0097). | +| **connectors** | `{ name: string; label: string; type: Enum<'saas' \| 'database' \| 'file_storage' \| 'message_queue' \| 'api' \| 'custom'>; description?: string; … }[]` | optional | External System Connectors. A provider-bound entry (has `provider`: openapi/mcp/rest) is materialized into a live, dispatchable connector at boot and referenced by flows via `connector_action`; credentials are `auth.credentialRef` references, never inline secrets. An entry with no `provider` is a catalog descriptor only (NOT dispatchable) — set `enabled: false` on deliberate descriptors. Unknown provider / name conflict ⇒ hard boot error; a credentialRef nothing resolves, or one the upstream refuses, degrades that instance until the credential is set (ADR-0097). | | **requires** | `string[]` | optional | Capability names this stack requires from the platform (canonical kebab-case tokens from PLATFORM_CAPABILITY_TOKENS; an unknown token is a defineStack error, declared-but-missing ⇒ fail-fast at startup) | | **tiers** | `string[]` | optional | Plugin tier presets to enable; overrides --preset | @@ -386,7 +386,7 @@ This schema accepts one of the following structures: | **hooks** | `{ name: string; label?: string; object: string \| string[]; events: Enum<'beforeFind' \| 'afterFind' \| 'beforeInsert' \| 'afterInsert' \| 'beforeUpdate' \| …>[]; … }[]` | optional | Object Lifecycle Hooks, as a JSON document carries them | | **mappings** | `{ name: string; label?: string; sourceFormat?: Enum<'csv' \| 'json' \| 'xml' \| 'sql'>; targetObject: string; … }[]` | optional | Data Import/Export Mappings | | **analyticsCubes** | `{ name: string; title?: string; description?: string; sql: string; … }[]` | optional | Analytics Semantic Layer Cubes | -| **connectors** | `{ name: string; label: string; type: Enum<'saas' \| 'database' \| 'file_storage' \| 'message_queue' \| 'api' \| 'custom'>; description?: string; … }[]` | optional | External System Connectors. A provider-bound entry (has `provider`: openapi/mcp/rest) is materialized into a live, dispatchable connector at boot and referenced by flows via `connector_action`; credentials are `auth.credentialRef` references, never inline secrets. An entry with no `provider` is a catalog descriptor only (NOT dispatchable) — set `enabled: false` on deliberate descriptors. Unknown provider / unresolvable credentialRef / name conflict ⇒ hard boot error (ADR-0097). | +| **connectors** | `{ name: string; label: string; type: Enum<'saas' \| 'database' \| 'file_storage' \| 'message_queue' \| 'api' \| 'custom'>; description?: string; … }[]` | optional | External System Connectors. A provider-bound entry (has `provider`: openapi/mcp/rest) is materialized into a live, dispatchable connector at boot and referenced by flows via `connector_action`; credentials are `auth.credentialRef` references, never inline secrets. An entry with no `provider` is a catalog descriptor only (NOT dispatchable) — set `enabled: false` on deliberate descriptors. Unknown provider / name conflict ⇒ hard boot error; a credentialRef nothing resolves, or one the upstream refuses, degrades that instance until the credential is set (ADR-0097). | | **requires** | `string[]` | optional | Capability names this stack requires from the platform (canonical kebab-case tokens from PLATFORM_CAPABILITY_TOKENS; an unknown token is a defineStack error, declared-but-missing ⇒ fail-fast at startup) | | **tiers** | `string[]` | optional | Plugin tier presets to enable; overrides --preset | diff --git a/content/docs/references/integration/connector.mdx b/content/docs/references/integration/connector.mdx index 29ada7d1831..3204666ec0f 100644 --- a/content/docs/references/integration/connector.mdx +++ b/content/docs/references/integration/connector.mdx @@ -188,7 +188,7 @@ const result = ConnectorSchema.parse(data); | **authentication** | `{ type: 'oauth2'; authorizationUrl: string; tokenUrl: string; clientId: string; … } \| { type: 'api-key'; key: string; headerName?: string; paramName?: string } \| { type: 'basic'; username: string; password: string } \| { type: 'bearer'; token: string } \| { type: 'none' }` | optional (default: `{"type":"none"}`) | Authentication configuration (runtime shape with inline secrets — plugin-supplied at registerConnector). Authored entries must not inline secrets: use `auth.credentialRef` on a provider-bound instance. | | **provider** | `string` | optional | Generic-executor key that materializes this declarative entry at boot (e.g. openapi/mcp/rest). Omit for a catalog-only descriptor. Unknown provider ⇒ hard boot error (ADR-0097). | | **providerConfig** | `Record` | optional | Provider-specific config validated by the provider factory at boot (e.g. `{ spec, baseUrl }` for openapi, where spec is an inline document, a package-relative file path like './billing-openapi.json', or an http(s) URL). Requires `provider`. | -| **auth** | `{ type: 'none' } \| { type: 'bearer'; credentialRef: string } \| { type: 'api-key'; credentialRef: string; headerName?: string; paramName?: string } \| { type: 'basic'; username: string; credentialRef: string }` | optional | Declarative instance auth — references credentials via `credentialRef` (resolved at boot), never inline secrets. Requires `provider` (ADR-0097). | +| **auth** | `{ type: 'none' } \| { type: 'bearer'; credentialRef: string } \| { type: 'api-key'; credentialRef: string; headerName?: string; paramName?: string } \| { type: 'basic'; username: string; credentialRef: string }` | optional | Declarative instance auth — references credentials via `credentialRef` (resolved when the instance materializes), never inline secrets. Requires `provider` (ADR-0097). | | **actions** | `{ key: string; label: string; description?: string; inputSchema?: Record; … }[]` | optional | | | **triggers** | `never` | optional | [REMOVED] `connector.triggers` was removed in @objectstack/spec 17 (ADR-0049 enforce-or-remove) — a connector trigger never started anything: `AutomationEngine.registerConnector` registers a connector's actions only, no polling loop read `intervalSeconds` (or the `interval` spelling it was renamed from), and no receiver was driven by a `webhook` trigger. Delete the key; the `ConnectorTrigger` shape leaves with it. To start work from an external system, write a flow that calls the connector's action in a `connector_action` node: for an external event, an `api` flow that the event's sender calls; for a scheduled pull, a `schedule` flow. Run `os migrate meta --from 17` to list the mechanical edits for existing sources; `--write` applies the ones it can prove, and you apply the rest by hand. | | **syncConfig** | `never` | optional | [REMOVED] `connector.syncConfig` was removed in @objectstack/spec 17 (ADR-0049 enforce-or-remove) — no engine ever ran a connector-attached sync: nothing read `strategy`, `direction`, `realtimeSync`, `timestampField`, `conflictResolution`, `batchSize`, `deleteMode` or `filters`, so the `latest_wins` and `soft_delete` defaults resolved and deleted nothing. Delete the key; the `DataSyncConfig` shape leaves with it. A sync is defined on its TARGET: a `mapping` (`targetObject`, `fieldMapping`, `mode`, `upsertKey`) whose `connectorSource` names the `rest` or `openapi` connector it pulls from, the read action and an optional timestamp `watermark`, with a `job` for the cadence. Its pull runs when a `job` drives it — a `job` whose `pull: { mapping }` names that mapping; the binding alone moves no rows. Run `os migrate meta --from 17` to list the mechanical edits for existing sources; `--write` applies the ones it can prove, and you apply the rest by hand. | @@ -485,7 +485,7 @@ Connector type | **authentication** | `{ type: 'oauth2'; authorizationUrl: string; tokenUrl: string; clientId: string; … } \| { type: 'api-key'; key: string; headerName?: string; paramName?: string } \| { type: 'basic'; username: string; password: string } \| { type: 'bearer'; token: string } \| { type: 'none' }` | optional (default: `{"type":"none"}`) | Authentication configuration (runtime shape with inline secrets — plugin-supplied at registerConnector). Authored entries must not inline secrets: use `auth.credentialRef` on a provider-bound instance. | | **provider** | `string` | optional | Generic-executor key that materializes this declarative entry at boot (e.g. openapi/mcp/rest). Omit for a catalog-only descriptor. Unknown provider ⇒ hard boot error (ADR-0097). | | **providerConfig** | `Record` | optional | Provider-specific config validated by the provider factory at boot (e.g. `{ spec, baseUrl }` for openapi, where spec is an inline document, a package-relative file path like './billing-openapi.json', or an http(s) URL). Requires `provider`. | -| **auth** | `{ type: 'none' } \| { type: 'bearer'; credentialRef: string } \| { type: 'api-key'; credentialRef: string; headerName?: string; paramName?: string } \| { type: 'basic'; username: string; credentialRef: string }` | optional | Declarative instance auth — references credentials via `credentialRef` (resolved at boot), never inline secrets. Requires `provider` (ADR-0097). | +| **auth** | `{ type: 'none' } \| { type: 'bearer'; credentialRef: string } \| { type: 'api-key'; credentialRef: string; headerName?: string; paramName?: string } \| { type: 'basic'; username: string; credentialRef: string }` | optional | Declarative instance auth — references credentials via `credentialRef` (resolved when the instance materializes), never inline secrets. Requires `provider` (ADR-0097). | | **actions** | `{ key: string; label: string; description?: string; inputSchema?: Record; … }[]` | optional | | | **triggers** | `never` | optional | [REMOVED] `connector.triggers` was removed in @objectstack/spec 17 (ADR-0049 enforce-or-remove) — a connector trigger never started anything: `AutomationEngine.registerConnector` registers a connector's actions only, no polling loop read `intervalSeconds` (or the `interval` spelling it was renamed from), and no receiver was driven by a `webhook` trigger. Delete the key; the `ConnectorTrigger` shape leaves with it. To start work from an external system, write a flow that calls the connector's action in a `connector_action` node: for an external event, an `api` flow that the event's sender calls; for a scheduled pull, a `schedule` flow. Run `os migrate meta --from 17` to list the mechanical edits for existing sources; `--write` applies the ones it can prove, and you apply the rest by hand. | | **syncConfig** | `never` | optional | [REMOVED] `connector.syncConfig` was removed in @objectstack/spec 17 (ADR-0049 enforce-or-remove) — no engine ever ran a connector-attached sync: nothing read `strategy`, `direction`, `realtimeSync`, `timestampField`, `conflictResolution`, `batchSize`, `deleteMode` or `filters`, so the `latest_wins` and `soft_delete` defaults resolved and deleted nothing. Delete the key; the `DataSyncConfig` shape leaves with it. A sync is defined on its TARGET: a `mapping` (`targetObject`, `fieldMapping`, `mode`, `upsertKey`) whose `connectorSource` names the `rest` or `openapi` connector it pulls from, the read action and an optional timestamp `watermark`, with a `job` for the cadence. Its pull runs when a `job` drives it — a `job` whose `pull: { mapping }` names that mapping; the binding alone moves no rows. Run `os migrate meta --from 17` to list the mechanical edits for existing sources; `--write` applies the ones it can prove, and you apply the rest by hand. | diff --git a/packages/services/service-automation/src/connector-materialization.test.ts b/packages/services/service-automation/src/connector-materialization.test.ts index e6c5ad18136..9d171e04ce9 100644 --- a/packages/services/service-automation/src/connector-materialization.test.ts +++ b/packages/services/service-automation/src/connector-materialization.test.ts @@ -6,8 +6,8 @@ // resolves `auth.credentialRef`, and registers the resulting `{ def, handlers }` // on the connector registry — so `connector_action` dispatches it and // `GET /connectors` lists it, indistinguishable from a hand-written connector. -// Boot fails loudly for unknown provider / unresolvable credentialRef / name -// conflict / factory failure. +// Boot fails loudly for unknown provider / name conflict / factory failure; a +// credentialRef nothing resolves degrades the instance naming it (#27237). import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; From 07b65dd402df31d1b88d8eb80f0e347f294b50a6 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 05:59:04 +0000 Subject: [PATCH 19/21] =?UTF-8?q?WIP(#27237):=20contract=20review=20r3=20?= =?UTF-8?q?=E2=80=94=20fit=20the=20eval=20sentence=20under=20its=20token?= =?UTF-8?q?=20ceiling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- skills/objectstack-automation/evals/integrations-declared.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/objectstack-automation/evals/integrations-declared.json b/skills/objectstack-automation/evals/integrations-declared.json index 167e8c3ff41..3ced2195463 100644 --- a/skills/objectstack-automation/evals/integrations-declared.json +++ b/skills/objectstack-automation/evals/integrations-declared.json @@ -14,7 +14,7 @@ { "id": 2, "prompt": "Every night, read our Stripe account balance and write the available amount and the sync time onto our own company's `crm_account` row (it is flagged `is_house_account`). The Stripe secret key must not appear in metadata.", - "expected_output": "A `connectors:` entry `defineConnector({ provider: 'openapi', providerConfig: { spec: './…/stripe.trimmed.json' }, auth: { type: 'bearer', credentialRef: 'STRIPE_SECRET_KEY' } })` over a trimmed, packaged OpenAPI 3.x document holding `GetBalance`; the administrator sets `STRIPE_SECRET_KEY` (an environment variable of that name, or `PUT /api/v1/automation/connectors/credential-refs/STRIPE_SECRET_KEY`); until it is set the instance is degraded, and the app still boots. A `type: 'schedule'` flow with the cadence on the start node's `config.schedule`, `runAs: 'system'`, a `connector_action` with `actionId: 'GetBalance'` and a literal `input: {}`, then an `update_record` on `crm_account` with `filter: { is_house_account: true }` and `fields` read from the node's answer via CEL (`bal.body.available[0].amount / 100.0`, `isoDatetime(now())`). `requires` lists `automation`, `triggers`, `job`. No `http` node with the key in a header, no plugin.", + "expected_output": "A `connectors:` entry `defineConnector({ provider: 'openapi', providerConfig: { spec: './…/stripe.trimmed.json' }, auth: { type: 'bearer', credentialRef: 'STRIPE_SECRET_KEY' } })` over a trimmed, packaged OpenAPI 3.x document holding `GetBalance`; the administrator sets `STRIPE_SECRET_KEY`; an unset ref degrades the instance, not boot. A `type: 'schedule'` flow with the cadence on the start node's `config.schedule`, `runAs: 'system'`, a `connector_action` with `actionId: 'GetBalance'` and a literal `input: {}`, then an `update_record` on `crm_account` with `filter: { is_house_account: true }` and `fields` read from the node's answer via CEL (`bal.body.available[0].amount / 100.0`, `isoDatetime(now())`). `requires` lists `automation`, `triggers`, `job`. No `http` node with the key in a header, no plugin.", "files": [], "assertions": { "must_contain": ["provider: 'openapi'", "credentialRef", "type: 'schedule'", "runAs: 'system'", "connector_action", "update_record", "dialect: 'cel'"], From 355edc06242f3335b3dd39bb13a67fcd820e982b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 10:12:58 +0000 Subject: [PATCH 20/21] WIP(#27237): regenerate the API reference pages the merge brought in without a text merge Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- content/docs/references/api/automation-api.mdx | 4 +++- content/docs/references/api/contract.mdx | 4 +++- content/docs/references/api/error-code-ledger.mdx | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/content/docs/references/api/automation-api.mdx b/content/docs/references/api/automation-api.mdx index 7b0038272ef..cbb8fa37c4b 100644 --- a/content/docs/references/api/automation-api.mdx +++ b/content/docs/references/api/automation-api.mdx @@ -347,7 +347,7 @@ const result = AutomationApiErrorCode.parse(data); | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | -| **code** | `Enum<'VALIDATION_ERROR' \| 'INVALID_FIELD' \| 'MISSING_REQUIRED_FIELD' \| 'INVALID_FORMAT' \| 'VALUE_TOO_LONG' \| 'VALUE_TOO_SHORT' \| 'VALUE_OUT_OF_RANGE' \| … +348 more>` | ✅ | The inner refusal's code in its WIRE spelling - the ADR-0112 member a record save of the same write answers in `error.code` (e.g. `UNIQUE_VIOLATION`); the closed set `ApiErrorSchema.code` parses against | +| **code** | `Enum<'VALIDATION_ERROR' \| 'INVALID_FIELD' \| 'MISSING_REQUIRED_FIELD' \| 'INVALID_FORMAT' \| 'VALUE_TOO_LONG' \| 'VALUE_TOO_SHORT' \| 'VALUE_OUT_OF_RANGE' \| … +350 more>` | ✅ | The inner refusal's code in its WIRE spelling - the ADR-0112 member a record save of the same write answers in `error.code` (e.g. `UNIQUE_VIOLATION`); the closed set `ApiErrorSchema.code` parses against | | **message** | `string` | ✅ | The inner refusal's own diagnostic text, verbatim, without the node and operation prefixes the run's failure text adds | ### Allowed Values: `FlowInnerRefusal.code` @@ -497,9 +497,11 @@ const result = AutomationApiErrorCode.parse(data); * `EXPORT_NOT_PERMITTED` * `EXTERNAL_DATASOURCE_ERROR` * `EXTERNAL_IMPORT_ERROR` +* `EXTERNAL_ROW_CAP_EXCEEDED` * `EXTERNAL_SCHEMA_MISMATCH` * `EXTERNAL_SCHEMA_MODE_VIOLATION` * `EXTERNAL_WRITE_FORBIDDEN` +* `EXTERNAL_WRITE_REFUSED` * `FEEDS_DISABLED` * `FIELD_VISIBILITY_UNRESOLVED` * `FILES_DISABLED` diff --git a/content/docs/references/api/contract.mdx b/content/docs/references/api/contract.mdx index e7ded07f5c2..3f71ca94085 100644 --- a/content/docs/references/api/contract.mdx +++ b/content/docs/references/api/contract.mdx @@ -28,7 +28,7 @@ const result = ApiErrorSchema.parse(data); | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | -| **code** | `Enum<'VALIDATION_ERROR' \| 'INVALID_FIELD' \| 'MISSING_REQUIRED_FIELD' \| 'INVALID_FORMAT' \| 'VALUE_TOO_LONG' \| 'VALUE_TOO_SHORT' \| 'VALUE_OUT_OF_RANGE' \| … +348 more>` | ✅ | Error code (e.g. VALIDATION_ERROR; StandardErrorCode ∪ the ledger the serving side registers — ERROR_CODE_LEDGER for framework packages) | +| **code** | `Enum<'VALIDATION_ERROR' \| 'INVALID_FIELD' \| 'MISSING_REQUIRED_FIELD' \| 'INVALID_FORMAT' \| 'VALUE_TOO_LONG' \| 'VALUE_TOO_SHORT' \| 'VALUE_OUT_OF_RANGE' \| … +350 more>` | ✅ | Error code (e.g. VALIDATION_ERROR; StandardErrorCode ∪ the ledger the serving side registers — ERROR_CODE_LEDGER for framework packages) | | **declaredCode** | `string` | optional | The producer-declared code, verbatim, when it is not a member of the closed `code` vocabulary — the open, author-authored channel (app-specific spellings; ADR-0112) | | **message** | `string` | ✅ | Readable error message | | **userMessage** | `string` | optional | Producer-marked user-facing refusal text, verbatim. Present exactly when the producer opted in at throw time; consumers render it to end users and keep their generic substitution for anything unmarked. Status-agnostic; never replaces `message`. | @@ -185,9 +185,11 @@ const result = ApiErrorSchema.parse(data); * `EXPORT_NOT_PERMITTED` * `EXTERNAL_DATASOURCE_ERROR` * `EXTERNAL_IMPORT_ERROR` +* `EXTERNAL_ROW_CAP_EXCEEDED` * `EXTERNAL_SCHEMA_MISMATCH` * `EXTERNAL_SCHEMA_MODE_VIOLATION` * `EXTERNAL_WRITE_FORBIDDEN` +* `EXTERNAL_WRITE_REFUSED` * `FEEDS_DISABLED` * `FIELD_VISIBILITY_UNRESOLVED` * `FILES_DISABLED` diff --git a/content/docs/references/api/error-code-ledger.mdx b/content/docs/references/api/error-code-ledger.mdx index 62c5e18f373..7dd97f55a02 100644 --- a/content/docs/references/api/error-code-ledger.mdx +++ b/content/docs/references/api/error-code-ledger.mdx @@ -352,9 +352,11 @@ const result = ErrorCode.parse(data); * `EXPORT_NOT_PERMITTED` * `EXTERNAL_DATASOURCE_ERROR` * `EXTERNAL_IMPORT_ERROR` +* `EXTERNAL_ROW_CAP_EXCEEDED` * `EXTERNAL_SCHEMA_MISMATCH` * `EXTERNAL_SCHEMA_MODE_VIOLATION` * `EXTERNAL_WRITE_FORBIDDEN` +* `EXTERNAL_WRITE_REFUSED` * `FEEDS_DISABLED` * `FIELD_VISIBILITY_UNRESOLVED` * `FILES_DISABLED` From 1818d6628e460c23aa34c095de58832f00fd63a4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 11:03:19 +0000 Subject: [PATCH 21/21] WIP(#27237): regenerate the API reference pages against main with #27338 and #27339 Claude-Session: https://claude.ai/code/session_01Utp6CaqVRa7ow6CjE1JwaG Co-authored-by: Claude --- content/docs/references/api/automation-api.mdx | 3 ++- content/docs/references/api/contract.mdx | 3 ++- content/docs/references/api/error-code-ledger.mdx | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/content/docs/references/api/automation-api.mdx b/content/docs/references/api/automation-api.mdx index cbb8fa37c4b..5ccb3a512d3 100644 --- a/content/docs/references/api/automation-api.mdx +++ b/content/docs/references/api/automation-api.mdx @@ -347,7 +347,7 @@ const result = AutomationApiErrorCode.parse(data); | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | -| **code** | `Enum<'VALIDATION_ERROR' \| 'INVALID_FIELD' \| 'MISSING_REQUIRED_FIELD' \| 'INVALID_FORMAT' \| 'VALUE_TOO_LONG' \| 'VALUE_TOO_SHORT' \| 'VALUE_OUT_OF_RANGE' \| … +350 more>` | ✅ | The inner refusal's code in its WIRE spelling - the ADR-0112 member a record save of the same write answers in `error.code` (e.g. `UNIQUE_VIOLATION`); the closed set `ApiErrorSchema.code` parses against | +| **code** | `Enum<'VALIDATION_ERROR' \| 'INVALID_FIELD' \| 'MISSING_REQUIRED_FIELD' \| 'INVALID_FORMAT' \| 'VALUE_TOO_LONG' \| 'VALUE_TOO_SHORT' \| 'VALUE_OUT_OF_RANGE' \| … +351 more>` | ✅ | The inner refusal's code in its WIRE spelling - the ADR-0112 member a record save of the same write answers in `error.code` (e.g. `UNIQUE_VIOLATION`); the closed set `ApiErrorSchema.code` parses against | | **message** | `string` | ✅ | The inner refusal's own diagnostic text, verbatim, without the node and operation prefixes the run's failure text adds | ### Allowed Values: `FlowInnerRefusal.code` @@ -480,6 +480,7 @@ const result = AutomationApiErrorCode.parse(data); * `ERR_AUTONUMBER_COLLISION` * `ERR_BULK_RESULT_MISMATCH` * `ERR_CROSS_DATASOURCE_TRANSACTION_WRITE` +* `ERR_CURRENCY_CODE_REQUIRED` * `ERR_DATASOURCE_UNAVAILABLE` * `ERR_DRIVER_CONNECT` * `ERR_FILE_CONSTRAINT` diff --git a/content/docs/references/api/contract.mdx b/content/docs/references/api/contract.mdx index 3f71ca94085..17517625f87 100644 --- a/content/docs/references/api/contract.mdx +++ b/content/docs/references/api/contract.mdx @@ -28,7 +28,7 @@ const result = ApiErrorSchema.parse(data); | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | -| **code** | `Enum<'VALIDATION_ERROR' \| 'INVALID_FIELD' \| 'MISSING_REQUIRED_FIELD' \| 'INVALID_FORMAT' \| 'VALUE_TOO_LONG' \| 'VALUE_TOO_SHORT' \| 'VALUE_OUT_OF_RANGE' \| … +350 more>` | ✅ | Error code (e.g. VALIDATION_ERROR; StandardErrorCode ∪ the ledger the serving side registers — ERROR_CODE_LEDGER for framework packages) | +| **code** | `Enum<'VALIDATION_ERROR' \| 'INVALID_FIELD' \| 'MISSING_REQUIRED_FIELD' \| 'INVALID_FORMAT' \| 'VALUE_TOO_LONG' \| 'VALUE_TOO_SHORT' \| 'VALUE_OUT_OF_RANGE' \| … +351 more>` | ✅ | Error code (e.g. VALIDATION_ERROR; StandardErrorCode ∪ the ledger the serving side registers — ERROR_CODE_LEDGER for framework packages) | | **declaredCode** | `string` | optional | The producer-declared code, verbatim, when it is not a member of the closed `code` vocabulary — the open, author-authored channel (app-specific spellings; ADR-0112) | | **message** | `string` | ✅ | Readable error message | | **userMessage** | `string` | optional | Producer-marked user-facing refusal text, verbatim. Present exactly when the producer opted in at throw time; consumers render it to end users and keep their generic substitution for anything unmarked. Status-agnostic; never replaces `message`. | @@ -168,6 +168,7 @@ const result = ApiErrorSchema.parse(data); * `ERR_AUTONUMBER_COLLISION` * `ERR_BULK_RESULT_MISMATCH` * `ERR_CROSS_DATASOURCE_TRANSACTION_WRITE` +* `ERR_CURRENCY_CODE_REQUIRED` * `ERR_DATASOURCE_UNAVAILABLE` * `ERR_DRIVER_CONNECT` * `ERR_FILE_CONSTRAINT` diff --git a/content/docs/references/api/error-code-ledger.mdx b/content/docs/references/api/error-code-ledger.mdx index 7dd97f55a02..549e302b03c 100644 --- a/content/docs/references/api/error-code-ledger.mdx +++ b/content/docs/references/api/error-code-ledger.mdx @@ -335,6 +335,7 @@ const result = ErrorCode.parse(data); * `ERR_AUTONUMBER_COLLISION` * `ERR_BULK_RESULT_MISMATCH` * `ERR_CROSS_DATASOURCE_TRANSACTION_WRITE` +* `ERR_CURRENCY_CODE_REQUIRED` * `ERR_DATASOURCE_UNAVAILABLE` * `ERR_DRIVER_CONNECT` * `ERR_FILE_CONSTRAINT`