From de7bcc4bcb5954d71037df117a05eff84f30b2fd Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 16:12:56 +0000 Subject: [PATCH 1/6] wip(objectql,metadata-protocol): internal-field evaluate-position refusals (#22646) Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01JfJfBUC3cQ6hhgm9MQK76T --- packages/metadata-protocol/src/protocol.ts | 187 ++++++++++++++++++++- packages/objectql/src/engine.ts | 52 +++++- 2 files changed, 235 insertions(+), 4 deletions(-) diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index 5107b57fb12..b5247b2bf0b 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -14,7 +14,7 @@ import { postureEnforcesWall, type TenancyPosture } from '@objectstack/spec/secu // `DiscoverySchema` "System Identity" field — never a literal again. import { resolveDiscoveryVersion } from './discovery-version.js'; import type { MetadataHostEngine } from './host-engine.js'; -import { omitInternalFieldsFromWriteResponse } from './write-response-internal-fields.js'; +import { collectInternalWriteResponseFields, omitInternalFieldsFromWriteResponse } from './write-response-internal-fields.js'; // [#17502] The served JSON Schema publishes what an author MAY write, so a // property no instance can satisfy — a `retiredKey()` tombstone, rendered // `{ not: {} }` — is dropped from it. See the module header for the channels @@ -12027,6 +12027,37 @@ export class ObjectStackProtocolImplementation implements if (names.length === 0) return; const gate = this.resolveQueryFields(object); if (!gate) return; + // [#22646] A field declared `internal: true` is never a search target, + // refused FIRST and regardless of whether it resolves into the scanned + // set: a non-hidden internal text column (an auth digest, a token) + // enters the auto-default, so `$searchFields` naming one would be + // ACCEPTED here and a `$search` over it would confirm a guessed prefix + // of a withheld credential. The engine already drops internal fields + // from the auto-default set it scans (a withhold), but an EXPLICIT + // narrowing to one has to be a loud 400 rather than a silent widening + // back to the default set — the #4254 posture. Judged by the ONE flag + // reader (`@objectstack/core`'s cross-package twin of + // `collectInternalReadFields`), the same reader the evaluate refusal uses. + const internalSet = new Set(collectInternalWriteResponseFields(gate.schema)); + const internalNamed = names.filter((n) => internalSet.has(n)); + if (internalNamed.length > 0) { + const first = internalNamed[0] as string; + const err: any = new Error( + `Field '${first}' on object '${object}' is declared \`internal: true\` and cannot be searched` + + (internalNamed.length > 1 ? ` (also: ${internalNamed.slice(1).join(', ')})` : '') + + `. Its value is never returned on a generic data exit, so '${param}' may not name it — a ` + + 'search over it would confirm a guessed value by whether the row comes back. Search ' + + 'another column instead.', + ); + err.code = 'INVALID_FIELD'; + err.status = 400; + err.httpStatus = 400; + err.field = first; + err.fields = internalNamed; + err.object = object; + err.param = param; + throw err; + } const { allowed, source } = resolveSearchFieldResolution({ fields: gate.fields, searchableFields: gate.schema?.searchableFields, @@ -12382,6 +12413,146 @@ export class ObjectStackProtocolImplementation implements throw err; } + /** + * [#22646] The data door's EVALUATE refusal for a field declared + * `internal: true` — "the value is never returned on a generic data exit" + * (#21197), extended from the row position (where it is a withhold — the + * engine OMITS the column from every row) to every position that would + * reveal the stored value by EVALUATING it. A filter or sort on such a + * column is a confirmation oracle — a predicate matches the row only when + * the guess is right, an order leaks the comparative value — so it is + * REFUSED rather than withheld, the posture this door already takes for the + * stored-metadata body/hash family ({@link storedMetadataBodyPredicateRefusal}, + * {@link storedMetadataHashEvaluateRefusal}) and the analytics door for an + * internal member. `INVALID_FIELD` / 400, the family's envelope, located at + * the object, the field and the offending position. + * + * It binds EVERY caller of this generic door, administrators included + * (#21197): there is no caller carve-out, the same stance the stored-body + * family takes here. The engine's privileged internal consumers (the + * API-key verifier's `where: { key }`, the share-link / SCIM / approval-token + * lookups) call the engine directly and never reach this door, so the + * flag's one legitimate use — a `where` match the engine resolves without + * ever returning the value — is untouched, exactly as #7823 kept the engine's + * write results whole while the generic-data-path ingress stripped them. + * + * Positions, with the ONE flag reader ({@link collectInternalWriteResponseFields}, + * `@objectstack/core`'s cross-package twin of objectql's + * `collectInternalReadFields` — identical rule, imported because this package + * sits below objectql, as the write-response strip and the analytics door do): + * + * - `where` / `filter` and each `aggregations[i].filter` — every column a + * filter READS, by the stricter collector that also walks a cross-field + * `{ $field }` comparand ({@link collectFilterReads}). + * - `orderBy` — the fields a sort names. + * - `groupBy` and `aggregations[].field` — defence in depth; the engine's + * `rejectCredentialAggregation` refuses these for every caller with the + * same code, and this door answers first so the envelope is the door's. + * - a one-level `expand` entry's own `where` / `orderBy`, judged against + * the TARGET object's internal fields: the nested sub-read's row strip + * already withholds the column, but a nested filter is a presence oracle + * on the related record (the expanded object appears only when the guess + * matches), so the evaluate positions are refused there too. Target + * OBJECT exposure is a different axis, filed separately (#22661). + */ + private assertNoInternalFieldEvaluated(object: string, options: Record): void { + const gate = this.resolveQueryFields(object); + if (gate) { + const internal = new Set(collectInternalWriteResponseFields(gate.schema)); + if (internal.size > 0) { + // where + the `filter` alias a direct bag may still carry. + for (const [slot, where] of [['filter', options.where], ['filter', options.filter]] as const) { + this.refuseInternalReads(object, where, internal, slot); + } + this.refuseInternalNames(object, this.orderByFieldNames(options.orderBy), internal, 'sort'); + this.refuseInternalNames(object, this.groupByFieldNames(options.groupBy), internal, 'groupBy'); + if (Array.isArray(options.aggregations)) { + options.aggregations.forEach((agg: any, i: number) => { + const field = agg?.field; + if (typeof field === 'string' && field !== '*') { + this.refuseInternalNames(object, [field], internal, `aggregations[${i}].field`); + } + this.refuseInternalReads(object, agg?.filter, internal, `aggregations[${i}].filter`); + }); + } + } + } + // One-level expand: each entry's own evaluate positions, against the + // TARGET object's internal fields. + const expand = options.expand; + if (gate && expand && typeof expand === 'object' && !Array.isArray(expand)) { + for (const [rel, entry] of Object.entries(expand as Record)) { + if (!entry || typeof entry !== 'object') continue; + const hasEvaluatePosition = entry.where !== undefined || entry.orderBy !== undefined; + if (!hasEvaluatePosition) continue; + const sourceField = gate.fields?.[rel]; + const target = sourceField != null && REFERENCE_VALUE_TYPES.has(sourceField.type) + ? referenceTargetOf(sourceField) + : undefined; + if (!target) continue; + const targetGate = this.resolveQueryFields(target); + if (!targetGate) continue; + const targetInternal = new Set(collectInternalWriteResponseFields(targetGate.schema)); + if (targetInternal.size === 0) continue; + this.refuseInternalReads(target, entry.where, targetInternal, `expand['${rel}'].filter`); + this.refuseInternalNames(target, this.orderByFieldNames(entry.orderBy), targetInternal, `expand['${rel}'].sort`); + } + } + } + + /** Field heads an `orderBy` names (string or `{ field }`), dotted head only. */ + private orderByFieldNames(orderBy: unknown): string[] { + if (!Array.isArray(orderBy)) return []; + return orderBy + .map((e) => (typeof e === 'string' ? e : (e as { field?: unknown })?.field)) + .filter((f): f is string => typeof f === 'string' && f.length > 0) + .map((f) => f.split('.')[0] as string); + } + + /** Field heads a `groupBy` names (string or `{ field }`), dotted head only. */ + private groupByFieldNames(groupBy: unknown): string[] { + if (!Array.isArray(groupBy)) return []; + return groupBy + .map((e) => (typeof e === 'string' ? e : (e as { field?: unknown })?.field)) + .filter((f): f is string => typeof f === 'string' && f.length > 0) + .map((f) => f.split('.')[0] as string); + } + + /** Refuse when any READ column of `where` (keys and `{ $field }` comparands) is internal. */ + private refuseInternalReads(object: string, where: unknown, internal: ReadonlySet, param: string): void { + if (where == null) return; + const reads = new Set(); + collectFilterReads(isFilterAST(where) ? parseFilterAST(where) : where, reads); + this.refuseInternalNames(object, [...reads], internal, param); + } + + /** Refuse when any of `names` is an internal field of `object`. */ + private refuseInternalNames(object: string, names: readonly string[], internal: ReadonlySet, param: string): void { + const hit = names.filter((n) => internal.has(n)); + if (hit.length === 0) return; + const first = hit[0] as string; + const verb = param.endsWith('sort') ? 'sort' : param === 'groupBy' ? 'group' : param.includes('filter') ? 'filter' : 'query'; + const err: any = new Error( + `Cannot ${verb} '${object}' by '${first}' (${param}): the query was not run. The ${first} field ` + + 'is declared `internal: true`, so its value is never returned on a generic data exit — the engine ' + + 'omits it from every row. ' + + (verb === 'sort' + ? 'A sort on it orders by that withheld value' + : verb === 'group' + ? 'A group key would be the withheld value, served as the key' + : 'A filter on it confirms a guessed value by whether the row comes back') + + ', so it is refused rather than evaluated. Use another field.', + ); + err.code = 'INVALID_FIELD'; + err.status = 400; + err.httpStatus = 400; + err.field = first; + err.fields = hit; + err.object = object; + err.param = param; + throw err; + } + async findData(request: { object: string, query?: any, context?: any }) { // [#3770] Existence first: an unregistered object is a 404 before any // query parameter is even parsed, so an unknown name can never be @@ -12805,6 +12976,20 @@ export class ObjectStackProtocolImplementation implements }); if (hashEvaluateRefusal) throw hashEvaluateRefusal; + // [#22646] A field declared `internal: true` is withheld from every + // generic exit (#21197) — the engine omits it from every row. The + // EVALUATE positions do not follow that by omission: a filter, sort, + // group-by, aggregate operand, per-aggregation filter or a one-level + // expand's own filter/sort on such a column reveals the stored value by + // evaluating it (a filter is a confirmation oracle, a sort leaks order). + // So they are refused here, at the generic data door, after the + // existence gates and before the engine — the same shape, envelope and + // caller-independence as the stored-metadata body/hash family above. + // Internal fields in a `$search` set are handled on their own axes: the + // auto-default never scans one (engine), an explicit `$searchFields` + // naming one is refused (the gate above, #4254 posture). + this.assertNoInternalFieldEvaluated(request.object, options); + // Route to engine.aggregate() when the query has GROUP BY / aggregations. // engine.find() does not do in-memory aggregation fallback, so without // this branch a spec-shape aggregate request would silently return diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index 3177422431a..d766e48b39e 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -26,6 +26,9 @@ import type { WriteObservabilityOptions } from '@objectstack/spec/contracts'; // engine is what `metadata-protocol.validateData` returns, so letting the two // drift would put a translation layer between a verdict and its contract. import type { ValidateDataIssue, ValidateDataResponse } from '@objectstack/spec/api'; +// [#22646] The ADR-0112 catalog — `rejectCredentialAggregation`'s refusal +// envelope carries `INVALID_FIELD` from it rather than a bare `Error`. +import { StandardErrorCode } from '@objectstack/spec/api'; import { parseAutonumberFormat, renderAutonumber, resolveAutonumberFormat, readAutonumberCounter, missingFieldValues, isTenancyDisabled, FILE_REFERENCE_TYPES, REFERENCE_VALUE_TYPES, referenceTargetOf, referenceCarrierOf, isFileIdToken, RAW_FILE_VALUES_CONTEXT_KEY, isCurrentUserDefaultToken, isNowDefaultToken, isMultiValueField, driverSupportsTransactions, AUDIT_PROVENANCE_FIELDS } from '@objectstack/spec/data'; import type { FileRefusedValue } from '@objectstack/spec/data'; // [#5158] Door 2's lowering sink — the SAME pair the protocol face (Door 1) @@ -12380,8 +12383,31 @@ export class ObjectQL implements IObjectQLEngine { if (raw != null && schema?.fields) { const requestedFields = (ast as any).searchFields ?? (typeof raw === 'object' ? raw?.fields : undefined); + // [#22646] A `search` never scans an `internal: true` column. The flag's + // promise ("the value is never returned on a generic data exit") is a + // withhold on the read path (the row strip) and a refusal everywhere the + // value would be revealed by evaluation; `$search` is such a position — + // a cross-field `$icontains` that confirms a guessed prefix of a withheld + // credential, matching the row only when the guess is right. The field is + // dropped from the set the expansion resolves over (a WITHHOLD, like the + // auto-default's `hidden`/credential-type exclusions one layer down), so + // the default set never scans it, for every verb that reaches here + // (`find` / `findOne` / `aggregate`, and `searchAll` through them). An + // EXPLICIT `$searchFields` naming one is refused loudly at the generic + // data door instead (`assertSearchFieldsAreSearchable`, + // `@objectstack/metadata-protocol`), the #4254 posture — an unhonoured + // narrowing must be a 400, not a silent widening. A direct engine caller + // naming one has it intersected away here, the engine's existing tolerant + // posture for an unknown requested search field. + const internalFields = collectInternalReadFields(schema); + const scanFields = internalFields.length === 0 + ? (schema.fields as any) + : Object.fromEntries( + Object.entries(schema.fields as Record) + .filter(([name]) => !internalFields.includes(name)), + ); const searchFilter = expandSearchToFilter(raw, { - fields: schema.fields as any, + fields: scanFields, searchableFields: (schema as any).searchableFields, requestedFields, // [ADR-0079] `nameField` is the canonical primary-title pointer; @@ -18486,6 +18512,16 @@ export class ObjectQL implements IObjectQLEngine { * Only the two output-bearing positions on `EngineAggregateOptions` carry * field names: `aggregations[].field` (skip COUNT(*) — undefined or '*') and * `groupBy[]` (a string, or a `{ field }` bucket object). + * + * The refusal carries the ADR-0112 envelope — `INVALID_FIELD` / 400, located + * at the object and the field(s) named. It used to be a bare `Error` with no + * `code` or `status`, so a 400-class author refusal reached the data door + * 500-shaped (`INTERNAL_ERROR`, an undeclared fault). The code matches the + * analytics door's own refusal of an `internal: true` member and the engine's + * structured-JSON group-by refusal: the verdict is about the NAME being a + * field that is never returned on a generic data exit, which `INVALID_FIELD` + * is the catalog member for. The envelope now binds the `secret` / `password` + * case too, which shared the bare `Error`. */ private rejectCredentialAggregation(object: string, query: EngineAggregateOptions): void { const schema = this._registry.getObject(object); @@ -18508,13 +18544,23 @@ export class ObjectQL implements IObjectQLEngine { const hit = protectedFields.filter((f) => referenced.has(f)); if (hit.length > 0) { - throw new Error( + const err = new Error( `Cannot aggregate credential field(s) ${hit.map((f) => `"${object}.${f}"`).join(', ')}: ` + 'secret/password fields are masked on read and `internal: true` fields are omitted ' + 'outright, so the value never leaves the engine on the generic data path; aggregating ' + 'them (group-by, min/max, array_agg, …) would surface it. ' + 'Refusing (fail-closed) — see ADR-0100.', - ); + ) as Error & { code?: string; status?: number; httpStatus?: number; field?: string; fields?: string[]; object?: string }; + // ADR-0112 envelope, the same number under D5's two spellings — `status` + // for the HTTP doors, `httpStatus` for a consumer holding the thrown + // error (the CLI `--json` envelope). + err.code = StandardErrorCode.enum.INVALID_FIELD; + err.status = 400; + err.httpStatus = 400; + err.field = hit[0]; + err.fields = hit; + err.object = object; + throw err; } } From d652e6713c40c043a783faff9a880f84c24377c1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 16:26:43 +0000 Subject: [PATCH 2/6] test(objectql,metadata-protocol): pin internal-field data-door positions + changeset (#22646) Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01JfJfBUC3cQ6hhgm9MQK76T --- ...2646-internal-field-data-door-positions.md | 38 +++++ ...data-door-internal-field-positions.test.ts | 161 ++++++++++++++++++ .../src/internal-field-positions.test.ts | 159 +++++++++++++++++ ...-field-data-door-positions.dogfood.test.ts | 100 +++++++++++ 4 files changed, 458 insertions(+) create mode 100644 .changeset/22646-internal-field-data-door-positions.md create mode 100644 packages/metadata-protocol/src/protocol.data-door-internal-field-positions.test.ts create mode 100644 packages/objectql/src/internal-field-positions.test.ts create mode 100644 packages/qa/dogfood/test/internal-field-data-door-positions.dogfood.test.ts diff --git a/.changeset/22646-internal-field-data-door-positions.md b/.changeset/22646-internal-field-data-door-positions.md new file mode 100644 index 00000000000..47e0d332165 --- /dev/null +++ b/.changeset/22646-internal-field-data-door-positions.md @@ -0,0 +1,38 @@ +--- +'@objectstack/objectql': minor +'@objectstack/metadata-protocol': minor +--- + +fix(objectql,metadata-protocol)!: the data door honours a field's `internal: true` in its filter, sort, group-by, aggregate, per-aggregation-filter, `$search` and one-level `$expand` positions, not only the row (#22646) + +Clause-②: no (narrowing) + + + +**BREAKING** (an accept-set narrowing), shipped as `minor` under the launch-window convention for breaking changes. + +A field declared `internal: true` is withheld from every generic exit (#21197): the engine omits it from every row, select-named included. The ROW position honoured that; the generic data door's EVALUATE positions did not, so a value the platform withholds from rows could still be learned through a position that answers by its stored value. + +**FROM.** On `origin/main`, for a member holding read and for an administrator alike, on the list and query data routes (`GET /api/v1/data/:object`, `POST /api/v1/data/:object/query`): + +- A **filter** naming such a field was admitted — the predicate reached the driver, so the row came back only when the guess matched: a confirmation oracle over a withheld value. +- A **sort** by such a field was admitted — the order leaks the comparative stored value. +- A **group-by** of, or an **aggregate operand** over, such a field was refused by the engine, but as an undeclared `500 INTERNAL_ERROR` (a bare `Error` with no `code` or `status`), not a declared refusal. +- A **per-aggregation filter** (`aggregations[].filter`) naming it, and a **filter beside an aggregation**, were admitted — the same oracle at a second filter position. +- A **`$search`** whose resolved set included such a field (a non-hidden `internal` text column enters the auto-default) scanned it — a cross-field substring match that confirms a guessed prefix. +- A one-level **`$expand`**'s own `where` / `orderBy` on the TARGET object's `internal` field was admitted into the expansion sub-read — the expanded record's presence is an oracle on the related row. + +**TO.** Each evaluate position is refused `400 INVALID_FIELD`, a declared ADR-0112 envelope naming the object and the field, before the engine is asked: + +- Filter, sort, group-by, aggregate operand, per-aggregation filter and a one-level expand's own filter/sort are refused at the generic data door (`@objectstack/metadata-protocol`), after the field-existence gates, in the shape and envelope of the stored-metadata body/hash family one axis over. +- The engine's `aggregate()` refusal (`rejectCredentialAggregation`) now carries the `INVALID_FIELD` / 400 envelope instead of the bare `Error`, so the group-by and aggregate-operand positions answer a declared `400` rather than a `500` — and the `secret` / `password` case that shared that bare `Error` is upgraded with it. +- A `$search` never scans an `internal: true` column: the field is dropped from the set the engine's search expansion resolves over (a withhold, like the auto-default's `hidden` / credential-type exclusions), and an explicit `$searchFields` naming one is refused `400 INVALID_FIELD` at the door rather than silently widened back to the default set. + +**Bindings.** + +- **Every caller.** The refusal is at the generic data door and takes no user, so an administrator is refused exactly as a member is (#21197). There is no system carve-out at the door. +- **The engine's privileged consumers are untouched.** The API-key verifier's `where: { key: }` lookup, the share-link / SCIM / approval-token lookups — all call the engine directly under a system context and never pass the generic data door, so authentication keeps working, exactly as #7823 kept the engine's write results whole while the generic-data-path ingress stripped them. +- **One flag reader.** Every position judges `collectInternalReadFields` (engine) or its byte-identical `@objectstack/core` twin `collectInternalWriteResponseFields` (the door, which sits below objectql), never a second list. +- **Unchanged.** An ordinary field in every position is served as before; an object that merely has an `internal` field is served for every other field. + +**Scope.** Target-OBJECT exposure through `$expand` is a separate axis (#22661); the cross-object search sweep (`searchAll`) is #22640's region and inherits the engine's search-set withholding through `engine.find` without a change to its own code. diff --git a/packages/metadata-protocol/src/protocol.data-door-internal-field-positions.test.ts b/packages/metadata-protocol/src/protocol.data-door-internal-field-positions.test.ts new file mode 100644 index 00000000000..ca167e37c17 --- /dev/null +++ b/packages/metadata-protocol/src/protocol.data-door-internal-field-positions.test.ts @@ -0,0 +1,161 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#22646] The generic data door refuses every EVALUATE position that names a + * field declared `internal: true`, before the engine is asked — the position + * enumeration this card's fix is pinned by. + * + * A field declared `internal: true` is withheld from every generic exit + * (#21197): the engine omits it from every row (a withhold — pinned on the row + * position in objectql's `internal-fields.test.ts`). The EVALUATE positions do + * NOT follow that by omission — a filter is a confirmation oracle, a sort leaks + * the comparative value, a group key IS the value — so this door REFUSES them, + * `INVALID_FIELD` / 400, the same envelope and caller-independence as the + * stored-metadata body/hash family one axis over. + * + * Each refusal is paired with a CONTROL on an ordinary field of the SAME object + * that must keep working and must reach the engine — a door that refused every + * query would pass every "refused" assertion and serve nothing. The door binds + * every caller (no context carve-out here); the member/admin end-to-end split + * is pinned on a real stack in `@objectstack/dogfood`. + * + * The position enumeration is the load-bearing shape: a position added later + * that does not pass through this door turns a case here red. + */ + +import { describe, expect, it, vi } from 'vitest'; +import type { ServiceObject } from '@objectstack/spec/data'; +import { ObjectStackProtocolImplementation } from './protocol.js'; + +const VAULT: ServiceObject = { + name: 'pv_vault', + label: 'Vault', + fields: { + id: { name: 'id', label: 'ID', type: 'text' }, + name: { name: 'name', label: 'Name', type: 'text' }, + rank: { name: 'rank', label: 'Rank', type: 'number' }, + // The flagged column — a one-way digest in a `text` column, the shape + // the flag exists for (#7728). Non-hidden, so nothing else withholds it. + secret_hash: { name: 'secret_hash', label: 'Secret', type: 'text', internal: true }, + }, +} as unknown as ServiceObject; + +const HOLDER: ServiceObject = { + name: 'pv_holder', + label: 'Holder', + fields: { + id: { name: 'id', label: 'ID', type: 'text' }, + name: { name: 'name', label: 'Name', type: 'text' }, + vault_id: { name: 'vault_id', label: 'Vault', type: 'lookup', reference: 'pv_vault' }, + }, +} as unknown as ServiceObject; + +const SCHEMAS: Record = { pv_vault: VAULT, pv_holder: HOLDER }; +const ROWS: Record[]> = { + pv_vault: [{ id: 'v1', name: 'alpha', rank: 1, secret_hash: 'h1' }], + pv_holder: [{ id: 'o1', name: 'owner', vault_id: 'v1' }], +}; + +function makeProtocol() { + const find = vi.fn(async (object: string, _o: any) => (ROWS[object] ?? []).map((r) => ({ ...r }))); + const findOne = vi.fn(async (object: string) => ({ ...(ROWS[object]?.[0] ?? {}) })); + const aggregate = vi.fn(async () => [{ n: 1 }]); + const count = vi.fn(async () => (ROWS.pv_vault?.length ?? 0)); + const engine: any = { registry: { getObject: (n: string) => SCHEMAS[n] }, find, findOne, count, aggregate }; + return { p: new ObjectStackProtocolImplementation(engine), find, aggregate, count }; +} + +async function refusal(run: () => Promise): Promise { + let resolved = false; + let caught: any; + try { await run(); resolved = true; } catch (e) { caught = e; } + expect(resolved, 'expected a refusal, but the query ran').toBe(false); + return caught; +} + +const F = 'secret_hash'; + +describe('[#22646] the data door refuses an `internal: true` field in every evaluate position', () => { + const cases: Array<[string, Record]> = [ + ['implicit filter', { [F]: 'h1' }], + ['filter (OData param)', { filter: JSON.stringify({ [F]: 'h1' }) }], + ['filter ($eq, POST body)', { where: { [F]: { $eq: 'h1' } } }], + ['filter ($startsWith)', { where: { [F]: { $startsWith: 'h' } } }], + ['filter (under $or)', { where: { $or: [{ [F]: 'h1' }, { name: '__none__' }] } }], + ['filter (cross-field $field comparand)', { where: { name: { $eq: { $field: F } } } }], + ['sort (wire param)', { sort: F }], + ['sort (descending)', { sort: `-${F}` }], + ['sort (orderBy node)', { orderBy: [{ field: F, order: 'asc' }] }], + ['group-by (string)', { groupBy: [F], aggregations: [{ function: 'count', alias: 'n' }] }], + ['group-by (object form)', { groupBy: [{ field: F }], aggregations: [{ function: 'count', alias: 'n' }] }], + ['aggregate operand', { aggregations: [{ function: 'max', field: F, alias: 'm' }] }], + ['aggregation filter', { aggregations: [{ function: 'count', alias: 'n', filter: { [F]: 'h1' } }] }], + ['filter beside an aggregation (aggregate where)', { where: { [F]: 'h1' }, aggregations: [{ function: 'count', alias: 'n' }] }], + ]; + for (const [label, query] of cases) { + it(`${label} is refused (INVALID_FIELD / 400), engine never asked`, async () => { + const { p, find, aggregate, count } = makeProtocol(); + const err = await refusal(() => p.findData({ object: 'pv_vault', query })); + expect(err.code).toBe('INVALID_FIELD'); + expect(err.status).toBe(400); + expect(err.field).toBe(F); + expect(err.object).toBe('pv_vault'); + expect(find).not.toHaveBeenCalled(); + expect(aggregate).not.toHaveBeenCalled(); + expect(count).not.toHaveBeenCalled(); + }); + } + + it('explicit $searchFields naming the internal field is refused (not silently widened)', async () => { + const { p, find } = makeProtocol(); + const err = await refusal(() => p.findData({ object: 'pv_vault', query: { search: 'h', searchFields: F } })); + expect(err.code).toBe('INVALID_FIELD'); + expect(err.status).toBe(400); + expect(err.field).toBe(F); + expect(find).not.toHaveBeenCalled(); + }); + + it("a one-level expand's own filter/sort on the TARGET object's internal field is refused", async () => { + for (const query of [ + { expand: { vault_id: { object: 'pv_vault', where: { [F]: 'h1' } } } }, + { expand: { vault_id: { object: 'pv_vault', orderBy: [{ field: F, order: 'asc' }] } } }, + ]) { + const { p, find } = makeProtocol(); + const err = await refusal(() => p.findData({ object: 'pv_holder', query })); + expect(err.code).toBe('INVALID_FIELD'); + expect(err.status).toBe(400); + expect(err.field).toBe(F); + expect(err.object).toBe('pv_vault'); + expect(find).not.toHaveBeenCalled(); + } + }); +}); + +describe('[#22646] CONTROL — an ordinary field of the same object is unchanged', () => { + it('a filter on `name` is served and reaches the engine', async () => { + const { p, find } = makeProtocol(); + const res: any = await p.findData({ object: 'pv_vault', query: { name: 'alpha' } }); + expect(find).toHaveBeenCalledTimes(1); + expect((res.records ?? res.data ?? res).length).toBe(1); + }); + it('a sort on `name` is served', async () => { + const { p, find } = makeProtocol(); + await p.findData({ object: 'pv_vault', query: { sort: 'name' } }); + expect(find).toHaveBeenCalledTimes(1); + }); + it('a group-by on `name` is served and reaches the engine', async () => { + const { p, aggregate } = makeProtocol(); + await p.findData({ object: 'pv_vault', query: { groupBy: ['name'], aggregations: [{ function: 'count', alias: 'n' }] } }); + expect(aggregate).toHaveBeenCalledTimes(1); + }); + it('an aggregate operand over the non-internal numeric `rank` is served', async () => { + const { p, aggregate } = makeProtocol(); + await p.findData({ object: 'pv_vault', query: { aggregations: [{ function: 'max', field: 'rank', alias: 'm' }] } }); + expect(aggregate).toHaveBeenCalledTimes(1); + }); + it('a $searchFields on the ordinary `name` field is served', async () => { + const { p, find } = makeProtocol(); + await p.findData({ object: 'pv_vault', query: { search: 'al', searchFields: 'name' } }); + expect(find).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/objectql/src/internal-field-positions.test.ts b/packages/objectql/src/internal-field-positions.test.ts new file mode 100644 index 00000000000..418a822a95e --- /dev/null +++ b/packages/objectql/src/internal-field-positions.test.ts @@ -0,0 +1,159 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#22646] The engine's own half of the `internal: true` data-door positions: + * + * - the aggregate refusal (group-by / aggregate operand) carries the ADR-0112 + * envelope — `INVALID_FIELD` / 400 — instead of the bare `Error` that + * reached the data door as an undeclared `500 INTERNAL_ERROR`; + * - a `$search` never scans an `internal: true` column — the field is dropped + * from the set the search expansion resolves over (a WITHHOLD, like the + * auto-default's `hidden` / credential-type exclusions), so a cross-field + * `$icontains` can never confirm a guessed prefix of a withheld credential. + * + * The filter / sort / per-aggregation-filter / explicit-`$searchFields` / + * expand positions are refused at the generic data door, pinned in + * `@objectstack/metadata-protocol`'s + * `protocol.data-door-internal-field-positions.test.ts`; the engine keeps + * serving them so the privileged internal consumers that call it directly (the + * API-key verifier's `where: { key }`) are untouched — pinned by + * `internal-fields.test.ts`'s "keeps the field usable as a WHERE filter". + * + * Refusal cases assert `code` AND `status` (ADR-0112): a bare + * `rejects.toThrow()` stays green against a naked `Error`. + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import { ObjectQL, type EngineReadOptions } from './engine.js'; +import type { ServiceObject } from '@objectstack/spec/data'; + +function makeCaptureDriver() { + const seenWhere: unknown[] = []; + const store = new Map>>(); + const s = (o: string) => { let m = store.get(o); if (!m) { m = new Map(); store.set(o, m); } return m; }; + let n = 0; + const driver: any = { + name: 'memory', version: '0', supports: {}, + async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, + async find(object: string, ast: any) { seenWhere.push(ast?.where); return [...s(object).values()].map((r) => ({ ...r })); }, + async findOne(object: string, ast: any) { seenWhere.push(ast?.where); const r = [...s(object).values()][0]; return r ? { ...r } : null; }, + async count(object: string, ast: any) { seenWhere.push(ast?.where); return s(object).size; }, + // No `aggregate` method: the engine falls back to its in-memory + // aggregation over find() rows, so the control buckets are real. + async create(object: string, data: any) { n += 1; const id = data.id ?? `r_${n}`; const row = { ...data, id }; s(object).set(id, row); return { ...row }; }, + async update(object: string, id: string, data: any) { const row = { ...s(object).get(id), ...data, id }; s(object).set(id, row); return { ...row }; }, + async upsert(object: string, data: any) { return this.create(object, data); }, + async delete(object: string, id: string) { return s(object).delete(id); }, + async bulkCreate(o: string, rows: any[]) { return Promise.all(rows.map((r) => this.create(o, r))); }, + async bulkUpdate() { return []; }, async bulkDelete() {}, + async beginTransaction() { return { commit: async () => {}, rollback: async () => {} }; }, async commit() {}, async rollback() {}, + }; + return { driver, seenWhere }; +} + +// A `token` column that is `internal: true` but NOT `hidden` — so the ONLY +// thing that can keep it out of the auto-default search set is the internal +// flag (a `hidden` column is already excluded regardless). +const flagged: ServiceObject = { + name: 'itest_vault', label: 'Vault', managedBy: 'better-auth', + fields: { + id: { name: 'id', label: 'ID', type: 'text' as const }, + name: { name: 'name', label: 'Name', type: 'text' as const }, + prefix: { name: 'prefix', label: 'Prefix', type: 'text' as const }, + rank: { name: 'rank', label: 'Rank', type: 'number' as const }, + token: { name: 'token', label: 'Token', type: 'text' as const, internal: true }, + }, +} as unknown as ServiceObject; + +const plain: ServiceObject = { + name: 'itest_plain', label: 'Plain', + fields: { + id: { name: 'id', label: 'ID', type: 'text' as const }, + name: { name: 'name', label: 'Name', type: 'text' as const }, + token: { name: 'token', label: 'Token', type: 'text' as const }, + }, +} as unknown as ServiceObject; + +const SYSTEM: EngineReadOptions = { context: { isSystem: true } }; + +async function buildEngine() { + const { driver, seenWhere } = makeCaptureDriver(); + const engine = new ObjectQL(); + engine.registerDriver(driver, true); + await engine.init(); + engine.registry.registerObject(flagged, 'internal-field-positions-test'); + engine.registry.registerObject(plain, 'internal-field-positions-test'); + return { engine, seenWhere }; +} + +describe('[#22646] the aggregate refusal carries the ADR-0112 envelope', () => { + let ctx: Awaited>; + beforeEach(async () => { ctx = await buildEngine(); }); + const seed = () => ctx.engine.insert('itest_vault', { name: 'k', prefix: 'osk_', rank: 1, token: 'h1' }, SYSTEM as any); + + const envelopeOf = async (run: () => Promise) => { + try { await run(); } catch (e: any) { return e; } + throw new Error('expected a refusal, but the query ran'); + }; + + it('group-by the flagged field: INVALID_FIELD / 400, located at object + field', async () => { + await seed(); + const err = await envelopeOf(() => ctx.engine.aggregate('itest_vault', { + aggregations: [{ function: 'count', alias: 'n' }], groupBy: ['token'], + }, SYSTEM)); + expect(err.code).toBe('INVALID_FIELD'); + expect(err.status).toBe(400); + expect(err.httpStatus).toBe(400); + expect(err.object).toBe('itest_vault'); + expect(err.field).toBe('token'); + }); + + it('aggregate operand over the flagged field: INVALID_FIELD / 400', async () => { + await seed(); + const err = await envelopeOf(() => ctx.engine.aggregate('itest_vault', { + aggregations: [{ function: 'max', field: 'token', alias: 'm' }], + }, SYSTEM)); + expect(err.code).toBe('INVALID_FIELD'); + expect(err.status).toBe(400); + }); + + it('CONTROL: an ordinary column still aggregates (the guard is not a no-op)', async () => { + await seed(); + await ctx.engine.insert('itest_vault', { name: 'k2', prefix: 'osk_', rank: 2, token: 'h2' }, SYSTEM as any); + const rows = await ctx.engine.aggregate('itest_vault', { + aggregations: [{ function: 'count', alias: 'n' }], groupBy: ['prefix'], + }, SYSTEM); + expect(Object.fromEntries(rows.map((r: any) => [r.prefix, Number(r.n)]))).toEqual({ osk_: 2 }); + }); +}); + +describe('[#22646] a $search never scans an `internal: true` column', () => { + let ctx: Awaited>; + beforeEach(async () => { ctx = await buildEngine(); }); + + const lastSearchWhere = () => ctx.seenWhere[ctx.seenWhere.length - 1]; + + it('the generated search filter names the ordinary text columns, never the flagged one', async () => { + await ctx.engine.insert('itest_vault', { name: 'alpha', prefix: 'osk_', rank: 1, token: 'h1' }, SYSTEM as any); + await ctx.engine.find('itest_vault', { search: 'alp', context: { isSystem: true } } as any); + const blob = JSON.stringify(lastSearchWhere() ?? null); + expect(blob).toContain('name'); + expect(blob).not.toContain('token'); + }); + + it('an explicit searchFields naming the flagged column is intersected away at the engine (withheld)', async () => { + await ctx.engine.insert('itest_vault', { name: 'alpha', prefix: 'osk_', rank: 1, token: 'h1' }, SYSTEM as any); + await ctx.engine.find('itest_vault', { search: 'h1', searchFields: ['token'], context: { isSystem: true } } as any); + const blob = JSON.stringify(lastSearchWhere() ?? null); + // The override narrowed to nothing scannable, so the engine falls back + // to the (internal-free) default set — never a clause on `token`. + expect(blob).not.toContain('token'); + }); + + it('CONTROL: an object with no flagged field scans its text columns including `token`', async () => { + await ctx.engine.insert('itest_plain', { name: 'alpha', token: 'h1' }, SYSTEM as any); + await ctx.engine.find('itest_plain', { search: 'alp', context: { isSystem: true } } as any); + const blob = JSON.stringify(lastSearchWhere() ?? null); + expect(blob).toContain('token'); + }); +}); diff --git a/packages/qa/dogfood/test/internal-field-data-door-positions.dogfood.test.ts b/packages/qa/dogfood/test/internal-field-data-door-positions.dogfood.test.ts new file mode 100644 index 00000000000..c056d6f647d --- /dev/null +++ b/packages/qa/dogfood/test/internal-field-data-door-positions.dogfood.test.ts @@ -0,0 +1,100 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#22646] A field declared `internal: true` is withheld from every generic + * exit (#21197). The ROW position honours that by OMISSION (pinned in + * `api-key-hash-not-serialized.dogfood.test.ts`); the EVALUATE positions of the + * generic data door did not. On `origin/main`, for `sys_api_key.key` (the + * stored auth digest, declared `internal: true`), the data door answered a + * FILTER on it (a confirmation oracle), a SORT by it (an order oracle), and + * refused a GROUP-BY / aggregate as an undeclared `500 INTERNAL_ERROR`. + * + * This drives every position through the real HTTP stack as BOTH a seeded + * platform admin and an ordinary member, because the gap bound both (#21197): + * a refusal that only an admin saw would miss the member the row strip binds, + * and vice versa. Each position must answer `400 INVALID_FIELD` — a declared + * ADR-0112 refusal, never a 200 and never a 500. + * + * The load-bearing NEGATIVE direction: the key must still AUTHENTICATE. The fix + * refuses the generic data door, not the engine's privileged verifier + * (`where: { key }` under a system context), so a strip that broke the lookup + * would satisfy every "refused" assertion and break every login — asserted + * last. The CONTROL (a filter on the ordinary `name` column) must still answer + * 200, or "refused" would pass for a door that refuses everything. + * + * ⚠️ No reproduction recipe and no value is stated here: the credential is + * minted through its one real door and never read back. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { type VerifyStack } from '@objectstack/verify'; +import { bootShowcase } from './showcase-boot.js'; + +const OBJ = 'sys_api_key'; +const F = 'key'; + +describe('[#22646] the data door refuses sys_api_key.key in every evaluate position, for member and admin', () => { + let stack: VerifyStack; + const personas: Record = {}; + + const mint = async (token: string, name: string): Promise => { + const res = await stack.apiAs(token, 'POST', '/keys', { name }); + expect(res.status, await res.clone().text()).toBe(201); + const body: any = await res.json(); + expect(typeof body?.data?.key).toBe('string'); // show-once mint path — the one time a client sees it + return String(body.data.id); + }; + + beforeAll(async () => { + stack = await bootShowcase({}); + const adminToken = await stack.signIn(); + const memberToken = await stack.signUp('internal.positions.22646@verify.test'); + personas.admin = { token: adminToken, id: await mint(adminToken, 'pos-admin') }; + personas.member = { token: memberToken, id: await mint(memberToken, 'pos-member') }; + }, 180_000); + + afterAll(async () => { await stack?.stop?.(); }); + + // Positions driven through the two data-door routes. `GET` carries the wire + // spellings ($orderby); `POST …/query` carries the QueryAST body. + const positions: Array<[string, (token: string) => Promise]> = [ + ['filter (POST where $eq)', (t) => stack.apiAs(t, 'POST', `/data/${OBJ}/query`, { where: { [F]: { $eq: 'x' } } })], + ['filter (GET implicit)', (t) => stack.apiAs(t, 'GET', `/data/${OBJ}?${F}=x`)], + ['sort (GET $orderby)', (t) => stack.apiAs(t, 'GET', `/data/${OBJ}?$orderby=${F}`)], + ['group-by (was 500)', (t) => stack.apiAs(t, 'POST', `/data/${OBJ}/query`, { groupBy: [F], aggregations: [{ function: 'count', alias: 'n' }] })], + ['aggregate operand (was 500)', (t) => stack.apiAs(t, 'POST', `/data/${OBJ}/query`, { aggregations: [{ function: 'max', field: F, alias: 'm' }] })], + ['aggregation filter', (t) => stack.apiAs(t, 'POST', `/data/${OBJ}/query`, { aggregations: [{ function: 'count', alias: 'n', filter: { [F]: 'x' } }] })], + ]; + + for (const [label, call] of positions) { + for (const persona of ['admin', 'member'] as const) { + it(`${label} — ${persona}: 400 INVALID_FIELD`, async () => { + const res = await call(personas[persona]!.token); + expect(res.status, `${label}/${persona}: ${await res.clone().text()}`).toBe(400); + const body: any = await res.json(); + expect(body?.error?.code ?? body?.code).toBe('INVALID_FIELD'); + }); + } + } + + it('CONTROL — a filter on the ordinary `name` column still answers 200 (admin)', async () => { + const res = await stack.apiAs(personas.admin!.token, 'POST', `/data/${OBJ}/query`, { where: { name: 'pos-admin' } }); + expect(res.status, await res.clone().text()).toBe(200); + }); + + it('CONTROL — a sort on the ordinary `name` column still answers 200 (member)', async () => { + const res = await stack.apiAs(personas.member!.token, 'GET', `/data/${OBJ}?$orderby=name`); + expect(res.status, await res.clone().text()).toBe(200); + }); + + it('NEGATIVE — the key still authenticates: the engine verifier lookup is untouched', async () => { + // Mint a fresh key and use it as the ONLY credential on a real read. + const res = await stack.apiAs(personas.admin!.token, 'POST', '/keys', { name: 'pos-auth' }); + expect(res.status).toBe(201); + const raw = String((await res.json())?.data?.key); + const authed = await stack.api(`/data/${OBJ}?$top=1`, { headers: { 'x-api-key': raw } }); + // 200 proves the verifier's `where: { key: hash }` lookup still resolves + // the principal — the fix refuses the generic data door, not the engine. + expect(authed.status, await authed.clone().text()).toBe(200); + }); +}); From b046e01947480bd825730074cd8d2f22310c8e34 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 17:06:20 +0000 Subject: [PATCH 3/6] test(objectql,metadata-protocol): pin fake-engine findOne predicate, drop read-options erasure, record double ledger (#22646) Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01JfJfBUC3cQ6hhgm9MQK76T --- .../src/protocol.data-door-internal-field-positions.test.ts | 6 +++++- packages/objectql/src/internal-field-positions.test.ts | 6 +++--- scripts/engine-double-contract.pinned.json | 5 +++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/metadata-protocol/src/protocol.data-door-internal-field-positions.test.ts b/packages/metadata-protocol/src/protocol.data-door-internal-field-positions.test.ts index ca167e37c17..efd44ff1503 100644 --- a/packages/metadata-protocol/src/protocol.data-door-internal-field-positions.test.ts +++ b/packages/metadata-protocol/src/protocol.data-door-internal-field-positions.test.ts @@ -25,6 +25,7 @@ import { describe, expect, it, vi } from 'vitest'; import type { ServiceObject } from '@objectstack/spec/data'; +import { assertEngineFindOnePredicate } from '@objectstack/metadata-core'; import { ObjectStackProtocolImplementation } from './protocol.js'; const VAULT: ServiceObject = { @@ -58,7 +59,10 @@ const ROWS: Record[]> = { function makeProtocol() { const find = vi.fn(async (object: string, _o: any) => (ROWS[object] ?? []).map((r) => ({ ...r }))); - const findOne = vi.fn(async (object: string) => ({ ...(ROWS[object]?.[0] ?? {}) })); + const findOne = vi.fn(async (object: string, o: any) => { + assertEngineFindOnePredicate(object, o); + return { ...(ROWS[object]?.[0] ?? {}) }; + }); const aggregate = vi.fn(async () => [{ n: 1 }]); const count = vi.fn(async () => (ROWS.pv_vault?.length ?? 0)); const engine: any = { registry: { getObject: (n: string) => SCHEMAS[n] }, find, findOne, count, aggregate }; diff --git a/packages/objectql/src/internal-field-positions.test.ts b/packages/objectql/src/internal-field-positions.test.ts index 418a822a95e..a65c2ec9309 100644 --- a/packages/objectql/src/internal-field-positions.test.ts +++ b/packages/objectql/src/internal-field-positions.test.ts @@ -135,7 +135,7 @@ describe('[#22646] a $search never scans an `internal: true` column', () => { it('the generated search filter names the ordinary text columns, never the flagged one', async () => { await ctx.engine.insert('itest_vault', { name: 'alpha', prefix: 'osk_', rank: 1, token: 'h1' }, SYSTEM as any); - await ctx.engine.find('itest_vault', { search: 'alp', context: { isSystem: true } } as any); + await ctx.engine.find('itest_vault', { search: 'alp' }, SYSTEM); const blob = JSON.stringify(lastSearchWhere() ?? null); expect(blob).toContain('name'); expect(blob).not.toContain('token'); @@ -143,7 +143,7 @@ describe('[#22646] a $search never scans an `internal: true` column', () => { it('an explicit searchFields naming the flagged column is intersected away at the engine (withheld)', async () => { await ctx.engine.insert('itest_vault', { name: 'alpha', prefix: 'osk_', rank: 1, token: 'h1' }, SYSTEM as any); - await ctx.engine.find('itest_vault', { search: 'h1', searchFields: ['token'], context: { isSystem: true } } as any); + await ctx.engine.find('itest_vault', { search: 'h1', searchFields: ['token'] }, SYSTEM); const blob = JSON.stringify(lastSearchWhere() ?? null); // The override narrowed to nothing scannable, so the engine falls back // to the (internal-free) default set — never a clause on `token`. @@ -152,7 +152,7 @@ describe('[#22646] a $search never scans an `internal: true` column', () => { it('CONTROL: an object with no flagged field scans its text columns including `token`', async () => { await ctx.engine.insert('itest_plain', { name: 'alpha', token: 'h1' }, SYSTEM as any); - await ctx.engine.find('itest_plain', { search: 'alp', context: { isSystem: true } } as any); + await ctx.engine.find('itest_plain', { search: 'alp' }, SYSTEM); const blob = JSON.stringify(lastSearchWhere() ?? null); expect(blob).toContain('token'); }); diff --git a/scripts/engine-double-contract.pinned.json b/scripts/engine-double-contract.pinned.json index 9164e60220e..3141d49a218 100644 --- a/scripts/engine-double-contract.pinned.json +++ b/scripts/engine-double-contract.pinned.json @@ -506,6 +506,11 @@ "verb": "update", "pinned": 1 }, + { + "file": "packages/metadata-protocol/src/protocol.data-door-internal-field-positions.test.ts", + "verb": "findOne", + "pinned": 1 + }, { "file": "packages/metadata-protocol/src/protocol.data-door-stored-content-hash.test.ts", "verb": "findOne", From 9d0cc98dabf120032ca204f4e9fa6b8855215865 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 17:36:11 +0000 Subject: [PATCH 4/6] fix(metadata-protocol): judge nested-relation conditions and every expand level for internal fields (#22646) Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01JfJfBUC3cQ6hhgm9MQK76T --- ...data-door-internal-field-positions.test.ts | 90 ++++++++- packages/metadata-protocol/src/protocol.ts | 185 +++++++++++++----- 2 files changed, 224 insertions(+), 51 deletions(-) diff --git a/packages/metadata-protocol/src/protocol.data-door-internal-field-positions.test.ts b/packages/metadata-protocol/src/protocol.data-door-internal-field-positions.test.ts index efd44ff1503..55907142a5c 100644 --- a/packages/metadata-protocol/src/protocol.data-door-internal-field-positions.test.ts +++ b/packages/metadata-protocol/src/protocol.data-door-internal-field-positions.test.ts @@ -38,6 +38,18 @@ const VAULT: ServiceObject = { // The flagged column — a one-way digest in a `text` column, the shape // the flag exists for (#7728). Non-hidden, so nothing else withholds it. secret_hash: { name: 'secret_hash', label: 'Secret', type: 'text', internal: true }, + inner_id: { name: 'inner_id', label: 'Inner', type: 'lookup', reference: 'pv_inner' }, + }, +} as unknown as ServiceObject; + +/** The grand target: two relation hops from `pv_holder`, with its own flagged column. */ +const INNER: ServiceObject = { + name: 'pv_inner', + label: 'Inner', + fields: { + id: { name: 'id', label: 'ID', type: 'text' }, + name: { name: 'name', label: 'Name', type: 'text' }, + inner_secret: { name: 'inner_secret', label: 'Inner Secret', type: 'text', internal: true }, }, } as unknown as ServiceObject; @@ -51,10 +63,11 @@ const HOLDER: ServiceObject = { }, } as unknown as ServiceObject; -const SCHEMAS: Record = { pv_vault: VAULT, pv_holder: HOLDER }; +const SCHEMAS: Record = { pv_vault: VAULT, pv_holder: HOLDER, pv_inner: INNER }; const ROWS: Record[]> = { pv_vault: [{ id: 'v1', name: 'alpha', rank: 1, secret_hash: 'h1' }], pv_holder: [{ id: 'o1', name: 'owner', vault_id: 'v1' }], + pv_inner: [{ id: 'i1', name: 'inner', inner_secret: 's1' }], }; function makeProtocol() { @@ -163,3 +176,78 @@ describe('[#22646] CONTROL — an ordinary field of the same object is unchanged expect(find).toHaveBeenCalledTimes(1); }); }); + +/** + * Rework round 1 — the positions the first pass did not walk, measured served + * on a real engine and SQL driver before this change (fixture objects only): + * a NESTED-RELATION condition evaluated on the related object, and an `expand` + * tree deeper than one level. Each is refused for a member AND an + * administrator: the door takes no caller carve-out, and the context is passed + * so a future carve-out would have to defeat these cases to land. + */ +const PERSONAS = { + member: { userId: 'u_member', roles: [] as string[] }, + admin: { userId: 'u_admin', roles: ['admin'], isPlatformAdmin: true }, +} as const; + +describe('[#22646] relation conditions and every expand level are judged on their own object', () => { + const cases: Array<[string, string, Record, string, string]> = [ + ['relation filter (nested-relation form)', 'pv_holder', { where: { vault_id: { [F]: 'h1' } } }, 'pv_vault', F], + ['relation filter under $or', 'pv_holder', { where: { $or: [{ vault_id: { [F]: 'h1' } }, { name: '__none__' }] } }, 'pv_vault', F], + ['relation filter (OData filter param)', 'pv_holder', { filter: JSON.stringify({ vault_id: { [F]: 'h1' } }) }, 'pv_vault', F], + ['relation filter beside an aggregation', 'pv_holder', { where: { vault_id: { [F]: 'h1' } }, aggregations: [{ function: 'count', alias: 'n' }] }, 'pv_vault', F], + [ + 'expand depth 2: the grand target\'s filter', + 'pv_holder', + { expand: { vault_id: { object: 'pv_vault', expand: { inner_id: { object: 'pv_inner', where: { inner_secret: 's1' } } } } } }, + 'pv_inner', + 'inner_secret', + ], + [ + 'expand depth 2: the grand target\'s sort', + 'pv_holder', + { expand: { vault_id: { object: 'pv_vault', expand: { inner_id: { object: 'pv_inner', orderBy: [{ field: 'inner_secret', order: 'asc' }] } } } } }, + 'pv_inner', + 'inner_secret', + ], + [ + 'expand depth 1: a relation condition onto the grand target', + 'pv_holder', + { expand: { vault_id: { object: 'pv_vault', where: { inner_id: { inner_secret: 's1' } } } } }, + 'pv_inner', + 'inner_secret', + ], + ]; + for (const [label, object, query, owner, field] of cases) { + for (const [persona, context] of Object.entries(PERSONAS)) { + it(`${label} — ${persona}: INVALID_FIELD / 400 on '${owner}.${field}', engine never asked`, async () => { + const { p, find, aggregate, count } = makeProtocol(); + const err = await refusal(() => p.findData({ object, query, context })); + expect(err.code).toBe('INVALID_FIELD'); + expect(err.status).toBe(400); + expect(err.object).toBe(owner); + expect(err.field).toBe(field); + expect(find).not.toHaveBeenCalled(); + expect(aggregate).not.toHaveBeenCalled(); + expect(count).not.toHaveBeenCalled(); + }); + } + } +}); + +describe('[#22646] CONTROL — ordinary fields through a relation and at depth are unchanged', () => { + const served: Array<[string, Record]> = [ + ['a relation condition on the related object\'s ordinary `name`', { where: { vault_id: { name: 'alpha' } } }], + ['an operator bag on the lookup itself (not a relation condition)', { where: { vault_id: { $eq: 'v1' } } }], + ['expand depth 2 with an ordinary filter', { expand: { vault_id: { object: 'pv_vault', expand: { inner_id: { object: 'pv_inner', where: { name: 'inner' } } } } } }], + ]; + for (const [label, query] of served) { + for (const [persona, context] of Object.entries(PERSONAS)) { + it(`${label} — ${persona}: served, the engine is asked`, async () => { + const { p, find } = makeProtocol(); + await p.findData({ object: 'pv_holder', query, context }); + expect(find).toHaveBeenCalledTimes(1); + }); + } + } +}); diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index b5247b2bf0b..4598531e303 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -4437,6 +4437,55 @@ function headSegment(name: string): string { return name.split('.')[0] as string; } +/** + * [#22646] How deep the data door's `internal: true` walk follows nested + * relation conditions and `expand` levels — a backstop against a + * self-referential in-process value, well above anything the engine executes + * (one nested-relation hop; `expand` stops at its own depth cap). + */ +const INTERNAL_FIELD_WALK_DEPTH = 8; + +/** + * [#22646] The NESTED-RELATION conditions a filter holds — a key naming a + * reference field of the object (`REFERENCE_VALUE_TYPES`) whose value is a + * plain record carrying at least one non-`$` key, i.e. a condition on the + * related object rather than an operator bag — with each site's target object. + * Logical combinators (and any other `$` key) are walked as conditions, so a + * site under `$or` / `$not` is found. Shared by the door's internal-field + * refusal; the engine's own relation lowering is the authority on whether a + * site RUNS, this only finds what a filter would evaluate on the related object. + */ +function relationConditionSites( + fields: Record | undefined, + root: unknown, +): Array<{ field: string; target: string; condition: Record }> { + const sites: Array<{ field: string; target: string; condition: Record }> = []; + if (!fields || typeof fields !== 'object') return sites; + const seen = new WeakSet(); + const pending: unknown[] = [root]; + while (pending.length > 0) { + const node = pending.pop(); + if (node === null || typeof node !== 'object' || seen.has(node)) continue; + seen.add(node); + if (Array.isArray(node)) { + for (const member of node) pending.push(member); + continue; + } + for (const [key, value] of Object.entries(node as Record)) { + if (key.startsWith('$')) { + pending.push(value); + continue; + } + const def = fields[key]; + if (def == null || !REFERENCE_VALUE_TYPES.has(def.type) || !isPlainRecord(value)) continue; + if (!Object.keys(value).some((k) => !k.startsWith('$'))) continue; + const target = referenceTargetOf(def); + if (target) sites.push({ field: key, target, condition: value }); + } + } + return sites; +} + /** * The walk {@link collectStoredMetadataFilterFields} runs over one filter * condition. A node is visited as a CONDITION (its non-`$` keys are columns) @@ -12448,55 +12497,99 @@ export class ObjectStackProtocolImplementation implements * - `groupBy` and `aggregations[].field` — defence in depth; the engine's * `rejectCredentialAggregation` refuses these for every caller with the * same code, and this door answers first so the envelope is the door's. - * - a one-level `expand` entry's own `where` / `orderBy`, judged against - * the TARGET object's internal fields: the nested sub-read's row strip - * already withholds the column, but a nested filter is a presence oracle - * on the related record (the expanded object appears only when the guess - * matches), so the evaluate positions are refused there too. Target - * OBJECT exposure is a different axis, filed separately (#22661). + * - a NESTED-RELATION condition inside any of those filters + * (`{ : { : … } }`), judged against + * the RELATED object's own internal fields: the engine lowers it by + * evaluating the condition on the related object + * (`ObjectQL.lowerRelationConditions`) and keeps the rows whose related + * record matched, so a related `internal` field there is the same + * confirmation oracle one hop away. Judged here, at the door, rather than + * in the engine's relation-site walk, so the engine's privileged callers + * keep the #7823 split (none of them reaches this door). + * - every `expand` entry's own `where` / `orderBy`, at EVERY depth of the + * expand tree, judged against that level's TARGET object: the nested + * sub-read's row strip already withholds the column, but a nested filter + * is a presence oracle on the related record (the expanded object appears + * only when the guess matches), and a deeper level's sub-read is one more + * `engine.find` that answers the same way. Target OBJECT exposure is a + * different axis, filed separately (#22661). + * + * Not judged here because the ingress already refuses them, measured: a + * DOTTED sort or group-by path crossing a relation (the sort gate's + * `dotted` verdict, `INVALID_SORT`; the group-by gate's unknown-field + * verdict, `INVALID_FIELD`), and a dotted filter key (the engine's dotted + * door). A dotted sort inside an `expand` entry is admitted but orders a + * batch the expansion re-keys by id with no `limit`, so it changes no answer. */ private assertNoInternalFieldEvaluated(object: string, options: Record): void { const gate = this.resolveQueryFields(object); - if (gate) { - const internal = new Set(collectInternalWriteResponseFields(gate.schema)); - if (internal.size > 0) { - // where + the `filter` alias a direct bag may still carry. - for (const [slot, where] of [['filter', options.where], ['filter', options.filter]] as const) { - this.refuseInternalReads(object, where, internal, slot); - } - this.refuseInternalNames(object, this.orderByFieldNames(options.orderBy), internal, 'sort'); - this.refuseInternalNames(object, this.groupByFieldNames(options.groupBy), internal, 'groupBy'); - if (Array.isArray(options.aggregations)) { - options.aggregations.forEach((agg: any, i: number) => { - const field = agg?.field; - if (typeof field === 'string' && field !== '*') { - this.refuseInternalNames(object, [field], internal, `aggregations[${i}].field`); - } - this.refuseInternalReads(object, agg?.filter, internal, `aggregations[${i}].filter`); - }); + if (!gate) return; + const internal = new Set(collectInternalWriteResponseFields(gate.schema)); + // where + the `filter` alias a direct bag may still carry, each with its + // nested-relation conditions. + this.refuseInternalInFilter(object, options.where, 'filter', 0); + this.refuseInternalInFilter(object, options.filter, 'filter', 0); + this.refuseInternalNames(object, this.orderByFieldNames(options.orderBy), internal, 'sort'); + this.refuseInternalNames(object, this.groupByFieldNames(options.groupBy), internal, 'groupBy'); + if (Array.isArray(options.aggregations)) { + options.aggregations.forEach((agg: any, i: number) => { + const field = agg?.field; + if (typeof field === 'string' && field !== '*') { + this.refuseInternalNames(object, [field], internal, `aggregations[${i}].field`); } - } + this.refuseInternalInFilter(object, agg?.filter, `aggregations[${i}].filter`, 0); + }); } - // One-level expand: each entry's own evaluate positions, against the - // TARGET object's internal fields. - const expand = options.expand; - if (gate && expand && typeof expand === 'object' && !Array.isArray(expand)) { - for (const [rel, entry] of Object.entries(expand as Record)) { - if (!entry || typeof entry !== 'object') continue; - const hasEvaluatePosition = entry.where !== undefined || entry.orderBy !== undefined; - if (!hasEvaluatePosition) continue; - const sourceField = gate.fields?.[rel]; - const target = sourceField != null && REFERENCE_VALUE_TYPES.has(sourceField.type) - ? referenceTargetOf(sourceField) - : undefined; - if (!target) continue; - const targetGate = this.resolveQueryFields(target); - if (!targetGate) continue; + // The whole expand tree, every level against its own target. + this.refuseInternalInExpand(object, options.expand, 'expand', 0); + } + + /** + * One filter's internal-field judgement: every column it reads on `object` + * ({@link collectFilterReads}), then each nested-relation condition it holds, + * recursively, on the related object. `param` names the position, extended + * by the relation field at each hop (`filter.owner_id`). + */ + private refuseInternalInFilter(object: string, where: unknown, param: string, depth: number): void { + if (where == null || depth > INTERNAL_FIELD_WALK_DEPTH) return; + const gate = this.resolveQueryFields(object); + if (!gate) return; + const condition = isFilterAST(where) ? parseFilterAST(where) : where; + const internal = new Set(collectInternalWriteResponseFields(gate.schema)); + if (internal.size > 0) { + const reads = new Set(); + collectFilterReads(condition, reads); + this.refuseInternalNames(object, [...reads], internal, param); + } + for (const site of relationConditionSites(gate.fields, condition)) { + this.refuseInternalInFilter(site.target, site.condition, `${param}.${site.field}`, depth + 1); + } + } + + /** + * Every level of an `expand` tree: each entry's own `where` (with its + * nested-relation conditions) and `orderBy`, judged against that entry's + * target object, then the entry's own `expand` with the target as source. + */ + private refuseInternalInExpand(source: string, expand: unknown, prefix: string, depth: number): void { + if (!isPlainRecord(expand) || depth > INTERNAL_FIELD_WALK_DEPTH) return; + const gate = this.resolveQueryFields(source); + if (!gate) return; + for (const [rel, entry] of Object.entries(expand)) { + if (!isPlainRecord(entry)) continue; + const sourceField = gate.fields?.[rel]; + const target = sourceField != null && REFERENCE_VALUE_TYPES.has(sourceField.type) + ? referenceTargetOf(sourceField) + : undefined; + if (!target) continue; + const path = `${prefix}['${rel}']`; + this.refuseInternalInFilter(target, entry.where, `${path}.filter`, 0); + const targetGate = this.resolveQueryFields(target); + if (targetGate) { const targetInternal = new Set(collectInternalWriteResponseFields(targetGate.schema)); - if (targetInternal.size === 0) continue; - this.refuseInternalReads(target, entry.where, targetInternal, `expand['${rel}'].filter`); - this.refuseInternalNames(target, this.orderByFieldNames(entry.orderBy), targetInternal, `expand['${rel}'].sort`); + this.refuseInternalNames(target, this.orderByFieldNames(entry.orderBy), targetInternal, `${path}.sort`); } + this.refuseInternalInExpand(target, entry.expand, `${path}.expand`, depth + 1); } } @@ -12518,14 +12611,6 @@ export class ObjectStackProtocolImplementation implements .map((f) => f.split('.')[0] as string); } - /** Refuse when any READ column of `where` (keys and `{ $field }` comparands) is internal. */ - private refuseInternalReads(object: string, where: unknown, internal: ReadonlySet, param: string): void { - if (where == null) return; - const reads = new Set(); - collectFilterReads(isFilterAST(where) ? parseFilterAST(where) : where, reads); - this.refuseInternalNames(object, [...reads], internal, param); - } - /** Refuse when any of `names` is an internal field of `object`. */ private refuseInternalNames(object: string, names: readonly string[], internal: ReadonlySet, param: string): void { const hit = names.filter((n) => internal.has(n)); From e4682bbf232f4c350ea41c0a485f8e59b50c29fc Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 17:39:27 +0000 Subject: [PATCH 5/6] docs(api): INVALID_FIELD also refuses an internal field named in an evaluate position (#22646) Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01JfJfBUC3cQ6hhgm9MQK76T --- content/docs/api/error-catalog.mdx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/content/docs/api/error-catalog.mdx b/content/docs/api/error-catalog.mdx index 129ff7551f8..e16e11a4ee8 100644 --- a/content/docs/api/error-catalog.mdx +++ b/content/docs/api/error-catalog.mdx @@ -147,6 +147,13 @@ searchable set gets its own message, and so does a value with the wrong shape an array entry that is not a string, or a value that is itself neither a comma-separated string nor an array of field names), `groupBy`, and `aggregations[].field`. +The same code also refuses a field that DOES exist but is declared `internal: true`, +when a read names it in a position that would evaluate its stored value: a filter +(including a condition on a related object reached through a lookup, and an +aggregation's own `filter`), a sort, a `groupBy`, an `aggregations[].field`, a +`searchFields` entry, or an `expand` entry's own filter or sort at any depth. Such a +field is never returned on a generic read, so it cannot be queried by either — the +refusal names the object that declares it. Query by another field. Off the request path the same code answers `backfillSummaryNulls`'s `recomputeUndefinedOnEmpty` (`os migrate summary-nulls --recompute-undefined-on-empty object.field`) when an entry is not a roll-up From b5d83b18c2edffd906c86916b54aa17adf8419ad Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 17:39:49 +0000 Subject: [PATCH 6/6] chore(changeset): name the relation-condition and every-depth expand positions (#22646) Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01JfJfBUC3cQ6hhgm9MQK76T --- .changeset/22646-internal-field-data-door-positions.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.changeset/22646-internal-field-data-door-positions.md b/.changeset/22646-internal-field-data-door-positions.md index 47e0d332165..79495c302c4 100644 --- a/.changeset/22646-internal-field-data-door-positions.md +++ b/.changeset/22646-internal-field-data-door-positions.md @@ -3,7 +3,7 @@ '@objectstack/metadata-protocol': minor --- -fix(objectql,metadata-protocol)!: the data door honours a field's `internal: true` in its filter, sort, group-by, aggregate, per-aggregation-filter, `$search` and one-level `$expand` positions, not only the row (#22646) +fix(objectql,metadata-protocol)!: the data door honours a field's `internal: true` in its filter (related-object conditions included), sort, group-by, aggregate, per-aggregation-filter, `$search` and `$expand` positions at every depth, not only the row (#22646) Clause-②: no (narrowing) @@ -20,11 +20,12 @@ A field declared `internal: true` is withheld from every generic exit (#21197): - A **group-by** of, or an **aggregate operand** over, such a field was refused by the engine, but as an undeclared `500 INTERNAL_ERROR` (a bare `Error` with no `code` or `status`), not a declared refusal. - A **per-aggregation filter** (`aggregations[].filter`) naming it, and a **filter beside an aggregation**, were admitted — the same oracle at a second filter position. - A **`$search`** whose resolved set included such a field (a non-hidden `internal` text column enters the auto-default) scanned it — a cross-field substring match that confirms a guessed prefix. -- A one-level **`$expand`**'s own `where` / `orderBy` on the TARGET object's `internal` field was admitted into the expansion sub-read — the expanded record's presence is an oracle on the related row. +- A **nested-relation condition** (a condition on a lookup's related object) naming the related object's `internal` field was evaluated on that object — the parent row came back only when the related value matched, the same oracle one hop away. +- An **`$expand`** entry's own `where` / `orderBy` on its TARGET object's `internal` field was admitted into the expansion sub-read, at the first level and at every deeper level — the expanded record's presence is an oracle on the related row. **TO.** Each evaluate position is refused `400 INVALID_FIELD`, a declared ADR-0112 envelope naming the object and the field, before the engine is asked: -- Filter, sort, group-by, aggregate operand, per-aggregation filter and a one-level expand's own filter/sort are refused at the generic data door (`@objectstack/metadata-protocol`), after the field-existence gates, in the shape and envelope of the stored-metadata body/hash family one axis over. +- Filter, sort, group-by, aggregate operand and per-aggregation filter are refused at the generic data door (`@objectstack/metadata-protocol`), after the field-existence gates, in the shape and envelope of the stored-metadata body/hash family one axis over. A nested-relation condition inside any filter is judged against the RELATED object's own `internal` fields, and every level of an `$expand` tree has its own filter / sort judged against that level's target object; the refusal names the object that declares the field. - The engine's `aggregate()` refusal (`rejectCredentialAggregation`) now carries the `INVALID_FIELD` / 400 envelope instead of the bare `Error`, so the group-by and aggregate-operand positions answer a declared `400` rather than a `500` — and the `secret` / `password` case that shared that bare `Error` is upgraded with it. - A `$search` never scans an `internal: true` column: the field is dropped from the set the engine's search expansion resolves over (a withhold, like the auto-default's `hidden` / credential-type exclusions), and an explicit `$searchFields` naming one is refused `400 INVALID_FIELD` at the door rather than silently widened back to the default set.