From 85663e96a45cbd19e3b26c01b8e4a6d36910c933 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 00:24:18 +0000 Subject: [PATCH 1/8] fix(objectql): refuse a plain group over a virtual field at the aggregate door A groupBy entry of any form naming a formula field (no stored column) is refused INVALID_FIELD / 400 at groupBy.N / groupBy.N.field before any driver is asked; a bucket over one keeps the date-bucket words. Claude-Session: https://claude.ai/code/session_016WV4cyTehJsfYGVMzcXLJD Co-authored-by: Claude --- .../src/aggregate-query-shape-door.ts | 102 ++++++++++++++---- .../engine-aggregate-query-shape-door.test.ts | 51 ++++++++- .../src/engine-delete-and-aggregate.ts | 11 +- 3 files changed, 140 insertions(+), 24 deletions(-) diff --git a/packages/objectql/src/aggregate-query-shape-door.ts b/packages/objectql/src/aggregate-query-shape-door.ts index 84e537e90e9..99b3bf59e3c 100644 --- a/packages/objectql/src/aggregate-query-shape-door.ts +++ b/packages/objectql/src/aggregate-query-shape-door.ts @@ -12,6 +12,10 @@ * used twice, an aggregation alias equal to a group column, or two group * entries projecting one column — {@link assertAggregatedColumnsAreDistinct}. * + * [#29167, #29304] …and, ahead of the first, a `groupBy` entry of any form — + * a name, `{ field }`, or a bucket — naming a VIRTUAL field, which no driver + * stores a column for — {@link assertGroupByTargetsAreStored}. + * * ## What ran before this door, measured on `origin/main` `cb0a2fcf4f` * * Through `findData` (the `POST /api/v1/data/:object/query` door) over three @@ -48,7 +52,7 @@ * ingress door answers an unknown one first; a system column the map does not * declare is bucketed as before), or a registry-less host. * - * ## A virtual field (`formula`) is refused, on its storage ground [#29167] + * ## A virtual field (`formula`) is refused, on its storage ground [#29167, #29304] * * A `formula` names no stored value class (its value is the expression's), so * the class reading above cannot judge it. It is refused on the ground the @@ -56,18 +60,26 @@ * VIRTUAL — computed on read, no column — judged by the spec's * `isVirtualSearchField` (`SEARCH_VIRTUAL_TYPES`), never a local type list, * and whatever its `returnType` says. Measured on `origin/main` `7788eedb` - * before this arm, a month bucket over a `formula` with `returnType: 'date'`: + * before the bucket arm (#29167), a month bucket over a `formula` with + * `returnType: 'date'`, and on `origin/main` `447226b2` before the plain arm + * (#29304), a plain group over a `formula` (`returnType` `date` or `text`): * * | caller | answer | * |:--|:--| * | a SQL driver | `INVALID_FIELD` / 400 from the driver's missing-column backstop: no `path`, and a "run schema sync" prescription that cannot help | * | a driver with no native `aggregate` (the in-memory aggregation) | `200`, one `null` group holding every row | * - * Both now answer {@link virtualDateBucketMessage}, `INVALID_FIELD` / 400 at - * `groupBy.N.dateGranularity`, with the remedy those axes give: store the - * value on a stored field and use that. `@objectstack/service-analytics`'s - * `dataset-date-bucket-door.ts` gives the same refusal on the two faces that - * reach no engine (the `/analytics/sql` dry run and the draft preview). + * Every `groupBy` entry is judged, whatever its form, by + * {@link assertGroupByTargetsAreStored}, ahead of the date-bucket door: a + * bucket answers {@link virtualDateBucketMessage} at + * `groupBy.N.dateGranularity` (the #29167 words), a plain group answers + * {@link virtualGroupMessage} at `groupBy.N` (a name) or `groupBy.N.field` + * (the object form) — `INVALID_FIELD` / 400 either way, with the remedy those + * axes give: store the value on a stored field and use that. + * `@objectstack/service-analytics`'s `dataset-date-bucket-door.ts` gives the + * same refusals on the faces that reach no engine (the `/analytics/sql` dry + * run and the draft preview), and its native-SQL strategy declines a grouped + * dimension over a virtual field, so that face reaches this door. * * ## The column names * @@ -127,11 +139,69 @@ function virtualDateBucketMessage( + 'that aggregates in memory put every row into one null group while the answer looked served.'; } +/** + * [#29304] The refusal of a plain group over a VIRTUAL field (see the module + * header): {@link virtualDateBucketMessage}'s words for a group key with no + * bucket. `@objectstack/service-analytics`'s `dataset-date-bucket-door.ts` + * carries the one copy, compared field for field by its tests. + */ +function virtualGroupMessage(object: string, path: string, field: string, type: string): string { + return `aggregate('${object}'): ${path} groups by '${field}', but '${field}' is a ${type} field — its value ` + + 'is computed on read and never stored, so no driver materialises a column to group by. The query was NOT ' + + `run. Denormalise the value onto '${object}' (a stored field, written when the source changes) and group ` + + 'by that. A formula field is virtual: with no column behind it the group key reached the driver as a ' + + 'column that does not exist, and an engine that aggregates in memory put every row into one null group ' + + 'while the answer looked served.'; +} + +/** + * Refuse a `groupBy` entry of any form — a name, `{ field }`, or a + * `{ field, dateGranularity }` bucket — naming a field the object declares + * VIRTUAL (`formula`; the spec's `isVirtualSearchField`), `INVALID_FIELD` / + * 400 at the position it names. See the module header. + */ +export function assertGroupByTargetsAreStored( + object: string, + schema: unknown, + groupBy: unknown, +): void { + if (!Array.isArray(groupBy) || groupBy.length === 0) return; + const fields = (schema as { fields?: unknown } | undefined)?.fields; + if (!fields || typeof fields !== 'object' || Array.isArray(fields)) return; + for (const [i, entry] of groupBy.entries()) { + const named = typeof entry === 'string'; + if (!named && (entry === null || typeof entry !== 'object' || Array.isArray(entry))) continue; + const { field, dateGranularity } = named + ? { field: entry, dateGranularity: undefined } + : entry as { field?: unknown; dateGranularity?: unknown }; + if (typeof field !== 'string' || !Object.prototype.hasOwnProperty.call(fields, field)) continue; + const declared = (fields as Record)[field]; + if (!isVirtualSearchField(declared as never)) continue; + const type = String(declared?.type); + if (dateGranularity != null) { + // [#29167] The bucket keeps its own words and position. + const path = `groupBy.${i}.dateGranularity`; + throw refusal( + 'INVALID_FIELD', + virtualDateBucketMessage(object, path, field, type, String(dateGranularity)), + { field, fields: [field], object, param: 'groupBy', path }, + ); + } + const path = named ? `groupBy.${i}` : `groupBy.${i}.field`; + throw refusal( + 'INVALID_FIELD', + virtualGroupMessage(object, path, field, type), + { field, fields: [field], object, param: 'groupBy', path }, + ); + } +} + /** * Refuse a `groupBy` entry whose `dateGranularity` buckets a field the object - * declares neither a date nor a datetime, or a virtual (`formula`) field — - * `INVALID_FIELD` / 400, naming the position, the field and its declared type. - * See the module header. + * declares neither a date nor a datetime — `INVALID_FIELD` / 400, naming the + * position, the field and its declared type. A virtual field is judged ahead + * of this door, by {@link assertGroupByTargetsAreStored}. See the module + * header. */ export function assertDateBucketTargetsAreTemporal( object: string, @@ -147,18 +217,6 @@ export function assertDateBucketTargetsAreTemporal( const { field, dateGranularity } = entry as { field?: unknown; dateGranularity?: unknown }; if (dateGranularity == null || typeof field !== 'string') continue; const path = `groupBy.${i}.dateGranularity`; - const declared = Object.prototype.hasOwnProperty.call(fields, field) - ? (fields as Record)[field] - : undefined; - // [#29167] A virtual field first: the class reading below has no class - // for it, so it would otherwise be skipped unjudged. - if (isVirtualSearchField(declared as never)) { - throw refusal( - 'INVALID_FIELD', - virtualDateBucketMessage(object, path, field, String(declared?.type), String(dateGranularity)), - { field, fields: [field], object, param: 'groupBy', path }, - ); - } classes ??= declaredFieldClasses(fields); const cls = classes.get(field); if (cls === undefined || cls === 'date' || cls === 'datetime') continue; diff --git a/packages/objectql/src/engine-aggregate-query-shape-door.test.ts b/packages/objectql/src/engine-aggregate-query-shape-door.test.ts index 7633565a6e4..9c7d3567627 100644 --- a/packages/objectql/src/engine-aggregate-query-shape-door.test.ts +++ b/packages/objectql/src/engine-aggregate-query-shape-door.test.ts @@ -14,6 +14,10 @@ * query runs the engine's in-memory aggregation over them: the controls below * read real buckets, and a refusal is proven to precede every read. * + * [#29304] …and a plain group (a name, or `{ field }`) over a virtual + * (`formula`) field, beside the #29167 bucket over one: `INVALID_FIELD` / 400 + * at `groupBy.N` / `groupBy.N.field`. + * * [#25819] The REST door's empty-list refusal is the protocol's own * (`protocol.empty-aggregate-axis.test.ts`); the wire half below runs it over * the real engine, which is what turned it into `404 OBJECT_NOT_FOUND`. @@ -128,6 +132,45 @@ describe('[#25818] the aggregate query shapes that answered a plausible wrong ta expect(reads, 'every refusal precedes the driver').toHaveLength(0); }); + it('[#29304] refuses a plain group over a virtual (formula) field, whatever its returnType: INVALID_FIELD / 400 at groupBy.N (a name) or groupBy.N.field — no read', async () => { + // Before this arm the in-memory aggregation served one null group holding + // every row, and a SQL driver refused a missing column with no path. + const cases: ReadonlyArray = [ + ['doubled', 'doubled', 'groupBy.1'], + ['opened_calc', 'opened_calc', 'groupBy.1'], + [{ field: 'opened_calc' }, 'opened_calc', 'groupBy.1.field'], + [{ field: 'doubled', alias: 'd' }, 'doubled', 'groupBy.1.field'], + ]; + for (const [entry, field, path] of cases) { + const label = JSON.stringify(entry); + const err = await refusalOf(aggregate({ groupBy: ['status', entry], aggregations: [COUNT] })); + expect(envelopeOf(err), label).toEqual({ code: 'INVALID_FIELD', status: 400, httpStatus: 400 }); + expect({ path: err?.path, param: err?.param, field: err?.field, fields: err?.fields, object: err?.object }, label) + .toEqual({ path, param: 'groupBy', field, fields: [field], object: OBJECT }); + expect(err!.message, label).toBe( + `aggregate('${OBJECT}'): ${path} groups by '${field}', but '${field}' is a formula field — its value is computed ` + + 'on read and never stored, so no driver materialises a column to group by. The query was NOT run. Denormalise ' + + `the value onto '${OBJECT}' (a stored field, written when the source changes) and group by that. A formula ` + + 'field is virtual: with no column behind it the group key reached the driver as a column that does not exist, ' + + 'and an engine that aggregates in memory put every row into one null group while the answer looked served.', + ); + } + // The first offending position is named, whichever axis would refuse later ones. + const first = await refusalOf(aggregate({ groupBy: ['doubled', { field: 'name', dateGranularity: 'month' }], aggregations: [COUNT] })); + expect({ code: first?.code, path: first?.path }).toEqual({ code: 'INVALID_FIELD', path: 'groupBy.0' }); + expect(reads, 'every refusal precedes the driver').toHaveLength(0); + }); + + it('[#29304] the REST door answers the plain-group refusal at its path, through findData and the wire mapping', async () => { + const protocol = new ObjectStackProtocolImplementation(engine); + const err = await refusalOf(protocol.findData({ object: OBJECT, query: { groupBy: ['opened_calc'], aggregations: [COUNT] } } as any)); + expect({ code: err?.code, status: err?.status, path: err?.path }).toEqual({ code: 'INVALID_FIELD', status: 400, path: 'groupBy.0' }); + const wire = mapDataError(err, OBJECT); + expect({ status: wire.status, code: wire.body.code }).toEqual({ status: 400, code: 'INVALID_FIELD' }); + expect(String(wire.body.error)).toContain("groupBy.0 groups by 'opened_calc', but 'opened_calc' is a formula field"); + expect(reads).toHaveLength(0); + }); + it('refuses a repeated aggregation alias: INVALID_QUERY / 400 at the second aggregations.N.alias — no read', async () => { const err = await refusalOf(aggregate({ aggregations: [COUNT, { function: 'sum', field: 'amount', alias: 'n' }], @@ -174,6 +217,12 @@ describe('[#25818] the aggregate query shapes that answered a plausible wrong ta // The declaration cannot type an undeclared column: no verdict. [#29167] A // formula under a dateGranularity is refused (above), no longer a control. await expect(aggregate({ groupBy: [{ field: 'created_at', dateGranularity: 'month' }], aggregations: [COUNT] })).resolves.toBeDefined(); + // [#29304] A stored field grouped plainly, as a name and in the object form, + // and a value aggregate beside no virtual group key: served. + await expect(aggregate({ groupBy: ['status'], aggregations: [COUNT] })) + .resolves.toEqual([{ status: 'open', n: 2 }, { status: 'won', n: 1 }]); + await expect(aggregate({ groupBy: [{ field: 'status' }], aggregations: [COUNT] })) + .resolves.toEqual([{ status: 'open', n: 2 }, { status: 'won', n: 1 }]); // An alias moves the group column, so the field's own name is free. await expect(aggregate({ groupBy: [{ field: 'status', alias: 's' }], aggregations: [{ function: 'count', alias: 'status' }] })) .resolves.toEqual([{ s: 'open', status: 2 }, { s: 'won', status: 1 }]); @@ -182,7 +231,7 @@ describe('[#25818] the aggregate query shapes that answered a plausible wrong ta groupBy: [{ field: 'opened_on', dateGranularity: 'year', alias: 'y' }, { field: 'opened_on', dateGranularity: 'month', alias: 'm' }], aggregations: [COUNT, { function: 'sum', field: 'amount', alias: 'total' }], })).resolves.toEqual([{ y: '2026', m: '2026-01', n: 1, total: 10 }, { y: '2026', m: '2026-02', n: 2, total: 50 }]); - expect(reads, "each served query read once").toHaveLength(5); + expect(reads, "each served query read once").toHaveLength(7); }); it('the REST door answers the engine\'s refusal, not a 200 — through findData and the wire mapping', async () => { diff --git a/packages/objectql/src/engine-delete-and-aggregate.ts b/packages/objectql/src/engine-delete-and-aggregate.ts index 73eef2bc244..d369164d0d2 100644 --- a/packages/objectql/src/engine-delete-and-aggregate.ts +++ b/packages/objectql/src/engine-delete-and-aggregate.ts @@ -40,7 +40,11 @@ import { assertNoInternalFieldEvaluated, rejectInternalFieldEvaluation } from '. import { assertGroupByNamesNoJsonStoredField } from './group-by-structured-json-door.js'; import { assertAggregationFieldTypesAccepted } from './aggregate-field-type-door.js'; import { assertRecordCurrencyAggregatesGrouped } from './record-currency-aggregate-door.js'; -import { assertAggregatedColumnsAreDistinct, assertDateBucketTargetsAreTemporal } from './aggregate-query-shape-door.js'; +import { + assertAggregatedColumnsAreDistinct, + assertDateBucketTargetsAreTemporal, + assertGroupByTargetsAreStored, +} from './aggregate-query-shape-door.js'; import { applyInMemoryAggregation } from './in-memory-aggregation.js'; import { aggregatedRowColumnClasses, aggregatedRowColumnTypes, aggregatedRowColumns, applyHaving, assertAggregationFilterIsEvaluable, assertHavingIsEvaluable, assertHavingIsFilterCondition } from './having-filter.js'; import { @@ -131,6 +135,11 @@ export class DeleteAndAggregateFacet { // (every row in one null bucket), and two columns of the aggregated row // under one name (the last value written won). After the two doors // above, so a JSON field under a date bucket keeps their words. + // [#29167, #29304] A group entry of any form over a VIRTUAL (`formula`) + // field first: no driver stores a column for it (a SQL driver refused + // it as a missing column, the in-memory aggregation served one null + // group), and the date-bucket door has no class to judge it by. + assertGroupByTargetsAreStored(object, this._registry.getObject(object), query.groupBy); assertDateBucketTargetsAreTemporal(object, this._registry.getObject(object), query.groupBy); assertAggregatedColumnsAreDistinct(object, query.groupBy, query.aggregations); // [#10576] The per-aggregation `filter` (`AggregationNodeSchema.filter`, From 9ac634685b3069eb82d1aebfba3b8e0ec50b4e89 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 00:26:07 +0000 Subject: [PATCH 2/8] fix(service-analytics): refuse a plain group over a virtual field on every analytics face Claude-Session: https://claude.ai/code/session_016WV4cyTehJsfYGVMzcXLJD Co-authored-by: Claude --- .../src/analytics-service.ts | 20 ++++-- .../src/dataset-date-bucket-door.ts | 68 +++++++++++++++---- .../src/strategies/native-sql-strategy.ts | 46 ++++++++++++- .../src/strategies/objectql-strategy.ts | 6 +- 4 files changed, 120 insertions(+), 20 deletions(-) diff --git a/packages/services/service-analytics/src/analytics-service.ts b/packages/services/service-analytics/src/analytics-service.ts index bca6a0582f4..2a19c3dc8e4 100644 --- a/packages/services/service-analytics/src/analytics-service.ts +++ b/packages/services/service-analytics/src/analytics-service.ts @@ -125,7 +125,7 @@ import { } from './dimension-labels.js'; import { evaluateAnalyticsQueryOverRows, lowerPreviewRowFilters } from './preview-evaluator.js'; import { assertDatasetGroupColumnsDistinct } from './dataset-group-column-door.js'; -import { assertDatasetDateBucketsTemporal } from './dataset-date-bucket-door.js'; +import { assertDatasetDateBucketsTemporal, assertDatasetGroupTargetsStored } from './dataset-date-bucket-door.js'; // [#25778] The one lookup of a dataset duration measure by name. import { durationMeasureIn } from './duration-measure-sql.js'; import { scopedMeasureFilter, scopedMeasureKey } from './scoped-measure-key.js'; @@ -2967,6 +2967,10 @@ export class AnalyticsService implements IAnalyticsService { // nor a datetime, with the refusal the engine gives both live faces // for the same keys (after the column refusal above, as there), // where the evaluator served one `null` (or `1970`) group. + // [#29304] A key of any form over a virtual (`formula`) field + // first, as the engine judges it first: the evaluator served one + // `null` group holding the total for a plain dimension over one. + assertDatasetGroupTargetsStored(aggregateObjectOf(cube), aggregateGroupKeys(q, cube), declaredType); assertDatasetDateBucketsTemporal(aggregateObjectOf(cube), aggregateGroupKeys(q, cube), declaredType); return evaluateAnalyticsQueryOverRows( q, @@ -5125,16 +5129,20 @@ export function inferredCallerMeasureSql(measure: string, cubeName: string): str * `executeAggregate`, so every bucketed query goes to the engine); reached by * a host composing `new AnalyticsService({ fallbackService })`, or a * `queryCapabilities` override with `objectqlAggregate: false`. + * + * [#29304] The storage-axis door (`assertDatasetGroupTargetsStored`) runs + * first, on the same keys: a key of any form over a virtual (`formula`) field + * is refused as the engine refuses it. */ function assertDelegatedDateBucketsTemporal(query: AnalyticsQuery, ctx: StrategyContext): void { const cube = ctx.getCube(query.cube!); if (!cube) return; const object = aggregateObjectOf(cube); - assertDatasetDateBucketsTemporal( - object, - aggregateGroupKeys(query, cube), - (field) => (ctx as DatasetScopedStrategyContext).declaredFieldType?.(object, field), - ); + const groupBy = aggregateGroupKeys(query, cube); + const declaredType = (field: string) => (ctx as DatasetScopedStrategyContext).declaredFieldType?.(object, field); + // [#29304] A key of any form over a virtual field first, as the engine. + assertDatasetGroupTargetsStored(object, groupBy, declaredType); + assertDatasetDateBucketsTemporal(object, groupBy, declaredType); } /** diff --git a/packages/services/service-analytics/src/dataset-date-bucket-door.ts b/packages/services/service-analytics/src/dataset-date-bucket-door.ts index 551aba5a64c..8e46c9b1bca 100644 --- a/packages/services/service-analytics/src/dataset-date-bucket-door.ts +++ b/packages/services/service-analytics/src/dataset-date-bucket-door.ts @@ -44,6 +44,19 @@ * both its query and its dry run (`analytics-service.ts`, * `assertDelegatedDateBucketsTemporal`). * + * [#29304] …and a PLAIN group key over a virtual field, which the engine's + * storage-axis door (`assertGroupByTargetsAreStored`) now refuses ahead of its + * date-bucket door, `INVALID_FIELD` / 400 at `groupBy.N`. Measured on + * `origin/main` `447226b2` before it, a dataset or cube dimension over a + * `formula` (`returnType` `date` or `text`): the dry run rendered + * `GROUP BY calc` on both compositions and the preview served one `null` + * group holding the total, while the native strategy answered + * `DATABASE_ERROR` / 500 and the ObjectQL strategy the SQL driver's + * missing-column backstop. {@link assertDatasetGroupTargetsStored} gives the + * engine's refusal on the faces that hand it no keys (the preview, the dry + * run, the delegated service), and `NativeSQLStrategy.canHandle` declines a + * grouped dimension over a virtual field so the live query reaches the engine. + * * Why a copy of the sentence: `@objectstack/objectql` is a dev dependency of * this package, never a runtime one, and its root does not export the door. * The message, code, status and keys are the engine's, and @@ -61,7 +74,7 @@ type DateBucketRefusal = Error & { field?: string; fields?: string[]; object?: string; param?: string; path?: string; }; -/** The refusal `assertDatasetDateBucketsTemporal` throws, with the engine's keys. */ +/** The refusal this module's two doors throw, with the engine's keys. */ function dateBucketRefusal(message: string, object: string, field: string, path: string): DateBucketRefusal { const err = new Error(message) as DateBucketRefusal; err.code = StandardErrorCode.enum.INVALID_FIELD; @@ -72,25 +85,27 @@ function dateBucketRefusal(message: string, object: string, field: string, path: } /** - * Refuse the first bucketed key in `groupBy` (built for a query on `object`) - * whose field `declaredType` reads as a virtual field, or as declared and - * neither a date nor a datetime, with the engine's refusal for it. + * [#29167, #29304] Refuse the first key in `groupBy` (built for a query on + * `object`), plain or bucketed, whose field `declaredType` reads as a VIRTUAL + * field (the spec's `isVirtualSearchField`), with the engine's refusal for it: + * `aggregate-query-shape-door.ts` `assertGroupByTargetsAreStored`, the bucket + * in `virtualDateBucketMessage`'s words, the plain key in + * `virtualGroupMessage`'s. Called ahead of + * {@link assertDatasetDateBucketsTemporal}, as the engine calls its two doors. */ -export function assertDatasetDateBucketsTemporal( +export function assertDatasetGroupTargetsStored( object: string, groupBy: readonly AggregateGroupKey[], declaredType: (field: string) => string | undefined, ): void { for (const [i, key] of groupBy.entries()) { - if (typeof key === 'string') continue; - const { field, dateGranularity } = key; + const field = typeof key === 'string' ? key : key.field; const type = declaredType(field); - const path = `groupBy.${i}.dateGranularity`; - // [#29167] The engine's virtual-field arm (`aggregate-query-shape-door.ts` - // `virtualDateBucketMessage`), judged first as there. - if (type !== undefined && isVirtualSearchField({ type } as never)) { + if (type === undefined || !isVirtualSearchField({ type } as never)) continue; + if (typeof key !== 'string') { + const path = `groupBy.${i}.dateGranularity`; throw dateBucketRefusal( - `aggregate('${object}'): ${path} buckets '${field}' by '${dateGranularity}', but '${field}' ` + `aggregate('${object}'): ${path} buckets '${field}' by '${key.dateGranularity}', but '${field}' ` + `is a ${type} field — its value is computed on read and never stored, so no driver materialises a ` + `column to bucket. The query was NOT run. Denormalise the value onto '${object}' (a stored date or ` + 'datetime field, written when the source changes) and bucket that. A formula field is virtual: with ' @@ -99,6 +114,35 @@ export function assertDatasetDateBucketsTemporal( object, field, path, ); } + const path = `groupBy.${i}`; + throw dateBucketRefusal( + `aggregate('${object}'): ${path} groups by '${field}', but '${field}' is a ${type} field — its value ` + + 'is computed on read and never stored, so no driver materialises a column to group by. The query was NOT ' + + `run. Denormalise the value onto '${object}' (a stored field, written when the source changes) and group ` + + 'by that. A formula field is virtual: with no column behind it the group key reached the driver as a ' + + 'column that does not exist, and an engine that aggregates in memory put every row into one null group ' + + 'while the answer looked served.', + object, field, path, + ); + } +} + +/** + * Refuse the first bucketed key in `groupBy` (built for a query on `object`) + * whose field `declaredType` reads as declared and neither a date nor a + * datetime, with the engine's refusal for it. A virtual field is judged ahead + * of this door, by {@link assertDatasetGroupTargetsStored}. + */ +export function assertDatasetDateBucketsTemporal( + object: string, + groupBy: readonly AggregateGroupKey[], + declaredType: (field: string) => string | undefined, +): void { + for (const [i, key] of groupBy.entries()) { + if (typeof key === 'string') continue; + const { field, dateGranularity } = key; + const type = declaredType(field); + const path = `groupBy.${i}.dateGranularity`; if (type === undefined || CALENDAR_DATE_TYPES.has(type) || INSTANT_TYPES.has(type)) continue; throw dateBucketRefusal( `aggregate('${object}'): ${path} buckets '${field}' by '${dateGranularity}', but '${field}' ` diff --git a/packages/services/service-analytics/src/strategies/native-sql-strategy.ts b/packages/services/service-analytics/src/strategies/native-sql-strategy.ts index 39e3792e423..74c48148f05 100644 --- a/packages/services/service-analytics/src/strategies/native-sql-strategy.ts +++ b/packages/services/service-analytics/src/strategies/native-sql-strategy.ts @@ -60,7 +60,7 @@ import { AGGREGATE_ANSWER_KIND, presentAsNumber } from '@objectstack/core'; // boolean-aggregand cast, and the one column-class predicate both read — the // operand rule `driver-sql`'s own `aggregate()` applies, defined once in core. import { aggregandColumnClass, aggregandOperandSql } from '@objectstack/core'; -import { emptyGroupValueFor } from '@objectstack/spec/data'; +import { emptyGroupValueFor, isVirtualSearchField } from '@objectstack/spec/data'; import { explicitDateRangeWindow } from '../date-range-array-arm.js'; /** @@ -838,6 +838,21 @@ export class NativeSQLStrategy implements AnalyticsStrategy { // decline above is: the query routes on, and the service refuses a // duration measure no native statement serves, with the reason. if (this.durationMeasureOnUnknownDialectIn(query, ctx)) return false; + // ── [#29304] DECLINE a grouped dimension over a VIRTUAL field ────────── + // + // A `formula` is computed on read: no table stores a column for it, so the + // `GROUP BY calc` this strategy compiles reached the database as a column + // that does not exist and answered `DATABASE_ERROR` / 500, where the engine + // refuses the same group key `INVALID_FIELD` / 400 at its position with the + // store-the-value remedy (`@objectstack/objectql`'s + // `assertGroupByTargetsAreStored`). The mechanism of the declines above: + // what this strategy cannot serve as the engine would, the engine serves — + // here, refuses — so the query routes to the ObjectQL strategy and every + // composition answers one refusal. Only a GROUPED member is judged: a + // `dimensions` entry (a bucketed `timeDimensions` entry already declines + // above). A stored dimension, and a query that groups by no virtual field, + // stay here. See {@link groupsByVirtualField}. + if (this.groupsByVirtualField(query, ctx)) return false; const caps = ctx.queryCapabilities(query.cube); return caps.nativeSql && typeof ctx.executeRawSql === 'function'; } @@ -861,6 +876,35 @@ export class NativeSQLStrategy implements AnalyticsStrategy { return objects.some((object) => findReadableIdsLeaf(ctx.getReadScope!(object)) !== null); } + /** + * [#29304] Does a `dimensions` entry of this query resolve to a column the + * host declares VIRTUAL (the spec's `isVirtualSearchField`: `formula`)? See + * the decline at {@link canHandle}. Each member resolves to the (object, + * column) {@link resolveStorageTarget} gives the compiler — the object a + * relationship path's last hop reaches for a dotted one — and is asked of the + * host's `declaredFieldType`. An expression `sql`, a member that does not + * resolve, and a host that wired no field types are not judged: "cannot + * answer, do not block", the tiering every such hook here takes. + */ + private groupsByVirtualField(query: AnalyticsQuery, ctx: StrategyContext): boolean { + const declared = (ctx as DatasetScopedStrategyContext).declaredFieldType; + if (typeof declared !== 'function') return false; + const cube = query.cube ? ctx.getCube(query.cube) : undefined; + if (!cube) return false; + const baseTable = this.extractObjectName(cube); + const referenceOf = relationshipReferenceOf(ctx); + return (query.dimensions ?? []).some((member) => { + let target: { object: string; field: string }; + try { + target = this.resolveStorageTarget(cube, member, baseTable, referenceOf); + } catch { + return false; + } + const type = declared.call(ctx, target.object, target.field); + return type !== undefined && isVirtualSearchField({ type } as never); + }); + } + /** * [#25778] Does this query request a dataset duration measure on a base * object whose SQL dialect the host cannot name? See the decline at diff --git a/packages/services/service-analytics/src/strategies/objectql-strategy.ts b/packages/services/service-analytics/src/strategies/objectql-strategy.ts index 2ee53c6109a..2f5cc472637 100644 --- a/packages/services/service-analytics/src/strategies/objectql-strategy.ts +++ b/packages/services/service-analytics/src/strategies/objectql-strategy.ts @@ -8,7 +8,7 @@ import { markFilterSubtreeProvenance } from '@objectstack/spec/data'; import { StandardErrorCode } from '@objectstack/spec/api'; import type { AnalyticsStrategy, StrategyContext, DatasetScopedStrategyContext } from './types.js'; import { assertDatasetGroupColumnsDistinct } from '../dataset-group-column-door.js'; -import { assertDatasetDateBucketsTemporal } from '../dataset-date-bucket-door.js'; +import { assertDatasetDateBucketsTemporal, assertDatasetGroupTargetsStored } from '../dataset-date-bucket-door.js'; import { declaredDatetimeLowering, declaredDateColumn, @@ -803,6 +803,10 @@ export class ObjectQLStrategy implements AnalyticsStrategy { // the query is served. A host that names no declared type is not judged, // as the engine judges no undeclared field. const declaredType = (ctx as DatasetScopedStrategyContext).declaredFieldType; + // [#29304] A key of any form over a virtual (`formula`) field first, as + // the engine judges it first: the echo printed `GROUP BY calc` for a + // column no table stores. + assertDatasetGroupTargetsStored(tableName, echoGroupBy, (field) => declaredType?.call(ctx, tableName, field)); assertDatasetDateBucketsTemporal(tableName, echoGroupBy, (field) => declaredType?.call(ctx, tableName, field)); const joinClauses: string[] = []; // [#21441] The bucket expression the driver itself groups by for its From 6e07b4054a21cf133d757e72389a3051673f5a74 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 00:27:17 +0000 Subject: [PATCH 3/8] fix(service-analytics): judge a cross-object attribute over a virtual field on the query and its echo Claude-Session: https://claude.ai/code/session_016WV4cyTehJsfYGVMzcXLJD Co-authored-by: Claude --- .../src/strategies/objectql-strategy.ts | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/packages/services/service-analytics/src/strategies/objectql-strategy.ts b/packages/services/service-analytics/src/strategies/objectql-strategy.ts index 2f5cc472637..cbeced88789 100644 --- a/packages/services/service-analytics/src/strategies/objectql-strategy.ts +++ b/packages/services/service-analytics/src/strategies/objectql-strategy.ts @@ -146,6 +146,27 @@ interface CrossObjectPlan { crossDims: CrossObjectPlanDim[]; } +/** + * [#29304] The engine's refusal of a VIRTUAL (`formula`) attribute on the + * cross-object path, judged by the service's copy + * (`dataset-date-bucket-door.ts` `assertDatasetGroupTargetsStored`) on the + * keys that path's FK-resolve aggregate hands the related object + * (`resolveFkAttr`: `groupBy: ['id', attr]`). The engine refuses that + * aggregate, but only once a base row carries a key to resolve: an empty base + * served the dimension, and the echo, which runs nothing, rendered a `LEFT + * JOIN` grouping by a column no table stores. Called by both, after the base + * keys are judged, so the query and its echo give one answer on any data. A + * host that wired no field types is not judged. + */ +function assertCrossObjectAttrsStored(plan: CrossObjectPlan | null | undefined, ctx: StrategyContext): void { + const declared = (ctx as DatasetScopedStrategyContext).declaredFieldType; + if (!plan || typeof declared !== 'function') return; + for (const cd of plan.crossDims) { + if (cd.attr === 'id') continue; + assertDatasetGroupTargetsStored(cd.refObject, ['id', cd.attr], (field) => declared.call(ctx, cd.refObject, field)); + } +} + /** * Why the echo of a date bucket refuses: the in-memory zone, or no driver * expression for the bucket (with the dialect the host names, `'unknown'` when @@ -808,6 +829,9 @@ export class ObjectQLStrategy implements AnalyticsStrategy { // column no table stores. assertDatasetGroupTargetsStored(tableName, echoGroupBy, (field) => declaredType?.call(ctx, tableName, field)); assertDatasetDateBucketsTemporal(tableName, echoGroupBy, (field) => declaredType?.call(ctx, tableName, field)); + // [#29304] …then a cross-object dimension over a virtual attribute, as + // `executeCrossObject` judges it after its base aggregate. + assertCrossObjectAttrsStored(plan, ctx); const joinClauses: string[] = []; // [#21441] The bucket expression the driver itself groups by for its // dialect, read from the `dateBucketSql` hook: `strftime('%Y-%m', …)` on @@ -1534,6 +1558,10 @@ export class ObjectQLStrategy implements AnalyticsStrategy { context: ctx.context, }); + // [#29304] A cross-object dimension over a virtual attribute, whether or + // not a base row carries a key to resolve. See {@link assertCrossObjectAttrsStored}. + assertCrossObjectAttrsStored(plan, ctx); + // Resolve each cross-object dim's FK → attribute, SCOPED to the referenced // object: a related record the caller cannot read never yields its // attribute, so it buckets as RESTRICTED (no leak; ADR-0021 D-C / #3602). From 3a02407b9a4b6c97a0b4c560b1d68e642cced407 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 00:28:53 +0000 Subject: [PATCH 4/8] test(service-analytics): pin the plain-group refusal over a formula on every analytics face Claude-Session: https://claude.ai/code/session_016WV4cyTehJsfYGVMzcXLJD Co-authored-by: Claude --- ...fallback-delegate-date-bucket-type.test.ts | 5 ++ .../objectql-echo-non-date-bucket.test.ts | 67 ++++++++++++++++++- .../preview-date-bucket-type.test.ts | 41 ++++++++++++ 3 files changed, 111 insertions(+), 2 deletions(-) diff --git a/packages/services/service-analytics/src/__tests__/fallback-delegate-date-bucket-type.test.ts b/packages/services/service-analytics/src/__tests__/fallback-delegate-date-bucket-type.test.ts index 61383c61fdb..39f55390a67 100644 --- a/packages/services/service-analytics/src/__tests__/fallback-delegate-date-bucket-type.test.ts +++ b/packages/services/service-analytics/src/__tests__/fallback-delegate-date-bucket-type.test.ts @@ -27,6 +27,9 @@ * capturing a pre-registered analytics service under a `queryCapabilities` * override with `objectqlAggregate: false`. * + * [#29304] …and a plain group over the `formula`, which the engine face now + * refuses at `groupBy.0`: the delegate reads no field type either. + * * The delegate here is a recording double: the pin is the service's face (the * refusal, and that the delegate was never called), not the delegate's fold. */ @@ -182,6 +185,8 @@ describe('[#29364] the fallback face refuses a date bucket over a field that is ['a formula by month', bucket(['calc', 'month']), 'calc', 'groupBy.0.dateGranularity'], ['a text column by month', bucket(['region', 'month']), 'region', 'groupBy.0.dateGranularity'], ['a formula after a datetime bucket', bucket(['closed_at', 'year'], ['calc', 'month']), 'calc', 'groupBy.1.dateGranularity'], + // [#29304] A plain group over the formula, no bucket. + ['a formula, grouped plainly', { cube: CUBE, measures: [`${CUBE}.amount_sum`], dimensions: [`${CUBE}.calc`] }, 'calc', 'groupBy.0'], ]; for (const [faceLabel, makeFace] of FACES) { diff --git a/packages/services/service-analytics/src/__tests__/objectql-echo-non-date-bucket.test.ts b/packages/services/service-analytics/src/__tests__/objectql-echo-non-date-bucket.test.ts index 665e63aa880..35f34c42a20 100644 --- a/packages/services/service-analytics/src/__tests__/objectql-echo-non-date-bucket.test.ts +++ b/packages/services/service-analytics/src/__tests__/objectql-echo-non-date-bucket.test.ts @@ -27,6 +27,13 @@ * from the SQL driver's missing-column backstop: both now answer the engine's * virtual-field refusal, field for field. * + * [#29304] …and a PLAIN group over a `formula`, which the native strategy + * compiled into `GROUP BY calc` and ran (`DATABASE_ERROR` / 500), the ObjectQL + * strategy handed the SQL driver's missing-column backstop, and the dry run + * rendered on both compositions (measured on `origin/main` `447226b2`). The + * native strategy now declines a grouped dimension over a virtual field, so + * the query and the dry run both answer the engine's plain-group refusal. + * * It now judges the keys `execute()` hands the engine with the service's one * reading of it (`dataset-date-bucket-door.ts`, #28852), after the `dateRange` * read and the column-asked-twice door as `query()` refuses, so it answers the @@ -34,7 +41,7 @@ * byte for byte (pinned below). */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; import { ObjectQL } from '@objectstack/objectql'; import { SqlDriver } from '@objectstack/driver-sql'; import type { Cube } from '@objectstack/spec/data'; @@ -58,6 +65,8 @@ const CUBES = [ // A time dimension over a text column, declaring one granularity: asked // raw, it is bucketed by that granularity (`withDeclaredGranularityDefaults`). region_month: { type: 'time', sql: 'region', label: 'Region month', granularities: ['month'] }, + // [#29304] A string dimension over the formula, asked raw: a plain group. + calc_label: { type: 'string', sql: 'calc', label: 'Calc label' }, }, }, ] as unknown as Cube[]; @@ -91,7 +100,15 @@ describe('[#28977] the /analytics/sql dry run refuses a date bucket over a colum engine = new ObjectQL({ logger: quiet } as never); engine.registerDriver(driver as never, true); await engine.init(); - engine.registry.registerObject({ name: ACCOUNT, label: 'Account', fields: { name: { name: 'name', type: 'text' } } } as never); + engine.registry.registerObject({ + name: ACCOUNT, + label: 'Account', + fields: { + name: { name: 'name', type: 'text' }, + // [#29304] A formula on the related object, read through `account.`. + name_calc: { name: 'name_calc', type: 'formula', expression: 'name', returnType: 'text' }, + }, + } as never); engine.registry.registerObject({ name: DEAL, label: 'Deal', @@ -181,6 +198,52 @@ describe('[#28977] the /analytics/sql dry run refuses a date bucket over a colum expect(asSql, 'the dry run vs the query').toEqual(asQuery); }); + /** The engine's plain-group refusal of the formula `field` on `object`, at `path`. */ + const plainGroupRefusal = (object: string, field: string, path: string) => ({ + message: + `aggregate('${object}'): ${path} groups by '${field}', but '${field}' is a formula field — its value is computed ` + + 'on read and never stored, so no driver materialises a column to group by. The query was NOT run. Denormalise ' + + `the value onto '${object}' (a stored field, written when the source changes) and group by that. A formula ` + + 'field is virtual: with no column behind it the group key reached the driver as a column that does not exist, ' + + 'and an engine that aggregates in memory put every row into one null group while the answer looked served.', + code: 'INVALID_FIELD', status: 400, httpStatus: 400, + object, param: 'groupBy', path, field, fields: [field], + }); + + it.each<[string, Query, [object: string, field: string, path: string]]>([ + ['an inferred dimension', onObject({ dimensions: ['calc'] }), [DEAL, 'calc', 'groupBy.0']], + ['after a stored dimension', onObject({ dimensions: ['region', 'calc'] }), [DEAL, 'calc', 'groupBy.1']], + ["a cube dimension's sql", onCube({ dimensions: ['calc_label'] }), [DEAL, 'calc', 'groupBy.0']], + // The cross-object path's FK-resolve aggregate on the related object + // (`groupBy: ['id', attr]`): the engine's position there, judged by the + // service before the related object is read. + ["a related object's formula through 'account.'", onObject({ dimensions: ['account.name_calc'] }), [ACCOUNT, 'name_calc', 'groupBy.1']], + ])('[#29304] refuses a plain group over a formula (%s) on the query and the dry run alike, where native answered 500 and the dry run rendered GROUP BY calc', async (_label, query, expected) => { + const { asQuery, asSql } = await bothRefusals(query); + expect(asQuery).toEqual(plainGroupRefusal(...expected)); + expect(asSql, 'the dry run vs the query').toEqual(asQuery); + }); + + it('[#29304] routes only a grouped formula off the native strategy: a stored dimension and a measure-only query stay native', async () => { + const aggregate = vi.spyOn(engine, 'aggregate'); + try { + // The default composition answers both on the native strategy, which runs + // raw SQL and never calls `engine.aggregate`. + for (const query of [onObject({ dimensions: ['region'] }), onObject({}), onCube({ dimensions: ['region'] })]) { + const label = JSON.stringify(query); + const result = await analytics.query({ ...query } as never); + expect(result.rows, label).toHaveLength(1); + expect(result.sql, label).toContain('COALESCE(SUM(amount), 0)'); + } + expect(aggregate, 'native: no engine aggregate').not.toHaveBeenCalled(); + // A grouped formula declines to the ObjectQL strategy, whose engine refuses it. + await expect(analytics.query({ ...onObject({ dimensions: ['calc'] }) } as never)).rejects.toMatchObject({ code: 'INVALID_FIELD', path: 'groupBy.0' }); + expect(aggregate, 'declined: the engine judged it').toHaveBeenCalledTimes(1); + } finally { + aggregate.mockRestore(); + } + }); + it.each<[string, Query, string]>([ ["beside a cross-object 'account.name' (its FK column first)", onObject({ dimensions: ['account.name'], timeDimensions: [td('region', 'month')] }), 'groupBy.1.dateGranularity'], ['after a raw date column and a datetime bucket', onObject({ dimensions: ['closed_on'], timeDimensions: [td('closed_at', 'month'), td('region', 'month')] }), 'groupBy.2.dateGranularity'], diff --git a/packages/services/service-analytics/src/__tests__/preview-date-bucket-type.test.ts b/packages/services/service-analytics/src/__tests__/preview-date-bucket-type.test.ts index 2c85fd83465..08a1af25355 100644 --- a/packages/services/service-analytics/src/__tests__/preview-date-bucket-type.test.ts +++ b/packages/services/service-analytics/src/__tests__/preview-date-bucket-type.test.ts @@ -32,6 +32,12 @@ * where the live strategy lowers them, so an unrecognised `dateRange` beside * either fault is refused about the `dateRange` on every face; the preview * answered the door's refusal. + * + * [#29304] …and a PLAIN dimension over a `formula` (`returnType` `date` or + * `text`), measured on `origin/main` `447226b2`: native `DATABASE_ERROR` / + * 500, ObjectQL the SQL driver's missing-column backstop (400, no `path`), the + * preview one `null` group holding the total. Every face now answers the + * engine's plain-group refusal at `groupBy.N`, field for field. */ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; @@ -64,6 +70,7 @@ const DATASET = DatasetSchema.parse({ { name: 'calc', field: 'calc', type: 'date' }, { name: 'region_month', field: 'region', type: 'date', dateGranularity: 'month' }, { name: 'calc_month', field: 'calc', type: 'date', dateGranularity: 'month' }, + { name: 'calc_s', field: 'calc_s', type: 'string' }, ], measures: [{ name: 'total', aggregate: 'sum', field: 'amount' }], }); @@ -111,6 +118,8 @@ describe('[#28852] a date bucket over a column that is not a date, on every face opened_time: { name: 'opened_time', type: 'time' }, // A formula the table stores no column for. calc: { name: 'calc', type: 'formula', expression: 'closed_on', returnType: 'date' }, + // [#29304] …and one whose result is text. + calc_s: { name: 'calc_s', type: 'formula', expression: 'region', returnType: 'text' }, }, } as never); await engine.syncSchemas(); @@ -216,6 +225,38 @@ describe('[#28852] a date bucket over a column that is not a date, on every face expect(refusal).toMatchObject({ path: 'groupBy.0.dateGranularity', field: 'calc', fields: ['calc'] }); }); + it.each([ + ['date', 'calc'], + ['text', 'calc_s'], + ])('[#29304] refuses a plain dimension over a formula (returnType %s) with the engine\'s refusal on every face, where native answered 500, ObjectQL the driver backstop, and the preview one null group', async (_label, field) => { + const refusal = await refusedAlike({ dimensions: [field], measures: ['total'] }); + expect(refusal).toEqual({ + message: + `aggregate('${OBJECT}'): groupBy.0 groups by '${field}', but '${field}' is a formula field — its value is ` + + 'computed on read and never stored, so no driver materialises a column to group by. The query was NOT run. ' + + `Denormalise the value onto '${OBJECT}' (a stored field, written when the source changes) and group by that. ` + + 'A formula field is virtual: with no column behind it the group key reached the driver as a column that does ' + + 'not exist, and an engine that aggregates in memory put every row into one null group while the answer looked ' + + 'served.', + code: 'INVALID_FIELD', status: 400, httpStatus: 400, + object: OBJECT, param: 'groupBy', path: 'groupBy.0', field, fields: [field], + }); + }); + + it('[#29304] names the position of a plain formula dimension after a stored one', async () => { + const refusal = await refusedAlike({ dimensions: ['region', 'calc_s'], measures: ['total'] }); + expect(refusal).toMatchObject({ path: 'groupBy.1', field: 'calc_s', fields: ['calc_s'] }); + }); + + it('[#29304] still serves a plain dimension over a stored column alike on every face', async () => { + for (const face of FACES) { + const result = await ask(face, { dimensions: ['region'], measures: ['total'] }); + const rows = (result.rows as Array>).map((r) => ({ ...r, total: Number(r.total) })); + expect(rows.sort((a, b) => String(a.region).localeCompare(String(b.region))), face) + .toEqual([{ region: 'amer', total: 100 }, { region: 'emea', total: 50 }]); + } + }); + /** Every face refuses `selection` with one refusal of `code`, message included. */ const refusedAlikeWith = async (selection: Selection, code: string) => { const refusals: Array> = []; From 98b05f5ae9ad2fdf579523e050c0ca975cfc8dd9 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 00:40:32 +0000 Subject: [PATCH 5/8] test: fold the plain-group refusal into the having and guard fixtures; changeset and dogfood pin Claude-Session: https://claude.ai/code/session_016WV4cyTehJsfYGVMzcXLJD Co-authored-by: Claude --- ...29304-formula-plain-group-refused-alike.md | 20 ++++++ ...regate-reference-verdict-positions.test.ts | 15 ++++- ...tics-formula-group-refusal.dogfood.test.ts | 64 +++++++++++++++++++ .../dimension-structured-json-door.test.ts | 7 ++ .../multi-value-json-stored-door.test.ts | 7 ++ .../preview-date-bucket-type.test.ts | 2 +- 6 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 .changeset/29304-formula-plain-group-refused-alike.md create mode 100644 packages/qa/dogfood/test/analytics-formula-group-refusal.dogfood.test.ts diff --git a/.changeset/29304-formula-plain-group-refused-alike.md b/.changeset/29304-formula-plain-group-refused-alike.md new file mode 100644 index 00000000000..e9b19effda1 --- /dev/null +++ b/.changeset/29304-formula-plain-group-refused-alike.md @@ -0,0 +1,20 @@ +--- +'@objectstack/objectql': patch +'@objectstack/service-analytics': patch +--- + +fix(objectql, service-analytics)!: a plain group over a formula field is refused alike on every analytics face, instead of a 500, a driver backstop, one null group or a rendered statement + +Clause-②: yes (narrowing) — `engine.aggregate` now refuses `INVALID_FIELD` / 400 a `groupBy` entry of any form (a name, `{ field }`, or a bucket) naming a virtual (`formula`) field, whatever its `returnType`, before any driver is asked: at `groupBy.N` for a name, `groupBy.N.field` for the object form, and `groupBy.N.dateGranularity` for a bucket, as before. `/analytics/query` on the native composition, `/analytics/sql` (`AnalyticsService.generateSql`), the draft preview of `queryDataset` and the delegated fallback face give the same refusal, field for field. Nothing is widened: no export of either package root, public member, answer member, error code or status is added. +Metadata-protocol: none + + + +**BREAKING** accept-set narrowing on `engine.aggregate` (`@objectstack/objectql`) and on every analytics face (`@objectstack/service-analytics`), for a plain group over a `formula` field only. + +- **Before.** Measured on `main` `447226b2` with a real `SqlDriver`, a dataset or cube dimension over a `formula` (`returnType` `date` or `text`). `/analytics/query` on the native composition answered `DATABASE_ERROR` / 500 (the native strategy ran `GROUP BY calc` itself); on the ObjectQL composition it answered the SQL driver's missing-column backstop, `INVALID_FIELD` / 400 with no `path` and a "run schema sync" prescription that cannot help. The `/analytics/sql` dry run rendered `SELECT calc AS "calc" … GROUP BY calc` on both compositions, and the draft preview served one `null` group holding the total. `engine.aggregate` on a driver with no native `aggregate` (the in-memory aggregation) served one `null` group holding every row. +- **Now.** The engine's aggregate door gains a storage axis (`assertGroupByTargetsAreStored`, ahead of the date-bucket door), judged by the spec's `isVirtualSearchField` (`SEARCH_VIRTUAL_TYPES`), the predicate the sort, filter, search and bucket axes already refuse a formula by. The service's copy (`dataset-date-bucket-door.ts`, `assertDatasetGroupTargetsStored`) gives the same refusal on the faces that hand the engine no keys: the dry run, the preview and the delegated fallback service. The native-SQL strategy declines a query whose `dimensions` entry resolves to a virtual field, so that query reaches the engine. A dimension through a relationship (`account.calc`) over a related object's formula is judged on the keys the cross-object path hands the related object, on the query and the dry run alike, whether or not a base row exists. +- **Unchanged.** A plain group over a stored field, and a date or datetime bucket, are served on every face, byte for byte, and stay on the native strategy where they were. A measure-only query stays native. The #29167 refusal of a bucket over a formula keeps its words and position. +- **Who sees it.** A caller grouping by a formula field: a located refusal with a remedy, instead of a 500, a backstop with the wrong advice, an empty-looking group or SQL for a statement that never runs. + +**The remedy.** Denormalise the value onto the object (a stored field, written when the source changes) and group by that. diff --git a/packages/objectql/src/engine-aggregate-reference-verdict-positions.test.ts b/packages/objectql/src/engine-aggregate-reference-verdict-positions.test.ts index 72a8f33cf96..584e85ca334 100644 --- a/packages/objectql/src/engine-aggregate-reference-verdict-positions.test.ts +++ b/packages/objectql/src/engine-aggregate-reference-verdict-positions.test.ts @@ -32,6 +32,10 @@ * the engine's groupBy door and its `min` / `max` door refuse it first. The * aggregated column's type carries no `multiple` flag, and the rows below * hold that it needs none. + * - [#29304] Nor does a `formula` column: it is virtual (no stored value to + * group), and the engine's groupBy storage door + * (`assertGroupByTargetsAreStored`) refuses a group over it first. A + * formula pair at `having` is refused there, before the verdict is asked. * * The `where` column is measured in `packages/rest` * (`aggregation-filter-where-doors.test.ts`), where a real `SqlDriver` answers @@ -228,6 +232,10 @@ function expectedAt(position: Position, target: string, referent: string): Expec // The `multiple` seam (pointer 5944816815): no aggregated column holds a list. return { kind: 'refused first by', door: 'the groupBy door (a multi-value or JSON field is not grouped by)', code: 'INVALID_FIELD' }; } + if (position === 'having' && (FIELDS[target].type === 'formula' || FIELDS[referent].type === 'formula')) { + // [#29304] No aggregated column holds a formula: a group over one is refused. + return { kind: 'refused first by', door: 'the groupBy storage door (a formula is not grouped by)', code: 'INVALID_FIELD' }; + } if (position === 'aggregations[i].filter' && FIELDS[target].type === 'formula') { // `where` meets the same door first: a formula is computed on read, never filtered on. return { kind: 'refused first by', door: 'the materializable-filter door (a formula key)', code: 'INVALID_FIELD' }; @@ -420,16 +428,17 @@ describe('[#21299] the card\'s four measured shapes are refused before any read, }); it('…and at having, an addDays pair from a formula projection, which was answered: refused, naming the formula', async () => { + // [#29304] The formula projection itself is refused now, at its groupBy + // position, before `having` is read: no aggregated column holds a formula. const { engine, calls } = await makeEngine(OBJ); const out = await outcomeOf(() => engine.aggregate(OBJECT, { groupBy: ['due_f', { field: 'due_on', dateGranularity: 'day', alias: 'due_day' }], aggregations: [{ function: 'count', alias: 'n' }], having: { due_f: { $lte: { $field: 'due_day', addDays: 1 } } }, } as unknown as EngineAggregateOptions), () => []); - expect(out).toMatchObject({ refused: true, code: 'INVALID_FILTER', status: 400 }); + expect(out).toMatchObject({ refused: true, code: 'INVALID_FIELD', status: 400 }); expect(calls).toEqual({ aggregate: 0, find: 0 }); - expect(out.message).toContain('the target field "due_f" (type "formula")'); - expect(out.message).toContain('{ "$field": "due_day" } with addDays'); + expect(out.message).toContain("groupBy.0 groups by 'due_f', but 'due_f' is a formula field"); }); it('the same-class control on the same object answers as before', async () => { diff --git a/packages/qa/dogfood/test/analytics-formula-group-refusal.dogfood.test.ts b/packages/qa/dogfood/test/analytics-formula-group-refusal.dogfood.test.ts new file mode 100644 index 00000000000..6448a4b799a --- /dev/null +++ b/packages/qa/dogfood/test/analytics-formula-group-refusal.dogfood.test.ts @@ -0,0 +1,64 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// END-TO-END gate for #29304: a plain group over a formula field is refused +// `INVALID_FIELD` / 400 at its position on `POST /analytics/query` and on the +// `POST /analytics/sql` dry run alike — proven on the stock showcase, through +// the real stack (HTTP → REST → analytics plugin → the native strategy's +// decline → the engine's aggregate door). +// +// `showcase_project.budget_remaining` is a `formula` (`returnType: 'currency'`): +// computed on read, no column. Measured on `main` `447226b2` with a real +// `SqlDriver`, a dimension over a formula answered `DATABASE_ERROR` / 500 on +// the native composition (the strategy ran `GROUP BY` over a column no table +// stores), and the dry run rendered that statement. +// +// Controls: a stored dimension (`status`) is served on both doors. + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { type VerifyStack } from '@objectstack/verify'; +import { bootShowcase } from './showcase-boot.js'; + +const CUBE = 'showcase_project'; + +describe('dogfood: a plain group over a formula field is refused alike on the query and the dry run (#29304)', () => { + let stack: VerifyStack; + let token: string; + + beforeAll(async () => { + stack = await bootShowcase(); + token = await stack.signIn(); + }, 180_000); + + afterAll(async () => { + await stack?.stop?.(); + }); + + async function ask(door: '/analytics/query' | '/analytics/sql', dimension: string) { + const res = await stack.apiAs(token, 'POST', door, { cube: CUBE, measures: ['count'], dimensions: [dimension] }); + return { status: res.status, body: (await res.json()) as Record }; + } + + for (const door of ['/analytics/query', '/analytics/sql'] as const) { + it(`${door}: a dimension over the formula → 400 INVALID_FIELD at groupBy.0, with the store-the-value remedy`, async () => { + const res = await ask(door, 'budget_remaining'); + expect(res.status, JSON.stringify(res.body)).toBe(400); + expect(res.body.code).toBe('INVALID_FIELD'); + const message = String(res.body.message ?? res.body.error); + expect(message).toContain( + `aggregate('${CUBE}'): groupBy.0 groups by 'budget_remaining', but 'budget_remaining' is a formula field`, + ); + expect(message).toContain(`Denormalise the value onto '${CUBE}' (a stored field, written when the source changes) and group by that.`); + }); + + it(`CONTROL ${door}: a stored dimension is served`, async () => { + const res = await ask(door, 'status'); + expect(res.status, JSON.stringify(res.body)).toBe(200); + }); + } + + it('the query and the dry run give one refusal, word for word', async () => { + const query = await ask('/analytics/query', 'budget_remaining'); + const sql = await ask('/analytics/sql', 'budget_remaining'); + expect(sql.body).toEqual(query.body); + }); +}); diff --git a/packages/services/service-analytics/src/__tests__/dimension-structured-json-door.test.ts b/packages/services/service-analytics/src/__tests__/dimension-structured-json-door.test.ts index 47aa3caeb65..73690dc125b 100644 --- a/packages/services/service-analytics/src/__tests__/dimension-structured-json-door.test.ts +++ b/packages/services/service-analytics/src/__tests__/dimension-structured-json-door.test.ts @@ -320,6 +320,13 @@ describe('what the door does not judge', () => { () => null, (e: Refusal) => ({ code: e.code, status: e.status, member: e.member }), ); + if (type === 'formula') { + // [#29304] Not this door's: the native strategy declines a grouped + // virtual field to the engine path, which this native-only harness + // does not configure — no refusal of this door's (no `member`). + expect(verdict, type).toEqual({ code: undefined, status: undefined, member: undefined }); + continue; + } expect(verdict, type).toEqual( STRUCTURED_JSON_TYPES.has(type) || isMultiValueField({ type }) ? { code: 'INVALID_FIELD', status: 400, member: `f_${type}` } diff --git a/packages/services/service-analytics/src/__tests__/multi-value-json-stored-door.test.ts b/packages/services/service-analytics/src/__tests__/multi-value-json-stored-door.test.ts index 9edaa654fee..053ec943cce 100644 --- a/packages/services/service-analytics/src/__tests__/multi-value-json-stored-door.test.ts +++ b/packages/services/service-analytics/src/__tests__/multi-value-json-stored-door.test.ts @@ -371,6 +371,13 @@ describe('GUARD the judged population is the spec predicates\', over every Field () => null, (e: Refusal) => ({ code: e.code, status: e.status, member: e.member }), ); + if (type === 'formula') { + // [#29304] Not this door's: the native strategy declines a grouped + // virtual field to the engine path, which this native-only harness + // does not configure — no refusal of this door's (no `member`). + expect(verdict, field).toEqual({ code: undefined, status: undefined, member: undefined }); + continue; + } const judged = STRUCTURED_JSON_TYPES.has(type) || isMultiValueField({ type, multiple }); expect(verdict, field).toEqual(judged ? { code: 'INVALID_FIELD', status: 400, member: field } : null); } diff --git a/packages/services/service-analytics/src/__tests__/preview-date-bucket-type.test.ts b/packages/services/service-analytics/src/__tests__/preview-date-bucket-type.test.ts index 08a1af25355..5a50582fd72 100644 --- a/packages/services/service-analytics/src/__tests__/preview-date-bucket-type.test.ts +++ b/packages/services/service-analytics/src/__tests__/preview-date-bucket-type.test.ts @@ -251,7 +251,7 @@ describe('[#28852] a date bucket over a column that is not a date, on every face it('[#29304] still serves a plain dimension over a stored column alike on every face', async () => { for (const face of FACES) { const result = await ask(face, { dimensions: ['region'], measures: ['total'] }); - const rows = (result.rows as Array>).map((r) => ({ ...r, total: Number(r.total) })); + const rows = (result.rows as Array>).map((r) => ({ region: r.region, total: Number(r.total) })); expect(rows.sort((a, b) => String(a.region).localeCompare(String(b.region))), face) .toEqual([{ region: 'amer', total: 100 }, { region: 'emea', total: 50 }]); } From 42c89af2530ede61e73c9e218918db7a6d1e0474 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 00:50:25 +0000 Subject: [PATCH 6/8] test(dogfood): read the refusal from the ADR-0112 error envelope Claude-Session: https://claude.ai/code/session_016WV4cyTehJsfYGVMzcXLJD Co-authored-by: Claude --- .../test/analytics-formula-group-refusal.dogfood.test.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/qa/dogfood/test/analytics-formula-group-refusal.dogfood.test.ts b/packages/qa/dogfood/test/analytics-formula-group-refusal.dogfood.test.ts index 6448a4b799a..bd898260e81 100644 --- a/packages/qa/dogfood/test/analytics-formula-group-refusal.dogfood.test.ts +++ b/packages/qa/dogfood/test/analytics-formula-group-refusal.dogfood.test.ts @@ -42,8 +42,9 @@ describe('dogfood: a plain group over a formula field is refused alike on the qu it(`${door}: a dimension over the formula → 400 INVALID_FIELD at groupBy.0, with the store-the-value remedy`, async () => { const res = await ask(door, 'budget_remaining'); expect(res.status, JSON.stringify(res.body)).toBe(400); - expect(res.body.code).toBe('INVALID_FIELD'); - const message = String(res.body.message ?? res.body.error); + // The ADR-0112 envelope: `{ success: false, error: { code, message, … } }`. + expect(res.body.error?.code, JSON.stringify(res.body)).toBe('INVALID_FIELD'); + const message = String(res.body.error?.message); expect(message).toContain( `aggregate('${CUBE}'): groupBy.0 groups by 'budget_remaining', but 'budget_remaining' is a formula field`, ); @@ -59,6 +60,7 @@ describe('dogfood: a plain group over a formula field is refused alike on the qu it('the query and the dry run give one refusal, word for word', async () => { const query = await ask('/analytics/query', 'budget_remaining'); const sql = await ask('/analytics/sql', 'budget_remaining'); - expect(sql.body).toEqual(query.body); + expect(sql.status).toBe(400); + expect(sql.body.error).toEqual(query.body.error); }); }); From 6bcd5781926296cd89927c3a276491c123bd2b8d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 01:58:24 +0000 Subject: [PATCH 7/8] =?UTF-8?q?chore(changeset):=20declare=20the=20formula?= =?UTF-8?q?-group=20refusal=20as=20Clause-=E2=91=A1=20no=20(narrowing)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PR widens nothing, so the declaration is `no (narrowing)`, as #29167 landed with; `yes (narrowing)` reads as widening one surface and narrowing another. The BREAKING heading "Who sees it" becomes "Who moves". Claude-Session: https://claude.ai/code/session_016WV4cyTehJsfYGVMzcXLJD Co-authored-by: Claude --- .changeset/29304-formula-plain-group-refused-alike.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/29304-formula-plain-group-refused-alike.md b/.changeset/29304-formula-plain-group-refused-alike.md index e9b19effda1..b93f45a93ba 100644 --- a/.changeset/29304-formula-plain-group-refused-alike.md +++ b/.changeset/29304-formula-plain-group-refused-alike.md @@ -5,7 +5,7 @@ fix(objectql, service-analytics)!: a plain group over a formula field is refused alike on every analytics face, instead of a 500, a driver backstop, one null group or a rendered statement -Clause-②: yes (narrowing) — `engine.aggregate` now refuses `INVALID_FIELD` / 400 a `groupBy` entry of any form (a name, `{ field }`, or a bucket) naming a virtual (`formula`) field, whatever its `returnType`, before any driver is asked: at `groupBy.N` for a name, `groupBy.N.field` for the object form, and `groupBy.N.dateGranularity` for a bucket, as before. `/analytics/query` on the native composition, `/analytics/sql` (`AnalyticsService.generateSql`), the draft preview of `queryDataset` and the delegated fallback face give the same refusal, field for field. Nothing is widened: no export of either package root, public member, answer member, error code or status is added. +Clause-②: no (narrowing) — `engine.aggregate` now refuses `INVALID_FIELD` / 400 a `groupBy` entry of any form (a name, `{ field }`, or a bucket) naming a virtual (`formula`) field, whatever its `returnType`, before any driver is asked: at `groupBy.N` for a name, `groupBy.N.field` for the object form, and `groupBy.N.dateGranularity` for a bucket, as before. `/analytics/query` on the native composition, `/analytics/sql` (`AnalyticsService.generateSql`), the draft preview of `queryDataset` and the delegated fallback face give the same refusal, field for field. Nothing is widened: no export of either package root, public member, answer member, error code or status is added. Metadata-protocol: none @@ -15,6 +15,6 @@ Metadata-protocol: none - **Before.** Measured on `main` `447226b2` with a real `SqlDriver`, a dataset or cube dimension over a `formula` (`returnType` `date` or `text`). `/analytics/query` on the native composition answered `DATABASE_ERROR` / 500 (the native strategy ran `GROUP BY calc` itself); on the ObjectQL composition it answered the SQL driver's missing-column backstop, `INVALID_FIELD` / 400 with no `path` and a "run schema sync" prescription that cannot help. The `/analytics/sql` dry run rendered `SELECT calc AS "calc" … GROUP BY calc` on both compositions, and the draft preview served one `null` group holding the total. `engine.aggregate` on a driver with no native `aggregate` (the in-memory aggregation) served one `null` group holding every row. - **Now.** The engine's aggregate door gains a storage axis (`assertGroupByTargetsAreStored`, ahead of the date-bucket door), judged by the spec's `isVirtualSearchField` (`SEARCH_VIRTUAL_TYPES`), the predicate the sort, filter, search and bucket axes already refuse a formula by. The service's copy (`dataset-date-bucket-door.ts`, `assertDatasetGroupTargetsStored`) gives the same refusal on the faces that hand the engine no keys: the dry run, the preview and the delegated fallback service. The native-SQL strategy declines a query whose `dimensions` entry resolves to a virtual field, so that query reaches the engine. A dimension through a relationship (`account.calc`) over a related object's formula is judged on the keys the cross-object path hands the related object, on the query and the dry run alike, whether or not a base row exists. - **Unchanged.** A plain group over a stored field, and a date or datetime bucket, are served on every face, byte for byte, and stay on the native strategy where they were. A measure-only query stays native. The #29167 refusal of a bucket over a formula keeps its words and position. -- **Who sees it.** A caller grouping by a formula field: a located refusal with a remedy, instead of a 500, a backstop with the wrong advice, an empty-looking group or SQL for a statement that never runs. +- **Who moves.** A caller grouping by a formula field: a located refusal with a remedy, instead of a 500, a backstop with the wrong advice, an empty-looking group or SQL for a statement that never runs. **The remedy.** Denormalise the value onto the object (a stored field, written when the source changes) and group by that. From 8ca24327db0b7fc317c901e8cc60653a004e5643 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 02:27:25 +0000 Subject: [PATCH 8/8] test(client): ledger the #29304 route pin's two producer analytics.query sites The envelope-caller census (#13079) counts producer-service analytics.query call sites. The #29304 route pin in objectql-echo-non-date-bucket.test.ts adds two, calling the plugin-registered AnalyticsService directly: NOT_SDK, as the file's existing sites. Ledger row 2 -> 4; service sites and NOT_SDK total 17 -> 19. No logic change. Claude-Session: https://claude.ai/code/session_016WV4cyTehJsfYGVMzcXLJD Co-authored-by: Claude --- packages/client/src/envelope-caller-census.test.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/client/src/envelope-caller-census.test.ts b/packages/client/src/envelope-caller-census.test.ts index bc2c4115be5..d99f910194f 100644 --- a/packages/client/src/envelope-caller-census.test.ts +++ b/packages/client/src/envelope-caller-census.test.ts @@ -534,8 +534,8 @@ const LEDGER: readonly LedgerRow[] = [ // ── [#28977] the /analytics/sql dry run vs the query for a date bucket over a column that is not a date: producer reads only ── { file: 'packages/services/service-analytics/src/__tests__/objectql-echo-non-date-bucket.test.ts', - method: 'analytics.query', receiver: 'service', count: 2, verdict: 'NOT_SDK', - why: 'the real AnalyticsService that AnalyticsServicePlugin registers over a live engine, called directly (no HTTP, no dispatcher envelope) to compare the query\'s refusal with the dry run\'s (a date bucket over a text, select, number, boolean or time column), and to read the `sql` the query serves for a date and a datetime bucket', + method: 'analytics.query', receiver: 'service', count: 4, verdict: 'NOT_SDK', + why: 'the real AnalyticsService that AnalyticsServicePlugin registers over a live engine, called directly (no HTTP, no dispatcher envelope) to compare the query\'s refusal with the dry run\'s (a date bucket over a text, select, number, boolean or time column), and to read the `sql` the query serves for a date and a datetime bucket; [#29304] and to read which strategy serves a stored, measure-only or cube dimension (native) and which refuses a formula dimension (the engine)', }, { file: 'packages/client/src/analytics-automation-json-erasure.test.ts', @@ -781,7 +781,10 @@ describe('#13079 §2 — positive controls on the matcher itself', () => { // bucket over a column that is not a date calls the same // plugin-registered AnalyticsService to compare the query's refusal // and served `sql` with `generateSql`'s. - expect(service.length, literalNote()).toBe(17); + // [#29304] Two more sites in that file: the route pin calls the same + // plugin-registered AnalyticsService to read which strategy serves a + // stored dimension and which refuses a formula one. + expect(service.length, literalNote()).toBe(19); expect([...new Set(service.map((s) => s.file))].sort()).toEqual([ 'packages/client/src/analytics-automation-json-erasure.test.ts', 'packages/drivers/driver-memory/src/memory-exists-non-boolean-refusal.test.ts', @@ -836,7 +839,7 @@ describe('#13079 §3 — every call site is classified', () => { expect(production, literalNote()).toEqual([]); }); - it('records the split: 18 payload pins, 10 result-insensitive, 17 not-SDK', () => { + it('records the split: 18 payload pins, 10 result-insensitive, 19 not-SDK', () => { expect(verdictTotal('PAYLOAD_DEPENDENT')).toBe(18); expect(verdictTotal('RESULT_INSENSITIVE')).toBe(10); // [#21441] 8 -> 9: the date-bucket echo pin's producer call (§2). @@ -844,7 +847,8 @@ describe('#13079 §3 — every call site is classified', () => { // [#28841] 11 -> 14: the dry-run parity pin's three producer calls (§2). // [#28852] 14 -> 15: its dateRange-precedence call. // [#28977] 15 -> 17: the non-date bucket dry-run parity pin's two producer calls (§2). - expect(verdictTotal('NOT_SDK')).toBe(17); + // [#29304] 17 -> 19: the same file's route pin, two producer calls (§2). + expect(verdictTotal('NOT_SDK')).toBe(19); // The three above are LEDGER sums and cannot move on a census reading; // this one is census-derived, so it carries the note. [#13874] expect(sdkSites.length, literalNote()).toBe(28);