diff --git a/.changeset/28435-data-eval-case3-env-credential-route.md b/.changeset/28435-data-eval-case3-env-credential-route.md new file mode 100644 index 00000000000..07297592ef8 --- /dev/null +++ b/.changeset/28435-data-eval-case3-env-credential-route.md @@ -0,0 +1,12 @@ +--- +'@objectstack/skills': patch +--- + +docs(skills): objectstack-data eval case 3 answers the code-defined datasource credential route and pins it + +Clause-②: no +Metadata-protocol: none + +- **Before.** `evals/hooks-security-seeds-datasources.json` case 3 (a read-only `legacy_customer` object over the `customers` table of a Postgres reporting database) answered "a `defineDatasource` entry (`credentialsRef`, never inline credentials)". `rules/datasources.md` now teaches that a datasource defined in code reads its password at connect from `OS_DATASOURCE__PASSWORD` behind the host switch `OS_DATASOURCE_ENV_CREDENTIALS`, and that nothing mints a `sys_secret:` handle for one; the eval's `must_contain` pinned neither route, so the eval stayed green while its model answer taught a handle the connect could never resolve. +- **Now.** The model answer names the environment route the rule teaches (`OS_DATASOURCE_REPORTING_PASSWORD` for a datasource named `reporting`, the switch and the hosts that default it on), `schemaMode: 'external'`, the object's `datasource` / `external.remoteName` / `columnMap` binding and the closed write gate (`allowWrites`, `writable`). `must_contain` keeps `defineDatasource`, `datasource`, `remoteName`, `columnMap` and `schemaMode` and adds `OS_DATASOURCE_ENV_CREDENTIALS` and `_PASSWORD`, so an answer that teaches only the old route goes red. +- Eval fixtures are measured but not ratcheted; no SKILL.md, rules, references, schema, key or behaviour changes. diff --git a/skills/objectstack-data/evals/hooks-security-seeds-datasources.json b/skills/objectstack-data/evals/hooks-security-seeds-datasources.json index 3bb4ee10e03..e1abb388de4 100644 --- a/skills/objectstack-data/evals/hooks-security-seeds-datasources.json +++ b/skills/objectstack-data/evals/hooks-security-seeds-datasources.json @@ -24,10 +24,10 @@ { "id": 3, "prompt": "Expose the `customers` table of our Postgres reporting database as a read-only `legacy_customer` object.", - "expected_output": "A `defineDatasource` entry (`credentialsRef`, never inline credentials); the object sets `datasource: ''`, `remoteName` for the table, `columnMap` for columns; `schemaMode` leaves the remote schema unaltered; the external write gate stays closed.", + "expected_output": "A `defineDatasource` entry (say `reporting`) with `schemaMode: 'external'` (the remote schema stays unaltered) and no credential in the declaration: the password is read at connect from `OS_DATASOURCE_REPORTING_PASSWORD` (the name upper-cased) while the host switch `OS_DATASOURCE_ENV_CREDENTIALS` is `on` (`os serve`/`start`/`dev` default it on); the object sets `datasource: 'reporting'` and `external: { remoteName: 'customers', columnMap }`; the write gate stays closed (no `allowWrites`, no `writable`).", "files": [], "assertions": { - "must_contain": ["defineDatasource", "datasource", "remoteName", "columnMap", "schemaMode"], + "must_contain": ["defineDatasource", "datasource", "remoteName", "columnMap", "schemaMode", "OS_DATASOURCE_ENV_CREDENTIALS", "_PASSWORD"], "must_not_contain": ["columnName:"] } },