Feat: Add configurable discovery

CHANGELOG.md

@@ -1,5 +1,41 @@
 # Changelog
 
+## v1.5.0
+
+### Added
+- Support for datamasque-python 1.2.2.
+  - `dm discover schema-results` handles matches with no label.
+  - `dm rulesets validate` and `dm libraries validate` print validation
+    errors for invalid YAML.
+- Support for Configurable Discovery:
+  - `dm discover configs` — list, get, defaults, create, delete, validate,
+    and status for discovery configs (`database` or `file`).
+  - `dm discover libraries` — list, get, create, delete, validate, and status
+    for discovery config libraries.
+  - `dm discover schema --config <name>` and `dm discover file
+    [--config <name>]` start discovery runs with or without a specific config.
+  - `dm discover config-snapshot <run-id>` downloads the discovery config a run
+    actually used.
+- `dm rulesets status` and `dm libraries status` — show a stored ruleset's or
+  library's validation state and errors.
+- `dm rulesets validate` and `dm discover configs validate` refuse YAML of
+  60 KiB or larger, which the server validates asynchronously; create it and
+  poll `status` instead.
+- Safe Data Preview: `dm discover schema-results` and `dm discover file-report`
+  include `safe_data_preview` in their `--json` output.
+
+### Changed
+- A declined confirmation prompt now exits 10 (`cancelled`) instead of 1,
+  so a decision is not reported as a failure. Ctrl-C still exits 1.
+- `dm run start --json` and `dm run retry --json` only emit the run id.
+  `"status": "queued"` is no longer returned.
+
+### Fixed
+- `dm rulesets generate`, `dm connections update --password`, and the
+  deprecated `dm system import` no longer fail.
+- File errors now name the file instead of printing a traceback.
+- Unhandled server and network errors now abort with a code, not a traceback.
+
 ## v1.4.0
 
 ### Added

README.md

@@ -138,17 +138,16 @@ dm connections delete <name>                    # Delete a connection
 ### Rulesets
 
 ```console
-dm rulesets list                                  # List all rulesets
-dm rulesets list --type file                      # Filter by type
-dm rulesets get <name>                            # Show ruleset details
+dm rulesets list [--type database|file]           # List all rulesets, or filter by type
+dm rulesets get <name> [--type database|file]     # Show details; --type disambiguates same-name rulesets
 dm rulesets get <name> --yaml                     # Print raw YAML only
-dm rulesets get <name> --type file                # Disambiguate same-name rulesets
-dm rulesets create --name <n> --file rules.yaml   # Create/update (type auto-detected from YAML)
-dm rulesets create --name <n> --file rules.yaml --type file  # Force a type
-dm rulesets delete <name> [--type file|database]  # Delete a ruleset
+dm rulesets create --name <n> --file rules.yaml   # Create/update (type read from the existing ruleset)
+dm rulesets create --name <n> --file rules.yaml [--type database|file]  # Force a type
+dm rulesets delete <name> [--type database|file]  # Delete a ruleset
 dm rulesets generate --file request.json          # Generate from schema
 dm rulesets generate --file req.json -o out.yaml  # Generate to file
-dm rulesets validate --file rules.yaml            # Validate against server
+dm rulesets validate --file rules.yaml --type database|file  # Validate against server (YAML under 60 KiB)
+dm rulesets status <name>                         # Validation status; poll after creating YAML of 60 KiB+
 dm rulesets export-bundle -o bundle.zip           # Export rulesets + libraries + seeds
 dm rulesets import-bundle --file bundle.zip       # Import a previously exported bundle
 dm rulesets import-bundle -f bundle.zip --overwrite-rulesets --overwrite-libraries  # Replace existing entries
@@ -164,6 +163,7 @@ dm libraries create --name <n> --file lib.yaml    # Create/update from file
 dm libraries create --name <n> --file lib.yaml --namespace pii  # With namespace
 dm libraries delete <name>                        # Delete a library
 dm libraries validate <name>                      # Re-validate against current server schema
+dm libraries status <name>                        # Validation status; poll after creating YAML of 60 KiB+
 dm libraries usage <name>                         # Show rulesets using it
 ```
 
@@ -216,11 +216,42 @@ dm users delete <username>                      # Delete a user
 ### Discovery
 
 ```console
-dm discover schema <connection>                 # Start a schema-discovery run
-dm discover schema-results <run-id>             # List schema-discovery results once the run finishes
-dm discover sdd-report <run-id>                 # Sensitive data discovery report
-dm discover db-report <run-id>                  # Database discovery CSV
-dm discover file-report <run-id>                # File discovery report
+dm discover schema <connection>                      # Schema discovery (built-in keyword-driven)
+dm discover schema <connection> --config <name>      # Schema discovery from a saved database config
+dm discover schema <connection> --json               # {"id": <run-id>}
+dm discover schema-results <run-id>                  # List schema-discovery results once the run finishes
+dm discover file <connection>                        # File data discovery (built-in keyword-driven)
+dm discover file <connection> --config <name>        # File data discovery from a saved file config
+dm discover file <connection> --json                 # {"id": <run-id>}
+dm discover sdd-report <run-id>                      # Sensitive data discovery report
+dm discover db-report <run-id>                       # Database discovery CSV
+dm discover file-report <run-id>                     # File discovery report
+dm discover config-snapshot <run-id> -o used.yaml    # Download the discovery config a run actually used
+```
+
+#### Discovery configs
+
+```console
+dm discover configs list [--type database|file]                       # List configs
+dm discover configs get <name> [--type database|file] [--yaml]        # Show details or raw YAML
+dm discover configs defaults [--type database|file] -o cfg.yaml       # Built-in default as a starting point
+dm discover configs create --name <n> -f cfg.yaml [--type database|file]  # Create/update from YAML
+dm discover configs delete <name> [--type database|file]              # Delete a config
+dm discover configs validate -f cfg.yaml --type database|file         # Validate against server (YAML under 60 KiB)
+dm discover configs status <name> [--type database|file]              # Validation status; poll after creating YAML of 60 KiB+
+```
+
+#### Discovery config libraries
+
+The same library can be imported by both database and file discovery configs.
+
+```console
+dm discover libraries list
+dm discover libraries get <name> [--namespace org] [--yaml]
+dm discover libraries create --name <n> --namespace org -f lib.yaml
+dm discover libraries delete <name> [--namespace org] [--force]    # --force if imported by configs
+dm discover libraries validate -f lib.yaml
+dm discover libraries status <name> [--namespace org]
 ```
 
 ### Seeds
@@ -308,6 +339,8 @@ empty on failure):
 |    7 | auth_failed       | credentials rejected by server                 |
 |    8 | conflict          | operation rejected by server state             |
 |    9 | transport_error   | network or TLS failure                         |
+|   10 | cancelled         | you answered no to a confirmation prompt       |
+|   11 | forbidden         | user lacks permission for the operation        |
 
 Exit codes are stable across minor versions. The `error.code` string in the
 JSON envelope mirrors these names.

claude-skills/datamasque-cli/skills/datamasque-cli/SKILL.md

@@ -23,9 +23,11 @@ In agent mode — auto-detected when stdout is not a TTY, `AI_AGENT` is set, or
 
 `error.code` is the stable identifier; branch on it rather than the message.
 The set is `not_found`, `invalid_input`, `ambiguous`, `auth_required`,
-`auth_failed`, `conflict`, `transport_error`, `error`. Exit code is non-zero
-on any error; exit 2 specifically means a CLI usage error (unknown flag,
-missing argument) from typer.
+`auth_failed`, `conflict`, `transport_error`, `cancelled`, `forbidden`,
+`error`. Exit code is non-zero on any error; exit 2 specifically means a CLI
+usage error (unknown flag, missing argument) from typer, exit 10 means the user
+declined a confirmation prompt, and exit 11 means the logged-in user lacks
+permission.
 
 `DM_OUTPUT=table` forces human-readable output.
 
@@ -55,7 +57,7 @@ Pass repeated `--options key=value` for server-side knobs
 - **Ruleset namespaces.** `database` and `file` rulesets share a name
   namespace, so `customers` can exist in both. `dm run start` reads the
   source connection's type and picks the matching ruleset automatically.
-  For `get` / `create` / `delete`, pass `--type file|database` only when
+  For `get` / `create` / `delete`, pass `--type database|file` only when
   two rows share the name and you need to disambiguate.
 
 - **File masking needs a destination.** Database masking is in-place;
@@ -89,6 +91,14 @@ Pass repeated `--options key=value` for server-side knobs
   then fetch results with `dm discover schema-results <id>` /
   `sdd-report` / `db-report` / `file-report`.
 
+- **Configurable discovery and Safe Data Preview.** Save a discovery config
+  with `dm discover configs create` (start from `dm discover configs defaults`),
+  then run `dm discover schema <connection> --config <name>`. When the config
+  enables in-data discovery with safe data preview, `dm discover schema-results
+  <id> --json` carries a `safe_data_preview` per column — value distributions,
+  patterns, and cardinality worth reading before choosing masks. It is JSON-only; 
+  `file-report --json` exposes the same per locator.
+
 - **`dm rulesets validate --file <file> --type <type>`** runs server-side
   validation without committing the ruleset. Use this before `create`
   when you want a clean failure mode for bad YAML.

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datamasque-cli"
-version = "1.4.0"
+version = "1.5.0"
 description = "Official command-line interface for the DataMasque data-masking platform."
 authors = [
     { name = "DataMasque Ltd" },
@@ -12,7 +12,8 @@ requires-python = ">=3.11"
 dependencies = [
     "typer>=0.15.0",
     "tomli-w>=1.0.0",
-    "datamasque-python>=1.0.0,<2",
+    "datamasque-python>=1.2.2,<2",
+    "pydantic>=2.5,<3",
 ]
 classifiers = [
     "Development Status :: 4 - Beta",
@@ -37,7 +38,7 @@ Repository = "https://github.com/datamasque/datamasque-cli"
 Issues = "https://github.com/datamasque/datamasque-cli/issues"
 
 [project.scripts]
-dm = "datamasque_cli.main:app"
+dm = "datamasque_cli.main:main"
 
 [dependency-groups]
 dev = [

src/datamasque_cli/client.py

@@ -14,7 +14,7 @@
 from datamasque.client.models.ifm import DataMasqueIfmInstanceConfig
 
 from datamasque_cli.config import Config, Profile, load_config
-from datamasque_cli.output import ErrorCode, abort
+from datamasque_cli.errors import ErrorCode, abort
 
 ENV_URL = "DATAMASQUE_URL"
 ENV_USERNAME = "DATAMASQUE_USERNAME"

src/datamasque_cli/commands/auth.py

@@ -6,7 +6,8 @@
 
 from datamasque_cli.client import get_client, profile_from_env
 from datamasque_cli.config import DEFAULT_PROFILE, Profile, load_config, save_config
-from datamasque_cli.output import ErrorCode, abort, print_info, print_success, print_table
+from datamasque_cli.errors import ErrorCode, abort
+from datamasque_cli.output import print_info, print_success, print_table
 
 # `login` and `status` handle connection errors locally
 # because they need softer behaviour than `get_client`'s hard abort:

src/datamasque_cli/commands/connections.py

@@ -2,12 +2,12 @@
 
 from __future__ import annotations
 
-import json
 from enum import StrEnum
 from pathlib import Path
 
 import typer
 from datamasque.client import DataMasqueClient
+from datamasque.client.exceptions import DataMasqueApiError
 from datamasque.client.models.connection import (
     AzureConnectionConfig,
     ConnectionConfig,
@@ -22,7 +22,9 @@
 )
 
 from datamasque_cli.client import get_client
-from datamasque_cli.output import ErrorCode, abort, print_success, redact_sensitive_fields, render_output
+from datamasque_cli.errors import ErrorCode, abort, abort_api_error, confirm_or_abort
+from datamasque_cli.fileio import read_json_object_or_abort
+from datamasque_cli.output import print_success, redact_sensitive_fields, render_output
 
 
 class ConnectionType(StrEnum):
@@ -205,8 +207,11 @@ def create_connection(
 
 def _create_from_file(client: DataMasqueClient, file: Path) -> None:
     """Create a connection from a JSON file."""
-    data = json.loads(file.read_text())
-    conn_type = _parse_connection_type(data.pop("type", "database"))
+    data = read_json_object_or_abort(file)
+    raw_type = data.pop("type", "database")
+    if not isinstance(raw_type, str):
+        abort(f'{file}: "type" must be a string.', code=ErrorCode.INVALID_INPUT)
+    conn_type = _parse_connection_type(raw_type)
 
     # Convert db_type string to enum for database connections.
     if conn_type is ConnectionType.DATABASE and "database_type" in data:
@@ -298,7 +303,10 @@ def test_connection(
     if match is None:
         abort(f"Connection '{name}' not found.", code=ErrorCode.NOT_FOUND)
 
-    response = client.make_request("POST", f"/api/connections/{match.id}/test/", data={})
+    try:
+        response = client.make_request("POST", f"/api/connections/{match.id}/test/", data={})
+    except DataMasqueApiError as exc:
+        abort_api_error(f"Connection '{match.name}' is not reachable", exc)
     body = response.json() if response.content else {}
     warning = body.get("message") if isinstance(body, dict) else None
 
@@ -347,7 +355,12 @@ def update_connection(
     if not updates:
         abort("Pass at least one field to update (e.g. --password, --host).", code=ErrorCode.INVALID_INPUT)
 
-    client.make_request("PATCH", f"/api/connections/{match.id}/", data=updates)
+    payload = dict(updates)
+    # `dbpassword` is the server's field name for the database password on connection PATCH.
+    if "password" in payload:
+        payload["dbpassword"] = payload.pop("password")
+
+    client.make_request("PATCH", f"/api/connections/{match.id}/", data=payload)
     print_success(f"Connection '{match.name}' updated: {', '.join(updates)}.")
 
 
@@ -363,7 +376,7 @@ def delete_connection(
         abort(f"Connection '{name}' not found.", code=ErrorCode.NOT_FOUND)
 
     if not is_confirmed:
-        typer.confirm(f"Delete connection '{name}'?", abort=True)
+        confirm_or_abort(f"Delete connection '{name}'?")
 
     client.delete_connection_by_name_if_exists(name)
     print_success(f"Connection '{name}' deleted.")