From 41097a256f9d0cceac660db5f16e4fc80a37d5c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Wed, 19 Aug 2026 14:39:38 +0800 Subject: [PATCH 1/2] feat(index): add opt-in discovery resource limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Indexing accepts whatever a repository contains. A tree carrying a vendored monorepo, a generated dump, or a runaway build directory is discovered in full, and the first sign of trouble is a host under memory pressure with nothing that attributes it to indexing. Add two opt-in limits evaluated during discovery against accepted source files only: index_max_files and index_max_source_mb. Both default to off, so nothing changes until an operator sets one. Crossing a limit fails the whole attempt with a structured resource_limit_exceeded result naming the resource, the observed value and the limit; no partial graph is published, and an existing serving index keeps answering. Limits are read from the CLI-managed _config.db and are not MCP request arguments. A supervised parent replaces any caller-supplied policy before spawning its worker, and the worker rejects a missing or incomplete contract, so the CLI, the daemon and the supervised worker all enforce the same decision. Both keys reach an operator through the existing config get/set/list/reset with no new subcommand. `set` suppresses its own generic message for them because the policy writer names the precise reason -- so that writer speaks on every failure it can return, including a validated value whose write then fails on a database that cannot be written. Exiting non-zero in silence is not an acceptable answer from a CLI. The two shell regressions that hand-roll the supervisor's worker argv carry that contract as well. Without it the worker exits before either guard can observe anything, and the guard would go quietly vacuous. Signed-off-by: 刘冲 --- Makefile.cbm | 2 + README.md | 6 + docs/CONFIGURATION.md | 10 + docs/INDEX_RESOURCE_LIMITS.md | 68 +++++ scripts/test-runtime.sh | 10 + src/cli/cli.c | 56 +++- src/cli/cli.h | 7 + src/daemon/application.c | 35 ++- src/discover/discover.c | 51 +++- src/discover/discover.h | 9 +- src/foundation/index_policy.c | 153 +++++++++++ src/foundation/index_policy.h | 53 ++++ src/mcp/mcp.c | 135 +++++++++- src/mcp/mcp_internal.h | 8 + src/pipeline/pipeline.c | 47 +++- src/pipeline/pipeline.h | 9 + src/pipeline/pipeline_incremental.c | 28 +- src/pipeline/pipeline_internal.h | 4 +- tests/test_daemon_application.c | 84 +++++- tests/test_discover.c | 161 +++++++++++ tests/test_index_policy.c | 403 ++++++++++++++++++++++++++++ tests/test_main.c | 2 + tests/test_pipeline.c | 68 +++++ tests/test_worker_error_response.sh | 2 +- tests/test_worker_watchdog.sh | 2 +- 25 files changed, 1369 insertions(+), 44 deletions(-) create mode 100644 docs/INDEX_RESOURCE_LIMITS.md create mode 100644 src/foundation/index_policy.c create mode 100644 src/foundation/index_policy.h create mode 100644 tests/test_index_policy.c diff --git a/Makefile.cbm b/Makefile.cbm index bf51a5a8c..7368467c5 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -224,6 +224,7 @@ FOUNDATION_SRCS = \ src/foundation/profile.c \ src/foundation/dump_verify.c \ src/foundation/limits.c \ + src/foundation/index_policy.c \ src/foundation/subprocess.c \ src/foundation/sha256.c \ src/foundation/secure_random.c \ @@ -502,6 +503,7 @@ TEST_FOUNDATION_SRCS = \ tests/test_str_intern.c \ tests/test_log.c \ tests/test_str_util.c \ + tests/test_index_policy.c \ tests/test_workspace.c \ tests/test_platform.c \ tests/test_diagnostics.c \ diff --git a/README.md b/README.md index ca39ef418..1aedf6b71 100644 --- a/README.md +++ b/README.md @@ -667,9 +667,15 @@ codebase-memory-mcp config list # show all settings codebase-memory-mcp config set auto_index true # auto-index on session start codebase-memory-mcp config set auto_index_limit 50000 # max files for auto-index codebase-memory-mcp config set auto_watch false # don't register background git watcher (default: true) +codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit +codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit codebase-memory-mcp config reset auto_index # reset to default ``` +The two `index_max_*` settings default to `off`. Exceeding one fails the complete +index attempt rather than publishing a partial graph; an existing serving index +is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). + ### Environment Variables | Variable | Default | Description | diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 0ee8058d1..3200c72c4 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -86,6 +86,16 @@ Current keys: |---|---|---| | `auto_index` | `false` | Automatically index new projects when an MCP session starts. | | `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. | +| `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | +| `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | + +The two `index_max_*` settings are independent and disabled by default. They +apply to explicit indexing, automatic indexing, and watcher re-indexing, but not +to `cross-repo-intelligence`, which does not scan repository source files. +Equality is allowed; exceeding either setting fails the complete index request +and preserves any previously serving database. See +[Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and +error-response details. ## 3. UI Settings diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md new file mode 100644 index 000000000..a3d57e015 --- /dev/null +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -0,0 +1,68 @@ +# Index resource limits + +Index resource limits are optional operator controls for repositories whose +discovery breadth is not known in advance. They are disabled by default so +existing large-repository workloads retain their current behavior. + +## Discovery settings + +| Key | Default | Accepted value | Protects | +|---|---:|---:|---| +| `index_max_files` | `off` | `off` or `1..10000000` | Accepted source-file count | +| `index_max_source_mb` | `off` | `off` or `1..1048576` | Accepted source-file bytes | + +Set or reset them with the normal configuration command: + +```bash +codebase-memory-mcp config set index_max_files 250000 +codebase-memory-mcp config set index_max_source_mb 16384 +codebase-memory-mcp config reset index_max_files +``` + +Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero, +negative values, suffixes, trailing characters, and values outside the stated +ranges are rejected without changing the stored value. + +## Counting and failure semantics + +`index_max_files` counts a file only after it passes directory pruning, ignore +rules, filename and suffix filters, language detection, and the existing +per-file size rule. `index_max_source_mb` sums the filesystem sizes of that same +accepted set. + +Equality is allowed. The first file or byte that makes an observed value greater +than its limit stops discovery. CBM discards the partial file list and does not +publish a partial graph as a complete index. + +For an explicit MCP request the error payload contains: + +```json +{ + "status": "error", + "code": "resource_limit_exceeded", + "stage": "discovery", + "resource": "files", + "observed": 250001, + "limit": 250000, + "unit": "files", + "retryable": true, + "serving_index_preserved": true, + "message": "Index discovery exceeded index_max_files" +} +``` + +The previous database remains available because publication occurs only after a +complete discovery and successful staged build. If no previous database exists, +`serving_index_preserved` is false. + +## Trust and compatibility + +Limits are read from the CLI-managed `_config.db`; they are not MCP request +arguments. A supervised parent replaces any caller-supplied internal policy +before spawning its worker, and the worker rejects a missing or incomplete +parent policy. + +These settings do not replace or increase `auto_index_limit`, change the 512 MiB +single-file cap, alter workspace-root authorization, or affect +`cross-repo-intelligence`. With both settings `off`, discovery follows the +existing unbounded path. diff --git a/scripts/test-runtime.sh b/scripts/test-runtime.sh index cb69cadfb..5d2c0c3ae 100644 --- a/scripts/test-runtime.sh +++ b/scripts/test-runtime.sh @@ -105,6 +105,16 @@ _cbm_test_runtime_daemon() { CBM_CACHE_DIR="$_CBM_TEST_RUNTIME_PRODUCT_CACHE" "$1" daemon "$2" } +# The supervisor resolves one resource policy per index and hands it to the +# worker in argv; a worker that finds no complete policy refuses to start rather +# than index unbounded. A shell test that spawns `cli --index-worker` itself +# stands in for the supervisor and owes the worker the same object. Mirrors +# cbm_mcp_index_policy_add_to_args: every key in cbm_index_policy_key_at, and +# nothing else. +cbm_test_index_worker_policy_json() { + printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"}' +} + cbm_test_runtime_cleanup() { local binary="${1:-}" root="$_CBM_TEST_RUNTIME_CREATED_ROOT" local name="${_CBM_TEST_RUNTIME_CREATED_ROOT##*/}" runtime_entry="" active=0 diff --git a/src/cli/cli.c b/src/cli/cli.c index d981af266..f0159925c 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -6734,6 +6734,25 @@ int cbm_config_delete(cbm_config_t *cfg, const char *key) { return rc; } +bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy, + char *error, size_t error_size) { + if (!cfg || !policy) { + if (error && error_size > 0) { + (void)snprintf(error, error_size, "index resource configuration is unavailable"); + } + return false; + } + cbm_index_policy_init(policy); + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + const char *key = cbm_index_policy_key_at(index); + const char *value = cbm_config_get(cfg, key, cbm_index_policy_default_value(key)); + if (!cbm_index_policy_set(policy, key, value, error, error_size)) { + return false; + } + } + return true; +} + /* ── Config CLI subcommand ────────────────────────────────────── */ /* THE config-key table. list, get, help, and key validation all read this one @@ -6756,6 +6775,8 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_CONFIG_UI_LANG, "auto", "Pin graph UI language: en, zh, or auto"}, {CBM_CONFIG_UI_ENABLED, "false", "Serve the graph UI on a loopback HTTP port"}, {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"}, + {CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"}, + {CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"}, }; /* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing @@ -6781,6 +6802,31 @@ static bool config_key_is_ui(const char *key) { return key && (strcmp(key, CBM_CONFIG_UI_ENABLED) == 0 || strcmp(key, CBM_CONFIG_UI_PORT) == 0); } +static bool config_key_is_index_policy(const char *key) { + return key && (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0 || + strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); +} + +static int config_index_policy_write(cbm_config_t *config, const char *key, const char *value) { + cbm_index_resource_policy_t candidate; + cbm_index_policy_init(&candidate); + char error[CLI_BUF_256]; + if (!cbm_index_policy_set(&candidate, key, value, error, sizeof(error))) { + (void)fprintf(stderr, "error: %s\n", error); + return CLI_ERR; + } + int rc = cbm_config_set(config, key, value); + if (rc != 0) { + /* The caller suppresses its own message for policy keys because this + * helper names the precise reason. That is only true if the helper + * speaks on every failure it can return: a validated value whose write + * then fails -- a locked or read-only _config.db -- used to exit + * non-zero having printed nothing at all. */ + (void)fprintf(stderr, "error: failed to set %s\n", key); + } + return rc; +} + static void config_ui_read(const char *key, char *out, size_t out_sz) { cbm_ui_config_t ui; cbm_ui_config_load(&ui); @@ -6918,10 +6964,16 @@ int cbm_cmd_config(int argc, char **argv) { rc = CLI_TRUE; } } else { - if (cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]) == 0) { + int set_rc = + config_key_is_index_policy(argv[CLI_SKIP_ONE]) + ? config_index_policy_write(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]) + : cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]); + if (set_rc == 0) { printf("%s = %s\n", argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]); } else { - (void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]); + if (!config_key_is_index_policy(argv[CLI_SKIP_ONE])) { + (void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]); + } rc = CLI_TRUE; } } diff --git a/src/cli/cli.h b/src/cli/cli.h index c2121a940..56769c42a 100644 --- a/src/cli/cli.h +++ b/src/cli/cli.h @@ -13,6 +13,8 @@ #include #include +#include "foundation/index_policy.h" + typedef struct cbm_mcp_server cbm_mcp_server_t; /* ── Version ──────────────────────────────────────────────────── */ @@ -417,6 +419,11 @@ int cbm_config_set(cbm_config_t *cfg, const char *key, const char *value); /* Delete a config key. Returns 0 on success. */ int cbm_config_delete(cbm_config_t *cfg, const char *key); +/* Load and validate the operator-controlled discovery policy. Invalid stored + * values fail closed instead of silently disabling a guard. */ +bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy, + char *error, size_t error_size); + /* Well-known config keys */ #define CBM_CONFIG_AUTO_INDEX "auto_index" #define CBM_CONFIG_AUTO_INDEX_LIMIT "auto_index_limit" diff --git a/src/daemon/application.c b/src/daemon/application.c index 7e4a35be7..6cd42504f 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -207,7 +207,8 @@ static bool application_unique_recovery_file(char out[APPLICATION_PATH_CAP], con static bool application_update_reap(cbm_daemon_application_t *application, bool wait, uint32_t timeout_ms); static void *application_job_thread(void *opaque); -static char *application_auto_index_args(const char *root_path); +static char *application_auto_index_args(cbm_daemon_application_t *application, + const char *root_path); static cbm_daemon_application_job_t *application_job_subscribe_locked( cbm_daemon_application_t *application, const char *project_key, const char *root_path, const char *args_json, application_job_subscribe_status_t *status_out); @@ -1399,7 +1400,7 @@ static void application_auto_index_retry_pending_locked(cbm_daemon_application_t application_refresh_watch_locked(session); continue; } - char *args = application_auto_index_args(root_path); + char *args = application_auto_index_args(application, root_path); if (!args) { continue; } @@ -1900,7 +1901,27 @@ static bool application_update_reap(cbm_daemon_application_t *application, bool } } -static char *application_auto_index_args(const char *root_path) { +static bool application_index_args_add_policy(cbm_daemon_application_t *application, + yyjson_mut_doc *document, yyjson_mut_val *root) { + cbm_config_t *owned_config = NULL; + cbm_config_t *config = application ? application->config : NULL; + if (!config) { + owned_config = cbm_config_open(cbm_resolve_cache_dir()); + config = owned_config; + } + cbm_index_resource_policy_t policy; + char error[CBM_SZ_256] = {0}; + bool loaded = cbm_config_load_index_policy(config, &policy, error, sizeof(error)); + cbm_config_close(owned_config); + if (!loaded) { + cbm_log_error("daemon.index.policy", "error", error); + return false; + } + return cbm_mcp_index_policy_add_to_args(document, root, &policy); +} + +static char *application_auto_index_args(cbm_daemon_application_t *application, + const char *root_path) { yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); yyjson_mut_val *root = document ? yyjson_mut_obj(document) : NULL; if (!document || !root) { @@ -1908,7 +1929,8 @@ static char *application_auto_index_args(const char *root_path) { return NULL; } yyjson_mut_doc_set_root(document, root); - char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) + char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) && + application_index_args_add_policy(application, document, root) ? yyjson_mut_write(document, 0, NULL) : NULL; yyjson_mut_doc_free(document); @@ -1951,7 +1973,7 @@ static void application_background_initialize_impl(cbm_daemon_application_sessio files); } bool args_required = auto_index_candidate && within_auto_index_limit; - char *args = args_required ? application_auto_index_args(root_path) : NULL; + char *args = args_required ? application_auto_index_args(application, root_path) : NULL; application_jobs_reap_completed(application); cbm_mutex_lock(&application->mutex); if (application->stopping || application_request_cancelled_locked(session)) { @@ -3351,7 +3373,8 @@ static int application_background_index(cbm_daemon_application_t *application, return -1; } yyjson_mut_doc_set_root(document, root); - bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root); + bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root) && + application_index_args_add_policy(application, document, root); char *default_project = cbm_project_name_from_path(canonical_root); bool custom_project = project_name[0] && (!default_project || strcmp(default_project, project_name) != 0); diff --git a/src/discover/discover.c b/src/discover/discover.c index fe1e4e19d..94cceaeb1 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -13,6 +13,7 @@ #include "foundation/constants.h" #include "foundation/compat_fs.h" +#include "foundation/limits.h" #include "foundation/workspace.h" #include "foundation/platform.h" #ifdef _WIN32 @@ -414,6 +415,10 @@ typedef struct { int capacity; int max_files; uint64_t deadline_ms; + const cbm_index_resource_policy_t *resource_policy; + cbm_index_resource_violation_t *resource_violation; + uint64_t source_files; + uint64_t source_bytes; bool count_only; bool collect_excluded; bool limit_exceeded; @@ -435,6 +440,21 @@ typedef struct { int ignored_total; } file_list_t; +static void file_list_resource_violation(file_list_t *fl, cbm_index_resource_t resource, + uint64_t observed, uint64_t limit) { + if (!fl || fl->limit_exceeded || fl->failed) { + return; + } + fl->limit_exceeded = true; + if (fl->resource_violation) { + *fl->resource_violation = (cbm_index_resource_violation_t){ + .resource = resource, + .observed = observed, + .limit = limit, + }; + } +} + static bool file_list_should_stop(file_list_t *fl) { if (!fl) { return true; @@ -504,6 +524,28 @@ static void fl_add(file_list_t *fl, const char *abs_path, const char *rel_path, fl->limit_exceeded = true; return; } + uint64_t source_size = size > 0 ? (uint64_t)size : 0; + bool resource_accepted = fl->resource_policy && source_size <= (uint64_t)cbm_max_file_bytes(); + if (resource_accepted) { + if (fl->resource_policy->max_files.enabled && + fl->source_files >= fl->resource_policy->max_files.value) { + file_list_resource_violation(fl, CBM_INDEX_RESOURCE_FILES, fl->source_files + 1U, + fl->resource_policy->max_files.value); + return; + } + if (fl->resource_policy->max_source_bytes.enabled) { + uint64_t limit = fl->resource_policy->max_source_bytes.value; + if (source_size > limit || fl->source_bytes > limit - source_size) { + uint64_t observed = source_size > UINT64_MAX - fl->source_bytes + ? UINT64_MAX + : fl->source_bytes + source_size; + file_list_resource_violation(fl, CBM_INDEX_RESOURCE_SOURCE_BYTES, observed, limit); + return; + } + fl->source_bytes += source_size; + } + fl->source_files++; + } if (fl->count_only) { fl->count++; return; @@ -1129,6 +1171,9 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc if (ignored_total_out) { *ignored_total_out = 0; } + if (opts && opts->resource_violation) { + *opts->resource_violation = (cbm_index_resource_violation_t){0}; + } if (!repo_path || !out || !count || (count_only && max_files < 0)) { return CBM_DISCOVER_ERROR; } @@ -1203,6 +1248,8 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc file_list_t fl = { .max_files = count_only ? max_files : -1, .deadline_ms = count_only ? deadline_ms : 0, + .resource_policy = opts ? opts->resource_policy : NULL, + .resource_violation = opts ? opts->resource_violation : NULL, .count_only = count_only, .collect_excluded = !count_only && excluded_out != NULL, .collect_ignored = !count_only && ignored_out != NULL, @@ -1225,11 +1272,11 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc } return fl.limit_exceeded ? CBM_DISCOVER_LIMIT_EXCEEDED : CBM_DISCOVER_OK; } - if (fl.failed) { + if (fl.failed || fl.limit_exceeded) { cbm_discover_free(fl.files, fl.count); cbm_discover_free_excluded(fl.excluded, fl.excluded_count); cbm_discover_free_ignored(fl.ignored, fl.ignored_count); - return CBM_DISCOVER_ERROR; + return fl.limit_exceeded ? CBM_DISCOVER_LIMIT_EXCEEDED : CBM_DISCOVER_ERROR; } *out = fl.files; diff --git a/src/discover/discover.h b/src/discover/discover.h index afb60fa36..59a5ed7a4 100644 --- a/src/discover/discover.h +++ b/src/discover/discover.h @@ -17,6 +17,7 @@ /* Use the existing CBMLanguage enum from extraction layer */ #include "cbm.h" +#include "foundation/index_policy.h" /* ── Language detection ──────────────────────────────────────────── */ @@ -131,9 +132,11 @@ typedef struct { } cbm_file_info_t; typedef struct { - cbm_index_mode_t mode; /* CBM_MODE_FULL or CBM_MODE_FAST */ - const char *ignore_file; /* path to .cbmignore file, or NULL */ - int64_t max_file_size; /* 0 = no limit */ + cbm_index_mode_t mode; /* discovery filtering mode */ + const char *ignore_file; /* .cbmignore path, or NULL */ + int64_t max_file_size; /* 0 = no per-file limit */ + const cbm_index_resource_policy_t *resource_policy; /* NULL = no resource limits */ + cbm_index_resource_violation_t *resource_violation; /* optional exact diagnostic */ } cbm_discover_opts_t; typedef enum { diff --git a/src/foundation/index_policy.c b/src/foundation/index_policy.c new file mode 100644 index 000000000..0d2b986bf --- /dev/null +++ b/src/foundation/index_policy.c @@ -0,0 +1,153 @@ +#include "foundation/index_policy.h" + +#include +#include + +static const char *const INDEX_POLICY_KEYS[] = { + CBM_INDEX_CONFIG_MAX_FILES, + CBM_INDEX_CONFIG_MAX_SOURCE_MB, +}; + +static void set_error(char *error, size_t error_size, const char *key, uint64_t maximum) { + if (error && error_size > 0) { + (void)snprintf(error, error_size, "%s must be off or an integer from 1 to %llu", key, + (unsigned long long)maximum); + } +} + +static bool parse_bounded_uint64(const char *value, uint64_t maximum, uint64_t *parsed) { + if (!value || !value[0] || !parsed) { + return false; + } + uint64_t result = 0; + for (const unsigned char *cursor = (const unsigned char *)value; *cursor; cursor++) { + if (*cursor < '0' || *cursor > '9') { + return false; + } + uint64_t digit = (uint64_t)(*cursor - '0'); + if (result > (maximum - digit) / 10U) { + return false; + } + result = result * 10U + digit; + } + if (result == 0) { + return false; + } + *parsed = result; + return true; +} + +void cbm_index_policy_init(cbm_index_resource_policy_t *policy) { + if (policy) { + *policy = (cbm_index_resource_policy_t){0}; + } +} + +bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy) { + return policy && (policy->max_files.enabled || policy->max_source_bytes.enabled); +} + +size_t cbm_index_policy_key_count(void) { + return sizeof(INDEX_POLICY_KEYS) / sizeof(INDEX_POLICY_KEYS[0]); +} + +const char *cbm_index_policy_key_at(size_t index) { + return index < cbm_index_policy_key_count() ? INDEX_POLICY_KEYS[index] : NULL; +} + +const char *cbm_index_policy_default_value(const char *key) { + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if (key && strcmp(key, INDEX_POLICY_KEYS[index]) == 0) { + return "off"; + } + } + return NULL; +} + +bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, const char *value, + char *error, size_t error_size) { + if (error && error_size > 0) { + error[0] = '\0'; + } + if (!policy || !key || !value) { + set_error(error, error_size, key ? key : "index resource limit", 0); + return false; + } + + cbm_index_limit_u64_t *target = NULL; + uint64_t maximum = 0; + uint64_t multiplier = 1; + if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) { + target = &policy->max_files; + maximum = CBM_INDEX_MAX_FILES_VALUE; + } else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) { + target = &policy->max_source_bytes; + maximum = CBM_INDEX_MAX_SOURCE_MB_VALUE; + multiplier = CBM_INDEX_MIB_BYTES; + } else { + set_error(error, error_size, key, 0); + return false; + } + + cbm_index_limit_u64_t candidate = {0}; + if (strcmp(value, "off") != 0) { + uint64_t parsed = 0; + if (!parse_bounded_uint64(value, maximum, &parsed)) { + set_error(error, error_size, key, maximum); + return false; + } + candidate.enabled = true; + candidate.value = parsed * multiplier; + } + *target = candidate; + return true; +} + +bool cbm_index_policy_format(const cbm_index_resource_policy_t *policy, const char *key, char *out, + size_t out_size) { + if (!policy || !key || !out || out_size == 0) { + return false; + } + const cbm_index_limit_u64_t *limit = NULL; + uint64_t divisor = 1; + if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) { + limit = &policy->max_files; + } else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) { + limit = &policy->max_source_bytes; + divisor = CBM_INDEX_MIB_BYTES; + } else { + return false; + } + int length = limit->enabled + ? snprintf(out, out_size, "%llu", (unsigned long long)(limit->value / divisor)) + : snprintf(out, out_size, "off"); + return length >= 0 && (size_t)length < out_size; +} + +const char *cbm_index_resource_name(cbm_index_resource_t resource) { + switch (resource) { + case CBM_INDEX_RESOURCE_FILES: + return "files"; + case CBM_INDEX_RESOURCE_SOURCE_BYTES: + return "source_bytes"; + case CBM_INDEX_RESOURCE_NONE: + default: + return "unknown"; + } +} + +const char *cbm_index_resource_unit(cbm_index_resource_t resource) { + return resource == CBM_INDEX_RESOURCE_FILES ? "files" : "bytes"; +} + +const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { + switch (resource) { + case CBM_INDEX_RESOURCE_FILES: + return CBM_INDEX_CONFIG_MAX_FILES; + case CBM_INDEX_RESOURCE_SOURCE_BYTES: + return CBM_INDEX_CONFIG_MAX_SOURCE_MB; + case CBM_INDEX_RESOURCE_NONE: + default: + return "index_resource_limit"; + } +} diff --git a/src/foundation/index_policy.h b/src/foundation/index_policy.h new file mode 100644 index 000000000..6a6c499fc --- /dev/null +++ b/src/foundation/index_policy.h @@ -0,0 +1,53 @@ +#ifndef CBM_INDEX_POLICY_H +#define CBM_INDEX_POLICY_H + +#include +#include +#include + +#define CBM_INDEX_CONFIG_MAX_FILES "index_max_files" +#define CBM_INDEX_CONFIG_MAX_SOURCE_MB "index_max_source_mb" + +#define CBM_INDEX_MAX_FILES_VALUE UINT64_C(10000000) +#define CBM_INDEX_MAX_SOURCE_MB_VALUE UINT64_C(1048576) +#define CBM_INDEX_MIB_BYTES UINT64_C(1048576) + +typedef struct { + bool enabled; + uint64_t value; +} cbm_index_limit_u64_t; + +typedef struct { + cbm_index_limit_u64_t max_files; + cbm_index_limit_u64_t max_source_bytes; +} cbm_index_resource_policy_t; + +typedef enum { + CBM_INDEX_RESOURCE_NONE = 0, + CBM_INDEX_RESOURCE_FILES, + CBM_INDEX_RESOURCE_SOURCE_BYTES, +} cbm_index_resource_t; + +typedef struct { + cbm_index_resource_t resource; + uint64_t observed; + uint64_t limit; +} cbm_index_resource_violation_t; + +void cbm_index_policy_init(cbm_index_resource_policy_t *policy); +bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy); + +size_t cbm_index_policy_key_count(void); +const char *cbm_index_policy_key_at(size_t index); +const char *cbm_index_policy_default_value(const char *key); + +bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, const char *value, + char *error, size_t error_size); +bool cbm_index_policy_format(const cbm_index_resource_policy_t *policy, const char *key, char *out, + size_t out_size); + +const char *cbm_index_resource_name(cbm_index_resource_t resource); +const char *cbm_index_resource_unit(cbm_index_resource_t resource); +const char *cbm_index_resource_config_key(cbm_index_resource_t resource); + +#endif diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 618d04469..f842e8f38 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -7758,6 +7758,63 @@ cbm_mcp_supervised_result_disposition_t cbm_mcp_supervised_result_disposition( return CBM_MCP_SUPERVISED_RESULT_CONTAINED_FAILURE; } +static bool index_policy_from_worker_args(const char *args, cbm_index_resource_policy_t *policy, + char *error, size_t error_size) { + yyjson_doc *doc = args ? yyjson_read(args, strlen(args), 0) : NULL; + yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; + yyjson_val *encoded = + root && yyjson_is_obj(root) ? yyjson_obj_get(root, "_cbm_index_policy") : NULL; + if (!encoded || !yyjson_is_obj(encoded) || + yyjson_obj_size(encoded) != cbm_index_policy_key_count()) { + yyjson_doc_free(doc); + (void)snprintf(error, error_size, "missing or incomplete trusted worker policy"); + return false; + } + + cbm_index_policy_init(policy); + bool valid = true; + for (size_t index = 0; valid && index < cbm_index_policy_key_count(); index++) { + const char *key = cbm_index_policy_key_at(index); + yyjson_val *value = yyjson_obj_get(encoded, key); + valid = value && yyjson_is_str(value) && + cbm_index_policy_set(policy, key, yyjson_get_str(value), error, error_size); + } + yyjson_doc_free(doc); + if (!valid && error && error_size > 0 && error[0] == '\0') { + (void)snprintf(error, error_size, "invalid trusted worker policy"); + } + return valid; +} + +static bool load_index_policy(cbm_mcp_server_t *srv, const char *args, + cbm_index_resource_policy_t *policy, char *error, size_t error_size) { + if (cbm_index_worker_active()) { + return index_policy_from_worker_args(args, policy, error, error_size); + } + cbm_config_t *owned_config = NULL; + cbm_config_t *config = srv ? srv->config : NULL; + if (!config) { + owned_config = cbm_config_open(cbm_resolve_cache_dir()); + config = owned_config; + } + bool loaded = cbm_config_load_index_policy(config, policy, error, error_size); + cbm_config_close(owned_config); + return loaded; +} + +bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, + const cbm_index_resource_policy_t *policy) { + yyjson_mut_val *encoded = yyjson_mut_obj(doc); + bool valid = encoded != NULL; + for (size_t index = 0; valid && index < cbm_index_policy_key_count(); index++) { + const char *key = cbm_index_policy_key_at(index); + char value[CBM_SZ_64]; + valid = cbm_index_policy_format(policy, key, value, sizeof(value)) && + yyjson_mut_obj_add_strcpy(doc, encoded, key, value); + } + return valid && yyjson_mut_obj_add_val(doc, root, "_cbm_index_policy", encoded); +} + /* Run index_repository in a supervised worker subprocess with skip-and-continue * (Stage 3c). Returns the response string (caller frees): * - the worker's own response on a clean first run (the common path); @@ -7981,10 +8038,20 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p if (!root_path || !root_path[0]) { return NULL; } + cbm_index_resource_policy_t policy; + char policy_error[CBM_SZ_256] = {0}; + if (!load_index_policy(srv, NULL, &policy, policy_error, sizeof(policy_error))) { + cbm_log_error("index.policy", "error", policy_error); + return NULL; + } yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); yyjson_mut_val *root = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root); - yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path); + if (!yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path) || + !cbm_mcp_index_policy_add_to_args(doc, root, &policy)) { + yyjson_mut_doc_free(doc); + return NULL; + } char *args = yy_doc_to_str(doc); yyjson_mut_doc_free(doc); if (!args) { @@ -8033,8 +8100,9 @@ static bool resolve_session_repo_path(cbm_mcp_server_t *srv, char **repo_path) { /* Preserve every index option while replacing all caller-supplied repo_path * keys with the one canonical path that was actually authorized. */ -static char *index_args_with_repo_path(const char *args, const char *canonical_repo_path) { - if (!args || !canonical_repo_path) { +static char *index_args_with_repo_path(const char *args, const char *canonical_repo_path, + const cbm_index_resource_policy_t *policy) { + if (!args || !canonical_repo_path || !policy) { return NULL; } yyjson_doc *source = yyjson_read(args, strlen(args), 0); @@ -8051,8 +8119,14 @@ static char *index_args_with_repo_path(const char *args, const char *canonical_r yyjson_mut_doc_free(copy); return NULL; } - (void)yyjson_mut_obj_remove_key(copy_root, "repo_path"); - if (!yyjson_mut_obj_add_strcpy(copy, copy_root, "repo_path", canonical_repo_path)) { + while (yyjson_mut_obj_get(copy_root, "repo_path")) { + (void)yyjson_mut_obj_remove_key(copy_root, "repo_path"); + } + while (yyjson_mut_obj_get(copy_root, "_cbm_index_policy")) { + (void)yyjson_mut_obj_remove_key(copy_root, "_cbm_index_policy"); + } + if (!yyjson_mut_obj_add_strcpy(copy, copy_root, "repo_path", canonical_repo_path) || + !cbm_mcp_index_policy_add_to_args(copy, copy_root, policy)) { yyjson_mut_doc_free(copy); return NULL; } @@ -8091,6 +8165,19 @@ static char *resolved_repo_path_from_project_arg(const char *args) { return root_path; } +static bool project_db_is_servable(const char *project, const char *db_path) { + cbm_store_t *store = db_path && db_path[0] ? cbm_store_open_path_query(db_path) : NULL; + if (!store) { + return false; + } + cbm_project_t stored_project = {0}; + bool servable = cbm_store_get_project(store, project, &stored_project) == CBM_STORE_OK && + stored_project.root_path && stored_project.root_path[0]; + cbm_project_free_fields(&stored_project); + cbm_store_close(store); + return servable; +} + static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); char *mode_str = cbm_mcp_get_string_arg(args, "mode"); @@ -8145,10 +8232,19 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { return result; } + cbm_index_resource_policy_t resource_policy; + char policy_error[CBM_SZ_256] = {0}; + if (!load_index_policy(srv, args, &resource_policy, policy_error, sizeof(policy_error))) { + free(mode_str); + free(name_override); + free(repo_path); + return cbm_mcp_text_result(policy_error, true); + } + /* A daemon session delegates the one physical write to its shared job * registry only after path canonicalization and workspace authorization. */ if (srv->index_executor) { - char *worker_args = index_args_with_repo_path(args, repo_path); + char *worker_args = index_args_with_repo_path(args, repo_path, &resource_policy); char *coordinated = worker_args ? srv->index_executor(srv->index_executor_context, repo_path, worker_args) : NULL; @@ -8180,7 +8276,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { * installs the same guard before running the in-process pipeline. A marked * host fails closed if preparation or worker startup cannot complete. */ if (cbm_index_supervisor_should_wrap()) { - char *worker_args = index_args_with_repo_path(args, repo_path); + char *worker_args = index_args_with_repo_path(args, repo_path, &resource_policy); if (!worker_args) { free(mutation_project); free(repo_path); @@ -8252,11 +8348,15 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { } free(name_override); cbm_pipeline_set_persistence(p, persistence); + cbm_pipeline_set_resource_policy(p, &resource_policy); char *project_name = heap_strdup(cbm_pipeline_project_name(p)); /* Bootstrap from artifact if no local DB exists */ try_artifact_bootstrap(project_name, repo_path); + char serving_db_path[CBM_SZ_1K]; + project_db_path(project_name, serving_db_path, sizeof(serving_db_path)); + bool serving_index_was_servable = project_db_is_servable(project_name, serving_db_path); /* Close cached store — pipeline will delete + recreate the .db file */ if (srv->owns_store && srv->store) { @@ -8288,6 +8388,8 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { cbm_file_error_t *file_errors = NULL; int file_error_count = 0; cbm_pipeline_get_file_errors(p, &file_errors, &file_error_count); + cbm_index_resource_violation_t resource_violation = {0}; + cbm_pipeline_get_resource_violation(p, &resource_violation); cbm_mem_collect(); /* return mimalloc pages to OS after large indexing */ @@ -8316,6 +8418,25 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { srv, doc, root, project_name, repo_path, persistence, p, excluded_dirs, excluded_count, file_errors, file_error_count, has_logfile ? logfile_path : NULL); yyjson_mut_obj_add_str(doc, root, "status", degraded ? "degraded" : "indexed"); + } else if (rc == CBM_PIPELINE_RESOURCE_LIMIT && + resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { + const char *config_key = cbm_index_resource_config_key(resource_violation.resource); + char message[CBM_SZ_256]; + (void)snprintf(message, sizeof(message), "Index discovery exceeded %s", config_key); + yyjson_mut_obj_add_str(doc, root, "status", "error"); + yyjson_mut_obj_add_str(doc, root, "code", "resource_limit_exceeded"); + yyjson_mut_obj_add_str(doc, root, "stage", "discovery"); + yyjson_mut_obj_add_str(doc, root, "resource", + cbm_index_resource_name(resource_violation.resource)); + yyjson_mut_obj_add_uint(doc, root, "observed", resource_violation.observed); + yyjson_mut_obj_add_uint(doc, root, "limit", resource_violation.limit); + yyjson_mut_obj_add_str(doc, root, "unit", + cbm_index_resource_unit(resource_violation.resource)); + yyjson_mut_obj_add_bool(doc, root, "retryable", true); + yyjson_mut_obj_add_bool(doc, root, "serving_index_preserved", + serving_index_was_servable && + project_db_is_servable(project_name, serving_db_path)); + yyjson_mut_obj_add_strcpy(doc, root, "message", message); } else { yyjson_mut_obj_add_str(doc, root, "status", "error"); yyjson_mut_obj_add_str(doc, root, "hint", diff --git a/src/mcp/mcp_internal.h b/src/mcp/mcp_internal.h index d9321c595..3f5532a0b 100644 --- a/src/mcp/mcp_internal.h +++ b/src/mcp/mcp_internal.h @@ -1,10 +1,13 @@ #ifndef CBM_MCP_INTERNAL_H #define CBM_MCP_INTERNAL_H +#include "foundation/index_policy.h" #include "mcp/mcp.h" #include "pipeline/pipeline.h" /* cbm_changed_hunk_t */ #include "store/store.h" /* cbm_node_t */ +#include + /* White-box fault injection for deterministic cross-platform quarantine * safety tests. This header is internal and is not part of the MCP API. */ typedef bool (*cbm_mcp_quarantine_test_hook_fn)(void *context, const char *step); @@ -25,6 +28,11 @@ bool cbm_mcp_server_release_pristine_memory_store(cbm_mcp_server_t *srv); * On success replaces and frees *response_io; on failure it is unchanged. */ bool cbm_mcp_jsonrpc_response_prepend_notice(char **response_io, const char *notice); +/* Encode the complete trusted policy on an internal worker request. Callers + * must remove any untrusted field with the same name before invoking this. */ +bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, + const cbm_index_resource_policy_t *policy); + enum { CBM_MCP_DEFAULT_AUTO_INDEX_LIMIT = 50000 }; /* Count indexable files with the pipeline's native full-mode discovery policy, diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 65ac75183..e23e7a86e 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -156,6 +156,8 @@ struct cbm_pipeline { atomic_int cancelled_storage; atomic_int *cancelled; bool persistence; /* write .codebase-memory/graph.db.zst after indexing */ + cbm_index_resource_policy_t resource_policy; + cbm_index_resource_violation_t resource_violation; /* Indexing state (set during run) */ cbm_gbuf_t *gbuf; @@ -304,6 +306,20 @@ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled) { } } +void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, + const cbm_index_resource_policy_t *policy) { + if (p && policy) { + p->resource_policy = *policy; + } +} + +void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p, + cbm_index_resource_violation_t *violation) { + if (violation) { + *violation = p ? p->resource_violation : (cbm_index_resource_violation_t){0}; + } +} + bool cbm_pipeline_set_project_name(cbm_pipeline_t *p, const char *name) { if (!p || !name || !name[0]) { return false; @@ -413,6 +429,14 @@ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p) { return p ? p->repo_path : NULL; } +const cbm_index_resource_policy_t *cbm_pipeline_resource_policy(const cbm_pipeline_t *p) { + return p && cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL; +} + +cbm_index_resource_violation_t *cbm_pipeline_resource_violation(cbm_pipeline_t *p) { + return p ? &p->resource_violation : NULL; +} + atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p) { return p ? p->cancelled : NULL; } @@ -1916,15 +1940,17 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas #if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API cbm_pipeline_persist_test_run_before_final_manifest(); #endif - if (cbm_pipeline_build_fresh_semantic_manifest(p->project_name, p->repo_path, p->mode, - &manifest, &manifest_count) != 0) { + int manifest_rc = + cbm_pipeline_build_fresh_semantic_manifest(p, p->project_name, &manifest, &manifest_count); + if (manifest_rc != 0) { cbm_log_error("pipeline.err", "phase", "semantic_manifest"); /* db_path and db_dir are this function's strdups; the success tail and * the publish-failure return release them, and these two aborts must * too -- LSan caught exactly these paths leaking both strings. */ free(db_dir); free(db_path); - return CBM_PIPELINE_ABORT_PRESERVE_DB; + return manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT + : CBM_PIPELINE_ABORT_PRESERVE_DB; } if (!cbm_pipeline_semantic_manifests_equal(baseline_manifest, baseline_count, manifest, manifest_count)) { @@ -2178,10 +2204,14 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { /* Phase 1: Discover files */ CBM_PROF_START(t_discover); + p->resource_violation = (cbm_index_resource_violation_t){0}; cbm_discover_opts_t opts = { .mode = p->requested_mode, .ignore_file = NULL, .max_file_size = 0, + .resource_policy = + cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL, + .resource_violation = &p->resource_violation, }; cbm_file_info_t *files = NULL; int file_count = 0; @@ -2206,7 +2236,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { cbm_log_info("pipeline.discover", "files", itoa_buf(file_count), "elapsed_ms", itoa_buf((int)elapsed_ms(t0))); if (rc != 0 || check_cancel(p)) { - rc = CBM_NOT_FOUND; + rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT : CBM_NOT_FOUND; goto cleanup; } @@ -2214,14 +2244,15 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { * bytes drive exact no-op comparison and are checked against a fresh * rediscovery immediately before any replacement is published. */ rc = mode_promoted - ? cbm_pipeline_build_fresh_semantic_manifest(p->project_name, p->repo_path, p->mode, - &baseline_manifest, &baseline_count) + ? cbm_pipeline_build_fresh_semantic_manifest(p, p->project_name, &baseline_manifest, + &baseline_count) : cbm_pipeline_build_semantic_manifest(p->project_name, p->repo_path, files, file_count, p->excluded_dirs, p->excluded_count, &p->git_ctx, p->userconfig, &baseline_manifest, &baseline_count); if (rc != 0) { - rc = CBM_PIPELINE_ABORT_PRESERVE_DB; + rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT + : CBM_PIPELINE_ABORT_PRESERVE_DB; goto cleanup; } @@ -2265,7 +2296,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { cbm_log_info("pipeline.rediscover", "requested_mode", pipeline_mode_name(p->requested_mode), "effective_mode", pipeline_mode_name(p->mode), "files", itoa_buf(file_count)); if (rc != 0 || check_cancel(p)) { - rc = CBM_NOT_FOUND; + rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT : CBM_NOT_FOUND; goto cleanup; } } diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 5cde0c0f0..c62b847db 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -30,6 +30,8 @@ typedef struct cbm_gbuf cbm_gbuf_t; typedef struct cbm_pipeline cbm_pipeline_t; +#define CBM_PIPELINE_RESOURCE_LIMIT (-5) + /* ── Index mode ─────────────────────────────────────────────────── */ #ifndef CBM_INDEX_MODE_T_DEFINED @@ -53,6 +55,13 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path, cbm * When enabled, the pipeline writes a compressed artifact after indexing. */ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled); +/* Apply a validated discovery resource policy. The value is copied. */ +void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, const cbm_index_resource_policy_t *policy); + +/* Copy the exact discovery violation from the most recent run. */ +void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p, + cbm_index_resource_violation_t *violation); + /* Free a pipeline and all its internal state. NULL-safe. */ void cbm_pipeline_free(cbm_pipeline_t *p); diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index d3c5247f5..d0db3fbd0 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -609,17 +609,20 @@ bool cbm_pipeline_semantic_manifests_equal(const cbm_file_hash_t *left, int left return equal; } -int cbm_pipeline_build_fresh_semantic_manifest(const char *project, const char *repo_path, int mode, +int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *project, cbm_file_hash_t **out, int *out_count) { - if (!project || !repo_path || !out || !out_count) { + const char *repo_path = cbm_pipeline_repo_path(p); + if (!p || !project || !repo_path || !out || !out_count) { return CBM_NOT_FOUND; } *out = NULL; *out_count = 0; cbm_discover_opts_t opts = { - .mode = (cbm_index_mode_t)mode, + .mode = (cbm_index_mode_t)cbm_pipeline_get_mode(p), .ignore_file = NULL, .max_file_size = 0, + .resource_policy = cbm_pipeline_resource_policy(p), + .resource_violation = cbm_pipeline_resource_violation(p), }; cbm_file_info_t *fresh_files = NULL; int fresh_file_count = 0; @@ -2224,13 +2227,13 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char #if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API cbm_pipeline_persist_test_run_before_final_manifest(); #endif - if (cbm_pipeline_build_fresh_semantic_manifest(project, cbm_pipeline_repo_path(p), - cbm_pipeline_get_mode(p), &manifest, - &manifest_count) != 0 || - !cbm_pipeline_semantic_manifests_equal(baseline_manifest, baseline_count, manifest, - manifest_count)) { + int manifest_rc = + cbm_pipeline_build_fresh_semantic_manifest(p, project, &manifest, &manifest_count); + if (manifest_rc != 0 || !cbm_pipeline_semantic_manifests_equal( + baseline_manifest, baseline_count, manifest, manifest_count)) { cbm_log_warn("delta.abort", "reason", "semantic_inputs_changed"); - result = CBM_PIPELINE_ABORT_PRESERVE_DB; + result = manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT + : CBM_PIPELINE_ABORT_PRESERVE_DB; goto out; } @@ -2816,8 +2819,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil #if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API cbm_pipeline_persist_test_run_before_final_manifest(); #endif - int manifest_rc = cbm_pipeline_build_fresh_semantic_manifest( - project, cbm_pipeline_repo_path(p), cbm_pipeline_get_mode(p), &manifest, &manifest_count); + int manifest_rc = + cbm_pipeline_build_fresh_semantic_manifest(p, project, &manifest, &manifest_count); if (manifest_rc != 0 || !cbm_pipeline_semantic_manifests_equal( baseline_manifest, baseline_count, manifest, manifest_count)) { cbm_log_warn("incremental.abort", "reason", "semantic_inputs_changed"); @@ -2827,7 +2830,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil free_mode_skipped(mode_skipped, mode_skipped_count); free(saved_adr); cbm_gbuf_free(existing); - return CBM_PIPELINE_ABORT_PRESERVE_DB; + return manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT + : CBM_PIPELINE_ABORT_PRESERVE_DB; } /* Step 7: atomically publish the complete staged generation. */ diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 754fa4ffa..c311bbd00 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -678,7 +678,7 @@ bool cbm_pipeline_semantic_manifests_equal(const cbm_file_hash_t *left, int left const cbm_file_hash_t *right, int right_count); /* Re-run discovery and hash its exact semantic inputs. Used at the publication * boundary so late additions/deletions cannot escape a frozen file list. */ -int cbm_pipeline_build_fresh_semantic_manifest(const char *project, const char *repo_path, int mode, +int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *project, cbm_file_hash_t **out, int *out_count); /* Compatibility contract persisted in coverage metadata. Increment when a @@ -760,6 +760,8 @@ void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *row /* Pipeline accessors for incremental use */ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p); +const cbm_index_resource_policy_t *cbm_pipeline_resource_policy(const cbm_pipeline_t *p); +cbm_index_resource_violation_t *cbm_pipeline_resource_violation(cbm_pipeline_t *p); atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p); /* Record committed graph size (#334 gate axis) from the incremental path, * which cannot see the opaque cbm_pipeline struct. Call before the dump. */ diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index c639ace2f..30bf0a430 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1308,6 +1308,7 @@ TEST(daemon_application_prune_clears_logical_watch_for_reregistration) { } enum { APP_FAKE_MAX_ATTEMPTS = 16 }; +enum { APP_FAKE_ARGS_CAP = 2048 }; typedef struct { atomic_int starts; @@ -1328,6 +1329,11 @@ typedef struct { char marker_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; char quarantine_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; char quarantine_seen[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; + char args_json[APP_FAKE_MAX_ATTEMPTS][APP_FAKE_ARGS_CAP]; + /* `starts` is incremented before the slot is filled, so waiting on it says + * a worker began, not that its arguments are readable. Each slot publishes + * itself instead, which is the edge a reader on another thread needs. */ + atomic_bool args_captured[APP_FAKE_MAX_ATTEMPTS]; size_t memory_budgets[APP_FAKE_MAX_ATTEMPTS]; } app_fake_worker_context_t; @@ -1392,6 +1398,9 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem atomic_init(&worker->cancelled, false); worker->result.exit_code = -1; if (worker->attempt < APP_FAKE_MAX_ATTEMPTS) { + (void)snprintf(context->args_json[worker->attempt], APP_FAKE_ARGS_CAP, "%s", + args_json ? args_json : ""); + atomic_store(&context->args_captured[worker->attempt], true); context->memory_budgets[worker->attempt] = memory_budget_bytes; if (marker_file) { (void)snprintf(context->marker_paths[worker->attempt], APP_TEST_PATH_CAP, "%s", @@ -1416,6 +1425,33 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem return 0; } +static bool app_wait_for_atomic_bool(atomic_bool *value, bool expected); + +static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, int attempt, + const char *max_files, const char *max_source_mb) { + if (!context || attempt < 0 || attempt >= APP_FAKE_MAX_ATTEMPTS) { + return false; + } + if (!app_wait_for_atomic_bool(&context->args_captured[attempt], true)) { + return false; + } + const char *args = context->args_json[attempt]; + yyjson_doc *document = yyjson_read(args, strlen(args), 0); + yyjson_val *root = document ? yyjson_doc_get_root(document) : NULL; + yyjson_val *policy = + root && yyjson_is_obj(root) ? yyjson_obj_get(root, "_cbm_index_policy") : NULL; + yyjson_val *files = + policy && yyjson_is_obj(policy) ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_FILES) : NULL; + yyjson_val *bytes = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB) + : NULL; + bool equal = files && bytes && yyjson_is_str(files) && yyjson_is_str(bytes) && + strcmp(yyjson_get_str(files), max_files) == 0 && + strcmp(yyjson_get_str(bytes), max_source_mb) == 0; + yyjson_doc_free(document); + return equal; +} + static cbm_index_worker_poll_t app_fake_worker_poll(void *opaque, cbm_daemon_application_worker_t handle, const cbm_index_worker_result_t **result_out) { @@ -1969,7 +2005,9 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { cbm_config_t *stored_config = cache_set ? cbm_config_open(cache) : NULL; bool config_ready = stored_config && cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && - cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0; + cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "3") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "4") == 0; char canonical_root[APP_TEST_PATH_CAP] = {0}; bool canonical = dirs_ok && cbm_canonical_path(root, canonical_root, sizeof(canonical_root)); char *project = canonical ? cbm_project_name_from_path(canonical_root) : NULL; @@ -2014,6 +2052,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { bool first_owned = first_initialized && project && app_wait_for_subscribers(application, project, 1) && app_wait_for_atomic_int(&fake.starts, 1); + bool auto_policy_propagated = first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4"); bool second_initialized = app_test_initialize_profile(&callbacks, sessions[1], root, CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); bool coalesced = first_owned && second_initialized && project && @@ -2075,6 +2114,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { ASSERT_TRUE(restricted_started_nothing); ASSERT_TRUE(first_initialized); ASSERT_TRUE(first_owned); + ASSERT_TRUE(auto_policy_propagated); ASSERT_TRUE(second_initialized); ASSERT_TRUE(coalesced); ASSERT_TRUE(restricted_disconnect_kept_job); @@ -2089,6 +2129,47 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { PASS(); } +TEST(daemon_application_programmatic_index_injects_resource_policy) { + char *root = th_mktempdir("cbm_app_policy_root"); + char *cache = th_mktempdir("cbm_app_policy_cache"); + ASSERT_NOT_NULL(root); + ASSERT_NOT_NULL(cache); + cbm_config_t *stored_config = cbm_config_open(cache); + bool configured = stored_config && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0; + + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + atomic_store(&fake.allow_completion, true); + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = { + .config = stored_config, + .worker_ops = &worker_ops, + }; + cbm_daemon_application_t *application = configured ? cbm_daemon_application_new(&config) : NULL; + int index_rc = + application ? cbm_daemon_application_index(application, "policy-programmatic", root) : -1; + bool policy_propagated = index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6"); + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + + cbm_daemon_application_free(application); + cbm_config_close(stored_config); + th_cleanup(root); + th_cleanup(cache); + ASSERT_TRUE(configured); + ASSERT_TRUE(policy_propagated); + ASSERT_TRUE(stopped); + PASS(); +} + TEST(daemon_application_auto_index_honors_tracked_file_limit) { app_env_backup_t cache_environment; bool cache_saved = app_env_backup_capture(&cache_environment, "CBM_CACHE_DIR"); @@ -5097,6 +5178,7 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_free_releases_live_watch_once); RUN_TEST(daemon_application_prune_clears_logical_watch_for_reregistration); RUN_TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions); + RUN_TEST(daemon_application_programmatic_index_injects_resource_policy); RUN_TEST(daemon_application_auto_index_honors_tracked_file_limit); RUN_TEST(daemon_application_auto_index_file_count_handles_literal_metacharacter_path); RUN_TEST(daemon_application_auto_index_file_count_supports_non_git_roots); diff --git a/tests/test_discover.c b/tests/test_discover.c index 7e0007b11..ac1ada5e9 100644 --- a/tests/test_discover.c +++ b/tests/test_discover.c @@ -475,6 +475,163 @@ TEST(discover_bounded_count_matches_shebang_discovery) { PASS(); } +TEST(discover_resource_policy_off_matches_legacy_discovery) { + char *base = th_mktempdir("cbm_disc_policy_off"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "src/first.c"), "int first;\n"); + th_write_file(TH_PATH(base, "src/second.py"), "second = 2\n"); + th_write_file(TH_PATH(base, "src/ignored.png"), "not source\n"); + + cbm_file_info_t *legacy_files = NULL; + int legacy_count = 0; + cbm_discover_opts_t legacy_opts = {.mode = CBM_MODE_FULL}; + ASSERT_EQ(cbm_discover(base, &legacy_opts, &legacy_files, &legacy_count), CBM_DISCOVER_OK); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + cbm_index_resource_violation_t violation = {0}; + cbm_file_info_t *policy_files = NULL; + int policy_count = 0; + cbm_discover_opts_t policy_opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_discover_status_t status = cbm_discover(base, &policy_opts, &policy_files, &policy_count); + + ASSERT_EQ(status, CBM_DISCOVER_OK); + ASSERT_EQ(policy_count, legacy_count); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_NONE); + for (int index = 0; index < legacy_count; index++) { + ASSERT_STR_EQ(policy_files[index].rel_path, legacy_files[index].rel_path); + ASSERT_EQ(policy_files[index].size, legacy_files[index].size); + } + + cbm_discover_free(legacy_files, legacy_count); + cbm_discover_free(policy_files, policy_count); + th_cleanup(base); + PASS(); +} + +TEST(discover_resource_file_limit_is_exact_and_counts_only_accepted_sources) { + char *base = th_mktempdir("cbm_disc_policy_files"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, ".gitignore"), "ignored.c\n"); + th_write_file(TH_PATH(base, "accepted.c"), "int accepted;\n"); + th_write_file(TH_PATH(base, "ignored.c"), "int ignored;\n"); + th_write_file(TH_PATH(base, "unsupported.png"), "not source\n"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK); + ASSERT_EQ(count, 1); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_NONE); + cbm_discover_free(files, count); + + th_write_file(TH_PATH(base, "second.py"), "second = 2\n"); + files = NULL; + count = 99; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + + th_cleanup(base); + PASS(); +} + +TEST(discover_resource_source_bytes_allows_equality_and_rejects_one_more_byte) { + char *base = th_mktempdir("cbm_disc_policy_bytes"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "exact.c"), "1234567"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_source_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 7}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK); + ASSERT_EQ(count, 1); + cbm_discover_free(files, count); + + th_write_file(TH_PATH(base, "plus.py"), "x"); + files = NULL; + count = 99; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_SOURCE_BYTES); + ASSERT_EQ(violation.observed, 8); + ASSERT_EQ(violation.limit, 7); + + th_cleanup(base); + PASS(); +} + +TEST(discover_resource_file_budget_excludes_existing_oversized_skip) { + char *base = th_mktempdir("cbm_disc_policy_oversized"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "accepted.c"), "x"); + th_write_file(TH_PATH(base, "oversized.py"), "123"); + const char *saved_limit = getenv("CBM_MAX_FILE_BYTES"); + char *saved_limit_copy = saved_limit ? strdup(saved_limit) : NULL; + cbm_setenv("CBM_MAX_FILE_BYTES", "2", 1); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + cbm_discover_status_t initial_status = cbm_discover(base, &opts, &files, &count); + bool oversized_did_not_consume_budget = initial_status == CBM_DISCOVER_OK && count == 2 && + violation.resource == CBM_INDEX_RESOURCE_NONE; + cbm_discover_free(files, count); + + th_write_file(TH_PATH(base, "second.c"), "y"); + files = NULL; + count = 0; + cbm_discover_status_t exceeded_status = cbm_discover(base, &opts, &files, &count); + bool accepted_sources_exceeded = exceeded_status == CBM_DISCOVER_LIMIT_EXCEEDED && + files == NULL && count == 0 && + violation.resource == CBM_INDEX_RESOURCE_FILES && + violation.observed == 2 && violation.limit == 1; + + if (saved_limit_copy) { + cbm_setenv("CBM_MAX_FILE_BYTES", saved_limit_copy, 1); + } else { + cbm_unsetenv("CBM_MAX_FILE_BYTES"); + } + free(saved_limit_copy); + th_cleanup(base); + + ASSERT_TRUE(oversized_did_not_consume_budget); + ASSERT_TRUE(accepted_sources_exceeded); + PASS(); +} + TEST(discover_skips_git_dir) { char *base = th_mktempdir("cbm_disc_git"); ASSERT(base != NULL); @@ -1767,6 +1924,10 @@ SUITE(discover) { RUN_TEST(discover_bounded_count_is_allocation_free_and_limit_exact); RUN_TEST(discover_bounded_count_fails_closed_after_deadline); RUN_TEST(discover_bounded_count_matches_shebang_discovery); + RUN_TEST(discover_resource_policy_off_matches_legacy_discovery); + RUN_TEST(discover_resource_file_limit_is_exact_and_counts_only_accepted_sources); + RUN_TEST(discover_resource_source_bytes_allows_equality_and_rejects_one_more_byte); + RUN_TEST(discover_resource_file_budget_excludes_existing_oversized_skip); RUN_TEST(discover_skips_git_dir); RUN_TEST(discover_with_gitignore); RUN_TEST(discover_with_global_xdg_ignore); diff --git a/tests/test_index_policy.c b/tests/test_index_policy.c new file mode 100644 index 000000000..d866a0631 --- /dev/null +++ b/tests/test_index_policy.c @@ -0,0 +1,403 @@ +#include "test_framework.h" +#include "test_helpers.h" + +#include "cli/cli.h" +#include "foundation/compat.h" +#include "foundation/index_policy.h" +#include "mcp/index_supervisor.h" +#include "mcp/mcp.h" +#include "store/store.h" + +#include +#include +#include +#include +#include +#include + +TEST(index_policy_defaults_are_disabled) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + + ASSERT_FALSE(policy.max_files.enabled); + ASSERT_FALSE(policy.max_source_bytes.enabled); + ASSERT_FALSE(cbm_index_policy_enabled(&policy)); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_FILES), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_SOURCE_MB), "off"); + PASS(); +} + +TEST(index_policy_file_limit_accepts_off_and_exact_range) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "1", error, sizeof(error))); + ASSERT_TRUE(policy.max_files.enabled); + ASSERT_EQ(policy.max_files.value, 1); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "10000000", error, + sizeof(error))); + ASSERT_EQ(policy.max_files.value, 10000000); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "off", error, sizeof(error))); + ASSERT_FALSE(policy.max_files.enabled); + PASS(); +} + +TEST(index_policy_source_limit_converts_mib_without_overflow) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "1", error, sizeof(error))); + ASSERT_TRUE(policy.max_source_bytes.enabled); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(1024) * UINT64_C(1024)); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "1048576", error, + sizeof(error))); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(1048576) * UINT64_C(1024) * UINT64_C(1024)); + PASS(); +} + +TEST(index_policy_invalid_value_is_rejected_atomically) { + static const char *const invalid[] = {"", "0", "-1", "1MB", + "1 ", "+1", "10000001", "18446744073709551616"}; + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "42", error, sizeof(error))); + + for (size_t index = 0; index < sizeof(invalid) / sizeof(invalid[0]); index++) { + cbm_index_resource_policy_t before = policy; + ASSERT_FALSE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, invalid[index], + error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + ASSERT_TRUE(error[0] != '\0'); + } + ASSERT_FALSE(cbm_index_policy_set(&policy, "index_unknown_limit", "1", error, sizeof(error))); + PASS(); +} + +TEST(index_policy_format_round_trips_public_values) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + char value[64]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "321", error, sizeof(error))); + ASSERT_TRUE(cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_FILES, value, sizeof(value))); + ASSERT_STR_EQ(value, "321"); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "7", error, sizeof(error))); + ASSERT_TRUE( + cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, value, sizeof(value))); + ASSERT_STR_EQ(value, "7"); + PASS(); +} + +TEST(index_policy_violation_metadata_is_stable) { + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_FILES), "files"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_SOURCE_BYTES), "source_bytes"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_FILES), "files"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_SOURCE_BYTES), "bytes"); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_FILES), + CBM_INDEX_CONFIG_MAX_FILES); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_SOURCE_BYTES), + CBM_INDEX_CONFIG_MAX_SOURCE_MB); + PASS(); +} + +TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { + char *cache = th_mktempdir("cbm_index_policy_config"); + ASSERT_NOT_NULL(cache); + cbm_config_t *config = cbm_config_open(cache); + ASSERT_NOT_NULL(config); + cbm_index_resource_policy_t policy; + char error[256]; + + ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); + ASSERT_FALSE(cbm_index_policy_enabled(&policy)); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "9"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "3"), 0); + ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); + ASSERT_EQ(policy.max_files.value, 9); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(3) * CBM_INDEX_MIB_BYTES); + + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "corrupt"), 0); + ASSERT_FALSE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); + ASSERT_TRUE(strstr(error, CBM_INDEX_CONFIG_MAX_FILES) != NULL); + + cbm_config_close(config); + th_cleanup(cache); + PASS(); +} + +TEST(index_policy_cli_lists_both_operator_keys) { + bool files_found = false; + bool bytes_found = false; + for (size_t index = 0; index < cbm_cli_config_key_count_for_testing(); index++) { + const char *key = cbm_cli_config_key_at_for_testing(index); + files_found = files_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0); + bytes_found = bytes_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); + } + ASSERT_TRUE(files_found); + ASSERT_TRUE(bytes_found); + PASS(); +} + +TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite) { + char *cache = th_mktempdir("cbm_index_policy_cli"); + ASSERT_NOT_NULL(cache); + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + + char *set_valid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "7"}; + char *set_invalid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "0"}; + char *reset[] = {"reset", CBM_INDEX_CONFIG_MAX_FILES}; + int valid_rc = cbm_cmd_config(3, set_valid); + cbm_config_t *config = cbm_config_open(cache); + const char *stored = + config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "missing") : "missing"; + bool valid_stored = strcmp(stored, "7") == 0; + int invalid_rc = cbm_cmd_config(3, set_invalid); + stored = config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "missing") : "missing"; + bool invalid_preserved = strcmp(stored, "7") == 0; + int reset_rc = cbm_cmd_config(2, reset); + stored = config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "off") : "missing"; + bool reset_to_default = strcmp(stored, "off") == 0; + cbm_config_close(config); + + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); + th_cleanup(cache); + + ASSERT_EQ(valid_rc, 0); + ASSERT_TRUE(valid_stored); + ASSERT_TRUE(invalid_rc != 0); + ASSERT_TRUE(invalid_preserved); + ASSERT_EQ(reset_rc, 0); + ASSERT_TRUE(reset_to_default); + PASS(); +} + +/* Capture stderr across one cbm_cmd_config invocation, mirroring the stdout + * idiom in test_cli.c: tmpfile+dup2+rewind is what works on the MinGW leg. */ +static int index_policy_config_stderr(int argc, char **argv, char *out, size_t cap) { + out[0] = '\0'; + FILE *capture = tmpfile(); + int saved = capture ? dup(fileno(stderr)) : -1; + if (!capture || saved < 0) { + if (capture) { + (void)fclose(capture); + } + if (saved >= 0) { + (void)close(saved); + } + return -1000; + } + (void)fflush(stderr); + if (dup2(fileno(capture), fileno(stderr)) < 0) { + (void)fclose(capture); + (void)close(saved); + return -1000; + } + int rc = cbm_cmd_config(argc, argv); + (void)fflush(stderr); + (void)dup2(saved, fileno(stderr)); + (void)close(saved); + rewind(capture); + size_t got = fread(out, 1, cap - 1, capture); + out[got] = '\0'; + (void)fclose(capture); + return rc; +} + +/* `config set` suppresses its own generic message for a policy key, trusting + * the policy writer to name the precise reason. That trust held only for a + * value the writer rejects. A value it accepts whose write then fails — a + * _config.db that cannot be written — exited non-zero having printed nothing + * at all, which is a CLI that failed in silence. + * + * The unwritable database here is a `config` that is a view: CREATE TABLE IF + * NOT EXISTS finds the name taken and succeeds, so the command gets past open + * with a healthy handle and fails at the INSERT. That is the exact path with + * no message, and it needs no file permissions, so it behaves the same on + * every leg and under a root CI container. */ +TEST(index_policy_cli_set_reports_a_failed_write) { + char *cache = th_mktempdir("cbm_index_policy_cli_write"); + ASSERT_NOT_NULL(cache); + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + + char db_path[1024]; + (void)snprintf(db_path, sizeof(db_path), "%s/_config.db", cache); + sqlite3 *db = NULL; + bool fixture_ready = + sqlite3_open(db_path, &db) == SQLITE_OK && + sqlite3_exec(db, "CREATE VIEW config(key, value) AS SELECT 'pinned', 'pinned'", NULL, NULL, + NULL) == SQLITE_OK; + if (db) { + (void)sqlite3_close(db); + } + + char captured[1024]; + char *set_valid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "7"}; + int rc = index_policy_config_stderr(3, set_valid, captured, sizeof(captured)); + + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); + th_cleanup(cache); + + ASSERT_TRUE(fixture_ready); + ASSERT_TRUE(rc != 0); + ASSERT_TRUE(strstr(captured, CBM_INDEX_CONFIG_MAX_FILES) != NULL); + PASS(); +} + +TEST(index_policy_worker_rejects_missing_parent_policy) { + char *repo = th_mktempdir("cbm_index_policy_worker"); + ASSERT_NOT_NULL(repo); + char args[1024]; + (void)snprintf(args, sizeof(args), "{\"repo_path\":\"%s\",\"mode\":\"fast\"}", repo); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + + cbm_index_set_worker_role(true, NULL); + char *response = server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + cbm_index_set_worker_role(false, NULL); + bool rejected = response && strstr(response, "missing or incomplete trusted worker policy"); + + free(response); + cbm_mcp_server_free(server); + th_cleanup(repo); + ASSERT_TRUE(rejected); + PASS(); +} + +TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index) { + char *repo = th_mktempdir("cbm_index_policy_mcp_repo"); + char *cache = th_mktempdir("cbm_index_policy_mcp_cache"); + ASSERT_NOT_NULL(repo); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first(void) { return 1; }\n"), 0); + + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + cbm_config_t *config = cbm_config_open(cache); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + if (server && config) { + cbm_mcp_server_set_config(server, config); + } + + char args[2048]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\"," + "\"mode\":\"fast\"}", + repo); + char *first_response = + server && config ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + bool first_indexed = first_response && strstr(first_response, "\\\"status\\\":\\\"indexed\\\""); + free(first_response); + + char db_path[2048]; + (void)snprintf(db_path, sizeof(db_path), "%s/ResourcePolicyFixture.db", cache); + cbm_store_t *before_store = cbm_store_open_path_query(db_path); + cbm_project_t before_project = {0}; + bool before_read = before_store && cbm_store_get_project(before_store, "ResourcePolicyFixture", + &before_project) == CBM_STORE_OK; + char *indexed_at_before = + before_read && before_project.indexed_at ? strdup(before_project.indexed_at) : NULL; + cbm_project_free_fields(&before_project); + cbm_store_close(before_store); + + bool configured = + config && cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "1") == 0 && + th_write_file(TH_PATH(repo, "second.py"), "def second():\n return 2\n") == 0; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\"," + "\"mode\":\"fast\",\"_cbm_index_policy\":{" + "\"index_max_files\":\"off\",\"index_max_source_mb\":\"off\"}}", + repo); + char *limited_response = + configured && server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + bool contract_ok = limited_response && strstr(limited_response, "resource_limit_exceeded") && + strstr(limited_response, "\\\"stage\\\":\\\"discovery\\\"") && + strstr(limited_response, "\\\"resource\\\":\\\"files\\\"") && + strstr(limited_response, "\\\"observed\\\":2") && + strstr(limited_response, "\\\"limit\\\":1") && + strstr(limited_response, "\\\"unit\\\":\\\"files\\\"") && + strstr(limited_response, "\\\"retryable\\\":true") && + strstr(limited_response, "\\\"serving_index_preserved\\\":true"); + free(limited_response); + + cbm_store_t *after_store = cbm_store_open_path_query(db_path); + cbm_project_t after_project = {0}; + bool after_read = after_store && cbm_store_get_project(after_store, "ResourcePolicyFixture", + &after_project) == CBM_STORE_OK; + bool generation_preserved = indexed_at_before && after_read && after_project.indexed_at && + strcmp(indexed_at_before, after_project.indexed_at) == 0; + cbm_project_free_fields(&after_project); + cbm_store_close(after_store); + free(indexed_at_before); + + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\"," + "\"mode\":\"cross-repo-intelligence\"}", + repo); + char *cross_response = server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + bool cross_repo_unaffected = + cross_response && strstr(cross_response, "resource_limit_exceeded") == NULL; + free(cross_response); + + cbm_mcp_server_free(server); + cbm_config_close(config); + (void)cbm_unlink(db_path); + char sidecar[2100]; + (void)snprintf(sidecar, sizeof(sidecar), "%s-wal", db_path); + (void)cbm_unlink(sidecar); + (void)snprintf(sidecar, sizeof(sidecar), "%s-shm", db_path); + (void)cbm_unlink(sidecar); + th_cleanup(repo); + th_cleanup(cache); + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); + + ASSERT_TRUE(first_indexed); + ASSERT_TRUE(configured); + ASSERT_TRUE(contract_ok); + ASSERT_TRUE(generation_preserved); + ASSERT_TRUE(cross_repo_unaffected); + PASS(); +} + +SUITE(index_policy) { + RUN_TEST(index_policy_defaults_are_disabled); + RUN_TEST(index_policy_file_limit_accepts_off_and_exact_range); + RUN_TEST(index_policy_source_limit_converts_mib_without_overflow); + RUN_TEST(index_policy_invalid_value_is_rejected_atomically); + RUN_TEST(index_policy_format_round_trips_public_values); + RUN_TEST(index_policy_violation_metadata_is_stable); + RUN_TEST(index_policy_config_loads_defaults_values_and_rejects_corruption); + RUN_TEST(index_policy_cli_lists_both_operator_keys); + RUN_TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite); + RUN_TEST(index_policy_cli_set_reports_a_failed_write); + RUN_TEST(index_policy_worker_rejects_missing_parent_policy); + RUN_TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index); +} diff --git a/tests/test_main.c b/tests/test_main.c index ba6f26d45..b38cc912b 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -677,6 +677,7 @@ extern void suite_dyn_array(void); extern void suite_str_intern(void); extern void suite_log(void); extern void suite_str_util(void); +extern void suite_index_policy(void); extern void suite_workspace(void); extern void suite_platform(void); extern void suite_diagnostics(void); @@ -924,6 +925,7 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(str_intern); RUN_SELECTED_SUITE(log); RUN_SELECTED_SUITE(str_util); + RUN_SELECTED_SUITE(index_policy); RUN_SELECTED_SUITE(workspace); RUN_SELECTED_SUITE(platform); RUN_SELECTED_SUITE(diagnostics); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 25878a8b3..5ef810908 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -151,6 +151,34 @@ TEST(pipeline_run_null) { PASS(); } +TEST(pipeline_discovery_limit_returns_exact_violation_without_publishing) { + char *repo = th_mktempdir("cbm_pipeline_discovery_limit"); + ASSERT_NOT_NULL(repo); + ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first;\n"), 0); + ASSERT_EQ(th_write_file(TH_PATH(repo, "second.py"), "second = 2\n"), 0); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/index.db", repo); + cbm_pipeline_t *pipeline = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(pipeline); + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_pipeline_set_resource_policy(pipeline, &policy); + + ASSERT_EQ(cbm_pipeline_run(pipeline), CBM_PIPELINE_RESOURCE_LIMIT); + cbm_index_resource_violation_t violation = {0}; + cbm_pipeline_get_resource_violation(pipeline, &violation); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + ASSERT_FALSE(cbm_file_exists(db_path)); + + cbm_pipeline_free(pipeline); + th_cleanup(repo); + PASS(); +} + /* ── Focused: file-backed store persistence ─────────────────────── */ TEST(store_file_persistence) { @@ -2845,6 +2873,44 @@ static void mutate_semantic_input_before_final_manifest(void *userdata) { mutation->write_rc = th_write_file(mutation->path, mutation->replacement); } +TEST(pipeline_late_source_limit_preserves_previous_generation) { + char *repo = th_mktempdir("cbm_pipeline_late_limit"); + ASSERT_NOT_NULL(repo); + ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first;\n"), 0); + char db_path[512]; + char late_path[512]; + snprintf(db_path, sizeof(db_path), "%s/index.db", repo); + snprintf(late_path, sizeof(late_path), "%s/late.py", repo); + + manifest_race_mutation_t mutation = { + .path = late_path, + .replacement = "late = 2\n", + .write_rc = -1, + }; + cbm_pipeline_incremental_test_before_final_manifest_once( + mutate_semantic_input_before_final_manifest, &mutation); + cbm_pipeline_t *pipeline = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(pipeline); + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_pipeline_set_resource_policy(pipeline, &policy); + int rc = cbm_pipeline_run(pipeline); + cbm_index_resource_violation_t violation = {0}; + cbm_pipeline_get_resource_violation(pipeline, &violation); + cbm_pipeline_free(pipeline); + cbm_pipeline_incremental_test_reset_faults(); + + ASSERT_EQ(mutation.write_rc, 0); + ASSERT_EQ(rc, CBM_PIPELINE_RESOURCE_LIMIT); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + ASSERT_FALSE(cbm_file_exists(db_path)); + th_cleanup(repo); + PASS(); +} + /* A graph and its exact manifest are one generation. If source bytes change * after extraction but before publication, the mixed generation must be * rejected and the previous live DB preserved. */ @@ -11951,6 +12017,8 @@ SUITE(pipeline) { RUN_TEST(pipeline_cancel); RUN_TEST(pipeline_cancel_null); RUN_TEST(pipeline_run_null); + RUN_TEST(pipeline_discovery_limit_returns_exact_violation_without_publishing); + RUN_TEST(pipeline_late_source_limit_preserves_previous_generation); /* Extraction back-pressure */ RUN_TEST(pipeline_backpressure_futile_nap_disengages); /* Sequential cross-LSP shared registry (ms-typescript quadratic) */ diff --git a/tests/test_worker_error_response.sh b/tests/test_worker_error_response.sh index 33e2db115..500c57ff7 100755 --- a/tests/test_worker_error_response.sh +++ b/tests/test_worker_error_response.sh @@ -48,7 +48,7 @@ trap cleanup EXIT missing="${tmpdir}/repository-does-not-exist" response="${tmpdir}/worker.response" -args="{\"repo_path\":\"${missing}\",\"mode\":\"fast\"}" +args="{\"repo_path\":\"${missing}\",\"mode\":\"fast\",$(cbm_test_index_worker_policy_json)}" if ! CBM_CACHE_DIR="${tmpdir}/cache-worker" \ "${BINARY}" cli --index-worker \ diff --git a/tests/test_worker_watchdog.sh b/tests/test_worker_watchdog.sh index 7a52603a0..e38be10bf 100755 --- a/tests/test_worker_watchdog.sh +++ b/tests/test_worker_watchdog.sh @@ -109,7 +109,7 @@ SH chmod +x "${tmpdir}/wrapper.sh" CBM_BINARY="${BINARY}" BUILD_FINGERPRINT="${BUILD_FINGERPRINT}" TMPDIR_PATH="${tmpdir}" \ - ARGS_JSON="{\"repo_path\":\"${tmpdir}/repo\"}" \ + ARGS_JSON="{\"repo_path\":\"${tmpdir}/repo\",$(cbm_test_index_worker_policy_json)}" \ CBM_TEST_HANG_ON=hang_me \ CBM_TEST_WORKER_DESCENDANT_PID_FILE="${tmpdir}/descendant.pid" \ "${tmpdir}/wrapper.sh" & From a1339ec90922542e78fd38cd2b1fde41dc7dabe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Wed, 19 Aug 2026 14:39:38 +0800 Subject: [PATCH 2/2] feat(index): add worker resource watchdogs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discovery limits bound what indexing accepts, not what it then costs. A repository well inside those bounds can still exhaust the host through parser memory, or simply never finish, and a supervised worker that hangs leaves the parent waiting with nothing to report. Add index_max_rss_mb and index_max_duration_seconds, enforced by the parent against the worker process tree rather than the worker process alone, so a runaway child cannot hide behind a small parent. Resident memory is sampled through the platform interface on macOS, Linux and Windows. Crossing a limit terminates the tree and yields one trusted, structured terminal result that attributes the failure to the resource that caused it. Both limits default to off. A measurement that cannot be taken fails the attempt instead of passing it: a watchdog that quietly stops watching is worse than no watchdog at all. The shell fixture that stands in for the supervisor names the two new keys. The worker accepts only a policy that spells out every key it knows, which is what keeps a stale supervisor from starting a worker it cannot bound. Signed-off-by: 刘冲 --- README.md | 4 +- docs/CONFIGURATION.md | 10 +- docs/INDEX_RESOURCE_LIMITS.md | 48 +++++- scripts/test-runtime.sh | 3 +- src/cli/cli.c | 10 +- src/daemon/application.c | 19 ++- src/foundation/index_policy.c | 126 +++++++++----- src/foundation/index_policy.h | 11 ++ src/foundation/subprocess.c | 285 ++++++++++++++++++++++++++++++++ src/foundation/subprocess.h | 18 ++ src/mcp/index_supervisor.c | 256 ++++++++++++++++++++++++++-- src/mcp/index_supervisor.h | 28 +++- src/mcp/mcp.c | 93 +++++++++-- src/mcp/mcp.h | 1 + src/mcp/mcp_internal.h | 4 + src/pipeline/pipeline.c | 5 +- tests/test_daemon_application.c | 132 +++++++++++++-- tests/test_index_policy.c | 72 +++++++- tests/test_index_supervisor.c | 269 ++++++++++++++++++++++++++++++ tests/test_mcp.c | 108 ++++++++++++ tests/test_subprocess.c | 47 ++++++ 21 files changed, 1451 insertions(+), 98 deletions(-) diff --git a/README.md b/README.md index 1aedf6b71..8c249b087 100644 --- a/README.md +++ b/README.md @@ -669,10 +669,12 @@ codebase-memory-mcp config set auto_index_limit 50000 # max files for auto-in codebase-memory-mcp config set auto_watch false # don't register background git watcher (default: true) codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit +codebase-memory-mcp config set index_max_rss_mb 8192 # optional worker-tree current RSS limit +codebase-memory-mcp config set index_max_duration_seconds 3600 # optional total worker duration codebase-memory-mcp config reset auto_index # reset to default ``` -The two `index_max_*` settings default to `off`. Exceeding one fails the complete +The four `index_max_*` settings default to `off`. Exceeding one fails the complete index attempt rather than publishing a partial graph; an existing serving index is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 3200c72c4..c3bdf8d96 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -88,12 +88,16 @@ Current keys: | `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. | | `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | | `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | +| `index_max_rss_mb` | `off` | Optional maximum current RSS in MiB for the complete contained index-worker process tree (`64..1048576`). | +| `index_max_duration_seconds` | `off` | Optional maximum total worker duration in seconds (`1..86400`). | -The two `index_max_*` settings are independent and disabled by default. They +The four `index_max_*` settings are independent and disabled by default. They apply to explicit indexing, automatic indexing, and watcher re-indexing, but not to `cross-repo-intelligence`, which does not scan repository source files. -Equality is allowed; exceeding either setting fails the complete index request -and preserves any previously serving database. See +Equality is allowed; exceeding any setting fails the complete index request and +preserves any previously serving database. Worker RSS covers descendants and is +not the same as the internal `CBM_MEM_BUDGET_MB` allocation budget. Total +duration is independent of the existing 15-minute no-log-progress timeout. See [Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and error-response details. diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index a3d57e015..66be4ecf4 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -1,8 +1,8 @@ # Index resource limits Index resource limits are optional operator controls for repositories whose -discovery breadth is not known in advance. They are disabled by default so -existing large-repository workloads retain their current behavior. +discovery breadth or worker runtime is not known in advance. They are disabled +by default so existing large-repository workloads retain their current behavior. ## Discovery settings @@ -23,6 +23,39 @@ Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero, negative values, suffixes, trailing characters, and values outside the stated ranges are rejected without changing the stored value. +## Worker settings + +| Key | Default | Accepted value | Protects | +|---|---:|---:|---| +| `index_max_rss_mb` | `off` | `off` or `64..1048576` | Current RSS of the complete worker process tree | +| `index_max_duration_seconds` | `off` | `off` or `1..86400` | Total worker wall-clock duration | + +```bash +codebase-memory-mcp config set index_max_rss_mb 8192 +codebase-memory-mcp config set index_max_duration_seconds 3600 +``` + +RSS is the current resident memory of the contained worker and every descendant, +not the worker's allocation budget and not peak memory. This hard watchdog is +separate from the internal `CBM_MEM_BUDGET_MB` soft budget. The supervisor +samples RSS at most once every 250 milliseconds so the watchdog does not turn +full process-table enumeration into a busy loop. + +Duration uses a monotonic clock from successful spawn. It is independent of the +existing 15-minute quiet timeout: continuous log progress does not reset total +duration, while the quiet timeout continues to identify a worker that stops +making progress. + +Equality is allowed. The first RSS or elapsed-duration observation above its +limit starts the existing graceful-to-force process-tree shutdown. CBM reports +terminal only after the tree is quiescent or a bounded containment failure is +explicitly surfaced. Resource termination is not retried and does not +quarantine a source file. + +If RSS is enabled and three consecutive probes cannot obtain any trustworthy +tree measurement while the root worker is still running, CBM fails closed with +`code=resource_probe_failed`. + ## Counting and failure semantics `index_max_files` counts a file only after it passes directory pruning, ignore @@ -55,6 +88,12 @@ The previous database remains available because publication occurs only after a complete discovery and successful staged build. If no previous database exists, `serving_index_preserved` is false. +Worker limit failures use the same shape with `stage=worker`, +`resource=rss_bytes` and `unit=bytes`, or `resource=duration_ms` and +`unit=milliseconds`. RSS measurement failures use `code=resource_probe_failed` +and omit `observed`, `limit`, and `unit` because no trustworthy observation was +available. + ## Trust and compatibility Limits are read from the CLI-managed `_config.db`; they are not MCP request @@ -64,5 +103,6 @@ parent policy. These settings do not replace or increase `auto_index_limit`, change the 512 MiB single-file cap, alter workspace-root authorization, or affect -`cross-repo-intelligence`. With both settings `off`, discovery follows the -existing unbounded path. +`cross-repo-intelligence`. With all settings `off`, discovery follows the +existing path and the supervisor performs no periodic RSS probe or total-duration +termination. diff --git a/scripts/test-runtime.sh b/scripts/test-runtime.sh index 5d2c0c3ae..e6e60b15a 100644 --- a/scripts/test-runtime.sh +++ b/scripts/test-runtime.sh @@ -112,7 +112,8 @@ _cbm_test_runtime_daemon() { # cbm_mcp_index_policy_add_to_args: every key in cbm_index_policy_key_at, and # nothing else. cbm_test_index_worker_policy_json() { - printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"}' + printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"' + printf '%s' ',"index_max_rss_mb":"off","index_max_duration_seconds":"off"}' } cbm_test_runtime_cleanup() { diff --git a/src/cli/cli.c b/src/cli/cli.c index f0159925c..8c52bb5eb 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -6777,6 +6777,8 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"}, {CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"}, {CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"}, + {CBM_INDEX_CONFIG_MAX_RSS_MB, "off", "Max worker process-tree RSS MiB, or off"}, + {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "off", "Max worker duration in seconds, or off"}, }; /* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing @@ -6803,8 +6805,12 @@ static bool config_key_is_ui(const char *key) { } static bool config_key_is_index_policy(const char *key) { - return key && (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0 || - strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); + for (size_t index = 0; key && index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, cbm_index_policy_key_at(index)) == 0) { + return true; + } + } + return false; } static int config_index_policy_write(cbm_config_t *config, const char *key, const char *value) { diff --git a/src/daemon/application.c b/src/daemon/application.c index 6cd42504f..dce9468f2 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -304,9 +304,18 @@ static int application_worker_start_default(void *context, const char *args_json const char *quarantine_file, cbm_daemon_application_worker_t *worker_out) { (void)context; + cbm_index_resource_policy_t resource_policy; + char error[CBM_SZ_256] = {0}; + if (!cbm_mcp_index_policy_from_internal_args(args_json, &resource_policy, error, + sizeof(error))) { + cbm_log_error("daemon.index.policy", "error", error); + *worker_out = NULL; + return -1; + } cbm_index_worker_handle_t *worker = NULL; - int result = cbm_index_worker_start(args_json, memory_budget_bytes, false, marker_file, - quarantine_file, &worker); + int result = + cbm_index_worker_start_with_policy(args_json, memory_budget_bytes, &resource_policy, false, + marker_file, quarantine_file, &worker); *worker_out = worker; return result; } @@ -1233,6 +1242,12 @@ static application_attempt_decision_t application_consume_attempt( application_attempt_free(attempt); return APPLICATION_ATTEMPT_DECISION_SUCCESS; } + if (disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + execution->response = + cbm_mcp_index_worker_resource_response(job->args_json, &attempt->result); + application_attempt_free(attempt); + return APPLICATION_ATTEMPT_DECISION_STOP; + } if (disposition == CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL) { execution->unsafe_terminal = true; execution->supervision_failed = diff --git a/src/foundation/index_policy.c b/src/foundation/index_policy.c index 0d2b986bf..80aa2f8a6 100644 --- a/src/foundation/index_policy.c +++ b/src/foundation/index_policy.c @@ -1,21 +1,38 @@ #include "foundation/index_policy.h" +#include #include #include -static const char *const INDEX_POLICY_KEYS[] = { - CBM_INDEX_CONFIG_MAX_FILES, - CBM_INDEX_CONFIG_MAX_SOURCE_MB, +typedef struct { + const char *key; + size_t field_offset; + uint64_t minimum; + uint64_t maximum; + uint64_t multiplier; +} index_policy_metadata_t; + +static const index_policy_metadata_t INDEX_POLICY_METADATA[] = { + {CBM_INDEX_CONFIG_MAX_FILES, offsetof(cbm_index_resource_policy_t, max_files), 1, + CBM_INDEX_MAX_FILES_VALUE, 1}, + {CBM_INDEX_CONFIG_MAX_SOURCE_MB, offsetof(cbm_index_resource_policy_t, max_source_bytes), 1, + CBM_INDEX_MAX_SOURCE_MB_VALUE, CBM_INDEX_MIB_BYTES}, + {CBM_INDEX_CONFIG_MAX_RSS_MB, offsetof(cbm_index_resource_policy_t, max_rss_bytes), + CBM_INDEX_MIN_RSS_MB_VALUE, CBM_INDEX_MAX_RSS_MB_VALUE, CBM_INDEX_MIB_BYTES}, + {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, offsetof(cbm_index_resource_policy_t, max_duration_ms), + 1, CBM_INDEX_MAX_DURATION_SECONDS_VALUE, UINT64_C(1000)}, }; -static void set_error(char *error, size_t error_size, const char *key, uint64_t maximum) { +static void set_error(char *error, size_t error_size, const char *key, uint64_t minimum, + uint64_t maximum) { if (error && error_size > 0) { - (void)snprintf(error, error_size, "%s must be off or an integer from 1 to %llu", key, - (unsigned long long)maximum); + (void)snprintf(error, error_size, "%s must be off or an integer from %llu to %llu", key, + (unsigned long long)minimum, (unsigned long long)maximum); } } -static bool parse_bounded_uint64(const char *value, uint64_t maximum, uint64_t *parsed) { +static bool parse_bounded_uint64(const char *value, uint64_t minimum, uint64_t maximum, + uint64_t *parsed) { if (!value || !value[0] || !parsed) { return false; } @@ -30,7 +47,7 @@ static bool parse_bounded_uint64(const char *value, uint64_t maximum, uint64_t * } result = result * 10U + digit; } - if (result == 0) { + if (result < minimum) { return false; } *parsed = result; @@ -44,20 +61,39 @@ void cbm_index_policy_init(cbm_index_resource_policy_t *policy) { } bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy) { + if (!policy) { + return false; + } + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + const cbm_index_limit_u64_t *limit = + (const cbm_index_limit_u64_t *)((const unsigned char *)policy + + INDEX_POLICY_METADATA[index].field_offset); + if (limit->enabled) { + return true; + } + } + return false; +} + +bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy) { return policy && (policy->max_files.enabled || policy->max_source_bytes.enabled); } +bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy) { + return policy && (policy->max_rss_bytes.enabled || policy->max_duration_ms.enabled); +} + size_t cbm_index_policy_key_count(void) { - return sizeof(INDEX_POLICY_KEYS) / sizeof(INDEX_POLICY_KEYS[0]); + return sizeof(INDEX_POLICY_METADATA) / sizeof(INDEX_POLICY_METADATA[0]); } const char *cbm_index_policy_key_at(size_t index) { - return index < cbm_index_policy_key_count() ? INDEX_POLICY_KEYS[index] : NULL; + return index < cbm_index_policy_key_count() ? INDEX_POLICY_METADATA[index].key : NULL; } const char *cbm_index_policy_default_value(const char *key) { for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { - if (key && strcmp(key, INDEX_POLICY_KEYS[index]) == 0) { + if (key && strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { return "off"; } } @@ -70,34 +106,33 @@ bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, error[0] = '\0'; } if (!policy || !key || !value) { - set_error(error, error_size, key ? key : "index resource limit", 0); + set_error(error, error_size, key ? key : "index resource limit", 0, 0); return false; } - cbm_index_limit_u64_t *target = NULL; - uint64_t maximum = 0; - uint64_t multiplier = 1; - if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) { - target = &policy->max_files; - maximum = CBM_INDEX_MAX_FILES_VALUE; - } else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) { - target = &policy->max_source_bytes; - maximum = CBM_INDEX_MAX_SOURCE_MB_VALUE; - multiplier = CBM_INDEX_MIB_BYTES; - } else { - set_error(error, error_size, key, 0); + const index_policy_metadata_t *metadata = NULL; + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { + metadata = &INDEX_POLICY_METADATA[index]; + break; + } + } + if (!metadata) { + set_error(error, error_size, key, 0, 0); return false; } + cbm_index_limit_u64_t *target = + (cbm_index_limit_u64_t *)((unsigned char *)policy + metadata->field_offset); cbm_index_limit_u64_t candidate = {0}; if (strcmp(value, "off") != 0) { uint64_t parsed = 0; - if (!parse_bounded_uint64(value, maximum, &parsed)) { - set_error(error, error_size, key, maximum); + if (!parse_bounded_uint64(value, metadata->minimum, metadata->maximum, &parsed)) { + set_error(error, error_size, key, metadata->minimum, metadata->maximum); return false; } candidate.enabled = true; - candidate.value = parsed * multiplier; + candidate.value = parsed * metadata->multiplier; } *target = candidate; return true; @@ -108,18 +143,21 @@ bool cbm_index_policy_format(const cbm_index_resource_policy_t *policy, const ch if (!policy || !key || !out || out_size == 0) { return false; } - const cbm_index_limit_u64_t *limit = NULL; - uint64_t divisor = 1; - if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) { - limit = &policy->max_files; - } else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) { - limit = &policy->max_source_bytes; - divisor = CBM_INDEX_MIB_BYTES; - } else { + const index_policy_metadata_t *metadata = NULL; + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { + metadata = &INDEX_POLICY_METADATA[index]; + break; + } + } + if (!metadata) { return false; } + const cbm_index_limit_u64_t *limit = + (const cbm_index_limit_u64_t *)((const unsigned char *)policy + metadata->field_offset); int length = limit->enabled - ? snprintf(out, out_size, "%llu", (unsigned long long)(limit->value / divisor)) + ? snprintf(out, out_size, "%llu", + (unsigned long long)(limit->value / metadata->multiplier)) : snprintf(out, out_size, "off"); return length >= 0 && (size_t)length < out_size; } @@ -130,6 +168,10 @@ const char *cbm_index_resource_name(cbm_index_resource_t resource) { return "files"; case CBM_INDEX_RESOURCE_SOURCE_BYTES: return "source_bytes"; + case CBM_INDEX_RESOURCE_RSS_BYTES: + return "rss_bytes"; + case CBM_INDEX_RESOURCE_DURATION_MS: + return "duration_ms"; case CBM_INDEX_RESOURCE_NONE: default: return "unknown"; @@ -137,7 +179,13 @@ const char *cbm_index_resource_name(cbm_index_resource_t resource) { } const char *cbm_index_resource_unit(cbm_index_resource_t resource) { - return resource == CBM_INDEX_RESOURCE_FILES ? "files" : "bytes"; + if (resource == CBM_INDEX_RESOURCE_FILES) { + return "files"; + } + if (resource == CBM_INDEX_RESOURCE_DURATION_MS) { + return "milliseconds"; + } + return "bytes"; } const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { @@ -146,6 +194,10 @@ const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { return CBM_INDEX_CONFIG_MAX_FILES; case CBM_INDEX_RESOURCE_SOURCE_BYTES: return CBM_INDEX_CONFIG_MAX_SOURCE_MB; + case CBM_INDEX_RESOURCE_RSS_BYTES: + return CBM_INDEX_CONFIG_MAX_RSS_MB; + case CBM_INDEX_RESOURCE_DURATION_MS: + return CBM_INDEX_CONFIG_MAX_DURATION_SECONDS; case CBM_INDEX_RESOURCE_NONE: default: return "index_resource_limit"; diff --git a/src/foundation/index_policy.h b/src/foundation/index_policy.h index 6a6c499fc..617a6ab7d 100644 --- a/src/foundation/index_policy.h +++ b/src/foundation/index_policy.h @@ -7,9 +7,14 @@ #define CBM_INDEX_CONFIG_MAX_FILES "index_max_files" #define CBM_INDEX_CONFIG_MAX_SOURCE_MB "index_max_source_mb" +#define CBM_INDEX_CONFIG_MAX_RSS_MB "index_max_rss_mb" +#define CBM_INDEX_CONFIG_MAX_DURATION_SECONDS "index_max_duration_seconds" #define CBM_INDEX_MAX_FILES_VALUE UINT64_C(10000000) #define CBM_INDEX_MAX_SOURCE_MB_VALUE UINT64_C(1048576) +#define CBM_INDEX_MIN_RSS_MB_VALUE UINT64_C(64) +#define CBM_INDEX_MAX_RSS_MB_VALUE UINT64_C(1048576) +#define CBM_INDEX_MAX_DURATION_SECONDS_VALUE UINT64_C(86400) #define CBM_INDEX_MIB_BYTES UINT64_C(1048576) typedef struct { @@ -20,12 +25,16 @@ typedef struct { typedef struct { cbm_index_limit_u64_t max_files; cbm_index_limit_u64_t max_source_bytes; + cbm_index_limit_u64_t max_rss_bytes; + cbm_index_limit_u64_t max_duration_ms; } cbm_index_resource_policy_t; typedef enum { CBM_INDEX_RESOURCE_NONE = 0, CBM_INDEX_RESOURCE_FILES, CBM_INDEX_RESOURCE_SOURCE_BYTES, + CBM_INDEX_RESOURCE_RSS_BYTES, + CBM_INDEX_RESOURCE_DURATION_MS, } cbm_index_resource_t; typedef struct { @@ -36,6 +45,8 @@ typedef struct { void cbm_index_policy_init(cbm_index_resource_policy_t *policy); bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy); size_t cbm_index_policy_key_count(void); const char *cbm_index_policy_key_at(size_t index); diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index 9174391f4..f6be6b93d 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -11,6 +11,7 @@ #include "platform.h" /* cbm_now_ms */ #include "sanitized.h" /* CBM_SANITIZED — spawn-retry budget */ +#include #include #include #include @@ -18,14 +19,17 @@ #ifdef _WIN32 #include +#include #include "win_utf8.h" /* cbm_utf8_to_wide — spawn the worker with a wide command line so a * non-ASCII repo path survives CreateProcess (#423/#20) */ #include /* free */ #else +#include #include #include #include #ifdef __APPLE__ +#include #include extern char **environ; #endif @@ -436,6 +440,287 @@ struct cbm_subprocess { #endif }; +static void cbm_rss_add_saturated(uint64_t *total, uint64_t value) { + if (value > UINT64_MAX - *total) { + *total = UINT64_MAX; + } else { + *total += value; + } +} + +#ifdef CBM_ENABLE_TEST_SEAMS +uint64_t cbm_subprocess_rss_sum_for_testing(const uint64_t *values, size_t count) { + uint64_t total = 0; + for (size_t index = 0; values && index < count; index++) { + cbm_rss_add_saturated(&total, values[index]); + } + return total; +} +#endif + +#ifdef _WIN32 +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + DWORD capacity = 32; + JOBOBJECT_BASIC_PROCESS_ID_LIST *processes = NULL; + for (;;) { + size_t bytes = sizeof(*processes) + (size_t)(capacity - 1) * sizeof(ULONG_PTR); + if (bytes > UINT32_MAX) { + free(processes); + return CBM_PROC_TREE_RSS_ERROR; + } + JOBOBJECT_BASIC_PROCESS_ID_LIST *grown = realloc(processes, bytes); + if (!grown) { + free(processes); + return CBM_PROC_TREE_RSS_ERROR; + } + processes = grown; + ZeroMemory(processes, bytes); + processes->NumberOfAssignedProcesses = capacity; + if (QueryInformationJobObject(process->job, JobObjectBasicProcessIdList, processes, + (DWORD)bytes, NULL)) { + break; + } + if (GetLastError() != ERROR_MORE_DATA || capacity > (UINT32_MAX / 2U)) { + free(processes); + return CBM_PROC_TREE_RSS_ERROR; + } + capacity *= 2U; + } + + uint64_t total = 0; + DWORD measured = 0; + bool root_failed = false; + for (DWORD index = 0; index < processes->NumberOfProcessIdsInList; index++) { + DWORD pid = (DWORD)processes->ProcessIdList[index]; + HANDLE member = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid); + if (!member) { + root_failed = root_failed || pid == process->process_id; + continue; + } + PROCESS_MEMORY_COUNTERS memory; + ZeroMemory(&memory, sizeof(memory)); + if (GetProcessMemoryInfo(member, &memory, sizeof(memory))) { + cbm_rss_add_saturated(&total, (uint64_t)memory.WorkingSetSize); + measured++; + } else if (pid == process->process_id) { + root_failed = true; + } + CloseHandle(member); + } + DWORD listed = processes->NumberOfProcessIdsInList; + free(processes); + if (root_failed || (measured == 0 && listed > 0)) { + return CBM_PROC_TREE_RSS_ERROR; + } + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#elif defined(__APPLE__) +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + int capacity = proc_listallpids(NULL, 0); + if (capacity <= 0 || capacity > INT_MAX / (int)sizeof(pid_t) - 64) { + return CBM_PROC_TREE_RSS_ERROR; + } + capacity += 64; + pid_t *pids = NULL; + int count = 0; + for (;;) { + pid_t *grown = realloc(pids, (size_t)capacity * sizeof(*pids)); + if (!grown) { + free(pids); + return CBM_PROC_TREE_RSS_ERROR; + } + pids = grown; + count = proc_listallpids(pids, capacity * (int)sizeof(*pids)); + if (count <= 0) { + free(pids); + return CBM_PROC_TREE_RSS_ERROR; + } + if (count < capacity) { + break; + } + if (capacity > INT_MAX / (int)sizeof(*pids) / 2) { + free(pids); + return CBM_PROC_TREE_RSS_ERROR; + } + capacity *= 2; + } + + uint64_t total = 0; + int measured = 0; + bool root_failed = false; + for (int index = 0; index < count; index++) { + struct proc_bsdinfo info; + if (pids[index] <= 0) { + continue; + } + if (proc_pidinfo(pids[index], PROC_PIDTBSDINFO, 0, &info, sizeof(info)) != + (int)sizeof(info)) { + root_failed = root_failed || pids[index] == process->pid; + continue; + } + if ((pid_t)info.pbi_pgid != process->pgid) { + continue; + } + struct rusage_info_v2 usage; + if (proc_pid_rusage(pids[index], RUSAGE_INFO_V2, (rusage_info_t *)&usage) != 0) { + root_failed = root_failed || pids[index] == process->pid; + continue; + } + cbm_rss_add_saturated(&total, usage.ri_resident_size); + measured++; + } + free(pids); + if (root_failed) { + return CBM_PROC_TREE_RSS_ERROR; + } + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#else +static bool cbm_proc_pid_name(const char *name) { + if (!name || !name[0]) { + return false; + } + for (const unsigned char *cursor = (const unsigned char *)name; *cursor; cursor++) { + if (*cursor < '0' || *cursor > '9') { + return false; + } + } + return true; +} + +static bool cbm_linux_proc_stat(const char *pid_name, pid_t *group, int64_t *rss_pages) { + char path[64]; + int length = snprintf(path, sizeof(path), "/proc/%s/stat", pid_name); + if (length <= 0 || length >= (int)sizeof(path)) { + return false; + } + FILE *stat_file = cbm_fopen(path, "rb"); + if (!stat_file) { + return false; + } + char stat_line[4096]; + bool read = fgets(stat_line, sizeof(stat_line), stat_file) != NULL; + (void)fclose(stat_file); + char *command_end = read ? strrchr(stat_line, ')') : NULL; + if (!command_end || command_end[1] != ' ') { + return false; + } + + char *save = NULL; + char *token = strtok_r(command_end + 2, " ", &save); + int field = 3; + bool have_group = false; + bool have_rss = false; + while (token) { + if (field == 5 || field == 24) { + char *end = NULL; + long long value = strtoll(token, &end, 10); + if (!end || *end != '\0') { + return false; + } + if (field == 5) { + *group = (pid_t)value; + have_group = true; + } else { + *rss_pages = (int64_t)value; + have_rss = true; + break; + } + } + token = strtok_r(NULL, " ", &save); + field++; + } + return have_group && have_rss; +} + +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + DIR *proc = opendir("/proc"); + if (!proc) { + return CBM_PROC_TREE_RSS_ERROR; + } + long page_size = sysconf(_SC_PAGESIZE); + if (page_size <= 0) { + (void)closedir(proc); + return CBM_PROC_TREE_RSS_ERROR; + } + + uint64_t total = 0; + int measured = 0; + bool root_failed = false; + struct dirent *entry; + while ((entry = readdir(proc)) != NULL) { + if (!cbm_proc_pid_name(entry->d_name)) { + continue; + } + pid_t group = 0; + int64_t pages = 0; + bool root_entry = strtol(entry->d_name, NULL, 10) == (long)process->pid; + if (!cbm_linux_proc_stat(entry->d_name, &group, &pages)) { + root_failed = root_failed || root_entry; + continue; + } + if (group != process->pgid || pages < 0) { + continue; + } + uint64_t page_count = (uint64_t)pages; + uint64_t bytes = page_count > UINT64_MAX / (uint64_t)page_size + ? UINT64_MAX + : page_count * (uint64_t)page_size; + cbm_rss_add_saturated(&total, bytes); + measured++; + } + (void)closedir(proc); + if (root_failed) { + return CBM_PROC_TREE_RSS_ERROR; + } + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#endif + +cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_bytes(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + if (!process || !rss_bytes) { + return CBM_PROC_TREE_RSS_ERROR; + } + *rss_bytes = 0; + return cbm_subprocess_tree_rss_platform(process, rss_bytes); +} + +bool cbm_subprocess_root_running(const cbm_subprocess_t *process) { + if (!process || process->root_reaped) { + return false; + } + int lifecycle = atomic_load_explicit(&process->lifecycle, memory_order_acquire); + return lifecycle == CBM_SUBPROCESS_ACTIVE || lifecycle == CBM_SUBPROCESS_CANCEL_REQUESTED; +} + +bool cbm_subprocess_termination_pending(const cbm_subprocess_t *process) { + return process && process->termination_started; +} + +bool cbm_subprocess_supervision_active(const cbm_subprocess_t *process) { + if (!process) { + return false; + } + int lifecycle = atomic_load_explicit(&process->lifecycle, memory_order_acquire); + return lifecycle == CBM_SUBPROCESS_ACTIVE || lifecycle == CBM_SUBPROCESS_CANCEL_REQUESTED; +} + static void cbm_subprocess_result_init(cbm_proc_result_t *result) { result->outcome = CBM_PROC_SPAWN_FAILED; result->exit_code = -1; diff --git a/src/foundation/subprocess.h b/src/foundation/subprocess.h index 62592c592..07998603c 100644 --- a/src/foundation/subprocess.h +++ b/src/foundation/subprocess.h @@ -22,6 +22,7 @@ #include #include /* size_t (cbm_build_win_cmdline) */ +#include /* How a supervised child ended. */ typedef enum { @@ -87,6 +88,12 @@ typedef enum { CBM_PROC_POLL_TERMINAL = 1 } cbm_proc_poll_t; +typedef enum { + CBM_PROC_TREE_RSS_ERROR = -1, + CBM_PROC_TREE_RSS_EMPTY = 0, + CBM_PROC_TREE_RSS_OK = 1, +} cbm_proc_tree_rss_status_t; + /* Spawn opts->bin and return immediately with a supervisor handle. On success, * *out owns the process until a terminal poll followed by destroy. Spawn copies * the option strings/argv it needs after return; log_ud remains caller-owned until @@ -117,6 +124,16 @@ int cbm_subprocess_spawn(const cbm_proc_opts_t *opts, cbm_subprocess_t **out); * tree when cancel_grace_ms elapses. Callers must keep polling to make progress. */ cbm_proc_poll_t cbm_subprocess_poll(cbm_subprocess_t *process, cbm_proc_result_t *out); +/* Read the current resident-set size of the complete contained process tree. + * OK returns an overflow-safe byte total, EMPTY means the owned tree currently + * has no observable members, and ERROR means no trustworthy measurement could + * be obtained. Individual processes that exit during enumeration are ignored. */ +cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_bytes(cbm_subprocess_t *process, + uint64_t *rss_bytes); +bool cbm_subprocess_root_running(const cbm_subprocess_t *process); +bool cbm_subprocess_termination_pending(const cbm_subprocess_t *process); +bool cbm_subprocess_supervision_active(const cbm_subprocess_t *process); + /* Record an explicit cancellation request without waiting. Safe to repeat and * safe to call from a cancellation thread while one owner thread polls. The * owner must stop cancellation producers before destroying the handle. true @@ -183,6 +200,7 @@ bool cbm_build_win_cmd_payload(char *buf, size_t cap, const char *cmd_executable * of hoping a loaded machine reproduces it. Test builds only. */ void cbm_subprocess_force_spawn_eagain_for_testing(int attempts); int cbm_subprocess_pending_spawn_eagain_for_testing(void); +uint64_t cbm_subprocess_rss_sum_for_testing(const uint64_t *values, size_t count); #endif #endif /* CBM_SUBPROCESS_H */ diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index b5257dfc1..b9d0212ef 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -390,8 +390,56 @@ enum { INDEX_WORKER_SYNC_POLL_NS = 10000000, INDEX_WORKER_RELAY_LINES_PER_POLL = 64, INDEX_WORKER_RELAY_BYTES_PER_POLL = 64 * 1024, + INDEX_WORKER_RSS_PROBE_FAILURE_LIMIT = 3, + INDEX_WORKER_RSS_PROBE_INTERVAL_MS = 250, }; +typedef enum { + INDEX_WORKER_TERMINATION_NONE = 0, + INDEX_WORKER_TERMINATION_CANCEL_PENDING, + INDEX_WORKER_TERMINATION_CANCEL, + INDEX_WORKER_TERMINATION_RESOURCE, +} index_worker_termination_reason_t; + +#ifdef CBM_ENABLE_TEST_SEAMS +static cbm_index_supervisor_clock_fn g_resource_clock; +static cbm_index_supervisor_rss_fn g_resource_rss; +static void *g_resource_hook_context; + +void cbm_index_supervisor_set_resource_hooks_for_testing(cbm_index_supervisor_clock_fn clock_fn, + cbm_index_supervisor_rss_fn rss_fn, + void *context) { + g_resource_clock = clock_fn; + g_resource_rss = rss_fn; + g_resource_hook_context = context; +} + +void cbm_index_supervisor_reset_resource_hooks_for_testing(void) { + g_resource_clock = NULL; + g_resource_rss = NULL; + g_resource_hook_context = NULL; +} +#endif + +static uint64_t worker_resource_now_ms(void) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_resource_clock) { + return g_resource_clock(g_resource_hook_context); + } +#endif + return cbm_now_ms(); +} + +static cbm_proc_tree_rss_status_t worker_resource_rss(cbm_subprocess_t *process, + uint64_t *rss_bytes) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_resource_rss) { + return g_resource_rss(process, rss_bytes, g_resource_hook_context); + } +#endif + return cbm_subprocess_tree_rss_bytes(process, rss_bytes); +} + struct cbm_index_worker_handle { cbm_subprocess_t *process; char response_path[INDEX_WORKER_PATH_CAP]; @@ -401,6 +449,12 @@ struct cbm_index_worker_handle { long relay_tail_pos; bool process_terminal; cbm_proc_result_t process_result; + cbm_index_resource_policy_t resource_policy; + uint64_t started_ms; + uint64_t last_rss_probe_ms; + unsigned int rss_probe_failures; + bool rss_probe_started; + atomic_int termination_reason; atomic_bool terminal; cbm_index_worker_result_t result; }; @@ -578,7 +632,8 @@ static bool worker_unique_file(char *out, size_t out_size, const char *kind) { static bool worker_result_succeeded(const cbm_index_worker_result_t *result) { return result && result->outcome == CBM_PROC_CLEAN && !result->cancellation_requested && - result->tree_quiesced && !result->supervision_failed; + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed && result->tree_quiesced && !result->supervision_failed; } static void worker_terminal_log(cbm_index_worker_handle_t *handle) { @@ -594,6 +649,13 @@ static void worker_terminal_log(cbm_index_worker_handle_t *handle) { } else if (handle->result.supervision_failed || !handle->result.tree_quiesced) { cbm_log_error("index.supervisor.containment_failed", "outcome", cbm_proc_outcome_str(handle->result.outcome), "log", handle->log_path); + } else if (handle->result.resource_probe_failed) { + cbm_log_error("index.supervisor.resource_probe_failed", "resource", "rss_bytes", "log", + handle->log_path); + } else if (handle->result.resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { + cbm_log_warn("index.supervisor.resource_limit", "resource", + cbm_index_resource_name(handle->result.resource_violation.resource), "log", + handle->log_path); } else if (handle->result.cancellation_requested) { cbm_log_warn("index.supervisor.worker_cancelled", "outcome", cbm_proc_outcome_str(handle->result.outcome), "log", handle->log_path); @@ -608,10 +670,87 @@ static void worker_terminal_log(cbm_index_worker_handle_t *handle) { } } -int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_bytes, - bool single_thread, const char *marker_file, - const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, cbm_index_worker_handle_t **handle_out) { +static void worker_request_resource_termination(cbm_index_worker_handle_t *handle, + cbm_index_resource_t resource, uint64_t observed, + uint64_t limit, bool probe_failed) { + int expected = INDEX_WORKER_TERMINATION_NONE; + if (!atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + INDEX_WORKER_TERMINATION_RESOURCE, + memory_order_acq_rel, memory_order_acquire)) { + return; + } + handle->result.resource_violation = (cbm_index_resource_violation_t){ + .resource = resource, .observed = observed, .limit = limit}; + handle->result.resource_probe_failed = probe_failed; + if (!cbm_subprocess_request_cancel(handle->process)) { + handle->result.resource_violation = (cbm_index_resource_violation_t){0}; + handle->result.resource_probe_failed = false; + expected = INDEX_WORKER_TERMINATION_RESOURCE; + (void)atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + INDEX_WORKER_TERMINATION_NONE, + memory_order_acq_rel, memory_order_acquire); + } +} + +static bool worker_check_resource_limits(cbm_index_worker_handle_t *handle) { + if (atomic_load_explicit(&handle->termination_reason, memory_order_acquire) != + INDEX_WORKER_TERMINATION_NONE || + cbm_subprocess_termination_pending(handle->process) || + !cbm_subprocess_supervision_active(handle->process)) { + return false; + } + uint64_t now = 0; + bool have_now = false; + bool rss_probe_failed = false; + if (handle->resource_policy.max_rss_bytes.enabled) { + now = worker_resource_now_ms(); + have_now = true; + bool probe_due = !handle->rss_probe_started || now < handle->last_rss_probe_ms || + now - handle->last_rss_probe_ms >= INDEX_WORKER_RSS_PROBE_INTERVAL_MS; + if (!probe_due) { + goto duration_check; + } + handle->rss_probe_started = true; + handle->last_rss_probe_ms = now; + uint64_t rss_bytes = 0; + cbm_proc_tree_rss_status_t status = worker_resource_rss(handle->process, &rss_bytes); + if (status == CBM_PROC_TREE_RSS_OK) { + handle->rss_probe_failures = 0; + if (rss_bytes > handle->resource_policy.max_rss_bytes.value) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_RSS_BYTES, rss_bytes, + handle->resource_policy.max_rss_bytes.value, + false); + return false; + } + } else if (status == CBM_PROC_TREE_RSS_ERROR) { + handle->rss_probe_failures++; + if (handle->rss_probe_failures >= INDEX_WORKER_RSS_PROBE_FAILURE_LIMIT) { + rss_probe_failed = true; + } + } else { + handle->rss_probe_failures = 0; + } + } +duration_check: + if (handle->resource_policy.max_duration_ms.enabled) { + if (!have_now) { + now = worker_resource_now_ms(); + } + uint64_t elapsed = now >= handle->started_ms ? now - handle->started_ms : 0; + if (elapsed > handle->resource_policy.max_duration_ms.value) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_DURATION_MS, elapsed, + handle->resource_policy.max_duration_ms.value, + false); + } + } + return rss_probe_failed; +} + +static int worker_start_internal(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, cbm_index_worker_handle_t **handle_out) { if (handle_out) { *handle_out = NULL; } @@ -643,6 +782,11 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ if (!handle) { return -1; } + cbm_index_policy_init(&handle->resource_policy); + if (resource_policy) { + handle->resource_policy = *resource_policy; + } + atomic_init(&handle->termination_reason, INDEX_WORKER_TERMINATION_NONE); atomic_init(&handle->terminal, false); handle->log_callback = log_callback; handle->log_context = log_context; @@ -703,15 +847,35 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ cbm_log_error("index.supervisor.spawn_failed", "action", "fail_closed"); return -1; } + if (handle->resource_policy.max_duration_ms.enabled) { + handle->started_ms = worker_resource_now_ms(); + } *handle_out = handle; return 0; } +int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_bytes, + bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, + quarantine_file, log_callback, log_context, handle_out); +} + int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out) { - return cbm_index_worker_start_with_log(args_json, memory_budget_bytes, single_thread, - marker_file, quarantine_file, NULL, NULL, handle_out); + return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, + quarantine_file, NULL, NULL, handle_out); +} + +int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, + cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, resource_policy, single_thread, + marker_file, quarantine_file, NULL, NULL, handle_out); } cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, @@ -728,10 +892,16 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, } bool relay_caught_up = true; if (!handle->process_terminal) { + bool rss_probe_failed = worker_check_resource_limits(handle); cbm_proc_result_t process_result; cbm_proc_poll_t state = cbm_subprocess_poll(handle->process, &process_result); relay_caught_up = worker_relay_log(handle); if (state == CBM_PROC_POLL_RUNNING) { + if (rss_probe_failed && cbm_subprocess_root_running(handle->process)) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_RSS_BYTES, 0, + handle->resource_policy.max_rss_bytes.value, + true); + } return CBM_INDEX_WORKER_POLL_RUNNING; } if (state != CBM_PROC_POLL_TERMINAL) { @@ -752,7 +922,12 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, handle->result.outcome = process_result->outcome; handle->result.exit_code = process_result->exit_code; handle->result.term_signal = process_result->term_signal; - handle->result.cancellation_requested = process_result->cancellation_requested; + int termination_reason = + atomic_load_explicit(&handle->termination_reason, memory_order_acquire); + handle->result.cancellation_requested = + termination_reason == INDEX_WORKER_TERMINATION_CANCEL || + (termination_reason == INDEX_WORKER_TERMINATION_CANCEL_PENDING && + process_result->cancellation_requested); handle->result.forced = process_result->forced; handle->result.tree_quiesced = process_result->tree_quiesced; handle->result.supervision_failed = process_result->supervision_failed; @@ -775,8 +950,36 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, } bool cbm_index_worker_request_cancel(cbm_index_worker_handle_t *handle) { - return handle && !atomic_load_explicit(&handle->terminal, memory_order_acquire) && - cbm_subprocess_request_cancel(handle->process); + if (!handle || atomic_load_explicit(&handle->terminal, memory_order_acquire)) { + return false; + } + int expected = INDEX_WORKER_TERMINATION_NONE; + bool claimed = atomic_compare_exchange_strong_explicit( + &handle->termination_reason, &expected, INDEX_WORKER_TERMINATION_CANCEL_PENDING, + memory_order_acq_rel, memory_order_acquire); + if (!claimed) { + if (expected == INDEX_WORKER_TERMINATION_CANCEL) { + return true; + } + if (expected != INDEX_WORKER_TERMINATION_CANCEL_PENDING) { + return false; + } + bool accepted = cbm_subprocess_request_cancel(handle->process); + if (accepted) { + expected = INDEX_WORKER_TERMINATION_CANCEL_PENDING; + (void)atomic_compare_exchange_strong_explicit( + &handle->termination_reason, &expected, INDEX_WORKER_TERMINATION_CANCEL, + memory_order_acq_rel, memory_order_acquire); + } + return accepted; + } + bool accepted = cbm_subprocess_request_cancel(handle->process); + expected = INDEX_WORKER_TERMINATION_CANCEL_PENDING; + (void)atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + accepted ? INDEX_WORKER_TERMINATION_CANCEL + : INDEX_WORKER_TERMINATION_NONE, + memory_order_acq_rel, memory_order_acquire); + return accepted; } const char *cbm_index_worker_response_path(const cbm_index_worker_handle_t *handle) { @@ -801,18 +1004,19 @@ void cbm_index_worker_destroy(cbm_index_worker_handle_t *handle) { free(handle); } -int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_thread, - const char *marker_file, const char *quarantine_file, - cbm_proc_log_cb log_callback, void *log_context, - const atomic_int *cancel_requested, - cbm_index_worker_result_t *result) { +static int worker_spawn_internal(const char *args_json, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, const atomic_int *cancel_requested, + cbm_index_worker_result_t *result) { if (!result) { return -1; } worker_result_init(result); cbm_index_worker_handle_t *handle = NULL; - if (cbm_index_worker_start_with_log(args_json, 0, single_thread, marker_file, quarantine_file, - log_callback, log_context, &handle) != 0) { + if (worker_start_internal(args_json, 0, resource_policy, single_thread, marker_file, + quarantine_file, log_callback, log_context, &handle) != 0) { return -1; } const cbm_index_worker_result_t *cached = NULL; @@ -838,6 +1042,24 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th return 0; } +int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + const atomic_int *cancel_requested, + cbm_index_worker_result_t *result) { + return worker_spawn_internal(args_json, NULL, single_thread, marker_file, quarantine_file, + log_callback, log_context, cancel_requested, result); +} + +int cbm_index_spawn_worker_with_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, + const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result) { + return worker_spawn_internal(args_json, resource_policy, single_thread, marker_file, + quarantine_file, log_callback, log_context, cancel_requested, + result); +} + int cbm_index_spawn_worker_with_log(const char *args_json, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, diff --git a/src/mcp/index_supervisor.h b/src/mcp/index_supervisor.h index 31259df90..d7e39f94d 100644 --- a/src/mcp/index_supervisor.h +++ b/src/mcp/index_supervisor.h @@ -22,8 +22,10 @@ #include #include +#include #include +#include "foundation/index_policy.h" #include "foundation/subprocess.h" /* cbm_proc_outcome_t */ /* Worker-role state, set once from the CLI arg parser (main.c) when this process @@ -138,8 +140,10 @@ typedef struct { bool tree_quiesced; bool supervision_failed; bool response_rejected; /* clean worker exceeded the bounded response protocol */ - char *response; /* worker result only after a contained, uncancelled CLEAN exit; - * borrowed for async polls, caller-owned from the sync wrapper */ + cbm_index_resource_violation_t resource_violation; + bool resource_probe_failed; + char *response; /* worker result only after a contained, uncancelled CLEAN exit; + * borrowed for async polls, caller-owned from the sync wrapper */ } cbm_index_worker_result_t; /* Daemon-owned, nonblocking supervisor for one contained worker process tree. */ @@ -157,6 +161,11 @@ typedef enum { int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out); +int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, + cbm_index_worker_handle_t **handle_out); /* Request-scoped variant used by interactive local CLI calls. The callback is * invoked by the owner thread while it polls the contained worker; log_context @@ -220,7 +229,22 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th cbm_proc_log_cb log_callback, void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result); +int cbm_index_spawn_worker_with_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, + const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result); void cbm_index_worker_result_free(cbm_index_worker_result_t *result); +#ifdef CBM_ENABLE_TEST_SEAMS +typedef uint64_t (*cbm_index_supervisor_clock_fn)(void *context); +typedef cbm_proc_tree_rss_status_t (*cbm_index_supervisor_rss_fn)(cbm_subprocess_t *process, + uint64_t *rss_bytes, + void *context); +void cbm_index_supervisor_set_resource_hooks_for_testing(cbm_index_supervisor_clock_fn clock_fn, + cbm_index_supervisor_rss_fn rss_fn, + void *context); +void cbm_index_supervisor_reset_resource_hooks_for_testing(void); +#endif + #endif /* CBM_INDEX_SUPERVISOR_H */ diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index f842e8f38..e23127c3a 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -7543,6 +7543,8 @@ static bool build_index_success_response(cbm_mcp_server_t *srv, yyjson_mut_doc * return degraded; } +static bool project_db_is_servable(const char *project, const char *db_path); + /* Build the response for a worker that crashed/hung/failed without producing a * result. The crash is already contained (this process survived); we report it * rather than dying. Precise skip-and-continue (quarantine the culprit, index the @@ -7608,6 +7610,56 @@ static char *build_worker_unsafe_terminal_response(const char *args, cbm_proc_ou return response; } +char *cbm_mcp_index_worker_resource_response(const char *args, + const cbm_index_worker_result_t *worker_result) { + char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); + char *name_override = cbm_mcp_get_string_arg(args, "name"); + char *project_name = + cbm_project_name_from_path(name_override && name_override[0] ? name_override : repo_path); + char db_path[CBM_SZ_1K] = {0}; + if (project_name) { + project_db_path(project_name, db_path, sizeof(db_path)); + } + + const cbm_index_resource_violation_t *violation = &worker_result->resource_violation; + const char *config_key = cbm_index_resource_config_key(violation->resource); + char message[CBM_SZ_256]; + (void)snprintf(message, sizeof(message), + worker_result->resource_probe_failed + ? "Worker resource measurement failed for %s" + : "Index worker exceeded %s", + config_key); + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root); + yyjson_mut_obj_add_str(doc, root, "status", "error"); + yyjson_mut_obj_add_str(doc, root, "code", + worker_result->resource_probe_failed ? "resource_probe_failed" + : "resource_limit_exceeded"); + yyjson_mut_obj_add_str(doc, root, "stage", "worker"); + yyjson_mut_obj_add_str(doc, root, "resource", cbm_index_resource_name(violation->resource)); + if (!worker_result->resource_probe_failed) { + yyjson_mut_obj_add_uint(doc, root, "observed", violation->observed); + yyjson_mut_obj_add_uint(doc, root, "limit", violation->limit); + yyjson_mut_obj_add_str(doc, root, "unit", cbm_index_resource_unit(violation->resource)); + } + yyjson_mut_obj_add_bool(doc, root, "retryable", true); + yyjson_mut_obj_add_bool(doc, root, "serving_index_preserved", + project_name && project_db_is_servable(project_name, db_path)); + yyjson_mut_obj_add_strcpy(doc, root, "message", message); + if (repo_path) { + yyjson_mut_obj_add_strcpy(doc, root, "repo_path", repo_path); + } + char *json = yy_doc_to_str(doc); + yyjson_mut_doc_free(doc); + free(project_name); + free(name_override); + free(repo_path); + char *response = cbm_mcp_text_result(json, true); + free(json); + return response; +} + /* Drop the cached store so the next query reopens whatever the worker wrote (each * worker is a fresh process that deletes + recreates the .db). NULL-safe: the * background watcher path (main.c) has no MCP server / cached store — the child @@ -7751,6 +7803,9 @@ cbm_mcp_supervised_result_disposition_t cbm_mcp_supervised_result_disposition( worker_result->supervision_failed) { return CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL; } + if (worker_result->resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { + return CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE; + } if (worker_result->outcome == CBM_PROC_CLEAN) { return worker_result->response ? CBM_MCP_SUPERVISED_RESULT_SUCCESS : CBM_MCP_SUPERVISED_RESULT_FALLBACK; @@ -7758,8 +7813,8 @@ cbm_mcp_supervised_result_disposition_t cbm_mcp_supervised_result_disposition( return CBM_MCP_SUPERVISED_RESULT_CONTAINED_FAILURE; } -static bool index_policy_from_worker_args(const char *args, cbm_index_resource_policy_t *policy, - char *error, size_t error_size) { +bool cbm_mcp_index_policy_from_internal_args(const char *args, cbm_index_resource_policy_t *policy, + char *error, size_t error_size) { yyjson_doc *doc = args ? yyjson_read(args, strlen(args), 0) : NULL; yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; yyjson_val *encoded = @@ -7789,7 +7844,7 @@ static bool index_policy_from_worker_args(const char *args, cbm_index_resource_p static bool load_index_policy(cbm_mcp_server_t *srv, const char *args, cbm_index_resource_policy_t *policy, char *error, size_t error_size) { if (cbm_index_worker_active()) { - return index_policy_from_worker_args(args, policy, error, error_size); + return cbm_mcp_index_policy_from_internal_args(args, policy, error, error_size); } cbm_config_t *owned_config = NULL; cbm_config_t *config = srv ? srv->config : NULL; @@ -7826,13 +7881,14 @@ bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, * - a contained-failure response only if even that cannot produce a clean run. * A physical CBM host never falls back to its in-process pipeline: an initial * start/protocol failure is returned as an explicit error response. */ -static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { +static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, + const cbm_index_resource_policy_t *resource_policy) { invalidate_cached_store(srv); /* First attempt: normal parallel run. */ cbm_index_worker_result_t wr; - int rc = cbm_index_spawn_worker_with_log_cancel( - args, false, NULL, NULL, srv ? srv->index_log_callback : NULL, + int rc = cbm_index_spawn_worker_with_policy_log_cancel( + args, resource_policy, false, NULL, NULL, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wr); cbm_mcp_supervised_result_disposition_t disposition = cbm_mcp_supervised_result_disposition(rc, &wr); @@ -7850,6 +7906,12 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { invalidate_cached_store(srv); return failure; } + if (disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + char *failure = cbm_mcp_index_worker_resource_response(args, &wr); + cbm_index_worker_result_free(&wr); + invalidate_cached_store(srv); + return failure; + } if (disposition == CBM_MCP_SUPERVISED_RESULT_SUCCESS) { /* Clean exit → transfer the worker's response (the common path). */ char *resp = wr.response; /* transfer ownership to caller (may be NULL) */ @@ -7906,8 +7968,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { bool terminal_cancelled = false; for (int i = 0; i < cap; i++) { cbm_index_worker_result_t wr2; - int rc2 = cbm_index_spawn_worker_with_log_cancel( - args, /*single_thread=*/false, marker_path, quarantine_path, + int rc2 = cbm_index_spawn_worker_with_policy_log_cancel( + args, resource_policy, /*single_thread=*/false, marker_path, quarantine_path, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wr2); cbm_mcp_supervised_result_disposition_t recovery_disposition = @@ -7924,6 +7986,11 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { cbm_index_worker_result_free(&wr2); break; } + if (recovery_disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + resp = cbm_mcp_index_worker_resource_response(args, &wr2); + cbm_index_worker_result_free(&wr2); + break; + } if (recovery_disposition == CBM_MCP_SUPERVISED_RESULT_SUCCESS) { resp = wr2.response; /* transfer ownership to caller */ wr2.response = NULL; @@ -7997,8 +8064,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { * so it cannot itself hang. Rare given monotonic progress. */ if (!resp && !unsafe_terminal && quarantined > 0) { cbm_index_worker_result_t wrp; - int rcp = cbm_index_spawn_worker_with_log_cancel( - args, /*single_thread=*/false, NULL, quarantine_path, + int rcp = cbm_index_spawn_worker_with_policy_log_cancel( + args, resource_policy, /*single_thread=*/false, NULL, quarantine_path, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wrp); cbm_mcp_supervised_result_disposition_t partial_disposition = @@ -8014,6 +8081,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { last_outcome = wrp.outcome; unsafe_terminal = true; terminal_cancelled = wrp.cancellation_requested; + } else if (partial_disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + resp = cbm_mcp_index_worker_resource_response(args, &wrp); } cbm_index_worker_result_free(&wrp); } @@ -8057,7 +8126,7 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p if (!args) { return NULL; } - char *resp = index_run_supervised(srv, args); + char *resp = index_run_supervised(srv, args, &policy); free(args); return resp; } @@ -8284,7 +8353,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { free(name_override); return cbm_mcp_text_result("failed to prepare supervised index request", true); } - char *supervised = index_run_supervised(srv, worker_args); + char *supervised = index_run_supervised(srv, worker_args, &resource_policy); free(worker_args); if (supervised) { free(mutation_project); diff --git a/src/mcp/mcp.h b/src/mcp/mcp.h index a70b6372b..320f3dda0 100644 --- a/src/mcp/mcp.h +++ b/src/mcp/mcp.h @@ -221,6 +221,7 @@ char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const ch typedef enum { CBM_MCP_SUPERVISED_RESULT_FALLBACK = 0, CBM_MCP_SUPERVISED_RESULT_SUCCESS, + CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE, CBM_MCP_SUPERVISED_RESULT_CONTAINED_FAILURE, CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL, } cbm_mcp_supervised_result_disposition_t; diff --git a/src/mcp/mcp_internal.h b/src/mcp/mcp_internal.h index 3f5532a0b..a2a6ac5cf 100644 --- a/src/mcp/mcp_internal.h +++ b/src/mcp/mcp_internal.h @@ -32,6 +32,10 @@ bool cbm_mcp_jsonrpc_response_prepend_notice(char **response_io, const char *not * must remove any untrusted field with the same name before invoking this. */ bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, const cbm_index_resource_policy_t *policy); +bool cbm_mcp_index_policy_from_internal_args(const char *args, cbm_index_resource_policy_t *policy, + char *error, size_t error_size); +char *cbm_mcp_index_worker_resource_response(const char *args, + const cbm_index_worker_result_t *worker_result); enum { CBM_MCP_DEFAULT_AUTO_INDEX_LIMIT = 50000 }; diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index e23e7a86e..cad1d9c14 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -430,7 +430,8 @@ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p) { } const cbm_index_resource_policy_t *cbm_pipeline_resource_policy(const cbm_pipeline_t *p) { - return p && cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL; + return p && cbm_index_policy_discovery_enabled(&p->resource_policy) ? &p->resource_policy + : NULL; } cbm_index_resource_violation_t *cbm_pipeline_resource_violation(cbm_pipeline_t *p) { @@ -2210,7 +2211,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { .ignore_file = NULL, .max_file_size = 0, .resource_policy = - cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL, + cbm_index_policy_discovery_enabled(&p->resource_policy) ? &p->resource_policy : NULL, .resource_violation = &p->resource_violation, }; cbm_file_info_t *files = NULL; diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index 30bf0a430..5518c565f 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1324,6 +1324,8 @@ typedef struct { cbm_project_lock_manager_t *project_locks; bool project_lock_busy_is_error; cbm_proc_outcome_t outcomes[APP_FAKE_MAX_ATTEMPTS]; + cbm_index_resource_violation_t resource_violations[APP_FAKE_MAX_ATTEMPTS]; + bool resource_probe_failures[APP_FAKE_MAX_ATTEMPTS]; const char *responses[APP_FAKE_MAX_ATTEMPTS]; const char *marker_payloads[APP_FAKE_MAX_ATTEMPTS]; char marker_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; @@ -1428,7 +1430,9 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem static bool app_wait_for_atomic_bool(atomic_bool *value, bool expected); static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, int attempt, - const char *max_files, const char *max_source_mb) { + const char *max_files, const char *max_source_mb, + const char *max_rss_mb, + const char *max_duration_seconds) { if (!context || attempt < 0 || attempt >= APP_FAKE_MAX_ATTEMPTS) { return false; } @@ -1445,9 +1449,18 @@ static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, in yyjson_val *bytes = policy && yyjson_is_obj(policy) ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB) : NULL; - bool equal = files && bytes && yyjson_is_str(files) && yyjson_is_str(bytes) && + yyjson_val *rss = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_RSS_MB) + : NULL; + yyjson_val *duration = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS) + : NULL; + bool equal = files && bytes && rss && duration && yyjson_is_str(files) && + yyjson_is_str(bytes) && yyjson_is_str(rss) && yyjson_is_str(duration) && strcmp(yyjson_get_str(files), max_files) == 0 && - strcmp(yyjson_get_str(bytes), max_source_mb) == 0; + strcmp(yyjson_get_str(bytes), max_source_mb) == 0 && + strcmp(yyjson_get_str(rss), max_rss_mb) == 0 && + strcmp(yyjson_get_str(duration), max_duration_seconds) == 0; yyjson_doc_free(document); return equal; } @@ -1489,6 +1502,12 @@ static cbm_index_worker_poll_t app_fake_worker_poll(void *opaque, worker->result.outcome = outcome; worker->result.exit_code = outcome == CBM_PROC_CLEAN ? 0 : -1; worker->result.tree_quiesced = true; + if (worker->attempt < APP_FAKE_MAX_ATTEMPTS) { + worker->result.resource_violation = + worker->context->resource_violations[worker->attempt]; + worker->result.resource_probe_failed = + worker->context->resource_probe_failures[worker->attempt]; + } const char *response = worker->attempt < APP_FAKE_MAX_ATTEMPTS && worker->context->responses[worker->attempt] ? worker->context->responses[worker->attempt] @@ -2003,11 +2022,13 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { bool dirs_ok = cbm_mkdtemp(root) != NULL && cbm_mkdtemp(cache) != NULL; bool cache_set = dirs_ok && cache_saved && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0; cbm_config_t *stored_config = cache_set ? cbm_config_open(cache) : NULL; - bool config_ready = stored_config && - cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && - cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "3") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "4") == 0; + bool config_ready = + stored_config && cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && + cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "3") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "4") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7") == 0; char canonical_root[APP_TEST_PATH_CAP] = {0}; bool canonical = dirs_ok && cbm_canonical_path(root, canonical_root, sizeof(canonical_root)); char *project = canonical ? cbm_project_name_from_path(canonical_root) : NULL; @@ -2052,7 +2073,8 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { bool first_owned = first_initialized && project && app_wait_for_subscribers(application, project, 1) && app_wait_for_atomic_int(&fake.starts, 1); - bool auto_policy_propagated = first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4"); + bool auto_policy_propagated = + first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4", "64", "7"); bool second_initialized = app_test_initialize_profile(&callbacks, sessions[1], root, CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); bool coalesced = first_owned && second_initialized && project && @@ -2135,9 +2157,11 @@ TEST(daemon_application_programmatic_index_injects_resource_policy) { ASSERT_NOT_NULL(root); ASSERT_NOT_NULL(cache); cbm_config_t *stored_config = cbm_config_open(cache); - bool configured = stored_config && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0; + bool configured = + stored_config && cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7") == 0; app_fake_worker_context_t fake; app_fake_worker_context_init(&fake); @@ -2157,7 +2181,8 @@ TEST(daemon_application_programmatic_index_injects_resource_policy) { cbm_daemon_application_t *application = configured ? cbm_daemon_application_new(&config) : NULL; int index_rc = application ? cbm_daemon_application_index(application, "policy-programmatic", root) : -1; - bool policy_propagated = index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6"); + bool policy_propagated = + index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6", "64", "7"); bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); cbm_daemon_application_free(application); @@ -2170,6 +2195,86 @@ TEST(daemon_application_programmatic_index_injects_resource_policy) { PASS(); } +TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { + char *cache = th_mktempdir("cbm_app_worker_resource_cache"); + cbm_config_t *stored_config = cache ? cbm_config_open(cache) : NULL; + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + atomic_store(&fake.scripted, true); + fake.outcomes[0] = CBM_PROC_KILLED; + fake.resource_violations[0] = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_DURATION_MS, .observed = 7001, .limit = 7000}; + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = { + .config = stored_config, + .worker_ops = &worker_ops, + }; + cbm_daemon_application_t *application = + stored_config ? cbm_daemon_application_new(&config) : NULL; + cbm_daemon_runtime_application_callbacks_t callbacks = + cbm_daemon_application_runtime_callbacks(application); + cbm_daemon_runtime_application_session_t *session = app_test_open(&callbacks, 4188); + char *root = th_mktempdir("cbm_app_worker_resource"); + uint8_t *context = NULL; + uint32_t context_length = 0; + uint8_t *tool = NULL; + uint32_t tool_length = 0; + char args[APP_TEST_PATH_CAP + 96]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"DaemonWorkerResourceFixture\"}", + root ? root : ""); + bool setup = cache && stored_config && application && session && root && + app_test_context_request(root, root, &context, &context_length) && + app_test_tool_request("index_repository", args, &tool, &tool_length); + uint8_t *response = NULL; + uint32_t response_length = 0; + if (setup) { + uint8_t *context_response = NULL; + uint32_t context_response_length = 0; + setup = app_test_request(&callbacks, session, context, context_length, &context_response, + &context_response_length) == CBM_DAEMON_RUNTIME_APPLICATION_OK; + free(context_response); + } + cbm_daemon_runtime_application_status_t status = + setup + ? app_test_request(&callbacks, session, tool, tool_length, &response, &response_length) + : CBM_DAEMON_RUNTIME_APPLICATION_TRANSPORT_ERROR; + bool structured = response && strstr((char *)response, "resource_limit_exceeded") && + strstr((char *)response, "\\\"stage\\\":\\\"worker\\\"") && + strstr((char *)response, "\\\"resource\\\":\\\"duration_ms\\\"") && + strstr((char *)response, "\\\"observed\\\":7001") && + strstr((char *)response, "\\\"limit\\\":7000") && + strstr((char *)response, "\\\"unit\\\":\\\"milliseconds\\\""); + if (session) { + callbacks.session_close(callbacks.context, session); + } + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + int starts = atomic_load(&fake.starts); + int destroys = atomic_load(&fake.destroys); + cbm_daemon_application_free(application); + cbm_config_close(stored_config); + free(response); + free(context); + free(tool); + th_cleanup(root); + th_cleanup(cache); + + ASSERT_TRUE(setup); + ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_OK); + ASSERT_TRUE(structured); + ASSERT_EQ(starts, 1); + ASSERT_EQ(destroys, 1); + ASSERT_TRUE(stopped); + PASS(); +} + TEST(daemon_application_auto_index_honors_tracked_file_limit) { app_env_backup_t cache_environment; bool cache_saved = app_env_backup_capture(&cache_environment, "CBM_CACHE_DIR"); @@ -5179,6 +5284,7 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_prune_clears_logical_watch_for_reregistration); RUN_TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions); RUN_TEST(daemon_application_programmatic_index_injects_resource_policy); + RUN_TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried); RUN_TEST(daemon_application_auto_index_honors_tracked_file_limit); RUN_TEST(daemon_application_auto_index_file_count_handles_literal_metacharacter_path); RUN_TEST(daemon_application_auto_index_file_count_supports_non_git_roots); diff --git a/tests/test_index_policy.c b/tests/test_index_policy.c index d866a0631..b33d64cdb 100644 --- a/tests/test_index_policy.c +++ b/tests/test_index_policy.c @@ -21,9 +21,15 @@ TEST(index_policy_defaults_are_disabled) { ASSERT_FALSE(policy.max_files.enabled); ASSERT_FALSE(policy.max_source_bytes.enabled); + ASSERT_FALSE(policy.max_rss_bytes.enabled); + ASSERT_FALSE(policy.max_duration_ms.enabled); ASSERT_FALSE(cbm_index_policy_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_discovery_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_worker_enabled(&policy)); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_FILES), "off"); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_SOURCE_MB), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_RSS_MB), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_DURATION_SECONDS), "off"); PASS(); } @@ -60,6 +66,38 @@ TEST(index_policy_source_limit_converts_mib_without_overflow) { PASS(); } +TEST(index_policy_worker_limits_validate_and_convert_units) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "64", error, sizeof(error))); + ASSERT_TRUE(cbm_index_policy_enabled(&policy)); + ASSERT_TRUE(cbm_index_policy_worker_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_discovery_enabled(&policy)); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(64) * CBM_INDEX_MIB_BYTES); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "1048576", error, + sizeof(error))); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(1048576) * CBM_INDEX_MIB_BYTES); + cbm_index_resource_policy_t before = policy; + ASSERT_FALSE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "63", error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "1", error, + sizeof(error))); + ASSERT_EQ(policy.max_duration_ms.value, 1000); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "86400", error, + sizeof(error))); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(86400000)); + before = policy; + ASSERT_FALSE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "86401", + error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + PASS(); +} + TEST(index_policy_invalid_value_is_rejected_atomically) { static const char *const invalid[] = {"", "0", "-1", "1MB", "1 ", "+1", "10000001", "18446744073709551616"}; @@ -95,6 +133,16 @@ TEST(index_policy_format_round_trips_public_values) { ASSERT_TRUE( cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, value, sizeof(value))); ASSERT_STR_EQ(value, "7"); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "64", error, sizeof(error))); + ASSERT_TRUE( + cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, value, sizeof(value))); + ASSERT_STR_EQ(value, "64"); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "9", error, + sizeof(error))); + ASSERT_TRUE(cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, value, + sizeof(value))); + ASSERT_STR_EQ(value, "9"); PASS(); } @@ -103,10 +151,18 @@ TEST(index_policy_violation_metadata_is_stable) { ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_SOURCE_BYTES), "source_bytes"); ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_FILES), "files"); ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_SOURCE_BYTES), "bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_RSS_BYTES), "rss_bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_DURATION_MS), "duration_ms"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_RSS_BYTES), "bytes"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_DURATION_MS), "milliseconds"); ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_FILES), CBM_INDEX_CONFIG_MAX_FILES); ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_SOURCE_BYTES), CBM_INDEX_CONFIG_MAX_SOURCE_MB); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_RSS_BYTES), + CBM_INDEX_CONFIG_MAX_RSS_MB); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_DURATION_MS), + CBM_INDEX_CONFIG_MAX_DURATION_SECONDS); PASS(); } @@ -122,9 +178,13 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { ASSERT_FALSE(cbm_index_policy_enabled(&policy)); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "9"), 0); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "3"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7"), 0); ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); ASSERT_EQ(policy.max_files.value, 9); ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(3) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(64) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(7000)); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "corrupt"), 0); ASSERT_FALSE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); @@ -135,16 +195,23 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { PASS(); } -TEST(index_policy_cli_lists_both_operator_keys) { +TEST(index_policy_cli_lists_all_operator_keys) { bool files_found = false; bool bytes_found = false; + bool rss_found = false; + bool duration_found = false; for (size_t index = 0; index < cbm_cli_config_key_count_for_testing(); index++) { const char *key = cbm_cli_config_key_at_for_testing(index); files_found = files_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0); bytes_found = bytes_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); + rss_found = rss_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_RSS_MB) == 0); + duration_found = + duration_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS) == 0); } ASSERT_TRUE(files_found); ASSERT_TRUE(bytes_found); + ASSERT_TRUE(rss_found); + ASSERT_TRUE(duration_found); PASS(); } @@ -391,11 +458,12 @@ SUITE(index_policy) { RUN_TEST(index_policy_defaults_are_disabled); RUN_TEST(index_policy_file_limit_accepts_off_and_exact_range); RUN_TEST(index_policy_source_limit_converts_mib_without_overflow); + RUN_TEST(index_policy_worker_limits_validate_and_convert_units); RUN_TEST(index_policy_invalid_value_is_rejected_atomically); RUN_TEST(index_policy_format_round_trips_public_values); RUN_TEST(index_policy_violation_metadata_is_stable); RUN_TEST(index_policy_config_loads_defaults_values_and_rejects_corruption); - RUN_TEST(index_policy_cli_lists_both_operator_keys); + RUN_TEST(index_policy_cli_lists_all_operator_keys); RUN_TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite); RUN_TEST(index_policy_cli_set_reports_a_failed_write); RUN_TEST(index_policy_worker_rejects_missing_parent_policy); diff --git a/tests/test_index_supervisor.c b/tests/test_index_supervisor.c index c67f61fc4..b061f0dd1 100644 --- a/tests/test_index_supervisor.c +++ b/tests/test_index_supervisor.c @@ -849,6 +849,269 @@ TEST(index_supervisor_killed_worker_log_is_never_empty_and_names_the_run) { PASS(); } +typedef struct { + uint64_t now_ms; + uint64_t rss_values[4]; + int rss_value_count; + int rss_calls; + int clock_calls; + cbm_proc_tree_rss_status_t rss_status; +} index_supervisor_resource_fake_t; + +static uint64_t index_supervisor_fake_clock(void *context) { + index_supervisor_resource_fake_t *fake = context; + fake->clock_calls++; + return fake->now_ms; +} + +static cbm_proc_tree_rss_status_t index_supervisor_fake_rss(cbm_subprocess_t *process, + uint64_t *rss_bytes, void *context) { + (void)process; + index_supervisor_resource_fake_t *fake = context; + int value_index = + fake->rss_calls < fake->rss_value_count ? fake->rss_calls : fake->rss_value_count - 1; + fake->rss_calls++; + if (fake->rss_status == CBM_PROC_TREE_RSS_OK && rss_bytes && value_index >= 0) { + *rss_bytes = fake->rss_values[value_index]; + } + return fake->rss_status; +} + +static cbm_index_resource_policy_t index_supervisor_test_worker_policy(uint64_t rss_bytes, + uint64_t duration_ms) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_rss_bytes = (cbm_index_limit_u64_t){.enabled = rss_bytes > 0, .value = rss_bytes}; + policy.max_duration_ms = + (cbm_index_limit_u64_t){.enabled = duration_ms > 0, .value = duration_ms}; + return policy; +} + +TEST(index_supervisor_disabled_limits_do_not_probe) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {UINT64_MAX}, + .rss_value_count = 1, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"clean\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + cbm_index_resource_t resource = + result ? result->resource_violation.resource : CBM_INDEX_RESOURCE_RSS_BYTES; + bool probe_failed = result && result->resource_probe_failed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(terminal); + ASSERT_EQ(fake.rss_calls, 0); + ASSERT_EQ(fake.clock_calls, 0); + ASSERT_EQ(resource, CBM_INDEX_RESOURCE_NONE); + ASSERT_FALSE(probe_failed); + PASS(); +} + +TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {limit, limit + 1}, + .rss_value_count = 2, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t equal_state = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms += 250; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + const cbm_index_worker_result_t *cached = NULL; + bool cached_terminal = + terminal && cbm_index_worker_poll(handle, &cached) == CBM_INDEX_WORKER_POLL_TERMINAL && + cached == result; + bool limited = terminal && result && + result->resource_violation.resource == CBM_INDEX_RESOURCE_RSS_BYTES && + result->resource_violation.observed == limit + 1 && + result->resource_violation.limit == limit && !result->cancellation_requested && + result->tree_quiesced && !result->supervision_failed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(equal_state, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(limited); + ASSERT_TRUE(cached_terminal); + PASS(); +} + +TEST(index_supervisor_duration_is_total_time_not_quiet_timeout) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_EMPTY, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + fake.now_ms = 1100; + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t equal_state = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms = 1101; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool limited = terminal && result && + result->resource_violation.resource == CBM_INDEX_RESOURCE_DURATION_MS && + result->resource_violation.observed == 1001 && + result->resource_violation.limit == 1000 && result->outcome != CBM_PROC_HANG && + result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(equal_state, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(limited); + PASS(); +} + +TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled) { + const char *saved_timeout = getenv("CBM_INDEX_WORKER_TIMEOUT_S"); + char *saved_timeout_copy = saved_timeout ? cbm_strdup(saved_timeout) : NULL; + (void)cbm_setenv("CBM_INDEX_WORKER_TIMEOUT_S", "1", 1); + cbm_index_supervisor_reset_resource_hooks_for_testing(); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 60000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool remained_hang = terminal && result && result->outcome == CBM_PROC_HANG && + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + index_supervisor_test_restore_env("CBM_INDEX_WORKER_TIMEOUT_S", saved_timeout_copy); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(terminal); + ASSERT_TRUE(remained_hang); + PASS(); +} + +TEST(index_supervisor_cancel_precedes_resource_probe) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {limit + 1}, + .rss_value_count = 1, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 1000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + fake.now_ms = 2000; + bool cancel_accepted = handle && cbm_index_worker_request_cancel(handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool cancelled = terminal && result && result->cancellation_requested && + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed && result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(cancel_accepted); + ASSERT_TRUE(cancelled); + ASSERT_EQ(fake.rss_calls, 0); + PASS(); +} + +TEST(index_supervisor_three_failed_rss_probes_fail_closed) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_ERROR, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t first = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + cbm_index_worker_poll_t throttled = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + int calls_before_interval = fake.rss_calls; + fake.now_ms += 250; + cbm_index_worker_poll_t second = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms += 250; + cbm_index_worker_poll_t third = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool failed_closed = terminal && result && result->resource_probe_failed && + result->resource_violation.resource == CBM_INDEX_RESOURCE_RSS_BYTES && + !result->cancellation_requested && result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(first, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(throttled, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(calls_before_interval, 1); + ASSERT_EQ(second, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(third, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(failed_closed); + ASSERT_EQ(fake.rss_calls, 3); + PASS(); +} + SUITE(index_supervisor) { RUN_TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar); RUN_TEST(index_supervisor_async_jobs_are_isolated_cancellable_and_terminal_cached); @@ -857,4 +1120,10 @@ SUITE(index_supervisor) { RUN_TEST(index_supervisor_drains_terminal_backlog_into_request_progress_callback); RUN_TEST(index_supervisor_oversized_response_is_contained_and_log_is_retained); RUN_TEST(index_supervisor_killed_worker_log_is_never_empty_and_names_the_run); + RUN_TEST(index_supervisor_disabled_limits_do_not_probe); + RUN_TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree); + RUN_TEST(index_supervisor_duration_is_total_time_not_quiet_timeout); + RUN_TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled); + RUN_TEST(index_supervisor_cancel_precedes_resource_probe); + RUN_TEST(index_supervisor_three_failed_rss_probes_fail_closed); } diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 278d33e68..37c0b9374 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -9601,6 +9601,17 @@ TEST(index_supervisor_unsafe_clean_is_never_fallback_or_recovery) { CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL); result.supervision_failed = false; + result.tree_quiesced = true; + result.outcome = CBM_PROC_KILLED; + result.resource_violation.resource = CBM_INDEX_RESOURCE_RSS_BYTES; + ASSERT_EQ(cbm_mcp_supervised_result_disposition(0, &result), + CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE); + result.tree_quiesced = false; + ASSERT_EQ(cbm_mcp_supervised_result_disposition(0, &result), + CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL); + + result.resource_violation = (cbm_index_resource_violation_t){0}; + result.tree_quiesced = true; result.outcome = CBM_PROC_CRASH; result.response = NULL; ASSERT_EQ(cbm_mcp_supervised_result_disposition(0, &result), @@ -9610,6 +9621,102 @@ TEST(index_supervisor_unsafe_clean_is_never_fallback_or_recovery) { PASS(); } +#ifndef _WIN32 +typedef struct { + int calls; +} index_worker_resource_clock_t; + +static uint64_t index_worker_resource_clock(void *context) { + index_worker_resource_clock_t *clock = context; + clock->calls++; + return clock->calls == 1 ? 100 : 1101; +} + +enum { + IDXRESOURCE_OK = 0, + IDXRESOURCE_SETUP = 71, + IDXRESOURCE_NO_RESPONSE = 72, + IDXRESOURCE_BAD_CONTRACT = 73, + IDXRESOURCE_RETRIED = 74, +}; + +static int index_worker_resource_response_check(const char *repo, const char *cache) { + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + cbm_unsetenv("CBM_INDEX_SUPERVISOR"); + cbm_index_supervisor_mark_host(); + cbm_config_t *config = cbm_config_open(cache); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + if (!config || !server || + cbm_config_set(config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "1") != 0) { + cbm_mcp_server_free(server); + cbm_config_close(config); + return IDXRESOURCE_SETUP; + } + cbm_mcp_server_set_config(server, config); + index_worker_resource_clock_t clock = {0}; + cbm_index_supervisor_set_resource_hooks_for_testing(index_worker_resource_clock, NULL, &clock); + int before = cbm_index_supervisor_spawn_count(); + char args[CBM_SZ_2K]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"mode\":\"fast\"," + "\"_cbm_index_policy\":{\"index_max_files\":\"off\"," + "\"index_max_source_mb\":\"off\",\"index_max_rss_mb\":\"off\"," + "\"index_max_duration_seconds\":\"off\"}}", + repo); + char *response = cbm_mcp_handle_tool(server, "index_repository", args); + int after = cbm_index_supervisor_spawn_count(); + cbm_index_supervisor_reset_resource_hooks_for_testing(); + int result = IDXRESOURCE_OK; + if (!response) { + result = IDXRESOURCE_NO_RESPONSE; + } else if (!strstr(response, "resource_limit_exceeded") || + !strstr(response, "\\\"stage\\\":\\\"worker\\\"") || + !strstr(response, "\\\"resource\\\":\\\"duration_ms\\\"") || + !strstr(response, "\\\"observed\\\":1001") || + !strstr(response, "\\\"limit\\\":1000") || + !strstr(response, "\\\"unit\\\":\\\"milliseconds\\\"")) { + result = IDXRESOURCE_BAD_CONTRACT; + } else if (after - before != 1) { + result = IDXRESOURCE_RETRIED; + } + free(response); + cbm_mcp_server_free(server); + cbm_config_close(config); + return result; +} +#endif + +TEST(index_worker_resource_limit_is_trusted_structured_and_not_retried) { +#ifdef _WIN32 + SKIP_PLATFORM("fork-isolated MCP host harness; supervisor state machine is cross-platform"); +#else + char *repo = th_mktempdir("cbm_worker_resource_repo"); + char *cache = th_mktempdir("cbm_worker_resource_cache"); + ASSERT_NOT_NULL(repo); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int main(void) { return 0; }\n"), 0); + fflush(NULL); + pid_t pid = fork(); + if (pid == 0) { + alarm(60); + _exit(index_worker_resource_response_check(repo, cache)); + } + int status = 0; + bool waited = waitpid(pid, &status, 0) == pid; + int exit_code = waited && WIFEXITED(status) ? WEXITSTATUS(status) : -1; + int signal_code = waited && WIFSIGNALED(status) ? WTERMSIG(status) : 0; + th_cleanup(repo); + th_cleanup(cache); + if (exit_code != IDXRESOURCE_OK) { + printf(" worker resource child exit=%d signal=%d\n", exit_code, signal_code); + } + ASSERT_TRUE(waited); + ASSERT_EQ(signal_code, 0); + ASSERT_EQ(exit_code, IDXRESOURCE_OK); + PASS(); +#endif +} + /* Child-side check: index a tiny fixture and verify it ran IN-PROCESS. * Distinct exit codes so the parent can report the exact failure mode. */ enum { @@ -11401,6 +11508,7 @@ SUITE(mcp) { RUN_TEST(index_repository_supervisor_uses_canonical_session_path); RUN_TEST(index_repository_cli_name_override_issue823); RUN_TEST(index_supervisor_unsafe_clean_is_never_fallback_or_recovery); + RUN_TEST(index_worker_resource_limit_is_trusted_structured_and_not_retried); RUN_TEST(index_supervisor_gate_requires_marked_host_issue845); RUN_TEST(index_supervisor_start_failure_is_fail_closed_in_real_host); RUN_TEST(index_bg_paths_route_through_supervisor_issue832); diff --git a/tests/test_subprocess.c b/tests/test_subprocess.c index 30fd1537d..82caff354 100644 --- a/tests/test_subprocess.c +++ b/tests/test_subprocess.c @@ -82,6 +82,12 @@ TEST(subprocess_outcome_str) { PASS(); } +TEST(subprocess_tree_rss_sum_saturates_on_overflow) { + const uint64_t values[] = {UINT64_MAX - 4, 4, 1}; + ASSERT_EQ(cbm_subprocess_rss_sum_for_testing(values, 3), UINT64_MAX); + PASS(); +} + /* ── Layer 2: real spawn/reap (POSIX) ─────────────────────────────────────── */ #ifndef _WIN32 @@ -430,6 +436,45 @@ TEST(subprocess_spawn_returns_while_child_is_running) { #endif } +TEST(subprocess_tree_rss_measures_contained_descendants) { +#ifdef _WIN32 + SKIP_PLATFORM("native Job Object process-tree RSS runs on Windows CI"); +#else + char pid_path[64]; + ASSERT_TRUE(make_tree_pid_path(pid_path)); + cbm_subprocess_t *process = NULL; + ASSERT_EQ(spawn_ignoring_tree(pid_path, 0, 100, &process), 0); + pid_t parent_pid = 0; + pid_t grandchild_pid = 0; + bool ready = wait_for_tree_pids(pid_path, process, &parent_pid, &grandchild_pid, 3000); + uint64_t rss_bytes = 0; + cbm_proc_tree_rss_status_t rss_status = + ready ? cbm_subprocess_tree_rss_bytes(process, &rss_bytes) : CBM_PROC_TREE_RSS_ERROR; + bool cancel_accepted = cbm_subprocess_request_cancel(process); + cbm_proc_result_t result; + bool terminal = poll_until_terminal(process, 5000, &result); + uint64_t final_rss = UINT64_MAX; + cbm_proc_tree_rss_status_t final_rss_status = + terminal ? cbm_subprocess_tree_rss_bytes(process, &final_rss) : CBM_PROC_TREE_RSS_ERROR; + if (terminal) { + cbm_subprocess_destroy(process); + } else { + force_probe_cleanup(parent_pid, grandchild_pid); + } + (void)unlink(pid_path); + + ASSERT_TRUE(ready); + ASSERT_EQ(rss_status, CBM_PROC_TREE_RSS_OK); + ASSERT_TRUE(rss_bytes > 0); + ASSERT_TRUE(cancel_accepted); + ASSERT_TRUE(terminal); + ASSERT_TRUE(result.tree_quiesced); + ASSERT_EQ(final_rss_status, CBM_PROC_TREE_RSS_EMPTY); + ASSERT_EQ(final_rss, 0); + PASS(); +#endif +} + TEST(subprocess_natural_completion_is_cached_across_polls) { #ifdef _WIN32 SKIP_PLATFORM("POSIX /bin/sh completion-cache probe; native Job Object coverage pending"); @@ -1027,6 +1072,7 @@ SUITE(subprocess) { RUN_TEST(subprocess_classify_non_fault_signal_is_killed); RUN_TEST(subprocess_classify_timeout_dominates); RUN_TEST(subprocess_outcome_str); + RUN_TEST(subprocess_tree_rss_sum_saturates_on_overflow); RUN_TEST(subprocess_run_clean); RUN_TEST(subprocess_run_exit_nonzero); RUN_TEST(subprocess_run_resolves_literal_binary_name_from_path); @@ -1037,6 +1083,7 @@ SUITE(subprocess) { RUN_TEST(subprocess_run_spawn_failure); RUN_TEST(subprocess_run_null_bin_rejected); RUN_TEST(subprocess_spawn_returns_while_child_is_running); + RUN_TEST(subprocess_tree_rss_measures_contained_descendants); RUN_TEST(subprocess_natural_completion_is_cached_across_polls); RUN_TEST(subprocess_cancel_is_idempotent_and_kills_ignoring_tree); RUN_TEST(subprocess_quiet_timeout_kills_ignoring_tree);