From 41097a256f9d0cceac660db5f16e4fc80a37d5c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Wed, 19 Aug 2026 14:39:38 +0800 Subject: [PATCH 1/6] feat(index): add opt-in discovery resource limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Indexing accepts whatever a repository contains. A tree carrying a vendored monorepo, a generated dump, or a runaway build directory is discovered in full, and the first sign of trouble is a host under memory pressure with nothing that attributes it to indexing. Add two opt-in limits evaluated during discovery against accepted source files only: index_max_files and index_max_source_mb. Both default to off, so nothing changes until an operator sets one. Crossing a limit fails the whole attempt with a structured resource_limit_exceeded result naming the resource, the observed value and the limit; no partial graph is published, and an existing serving index keeps answering. Limits are read from the CLI-managed _config.db and are not MCP request arguments. A supervised parent replaces any caller-supplied policy before spawning its worker, and the worker rejects a missing or incomplete contract, so the CLI, the daemon and the supervised worker all enforce the same decision. Both keys reach an operator through the existing config get/set/list/reset with no new subcommand. `set` suppresses its own generic message for them because the policy writer names the precise reason -- so that writer speaks on every failure it can return, including a validated value whose write then fails on a database that cannot be written. Exiting non-zero in silence is not an acceptable answer from a CLI. The two shell regressions that hand-roll the supervisor's worker argv carry that contract as well. Without it the worker exits before either guard can observe anything, and the guard would go quietly vacuous. Signed-off-by: 刘冲 --- Makefile.cbm | 2 + README.md | 6 + docs/CONFIGURATION.md | 10 + docs/INDEX_RESOURCE_LIMITS.md | 68 +++++ scripts/test-runtime.sh | 10 + src/cli/cli.c | 56 +++- src/cli/cli.h | 7 + src/daemon/application.c | 35 ++- src/discover/discover.c | 51 +++- src/discover/discover.h | 9 +- src/foundation/index_policy.c | 153 +++++++++++ src/foundation/index_policy.h | 53 ++++ src/mcp/mcp.c | 135 +++++++++- src/mcp/mcp_internal.h | 8 + src/pipeline/pipeline.c | 47 +++- src/pipeline/pipeline.h | 9 + src/pipeline/pipeline_incremental.c | 28 +- src/pipeline/pipeline_internal.h | 4 +- tests/test_daemon_application.c | 84 +++++- tests/test_discover.c | 161 +++++++++++ tests/test_index_policy.c | 403 ++++++++++++++++++++++++++++ tests/test_main.c | 2 + tests/test_pipeline.c | 68 +++++ tests/test_worker_error_response.sh | 2 +- tests/test_worker_watchdog.sh | 2 +- 25 files changed, 1369 insertions(+), 44 deletions(-) create mode 100644 docs/INDEX_RESOURCE_LIMITS.md create mode 100644 src/foundation/index_policy.c create mode 100644 src/foundation/index_policy.h create mode 100644 tests/test_index_policy.c diff --git a/Makefile.cbm b/Makefile.cbm index bf51a5a8c..7368467c5 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -224,6 +224,7 @@ FOUNDATION_SRCS = \ src/foundation/profile.c \ src/foundation/dump_verify.c \ src/foundation/limits.c \ + src/foundation/index_policy.c \ src/foundation/subprocess.c \ src/foundation/sha256.c \ src/foundation/secure_random.c \ @@ -502,6 +503,7 @@ TEST_FOUNDATION_SRCS = \ tests/test_str_intern.c \ tests/test_log.c \ tests/test_str_util.c \ + tests/test_index_policy.c \ tests/test_workspace.c \ tests/test_platform.c \ tests/test_diagnostics.c \ diff --git a/README.md b/README.md index ca39ef418..1aedf6b71 100644 --- a/README.md +++ b/README.md @@ -667,9 +667,15 @@ codebase-memory-mcp config list # show all settings codebase-memory-mcp config set auto_index true # auto-index on session start codebase-memory-mcp config set auto_index_limit 50000 # max files for auto-index codebase-memory-mcp config set auto_watch false # don't register background git watcher (default: true) +codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit +codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit codebase-memory-mcp config reset auto_index # reset to default ``` +The two `index_max_*` settings default to `off`. Exceeding one fails the complete +index attempt rather than publishing a partial graph; an existing serving index +is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). + ### Environment Variables | Variable | Default | Description | diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 0ee8058d1..3200c72c4 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -86,6 +86,16 @@ Current keys: |---|---|---| | `auto_index` | `false` | Automatically index new projects when an MCP session starts. | | `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. | +| `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | +| `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | + +The two `index_max_*` settings are independent and disabled by default. They +apply to explicit indexing, automatic indexing, and watcher re-indexing, but not +to `cross-repo-intelligence`, which does not scan repository source files. +Equality is allowed; exceeding either setting fails the complete index request +and preserves any previously serving database. See +[Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and +error-response details. ## 3. UI Settings diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md new file mode 100644 index 000000000..a3d57e015 --- /dev/null +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -0,0 +1,68 @@ +# Index resource limits + +Index resource limits are optional operator controls for repositories whose +discovery breadth is not known in advance. They are disabled by default so +existing large-repository workloads retain their current behavior. + +## Discovery settings + +| Key | Default | Accepted value | Protects | +|---|---:|---:|---| +| `index_max_files` | `off` | `off` or `1..10000000` | Accepted source-file count | +| `index_max_source_mb` | `off` | `off` or `1..1048576` | Accepted source-file bytes | + +Set or reset them with the normal configuration command: + +```bash +codebase-memory-mcp config set index_max_files 250000 +codebase-memory-mcp config set index_max_source_mb 16384 +codebase-memory-mcp config reset index_max_files +``` + +Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero, +negative values, suffixes, trailing characters, and values outside the stated +ranges are rejected without changing the stored value. + +## Counting and failure semantics + +`index_max_files` counts a file only after it passes directory pruning, ignore +rules, filename and suffix filters, language detection, and the existing +per-file size rule. `index_max_source_mb` sums the filesystem sizes of that same +accepted set. + +Equality is allowed. The first file or byte that makes an observed value greater +than its limit stops discovery. CBM discards the partial file list and does not +publish a partial graph as a complete index. + +For an explicit MCP request the error payload contains: + +```json +{ + "status": "error", + "code": "resource_limit_exceeded", + "stage": "discovery", + "resource": "files", + "observed": 250001, + "limit": 250000, + "unit": "files", + "retryable": true, + "serving_index_preserved": true, + "message": "Index discovery exceeded index_max_files" +} +``` + +The previous database remains available because publication occurs only after a +complete discovery and successful staged build. If no previous database exists, +`serving_index_preserved` is false. + +## Trust and compatibility + +Limits are read from the CLI-managed `_config.db`; they are not MCP request +arguments. A supervised parent replaces any caller-supplied internal policy +before spawning its worker, and the worker rejects a missing or incomplete +parent policy. + +These settings do not replace or increase `auto_index_limit`, change the 512 MiB +single-file cap, alter workspace-root authorization, or affect +`cross-repo-intelligence`. With both settings `off`, discovery follows the +existing unbounded path. diff --git a/scripts/test-runtime.sh b/scripts/test-runtime.sh index cb69cadfb..5d2c0c3ae 100644 --- a/scripts/test-runtime.sh +++ b/scripts/test-runtime.sh @@ -105,6 +105,16 @@ _cbm_test_runtime_daemon() { CBM_CACHE_DIR="$_CBM_TEST_RUNTIME_PRODUCT_CACHE" "$1" daemon "$2" } +# The supervisor resolves one resource policy per index and hands it to the +# worker in argv; a worker that finds no complete policy refuses to start rather +# than index unbounded. A shell test that spawns `cli --index-worker` itself +# stands in for the supervisor and owes the worker the same object. Mirrors +# cbm_mcp_index_policy_add_to_args: every key in cbm_index_policy_key_at, and +# nothing else. +cbm_test_index_worker_policy_json() { + printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"}' +} + cbm_test_runtime_cleanup() { local binary="${1:-}" root="$_CBM_TEST_RUNTIME_CREATED_ROOT" local name="${_CBM_TEST_RUNTIME_CREATED_ROOT##*/}" runtime_entry="" active=0 diff --git a/src/cli/cli.c b/src/cli/cli.c index d981af266..f0159925c 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -6734,6 +6734,25 @@ int cbm_config_delete(cbm_config_t *cfg, const char *key) { return rc; } +bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy, + char *error, size_t error_size) { + if (!cfg || !policy) { + if (error && error_size > 0) { + (void)snprintf(error, error_size, "index resource configuration is unavailable"); + } + return false; + } + cbm_index_policy_init(policy); + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + const char *key = cbm_index_policy_key_at(index); + const char *value = cbm_config_get(cfg, key, cbm_index_policy_default_value(key)); + if (!cbm_index_policy_set(policy, key, value, error, error_size)) { + return false; + } + } + return true; +} + /* ── Config CLI subcommand ────────────────────────────────────── */ /* THE config-key table. list, get, help, and key validation all read this one @@ -6756,6 +6775,8 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_CONFIG_UI_LANG, "auto", "Pin graph UI language: en, zh, or auto"}, {CBM_CONFIG_UI_ENABLED, "false", "Serve the graph UI on a loopback HTTP port"}, {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"}, + {CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"}, + {CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"}, }; /* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing @@ -6781,6 +6802,31 @@ static bool config_key_is_ui(const char *key) { return key && (strcmp(key, CBM_CONFIG_UI_ENABLED) == 0 || strcmp(key, CBM_CONFIG_UI_PORT) == 0); } +static bool config_key_is_index_policy(const char *key) { + return key && (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0 || + strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); +} + +static int config_index_policy_write(cbm_config_t *config, const char *key, const char *value) { + cbm_index_resource_policy_t candidate; + cbm_index_policy_init(&candidate); + char error[CLI_BUF_256]; + if (!cbm_index_policy_set(&candidate, key, value, error, sizeof(error))) { + (void)fprintf(stderr, "error: %s\n", error); + return CLI_ERR; + } + int rc = cbm_config_set(config, key, value); + if (rc != 0) { + /* The caller suppresses its own message for policy keys because this + * helper names the precise reason. That is only true if the helper + * speaks on every failure it can return: a validated value whose write + * then fails -- a locked or read-only _config.db -- used to exit + * non-zero having printed nothing at all. */ + (void)fprintf(stderr, "error: failed to set %s\n", key); + } + return rc; +} + static void config_ui_read(const char *key, char *out, size_t out_sz) { cbm_ui_config_t ui; cbm_ui_config_load(&ui); @@ -6918,10 +6964,16 @@ int cbm_cmd_config(int argc, char **argv) { rc = CLI_TRUE; } } else { - if (cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]) == 0) { + int set_rc = + config_key_is_index_policy(argv[CLI_SKIP_ONE]) + ? config_index_policy_write(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]) + : cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]); + if (set_rc == 0) { printf("%s = %s\n", argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]); } else { - (void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]); + if (!config_key_is_index_policy(argv[CLI_SKIP_ONE])) { + (void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]); + } rc = CLI_TRUE; } } diff --git a/src/cli/cli.h b/src/cli/cli.h index c2121a940..56769c42a 100644 --- a/src/cli/cli.h +++ b/src/cli/cli.h @@ -13,6 +13,8 @@ #include #include +#include "foundation/index_policy.h" + typedef struct cbm_mcp_server cbm_mcp_server_t; /* ── Version ──────────────────────────────────────────────────── */ @@ -417,6 +419,11 @@ int cbm_config_set(cbm_config_t *cfg, const char *key, const char *value); /* Delete a config key. Returns 0 on success. */ int cbm_config_delete(cbm_config_t *cfg, const char *key); +/* Load and validate the operator-controlled discovery policy. Invalid stored + * values fail closed instead of silently disabling a guard. */ +bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy, + char *error, size_t error_size); + /* Well-known config keys */ #define CBM_CONFIG_AUTO_INDEX "auto_index" #define CBM_CONFIG_AUTO_INDEX_LIMIT "auto_index_limit" diff --git a/src/daemon/application.c b/src/daemon/application.c index 7e4a35be7..6cd42504f 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -207,7 +207,8 @@ static bool application_unique_recovery_file(char out[APPLICATION_PATH_CAP], con static bool application_update_reap(cbm_daemon_application_t *application, bool wait, uint32_t timeout_ms); static void *application_job_thread(void *opaque); -static char *application_auto_index_args(const char *root_path); +static char *application_auto_index_args(cbm_daemon_application_t *application, + const char *root_path); static cbm_daemon_application_job_t *application_job_subscribe_locked( cbm_daemon_application_t *application, const char *project_key, const char *root_path, const char *args_json, application_job_subscribe_status_t *status_out); @@ -1399,7 +1400,7 @@ static void application_auto_index_retry_pending_locked(cbm_daemon_application_t application_refresh_watch_locked(session); continue; } - char *args = application_auto_index_args(root_path); + char *args = application_auto_index_args(application, root_path); if (!args) { continue; } @@ -1900,7 +1901,27 @@ static bool application_update_reap(cbm_daemon_application_t *application, bool } } -static char *application_auto_index_args(const char *root_path) { +static bool application_index_args_add_policy(cbm_daemon_application_t *application, + yyjson_mut_doc *document, yyjson_mut_val *root) { + cbm_config_t *owned_config = NULL; + cbm_config_t *config = application ? application->config : NULL; + if (!config) { + owned_config = cbm_config_open(cbm_resolve_cache_dir()); + config = owned_config; + } + cbm_index_resource_policy_t policy; + char error[CBM_SZ_256] = {0}; + bool loaded = cbm_config_load_index_policy(config, &policy, error, sizeof(error)); + cbm_config_close(owned_config); + if (!loaded) { + cbm_log_error("daemon.index.policy", "error", error); + return false; + } + return cbm_mcp_index_policy_add_to_args(document, root, &policy); +} + +static char *application_auto_index_args(cbm_daemon_application_t *application, + const char *root_path) { yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); yyjson_mut_val *root = document ? yyjson_mut_obj(document) : NULL; if (!document || !root) { @@ -1908,7 +1929,8 @@ static char *application_auto_index_args(const char *root_path) { return NULL; } yyjson_mut_doc_set_root(document, root); - char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) + char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) && + application_index_args_add_policy(application, document, root) ? yyjson_mut_write(document, 0, NULL) : NULL; yyjson_mut_doc_free(document); @@ -1951,7 +1973,7 @@ static void application_background_initialize_impl(cbm_daemon_application_sessio files); } bool args_required = auto_index_candidate && within_auto_index_limit; - char *args = args_required ? application_auto_index_args(root_path) : NULL; + char *args = args_required ? application_auto_index_args(application, root_path) : NULL; application_jobs_reap_completed(application); cbm_mutex_lock(&application->mutex); if (application->stopping || application_request_cancelled_locked(session)) { @@ -3351,7 +3373,8 @@ static int application_background_index(cbm_daemon_application_t *application, return -1; } yyjson_mut_doc_set_root(document, root); - bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root); + bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root) && + application_index_args_add_policy(application, document, root); char *default_project = cbm_project_name_from_path(canonical_root); bool custom_project = project_name[0] && (!default_project || strcmp(default_project, project_name) != 0); diff --git a/src/discover/discover.c b/src/discover/discover.c index fe1e4e19d..94cceaeb1 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -13,6 +13,7 @@ #include "foundation/constants.h" #include "foundation/compat_fs.h" +#include "foundation/limits.h" #include "foundation/workspace.h" #include "foundation/platform.h" #ifdef _WIN32 @@ -414,6 +415,10 @@ typedef struct { int capacity; int max_files; uint64_t deadline_ms; + const cbm_index_resource_policy_t *resource_policy; + cbm_index_resource_violation_t *resource_violation; + uint64_t source_files; + uint64_t source_bytes; bool count_only; bool collect_excluded; bool limit_exceeded; @@ -435,6 +440,21 @@ typedef struct { int ignored_total; } file_list_t; +static void file_list_resource_violation(file_list_t *fl, cbm_index_resource_t resource, + uint64_t observed, uint64_t limit) { + if (!fl || fl->limit_exceeded || fl->failed) { + return; + } + fl->limit_exceeded = true; + if (fl->resource_violation) { + *fl->resource_violation = (cbm_index_resource_violation_t){ + .resource = resource, + .observed = observed, + .limit = limit, + }; + } +} + static bool file_list_should_stop(file_list_t *fl) { if (!fl) { return true; @@ -504,6 +524,28 @@ static void fl_add(file_list_t *fl, const char *abs_path, const char *rel_path, fl->limit_exceeded = true; return; } + uint64_t source_size = size > 0 ? (uint64_t)size : 0; + bool resource_accepted = fl->resource_policy && source_size <= (uint64_t)cbm_max_file_bytes(); + if (resource_accepted) { + if (fl->resource_policy->max_files.enabled && + fl->source_files >= fl->resource_policy->max_files.value) { + file_list_resource_violation(fl, CBM_INDEX_RESOURCE_FILES, fl->source_files + 1U, + fl->resource_policy->max_files.value); + return; + } + if (fl->resource_policy->max_source_bytes.enabled) { + uint64_t limit = fl->resource_policy->max_source_bytes.value; + if (source_size > limit || fl->source_bytes > limit - source_size) { + uint64_t observed = source_size > UINT64_MAX - fl->source_bytes + ? UINT64_MAX + : fl->source_bytes + source_size; + file_list_resource_violation(fl, CBM_INDEX_RESOURCE_SOURCE_BYTES, observed, limit); + return; + } + fl->source_bytes += source_size; + } + fl->source_files++; + } if (fl->count_only) { fl->count++; return; @@ -1129,6 +1171,9 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc if (ignored_total_out) { *ignored_total_out = 0; } + if (opts && opts->resource_violation) { + *opts->resource_violation = (cbm_index_resource_violation_t){0}; + } if (!repo_path || !out || !count || (count_only && max_files < 0)) { return CBM_DISCOVER_ERROR; } @@ -1203,6 +1248,8 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc file_list_t fl = { .max_files = count_only ? max_files : -1, .deadline_ms = count_only ? deadline_ms : 0, + .resource_policy = opts ? opts->resource_policy : NULL, + .resource_violation = opts ? opts->resource_violation : NULL, .count_only = count_only, .collect_excluded = !count_only && excluded_out != NULL, .collect_ignored = !count_only && ignored_out != NULL, @@ -1225,11 +1272,11 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc } return fl.limit_exceeded ? CBM_DISCOVER_LIMIT_EXCEEDED : CBM_DISCOVER_OK; } - if (fl.failed) { + if (fl.failed || fl.limit_exceeded) { cbm_discover_free(fl.files, fl.count); cbm_discover_free_excluded(fl.excluded, fl.excluded_count); cbm_discover_free_ignored(fl.ignored, fl.ignored_count); - return CBM_DISCOVER_ERROR; + return fl.limit_exceeded ? CBM_DISCOVER_LIMIT_EXCEEDED : CBM_DISCOVER_ERROR; } *out = fl.files; diff --git a/src/discover/discover.h b/src/discover/discover.h index afb60fa36..59a5ed7a4 100644 --- a/src/discover/discover.h +++ b/src/discover/discover.h @@ -17,6 +17,7 @@ /* Use the existing CBMLanguage enum from extraction layer */ #include "cbm.h" +#include "foundation/index_policy.h" /* ── Language detection ──────────────────────────────────────────── */ @@ -131,9 +132,11 @@ typedef struct { } cbm_file_info_t; typedef struct { - cbm_index_mode_t mode; /* CBM_MODE_FULL or CBM_MODE_FAST */ - const char *ignore_file; /* path to .cbmignore file, or NULL */ - int64_t max_file_size; /* 0 = no limit */ + cbm_index_mode_t mode; /* discovery filtering mode */ + const char *ignore_file; /* .cbmignore path, or NULL */ + int64_t max_file_size; /* 0 = no per-file limit */ + const cbm_index_resource_policy_t *resource_policy; /* NULL = no resource limits */ + cbm_index_resource_violation_t *resource_violation; /* optional exact diagnostic */ } cbm_discover_opts_t; typedef enum { diff --git a/src/foundation/index_policy.c b/src/foundation/index_policy.c new file mode 100644 index 000000000..0d2b986bf --- /dev/null +++ b/src/foundation/index_policy.c @@ -0,0 +1,153 @@ +#include "foundation/index_policy.h" + +#include +#include + +static const char *const INDEX_POLICY_KEYS[] = { + CBM_INDEX_CONFIG_MAX_FILES, + CBM_INDEX_CONFIG_MAX_SOURCE_MB, +}; + +static void set_error(char *error, size_t error_size, const char *key, uint64_t maximum) { + if (error && error_size > 0) { + (void)snprintf(error, error_size, "%s must be off or an integer from 1 to %llu", key, + (unsigned long long)maximum); + } +} + +static bool parse_bounded_uint64(const char *value, uint64_t maximum, uint64_t *parsed) { + if (!value || !value[0] || !parsed) { + return false; + } + uint64_t result = 0; + for (const unsigned char *cursor = (const unsigned char *)value; *cursor; cursor++) { + if (*cursor < '0' || *cursor > '9') { + return false; + } + uint64_t digit = (uint64_t)(*cursor - '0'); + if (result > (maximum - digit) / 10U) { + return false; + } + result = result * 10U + digit; + } + if (result == 0) { + return false; + } + *parsed = result; + return true; +} + +void cbm_index_policy_init(cbm_index_resource_policy_t *policy) { + if (policy) { + *policy = (cbm_index_resource_policy_t){0}; + } +} + +bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy) { + return policy && (policy->max_files.enabled || policy->max_source_bytes.enabled); +} + +size_t cbm_index_policy_key_count(void) { + return sizeof(INDEX_POLICY_KEYS) / sizeof(INDEX_POLICY_KEYS[0]); +} + +const char *cbm_index_policy_key_at(size_t index) { + return index < cbm_index_policy_key_count() ? INDEX_POLICY_KEYS[index] : NULL; +} + +const char *cbm_index_policy_default_value(const char *key) { + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if (key && strcmp(key, INDEX_POLICY_KEYS[index]) == 0) { + return "off"; + } + } + return NULL; +} + +bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, const char *value, + char *error, size_t error_size) { + if (error && error_size > 0) { + error[0] = '\0'; + } + if (!policy || !key || !value) { + set_error(error, error_size, key ? key : "index resource limit", 0); + return false; + } + + cbm_index_limit_u64_t *target = NULL; + uint64_t maximum = 0; + uint64_t multiplier = 1; + if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) { + target = &policy->max_files; + maximum = CBM_INDEX_MAX_FILES_VALUE; + } else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) { + target = &policy->max_source_bytes; + maximum = CBM_INDEX_MAX_SOURCE_MB_VALUE; + multiplier = CBM_INDEX_MIB_BYTES; + } else { + set_error(error, error_size, key, 0); + return false; + } + + cbm_index_limit_u64_t candidate = {0}; + if (strcmp(value, "off") != 0) { + uint64_t parsed = 0; + if (!parse_bounded_uint64(value, maximum, &parsed)) { + set_error(error, error_size, key, maximum); + return false; + } + candidate.enabled = true; + candidate.value = parsed * multiplier; + } + *target = candidate; + return true; +} + +bool cbm_index_policy_format(const cbm_index_resource_policy_t *policy, const char *key, char *out, + size_t out_size) { + if (!policy || !key || !out || out_size == 0) { + return false; + } + const cbm_index_limit_u64_t *limit = NULL; + uint64_t divisor = 1; + if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) { + limit = &policy->max_files; + } else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) { + limit = &policy->max_source_bytes; + divisor = CBM_INDEX_MIB_BYTES; + } else { + return false; + } + int length = limit->enabled + ? snprintf(out, out_size, "%llu", (unsigned long long)(limit->value / divisor)) + : snprintf(out, out_size, "off"); + return length >= 0 && (size_t)length < out_size; +} + +const char *cbm_index_resource_name(cbm_index_resource_t resource) { + switch (resource) { + case CBM_INDEX_RESOURCE_FILES: + return "files"; + case CBM_INDEX_RESOURCE_SOURCE_BYTES: + return "source_bytes"; + case CBM_INDEX_RESOURCE_NONE: + default: + return "unknown"; + } +} + +const char *cbm_index_resource_unit(cbm_index_resource_t resource) { + return resource == CBM_INDEX_RESOURCE_FILES ? "files" : "bytes"; +} + +const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { + switch (resource) { + case CBM_INDEX_RESOURCE_FILES: + return CBM_INDEX_CONFIG_MAX_FILES; + case CBM_INDEX_RESOURCE_SOURCE_BYTES: + return CBM_INDEX_CONFIG_MAX_SOURCE_MB; + case CBM_INDEX_RESOURCE_NONE: + default: + return "index_resource_limit"; + } +} diff --git a/src/foundation/index_policy.h b/src/foundation/index_policy.h new file mode 100644 index 000000000..6a6c499fc --- /dev/null +++ b/src/foundation/index_policy.h @@ -0,0 +1,53 @@ +#ifndef CBM_INDEX_POLICY_H +#define CBM_INDEX_POLICY_H + +#include +#include +#include + +#define CBM_INDEX_CONFIG_MAX_FILES "index_max_files" +#define CBM_INDEX_CONFIG_MAX_SOURCE_MB "index_max_source_mb" + +#define CBM_INDEX_MAX_FILES_VALUE UINT64_C(10000000) +#define CBM_INDEX_MAX_SOURCE_MB_VALUE UINT64_C(1048576) +#define CBM_INDEX_MIB_BYTES UINT64_C(1048576) + +typedef struct { + bool enabled; + uint64_t value; +} cbm_index_limit_u64_t; + +typedef struct { + cbm_index_limit_u64_t max_files; + cbm_index_limit_u64_t max_source_bytes; +} cbm_index_resource_policy_t; + +typedef enum { + CBM_INDEX_RESOURCE_NONE = 0, + CBM_INDEX_RESOURCE_FILES, + CBM_INDEX_RESOURCE_SOURCE_BYTES, +} cbm_index_resource_t; + +typedef struct { + cbm_index_resource_t resource; + uint64_t observed; + uint64_t limit; +} cbm_index_resource_violation_t; + +void cbm_index_policy_init(cbm_index_resource_policy_t *policy); +bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy); + +size_t cbm_index_policy_key_count(void); +const char *cbm_index_policy_key_at(size_t index); +const char *cbm_index_policy_default_value(const char *key); + +bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, const char *value, + char *error, size_t error_size); +bool cbm_index_policy_format(const cbm_index_resource_policy_t *policy, const char *key, char *out, + size_t out_size); + +const char *cbm_index_resource_name(cbm_index_resource_t resource); +const char *cbm_index_resource_unit(cbm_index_resource_t resource); +const char *cbm_index_resource_config_key(cbm_index_resource_t resource); + +#endif diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 618d04469..f842e8f38 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -7758,6 +7758,63 @@ cbm_mcp_supervised_result_disposition_t cbm_mcp_supervised_result_disposition( return CBM_MCP_SUPERVISED_RESULT_CONTAINED_FAILURE; } +static bool index_policy_from_worker_args(const char *args, cbm_index_resource_policy_t *policy, + char *error, size_t error_size) { + yyjson_doc *doc = args ? yyjson_read(args, strlen(args), 0) : NULL; + yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; + yyjson_val *encoded = + root && yyjson_is_obj(root) ? yyjson_obj_get(root, "_cbm_index_policy") : NULL; + if (!encoded || !yyjson_is_obj(encoded) || + yyjson_obj_size(encoded) != cbm_index_policy_key_count()) { + yyjson_doc_free(doc); + (void)snprintf(error, error_size, "missing or incomplete trusted worker policy"); + return false; + } + + cbm_index_policy_init(policy); + bool valid = true; + for (size_t index = 0; valid && index < cbm_index_policy_key_count(); index++) { + const char *key = cbm_index_policy_key_at(index); + yyjson_val *value = yyjson_obj_get(encoded, key); + valid = value && yyjson_is_str(value) && + cbm_index_policy_set(policy, key, yyjson_get_str(value), error, error_size); + } + yyjson_doc_free(doc); + if (!valid && error && error_size > 0 && error[0] == '\0') { + (void)snprintf(error, error_size, "invalid trusted worker policy"); + } + return valid; +} + +static bool load_index_policy(cbm_mcp_server_t *srv, const char *args, + cbm_index_resource_policy_t *policy, char *error, size_t error_size) { + if (cbm_index_worker_active()) { + return index_policy_from_worker_args(args, policy, error, error_size); + } + cbm_config_t *owned_config = NULL; + cbm_config_t *config = srv ? srv->config : NULL; + if (!config) { + owned_config = cbm_config_open(cbm_resolve_cache_dir()); + config = owned_config; + } + bool loaded = cbm_config_load_index_policy(config, policy, error, error_size); + cbm_config_close(owned_config); + return loaded; +} + +bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, + const cbm_index_resource_policy_t *policy) { + yyjson_mut_val *encoded = yyjson_mut_obj(doc); + bool valid = encoded != NULL; + for (size_t index = 0; valid && index < cbm_index_policy_key_count(); index++) { + const char *key = cbm_index_policy_key_at(index); + char value[CBM_SZ_64]; + valid = cbm_index_policy_format(policy, key, value, sizeof(value)) && + yyjson_mut_obj_add_strcpy(doc, encoded, key, value); + } + return valid && yyjson_mut_obj_add_val(doc, root, "_cbm_index_policy", encoded); +} + /* Run index_repository in a supervised worker subprocess with skip-and-continue * (Stage 3c). Returns the response string (caller frees): * - the worker's own response on a clean first run (the common path); @@ -7981,10 +8038,20 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p if (!root_path || !root_path[0]) { return NULL; } + cbm_index_resource_policy_t policy; + char policy_error[CBM_SZ_256] = {0}; + if (!load_index_policy(srv, NULL, &policy, policy_error, sizeof(policy_error))) { + cbm_log_error("index.policy", "error", policy_error); + return NULL; + } yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); yyjson_mut_val *root = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root); - yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path); + if (!yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path) || + !cbm_mcp_index_policy_add_to_args(doc, root, &policy)) { + yyjson_mut_doc_free(doc); + return NULL; + } char *args = yy_doc_to_str(doc); yyjson_mut_doc_free(doc); if (!args) { @@ -8033,8 +8100,9 @@ static bool resolve_session_repo_path(cbm_mcp_server_t *srv, char **repo_path) { /* Preserve every index option while replacing all caller-supplied repo_path * keys with the one canonical path that was actually authorized. */ -static char *index_args_with_repo_path(const char *args, const char *canonical_repo_path) { - if (!args || !canonical_repo_path) { +static char *index_args_with_repo_path(const char *args, const char *canonical_repo_path, + const cbm_index_resource_policy_t *policy) { + if (!args || !canonical_repo_path || !policy) { return NULL; } yyjson_doc *source = yyjson_read(args, strlen(args), 0); @@ -8051,8 +8119,14 @@ static char *index_args_with_repo_path(const char *args, const char *canonical_r yyjson_mut_doc_free(copy); return NULL; } - (void)yyjson_mut_obj_remove_key(copy_root, "repo_path"); - if (!yyjson_mut_obj_add_strcpy(copy, copy_root, "repo_path", canonical_repo_path)) { + while (yyjson_mut_obj_get(copy_root, "repo_path")) { + (void)yyjson_mut_obj_remove_key(copy_root, "repo_path"); + } + while (yyjson_mut_obj_get(copy_root, "_cbm_index_policy")) { + (void)yyjson_mut_obj_remove_key(copy_root, "_cbm_index_policy"); + } + if (!yyjson_mut_obj_add_strcpy(copy, copy_root, "repo_path", canonical_repo_path) || + !cbm_mcp_index_policy_add_to_args(copy, copy_root, policy)) { yyjson_mut_doc_free(copy); return NULL; } @@ -8091,6 +8165,19 @@ static char *resolved_repo_path_from_project_arg(const char *args) { return root_path; } +static bool project_db_is_servable(const char *project, const char *db_path) { + cbm_store_t *store = db_path && db_path[0] ? cbm_store_open_path_query(db_path) : NULL; + if (!store) { + return false; + } + cbm_project_t stored_project = {0}; + bool servable = cbm_store_get_project(store, project, &stored_project) == CBM_STORE_OK && + stored_project.root_path && stored_project.root_path[0]; + cbm_project_free_fields(&stored_project); + cbm_store_close(store); + return servable; +} + static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); char *mode_str = cbm_mcp_get_string_arg(args, "mode"); @@ -8145,10 +8232,19 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { return result; } + cbm_index_resource_policy_t resource_policy; + char policy_error[CBM_SZ_256] = {0}; + if (!load_index_policy(srv, args, &resource_policy, policy_error, sizeof(policy_error))) { + free(mode_str); + free(name_override); + free(repo_path); + return cbm_mcp_text_result(policy_error, true); + } + /* A daemon session delegates the one physical write to its shared job * registry only after path canonicalization and workspace authorization. */ if (srv->index_executor) { - char *worker_args = index_args_with_repo_path(args, repo_path); + char *worker_args = index_args_with_repo_path(args, repo_path, &resource_policy); char *coordinated = worker_args ? srv->index_executor(srv->index_executor_context, repo_path, worker_args) : NULL; @@ -8180,7 +8276,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { * installs the same guard before running the in-process pipeline. A marked * host fails closed if preparation or worker startup cannot complete. */ if (cbm_index_supervisor_should_wrap()) { - char *worker_args = index_args_with_repo_path(args, repo_path); + char *worker_args = index_args_with_repo_path(args, repo_path, &resource_policy); if (!worker_args) { free(mutation_project); free(repo_path); @@ -8252,11 +8348,15 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { } free(name_override); cbm_pipeline_set_persistence(p, persistence); + cbm_pipeline_set_resource_policy(p, &resource_policy); char *project_name = heap_strdup(cbm_pipeline_project_name(p)); /* Bootstrap from artifact if no local DB exists */ try_artifact_bootstrap(project_name, repo_path); + char serving_db_path[CBM_SZ_1K]; + project_db_path(project_name, serving_db_path, sizeof(serving_db_path)); + bool serving_index_was_servable = project_db_is_servable(project_name, serving_db_path); /* Close cached store — pipeline will delete + recreate the .db file */ if (srv->owns_store && srv->store) { @@ -8288,6 +8388,8 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { cbm_file_error_t *file_errors = NULL; int file_error_count = 0; cbm_pipeline_get_file_errors(p, &file_errors, &file_error_count); + cbm_index_resource_violation_t resource_violation = {0}; + cbm_pipeline_get_resource_violation(p, &resource_violation); cbm_mem_collect(); /* return mimalloc pages to OS after large indexing */ @@ -8316,6 +8418,25 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { srv, doc, root, project_name, repo_path, persistence, p, excluded_dirs, excluded_count, file_errors, file_error_count, has_logfile ? logfile_path : NULL); yyjson_mut_obj_add_str(doc, root, "status", degraded ? "degraded" : "indexed"); + } else if (rc == CBM_PIPELINE_RESOURCE_LIMIT && + resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { + const char *config_key = cbm_index_resource_config_key(resource_violation.resource); + char message[CBM_SZ_256]; + (void)snprintf(message, sizeof(message), "Index discovery exceeded %s", config_key); + yyjson_mut_obj_add_str(doc, root, "status", "error"); + yyjson_mut_obj_add_str(doc, root, "code", "resource_limit_exceeded"); + yyjson_mut_obj_add_str(doc, root, "stage", "discovery"); + yyjson_mut_obj_add_str(doc, root, "resource", + cbm_index_resource_name(resource_violation.resource)); + yyjson_mut_obj_add_uint(doc, root, "observed", resource_violation.observed); + yyjson_mut_obj_add_uint(doc, root, "limit", resource_violation.limit); + yyjson_mut_obj_add_str(doc, root, "unit", + cbm_index_resource_unit(resource_violation.resource)); + yyjson_mut_obj_add_bool(doc, root, "retryable", true); + yyjson_mut_obj_add_bool(doc, root, "serving_index_preserved", + serving_index_was_servable && + project_db_is_servable(project_name, serving_db_path)); + yyjson_mut_obj_add_strcpy(doc, root, "message", message); } else { yyjson_mut_obj_add_str(doc, root, "status", "error"); yyjson_mut_obj_add_str(doc, root, "hint", diff --git a/src/mcp/mcp_internal.h b/src/mcp/mcp_internal.h index d9321c595..3f5532a0b 100644 --- a/src/mcp/mcp_internal.h +++ b/src/mcp/mcp_internal.h @@ -1,10 +1,13 @@ #ifndef CBM_MCP_INTERNAL_H #define CBM_MCP_INTERNAL_H +#include "foundation/index_policy.h" #include "mcp/mcp.h" #include "pipeline/pipeline.h" /* cbm_changed_hunk_t */ #include "store/store.h" /* cbm_node_t */ +#include + /* White-box fault injection for deterministic cross-platform quarantine * safety tests. This header is internal and is not part of the MCP API. */ typedef bool (*cbm_mcp_quarantine_test_hook_fn)(void *context, const char *step); @@ -25,6 +28,11 @@ bool cbm_mcp_server_release_pristine_memory_store(cbm_mcp_server_t *srv); * On success replaces and frees *response_io; on failure it is unchanged. */ bool cbm_mcp_jsonrpc_response_prepend_notice(char **response_io, const char *notice); +/* Encode the complete trusted policy on an internal worker request. Callers + * must remove any untrusted field with the same name before invoking this. */ +bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, + const cbm_index_resource_policy_t *policy); + enum { CBM_MCP_DEFAULT_AUTO_INDEX_LIMIT = 50000 }; /* Count indexable files with the pipeline's native full-mode discovery policy, diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 65ac75183..e23e7a86e 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -156,6 +156,8 @@ struct cbm_pipeline { atomic_int cancelled_storage; atomic_int *cancelled; bool persistence; /* write .codebase-memory/graph.db.zst after indexing */ + cbm_index_resource_policy_t resource_policy; + cbm_index_resource_violation_t resource_violation; /* Indexing state (set during run) */ cbm_gbuf_t *gbuf; @@ -304,6 +306,20 @@ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled) { } } +void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, + const cbm_index_resource_policy_t *policy) { + if (p && policy) { + p->resource_policy = *policy; + } +} + +void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p, + cbm_index_resource_violation_t *violation) { + if (violation) { + *violation = p ? p->resource_violation : (cbm_index_resource_violation_t){0}; + } +} + bool cbm_pipeline_set_project_name(cbm_pipeline_t *p, const char *name) { if (!p || !name || !name[0]) { return false; @@ -413,6 +429,14 @@ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p) { return p ? p->repo_path : NULL; } +const cbm_index_resource_policy_t *cbm_pipeline_resource_policy(const cbm_pipeline_t *p) { + return p && cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL; +} + +cbm_index_resource_violation_t *cbm_pipeline_resource_violation(cbm_pipeline_t *p) { + return p ? &p->resource_violation : NULL; +} + atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p) { return p ? p->cancelled : NULL; } @@ -1916,15 +1940,17 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas #if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API cbm_pipeline_persist_test_run_before_final_manifest(); #endif - if (cbm_pipeline_build_fresh_semantic_manifest(p->project_name, p->repo_path, p->mode, - &manifest, &manifest_count) != 0) { + int manifest_rc = + cbm_pipeline_build_fresh_semantic_manifest(p, p->project_name, &manifest, &manifest_count); + if (manifest_rc != 0) { cbm_log_error("pipeline.err", "phase", "semantic_manifest"); /* db_path and db_dir are this function's strdups; the success tail and * the publish-failure return release them, and these two aborts must * too -- LSan caught exactly these paths leaking both strings. */ free(db_dir); free(db_path); - return CBM_PIPELINE_ABORT_PRESERVE_DB; + return manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT + : CBM_PIPELINE_ABORT_PRESERVE_DB; } if (!cbm_pipeline_semantic_manifests_equal(baseline_manifest, baseline_count, manifest, manifest_count)) { @@ -2178,10 +2204,14 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { /* Phase 1: Discover files */ CBM_PROF_START(t_discover); + p->resource_violation = (cbm_index_resource_violation_t){0}; cbm_discover_opts_t opts = { .mode = p->requested_mode, .ignore_file = NULL, .max_file_size = 0, + .resource_policy = + cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL, + .resource_violation = &p->resource_violation, }; cbm_file_info_t *files = NULL; int file_count = 0; @@ -2206,7 +2236,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { cbm_log_info("pipeline.discover", "files", itoa_buf(file_count), "elapsed_ms", itoa_buf((int)elapsed_ms(t0))); if (rc != 0 || check_cancel(p)) { - rc = CBM_NOT_FOUND; + rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT : CBM_NOT_FOUND; goto cleanup; } @@ -2214,14 +2244,15 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { * bytes drive exact no-op comparison and are checked against a fresh * rediscovery immediately before any replacement is published. */ rc = mode_promoted - ? cbm_pipeline_build_fresh_semantic_manifest(p->project_name, p->repo_path, p->mode, - &baseline_manifest, &baseline_count) + ? cbm_pipeline_build_fresh_semantic_manifest(p, p->project_name, &baseline_manifest, + &baseline_count) : cbm_pipeline_build_semantic_manifest(p->project_name, p->repo_path, files, file_count, p->excluded_dirs, p->excluded_count, &p->git_ctx, p->userconfig, &baseline_manifest, &baseline_count); if (rc != 0) { - rc = CBM_PIPELINE_ABORT_PRESERVE_DB; + rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT + : CBM_PIPELINE_ABORT_PRESERVE_DB; goto cleanup; } @@ -2265,7 +2296,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { cbm_log_info("pipeline.rediscover", "requested_mode", pipeline_mode_name(p->requested_mode), "effective_mode", pipeline_mode_name(p->mode), "files", itoa_buf(file_count)); if (rc != 0 || check_cancel(p)) { - rc = CBM_NOT_FOUND; + rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT : CBM_NOT_FOUND; goto cleanup; } } diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 5cde0c0f0..c62b847db 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -30,6 +30,8 @@ typedef struct cbm_gbuf cbm_gbuf_t; typedef struct cbm_pipeline cbm_pipeline_t; +#define CBM_PIPELINE_RESOURCE_LIMIT (-5) + /* ── Index mode ─────────────────────────────────────────────────── */ #ifndef CBM_INDEX_MODE_T_DEFINED @@ -53,6 +55,13 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path, cbm * When enabled, the pipeline writes a compressed artifact after indexing. */ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled); +/* Apply a validated discovery resource policy. The value is copied. */ +void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, const cbm_index_resource_policy_t *policy); + +/* Copy the exact discovery violation from the most recent run. */ +void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p, + cbm_index_resource_violation_t *violation); + /* Free a pipeline and all its internal state. NULL-safe. */ void cbm_pipeline_free(cbm_pipeline_t *p); diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index d3c5247f5..d0db3fbd0 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -609,17 +609,20 @@ bool cbm_pipeline_semantic_manifests_equal(const cbm_file_hash_t *left, int left return equal; } -int cbm_pipeline_build_fresh_semantic_manifest(const char *project, const char *repo_path, int mode, +int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *project, cbm_file_hash_t **out, int *out_count) { - if (!project || !repo_path || !out || !out_count) { + const char *repo_path = cbm_pipeline_repo_path(p); + if (!p || !project || !repo_path || !out || !out_count) { return CBM_NOT_FOUND; } *out = NULL; *out_count = 0; cbm_discover_opts_t opts = { - .mode = (cbm_index_mode_t)mode, + .mode = (cbm_index_mode_t)cbm_pipeline_get_mode(p), .ignore_file = NULL, .max_file_size = 0, + .resource_policy = cbm_pipeline_resource_policy(p), + .resource_violation = cbm_pipeline_resource_violation(p), }; cbm_file_info_t *fresh_files = NULL; int fresh_file_count = 0; @@ -2224,13 +2227,13 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char #if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API cbm_pipeline_persist_test_run_before_final_manifest(); #endif - if (cbm_pipeline_build_fresh_semantic_manifest(project, cbm_pipeline_repo_path(p), - cbm_pipeline_get_mode(p), &manifest, - &manifest_count) != 0 || - !cbm_pipeline_semantic_manifests_equal(baseline_manifest, baseline_count, manifest, - manifest_count)) { + int manifest_rc = + cbm_pipeline_build_fresh_semantic_manifest(p, project, &manifest, &manifest_count); + if (manifest_rc != 0 || !cbm_pipeline_semantic_manifests_equal( + baseline_manifest, baseline_count, manifest, manifest_count)) { cbm_log_warn("delta.abort", "reason", "semantic_inputs_changed"); - result = CBM_PIPELINE_ABORT_PRESERVE_DB; + result = manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT + : CBM_PIPELINE_ABORT_PRESERVE_DB; goto out; } @@ -2816,8 +2819,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil #if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API cbm_pipeline_persist_test_run_before_final_manifest(); #endif - int manifest_rc = cbm_pipeline_build_fresh_semantic_manifest( - project, cbm_pipeline_repo_path(p), cbm_pipeline_get_mode(p), &manifest, &manifest_count); + int manifest_rc = + cbm_pipeline_build_fresh_semantic_manifest(p, project, &manifest, &manifest_count); if (manifest_rc != 0 || !cbm_pipeline_semantic_manifests_equal( baseline_manifest, baseline_count, manifest, manifest_count)) { cbm_log_warn("incremental.abort", "reason", "semantic_inputs_changed"); @@ -2827,7 +2830,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil free_mode_skipped(mode_skipped, mode_skipped_count); free(saved_adr); cbm_gbuf_free(existing); - return CBM_PIPELINE_ABORT_PRESERVE_DB; + return manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT + : CBM_PIPELINE_ABORT_PRESERVE_DB; } /* Step 7: atomically publish the complete staged generation. */ diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 754fa4ffa..c311bbd00 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -678,7 +678,7 @@ bool cbm_pipeline_semantic_manifests_equal(const cbm_file_hash_t *left, int left const cbm_file_hash_t *right, int right_count); /* Re-run discovery and hash its exact semantic inputs. Used at the publication * boundary so late additions/deletions cannot escape a frozen file list. */ -int cbm_pipeline_build_fresh_semantic_manifest(const char *project, const char *repo_path, int mode, +int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *project, cbm_file_hash_t **out, int *out_count); /* Compatibility contract persisted in coverage metadata. Increment when a @@ -760,6 +760,8 @@ void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *row /* Pipeline accessors for incremental use */ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p); +const cbm_index_resource_policy_t *cbm_pipeline_resource_policy(const cbm_pipeline_t *p); +cbm_index_resource_violation_t *cbm_pipeline_resource_violation(cbm_pipeline_t *p); atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p); /* Record committed graph size (#334 gate axis) from the incremental path, * which cannot see the opaque cbm_pipeline struct. Call before the dump. */ diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index c639ace2f..30bf0a430 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1308,6 +1308,7 @@ TEST(daemon_application_prune_clears_logical_watch_for_reregistration) { } enum { APP_FAKE_MAX_ATTEMPTS = 16 }; +enum { APP_FAKE_ARGS_CAP = 2048 }; typedef struct { atomic_int starts; @@ -1328,6 +1329,11 @@ typedef struct { char marker_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; char quarantine_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; char quarantine_seen[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; + char args_json[APP_FAKE_MAX_ATTEMPTS][APP_FAKE_ARGS_CAP]; + /* `starts` is incremented before the slot is filled, so waiting on it says + * a worker began, not that its arguments are readable. Each slot publishes + * itself instead, which is the edge a reader on another thread needs. */ + atomic_bool args_captured[APP_FAKE_MAX_ATTEMPTS]; size_t memory_budgets[APP_FAKE_MAX_ATTEMPTS]; } app_fake_worker_context_t; @@ -1392,6 +1398,9 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem atomic_init(&worker->cancelled, false); worker->result.exit_code = -1; if (worker->attempt < APP_FAKE_MAX_ATTEMPTS) { + (void)snprintf(context->args_json[worker->attempt], APP_FAKE_ARGS_CAP, "%s", + args_json ? args_json : ""); + atomic_store(&context->args_captured[worker->attempt], true); context->memory_budgets[worker->attempt] = memory_budget_bytes; if (marker_file) { (void)snprintf(context->marker_paths[worker->attempt], APP_TEST_PATH_CAP, "%s", @@ -1416,6 +1425,33 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem return 0; } +static bool app_wait_for_atomic_bool(atomic_bool *value, bool expected); + +static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, int attempt, + const char *max_files, const char *max_source_mb) { + if (!context || attempt < 0 || attempt >= APP_FAKE_MAX_ATTEMPTS) { + return false; + } + if (!app_wait_for_atomic_bool(&context->args_captured[attempt], true)) { + return false; + } + const char *args = context->args_json[attempt]; + yyjson_doc *document = yyjson_read(args, strlen(args), 0); + yyjson_val *root = document ? yyjson_doc_get_root(document) : NULL; + yyjson_val *policy = + root && yyjson_is_obj(root) ? yyjson_obj_get(root, "_cbm_index_policy") : NULL; + yyjson_val *files = + policy && yyjson_is_obj(policy) ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_FILES) : NULL; + yyjson_val *bytes = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB) + : NULL; + bool equal = files && bytes && yyjson_is_str(files) && yyjson_is_str(bytes) && + strcmp(yyjson_get_str(files), max_files) == 0 && + strcmp(yyjson_get_str(bytes), max_source_mb) == 0; + yyjson_doc_free(document); + return equal; +} + static cbm_index_worker_poll_t app_fake_worker_poll(void *opaque, cbm_daemon_application_worker_t handle, const cbm_index_worker_result_t **result_out) { @@ -1969,7 +2005,9 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { cbm_config_t *stored_config = cache_set ? cbm_config_open(cache) : NULL; bool config_ready = stored_config && cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && - cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0; + cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "3") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "4") == 0; char canonical_root[APP_TEST_PATH_CAP] = {0}; bool canonical = dirs_ok && cbm_canonical_path(root, canonical_root, sizeof(canonical_root)); char *project = canonical ? cbm_project_name_from_path(canonical_root) : NULL; @@ -2014,6 +2052,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { bool first_owned = first_initialized && project && app_wait_for_subscribers(application, project, 1) && app_wait_for_atomic_int(&fake.starts, 1); + bool auto_policy_propagated = first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4"); bool second_initialized = app_test_initialize_profile(&callbacks, sessions[1], root, CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); bool coalesced = first_owned && second_initialized && project && @@ -2075,6 +2114,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { ASSERT_TRUE(restricted_started_nothing); ASSERT_TRUE(first_initialized); ASSERT_TRUE(first_owned); + ASSERT_TRUE(auto_policy_propagated); ASSERT_TRUE(second_initialized); ASSERT_TRUE(coalesced); ASSERT_TRUE(restricted_disconnect_kept_job); @@ -2089,6 +2129,47 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { PASS(); } +TEST(daemon_application_programmatic_index_injects_resource_policy) { + char *root = th_mktempdir("cbm_app_policy_root"); + char *cache = th_mktempdir("cbm_app_policy_cache"); + ASSERT_NOT_NULL(root); + ASSERT_NOT_NULL(cache); + cbm_config_t *stored_config = cbm_config_open(cache); + bool configured = stored_config && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0; + + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + atomic_store(&fake.allow_completion, true); + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = { + .config = stored_config, + .worker_ops = &worker_ops, + }; + cbm_daemon_application_t *application = configured ? cbm_daemon_application_new(&config) : NULL; + int index_rc = + application ? cbm_daemon_application_index(application, "policy-programmatic", root) : -1; + bool policy_propagated = index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6"); + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + + cbm_daemon_application_free(application); + cbm_config_close(stored_config); + th_cleanup(root); + th_cleanup(cache); + ASSERT_TRUE(configured); + ASSERT_TRUE(policy_propagated); + ASSERT_TRUE(stopped); + PASS(); +} + TEST(daemon_application_auto_index_honors_tracked_file_limit) { app_env_backup_t cache_environment; bool cache_saved = app_env_backup_capture(&cache_environment, "CBM_CACHE_DIR"); @@ -5097,6 +5178,7 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_free_releases_live_watch_once); RUN_TEST(daemon_application_prune_clears_logical_watch_for_reregistration); RUN_TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions); + RUN_TEST(daemon_application_programmatic_index_injects_resource_policy); RUN_TEST(daemon_application_auto_index_honors_tracked_file_limit); RUN_TEST(daemon_application_auto_index_file_count_handles_literal_metacharacter_path); RUN_TEST(daemon_application_auto_index_file_count_supports_non_git_roots); diff --git a/tests/test_discover.c b/tests/test_discover.c index 7e0007b11..ac1ada5e9 100644 --- a/tests/test_discover.c +++ b/tests/test_discover.c @@ -475,6 +475,163 @@ TEST(discover_bounded_count_matches_shebang_discovery) { PASS(); } +TEST(discover_resource_policy_off_matches_legacy_discovery) { + char *base = th_mktempdir("cbm_disc_policy_off"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "src/first.c"), "int first;\n"); + th_write_file(TH_PATH(base, "src/second.py"), "second = 2\n"); + th_write_file(TH_PATH(base, "src/ignored.png"), "not source\n"); + + cbm_file_info_t *legacy_files = NULL; + int legacy_count = 0; + cbm_discover_opts_t legacy_opts = {.mode = CBM_MODE_FULL}; + ASSERT_EQ(cbm_discover(base, &legacy_opts, &legacy_files, &legacy_count), CBM_DISCOVER_OK); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + cbm_index_resource_violation_t violation = {0}; + cbm_file_info_t *policy_files = NULL; + int policy_count = 0; + cbm_discover_opts_t policy_opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_discover_status_t status = cbm_discover(base, &policy_opts, &policy_files, &policy_count); + + ASSERT_EQ(status, CBM_DISCOVER_OK); + ASSERT_EQ(policy_count, legacy_count); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_NONE); + for (int index = 0; index < legacy_count; index++) { + ASSERT_STR_EQ(policy_files[index].rel_path, legacy_files[index].rel_path); + ASSERT_EQ(policy_files[index].size, legacy_files[index].size); + } + + cbm_discover_free(legacy_files, legacy_count); + cbm_discover_free(policy_files, policy_count); + th_cleanup(base); + PASS(); +} + +TEST(discover_resource_file_limit_is_exact_and_counts_only_accepted_sources) { + char *base = th_mktempdir("cbm_disc_policy_files"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, ".gitignore"), "ignored.c\n"); + th_write_file(TH_PATH(base, "accepted.c"), "int accepted;\n"); + th_write_file(TH_PATH(base, "ignored.c"), "int ignored;\n"); + th_write_file(TH_PATH(base, "unsupported.png"), "not source\n"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK); + ASSERT_EQ(count, 1); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_NONE); + cbm_discover_free(files, count); + + th_write_file(TH_PATH(base, "second.py"), "second = 2\n"); + files = NULL; + count = 99; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + + th_cleanup(base); + PASS(); +} + +TEST(discover_resource_source_bytes_allows_equality_and_rejects_one_more_byte) { + char *base = th_mktempdir("cbm_disc_policy_bytes"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "exact.c"), "1234567"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_source_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 7}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK); + ASSERT_EQ(count, 1); + cbm_discover_free(files, count); + + th_write_file(TH_PATH(base, "plus.py"), "x"); + files = NULL; + count = 99; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_SOURCE_BYTES); + ASSERT_EQ(violation.observed, 8); + ASSERT_EQ(violation.limit, 7); + + th_cleanup(base); + PASS(); +} + +TEST(discover_resource_file_budget_excludes_existing_oversized_skip) { + char *base = th_mktempdir("cbm_disc_policy_oversized"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "accepted.c"), "x"); + th_write_file(TH_PATH(base, "oversized.py"), "123"); + const char *saved_limit = getenv("CBM_MAX_FILE_BYTES"); + char *saved_limit_copy = saved_limit ? strdup(saved_limit) : NULL; + cbm_setenv("CBM_MAX_FILE_BYTES", "2", 1); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + cbm_discover_status_t initial_status = cbm_discover(base, &opts, &files, &count); + bool oversized_did_not_consume_budget = initial_status == CBM_DISCOVER_OK && count == 2 && + violation.resource == CBM_INDEX_RESOURCE_NONE; + cbm_discover_free(files, count); + + th_write_file(TH_PATH(base, "second.c"), "y"); + files = NULL; + count = 0; + cbm_discover_status_t exceeded_status = cbm_discover(base, &opts, &files, &count); + bool accepted_sources_exceeded = exceeded_status == CBM_DISCOVER_LIMIT_EXCEEDED && + files == NULL && count == 0 && + violation.resource == CBM_INDEX_RESOURCE_FILES && + violation.observed == 2 && violation.limit == 1; + + if (saved_limit_copy) { + cbm_setenv("CBM_MAX_FILE_BYTES", saved_limit_copy, 1); + } else { + cbm_unsetenv("CBM_MAX_FILE_BYTES"); + } + free(saved_limit_copy); + th_cleanup(base); + + ASSERT_TRUE(oversized_did_not_consume_budget); + ASSERT_TRUE(accepted_sources_exceeded); + PASS(); +} + TEST(discover_skips_git_dir) { char *base = th_mktempdir("cbm_disc_git"); ASSERT(base != NULL); @@ -1767,6 +1924,10 @@ SUITE(discover) { RUN_TEST(discover_bounded_count_is_allocation_free_and_limit_exact); RUN_TEST(discover_bounded_count_fails_closed_after_deadline); RUN_TEST(discover_bounded_count_matches_shebang_discovery); + RUN_TEST(discover_resource_policy_off_matches_legacy_discovery); + RUN_TEST(discover_resource_file_limit_is_exact_and_counts_only_accepted_sources); + RUN_TEST(discover_resource_source_bytes_allows_equality_and_rejects_one_more_byte); + RUN_TEST(discover_resource_file_budget_excludes_existing_oversized_skip); RUN_TEST(discover_skips_git_dir); RUN_TEST(discover_with_gitignore); RUN_TEST(discover_with_global_xdg_ignore); diff --git a/tests/test_index_policy.c b/tests/test_index_policy.c new file mode 100644 index 000000000..d866a0631 --- /dev/null +++ b/tests/test_index_policy.c @@ -0,0 +1,403 @@ +#include "test_framework.h" +#include "test_helpers.h" + +#include "cli/cli.h" +#include "foundation/compat.h" +#include "foundation/index_policy.h" +#include "mcp/index_supervisor.h" +#include "mcp/mcp.h" +#include "store/store.h" + +#include +#include +#include +#include +#include +#include + +TEST(index_policy_defaults_are_disabled) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + + ASSERT_FALSE(policy.max_files.enabled); + ASSERT_FALSE(policy.max_source_bytes.enabled); + ASSERT_FALSE(cbm_index_policy_enabled(&policy)); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_FILES), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_SOURCE_MB), "off"); + PASS(); +} + +TEST(index_policy_file_limit_accepts_off_and_exact_range) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "1", error, sizeof(error))); + ASSERT_TRUE(policy.max_files.enabled); + ASSERT_EQ(policy.max_files.value, 1); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "10000000", error, + sizeof(error))); + ASSERT_EQ(policy.max_files.value, 10000000); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "off", error, sizeof(error))); + ASSERT_FALSE(policy.max_files.enabled); + PASS(); +} + +TEST(index_policy_source_limit_converts_mib_without_overflow) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "1", error, sizeof(error))); + ASSERT_TRUE(policy.max_source_bytes.enabled); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(1024) * UINT64_C(1024)); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "1048576", error, + sizeof(error))); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(1048576) * UINT64_C(1024) * UINT64_C(1024)); + PASS(); +} + +TEST(index_policy_invalid_value_is_rejected_atomically) { + static const char *const invalid[] = {"", "0", "-1", "1MB", + "1 ", "+1", "10000001", "18446744073709551616"}; + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "42", error, sizeof(error))); + + for (size_t index = 0; index < sizeof(invalid) / sizeof(invalid[0]); index++) { + cbm_index_resource_policy_t before = policy; + ASSERT_FALSE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, invalid[index], + error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + ASSERT_TRUE(error[0] != '\0'); + } + ASSERT_FALSE(cbm_index_policy_set(&policy, "index_unknown_limit", "1", error, sizeof(error))); + PASS(); +} + +TEST(index_policy_format_round_trips_public_values) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + char value[64]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "321", error, sizeof(error))); + ASSERT_TRUE(cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_FILES, value, sizeof(value))); + ASSERT_STR_EQ(value, "321"); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "7", error, sizeof(error))); + ASSERT_TRUE( + cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, value, sizeof(value))); + ASSERT_STR_EQ(value, "7"); + PASS(); +} + +TEST(index_policy_violation_metadata_is_stable) { + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_FILES), "files"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_SOURCE_BYTES), "source_bytes"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_FILES), "files"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_SOURCE_BYTES), "bytes"); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_FILES), + CBM_INDEX_CONFIG_MAX_FILES); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_SOURCE_BYTES), + CBM_INDEX_CONFIG_MAX_SOURCE_MB); + PASS(); +} + +TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { + char *cache = th_mktempdir("cbm_index_policy_config"); + ASSERT_NOT_NULL(cache); + cbm_config_t *config = cbm_config_open(cache); + ASSERT_NOT_NULL(config); + cbm_index_resource_policy_t policy; + char error[256]; + + ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); + ASSERT_FALSE(cbm_index_policy_enabled(&policy)); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "9"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "3"), 0); + ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); + ASSERT_EQ(policy.max_files.value, 9); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(3) * CBM_INDEX_MIB_BYTES); + + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "corrupt"), 0); + ASSERT_FALSE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); + ASSERT_TRUE(strstr(error, CBM_INDEX_CONFIG_MAX_FILES) != NULL); + + cbm_config_close(config); + th_cleanup(cache); + PASS(); +} + +TEST(index_policy_cli_lists_both_operator_keys) { + bool files_found = false; + bool bytes_found = false; + for (size_t index = 0; index < cbm_cli_config_key_count_for_testing(); index++) { + const char *key = cbm_cli_config_key_at_for_testing(index); + files_found = files_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0); + bytes_found = bytes_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); + } + ASSERT_TRUE(files_found); + ASSERT_TRUE(bytes_found); + PASS(); +} + +TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite) { + char *cache = th_mktempdir("cbm_index_policy_cli"); + ASSERT_NOT_NULL(cache); + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + + char *set_valid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "7"}; + char *set_invalid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "0"}; + char *reset[] = {"reset", CBM_INDEX_CONFIG_MAX_FILES}; + int valid_rc = cbm_cmd_config(3, set_valid); + cbm_config_t *config = cbm_config_open(cache); + const char *stored = + config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "missing") : "missing"; + bool valid_stored = strcmp(stored, "7") == 0; + int invalid_rc = cbm_cmd_config(3, set_invalid); + stored = config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "missing") : "missing"; + bool invalid_preserved = strcmp(stored, "7") == 0; + int reset_rc = cbm_cmd_config(2, reset); + stored = config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "off") : "missing"; + bool reset_to_default = strcmp(stored, "off") == 0; + cbm_config_close(config); + + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); + th_cleanup(cache); + + ASSERT_EQ(valid_rc, 0); + ASSERT_TRUE(valid_stored); + ASSERT_TRUE(invalid_rc != 0); + ASSERT_TRUE(invalid_preserved); + ASSERT_EQ(reset_rc, 0); + ASSERT_TRUE(reset_to_default); + PASS(); +} + +/* Capture stderr across one cbm_cmd_config invocation, mirroring the stdout + * idiom in test_cli.c: tmpfile+dup2+rewind is what works on the MinGW leg. */ +static int index_policy_config_stderr(int argc, char **argv, char *out, size_t cap) { + out[0] = '\0'; + FILE *capture = tmpfile(); + int saved = capture ? dup(fileno(stderr)) : -1; + if (!capture || saved < 0) { + if (capture) { + (void)fclose(capture); + } + if (saved >= 0) { + (void)close(saved); + } + return -1000; + } + (void)fflush(stderr); + if (dup2(fileno(capture), fileno(stderr)) < 0) { + (void)fclose(capture); + (void)close(saved); + return -1000; + } + int rc = cbm_cmd_config(argc, argv); + (void)fflush(stderr); + (void)dup2(saved, fileno(stderr)); + (void)close(saved); + rewind(capture); + size_t got = fread(out, 1, cap - 1, capture); + out[got] = '\0'; + (void)fclose(capture); + return rc; +} + +/* `config set` suppresses its own generic message for a policy key, trusting + * the policy writer to name the precise reason. That trust held only for a + * value the writer rejects. A value it accepts whose write then fails — a + * _config.db that cannot be written — exited non-zero having printed nothing + * at all, which is a CLI that failed in silence. + * + * The unwritable database here is a `config` that is a view: CREATE TABLE IF + * NOT EXISTS finds the name taken and succeeds, so the command gets past open + * with a healthy handle and fails at the INSERT. That is the exact path with + * no message, and it needs no file permissions, so it behaves the same on + * every leg and under a root CI container. */ +TEST(index_policy_cli_set_reports_a_failed_write) { + char *cache = th_mktempdir("cbm_index_policy_cli_write"); + ASSERT_NOT_NULL(cache); + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + + char db_path[1024]; + (void)snprintf(db_path, sizeof(db_path), "%s/_config.db", cache); + sqlite3 *db = NULL; + bool fixture_ready = + sqlite3_open(db_path, &db) == SQLITE_OK && + sqlite3_exec(db, "CREATE VIEW config(key, value) AS SELECT 'pinned', 'pinned'", NULL, NULL, + NULL) == SQLITE_OK; + if (db) { + (void)sqlite3_close(db); + } + + char captured[1024]; + char *set_valid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "7"}; + int rc = index_policy_config_stderr(3, set_valid, captured, sizeof(captured)); + + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); + th_cleanup(cache); + + ASSERT_TRUE(fixture_ready); + ASSERT_TRUE(rc != 0); + ASSERT_TRUE(strstr(captured, CBM_INDEX_CONFIG_MAX_FILES) != NULL); + PASS(); +} + +TEST(index_policy_worker_rejects_missing_parent_policy) { + char *repo = th_mktempdir("cbm_index_policy_worker"); + ASSERT_NOT_NULL(repo); + char args[1024]; + (void)snprintf(args, sizeof(args), "{\"repo_path\":\"%s\",\"mode\":\"fast\"}", repo); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + + cbm_index_set_worker_role(true, NULL); + char *response = server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + cbm_index_set_worker_role(false, NULL); + bool rejected = response && strstr(response, "missing or incomplete trusted worker policy"); + + free(response); + cbm_mcp_server_free(server); + th_cleanup(repo); + ASSERT_TRUE(rejected); + PASS(); +} + +TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index) { + char *repo = th_mktempdir("cbm_index_policy_mcp_repo"); + char *cache = th_mktempdir("cbm_index_policy_mcp_cache"); + ASSERT_NOT_NULL(repo); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first(void) { return 1; }\n"), 0); + + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + cbm_config_t *config = cbm_config_open(cache); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + if (server && config) { + cbm_mcp_server_set_config(server, config); + } + + char args[2048]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\"," + "\"mode\":\"fast\"}", + repo); + char *first_response = + server && config ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + bool first_indexed = first_response && strstr(first_response, "\\\"status\\\":\\\"indexed\\\""); + free(first_response); + + char db_path[2048]; + (void)snprintf(db_path, sizeof(db_path), "%s/ResourcePolicyFixture.db", cache); + cbm_store_t *before_store = cbm_store_open_path_query(db_path); + cbm_project_t before_project = {0}; + bool before_read = before_store && cbm_store_get_project(before_store, "ResourcePolicyFixture", + &before_project) == CBM_STORE_OK; + char *indexed_at_before = + before_read && before_project.indexed_at ? strdup(before_project.indexed_at) : NULL; + cbm_project_free_fields(&before_project); + cbm_store_close(before_store); + + bool configured = + config && cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "1") == 0 && + th_write_file(TH_PATH(repo, "second.py"), "def second():\n return 2\n") == 0; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\"," + "\"mode\":\"fast\",\"_cbm_index_policy\":{" + "\"index_max_files\":\"off\",\"index_max_source_mb\":\"off\"}}", + repo); + char *limited_response = + configured && server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + bool contract_ok = limited_response && strstr(limited_response, "resource_limit_exceeded") && + strstr(limited_response, "\\\"stage\\\":\\\"discovery\\\"") && + strstr(limited_response, "\\\"resource\\\":\\\"files\\\"") && + strstr(limited_response, "\\\"observed\\\":2") && + strstr(limited_response, "\\\"limit\\\":1") && + strstr(limited_response, "\\\"unit\\\":\\\"files\\\"") && + strstr(limited_response, "\\\"retryable\\\":true") && + strstr(limited_response, "\\\"serving_index_preserved\\\":true"); + free(limited_response); + + cbm_store_t *after_store = cbm_store_open_path_query(db_path); + cbm_project_t after_project = {0}; + bool after_read = after_store && cbm_store_get_project(after_store, "ResourcePolicyFixture", + &after_project) == CBM_STORE_OK; + bool generation_preserved = indexed_at_before && after_read && after_project.indexed_at && + strcmp(indexed_at_before, after_project.indexed_at) == 0; + cbm_project_free_fields(&after_project); + cbm_store_close(after_store); + free(indexed_at_before); + + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\"," + "\"mode\":\"cross-repo-intelligence\"}", + repo); + char *cross_response = server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + bool cross_repo_unaffected = + cross_response && strstr(cross_response, "resource_limit_exceeded") == NULL; + free(cross_response); + + cbm_mcp_server_free(server); + cbm_config_close(config); + (void)cbm_unlink(db_path); + char sidecar[2100]; + (void)snprintf(sidecar, sizeof(sidecar), "%s-wal", db_path); + (void)cbm_unlink(sidecar); + (void)snprintf(sidecar, sizeof(sidecar), "%s-shm", db_path); + (void)cbm_unlink(sidecar); + th_cleanup(repo); + th_cleanup(cache); + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); + + ASSERT_TRUE(first_indexed); + ASSERT_TRUE(configured); + ASSERT_TRUE(contract_ok); + ASSERT_TRUE(generation_preserved); + ASSERT_TRUE(cross_repo_unaffected); + PASS(); +} + +SUITE(index_policy) { + RUN_TEST(index_policy_defaults_are_disabled); + RUN_TEST(index_policy_file_limit_accepts_off_and_exact_range); + RUN_TEST(index_policy_source_limit_converts_mib_without_overflow); + RUN_TEST(index_policy_invalid_value_is_rejected_atomically); + RUN_TEST(index_policy_format_round_trips_public_values); + RUN_TEST(index_policy_violation_metadata_is_stable); + RUN_TEST(index_policy_config_loads_defaults_values_and_rejects_corruption); + RUN_TEST(index_policy_cli_lists_both_operator_keys); + RUN_TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite); + RUN_TEST(index_policy_cli_set_reports_a_failed_write); + RUN_TEST(index_policy_worker_rejects_missing_parent_policy); + RUN_TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index); +} diff --git a/tests/test_main.c b/tests/test_main.c index ba6f26d45..b38cc912b 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -677,6 +677,7 @@ extern void suite_dyn_array(void); extern void suite_str_intern(void); extern void suite_log(void); extern void suite_str_util(void); +extern void suite_index_policy(void); extern void suite_workspace(void); extern void suite_platform(void); extern void suite_diagnostics(void); @@ -924,6 +925,7 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(str_intern); RUN_SELECTED_SUITE(log); RUN_SELECTED_SUITE(str_util); + RUN_SELECTED_SUITE(index_policy); RUN_SELECTED_SUITE(workspace); RUN_SELECTED_SUITE(platform); RUN_SELECTED_SUITE(diagnostics); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 25878a8b3..5ef810908 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -151,6 +151,34 @@ TEST(pipeline_run_null) { PASS(); } +TEST(pipeline_discovery_limit_returns_exact_violation_without_publishing) { + char *repo = th_mktempdir("cbm_pipeline_discovery_limit"); + ASSERT_NOT_NULL(repo); + ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first;\n"), 0); + ASSERT_EQ(th_write_file(TH_PATH(repo, "second.py"), "second = 2\n"), 0); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/index.db", repo); + cbm_pipeline_t *pipeline = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(pipeline); + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_pipeline_set_resource_policy(pipeline, &policy); + + ASSERT_EQ(cbm_pipeline_run(pipeline), CBM_PIPELINE_RESOURCE_LIMIT); + cbm_index_resource_violation_t violation = {0}; + cbm_pipeline_get_resource_violation(pipeline, &violation); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + ASSERT_FALSE(cbm_file_exists(db_path)); + + cbm_pipeline_free(pipeline); + th_cleanup(repo); + PASS(); +} + /* ── Focused: file-backed store persistence ─────────────────────── */ TEST(store_file_persistence) { @@ -2845,6 +2873,44 @@ static void mutate_semantic_input_before_final_manifest(void *userdata) { mutation->write_rc = th_write_file(mutation->path, mutation->replacement); } +TEST(pipeline_late_source_limit_preserves_previous_generation) { + char *repo = th_mktempdir("cbm_pipeline_late_limit"); + ASSERT_NOT_NULL(repo); + ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first;\n"), 0); + char db_path[512]; + char late_path[512]; + snprintf(db_path, sizeof(db_path), "%s/index.db", repo); + snprintf(late_path, sizeof(late_path), "%s/late.py", repo); + + manifest_race_mutation_t mutation = { + .path = late_path, + .replacement = "late = 2\n", + .write_rc = -1, + }; + cbm_pipeline_incremental_test_before_final_manifest_once( + mutate_semantic_input_before_final_manifest, &mutation); + cbm_pipeline_t *pipeline = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(pipeline); + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_pipeline_set_resource_policy(pipeline, &policy); + int rc = cbm_pipeline_run(pipeline); + cbm_index_resource_violation_t violation = {0}; + cbm_pipeline_get_resource_violation(pipeline, &violation); + cbm_pipeline_free(pipeline); + cbm_pipeline_incremental_test_reset_faults(); + + ASSERT_EQ(mutation.write_rc, 0); + ASSERT_EQ(rc, CBM_PIPELINE_RESOURCE_LIMIT); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + ASSERT_FALSE(cbm_file_exists(db_path)); + th_cleanup(repo); + PASS(); +} + /* A graph and its exact manifest are one generation. If source bytes change * after extraction but before publication, the mixed generation must be * rejected and the previous live DB preserved. */ @@ -11951,6 +12017,8 @@ SUITE(pipeline) { RUN_TEST(pipeline_cancel); RUN_TEST(pipeline_cancel_null); RUN_TEST(pipeline_run_null); + RUN_TEST(pipeline_discovery_limit_returns_exact_violation_without_publishing); + RUN_TEST(pipeline_late_source_limit_preserves_previous_generation); /* Extraction back-pressure */ RUN_TEST(pipeline_backpressure_futile_nap_disengages); /* Sequential cross-LSP shared registry (ms-typescript quadratic) */ diff --git a/tests/test_worker_error_response.sh b/tests/test_worker_error_response.sh index 33e2db115..500c57ff7 100755 --- a/tests/test_worker_error_response.sh +++ b/tests/test_worker_error_response.sh @@ -48,7 +48,7 @@ trap cleanup EXIT missing="${tmpdir}/repository-does-not-exist" response="${tmpdir}/worker.response" -args="{\"repo_path\":\"${missing}\",\"mode\":\"fast\"}" +args="{\"repo_path\":\"${missing}\",\"mode\":\"fast\",$(cbm_test_index_worker_policy_json)}" if ! CBM_CACHE_DIR="${tmpdir}/cache-worker" \ "${BINARY}" cli --index-worker \ diff --git a/tests/test_worker_watchdog.sh b/tests/test_worker_watchdog.sh index 7a52603a0..e38be10bf 100755 --- a/tests/test_worker_watchdog.sh +++ b/tests/test_worker_watchdog.sh @@ -109,7 +109,7 @@ SH chmod +x "${tmpdir}/wrapper.sh" CBM_BINARY="${BINARY}" BUILD_FINGERPRINT="${BUILD_FINGERPRINT}" TMPDIR_PATH="${tmpdir}" \ - ARGS_JSON="{\"repo_path\":\"${tmpdir}/repo\"}" \ + ARGS_JSON="{\"repo_path\":\"${tmpdir}/repo\",$(cbm_test_index_worker_policy_json)}" \ CBM_TEST_HANG_ON=hang_me \ CBM_TEST_WORKER_DESCENDANT_PID_FILE="${tmpdir}/descendant.pid" \ "${tmpdir}/wrapper.sh" & From a1339ec90922542e78fd38cd2b1fde41dc7dabe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Wed, 19 Aug 2026 14:39:38 +0800 Subject: [PATCH 2/6] feat(index): add worker resource watchdogs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discovery limits bound what indexing accepts, not what it then costs. A repository well inside those bounds can still exhaust the host through parser memory, or simply never finish, and a supervised worker that hangs leaves the parent waiting with nothing to report. Add index_max_rss_mb and index_max_duration_seconds, enforced by the parent against the worker process tree rather than the worker process alone, so a runaway child cannot hide behind a small parent. Resident memory is sampled through the platform interface on macOS, Linux and Windows. Crossing a limit terminates the tree and yields one trusted, structured terminal result that attributes the failure to the resource that caused it. Both limits default to off. A measurement that cannot be taken fails the attempt instead of passing it: a watchdog that quietly stops watching is worse than no watchdog at all. The shell fixture that stands in for the supervisor names the two new keys. The worker accepts only a policy that spells out every key it knows, which is what keeps a stale supervisor from starting a worker it cannot bound. Signed-off-by: 刘冲 --- README.md | 4 +- docs/CONFIGURATION.md | 10 +- docs/INDEX_RESOURCE_LIMITS.md | 48 +++++- scripts/test-runtime.sh | 3 +- src/cli/cli.c | 10 +- src/daemon/application.c | 19 ++- src/foundation/index_policy.c | 126 +++++++++----- src/foundation/index_policy.h | 11 ++ src/foundation/subprocess.c | 285 ++++++++++++++++++++++++++++++++ src/foundation/subprocess.h | 18 ++ src/mcp/index_supervisor.c | 256 ++++++++++++++++++++++++++-- src/mcp/index_supervisor.h | 28 +++- src/mcp/mcp.c | 93 +++++++++-- src/mcp/mcp.h | 1 + src/mcp/mcp_internal.h | 4 + src/pipeline/pipeline.c | 5 +- tests/test_daemon_application.c | 132 +++++++++++++-- tests/test_index_policy.c | 72 +++++++- tests/test_index_supervisor.c | 269 ++++++++++++++++++++++++++++++ tests/test_mcp.c | 108 ++++++++++++ tests/test_subprocess.c | 47 ++++++ 21 files changed, 1451 insertions(+), 98 deletions(-) diff --git a/README.md b/README.md index 1aedf6b71..8c249b087 100644 --- a/README.md +++ b/README.md @@ -669,10 +669,12 @@ codebase-memory-mcp config set auto_index_limit 50000 # max files for auto-in codebase-memory-mcp config set auto_watch false # don't register background git watcher (default: true) codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit +codebase-memory-mcp config set index_max_rss_mb 8192 # optional worker-tree current RSS limit +codebase-memory-mcp config set index_max_duration_seconds 3600 # optional total worker duration codebase-memory-mcp config reset auto_index # reset to default ``` -The two `index_max_*` settings default to `off`. Exceeding one fails the complete +The four `index_max_*` settings default to `off`. Exceeding one fails the complete index attempt rather than publishing a partial graph; an existing serving index is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 3200c72c4..c3bdf8d96 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -88,12 +88,16 @@ Current keys: | `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. | | `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | | `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | +| `index_max_rss_mb` | `off` | Optional maximum current RSS in MiB for the complete contained index-worker process tree (`64..1048576`). | +| `index_max_duration_seconds` | `off` | Optional maximum total worker duration in seconds (`1..86400`). | -The two `index_max_*` settings are independent and disabled by default. They +The four `index_max_*` settings are independent and disabled by default. They apply to explicit indexing, automatic indexing, and watcher re-indexing, but not to `cross-repo-intelligence`, which does not scan repository source files. -Equality is allowed; exceeding either setting fails the complete index request -and preserves any previously serving database. See +Equality is allowed; exceeding any setting fails the complete index request and +preserves any previously serving database. Worker RSS covers descendants and is +not the same as the internal `CBM_MEM_BUDGET_MB` allocation budget. Total +duration is independent of the existing 15-minute no-log-progress timeout. See [Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and error-response details. diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index a3d57e015..66be4ecf4 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -1,8 +1,8 @@ # Index resource limits Index resource limits are optional operator controls for repositories whose -discovery breadth is not known in advance. They are disabled by default so -existing large-repository workloads retain their current behavior. +discovery breadth or worker runtime is not known in advance. They are disabled +by default so existing large-repository workloads retain their current behavior. ## Discovery settings @@ -23,6 +23,39 @@ Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero, negative values, suffixes, trailing characters, and values outside the stated ranges are rejected without changing the stored value. +## Worker settings + +| Key | Default | Accepted value | Protects | +|---|---:|---:|---| +| `index_max_rss_mb` | `off` | `off` or `64..1048576` | Current RSS of the complete worker process tree | +| `index_max_duration_seconds` | `off` | `off` or `1..86400` | Total worker wall-clock duration | + +```bash +codebase-memory-mcp config set index_max_rss_mb 8192 +codebase-memory-mcp config set index_max_duration_seconds 3600 +``` + +RSS is the current resident memory of the contained worker and every descendant, +not the worker's allocation budget and not peak memory. This hard watchdog is +separate from the internal `CBM_MEM_BUDGET_MB` soft budget. The supervisor +samples RSS at most once every 250 milliseconds so the watchdog does not turn +full process-table enumeration into a busy loop. + +Duration uses a monotonic clock from successful spawn. It is independent of the +existing 15-minute quiet timeout: continuous log progress does not reset total +duration, while the quiet timeout continues to identify a worker that stops +making progress. + +Equality is allowed. The first RSS or elapsed-duration observation above its +limit starts the existing graceful-to-force process-tree shutdown. CBM reports +terminal only after the tree is quiescent or a bounded containment failure is +explicitly surfaced. Resource termination is not retried and does not +quarantine a source file. + +If RSS is enabled and three consecutive probes cannot obtain any trustworthy +tree measurement while the root worker is still running, CBM fails closed with +`code=resource_probe_failed`. + ## Counting and failure semantics `index_max_files` counts a file only after it passes directory pruning, ignore @@ -55,6 +88,12 @@ The previous database remains available because publication occurs only after a complete discovery and successful staged build. If no previous database exists, `serving_index_preserved` is false. +Worker limit failures use the same shape with `stage=worker`, +`resource=rss_bytes` and `unit=bytes`, or `resource=duration_ms` and +`unit=milliseconds`. RSS measurement failures use `code=resource_probe_failed` +and omit `observed`, `limit`, and `unit` because no trustworthy observation was +available. + ## Trust and compatibility Limits are read from the CLI-managed `_config.db`; they are not MCP request @@ -64,5 +103,6 @@ parent policy. These settings do not replace or increase `auto_index_limit`, change the 512 MiB single-file cap, alter workspace-root authorization, or affect -`cross-repo-intelligence`. With both settings `off`, discovery follows the -existing unbounded path. +`cross-repo-intelligence`. With all settings `off`, discovery follows the +existing path and the supervisor performs no periodic RSS probe or total-duration +termination. diff --git a/scripts/test-runtime.sh b/scripts/test-runtime.sh index 5d2c0c3ae..e6e60b15a 100644 --- a/scripts/test-runtime.sh +++ b/scripts/test-runtime.sh @@ -112,7 +112,8 @@ _cbm_test_runtime_daemon() { # cbm_mcp_index_policy_add_to_args: every key in cbm_index_policy_key_at, and # nothing else. cbm_test_index_worker_policy_json() { - printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"}' + printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"' + printf '%s' ',"index_max_rss_mb":"off","index_max_duration_seconds":"off"}' } cbm_test_runtime_cleanup() { diff --git a/src/cli/cli.c b/src/cli/cli.c index f0159925c..8c52bb5eb 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -6777,6 +6777,8 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"}, {CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"}, {CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"}, + {CBM_INDEX_CONFIG_MAX_RSS_MB, "off", "Max worker process-tree RSS MiB, or off"}, + {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "off", "Max worker duration in seconds, or off"}, }; /* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing @@ -6803,8 +6805,12 @@ static bool config_key_is_ui(const char *key) { } static bool config_key_is_index_policy(const char *key) { - return key && (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0 || - strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); + for (size_t index = 0; key && index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, cbm_index_policy_key_at(index)) == 0) { + return true; + } + } + return false; } static int config_index_policy_write(cbm_config_t *config, const char *key, const char *value) { diff --git a/src/daemon/application.c b/src/daemon/application.c index 6cd42504f..dce9468f2 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -304,9 +304,18 @@ static int application_worker_start_default(void *context, const char *args_json const char *quarantine_file, cbm_daemon_application_worker_t *worker_out) { (void)context; + cbm_index_resource_policy_t resource_policy; + char error[CBM_SZ_256] = {0}; + if (!cbm_mcp_index_policy_from_internal_args(args_json, &resource_policy, error, + sizeof(error))) { + cbm_log_error("daemon.index.policy", "error", error); + *worker_out = NULL; + return -1; + } cbm_index_worker_handle_t *worker = NULL; - int result = cbm_index_worker_start(args_json, memory_budget_bytes, false, marker_file, - quarantine_file, &worker); + int result = + cbm_index_worker_start_with_policy(args_json, memory_budget_bytes, &resource_policy, false, + marker_file, quarantine_file, &worker); *worker_out = worker; return result; } @@ -1233,6 +1242,12 @@ static application_attempt_decision_t application_consume_attempt( application_attempt_free(attempt); return APPLICATION_ATTEMPT_DECISION_SUCCESS; } + if (disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + execution->response = + cbm_mcp_index_worker_resource_response(job->args_json, &attempt->result); + application_attempt_free(attempt); + return APPLICATION_ATTEMPT_DECISION_STOP; + } if (disposition == CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL) { execution->unsafe_terminal = true; execution->supervision_failed = diff --git a/src/foundation/index_policy.c b/src/foundation/index_policy.c index 0d2b986bf..80aa2f8a6 100644 --- a/src/foundation/index_policy.c +++ b/src/foundation/index_policy.c @@ -1,21 +1,38 @@ #include "foundation/index_policy.h" +#include #include #include -static const char *const INDEX_POLICY_KEYS[] = { - CBM_INDEX_CONFIG_MAX_FILES, - CBM_INDEX_CONFIG_MAX_SOURCE_MB, +typedef struct { + const char *key; + size_t field_offset; + uint64_t minimum; + uint64_t maximum; + uint64_t multiplier; +} index_policy_metadata_t; + +static const index_policy_metadata_t INDEX_POLICY_METADATA[] = { + {CBM_INDEX_CONFIG_MAX_FILES, offsetof(cbm_index_resource_policy_t, max_files), 1, + CBM_INDEX_MAX_FILES_VALUE, 1}, + {CBM_INDEX_CONFIG_MAX_SOURCE_MB, offsetof(cbm_index_resource_policy_t, max_source_bytes), 1, + CBM_INDEX_MAX_SOURCE_MB_VALUE, CBM_INDEX_MIB_BYTES}, + {CBM_INDEX_CONFIG_MAX_RSS_MB, offsetof(cbm_index_resource_policy_t, max_rss_bytes), + CBM_INDEX_MIN_RSS_MB_VALUE, CBM_INDEX_MAX_RSS_MB_VALUE, CBM_INDEX_MIB_BYTES}, + {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, offsetof(cbm_index_resource_policy_t, max_duration_ms), + 1, CBM_INDEX_MAX_DURATION_SECONDS_VALUE, UINT64_C(1000)}, }; -static void set_error(char *error, size_t error_size, const char *key, uint64_t maximum) { +static void set_error(char *error, size_t error_size, const char *key, uint64_t minimum, + uint64_t maximum) { if (error && error_size > 0) { - (void)snprintf(error, error_size, "%s must be off or an integer from 1 to %llu", key, - (unsigned long long)maximum); + (void)snprintf(error, error_size, "%s must be off or an integer from %llu to %llu", key, + (unsigned long long)minimum, (unsigned long long)maximum); } } -static bool parse_bounded_uint64(const char *value, uint64_t maximum, uint64_t *parsed) { +static bool parse_bounded_uint64(const char *value, uint64_t minimum, uint64_t maximum, + uint64_t *parsed) { if (!value || !value[0] || !parsed) { return false; } @@ -30,7 +47,7 @@ static bool parse_bounded_uint64(const char *value, uint64_t maximum, uint64_t * } result = result * 10U + digit; } - if (result == 0) { + if (result < minimum) { return false; } *parsed = result; @@ -44,20 +61,39 @@ void cbm_index_policy_init(cbm_index_resource_policy_t *policy) { } bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy) { + if (!policy) { + return false; + } + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + const cbm_index_limit_u64_t *limit = + (const cbm_index_limit_u64_t *)((const unsigned char *)policy + + INDEX_POLICY_METADATA[index].field_offset); + if (limit->enabled) { + return true; + } + } + return false; +} + +bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy) { return policy && (policy->max_files.enabled || policy->max_source_bytes.enabled); } +bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy) { + return policy && (policy->max_rss_bytes.enabled || policy->max_duration_ms.enabled); +} + size_t cbm_index_policy_key_count(void) { - return sizeof(INDEX_POLICY_KEYS) / sizeof(INDEX_POLICY_KEYS[0]); + return sizeof(INDEX_POLICY_METADATA) / sizeof(INDEX_POLICY_METADATA[0]); } const char *cbm_index_policy_key_at(size_t index) { - return index < cbm_index_policy_key_count() ? INDEX_POLICY_KEYS[index] : NULL; + return index < cbm_index_policy_key_count() ? INDEX_POLICY_METADATA[index].key : NULL; } const char *cbm_index_policy_default_value(const char *key) { for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { - if (key && strcmp(key, INDEX_POLICY_KEYS[index]) == 0) { + if (key && strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { return "off"; } } @@ -70,34 +106,33 @@ bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, error[0] = '\0'; } if (!policy || !key || !value) { - set_error(error, error_size, key ? key : "index resource limit", 0); + set_error(error, error_size, key ? key : "index resource limit", 0, 0); return false; } - cbm_index_limit_u64_t *target = NULL; - uint64_t maximum = 0; - uint64_t multiplier = 1; - if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) { - target = &policy->max_files; - maximum = CBM_INDEX_MAX_FILES_VALUE; - } else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) { - target = &policy->max_source_bytes; - maximum = CBM_INDEX_MAX_SOURCE_MB_VALUE; - multiplier = CBM_INDEX_MIB_BYTES; - } else { - set_error(error, error_size, key, 0); + const index_policy_metadata_t *metadata = NULL; + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { + metadata = &INDEX_POLICY_METADATA[index]; + break; + } + } + if (!metadata) { + set_error(error, error_size, key, 0, 0); return false; } + cbm_index_limit_u64_t *target = + (cbm_index_limit_u64_t *)((unsigned char *)policy + metadata->field_offset); cbm_index_limit_u64_t candidate = {0}; if (strcmp(value, "off") != 0) { uint64_t parsed = 0; - if (!parse_bounded_uint64(value, maximum, &parsed)) { - set_error(error, error_size, key, maximum); + if (!parse_bounded_uint64(value, metadata->minimum, metadata->maximum, &parsed)) { + set_error(error, error_size, key, metadata->minimum, metadata->maximum); return false; } candidate.enabled = true; - candidate.value = parsed * multiplier; + candidate.value = parsed * metadata->multiplier; } *target = candidate; return true; @@ -108,18 +143,21 @@ bool cbm_index_policy_format(const cbm_index_resource_policy_t *policy, const ch if (!policy || !key || !out || out_size == 0) { return false; } - const cbm_index_limit_u64_t *limit = NULL; - uint64_t divisor = 1; - if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) { - limit = &policy->max_files; - } else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) { - limit = &policy->max_source_bytes; - divisor = CBM_INDEX_MIB_BYTES; - } else { + const index_policy_metadata_t *metadata = NULL; + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { + metadata = &INDEX_POLICY_METADATA[index]; + break; + } + } + if (!metadata) { return false; } + const cbm_index_limit_u64_t *limit = + (const cbm_index_limit_u64_t *)((const unsigned char *)policy + metadata->field_offset); int length = limit->enabled - ? snprintf(out, out_size, "%llu", (unsigned long long)(limit->value / divisor)) + ? snprintf(out, out_size, "%llu", + (unsigned long long)(limit->value / metadata->multiplier)) : snprintf(out, out_size, "off"); return length >= 0 && (size_t)length < out_size; } @@ -130,6 +168,10 @@ const char *cbm_index_resource_name(cbm_index_resource_t resource) { return "files"; case CBM_INDEX_RESOURCE_SOURCE_BYTES: return "source_bytes"; + case CBM_INDEX_RESOURCE_RSS_BYTES: + return "rss_bytes"; + case CBM_INDEX_RESOURCE_DURATION_MS: + return "duration_ms"; case CBM_INDEX_RESOURCE_NONE: default: return "unknown"; @@ -137,7 +179,13 @@ const char *cbm_index_resource_name(cbm_index_resource_t resource) { } const char *cbm_index_resource_unit(cbm_index_resource_t resource) { - return resource == CBM_INDEX_RESOURCE_FILES ? "files" : "bytes"; + if (resource == CBM_INDEX_RESOURCE_FILES) { + return "files"; + } + if (resource == CBM_INDEX_RESOURCE_DURATION_MS) { + return "milliseconds"; + } + return "bytes"; } const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { @@ -146,6 +194,10 @@ const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { return CBM_INDEX_CONFIG_MAX_FILES; case CBM_INDEX_RESOURCE_SOURCE_BYTES: return CBM_INDEX_CONFIG_MAX_SOURCE_MB; + case CBM_INDEX_RESOURCE_RSS_BYTES: + return CBM_INDEX_CONFIG_MAX_RSS_MB; + case CBM_INDEX_RESOURCE_DURATION_MS: + return CBM_INDEX_CONFIG_MAX_DURATION_SECONDS; case CBM_INDEX_RESOURCE_NONE: default: return "index_resource_limit"; diff --git a/src/foundation/index_policy.h b/src/foundation/index_policy.h index 6a6c499fc..617a6ab7d 100644 --- a/src/foundation/index_policy.h +++ b/src/foundation/index_policy.h @@ -7,9 +7,14 @@ #define CBM_INDEX_CONFIG_MAX_FILES "index_max_files" #define CBM_INDEX_CONFIG_MAX_SOURCE_MB "index_max_source_mb" +#define CBM_INDEX_CONFIG_MAX_RSS_MB "index_max_rss_mb" +#define CBM_INDEX_CONFIG_MAX_DURATION_SECONDS "index_max_duration_seconds" #define CBM_INDEX_MAX_FILES_VALUE UINT64_C(10000000) #define CBM_INDEX_MAX_SOURCE_MB_VALUE UINT64_C(1048576) +#define CBM_INDEX_MIN_RSS_MB_VALUE UINT64_C(64) +#define CBM_INDEX_MAX_RSS_MB_VALUE UINT64_C(1048576) +#define CBM_INDEX_MAX_DURATION_SECONDS_VALUE UINT64_C(86400) #define CBM_INDEX_MIB_BYTES UINT64_C(1048576) typedef struct { @@ -20,12 +25,16 @@ typedef struct { typedef struct { cbm_index_limit_u64_t max_files; cbm_index_limit_u64_t max_source_bytes; + cbm_index_limit_u64_t max_rss_bytes; + cbm_index_limit_u64_t max_duration_ms; } cbm_index_resource_policy_t; typedef enum { CBM_INDEX_RESOURCE_NONE = 0, CBM_INDEX_RESOURCE_FILES, CBM_INDEX_RESOURCE_SOURCE_BYTES, + CBM_INDEX_RESOURCE_RSS_BYTES, + CBM_INDEX_RESOURCE_DURATION_MS, } cbm_index_resource_t; typedef struct { @@ -36,6 +45,8 @@ typedef struct { void cbm_index_policy_init(cbm_index_resource_policy_t *policy); bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy); size_t cbm_index_policy_key_count(void); const char *cbm_index_policy_key_at(size_t index); diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index 9174391f4..f6be6b93d 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -11,6 +11,7 @@ #include "platform.h" /* cbm_now_ms */ #include "sanitized.h" /* CBM_SANITIZED — spawn-retry budget */ +#include #include #include #include @@ -18,14 +19,17 @@ #ifdef _WIN32 #include +#include #include "win_utf8.h" /* cbm_utf8_to_wide — spawn the worker with a wide command line so a * non-ASCII repo path survives CreateProcess (#423/#20) */ #include /* free */ #else +#include #include #include #include #ifdef __APPLE__ +#include #include extern char **environ; #endif @@ -436,6 +440,287 @@ struct cbm_subprocess { #endif }; +static void cbm_rss_add_saturated(uint64_t *total, uint64_t value) { + if (value > UINT64_MAX - *total) { + *total = UINT64_MAX; + } else { + *total += value; + } +} + +#ifdef CBM_ENABLE_TEST_SEAMS +uint64_t cbm_subprocess_rss_sum_for_testing(const uint64_t *values, size_t count) { + uint64_t total = 0; + for (size_t index = 0; values && index < count; index++) { + cbm_rss_add_saturated(&total, values[index]); + } + return total; +} +#endif + +#ifdef _WIN32 +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + DWORD capacity = 32; + JOBOBJECT_BASIC_PROCESS_ID_LIST *processes = NULL; + for (;;) { + size_t bytes = sizeof(*processes) + (size_t)(capacity - 1) * sizeof(ULONG_PTR); + if (bytes > UINT32_MAX) { + free(processes); + return CBM_PROC_TREE_RSS_ERROR; + } + JOBOBJECT_BASIC_PROCESS_ID_LIST *grown = realloc(processes, bytes); + if (!grown) { + free(processes); + return CBM_PROC_TREE_RSS_ERROR; + } + processes = grown; + ZeroMemory(processes, bytes); + processes->NumberOfAssignedProcesses = capacity; + if (QueryInformationJobObject(process->job, JobObjectBasicProcessIdList, processes, + (DWORD)bytes, NULL)) { + break; + } + if (GetLastError() != ERROR_MORE_DATA || capacity > (UINT32_MAX / 2U)) { + free(processes); + return CBM_PROC_TREE_RSS_ERROR; + } + capacity *= 2U; + } + + uint64_t total = 0; + DWORD measured = 0; + bool root_failed = false; + for (DWORD index = 0; index < processes->NumberOfProcessIdsInList; index++) { + DWORD pid = (DWORD)processes->ProcessIdList[index]; + HANDLE member = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid); + if (!member) { + root_failed = root_failed || pid == process->process_id; + continue; + } + PROCESS_MEMORY_COUNTERS memory; + ZeroMemory(&memory, sizeof(memory)); + if (GetProcessMemoryInfo(member, &memory, sizeof(memory))) { + cbm_rss_add_saturated(&total, (uint64_t)memory.WorkingSetSize); + measured++; + } else if (pid == process->process_id) { + root_failed = true; + } + CloseHandle(member); + } + DWORD listed = processes->NumberOfProcessIdsInList; + free(processes); + if (root_failed || (measured == 0 && listed > 0)) { + return CBM_PROC_TREE_RSS_ERROR; + } + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#elif defined(__APPLE__) +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + int capacity = proc_listallpids(NULL, 0); + if (capacity <= 0 || capacity > INT_MAX / (int)sizeof(pid_t) - 64) { + return CBM_PROC_TREE_RSS_ERROR; + } + capacity += 64; + pid_t *pids = NULL; + int count = 0; + for (;;) { + pid_t *grown = realloc(pids, (size_t)capacity * sizeof(*pids)); + if (!grown) { + free(pids); + return CBM_PROC_TREE_RSS_ERROR; + } + pids = grown; + count = proc_listallpids(pids, capacity * (int)sizeof(*pids)); + if (count <= 0) { + free(pids); + return CBM_PROC_TREE_RSS_ERROR; + } + if (count < capacity) { + break; + } + if (capacity > INT_MAX / (int)sizeof(*pids) / 2) { + free(pids); + return CBM_PROC_TREE_RSS_ERROR; + } + capacity *= 2; + } + + uint64_t total = 0; + int measured = 0; + bool root_failed = false; + for (int index = 0; index < count; index++) { + struct proc_bsdinfo info; + if (pids[index] <= 0) { + continue; + } + if (proc_pidinfo(pids[index], PROC_PIDTBSDINFO, 0, &info, sizeof(info)) != + (int)sizeof(info)) { + root_failed = root_failed || pids[index] == process->pid; + continue; + } + if ((pid_t)info.pbi_pgid != process->pgid) { + continue; + } + struct rusage_info_v2 usage; + if (proc_pid_rusage(pids[index], RUSAGE_INFO_V2, (rusage_info_t *)&usage) != 0) { + root_failed = root_failed || pids[index] == process->pid; + continue; + } + cbm_rss_add_saturated(&total, usage.ri_resident_size); + measured++; + } + free(pids); + if (root_failed) { + return CBM_PROC_TREE_RSS_ERROR; + } + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#else +static bool cbm_proc_pid_name(const char *name) { + if (!name || !name[0]) { + return false; + } + for (const unsigned char *cursor = (const unsigned char *)name; *cursor; cursor++) { + if (*cursor < '0' || *cursor > '9') { + return false; + } + } + return true; +} + +static bool cbm_linux_proc_stat(const char *pid_name, pid_t *group, int64_t *rss_pages) { + char path[64]; + int length = snprintf(path, sizeof(path), "/proc/%s/stat", pid_name); + if (length <= 0 || length >= (int)sizeof(path)) { + return false; + } + FILE *stat_file = cbm_fopen(path, "rb"); + if (!stat_file) { + return false; + } + char stat_line[4096]; + bool read = fgets(stat_line, sizeof(stat_line), stat_file) != NULL; + (void)fclose(stat_file); + char *command_end = read ? strrchr(stat_line, ')') : NULL; + if (!command_end || command_end[1] != ' ') { + return false; + } + + char *save = NULL; + char *token = strtok_r(command_end + 2, " ", &save); + int field = 3; + bool have_group = false; + bool have_rss = false; + while (token) { + if (field == 5 || field == 24) { + char *end = NULL; + long long value = strtoll(token, &end, 10); + if (!end || *end != '\0') { + return false; + } + if (field == 5) { + *group = (pid_t)value; + have_group = true; + } else { + *rss_pages = (int64_t)value; + have_rss = true; + break; + } + } + token = strtok_r(NULL, " ", &save); + field++; + } + return have_group && have_rss; +} + +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + DIR *proc = opendir("/proc"); + if (!proc) { + return CBM_PROC_TREE_RSS_ERROR; + } + long page_size = sysconf(_SC_PAGESIZE); + if (page_size <= 0) { + (void)closedir(proc); + return CBM_PROC_TREE_RSS_ERROR; + } + + uint64_t total = 0; + int measured = 0; + bool root_failed = false; + struct dirent *entry; + while ((entry = readdir(proc)) != NULL) { + if (!cbm_proc_pid_name(entry->d_name)) { + continue; + } + pid_t group = 0; + int64_t pages = 0; + bool root_entry = strtol(entry->d_name, NULL, 10) == (long)process->pid; + if (!cbm_linux_proc_stat(entry->d_name, &group, &pages)) { + root_failed = root_failed || root_entry; + continue; + } + if (group != process->pgid || pages < 0) { + continue; + } + uint64_t page_count = (uint64_t)pages; + uint64_t bytes = page_count > UINT64_MAX / (uint64_t)page_size + ? UINT64_MAX + : page_count * (uint64_t)page_size; + cbm_rss_add_saturated(&total, bytes); + measured++; + } + (void)closedir(proc); + if (root_failed) { + return CBM_PROC_TREE_RSS_ERROR; + } + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#endif + +cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_bytes(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + if (!process || !rss_bytes) { + return CBM_PROC_TREE_RSS_ERROR; + } + *rss_bytes = 0; + return cbm_subprocess_tree_rss_platform(process, rss_bytes); +} + +bool cbm_subprocess_root_running(const cbm_subprocess_t *process) { + if (!process || process->root_reaped) { + return false; + } + int lifecycle = atomic_load_explicit(&process->lifecycle, memory_order_acquire); + return lifecycle == CBM_SUBPROCESS_ACTIVE || lifecycle == CBM_SUBPROCESS_CANCEL_REQUESTED; +} + +bool cbm_subprocess_termination_pending(const cbm_subprocess_t *process) { + return process && process->termination_started; +} + +bool cbm_subprocess_supervision_active(const cbm_subprocess_t *process) { + if (!process) { + return false; + } + int lifecycle = atomic_load_explicit(&process->lifecycle, memory_order_acquire); + return lifecycle == CBM_SUBPROCESS_ACTIVE || lifecycle == CBM_SUBPROCESS_CANCEL_REQUESTED; +} + static void cbm_subprocess_result_init(cbm_proc_result_t *result) { result->outcome = CBM_PROC_SPAWN_FAILED; result->exit_code = -1; diff --git a/src/foundation/subprocess.h b/src/foundation/subprocess.h index 62592c592..07998603c 100644 --- a/src/foundation/subprocess.h +++ b/src/foundation/subprocess.h @@ -22,6 +22,7 @@ #include #include /* size_t (cbm_build_win_cmdline) */ +#include /* How a supervised child ended. */ typedef enum { @@ -87,6 +88,12 @@ typedef enum { CBM_PROC_POLL_TERMINAL = 1 } cbm_proc_poll_t; +typedef enum { + CBM_PROC_TREE_RSS_ERROR = -1, + CBM_PROC_TREE_RSS_EMPTY = 0, + CBM_PROC_TREE_RSS_OK = 1, +} cbm_proc_tree_rss_status_t; + /* Spawn opts->bin and return immediately with a supervisor handle. On success, * *out owns the process until a terminal poll followed by destroy. Spawn copies * the option strings/argv it needs after return; log_ud remains caller-owned until @@ -117,6 +124,16 @@ int cbm_subprocess_spawn(const cbm_proc_opts_t *opts, cbm_subprocess_t **out); * tree when cancel_grace_ms elapses. Callers must keep polling to make progress. */ cbm_proc_poll_t cbm_subprocess_poll(cbm_subprocess_t *process, cbm_proc_result_t *out); +/* Read the current resident-set size of the complete contained process tree. + * OK returns an overflow-safe byte total, EMPTY means the owned tree currently + * has no observable members, and ERROR means no trustworthy measurement could + * be obtained. Individual processes that exit during enumeration are ignored. */ +cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_bytes(cbm_subprocess_t *process, + uint64_t *rss_bytes); +bool cbm_subprocess_root_running(const cbm_subprocess_t *process); +bool cbm_subprocess_termination_pending(const cbm_subprocess_t *process); +bool cbm_subprocess_supervision_active(const cbm_subprocess_t *process); + /* Record an explicit cancellation request without waiting. Safe to repeat and * safe to call from a cancellation thread while one owner thread polls. The * owner must stop cancellation producers before destroying the handle. true @@ -183,6 +200,7 @@ bool cbm_build_win_cmd_payload(char *buf, size_t cap, const char *cmd_executable * of hoping a loaded machine reproduces it. Test builds only. */ void cbm_subprocess_force_spawn_eagain_for_testing(int attempts); int cbm_subprocess_pending_spawn_eagain_for_testing(void); +uint64_t cbm_subprocess_rss_sum_for_testing(const uint64_t *values, size_t count); #endif #endif /* CBM_SUBPROCESS_H */ diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index b5257dfc1..b9d0212ef 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -390,8 +390,56 @@ enum { INDEX_WORKER_SYNC_POLL_NS = 10000000, INDEX_WORKER_RELAY_LINES_PER_POLL = 64, INDEX_WORKER_RELAY_BYTES_PER_POLL = 64 * 1024, + INDEX_WORKER_RSS_PROBE_FAILURE_LIMIT = 3, + INDEX_WORKER_RSS_PROBE_INTERVAL_MS = 250, }; +typedef enum { + INDEX_WORKER_TERMINATION_NONE = 0, + INDEX_WORKER_TERMINATION_CANCEL_PENDING, + INDEX_WORKER_TERMINATION_CANCEL, + INDEX_WORKER_TERMINATION_RESOURCE, +} index_worker_termination_reason_t; + +#ifdef CBM_ENABLE_TEST_SEAMS +static cbm_index_supervisor_clock_fn g_resource_clock; +static cbm_index_supervisor_rss_fn g_resource_rss; +static void *g_resource_hook_context; + +void cbm_index_supervisor_set_resource_hooks_for_testing(cbm_index_supervisor_clock_fn clock_fn, + cbm_index_supervisor_rss_fn rss_fn, + void *context) { + g_resource_clock = clock_fn; + g_resource_rss = rss_fn; + g_resource_hook_context = context; +} + +void cbm_index_supervisor_reset_resource_hooks_for_testing(void) { + g_resource_clock = NULL; + g_resource_rss = NULL; + g_resource_hook_context = NULL; +} +#endif + +static uint64_t worker_resource_now_ms(void) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_resource_clock) { + return g_resource_clock(g_resource_hook_context); + } +#endif + return cbm_now_ms(); +} + +static cbm_proc_tree_rss_status_t worker_resource_rss(cbm_subprocess_t *process, + uint64_t *rss_bytes) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_resource_rss) { + return g_resource_rss(process, rss_bytes, g_resource_hook_context); + } +#endif + return cbm_subprocess_tree_rss_bytes(process, rss_bytes); +} + struct cbm_index_worker_handle { cbm_subprocess_t *process; char response_path[INDEX_WORKER_PATH_CAP]; @@ -401,6 +449,12 @@ struct cbm_index_worker_handle { long relay_tail_pos; bool process_terminal; cbm_proc_result_t process_result; + cbm_index_resource_policy_t resource_policy; + uint64_t started_ms; + uint64_t last_rss_probe_ms; + unsigned int rss_probe_failures; + bool rss_probe_started; + atomic_int termination_reason; atomic_bool terminal; cbm_index_worker_result_t result; }; @@ -578,7 +632,8 @@ static bool worker_unique_file(char *out, size_t out_size, const char *kind) { static bool worker_result_succeeded(const cbm_index_worker_result_t *result) { return result && result->outcome == CBM_PROC_CLEAN && !result->cancellation_requested && - result->tree_quiesced && !result->supervision_failed; + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed && result->tree_quiesced && !result->supervision_failed; } static void worker_terminal_log(cbm_index_worker_handle_t *handle) { @@ -594,6 +649,13 @@ static void worker_terminal_log(cbm_index_worker_handle_t *handle) { } else if (handle->result.supervision_failed || !handle->result.tree_quiesced) { cbm_log_error("index.supervisor.containment_failed", "outcome", cbm_proc_outcome_str(handle->result.outcome), "log", handle->log_path); + } else if (handle->result.resource_probe_failed) { + cbm_log_error("index.supervisor.resource_probe_failed", "resource", "rss_bytes", "log", + handle->log_path); + } else if (handle->result.resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { + cbm_log_warn("index.supervisor.resource_limit", "resource", + cbm_index_resource_name(handle->result.resource_violation.resource), "log", + handle->log_path); } else if (handle->result.cancellation_requested) { cbm_log_warn("index.supervisor.worker_cancelled", "outcome", cbm_proc_outcome_str(handle->result.outcome), "log", handle->log_path); @@ -608,10 +670,87 @@ static void worker_terminal_log(cbm_index_worker_handle_t *handle) { } } -int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_bytes, - bool single_thread, const char *marker_file, - const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, cbm_index_worker_handle_t **handle_out) { +static void worker_request_resource_termination(cbm_index_worker_handle_t *handle, + cbm_index_resource_t resource, uint64_t observed, + uint64_t limit, bool probe_failed) { + int expected = INDEX_WORKER_TERMINATION_NONE; + if (!atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + INDEX_WORKER_TERMINATION_RESOURCE, + memory_order_acq_rel, memory_order_acquire)) { + return; + } + handle->result.resource_violation = (cbm_index_resource_violation_t){ + .resource = resource, .observed = observed, .limit = limit}; + handle->result.resource_probe_failed = probe_failed; + if (!cbm_subprocess_request_cancel(handle->process)) { + handle->result.resource_violation = (cbm_index_resource_violation_t){0}; + handle->result.resource_probe_failed = false; + expected = INDEX_WORKER_TERMINATION_RESOURCE; + (void)atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + INDEX_WORKER_TERMINATION_NONE, + memory_order_acq_rel, memory_order_acquire); + } +} + +static bool worker_check_resource_limits(cbm_index_worker_handle_t *handle) { + if (atomic_load_explicit(&handle->termination_reason, memory_order_acquire) != + INDEX_WORKER_TERMINATION_NONE || + cbm_subprocess_termination_pending(handle->process) || + !cbm_subprocess_supervision_active(handle->process)) { + return false; + } + uint64_t now = 0; + bool have_now = false; + bool rss_probe_failed = false; + if (handle->resource_policy.max_rss_bytes.enabled) { + now = worker_resource_now_ms(); + have_now = true; + bool probe_due = !handle->rss_probe_started || now < handle->last_rss_probe_ms || + now - handle->last_rss_probe_ms >= INDEX_WORKER_RSS_PROBE_INTERVAL_MS; + if (!probe_due) { + goto duration_check; + } + handle->rss_probe_started = true; + handle->last_rss_probe_ms = now; + uint64_t rss_bytes = 0; + cbm_proc_tree_rss_status_t status = worker_resource_rss(handle->process, &rss_bytes); + if (status == CBM_PROC_TREE_RSS_OK) { + handle->rss_probe_failures = 0; + if (rss_bytes > handle->resource_policy.max_rss_bytes.value) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_RSS_BYTES, rss_bytes, + handle->resource_policy.max_rss_bytes.value, + false); + return false; + } + } else if (status == CBM_PROC_TREE_RSS_ERROR) { + handle->rss_probe_failures++; + if (handle->rss_probe_failures >= INDEX_WORKER_RSS_PROBE_FAILURE_LIMIT) { + rss_probe_failed = true; + } + } else { + handle->rss_probe_failures = 0; + } + } +duration_check: + if (handle->resource_policy.max_duration_ms.enabled) { + if (!have_now) { + now = worker_resource_now_ms(); + } + uint64_t elapsed = now >= handle->started_ms ? now - handle->started_ms : 0; + if (elapsed > handle->resource_policy.max_duration_ms.value) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_DURATION_MS, elapsed, + handle->resource_policy.max_duration_ms.value, + false); + } + } + return rss_probe_failed; +} + +static int worker_start_internal(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, cbm_index_worker_handle_t **handle_out) { if (handle_out) { *handle_out = NULL; } @@ -643,6 +782,11 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ if (!handle) { return -1; } + cbm_index_policy_init(&handle->resource_policy); + if (resource_policy) { + handle->resource_policy = *resource_policy; + } + atomic_init(&handle->termination_reason, INDEX_WORKER_TERMINATION_NONE); atomic_init(&handle->terminal, false); handle->log_callback = log_callback; handle->log_context = log_context; @@ -703,15 +847,35 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ cbm_log_error("index.supervisor.spawn_failed", "action", "fail_closed"); return -1; } + if (handle->resource_policy.max_duration_ms.enabled) { + handle->started_ms = worker_resource_now_ms(); + } *handle_out = handle; return 0; } +int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_bytes, + bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, + quarantine_file, log_callback, log_context, handle_out); +} + int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out) { - return cbm_index_worker_start_with_log(args_json, memory_budget_bytes, single_thread, - marker_file, quarantine_file, NULL, NULL, handle_out); + return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, + quarantine_file, NULL, NULL, handle_out); +} + +int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, + cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, resource_policy, single_thread, + marker_file, quarantine_file, NULL, NULL, handle_out); } cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, @@ -728,10 +892,16 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, } bool relay_caught_up = true; if (!handle->process_terminal) { + bool rss_probe_failed = worker_check_resource_limits(handle); cbm_proc_result_t process_result; cbm_proc_poll_t state = cbm_subprocess_poll(handle->process, &process_result); relay_caught_up = worker_relay_log(handle); if (state == CBM_PROC_POLL_RUNNING) { + if (rss_probe_failed && cbm_subprocess_root_running(handle->process)) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_RSS_BYTES, 0, + handle->resource_policy.max_rss_bytes.value, + true); + } return CBM_INDEX_WORKER_POLL_RUNNING; } if (state != CBM_PROC_POLL_TERMINAL) { @@ -752,7 +922,12 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, handle->result.outcome = process_result->outcome; handle->result.exit_code = process_result->exit_code; handle->result.term_signal = process_result->term_signal; - handle->result.cancellation_requested = process_result->cancellation_requested; + int termination_reason = + atomic_load_explicit(&handle->termination_reason, memory_order_acquire); + handle->result.cancellation_requested = + termination_reason == INDEX_WORKER_TERMINATION_CANCEL || + (termination_reason == INDEX_WORKER_TERMINATION_CANCEL_PENDING && + process_result->cancellation_requested); handle->result.forced = process_result->forced; handle->result.tree_quiesced = process_result->tree_quiesced; handle->result.supervision_failed = process_result->supervision_failed; @@ -775,8 +950,36 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, } bool cbm_index_worker_request_cancel(cbm_index_worker_handle_t *handle) { - return handle && !atomic_load_explicit(&handle->terminal, memory_order_acquire) && - cbm_subprocess_request_cancel(handle->process); + if (!handle || atomic_load_explicit(&handle->terminal, memory_order_acquire)) { + return false; + } + int expected = INDEX_WORKER_TERMINATION_NONE; + bool claimed = atomic_compare_exchange_strong_explicit( + &handle->termination_reason, &expected, INDEX_WORKER_TERMINATION_CANCEL_PENDING, + memory_order_acq_rel, memory_order_acquire); + if (!claimed) { + if (expected == INDEX_WORKER_TERMINATION_CANCEL) { + return true; + } + if (expected != INDEX_WORKER_TERMINATION_CANCEL_PENDING) { + return false; + } + bool accepted = cbm_subprocess_request_cancel(handle->process); + if (accepted) { + expected = INDEX_WORKER_TERMINATION_CANCEL_PENDING; + (void)atomic_compare_exchange_strong_explicit( + &handle->termination_reason, &expected, INDEX_WORKER_TERMINATION_CANCEL, + memory_order_acq_rel, memory_order_acquire); + } + return accepted; + } + bool accepted = cbm_subprocess_request_cancel(handle->process); + expected = INDEX_WORKER_TERMINATION_CANCEL_PENDING; + (void)atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + accepted ? INDEX_WORKER_TERMINATION_CANCEL + : INDEX_WORKER_TERMINATION_NONE, + memory_order_acq_rel, memory_order_acquire); + return accepted; } const char *cbm_index_worker_response_path(const cbm_index_worker_handle_t *handle) { @@ -801,18 +1004,19 @@ void cbm_index_worker_destroy(cbm_index_worker_handle_t *handle) { free(handle); } -int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_thread, - const char *marker_file, const char *quarantine_file, - cbm_proc_log_cb log_callback, void *log_context, - const atomic_int *cancel_requested, - cbm_index_worker_result_t *result) { +static int worker_spawn_internal(const char *args_json, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, const atomic_int *cancel_requested, + cbm_index_worker_result_t *result) { if (!result) { return -1; } worker_result_init(result); cbm_index_worker_handle_t *handle = NULL; - if (cbm_index_worker_start_with_log(args_json, 0, single_thread, marker_file, quarantine_file, - log_callback, log_context, &handle) != 0) { + if (worker_start_internal(args_json, 0, resource_policy, single_thread, marker_file, + quarantine_file, log_callback, log_context, &handle) != 0) { return -1; } const cbm_index_worker_result_t *cached = NULL; @@ -838,6 +1042,24 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th return 0; } +int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + const atomic_int *cancel_requested, + cbm_index_worker_result_t *result) { + return worker_spawn_internal(args_json, NULL, single_thread, marker_file, quarantine_file, + log_callback, log_context, cancel_requested, result); +} + +int cbm_index_spawn_worker_with_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, + const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result) { + return worker_spawn_internal(args_json, resource_policy, single_thread, marker_file, + quarantine_file, log_callback, log_context, cancel_requested, + result); +} + int cbm_index_spawn_worker_with_log(const char *args_json, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, diff --git a/src/mcp/index_supervisor.h b/src/mcp/index_supervisor.h index 31259df90..d7e39f94d 100644 --- a/src/mcp/index_supervisor.h +++ b/src/mcp/index_supervisor.h @@ -22,8 +22,10 @@ #include #include +#include #include +#include "foundation/index_policy.h" #include "foundation/subprocess.h" /* cbm_proc_outcome_t */ /* Worker-role state, set once from the CLI arg parser (main.c) when this process @@ -138,8 +140,10 @@ typedef struct { bool tree_quiesced; bool supervision_failed; bool response_rejected; /* clean worker exceeded the bounded response protocol */ - char *response; /* worker result only after a contained, uncancelled CLEAN exit; - * borrowed for async polls, caller-owned from the sync wrapper */ + cbm_index_resource_violation_t resource_violation; + bool resource_probe_failed; + char *response; /* worker result only after a contained, uncancelled CLEAN exit; + * borrowed for async polls, caller-owned from the sync wrapper */ } cbm_index_worker_result_t; /* Daemon-owned, nonblocking supervisor for one contained worker process tree. */ @@ -157,6 +161,11 @@ typedef enum { int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out); +int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, + cbm_index_worker_handle_t **handle_out); /* Request-scoped variant used by interactive local CLI calls. The callback is * invoked by the owner thread while it polls the contained worker; log_context @@ -220,7 +229,22 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th cbm_proc_log_cb log_callback, void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result); +int cbm_index_spawn_worker_with_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, + const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result); void cbm_index_worker_result_free(cbm_index_worker_result_t *result); +#ifdef CBM_ENABLE_TEST_SEAMS +typedef uint64_t (*cbm_index_supervisor_clock_fn)(void *context); +typedef cbm_proc_tree_rss_status_t (*cbm_index_supervisor_rss_fn)(cbm_subprocess_t *process, + uint64_t *rss_bytes, + void *context); +void cbm_index_supervisor_set_resource_hooks_for_testing(cbm_index_supervisor_clock_fn clock_fn, + cbm_index_supervisor_rss_fn rss_fn, + void *context); +void cbm_index_supervisor_reset_resource_hooks_for_testing(void); +#endif + #endif /* CBM_INDEX_SUPERVISOR_H */ diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index f842e8f38..e23127c3a 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -7543,6 +7543,8 @@ static bool build_index_success_response(cbm_mcp_server_t *srv, yyjson_mut_doc * return degraded; } +static bool project_db_is_servable(const char *project, const char *db_path); + /* Build the response for a worker that crashed/hung/failed without producing a * result. The crash is already contained (this process survived); we report it * rather than dying. Precise skip-and-continue (quarantine the culprit, index the @@ -7608,6 +7610,56 @@ static char *build_worker_unsafe_terminal_response(const char *args, cbm_proc_ou return response; } +char *cbm_mcp_index_worker_resource_response(const char *args, + const cbm_index_worker_result_t *worker_result) { + char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); + char *name_override = cbm_mcp_get_string_arg(args, "name"); + char *project_name = + cbm_project_name_from_path(name_override && name_override[0] ? name_override : repo_path); + char db_path[CBM_SZ_1K] = {0}; + if (project_name) { + project_db_path(project_name, db_path, sizeof(db_path)); + } + + const cbm_index_resource_violation_t *violation = &worker_result->resource_violation; + const char *config_key = cbm_index_resource_config_key(violation->resource); + char message[CBM_SZ_256]; + (void)snprintf(message, sizeof(message), + worker_result->resource_probe_failed + ? "Worker resource measurement failed for %s" + : "Index worker exceeded %s", + config_key); + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root); + yyjson_mut_obj_add_str(doc, root, "status", "error"); + yyjson_mut_obj_add_str(doc, root, "code", + worker_result->resource_probe_failed ? "resource_probe_failed" + : "resource_limit_exceeded"); + yyjson_mut_obj_add_str(doc, root, "stage", "worker"); + yyjson_mut_obj_add_str(doc, root, "resource", cbm_index_resource_name(violation->resource)); + if (!worker_result->resource_probe_failed) { + yyjson_mut_obj_add_uint(doc, root, "observed", violation->observed); + yyjson_mut_obj_add_uint(doc, root, "limit", violation->limit); + yyjson_mut_obj_add_str(doc, root, "unit", cbm_index_resource_unit(violation->resource)); + } + yyjson_mut_obj_add_bool(doc, root, "retryable", true); + yyjson_mut_obj_add_bool(doc, root, "serving_index_preserved", + project_name && project_db_is_servable(project_name, db_path)); + yyjson_mut_obj_add_strcpy(doc, root, "message", message); + if (repo_path) { + yyjson_mut_obj_add_strcpy(doc, root, "repo_path", repo_path); + } + char *json = yy_doc_to_str(doc); + yyjson_mut_doc_free(doc); + free(project_name); + free(name_override); + free(repo_path); + char *response = cbm_mcp_text_result(json, true); + free(json); + return response; +} + /* Drop the cached store so the next query reopens whatever the worker wrote (each * worker is a fresh process that deletes + recreates the .db). NULL-safe: the * background watcher path (main.c) has no MCP server / cached store — the child @@ -7751,6 +7803,9 @@ cbm_mcp_supervised_result_disposition_t cbm_mcp_supervised_result_disposition( worker_result->supervision_failed) { return CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL; } + if (worker_result->resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { + return CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE; + } if (worker_result->outcome == CBM_PROC_CLEAN) { return worker_result->response ? CBM_MCP_SUPERVISED_RESULT_SUCCESS : CBM_MCP_SUPERVISED_RESULT_FALLBACK; @@ -7758,8 +7813,8 @@ cbm_mcp_supervised_result_disposition_t cbm_mcp_supervised_result_disposition( return CBM_MCP_SUPERVISED_RESULT_CONTAINED_FAILURE; } -static bool index_policy_from_worker_args(const char *args, cbm_index_resource_policy_t *policy, - char *error, size_t error_size) { +bool cbm_mcp_index_policy_from_internal_args(const char *args, cbm_index_resource_policy_t *policy, + char *error, size_t error_size) { yyjson_doc *doc = args ? yyjson_read(args, strlen(args), 0) : NULL; yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; yyjson_val *encoded = @@ -7789,7 +7844,7 @@ static bool index_policy_from_worker_args(const char *args, cbm_index_resource_p static bool load_index_policy(cbm_mcp_server_t *srv, const char *args, cbm_index_resource_policy_t *policy, char *error, size_t error_size) { if (cbm_index_worker_active()) { - return index_policy_from_worker_args(args, policy, error, error_size); + return cbm_mcp_index_policy_from_internal_args(args, policy, error, error_size); } cbm_config_t *owned_config = NULL; cbm_config_t *config = srv ? srv->config : NULL; @@ -7826,13 +7881,14 @@ bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, * - a contained-failure response only if even that cannot produce a clean run. * A physical CBM host never falls back to its in-process pipeline: an initial * start/protocol failure is returned as an explicit error response. */ -static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { +static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, + const cbm_index_resource_policy_t *resource_policy) { invalidate_cached_store(srv); /* First attempt: normal parallel run. */ cbm_index_worker_result_t wr; - int rc = cbm_index_spawn_worker_with_log_cancel( - args, false, NULL, NULL, srv ? srv->index_log_callback : NULL, + int rc = cbm_index_spawn_worker_with_policy_log_cancel( + args, resource_policy, false, NULL, NULL, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wr); cbm_mcp_supervised_result_disposition_t disposition = cbm_mcp_supervised_result_disposition(rc, &wr); @@ -7850,6 +7906,12 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { invalidate_cached_store(srv); return failure; } + if (disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + char *failure = cbm_mcp_index_worker_resource_response(args, &wr); + cbm_index_worker_result_free(&wr); + invalidate_cached_store(srv); + return failure; + } if (disposition == CBM_MCP_SUPERVISED_RESULT_SUCCESS) { /* Clean exit → transfer the worker's response (the common path). */ char *resp = wr.response; /* transfer ownership to caller (may be NULL) */ @@ -7906,8 +7968,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { bool terminal_cancelled = false; for (int i = 0; i < cap; i++) { cbm_index_worker_result_t wr2; - int rc2 = cbm_index_spawn_worker_with_log_cancel( - args, /*single_thread=*/false, marker_path, quarantine_path, + int rc2 = cbm_index_spawn_worker_with_policy_log_cancel( + args, resource_policy, /*single_thread=*/false, marker_path, quarantine_path, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wr2); cbm_mcp_supervised_result_disposition_t recovery_disposition = @@ -7924,6 +7986,11 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { cbm_index_worker_result_free(&wr2); break; } + if (recovery_disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + resp = cbm_mcp_index_worker_resource_response(args, &wr2); + cbm_index_worker_result_free(&wr2); + break; + } if (recovery_disposition == CBM_MCP_SUPERVISED_RESULT_SUCCESS) { resp = wr2.response; /* transfer ownership to caller */ wr2.response = NULL; @@ -7997,8 +8064,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { * so it cannot itself hang. Rare given monotonic progress. */ if (!resp && !unsafe_terminal && quarantined > 0) { cbm_index_worker_result_t wrp; - int rcp = cbm_index_spawn_worker_with_log_cancel( - args, /*single_thread=*/false, NULL, quarantine_path, + int rcp = cbm_index_spawn_worker_with_policy_log_cancel( + args, resource_policy, /*single_thread=*/false, NULL, quarantine_path, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wrp); cbm_mcp_supervised_result_disposition_t partial_disposition = @@ -8014,6 +8081,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { last_outcome = wrp.outcome; unsafe_terminal = true; terminal_cancelled = wrp.cancellation_requested; + } else if (partial_disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + resp = cbm_mcp_index_worker_resource_response(args, &wrp); } cbm_index_worker_result_free(&wrp); } @@ -8057,7 +8126,7 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p if (!args) { return NULL; } - char *resp = index_run_supervised(srv, args); + char *resp = index_run_supervised(srv, args, &policy); free(args); return resp; } @@ -8284,7 +8353,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { free(name_override); return cbm_mcp_text_result("failed to prepare supervised index request", true); } - char *supervised = index_run_supervised(srv, worker_args); + char *supervised = index_run_supervised(srv, worker_args, &resource_policy); free(worker_args); if (supervised) { free(mutation_project); diff --git a/src/mcp/mcp.h b/src/mcp/mcp.h index a70b6372b..320f3dda0 100644 --- a/src/mcp/mcp.h +++ b/src/mcp/mcp.h @@ -221,6 +221,7 @@ char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const ch typedef enum { CBM_MCP_SUPERVISED_RESULT_FALLBACK = 0, CBM_MCP_SUPERVISED_RESULT_SUCCESS, + CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE, CBM_MCP_SUPERVISED_RESULT_CONTAINED_FAILURE, CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL, } cbm_mcp_supervised_result_disposition_t; diff --git a/src/mcp/mcp_internal.h b/src/mcp/mcp_internal.h index 3f5532a0b..a2a6ac5cf 100644 --- a/src/mcp/mcp_internal.h +++ b/src/mcp/mcp_internal.h @@ -32,6 +32,10 @@ bool cbm_mcp_jsonrpc_response_prepend_notice(char **response_io, const char *not * must remove any untrusted field with the same name before invoking this. */ bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, const cbm_index_resource_policy_t *policy); +bool cbm_mcp_index_policy_from_internal_args(const char *args, cbm_index_resource_policy_t *policy, + char *error, size_t error_size); +char *cbm_mcp_index_worker_resource_response(const char *args, + const cbm_index_worker_result_t *worker_result); enum { CBM_MCP_DEFAULT_AUTO_INDEX_LIMIT = 50000 }; diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index e23e7a86e..cad1d9c14 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -430,7 +430,8 @@ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p) { } const cbm_index_resource_policy_t *cbm_pipeline_resource_policy(const cbm_pipeline_t *p) { - return p && cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL; + return p && cbm_index_policy_discovery_enabled(&p->resource_policy) ? &p->resource_policy + : NULL; } cbm_index_resource_violation_t *cbm_pipeline_resource_violation(cbm_pipeline_t *p) { @@ -2210,7 +2211,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { .ignore_file = NULL, .max_file_size = 0, .resource_policy = - cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL, + cbm_index_policy_discovery_enabled(&p->resource_policy) ? &p->resource_policy : NULL, .resource_violation = &p->resource_violation, }; cbm_file_info_t *files = NULL; diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index 30bf0a430..5518c565f 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1324,6 +1324,8 @@ typedef struct { cbm_project_lock_manager_t *project_locks; bool project_lock_busy_is_error; cbm_proc_outcome_t outcomes[APP_FAKE_MAX_ATTEMPTS]; + cbm_index_resource_violation_t resource_violations[APP_FAKE_MAX_ATTEMPTS]; + bool resource_probe_failures[APP_FAKE_MAX_ATTEMPTS]; const char *responses[APP_FAKE_MAX_ATTEMPTS]; const char *marker_payloads[APP_FAKE_MAX_ATTEMPTS]; char marker_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; @@ -1428,7 +1430,9 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem static bool app_wait_for_atomic_bool(atomic_bool *value, bool expected); static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, int attempt, - const char *max_files, const char *max_source_mb) { + const char *max_files, const char *max_source_mb, + const char *max_rss_mb, + const char *max_duration_seconds) { if (!context || attempt < 0 || attempt >= APP_FAKE_MAX_ATTEMPTS) { return false; } @@ -1445,9 +1449,18 @@ static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, in yyjson_val *bytes = policy && yyjson_is_obj(policy) ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB) : NULL; - bool equal = files && bytes && yyjson_is_str(files) && yyjson_is_str(bytes) && + yyjson_val *rss = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_RSS_MB) + : NULL; + yyjson_val *duration = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS) + : NULL; + bool equal = files && bytes && rss && duration && yyjson_is_str(files) && + yyjson_is_str(bytes) && yyjson_is_str(rss) && yyjson_is_str(duration) && strcmp(yyjson_get_str(files), max_files) == 0 && - strcmp(yyjson_get_str(bytes), max_source_mb) == 0; + strcmp(yyjson_get_str(bytes), max_source_mb) == 0 && + strcmp(yyjson_get_str(rss), max_rss_mb) == 0 && + strcmp(yyjson_get_str(duration), max_duration_seconds) == 0; yyjson_doc_free(document); return equal; } @@ -1489,6 +1502,12 @@ static cbm_index_worker_poll_t app_fake_worker_poll(void *opaque, worker->result.outcome = outcome; worker->result.exit_code = outcome == CBM_PROC_CLEAN ? 0 : -1; worker->result.tree_quiesced = true; + if (worker->attempt < APP_FAKE_MAX_ATTEMPTS) { + worker->result.resource_violation = + worker->context->resource_violations[worker->attempt]; + worker->result.resource_probe_failed = + worker->context->resource_probe_failures[worker->attempt]; + } const char *response = worker->attempt < APP_FAKE_MAX_ATTEMPTS && worker->context->responses[worker->attempt] ? worker->context->responses[worker->attempt] @@ -2003,11 +2022,13 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { bool dirs_ok = cbm_mkdtemp(root) != NULL && cbm_mkdtemp(cache) != NULL; bool cache_set = dirs_ok && cache_saved && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0; cbm_config_t *stored_config = cache_set ? cbm_config_open(cache) : NULL; - bool config_ready = stored_config && - cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && - cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "3") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "4") == 0; + bool config_ready = + stored_config && cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && + cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "3") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "4") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7") == 0; char canonical_root[APP_TEST_PATH_CAP] = {0}; bool canonical = dirs_ok && cbm_canonical_path(root, canonical_root, sizeof(canonical_root)); char *project = canonical ? cbm_project_name_from_path(canonical_root) : NULL; @@ -2052,7 +2073,8 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { bool first_owned = first_initialized && project && app_wait_for_subscribers(application, project, 1) && app_wait_for_atomic_int(&fake.starts, 1); - bool auto_policy_propagated = first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4"); + bool auto_policy_propagated = + first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4", "64", "7"); bool second_initialized = app_test_initialize_profile(&callbacks, sessions[1], root, CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); bool coalesced = first_owned && second_initialized && project && @@ -2135,9 +2157,11 @@ TEST(daemon_application_programmatic_index_injects_resource_policy) { ASSERT_NOT_NULL(root); ASSERT_NOT_NULL(cache); cbm_config_t *stored_config = cbm_config_open(cache); - bool configured = stored_config && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0; + bool configured = + stored_config && cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7") == 0; app_fake_worker_context_t fake; app_fake_worker_context_init(&fake); @@ -2157,7 +2181,8 @@ TEST(daemon_application_programmatic_index_injects_resource_policy) { cbm_daemon_application_t *application = configured ? cbm_daemon_application_new(&config) : NULL; int index_rc = application ? cbm_daemon_application_index(application, "policy-programmatic", root) : -1; - bool policy_propagated = index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6"); + bool policy_propagated = + index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6", "64", "7"); bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); cbm_daemon_application_free(application); @@ -2170,6 +2195,86 @@ TEST(daemon_application_programmatic_index_injects_resource_policy) { PASS(); } +TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { + char *cache = th_mktempdir("cbm_app_worker_resource_cache"); + cbm_config_t *stored_config = cache ? cbm_config_open(cache) : NULL; + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + atomic_store(&fake.scripted, true); + fake.outcomes[0] = CBM_PROC_KILLED; + fake.resource_violations[0] = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_DURATION_MS, .observed = 7001, .limit = 7000}; + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = { + .config = stored_config, + .worker_ops = &worker_ops, + }; + cbm_daemon_application_t *application = + stored_config ? cbm_daemon_application_new(&config) : NULL; + cbm_daemon_runtime_application_callbacks_t callbacks = + cbm_daemon_application_runtime_callbacks(application); + cbm_daemon_runtime_application_session_t *session = app_test_open(&callbacks, 4188); + char *root = th_mktempdir("cbm_app_worker_resource"); + uint8_t *context = NULL; + uint32_t context_length = 0; + uint8_t *tool = NULL; + uint32_t tool_length = 0; + char args[APP_TEST_PATH_CAP + 96]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"DaemonWorkerResourceFixture\"}", + root ? root : ""); + bool setup = cache && stored_config && application && session && root && + app_test_context_request(root, root, &context, &context_length) && + app_test_tool_request("index_repository", args, &tool, &tool_length); + uint8_t *response = NULL; + uint32_t response_length = 0; + if (setup) { + uint8_t *context_response = NULL; + uint32_t context_response_length = 0; + setup = app_test_request(&callbacks, session, context, context_length, &context_response, + &context_response_length) == CBM_DAEMON_RUNTIME_APPLICATION_OK; + free(context_response); + } + cbm_daemon_runtime_application_status_t status = + setup + ? app_test_request(&callbacks, session, tool, tool_length, &response, &response_length) + : CBM_DAEMON_RUNTIME_APPLICATION_TRANSPORT_ERROR; + bool structured = response && strstr((char *)response, "resource_limit_exceeded") && + strstr((char *)response, "\\\"stage\\\":\\\"worker\\\"") && + strstr((char *)response, "\\\"resource\\\":\\\"duration_ms\\\"") && + strstr((char *)response, "\\\"observed\\\":7001") && + strstr((char *)response, "\\\"limit\\\":7000") && + strstr((char *)response, "\\\"unit\\\":\\\"milliseconds\\\""); + if (session) { + callbacks.session_close(callbacks.context, session); + } + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + int starts = atomic_load(&fake.starts); + int destroys = atomic_load(&fake.destroys); + cbm_daemon_application_free(application); + cbm_config_close(stored_config); + free(response); + free(context); + free(tool); + th_cleanup(root); + th_cleanup(cache); + + ASSERT_TRUE(setup); + ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_OK); + ASSERT_TRUE(structured); + ASSERT_EQ(starts, 1); + ASSERT_EQ(destroys, 1); + ASSERT_TRUE(stopped); + PASS(); +} + TEST(daemon_application_auto_index_honors_tracked_file_limit) { app_env_backup_t cache_environment; bool cache_saved = app_env_backup_capture(&cache_environment, "CBM_CACHE_DIR"); @@ -5179,6 +5284,7 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_prune_clears_logical_watch_for_reregistration); RUN_TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions); RUN_TEST(daemon_application_programmatic_index_injects_resource_policy); + RUN_TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried); RUN_TEST(daemon_application_auto_index_honors_tracked_file_limit); RUN_TEST(daemon_application_auto_index_file_count_handles_literal_metacharacter_path); RUN_TEST(daemon_application_auto_index_file_count_supports_non_git_roots); diff --git a/tests/test_index_policy.c b/tests/test_index_policy.c index d866a0631..b33d64cdb 100644 --- a/tests/test_index_policy.c +++ b/tests/test_index_policy.c @@ -21,9 +21,15 @@ TEST(index_policy_defaults_are_disabled) { ASSERT_FALSE(policy.max_files.enabled); ASSERT_FALSE(policy.max_source_bytes.enabled); + ASSERT_FALSE(policy.max_rss_bytes.enabled); + ASSERT_FALSE(policy.max_duration_ms.enabled); ASSERT_FALSE(cbm_index_policy_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_discovery_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_worker_enabled(&policy)); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_FILES), "off"); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_SOURCE_MB), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_RSS_MB), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_DURATION_SECONDS), "off"); PASS(); } @@ -60,6 +66,38 @@ TEST(index_policy_source_limit_converts_mib_without_overflow) { PASS(); } +TEST(index_policy_worker_limits_validate_and_convert_units) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "64", error, sizeof(error))); + ASSERT_TRUE(cbm_index_policy_enabled(&policy)); + ASSERT_TRUE(cbm_index_policy_worker_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_discovery_enabled(&policy)); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(64) * CBM_INDEX_MIB_BYTES); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "1048576", error, + sizeof(error))); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(1048576) * CBM_INDEX_MIB_BYTES); + cbm_index_resource_policy_t before = policy; + ASSERT_FALSE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "63", error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "1", error, + sizeof(error))); + ASSERT_EQ(policy.max_duration_ms.value, 1000); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "86400", error, + sizeof(error))); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(86400000)); + before = policy; + ASSERT_FALSE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "86401", + error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + PASS(); +} + TEST(index_policy_invalid_value_is_rejected_atomically) { static const char *const invalid[] = {"", "0", "-1", "1MB", "1 ", "+1", "10000001", "18446744073709551616"}; @@ -95,6 +133,16 @@ TEST(index_policy_format_round_trips_public_values) { ASSERT_TRUE( cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, value, sizeof(value))); ASSERT_STR_EQ(value, "7"); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "64", error, sizeof(error))); + ASSERT_TRUE( + cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, value, sizeof(value))); + ASSERT_STR_EQ(value, "64"); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "9", error, + sizeof(error))); + ASSERT_TRUE(cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, value, + sizeof(value))); + ASSERT_STR_EQ(value, "9"); PASS(); } @@ -103,10 +151,18 @@ TEST(index_policy_violation_metadata_is_stable) { ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_SOURCE_BYTES), "source_bytes"); ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_FILES), "files"); ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_SOURCE_BYTES), "bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_RSS_BYTES), "rss_bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_DURATION_MS), "duration_ms"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_RSS_BYTES), "bytes"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_DURATION_MS), "milliseconds"); ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_FILES), CBM_INDEX_CONFIG_MAX_FILES); ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_SOURCE_BYTES), CBM_INDEX_CONFIG_MAX_SOURCE_MB); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_RSS_BYTES), + CBM_INDEX_CONFIG_MAX_RSS_MB); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_DURATION_MS), + CBM_INDEX_CONFIG_MAX_DURATION_SECONDS); PASS(); } @@ -122,9 +178,13 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { ASSERT_FALSE(cbm_index_policy_enabled(&policy)); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "9"), 0); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "3"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7"), 0); ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); ASSERT_EQ(policy.max_files.value, 9); ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(3) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(64) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(7000)); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "corrupt"), 0); ASSERT_FALSE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); @@ -135,16 +195,23 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { PASS(); } -TEST(index_policy_cli_lists_both_operator_keys) { +TEST(index_policy_cli_lists_all_operator_keys) { bool files_found = false; bool bytes_found = false; + bool rss_found = false; + bool duration_found = false; for (size_t index = 0; index < cbm_cli_config_key_count_for_testing(); index++) { const char *key = cbm_cli_config_key_at_for_testing(index); files_found = files_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0); bytes_found = bytes_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); + rss_found = rss_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_RSS_MB) == 0); + duration_found = + duration_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS) == 0); } ASSERT_TRUE(files_found); ASSERT_TRUE(bytes_found); + ASSERT_TRUE(rss_found); + ASSERT_TRUE(duration_found); PASS(); } @@ -391,11 +458,12 @@ SUITE(index_policy) { RUN_TEST(index_policy_defaults_are_disabled); RUN_TEST(index_policy_file_limit_accepts_off_and_exact_range); RUN_TEST(index_policy_source_limit_converts_mib_without_overflow); + RUN_TEST(index_policy_worker_limits_validate_and_convert_units); RUN_TEST(index_policy_invalid_value_is_rejected_atomically); RUN_TEST(index_policy_format_round_trips_public_values); RUN_TEST(index_policy_violation_metadata_is_stable); RUN_TEST(index_policy_config_loads_defaults_values_and_rejects_corruption); - RUN_TEST(index_policy_cli_lists_both_operator_keys); + RUN_TEST(index_policy_cli_lists_all_operator_keys); RUN_TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite); RUN_TEST(index_policy_cli_set_reports_a_failed_write); RUN_TEST(index_policy_worker_rejects_missing_parent_policy); diff --git a/tests/test_index_supervisor.c b/tests/test_index_supervisor.c index c67f61fc4..b061f0dd1 100644 --- a/tests/test_index_supervisor.c +++ b/tests/test_index_supervisor.c @@ -849,6 +849,269 @@ TEST(index_supervisor_killed_worker_log_is_never_empty_and_names_the_run) { PASS(); } +typedef struct { + uint64_t now_ms; + uint64_t rss_values[4]; + int rss_value_count; + int rss_calls; + int clock_calls; + cbm_proc_tree_rss_status_t rss_status; +} index_supervisor_resource_fake_t; + +static uint64_t index_supervisor_fake_clock(void *context) { + index_supervisor_resource_fake_t *fake = context; + fake->clock_calls++; + return fake->now_ms; +} + +static cbm_proc_tree_rss_status_t index_supervisor_fake_rss(cbm_subprocess_t *process, + uint64_t *rss_bytes, void *context) { + (void)process; + index_supervisor_resource_fake_t *fake = context; + int value_index = + fake->rss_calls < fake->rss_value_count ? fake->rss_calls : fake->rss_value_count - 1; + fake->rss_calls++; + if (fake->rss_status == CBM_PROC_TREE_RSS_OK && rss_bytes && value_index >= 0) { + *rss_bytes = fake->rss_values[value_index]; + } + return fake->rss_status; +} + +static cbm_index_resource_policy_t index_supervisor_test_worker_policy(uint64_t rss_bytes, + uint64_t duration_ms) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_rss_bytes = (cbm_index_limit_u64_t){.enabled = rss_bytes > 0, .value = rss_bytes}; + policy.max_duration_ms = + (cbm_index_limit_u64_t){.enabled = duration_ms > 0, .value = duration_ms}; + return policy; +} + +TEST(index_supervisor_disabled_limits_do_not_probe) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {UINT64_MAX}, + .rss_value_count = 1, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"clean\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + cbm_index_resource_t resource = + result ? result->resource_violation.resource : CBM_INDEX_RESOURCE_RSS_BYTES; + bool probe_failed = result && result->resource_probe_failed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(terminal); + ASSERT_EQ(fake.rss_calls, 0); + ASSERT_EQ(fake.clock_calls, 0); + ASSERT_EQ(resource, CBM_INDEX_RESOURCE_NONE); + ASSERT_FALSE(probe_failed); + PASS(); +} + +TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {limit, limit + 1}, + .rss_value_count = 2, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t equal_state = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms += 250; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + const cbm_index_worker_result_t *cached = NULL; + bool cached_terminal = + terminal && cbm_index_worker_poll(handle, &cached) == CBM_INDEX_WORKER_POLL_TERMINAL && + cached == result; + bool limited = terminal && result && + result->resource_violation.resource == CBM_INDEX_RESOURCE_RSS_BYTES && + result->resource_violation.observed == limit + 1 && + result->resource_violation.limit == limit && !result->cancellation_requested && + result->tree_quiesced && !result->supervision_failed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(equal_state, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(limited); + ASSERT_TRUE(cached_terminal); + PASS(); +} + +TEST(index_supervisor_duration_is_total_time_not_quiet_timeout) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_EMPTY, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + fake.now_ms = 1100; + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t equal_state = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms = 1101; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool limited = terminal && result && + result->resource_violation.resource == CBM_INDEX_RESOURCE_DURATION_MS && + result->resource_violation.observed == 1001 && + result->resource_violation.limit == 1000 && result->outcome != CBM_PROC_HANG && + result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(equal_state, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(limited); + PASS(); +} + +TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled) { + const char *saved_timeout = getenv("CBM_INDEX_WORKER_TIMEOUT_S"); + char *saved_timeout_copy = saved_timeout ? cbm_strdup(saved_timeout) : NULL; + (void)cbm_setenv("CBM_INDEX_WORKER_TIMEOUT_S", "1", 1); + cbm_index_supervisor_reset_resource_hooks_for_testing(); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 60000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool remained_hang = terminal && result && result->outcome == CBM_PROC_HANG && + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + index_supervisor_test_restore_env("CBM_INDEX_WORKER_TIMEOUT_S", saved_timeout_copy); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(terminal); + ASSERT_TRUE(remained_hang); + PASS(); +} + +TEST(index_supervisor_cancel_precedes_resource_probe) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {limit + 1}, + .rss_value_count = 1, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 1000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + fake.now_ms = 2000; + bool cancel_accepted = handle && cbm_index_worker_request_cancel(handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool cancelled = terminal && result && result->cancellation_requested && + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed && result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(cancel_accepted); + ASSERT_TRUE(cancelled); + ASSERT_EQ(fake.rss_calls, 0); + PASS(); +} + +TEST(index_supervisor_three_failed_rss_probes_fail_closed) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_ERROR, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t first = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + cbm_index_worker_poll_t throttled = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + int calls_before_interval = fake.rss_calls; + fake.now_ms += 250; + cbm_index_worker_poll_t second = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms += 250; + cbm_index_worker_poll_t third = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool failed_closed = terminal && result && result->resource_probe_failed && + result->resource_violation.resource == CBM_INDEX_RESOURCE_RSS_BYTES && + !result->cancellation_requested && result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(first, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(throttled, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(calls_before_interval, 1); + ASSERT_EQ(second, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(third, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(failed_closed); + ASSERT_EQ(fake.rss_calls, 3); + PASS(); +} + SUITE(index_supervisor) { RUN_TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar); RUN_TEST(index_supervisor_async_jobs_are_isolated_cancellable_and_terminal_cached); @@ -857,4 +1120,10 @@ SUITE(index_supervisor) { RUN_TEST(index_supervisor_drains_terminal_backlog_into_request_progress_callback); RUN_TEST(index_supervisor_oversized_response_is_contained_and_log_is_retained); RUN_TEST(index_supervisor_killed_worker_log_is_never_empty_and_names_the_run); + RUN_TEST(index_supervisor_disabled_limits_do_not_probe); + RUN_TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree); + RUN_TEST(index_supervisor_duration_is_total_time_not_quiet_timeout); + RUN_TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled); + RUN_TEST(index_supervisor_cancel_precedes_resource_probe); + RUN_TEST(index_supervisor_three_failed_rss_probes_fail_closed); } diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 278d33e68..37c0b9374 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -9601,6 +9601,17 @@ TEST(index_supervisor_unsafe_clean_is_never_fallback_or_recovery) { CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL); result.supervision_failed = false; + result.tree_quiesced = true; + result.outcome = CBM_PROC_KILLED; + result.resource_violation.resource = CBM_INDEX_RESOURCE_RSS_BYTES; + ASSERT_EQ(cbm_mcp_supervised_result_disposition(0, &result), + CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE); + result.tree_quiesced = false; + ASSERT_EQ(cbm_mcp_supervised_result_disposition(0, &result), + CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL); + + result.resource_violation = (cbm_index_resource_violation_t){0}; + result.tree_quiesced = true; result.outcome = CBM_PROC_CRASH; result.response = NULL; ASSERT_EQ(cbm_mcp_supervised_result_disposition(0, &result), @@ -9610,6 +9621,102 @@ TEST(index_supervisor_unsafe_clean_is_never_fallback_or_recovery) { PASS(); } +#ifndef _WIN32 +typedef struct { + int calls; +} index_worker_resource_clock_t; + +static uint64_t index_worker_resource_clock(void *context) { + index_worker_resource_clock_t *clock = context; + clock->calls++; + return clock->calls == 1 ? 100 : 1101; +} + +enum { + IDXRESOURCE_OK = 0, + IDXRESOURCE_SETUP = 71, + IDXRESOURCE_NO_RESPONSE = 72, + IDXRESOURCE_BAD_CONTRACT = 73, + IDXRESOURCE_RETRIED = 74, +}; + +static int index_worker_resource_response_check(const char *repo, const char *cache) { + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + cbm_unsetenv("CBM_INDEX_SUPERVISOR"); + cbm_index_supervisor_mark_host(); + cbm_config_t *config = cbm_config_open(cache); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + if (!config || !server || + cbm_config_set(config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "1") != 0) { + cbm_mcp_server_free(server); + cbm_config_close(config); + return IDXRESOURCE_SETUP; + } + cbm_mcp_server_set_config(server, config); + index_worker_resource_clock_t clock = {0}; + cbm_index_supervisor_set_resource_hooks_for_testing(index_worker_resource_clock, NULL, &clock); + int before = cbm_index_supervisor_spawn_count(); + char args[CBM_SZ_2K]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"mode\":\"fast\"," + "\"_cbm_index_policy\":{\"index_max_files\":\"off\"," + "\"index_max_source_mb\":\"off\",\"index_max_rss_mb\":\"off\"," + "\"index_max_duration_seconds\":\"off\"}}", + repo); + char *response = cbm_mcp_handle_tool(server, "index_repository", args); + int after = cbm_index_supervisor_spawn_count(); + cbm_index_supervisor_reset_resource_hooks_for_testing(); + int result = IDXRESOURCE_OK; + if (!response) { + result = IDXRESOURCE_NO_RESPONSE; + } else if (!strstr(response, "resource_limit_exceeded") || + !strstr(response, "\\\"stage\\\":\\\"worker\\\"") || + !strstr(response, "\\\"resource\\\":\\\"duration_ms\\\"") || + !strstr(response, "\\\"observed\\\":1001") || + !strstr(response, "\\\"limit\\\":1000") || + !strstr(response, "\\\"unit\\\":\\\"milliseconds\\\"")) { + result = IDXRESOURCE_BAD_CONTRACT; + } else if (after - before != 1) { + result = IDXRESOURCE_RETRIED; + } + free(response); + cbm_mcp_server_free(server); + cbm_config_close(config); + return result; +} +#endif + +TEST(index_worker_resource_limit_is_trusted_structured_and_not_retried) { +#ifdef _WIN32 + SKIP_PLATFORM("fork-isolated MCP host harness; supervisor state machine is cross-platform"); +#else + char *repo = th_mktempdir("cbm_worker_resource_repo"); + char *cache = th_mktempdir("cbm_worker_resource_cache"); + ASSERT_NOT_NULL(repo); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int main(void) { return 0; }\n"), 0); + fflush(NULL); + pid_t pid = fork(); + if (pid == 0) { + alarm(60); + _exit(index_worker_resource_response_check(repo, cache)); + } + int status = 0; + bool waited = waitpid(pid, &status, 0) == pid; + int exit_code = waited && WIFEXITED(status) ? WEXITSTATUS(status) : -1; + int signal_code = waited && WIFSIGNALED(status) ? WTERMSIG(status) : 0; + th_cleanup(repo); + th_cleanup(cache); + if (exit_code != IDXRESOURCE_OK) { + printf(" worker resource child exit=%d signal=%d\n", exit_code, signal_code); + } + ASSERT_TRUE(waited); + ASSERT_EQ(signal_code, 0); + ASSERT_EQ(exit_code, IDXRESOURCE_OK); + PASS(); +#endif +} + /* Child-side check: index a tiny fixture and verify it ran IN-PROCESS. * Distinct exit codes so the parent can report the exact failure mode. */ enum { @@ -11401,6 +11508,7 @@ SUITE(mcp) { RUN_TEST(index_repository_supervisor_uses_canonical_session_path); RUN_TEST(index_repository_cli_name_override_issue823); RUN_TEST(index_supervisor_unsafe_clean_is_never_fallback_or_recovery); + RUN_TEST(index_worker_resource_limit_is_trusted_structured_and_not_retried); RUN_TEST(index_supervisor_gate_requires_marked_host_issue845); RUN_TEST(index_supervisor_start_failure_is_fail_closed_in_real_host); RUN_TEST(index_bg_paths_route_through_supervisor_issue832); diff --git a/tests/test_subprocess.c b/tests/test_subprocess.c index 30fd1537d..82caff354 100644 --- a/tests/test_subprocess.c +++ b/tests/test_subprocess.c @@ -82,6 +82,12 @@ TEST(subprocess_outcome_str) { PASS(); } +TEST(subprocess_tree_rss_sum_saturates_on_overflow) { + const uint64_t values[] = {UINT64_MAX - 4, 4, 1}; + ASSERT_EQ(cbm_subprocess_rss_sum_for_testing(values, 3), UINT64_MAX); + PASS(); +} + /* ── Layer 2: real spawn/reap (POSIX) ─────────────────────────────────────── */ #ifndef _WIN32 @@ -430,6 +436,45 @@ TEST(subprocess_spawn_returns_while_child_is_running) { #endif } +TEST(subprocess_tree_rss_measures_contained_descendants) { +#ifdef _WIN32 + SKIP_PLATFORM("native Job Object process-tree RSS runs on Windows CI"); +#else + char pid_path[64]; + ASSERT_TRUE(make_tree_pid_path(pid_path)); + cbm_subprocess_t *process = NULL; + ASSERT_EQ(spawn_ignoring_tree(pid_path, 0, 100, &process), 0); + pid_t parent_pid = 0; + pid_t grandchild_pid = 0; + bool ready = wait_for_tree_pids(pid_path, process, &parent_pid, &grandchild_pid, 3000); + uint64_t rss_bytes = 0; + cbm_proc_tree_rss_status_t rss_status = + ready ? cbm_subprocess_tree_rss_bytes(process, &rss_bytes) : CBM_PROC_TREE_RSS_ERROR; + bool cancel_accepted = cbm_subprocess_request_cancel(process); + cbm_proc_result_t result; + bool terminal = poll_until_terminal(process, 5000, &result); + uint64_t final_rss = UINT64_MAX; + cbm_proc_tree_rss_status_t final_rss_status = + terminal ? cbm_subprocess_tree_rss_bytes(process, &final_rss) : CBM_PROC_TREE_RSS_ERROR; + if (terminal) { + cbm_subprocess_destroy(process); + } else { + force_probe_cleanup(parent_pid, grandchild_pid); + } + (void)unlink(pid_path); + + ASSERT_TRUE(ready); + ASSERT_EQ(rss_status, CBM_PROC_TREE_RSS_OK); + ASSERT_TRUE(rss_bytes > 0); + ASSERT_TRUE(cancel_accepted); + ASSERT_TRUE(terminal); + ASSERT_TRUE(result.tree_quiesced); + ASSERT_EQ(final_rss_status, CBM_PROC_TREE_RSS_EMPTY); + ASSERT_EQ(final_rss, 0); + PASS(); +#endif +} + TEST(subprocess_natural_completion_is_cached_across_polls) { #ifdef _WIN32 SKIP_PLATFORM("POSIX /bin/sh completion-cache probe; native Job Object coverage pending"); @@ -1027,6 +1072,7 @@ SUITE(subprocess) { RUN_TEST(subprocess_classify_non_fault_signal_is_killed); RUN_TEST(subprocess_classify_timeout_dominates); RUN_TEST(subprocess_outcome_str); + RUN_TEST(subprocess_tree_rss_sum_saturates_on_overflow); RUN_TEST(subprocess_run_clean); RUN_TEST(subprocess_run_exit_nonzero); RUN_TEST(subprocess_run_resolves_literal_binary_name_from_path); @@ -1037,6 +1083,7 @@ SUITE(subprocess) { RUN_TEST(subprocess_run_spawn_failure); RUN_TEST(subprocess_run_null_bin_rejected); RUN_TEST(subprocess_spawn_returns_while_child_is_running); + RUN_TEST(subprocess_tree_rss_measures_contained_descendants); RUN_TEST(subprocess_natural_completion_is_cached_across_polls); RUN_TEST(subprocess_cancel_is_idempotent_and_kills_ignoring_tree); RUN_TEST(subprocess_quiet_timeout_kills_ignoring_tree); From cf862a0a48f92f94d7df295c4f7379b39f224d40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Wed, 19 Aug 2026 14:39:38 +0800 Subject: [PATCH 3/6] feat(index): enforce opt-in storage resource limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An index that fits in memory and finishes in time can still fill the disk. Publication needs room for the staging artifacts and the final database at the same time, and running out of space during publication is the one failure that can cost a working index. Add index_cache_max_mb and index_min_free_disk_mb, measured before staging and again before publication, together with internal ceilings on the final database, the staging artifacts and the task temporary directory. Those three have no public keys because they are only meaningful as part of one composed decision. Crossing any of them fails the attempt before the old database is touched, so atomic publication is unchanged and the previous index keeps serving. Staging cleanup is scoped by a private per-task token, so a worker removes only the artifacts it created and never a concurrent run's. An old database is treated as replaceable only after an integrity verdict distinguishes real corruption from a transient busy error. A probe that cannot complete fails closed. Both public keys default to off. The shell fixture that stands in for the supervisor names them too, so the worker still recognises the policy it is handed. Signed-off-by: 刘冲 --- README.md | 10 +- docs/CONFIGURATION.md | 19 +- docs/INDEX_RESOURCE_LIMITS.md | 29 ++- scripts/test-runtime.sh | 3 +- src/cli/cli.c | 3 + src/daemon/application.c | 13 +- src/daemon/bootstrap.c | 21 ++ src/foundation/compat_fs.c | 346 ++++++++++++++++++++++++++++++- src/foundation/compat_fs.h | 26 +++ src/foundation/index_policy.c | 92 ++++++-- src/foundation/index_policy.h | 32 +++ src/main.c | 6 +- src/mcp/index_supervisor.c | 173 ++++++++++++++-- src/mcp/index_supervisor.h | 21 +- src/mcp/mcp.c | 82 ++++++-- src/mcp/mcp_internal.h | 1 + src/pipeline/pipeline.c | 282 ++++++++++++++++++++++++- src/pipeline/pipeline.h | 15 ++ src/pipeline/pipeline_internal.h | 3 + tests/test_daemon_application.c | 62 +++++- tests/test_index_policy.c | 172 +++++++++++++++ tests/test_index_supervisor.c | 77 ++++++- tests/test_main.c | 2 +- tests/test_mcp.c | 77 ++++++- tests/test_pipeline.c | 209 +++++++++++++++++++ tests/test_platform.c | 93 +++++++++ 26 files changed, 1770 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 8c249b087..93312dca0 100644 --- a/README.md +++ b/README.md @@ -671,12 +671,16 @@ codebase-memory-mcp config set index_max_files 250000 # optional per-index so codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit codebase-memory-mcp config set index_max_rss_mb 8192 # optional worker-tree current RSS limit codebase-memory-mcp config set index_max_duration_seconds 3600 # optional total worker duration +codebase-memory-mcp config set index_cache_max_mb 32768 # optional projected cache-size limit +codebase-memory-mcp config set index_min_free_disk_mb 4096 # optional free-space reserve codebase-memory-mcp config reset auto_index # reset to default ``` -The four `index_max_*` settings default to `off`. Exceeding one fails the complete -index attempt rather than publishing a partial graph; an existing serving index -is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). +The six index resource settings default to `off`. Exceeding one fails the +complete index attempt rather than publishing a partial graph; an existing +serving index is preserved. Storage probes also fail closed when an enabled +measurement cannot be completed. See +[Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). ### Environment Variables diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index c3bdf8d96..6538ce3c0 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -90,14 +90,17 @@ Current keys: | `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | | `index_max_rss_mb` | `off` | Optional maximum current RSS in MiB for the complete contained index-worker process tree (`64..1048576`). | | `index_max_duration_seconds` | `off` | Optional maximum total worker duration in seconds (`1..86400`). | - -The four `index_max_*` settings are independent and disabled by default. They -apply to explicit indexing, automatic indexing, and watcher re-indexing, but not -to `cross-repo-intelligence`, which does not scan repository source files. -Equality is allowed; exceeding any setting fails the complete index request and -preserves any previously serving database. Worker RSS covers descendants and is -not the same as the internal `CBM_MEM_BUDGET_MB` allocation budget. Total -duration is independent of the existing 15-minute no-log-progress timeout. See +| `index_cache_max_mb` | `off` | Optional maximum projected cache size in MiB after publication. | +| `index_min_free_disk_mb` | `off` | Optional minimum free MiB reserved on the cache filesystem while indexing. | + +The six index resource settings are independent and disabled by default. +They apply to explicit indexing, automatic indexing, and watcher re-indexing, +but not to `cross-repo-intelligence`, which does not scan or publish repository +source indexes. Equality is allowed; exceeding a setting fails the complete +index request and preserves any previously serving database. Worker RSS covers +descendants and is not the same as the internal `CBM_MEM_BUDGET_MB` allocation +budget. Total duration is independent of the existing 15-minute no-log-progress +timeout. See [Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and error-response details. diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index 66be4ecf4..68fce77f8 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -94,6 +94,31 @@ Worker limit failures use the same shape with `stage=worker`, and omit `observed`, `limit`, and `unit` because no trustworthy observation was available. +## Storage settings + +| Key | Default | Accepted value | Protects | +|---|---:|---:|---| +| `index_cache_max_mb` | `off` | `off` or `1..1048576` | Projected cache bytes after replacement | +| `index_min_free_disk_mb` | `off` | `off` or `1..1048576` | Free bytes reserved on the cache filesystem | + +Set or reset these keys through the same `config set` and `config reset` +commands. With both keys `off`, indexing does not scan the cache tree or probe +filesystem capacity. + +Projected cache usage is the current cache size, minus the old project +database and SQLite sidecars only when that generation is confirmed valid and +replaceable, plus the current operation's staging artifacts. Other projects +and unrelated files always count toward the limit and are never evicted. + +Free space is checked before staging, after the staged build completes, and +immediately before atomic publication. Equality is allowed. An enabled +measurement that cannot be completed fails closed with +`code: "resource_probe_failed"` and +`stage: "storage"`. A limit breach uses `code: "resource_limit_exceeded"`. +Both cases preserve the old serving database. Staging files created by a +terminated supervised worker are tagged with a private task token and removed +after its process tree is quiescent; cleanup cannot match another attempt. + ## Trust and compatibility Limits are read from the CLI-managed `_config.db`; they are not MCP request @@ -104,5 +129,5 @@ parent policy. These settings do not replace or increase `auto_index_limit`, change the 512 MiB single-file cap, alter workspace-root authorization, or affect `cross-repo-intelligence`. With all settings `off`, discovery follows the -existing path and the supervisor performs no periodic RSS probe or total-duration -termination. +existing path, the supervisor performs no periodic RSS probe or total-duration +termination, and the pipeline performs no cache-tree or free-space probe. diff --git a/scripts/test-runtime.sh b/scripts/test-runtime.sh index e6e60b15a..b7e3b885a 100644 --- a/scripts/test-runtime.sh +++ b/scripts/test-runtime.sh @@ -113,7 +113,8 @@ _cbm_test_runtime_daemon() { # nothing else. cbm_test_index_worker_policy_json() { printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"' - printf '%s' ',"index_max_rss_mb":"off","index_max_duration_seconds":"off"}' + printf '%s' ',"index_max_rss_mb":"off","index_max_duration_seconds":"off"' + printf '%s' ',"index_cache_max_mb":"off","index_min_free_disk_mb":"off"}' } cbm_test_runtime_cleanup() { diff --git a/src/cli/cli.c b/src/cli/cli.c index 8c52bb5eb..9ae0a819c 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -6779,6 +6779,9 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"}, {CBM_INDEX_CONFIG_MAX_RSS_MB, "off", "Max worker process-tree RSS MiB, or off"}, {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "off", "Max worker duration in seconds, or off"}, + {CBM_INDEX_CONFIG_CACHE_MAX_MB, "off", "Max projected cache MiB after publish, or off"}, + {CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, "off", + "Minimum free cache-filesystem MiB during indexing, or off"}, }; /* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing diff --git a/src/daemon/application.c b/src/daemon/application.c index dce9468f2..3dc14161b 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -306,16 +306,19 @@ static int application_worker_start_default(void *context, const char *args_json (void)context; cbm_index_resource_policy_t resource_policy; char error[CBM_SZ_256] = {0}; + char task_db_path[CBM_SZ_1K]; if (!cbm_mcp_index_policy_from_internal_args(args_json, &resource_policy, error, - sizeof(error))) { - cbm_log_error("daemon.index.policy", "error", error); + sizeof(error)) || + !cbm_mcp_index_task_db_path(args_json, task_db_path, sizeof(task_db_path))) { + cbm_log_error("daemon.index.worker_policy", "error", + error[0] ? error : "could not resolve worker index path"); *worker_out = NULL; return -1; } cbm_index_worker_handle_t *worker = NULL; - int result = - cbm_index_worker_start_with_policy(args_json, memory_budget_bytes, &resource_policy, false, - marker_file, quarantine_file, &worker); + int result = cbm_index_worker_start_with_storage_policy( + args_json, memory_budget_bytes, &resource_policy, task_db_path, false, marker_file, + quarantine_file, NULL, NULL, &worker); *worker_out = worker; return result; } diff --git a/src/daemon/bootstrap.c b/src/daemon/bootstrap.c index ee61261d3..c3fecbf77 100644 --- a/src/daemon/bootstrap.c +++ b/src/daemon/bootstrap.c @@ -95,6 +95,21 @@ static bool bootstrap_worker_budget_valid(const char *text) { return value > 0; } +static bool bootstrap_worker_stage_token_valid(const char *token) { + size_t length = token ? strlen(token) : 0; + if (length < 6U || length >= 64U) { + return false; + } + for (size_t index = 0; index < length; index++) { + unsigned char ch = (unsigned char)token[index]; + if (!((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || + ch == '-' || ch == '_')) { + return false; + } + } + return true; +} + /* Keep the bootstrap role boundary exact and fail closed before any client or * worker state is initialized. index_supervisor owns the matching builder and * performs the captured-build comparison after this syntax-only classification. */ @@ -129,6 +144,12 @@ static bool bootstrap_worker_argv_exact(int argc, char *const argv[]) { } next += 2; } + if (next < argc && bootstrap_arg_is(argv[next], "--index-worker-stage-token")) { + if (next + 1 >= argc || !bootstrap_worker_stage_token_valid(argv[next + 1])) { + return false; + } + next += 2; + } return next == argc; } diff --git a/src/foundation/compat_fs.c b/src/foundation/compat_fs.c index 2610c421f..2b82156e0 100644 --- a/src/foundation/compat_fs.c +++ b/src/foundation/compat_fs.c @@ -121,19 +121,23 @@ cbm_dirent_t *cbm_readdir(cbm_dir_t *d) { return &d->entry; } -int cbm_path_info_utf8(const char *path, cbm_path_info_t *out) { +cbm_path_probe_status_t cbm_path_probe_info_utf8(const char *path, cbm_path_info_t *out) { if (!path || !out) { - return CBM_NOT_FOUND; + return CBM_PATH_PROBE_ERROR; } wchar_t *wpath = cbm_path_to_wide(path); if (!wpath) { - return CBM_NOT_FOUND; + return CBM_PATH_PROBE_ERROR; } WIN32_FILE_ATTRIBUTE_DATA data; BOOL ok = GetFileAttributesExW(wpath, GetFileExInfoStandard, &data); free(wpath); if (!ok) { - return CBM_NOT_FOUND; + DWORD error = GetLastError(); + return error == ERROR_FILE_NOT_FOUND || error == ERROR_PATH_NOT_FOUND || + error == ERROR_INVALID_NAME + ? CBM_PATH_PROBE_NOT_FOUND + : CBM_PATH_PROBE_ERROR; } memset(out, 0, sizeof(*out)); out->is_directory = (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; @@ -145,6 +149,9 @@ int cbm_path_info_utf8(const char *path, cbm_path_info_t *out) { * reports all four halves as assigned-but-never-read. This form says the * same thing without the union, so the checker needs no exception. */ uint64_t file_size = ((uint64_t)data.nFileSizeHigh << 32) | (uint64_t)data.nFileSizeLow; + if (file_size > INT64_MAX) { + return CBM_PATH_PROBE_ERROR; + } out->size = (int64_t)file_size; uint64_t written = ((uint64_t)data.ftLastWriteTime.dwHighDateTime << 32) | (uint64_t)data.ftLastWriteTime.dwLowDateTime; @@ -154,7 +161,29 @@ int cbm_path_info_utf8(const char *path, cbm_path_info_t *out) { written >= windows_to_unix_ticks ? (int64_t)((written - windows_to_unix_ticks) * NANOSECONDS_PER_WINDOWS_TICK) : 0; - return 0; + return CBM_PATH_PROBE_OK; +} + +int cbm_path_info_utf8(const char *path, cbm_path_info_t *out) { + return cbm_path_probe_info_utf8(path, out) == CBM_PATH_PROBE_OK ? 0 : CBM_NOT_FOUND; +} + +bool cbm_filesystem_free_bytes(const char *path, uint64_t *bytes_out) { + if (!path || !bytes_out) { + return false; + } + wchar_t *wpath = cbm_path_to_wide(path); + if (!wpath) { + return false; + } + ULARGE_INTEGER available; + BOOL ok = GetDiskFreeSpaceExW(wpath, &available, NULL, NULL); + free(wpath); + if (!ok) { + return false; + } + *bytes_out = available.QuadPart; + return true; } void cbm_closedir(cbm_dir_t *d) { @@ -728,6 +757,7 @@ int cbm_exec_no_shell(const char *const *argv) { #include #include #include +#include #include #include @@ -793,13 +823,14 @@ cbm_dirent_t *cbm_readdir(cbm_dir_t *d) { return NULL; } -int cbm_path_info_utf8(const char *path, cbm_path_info_t *out) { +cbm_path_probe_status_t cbm_path_probe_info_utf8(const char *path, cbm_path_info_t *out) { if (!path || !out) { - return CBM_NOT_FOUND; + return CBM_PATH_PROBE_ERROR; } struct stat state; if (lstat(path, &state) != 0) { - return CBM_NOT_FOUND; + return errno == ENOENT || errno == ENOTDIR ? CBM_PATH_PROBE_NOT_FOUND + : CBM_PATH_PROBE_ERROR; } memset(out, 0, sizeof(*out)); out->is_regular = S_ISREG(state.st_mode); @@ -813,7 +844,28 @@ int cbm_path_info_utf8(const char *path, cbm_path_info_t *out) { out->mtime_ns = ((int64_t)state.st_mtim.tv_sec * INT64_C(1000000000)) + (int64_t)state.st_mtim.tv_nsec; #endif - return 0; + return CBM_PATH_PROBE_OK; +} + +int cbm_path_info_utf8(const char *path, cbm_path_info_t *out) { + return cbm_path_probe_info_utf8(path, out) == CBM_PATH_PROBE_OK ? 0 : CBM_NOT_FOUND; +} + +bool cbm_filesystem_free_bytes(const char *path, uint64_t *bytes_out) { + if (!path || !bytes_out) { + return false; + } + struct statvfs status; + if (statvfs(path, &status) != 0) { + return false; + } + uint64_t blocks = (uint64_t)status.f_bavail; + uint64_t fragment = status.f_frsize != 0 ? (uint64_t)status.f_frsize : (uint64_t)status.f_bsize; + if (fragment == 0) { + return false; + } + *bytes_out = blocks > UINT64_MAX / fragment ? UINT64_MAX : blocks * fragment; + return true; } void cbm_closedir(cbm_dir_t *d) { @@ -953,6 +1005,282 @@ int cbm_exec_no_shell(const char *const *argv) { #endif /* _WIN32 */ +static uint64_t fs_add_saturated(uint64_t left, uint64_t right) { + return left > UINT64_MAX - right ? UINT64_MAX : left + right; +} + +static bool directory_size_recursive(const char *path, unsigned int depth, uint64_t *total) { + enum { CBM_DIRECTORY_SCAN_MAX_DEPTH = 1024 }; + if (!path || !total || depth > CBM_DIRECTORY_SCAN_MAX_DEPTH) { + return false; + } + cbm_dir_t *directory = cbm_opendir(path); + if (!directory) { + return false; + } + bool ok = true; + cbm_dirent_t *entry; + while (ok && (entry = cbm_readdir(directory)) != NULL) { + size_t path_len = strlen(path); + size_t name_len = strlen(entry->name); + bool separator_needed = + path_len > 0 && path[path_len - 1] != '/' && path[path_len - 1] != '\\'; + if (path_len > SIZE_MAX - name_len - (separator_needed ? 2U : 1U)) { + ok = false; + break; + } + size_t child_size = path_len + name_len + (separator_needed ? 2U : 1U); + char *child = (char *)malloc(child_size); + if (!child) { + ok = false; + break; + } + int written = + snprintf(child, child_size, separator_needed ? "%s/%s" : "%s%s", path, entry->name); + cbm_path_info_t info; + cbm_path_probe_status_t status = written > 0 && (size_t)written < child_size + ? cbm_path_probe_info_utf8(child, &info) + : CBM_PATH_PROBE_ERROR; + if (status == CBM_PATH_PROBE_ERROR) { + ok = false; + } else if (status == CBM_PATH_PROBE_OK && !info.is_symlink && info.is_regular) { + if (info.size < 0) { + ok = false; + } else { + *total = fs_add_saturated(*total, (uint64_t)info.size); + } + } else if (status == CBM_PATH_PROBE_OK && !info.is_symlink && info.is_directory) { + ok = directory_size_recursive(child, depth + 1U, total); + } + free(child); + } + cbm_closedir(directory); + return ok; +} + +bool cbm_directory_size_bytes(const char *path, uint64_t *bytes_out) { + if (!path || !bytes_out) { + return false; + } + *bytes_out = 0; + return directory_size_recursive(path, 0, bytes_out); +} + +bool cbm_db_artifact_bytes(const char *db_path, uint64_t *bytes_out) { + if (!db_path || !bytes_out) { + return false; + } + *bytes_out = 0; + static const char *const suffixes[] = {"", "-wal", "-shm", "-journal"}; + size_t base_len = strlen(db_path); + for (size_t index = 0; index < sizeof(suffixes) / sizeof(suffixes[0]); index++) { + size_t suffix_len = strlen(suffixes[index]); + if (base_len > SIZE_MAX - suffix_len - 1U) { + return false; + } + size_t path_size = base_len + suffix_len + 1U; + char *path = (char *)malloc(path_size); + if (!path) { + return false; + } + (void)snprintf(path, path_size, "%s%s", db_path, suffixes[index]); + cbm_path_info_t info; + cbm_path_probe_status_t status = cbm_path_probe_info_utf8(path, &info); + free(path); + if (status == CBM_PATH_PROBE_NOT_FOUND) { + continue; + } + if (status != CBM_PATH_PROBE_OK || !info.is_regular || info.is_symlink || info.size < 0) { + return false; + } + *bytes_out = fs_add_saturated(*bytes_out, (uint64_t)info.size); + } + return true; +} + +static bool fs_optional_regular_bytes(const char *path, uint64_t *total) { + if (!path || !path[0]) { + return true; + } + cbm_path_info_t info; + cbm_path_probe_status_t status = cbm_path_probe_info_utf8(path, &info); + if (status == CBM_PATH_PROBE_NOT_FOUND) { + return true; + } + if (status != CBM_PATH_PROBE_OK || !info.is_regular || info.is_symlink || info.size < 0) { + return false; + } + *total = fs_add_saturated(*total, (uint64_t)info.size); + return true; +} + +bool cbm_index_task_temp_bytes(const char *final_db_path, const char *stage_token, + const char *log_path, const char *response_path, + uint64_t *bytes_out) { + if (!final_db_path || !final_db_path[0] || !stage_token || !stage_token[0] || !bytes_out) { + return false; + } + *bytes_out = 0; + if (!fs_optional_regular_bytes(log_path, bytes_out) || + !fs_optional_regular_bytes(response_path, bytes_out)) { + return false; + } + const char *basename = strrchr(final_db_path, '/'); +#ifdef _WIN32 + const char *backslash = strrchr(final_db_path, '\\'); + if (backslash && (!basename || backslash > basename)) { + basename = backslash; + } +#endif + basename = basename ? basename + 1 : final_db_path; + size_t directory_len = (size_t)(basename - final_db_path); + char *directory = NULL; + if (directory_len == 0) { + directory = strdup("."); + directory_len = 1; + } else { + directory = (char *)malloc(directory_len + 1U); + if (directory) { + memcpy(directory, final_db_path, directory_len); + while (directory_len > 1U && (directory[directory_len - 1U] == '/' || + directory[directory_len - 1U] == '\\')) { + directory_len--; + } + directory[directory_len] = '\0'; + } + } + size_t basename_len = strlen(basename); + size_t token_len = strlen(stage_token); + static const char stage_marker[] = ".stage."; + if (!directory || token_len > SIZE_MAX - sizeof(stage_marker) - 1U || + basename_len > SIZE_MAX - token_len - sizeof(stage_marker) - 1U) { + free(directory); + return false; + } + size_t prefix_size = basename_len + (sizeof(stage_marker) - 1U) + token_len + 2U; + char *prefix = (char *)malloc(prefix_size); + if (!prefix) { + free(directory); + return false; + } + (void)snprintf(prefix, prefix_size, "%s%s%s.", basename, stage_marker, stage_token); + cbm_dir_t *dir = cbm_opendir(directory); + if (!dir) { + free(prefix); + free(directory); + return false; + } + bool ok = true; + cbm_dirent_t *entry; + while (ok && (entry = cbm_readdir(dir)) != NULL) { + if (strncmp(entry->name, prefix, prefix_size - 1U) != 0) { + continue; + } + size_t name_len = strlen(entry->name); + if (directory_len > SIZE_MAX - name_len - 2U) { + ok = false; + break; + } + size_t path_size = directory_len + name_len + 2U; + char *path = (char *)malloc(path_size); + if (!path) { + ok = false; + break; + } + (void)snprintf(path, path_size, "%s/%s", directory, entry->name); + ok = fs_optional_regular_bytes(path, bytes_out); + free(path); + } + cbm_closedir(dir); + free(prefix); + free(directory); + return ok; +} + +bool cbm_index_staging_cleanup(const char *final_db_path, const char *stage_token) { + if (!final_db_path || !final_db_path[0] || !stage_token || !stage_token[0]) { + return false; + } + const char *basename = strrchr(final_db_path, '/'); +#ifdef _WIN32 + const char *backslash = strrchr(final_db_path, '\\'); + if (backslash && (!basename || backslash > basename)) { + basename = backslash; + } +#endif + basename = basename ? basename + 1 : final_db_path; + size_t directory_len = (size_t)(basename - final_db_path); + char *directory = NULL; + if (directory_len == 0) { + directory = strdup("."); + directory_len = 1; + } else { + directory = (char *)malloc(directory_len + 1U); + if (directory) { + memcpy(directory, final_db_path, directory_len); + while (directory_len > 1U && (directory[directory_len - 1U] == '/' || + directory[directory_len - 1U] == '\\')) { + directory_len--; + } + directory[directory_len] = '\0'; + } + } + static const char stage_marker[] = ".stage."; + size_t basename_len = strlen(basename); + size_t token_len = strlen(stage_token); + if (!directory || token_len > SIZE_MAX - sizeof(stage_marker) - 1U || + basename_len > SIZE_MAX - token_len - sizeof(stage_marker) - 1U) { + free(directory); + return false; + } + size_t prefix_size = basename_len + (sizeof(stage_marker) - 1U) + token_len + 2U; + char *prefix = (char *)malloc(prefix_size); + if (!prefix) { + free(directory); + return false; + } + (void)snprintf(prefix, prefix_size, "%s%s%s.", basename, stage_marker, stage_token); + cbm_dir_t *dir = cbm_opendir(directory); + if (!dir) { + free(prefix); + free(directory); + return false; + } + bool ok = true; + cbm_dirent_t *entry; + while ((entry = cbm_readdir(dir)) != NULL) { + if (strncmp(entry->name, prefix, prefix_size - 1U) != 0) { + continue; + } + size_t name_len = strlen(entry->name); + if (directory_len > SIZE_MAX - name_len - 2U) { + ok = false; + continue; + } + size_t path_size = directory_len + name_len + 2U; + char *path = (char *)malloc(path_size); + if (!path) { + ok = false; + continue; + } + (void)snprintf(path, path_size, "%s/%s", directory, entry->name); + cbm_path_info_t info; + cbm_path_probe_status_t status = cbm_path_probe_info_utf8(path, &info); + if (status == CBM_PATH_PROBE_OK && !info.is_symlink && info.is_regular) { + if (cbm_unlink(path) != 0 && errno != ENOENT) { + ok = false; + } + } else if (status != CBM_PATH_PROBE_NOT_FOUND) { + ok = false; + } + free(path); + } + cbm_closedir(dir); + free(prefix); + free(directory); + return ok; +} + /* Canonicalize an EXISTING path (collapse `..`, resolve links/junctions): * realpath on POSIX; a final path queried from an opened handle on Windows. * The previous Windows callers used the ANSI CRT (_access/_fullpath) on UTF-8 diff --git a/src/foundation/compat_fs.h b/src/foundation/compat_fs.h index 240ebceae..51250e7d3 100644 --- a/src/foundation/compat_fs.h +++ b/src/foundation/compat_fs.h @@ -36,8 +36,34 @@ typedef struct { int64_t mtime_ns; } cbm_path_info_t; +typedef enum { + CBM_PATH_PROBE_ERROR = -1, + CBM_PATH_PROBE_OK = 0, + CBM_PATH_PROBE_NOT_FOUND = 1, +} cbm_path_probe_status_t; + /* Returns 0 on success and -1 when the path cannot be inspected. */ int cbm_path_info_utf8(const char *path, cbm_path_info_t *out); +cbm_path_probe_status_t cbm_path_probe_info_utf8(const char *path, cbm_path_info_t *out); + +/* Measure logical regular-file bytes beneath path without following symlinks + * or Windows reparse points. Saturates at UINT64_MAX. */ +bool cbm_directory_size_bytes(const char *path, uint64_t *bytes_out); + +/* Return bytes available to the current process on the filesystem containing + * an existing path (statvfs / GetDiskFreeSpaceExW). */ +bool cbm_filesystem_free_bytes(const char *path, uint64_t *bytes_out); + +/* Sum a SQLite main file and its -wal/-shm/-journal sidecars when present. + * Missing artifacts contribute zero; inspection errors fail the measurement. */ +bool cbm_db_artifact_bytes(const char *db_path, uint64_t *bytes_out); + +/* Sum this indexing attempt's explicit worker log/response plus every regular + * sibling whose name starts with ".stage.". */ +bool cbm_index_task_temp_bytes(const char *final_db_path, const char *stage_token, + const char *log_path, const char *response_path, + uint64_t *bytes_out); +bool cbm_index_staging_cleanup(const char *final_db_path, const char *stage_token); /* Open a directory for iteration. Returns NULL on error. */ cbm_dir_t *cbm_opendir(const char *path); diff --git a/src/foundation/index_policy.c b/src/foundation/index_policy.c index 80aa2f8a6..9c0f22e9c 100644 --- a/src/foundation/index_policy.c +++ b/src/foundation/index_policy.c @@ -21,6 +21,10 @@ static const index_policy_metadata_t INDEX_POLICY_METADATA[] = { CBM_INDEX_MIN_RSS_MB_VALUE, CBM_INDEX_MAX_RSS_MB_VALUE, CBM_INDEX_MIB_BYTES}, {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, offsetof(cbm_index_resource_policy_t, max_duration_ms), 1, CBM_INDEX_MAX_DURATION_SECONDS_VALUE, UINT64_C(1000)}, + {CBM_INDEX_CONFIG_CACHE_MAX_MB, offsetof(cbm_index_resource_policy_t, max_cache_bytes), 1, + CBM_INDEX_MAX_STORAGE_MB_VALUE, CBM_INDEX_MIB_BYTES}, + {CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, offsetof(cbm_index_resource_policy_t, min_free_disk_bytes), + 1, CBM_INDEX_MAX_STORAGE_MB_VALUE, CBM_INDEX_MIB_BYTES}, }; static void set_error(char *error, size_t error_size, const char *key, uint64_t minimum, @@ -61,18 +65,8 @@ void cbm_index_policy_init(cbm_index_resource_policy_t *policy) { } bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy) { - if (!policy) { - return false; - } - for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { - const cbm_index_limit_u64_t *limit = - (const cbm_index_limit_u64_t *)((const unsigned char *)policy + - INDEX_POLICY_METADATA[index].field_offset); - if (limit->enabled) { - return true; - } - } - return false; + return cbm_index_policy_discovery_enabled(policy) || cbm_index_policy_worker_enabled(policy) || + cbm_index_policy_storage_enabled(policy); } bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy) { @@ -83,6 +77,62 @@ bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy) return policy && (policy->max_rss_bytes.enabled || policy->max_duration_ms.enabled); } +bool cbm_index_policy_storage_enabled(const cbm_index_resource_policy_t *policy) { + return policy && (policy->max_cache_bytes.enabled || policy->min_free_disk_bytes.enabled || + policy->max_final_db_bytes.enabled || policy->max_staging_bytes.enabled || + policy->max_task_temp_bytes.enabled); +} + +static uint64_t add_saturated(uint64_t left, uint64_t right) { + return left > UINT64_MAX - right ? UINT64_MAX : left + right; +} + +static bool storage_within_max(cbm_index_resource_t resource, uint64_t observed, + const cbm_index_limit_u64_t *limit, + cbm_index_resource_violation_t *violation) { + if (!limit->enabled || observed <= limit->value) { + return true; + } + *violation = (cbm_index_resource_violation_t){ + .resource = resource, + .observed = observed, + .limit = limit->value, + }; + return false; +} + +bool cbm_index_policy_check_storage(const cbm_index_resource_policy_t *policy, + const cbm_index_storage_sample_t *sample, + cbm_index_resource_violation_t *violation) { + if (!policy || !sample || !violation) { + return false; + } + *violation = (cbm_index_resource_violation_t){0}; + uint64_t remaining_cache = sample->current_cache_bytes > sample->replaceable_old_bytes + ? sample->current_cache_bytes - sample->replaceable_old_bytes + : 0; + uint64_t projected_cache = add_saturated(remaining_cache, sample->operation_bytes); + if (!storage_within_max(CBM_INDEX_RESOURCE_CACHE_BYTES, projected_cache, + &policy->max_cache_bytes, violation)) { + return false; + } + if (policy->min_free_disk_bytes.enabled && + sample->free_disk_bytes < policy->min_free_disk_bytes.value) { + *violation = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_FREE_DISK_BYTES, + .observed = sample->free_disk_bytes, + .limit = policy->min_free_disk_bytes.value, + }; + return false; + } + return storage_within_max(CBM_INDEX_RESOURCE_FINAL_DB_BYTES, sample->final_db_bytes, + &policy->max_final_db_bytes, violation) && + storage_within_max(CBM_INDEX_RESOURCE_STAGING_BYTES, sample->staging_bytes, + &policy->max_staging_bytes, violation) && + storage_within_max(CBM_INDEX_RESOURCE_TASK_TEMP_BYTES, sample->task_temp_bytes, + &policy->max_task_temp_bytes, violation); +} + size_t cbm_index_policy_key_count(void) { return sizeof(INDEX_POLICY_METADATA) / sizeof(INDEX_POLICY_METADATA[0]); } @@ -172,6 +222,16 @@ const char *cbm_index_resource_name(cbm_index_resource_t resource) { return "rss_bytes"; case CBM_INDEX_RESOURCE_DURATION_MS: return "duration_ms"; + case CBM_INDEX_RESOURCE_CACHE_BYTES: + return "cache_bytes"; + case CBM_INDEX_RESOURCE_FREE_DISK_BYTES: + return "free_disk_bytes"; + case CBM_INDEX_RESOURCE_FINAL_DB_BYTES: + return "final_db_bytes"; + case CBM_INDEX_RESOURCE_STAGING_BYTES: + return "staging_bytes"; + case CBM_INDEX_RESOURCE_TASK_TEMP_BYTES: + return "task_temp_bytes"; case CBM_INDEX_RESOURCE_NONE: default: return "unknown"; @@ -198,6 +258,14 @@ const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { return CBM_INDEX_CONFIG_MAX_RSS_MB; case CBM_INDEX_RESOURCE_DURATION_MS: return CBM_INDEX_CONFIG_MAX_DURATION_SECONDS; + case CBM_INDEX_RESOURCE_CACHE_BYTES: + return CBM_INDEX_CONFIG_CACHE_MAX_MB; + case CBM_INDEX_RESOURCE_FREE_DISK_BYTES: + return CBM_INDEX_CONFIG_MIN_FREE_DISK_MB; + case CBM_INDEX_RESOURCE_FINAL_DB_BYTES: + case CBM_INDEX_RESOURCE_STAGING_BYTES: + case CBM_INDEX_RESOURCE_TASK_TEMP_BYTES: + return "index_resource_profile"; case CBM_INDEX_RESOURCE_NONE: default: return "index_resource_limit"; diff --git a/src/foundation/index_policy.h b/src/foundation/index_policy.h index 617a6ab7d..bac059f2e 100644 --- a/src/foundation/index_policy.h +++ b/src/foundation/index_policy.h @@ -9,12 +9,15 @@ #define CBM_INDEX_CONFIG_MAX_SOURCE_MB "index_max_source_mb" #define CBM_INDEX_CONFIG_MAX_RSS_MB "index_max_rss_mb" #define CBM_INDEX_CONFIG_MAX_DURATION_SECONDS "index_max_duration_seconds" +#define CBM_INDEX_CONFIG_CACHE_MAX_MB "index_cache_max_mb" +#define CBM_INDEX_CONFIG_MIN_FREE_DISK_MB "index_min_free_disk_mb" #define CBM_INDEX_MAX_FILES_VALUE UINT64_C(10000000) #define CBM_INDEX_MAX_SOURCE_MB_VALUE UINT64_C(1048576) #define CBM_INDEX_MIN_RSS_MB_VALUE UINT64_C(64) #define CBM_INDEX_MAX_RSS_MB_VALUE UINT64_C(1048576) #define CBM_INDEX_MAX_DURATION_SECONDS_VALUE UINT64_C(86400) +#define CBM_INDEX_MAX_STORAGE_MB_VALUE UINT64_C(1048576) #define CBM_INDEX_MIB_BYTES UINT64_C(1048576) typedef struct { @@ -27,6 +30,15 @@ typedef struct { cbm_index_limit_u64_t max_source_bytes; cbm_index_limit_u64_t max_rss_bytes; cbm_index_limit_u64_t max_duration_ms; + cbm_index_limit_u64_t max_cache_bytes; + cbm_index_limit_u64_t min_free_disk_bytes; + /* Internal storage dimensions with no public config key. Nothing enables + * them on its own; they exist so that a single composed decision can bound + * the database, the staging artifacts and the task temporary directory + * together, which no individual key can express. */ + cbm_index_limit_u64_t max_final_db_bytes; + cbm_index_limit_u64_t max_staging_bytes; + cbm_index_limit_u64_t max_task_temp_bytes; } cbm_index_resource_policy_t; typedef enum { @@ -35,18 +47,38 @@ typedef enum { CBM_INDEX_RESOURCE_SOURCE_BYTES, CBM_INDEX_RESOURCE_RSS_BYTES, CBM_INDEX_RESOURCE_DURATION_MS, + CBM_INDEX_RESOURCE_CACHE_BYTES, + CBM_INDEX_RESOURCE_FREE_DISK_BYTES, + CBM_INDEX_RESOURCE_FINAL_DB_BYTES, + CBM_INDEX_RESOURCE_STAGING_BYTES, + CBM_INDEX_RESOURCE_TASK_TEMP_BYTES, } cbm_index_resource_t; typedef struct { cbm_index_resource_t resource; uint64_t observed; uint64_t limit; + bool probe_failed; } cbm_index_resource_violation_t; +typedef struct { + uint64_t current_cache_bytes; + uint64_t replaceable_old_bytes; + uint64_t operation_bytes; + uint64_t free_disk_bytes; + uint64_t final_db_bytes; + uint64_t staging_bytes; + uint64_t task_temp_bytes; +} cbm_index_storage_sample_t; + void cbm_index_policy_init(cbm_index_resource_policy_t *policy); bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy); bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy); bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_storage_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_check_storage(const cbm_index_resource_policy_t *policy, + const cbm_index_storage_sample_t *sample, + cbm_index_resource_violation_t *violation); size_t cbm_index_policy_key_count(void); const char *cbm_index_policy_key_at(size_t index); diff --git a/src/main.c b/src/main.c index 288dec357..e7f22e60a 100644 --- a/src/main.c +++ b/src/main.c @@ -687,9 +687,11 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo const char *worker_marker = cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_MARKER_ARG); const char *worker_quarantine = cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_QUARANTINE_ARG); + const char *worker_stage_token = + cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_STAGE_TOKEN_ARG); cbm_index_set_worker_role_options(index_worker, response_out, worker_single_thread, worker_marker, worker_quarantine, - cbm_index_worker_memory_budget_bytes()); + cbm_index_worker_memory_budget_bytes(), worker_stage_token); if (argc < MAIN_MIN_ARGC) { (void)fprintf(stderr, CLI_USAGE); @@ -2718,7 +2720,7 @@ int main(int argc, char **argv) { } cbm_index_set_worker_role_options(true, invocation.response_out, invocation.single_thread, invocation.marker_file, invocation.quarantine_file, - invocation.memory_budget_bytes); + invocation.memory_budget_bytes, invocation.stage_token); #ifndef _WIN32 /* Split into three ordered steps rather than one condition, because the * ORDER is load-bearing and the middle step only exists in test builds: diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index b9d0212ef..c485b9a75 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -67,7 +67,7 @@ static void worker_set_local_env(const char *name, const char *value) { void cbm_index_set_worker_role_options(bool is_worker, const char *response_out, bool single_thread, const char *marker_file, const char *quarantine_file, - size_t memory_budget_bytes) { + size_t memory_budget_bytes, const char *stage_token) { cbm_index_set_worker_role(is_worker, response_out); g_worker_memory_budget_bytes = is_worker ? memory_budget_bytes : 0; if (!is_worker) { @@ -76,6 +76,7 @@ void cbm_index_set_worker_role_options(bool is_worker, const char *response_out, worker_set_local_env("CBM_INDEX_SINGLE_THREAD", single_thread ? "1" : NULL); worker_set_local_env("CBM_INDEX_MARKER_FILE", marker_file); worker_set_local_env("CBM_INDEX_QUARANTINE_FILE", quarantine_file); + worker_set_local_env("CBM_INDEX_STAGE_TOKEN", stage_token); } bool cbm_index_worker_active(void) { @@ -161,6 +162,21 @@ static bool worker_fingerprint_valid(const char *fingerprint) { return true; } +static bool worker_stage_token_valid(const char *token) { + size_t length = token ? strlen(token) : 0; + if (length < 6U || length >= CBM_SZ_64) { + return false; + } + for (size_t index = 0; index < length; index++) { + unsigned char ch = (unsigned char)token[index]; + if (!((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || + ch == '-' || ch == '_')) { + return false; + } + } + return true; +} + static bool worker_parse_positive_size(const char *text, size_t *value_out) { if (!text || !text[0] || !value_out) { return false; @@ -238,6 +254,13 @@ cbm_index_worker_argv_status_t cbm_index_worker_parse_process_argv( parsed.quarantine_file = argv[next + 1]; next += 2; } + if (next < argc && argv[next] && strcmp(argv[next], CBM_INDEX_WORKER_STAGE_TOKEN_ARG) == 0) { + if (next + 1 >= argc || !worker_stage_token_valid(argv[next + 1])) { + return CBM_INDEX_WORKER_ARGV_INVALID; + } + parsed.stage_token = argv[next + 1]; + next += 2; + } if (next != argc) { return CBM_INDEX_WORKER_ARGV_INVALID; } @@ -386,12 +409,13 @@ static char *slurp_worker_response(const char *path, worker_response_read_status enum { INDEX_WORKER_PATH_CAP = CBM_SZ_4K, - INDEX_WORKER_ARGV_CAP = 17, + INDEX_WORKER_ARGV_CAP = 19, INDEX_WORKER_SYNC_POLL_NS = 10000000, INDEX_WORKER_RELAY_LINES_PER_POLL = 64, INDEX_WORKER_RELAY_BYTES_PER_POLL = 64 * 1024, INDEX_WORKER_RSS_PROBE_FAILURE_LIMIT = 3, INDEX_WORKER_RSS_PROBE_INTERVAL_MS = 250, + INDEX_WORKER_TASK_TEMP_PROBE_INTERVAL_MS = 250, }; typedef enum { @@ -405,6 +429,8 @@ typedef enum { static cbm_index_supervisor_clock_fn g_resource_clock; static cbm_index_supervisor_rss_fn g_resource_rss; static void *g_resource_hook_context; +static cbm_index_supervisor_task_temp_fn g_task_temp_hook; +static void *g_task_temp_hook_context; void cbm_index_supervisor_set_resource_hooks_for_testing(cbm_index_supervisor_clock_fn clock_fn, cbm_index_supervisor_rss_fn rss_fn, @@ -419,6 +445,17 @@ void cbm_index_supervisor_reset_resource_hooks_for_testing(void) { g_resource_rss = NULL; g_resource_hook_context = NULL; } + +void cbm_index_supervisor_set_task_temp_hook_for_testing( + cbm_index_supervisor_task_temp_fn task_temp_fn, void *context) { + g_task_temp_hook = task_temp_fn; + g_task_temp_hook_context = context; +} + +void cbm_index_supervisor_reset_task_temp_hook_for_testing(void) { + g_task_temp_hook = NULL; + g_task_temp_hook_context = NULL; +} #endif static uint64_t worker_resource_now_ms(void) { @@ -450,10 +487,14 @@ struct cbm_index_worker_handle { bool process_terminal; cbm_proc_result_t process_result; cbm_index_resource_policy_t resource_policy; + char task_db_path[INDEX_WORKER_PATH_CAP]; + char stage_token[CBM_SZ_64]; uint64_t started_ms; uint64_t last_rss_probe_ms; + uint64_t last_task_temp_probe_ms; unsigned int rss_probe_failures; bool rss_probe_started; + bool task_temp_probe_started; atomic_int termination_reason; atomic_bool terminal; cbm_index_worker_result_t result; @@ -680,7 +721,7 @@ static void worker_request_resource_termination(cbm_index_worker_handle_t *handl return; } handle->result.resource_violation = (cbm_index_resource_violation_t){ - .resource = resource, .observed = observed, .limit = limit}; + .resource = resource, .observed = observed, .limit = limit, .probe_failed = probe_failed}; handle->result.resource_probe_failed = probe_failed; if (!cbm_subprocess_request_cancel(handle->process)) { handle->result.resource_violation = (cbm_index_resource_violation_t){0}; @@ -746,11 +787,49 @@ static bool worker_check_resource_limits(cbm_index_worker_handle_t *handle) { return rss_probe_failed; } +static bool worker_task_temp_bytes(cbm_index_worker_handle_t *handle, uint64_t *bytes_out) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_task_temp_hook) { + return g_task_temp_hook(handle->task_db_path, handle->log_path, handle->response_path, + bytes_out, g_task_temp_hook_context); + } +#endif + return cbm_index_task_temp_bytes(handle->task_db_path, handle->stage_token, handle->log_path, + handle->response_path, bytes_out); +} + +static void worker_check_task_temp_limit(cbm_index_worker_handle_t *handle) { + const cbm_index_limit_u64_t *limit = &handle->resource_policy.max_task_temp_bytes; + if (!limit->enabled || + atomic_load_explicit(&handle->termination_reason, memory_order_acquire) != + INDEX_WORKER_TERMINATION_NONE || + cbm_subprocess_termination_pending(handle->process) || + !cbm_subprocess_supervision_active(handle->process)) { + return; + } + uint64_t now = worker_resource_now_ms(); + bool probe_due = + !handle->task_temp_probe_started || now < handle->last_task_temp_probe_ms || + now - handle->last_task_temp_probe_ms >= INDEX_WORKER_TASK_TEMP_PROBE_INTERVAL_MS; + if (!probe_due) { + return; + } + handle->task_temp_probe_started = true; + handle->last_task_temp_probe_ms = now; + uint64_t observed = 0; + bool measured = worker_task_temp_bytes(handle, &observed); + if (!measured || observed > limit->value) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_TASK_TEMP_BYTES, observed, + limit->value, !measured); + } +} + static int worker_start_internal(const char *args_json, size_t memory_budget_bytes, const cbm_index_resource_policy_t *resource_policy, - bool single_thread, const char *marker_file, - const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, cbm_index_worker_handle_t **handle_out) { + const char *task_db_path, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + cbm_index_worker_handle_t **handle_out) { if (handle_out) { *handle_out = NULL; } @@ -786,6 +865,15 @@ static int worker_start_internal(const char *args_json, size_t memory_budget_byt if (resource_policy) { handle->resource_policy = *resource_policy; } + if (handle->resource_policy.max_task_temp_bytes.enabled) { + int written = task_db_path ? snprintf(handle->task_db_path, sizeof(handle->task_db_path), + "%s", task_db_path) + : -1; + if (written <= 0 || written >= (int)sizeof(handle->task_db_path)) { + free(handle); + return -1; + } + } atomic_init(&handle->termination_reason, INDEX_WORKER_TERMINATION_NONE); atomic_init(&handle->terminal, false); handle->log_callback = log_callback; @@ -798,6 +886,17 @@ static int worker_start_internal(const char *args_json, size_t memory_budget_byt free(handle); return -1; } + const char *random_suffix = strrchr(handle->response_path, '-'); + int token_written = random_suffix ? snprintf(handle->stage_token, sizeof(handle->stage_token), + "%ld%s", (long)worker_getpid(), random_suffix) + : -1; + if (token_written <= 0 || token_written >= (int)sizeof(handle->stage_token) || + !worker_stage_token_valid(handle->stage_token)) { + (void)cbm_unlink(handle->response_path); + (void)cbm_unlink(handle->log_path); + free(handle); + return -1; + } const char *argv[INDEX_WORKER_ARGV_CAP]; size_t argc = 0; @@ -827,6 +926,8 @@ static int worker_start_internal(const char *args_json, size_t memory_budget_byt argv[argc++] = CBM_INDEX_WORKER_QUARANTINE_ARG; argv[argc++] = quarantine_file; } + argv[argc++] = CBM_INDEX_WORKER_STAGE_TOKEN_ARG; + argv[argc++] = handle->stage_token; argv[argc] = NULL; cbm_proc_opts_t options = {0}; @@ -858,15 +959,16 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, cbm_index_worker_handle_t **handle_out) { - return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, - quarantine_file, log_callback, log_context, handle_out); + return worker_start_internal(args_json, memory_budget_bytes, NULL, NULL, single_thread, + marker_file, quarantine_file, log_callback, log_context, + handle_out); } int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out) { - return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, - quarantine_file, NULL, NULL, handle_out); + return worker_start_internal(args_json, memory_budget_bytes, NULL, NULL, single_thread, + marker_file, quarantine_file, NULL, NULL, handle_out); } int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, @@ -874,8 +976,20 @@ int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budg bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out) { - return worker_start_internal(args_json, memory_budget_bytes, resource_policy, single_thread, - marker_file, quarantine_file, NULL, NULL, handle_out); + return worker_start_internal(args_json, memory_budget_bytes, resource_policy, NULL, + single_thread, marker_file, quarantine_file, NULL, NULL, + handle_out); +} + +int cbm_index_worker_start_with_storage_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + const char *task_db_path, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, resource_policy, task_db_path, + single_thread, marker_file, quarantine_file, log_callback, + log_context, handle_out); } cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, @@ -893,6 +1007,7 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, bool relay_caught_up = true; if (!handle->process_terminal) { bool rss_probe_failed = worker_check_resource_limits(handle); + worker_check_task_temp_limit(handle); cbm_proc_result_t process_result; cbm_proc_poll_t state = cbm_subprocess_poll(handle->process, &process_result); relay_caught_up = worker_relay_log(handle); @@ -942,6 +1057,14 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, handle->result.exit_code = -1; } } + if (termination_reason == INDEX_WORKER_TERMINATION_RESOURCE && + handle->result.resource_violation.resource == CBM_INDEX_RESOURCE_TASK_TEMP_BYTES) { + if (!handle->result.tree_quiesced || handle->result.supervision_failed) { + cbm_log_warn("index.supervisor.staging_cleanup_deferred", "path", handle->task_db_path); + } else if (!cbm_index_staging_cleanup(handle->task_db_path, handle->stage_token)) { + cbm_log_warn("index.supervisor.staging_cleanup_failed", "path", handle->task_db_path); + } + } (void)cbm_unlink(handle->response_path); worker_terminal_log(handle); atomic_store_explicit(&handle->terminal, true, memory_order_release); @@ -1006,17 +1129,19 @@ void cbm_index_worker_destroy(cbm_index_worker_handle_t *handle) { static int worker_spawn_internal(const char *args_json, const cbm_index_resource_policy_t *resource_policy, - bool single_thread, const char *marker_file, - const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, const atomic_int *cancel_requested, + const char *task_db_path, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + const atomic_int *cancel_requested, cbm_index_worker_result_t *result) { if (!result) { return -1; } worker_result_init(result); cbm_index_worker_handle_t *handle = NULL; - if (worker_start_internal(args_json, 0, resource_policy, single_thread, marker_file, - quarantine_file, log_callback, log_context, &handle) != 0) { + if (worker_start_internal(args_json, 0, resource_policy, task_db_path, single_thread, + marker_file, quarantine_file, log_callback, log_context, + &handle) != 0) { return -1; } const cbm_index_worker_result_t *cached = NULL; @@ -1047,7 +1172,7 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th cbm_proc_log_cb log_callback, void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result) { - return worker_spawn_internal(args_json, NULL, single_thread, marker_file, quarantine_file, + return worker_spawn_internal(args_json, NULL, NULL, single_thread, marker_file, quarantine_file, log_callback, log_context, cancel_requested, result); } @@ -1055,11 +1180,21 @@ int cbm_index_spawn_worker_with_policy_log_cancel( const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result) { - return worker_spawn_internal(args_json, resource_policy, single_thread, marker_file, + return worker_spawn_internal(args_json, resource_policy, NULL, single_thread, marker_file, quarantine_file, log_callback, log_context, cancel_requested, result); } +int cbm_index_spawn_worker_with_storage_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, + const char *task_db_path, bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, + const atomic_int *cancel_requested, cbm_index_worker_result_t *result) { + return worker_spawn_internal(args_json, resource_policy, task_db_path, single_thread, + marker_file, quarantine_file, log_callback, log_context, + cancel_requested, result); +} + int cbm_index_spawn_worker_with_log(const char *args_json, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, diff --git a/src/mcp/index_supervisor.h b/src/mcp/index_supervisor.h index d7e39f94d..e8a3ef7db 100644 --- a/src/mcp/index_supervisor.h +++ b/src/mcp/index_supervisor.h @@ -42,11 +42,12 @@ void cbm_index_set_worker_role(bool is_worker, const char *response_out); #define CBM_INDEX_WORKER_QUARANTINE_ARG "--index-worker-quarantine" #define CBM_INDEX_WORKER_MEMORY_BUDGET_ARG "--index-worker-memory-budget-bytes" #define CBM_INDEX_WORKER_BUILD_ARG "--index-worker-build" +#define CBM_INDEX_WORKER_STAGE_TOKEN_ARG "--index-worker-stage-token" #define CBM_INDEX_WORKER_BUILD_FINGERPRINT_LENGTH 64U #define CBM_INDEX_WORKER_BUILD_FINGERPRINT_SIZE 65U void cbm_index_set_worker_role_options(bool is_worker, const char *response_out, bool single_thread, const char *marker_file, const char *quarantine_file, - size_t memory_budget_bytes); + size_t memory_budget_bytes, const char *stage_token); bool cbm_index_worker_active(void); const char *cbm_index_worker_response_out(void); size_t cbm_index_worker_memory_budget_bytes(void); @@ -86,6 +87,7 @@ typedef struct { const char *marker_file; const char *quarantine_file; size_t memory_budget_bytes; + const char *stage_token; } cbm_index_worker_invocation_t; typedef enum { @@ -166,6 +168,12 @@ int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budg bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out); +int cbm_index_worker_start_with_storage_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + const char *task_db_path, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + cbm_index_worker_handle_t **handle_out); /* Request-scoped variant used by interactive local CLI calls. The callback is * invoked by the owner thread while it polls the contained worker; log_context @@ -233,6 +241,11 @@ int cbm_index_spawn_worker_with_policy_log_cancel( const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result); +int cbm_index_spawn_worker_with_storage_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, + const char *task_db_path, bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, + const atomic_int *cancel_requested, cbm_index_worker_result_t *result); void cbm_index_worker_result_free(cbm_index_worker_result_t *result); @@ -245,6 +258,12 @@ void cbm_index_supervisor_set_resource_hooks_for_testing(cbm_index_supervisor_cl cbm_index_supervisor_rss_fn rss_fn, void *context); void cbm_index_supervisor_reset_resource_hooks_for_testing(void); +typedef bool (*cbm_index_supervisor_task_temp_fn)(const char *final_db_path, const char *log_path, + const char *response_path, uint64_t *bytes_out, + void *context); +void cbm_index_supervisor_set_task_temp_hook_for_testing( + cbm_index_supervisor_task_temp_fn task_temp_fn, void *context); +void cbm_index_supervisor_reset_task_temp_hook_for_testing(void); #endif #endif /* CBM_INDEX_SUPERVISOR_H */ diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index e23127c3a..887848c2b 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -7549,6 +7549,8 @@ static bool project_db_is_servable(const char *project, const char *db_path); * result. The crash is already contained (this process survived); we report it * rather than dying. Precise skip-and-continue (quarantine the culprit, index the * rest) is layered on in the probe stage. */ +static bool project_db_is_servable(const char *project, const char *db_path); + static char *build_worker_failure_response(const char *args, cbm_proc_outcome_t outcome) { char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); @@ -7612,6 +7614,9 @@ static char *build_worker_unsafe_terminal_response(const char *args, cbm_proc_ou char *cbm_mcp_index_worker_resource_response(const char *args, const cbm_index_worker_result_t *worker_result) { + if (!worker_result || worker_result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE) { + return NULL; + } char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); char *name_override = cbm_mcp_get_string_arg(args, "name"); char *project_name = @@ -7622,23 +7627,23 @@ char *cbm_mcp_index_worker_resource_response(const char *args, } const cbm_index_resource_violation_t *violation = &worker_result->resource_violation; + bool probe_failed = worker_result->resource_probe_failed || violation->probe_failed; + bool worker_stage = violation->resource == CBM_INDEX_RESOURCE_RSS_BYTES || + violation->resource == CBM_INDEX_RESOURCE_DURATION_MS; const char *config_key = cbm_index_resource_config_key(violation->resource); char message[CBM_SZ_256]; (void)snprintf(message, sizeof(message), - worker_result->resource_probe_failed - ? "Worker resource measurement failed for %s" - : "Index worker exceeded %s", + probe_failed ? "Index resource measurement failed for %s" : "Index exceeded %s", config_key); yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); yyjson_mut_val *root = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root); yyjson_mut_obj_add_str(doc, root, "status", "error"); yyjson_mut_obj_add_str(doc, root, "code", - worker_result->resource_probe_failed ? "resource_probe_failed" - : "resource_limit_exceeded"); - yyjson_mut_obj_add_str(doc, root, "stage", "worker"); + probe_failed ? "resource_probe_failed" : "resource_limit_exceeded"); + yyjson_mut_obj_add_str(doc, root, "stage", worker_stage ? "worker" : "storage"); yyjson_mut_obj_add_str(doc, root, "resource", cbm_index_resource_name(violation->resource)); - if (!worker_result->resource_probe_failed) { + if (!probe_failed) { yyjson_mut_obj_add_uint(doc, root, "observed", violation->observed); yyjson_mut_obj_add_uint(doc, root, "limit", violation->limit); yyjson_mut_obj_add_str(doc, root, "unit", cbm_index_resource_unit(violation->resource)); @@ -7841,6 +7846,24 @@ bool cbm_mcp_index_policy_from_internal_args(const char *args, cbm_index_resourc return valid; } +bool cbm_mcp_index_task_db_path(const char *args, char *path_out, size_t path_size) { + if (!path_out || path_size == 0) { + return false; + } + path_out[0] = '\0'; + char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); + char *name_override = cbm_mcp_get_string_arg(args, "name"); + char *project = + cbm_project_name_from_path(name_override && name_override[0] ? name_override : repo_path); + char cache_path[CBM_SZ_1K]; + cache_dir(cache_path, sizeof(cache_path)); + int written = project ? snprintf(path_out, path_size, "%s/%s.db", cache_path, project) : -1; + free(project); + free(name_override); + free(repo_path); + return written > 0 && (size_t)written < path_size; +} + static bool load_index_policy(cbm_mcp_server_t *srv, const char *args, cbm_index_resource_policy_t *policy, char *error, size_t error_size) { if (cbm_index_worker_active()) { @@ -7884,12 +7907,17 @@ bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, const cbm_index_resource_policy_t *resource_policy) { invalidate_cached_store(srv); + char task_db_path[CBM_SZ_1K]; + if (!resource_policy || !cbm_mcp_index_task_db_path(args, task_db_path, sizeof(task_db_path))) { + return cbm_mcp_text_result("could not resolve worker index path", true); + } /* First attempt: normal parallel run. */ cbm_index_worker_result_t wr; - int rc = cbm_index_spawn_worker_with_policy_log_cancel( - args, resource_policy, false, NULL, NULL, srv ? srv->index_log_callback : NULL, - srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wr); + int rc = cbm_index_spawn_worker_with_storage_policy_log_cancel( + args, resource_policy, task_db_path, false, NULL, NULL, + srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, + srv ? &srv->pipeline_cancel_requested : NULL, &wr); cbm_mcp_supervised_result_disposition_t disposition = cbm_mcp_supervised_result_disposition(rc, &wr); @@ -7920,7 +7948,6 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, invalidate_cached_store(srv); return resp; } - /* Crash / hang / nonzero exit → skip-and-continue recovery. Re-run the * worker PARALLEL (there are no sequential production runs) with the * per-file marker JOURNAL armed; after each failed run the journal's @@ -7968,10 +7995,11 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, bool terminal_cancelled = false; for (int i = 0; i < cap; i++) { cbm_index_worker_result_t wr2; - int rc2 = cbm_index_spawn_worker_with_policy_log_cancel( - args, resource_policy, /*single_thread=*/false, marker_path, quarantine_path, - srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, - srv ? &srv->pipeline_cancel_requested : NULL, &wr2); + int rc2 = cbm_index_spawn_worker_with_storage_policy_log_cancel( + args, resource_policy, task_db_path, /*single_thread=*/false, marker_path, + quarantine_path, srv ? srv->index_log_callback : NULL, + srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, + &wr2); cbm_mcp_supervised_result_disposition_t recovery_disposition = cbm_mcp_supervised_result_disposition(rc2, &wr2); if (recovery_disposition == CBM_MCP_SUPERVISED_RESULT_FALLBACK) { @@ -8064,8 +8092,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, * so it cannot itself hang. Rare given monotonic progress. */ if (!resp && !unsafe_terminal && quarantined > 0) { cbm_index_worker_result_t wrp; - int rcp = cbm_index_spawn_worker_with_policy_log_cancel( - args, resource_policy, /*single_thread=*/false, NULL, quarantine_path, + int rcp = cbm_index_spawn_worker_with_storage_policy_log_cancel( + args, resource_policy, task_db_path, /*single_thread=*/false, NULL, quarantine_path, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wrp); cbm_mcp_supervised_result_disposition_t partial_disposition = @@ -8490,11 +8518,18 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { } else if (rc == CBM_PIPELINE_RESOURCE_LIMIT && resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { const char *config_key = cbm_index_resource_config_key(resource_violation.resource); + bool discovery_resource = resource_violation.resource == CBM_INDEX_RESOURCE_FILES || + resource_violation.resource == CBM_INDEX_RESOURCE_SOURCE_BYTES; char message[CBM_SZ_256]; - (void)snprintf(message, sizeof(message), "Index discovery exceeded %s", config_key); + (void)snprintf(message, sizeof(message), + resource_violation.probe_failed ? "Index resource probe failed for %s" + : "Index exceeded %s", + config_key); yyjson_mut_obj_add_str(doc, root, "status", "error"); - yyjson_mut_obj_add_str(doc, root, "code", "resource_limit_exceeded"); - yyjson_mut_obj_add_str(doc, root, "stage", "discovery"); + yyjson_mut_obj_add_str(doc, root, "code", + resource_violation.probe_failed ? "resource_probe_failed" + : "resource_limit_exceeded"); + yyjson_mut_obj_add_str(doc, root, "stage", discovery_resource ? "discovery" : "storage"); yyjson_mut_obj_add_str(doc, root, "resource", cbm_index_resource_name(resource_violation.resource)); yyjson_mut_obj_add_uint(doc, root, "observed", resource_violation.observed); @@ -11733,11 +11768,18 @@ static void *autoindex_thread(void *arg) { return NULL; } + cbm_index_resource_policy_t resource_policy; + char policy_error[CBM_SZ_256] = {0}; + if (!load_index_policy(srv, NULL, &resource_policy, policy_error, sizeof(policy_error))) { + cbm_log_warn("autoindex.err", "msg", "resource_policy_load_failed", "error", policy_error); + return NULL; + } cbm_pipeline_t *p = cbm_pipeline_new(srv->session_root, NULL, CBM_MODE_FULL); if (!p) { cbm_log_warn("autoindex.err", "msg", "pipeline_create_failed"); return NULL; } + cbm_pipeline_set_resource_policy(p, &resource_policy); /* Block until any concurrent pipeline finishes */ cbm_pipeline_lock(); diff --git a/src/mcp/mcp_internal.h b/src/mcp/mcp_internal.h index a2a6ac5cf..b42b43315 100644 --- a/src/mcp/mcp_internal.h +++ b/src/mcp/mcp_internal.h @@ -34,6 +34,7 @@ bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, const cbm_index_resource_policy_t *policy); bool cbm_mcp_index_policy_from_internal_args(const char *args, cbm_index_resource_policy_t *policy, char *error, size_t error_size); +bool cbm_mcp_index_task_db_path(const char *args, char *path_out, size_t path_size); char *cbm_mcp_index_worker_resource_response(const char *args, const cbm_index_worker_result_t *worker_result); diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index cad1d9c14..0e479558c 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -158,6 +158,10 @@ struct cbm_pipeline { bool persistence; /* write .codebase-memory/graph.db.zst after indexing */ cbm_index_resource_policy_t resource_policy; cbm_index_resource_violation_t resource_violation; +#ifdef CBM_ENABLE_TEST_SEAMS + cbm_pipeline_storage_probe_fn storage_probe; + void *storage_probe_context; +#endif /* Indexing state (set during run) */ cbm_gbuf_t *gbuf; @@ -320,6 +324,17 @@ void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p, } } +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_pipeline_set_storage_probe_for_testing(cbm_pipeline_t *p, + cbm_pipeline_storage_probe_fn probe, + void *context) { + if (p) { + p->storage_probe = probe; + p->storage_probe_context = context; + } +} +#endif + bool cbm_pipeline_set_project_name(cbm_pipeline_t *p, const char *name) { if (!p || !name || !name[0]) { return false; @@ -1696,7 +1711,6 @@ int cbm_pipeline_publish_generation(const cbm_pipeline_generation_t *generation) if (generation->cancelled && atomic_load(generation->cancelled)) { return CBM_PIPELINE_ABORT_PRESERVE_DB; } - /* The staging name must be unpredictable and created exclusively. It used * to be ".stage..", which any other process can compute * in advance; this path is then unlinked and written, so in a @@ -2366,6 +2380,216 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { return rc; } +static const cbm_index_limit_u64_t *storage_limit_for_resource( + const cbm_index_resource_policy_t *policy, cbm_index_resource_t resource) { + if (!policy) { + return NULL; + } + switch (resource) { + case CBM_INDEX_RESOURCE_CACHE_BYTES: + return &policy->max_cache_bytes; + case CBM_INDEX_RESOURCE_FREE_DISK_BYTES: + return &policy->min_free_disk_bytes; + case CBM_INDEX_RESOURCE_FINAL_DB_BYTES: + return &policy->max_final_db_bytes; + case CBM_INDEX_RESOURCE_STAGING_BYTES: + return &policy->max_staging_bytes; + case CBM_INDEX_RESOURCE_TASK_TEMP_BYTES: + return &policy->max_task_temp_bytes; + default: + return NULL; + } +} + +static cbm_index_resource_t storage_enabled_probe_resource( + const cbm_index_resource_policy_t *policy) { + if (policy->max_cache_bytes.enabled) { + return CBM_INDEX_RESOURCE_CACHE_BYTES; + } + if (policy->min_free_disk_bytes.enabled) { + return CBM_INDEX_RESOURCE_FREE_DISK_BYTES; + } + if (policy->max_staging_bytes.enabled) { + return CBM_INDEX_RESOURCE_STAGING_BYTES; + } + if (policy->max_task_temp_bytes.enabled) { + return CBM_INDEX_RESOURCE_TASK_TEMP_BYTES; + } + return policy->max_final_db_bytes.enabled ? CBM_INDEX_RESOURCE_FINAL_DB_BYTES + : CBM_INDEX_RESOURCE_NONE; +} + +static char *storage_root_from_final_path(const char *final_path) { + if (!final_path) { + return NULL; + } + char *root = strdup(final_path); + if (!root) { + return NULL; + } + char *slash = strrchr(root, '/'); +#ifdef _WIN32 + char *backslash = strrchr(root, '\\'); + if (backslash && (!slash || backslash > slash)) { + slash = backslash; + } +#endif + if (!slash) { + free(root); + return strdup("."); + } + if (slash == root) { + slash[1] = '\0'; + } else { + *slash = '\0'; + } + return root; +} + +static bool storage_regular_file_bytes(const char *path, uint64_t *bytes_out) { + if (!path || !bytes_out) { + return false; + } + cbm_path_info_t info; + cbm_path_probe_status_t status = cbm_path_probe_info_utf8(path, &info); + if (status == CBM_PATH_PROBE_NOT_FOUND) { + *bytes_out = 0; + return true; + } + if (status != CBM_PATH_PROBE_OK || !info.is_regular || info.is_symlink || info.size < 0) { + return false; + } + *bytes_out = (uint64_t)info.size; + return true; +} + +static bool storage_replaceable_old_bytes(const char *final_path, uint64_t *bytes_out) { + if (!final_path || !bytes_out) { + return false; + } + *bytes_out = 0; + cbm_path_info_t info; + cbm_path_probe_status_t status = cbm_path_probe_info_utf8(final_path, &info); + if (status == CBM_PATH_PROBE_NOT_FOUND) { + return true; + } + if (status != CBM_PATH_PROBE_OK || !info.is_regular || info.is_symlink) { + return false; + } + cbm_store_t *store = cbm_store_open_path_query(final_path); + if (!store) { + /* A file that cannot be opened is not proven replaceable. Keep its + * bytes in the projection rather than blocking repair of a confirmed + * corrupt/non-database destination. */ + return true; + } + cbm_integrity_verdict_t verdict = cbm_store_check_integrity_verdict(store); + cbm_store_close(store); + if (verdict == CBM_INTEGRITY_TRANSIENT) { + return false; + } + /* A generation that cannot be proven healthy remains in current_cache but is + * not deducted: publication may quarantine it instead of replacing it. */ + return verdict != CBM_INTEGRITY_OK || cbm_db_artifact_bytes(final_path, bytes_out); +} + +static bool storage_probe_default(const cbm_index_resource_policy_t *policy, + cbm_pipeline_storage_checkpoint_t checkpoint, + const char *final_path, const char *staging_path, + cbm_index_storage_sample_t *sample, + cbm_index_resource_t *failed_resource) { + *sample = (cbm_index_storage_sample_t){0}; + *failed_resource = CBM_INDEX_RESOURCE_NONE; + char *storage_root = NULL; + if (policy->max_cache_bytes.enabled || policy->min_free_disk_bytes.enabled) { + storage_root = storage_root_from_final_path(final_path); + if (!storage_root) { + *failed_resource = storage_enabled_probe_resource(policy); + return false; + } + } + uint64_t operation_bytes = 0; + bool need_operation = + staging_path && + (policy->max_cache_bytes.enabled || policy->max_staging_bytes.enabled || + policy->max_task_temp_bytes.enabled || + (checkpoint == CBM_PIPELINE_STORAGE_PREPUBLISH && policy->max_final_db_bytes.enabled)); + if (need_operation && !cbm_db_artifact_bytes(staging_path, &operation_bytes)) { + *failed_resource = + policy->max_staging_bytes.enabled + ? CBM_INDEX_RESOURCE_STAGING_BYTES + : (policy->max_task_temp_bytes.enabled + ? CBM_INDEX_RESOURCE_TASK_TEMP_BYTES + : (policy->max_final_db_bytes.enabled ? CBM_INDEX_RESOURCE_FINAL_DB_BYTES + : CBM_INDEX_RESOURCE_CACHE_BYTES)); + free(storage_root); + return false; + } + if (policy->max_cache_bytes.enabled) { + uint64_t cache_with_operation = 0; + if (!cbm_directory_size_bytes(storage_root, &cache_with_operation) || + !storage_replaceable_old_bytes(final_path, &sample->replaceable_old_bytes)) { + *failed_resource = CBM_INDEX_RESOURCE_CACHE_BYTES; + free(storage_root); + return false; + } + sample->current_cache_bytes = + cache_with_operation > operation_bytes ? cache_with_operation - operation_bytes : 0; + sample->operation_bytes = operation_bytes; + } + if (policy->min_free_disk_bytes.enabled && + !cbm_filesystem_free_bytes(storage_root, &sample->free_disk_bytes)) { + *failed_resource = CBM_INDEX_RESOURCE_FREE_DISK_BYTES; + free(storage_root); + return false; + } + free(storage_root); + if (checkpoint != CBM_PIPELINE_STORAGE_PREFLIGHT) { + sample->staging_bytes = operation_bytes; + sample->task_temp_bytes = operation_bytes; + } + if (checkpoint == CBM_PIPELINE_STORAGE_PREPUBLISH && policy->max_final_db_bytes.enabled && + !storage_regular_file_bytes(staging_path, &sample->final_db_bytes)) { + *failed_resource = CBM_INDEX_RESOURCE_FINAL_DB_BYTES; + return false; + } + return true; +} + +int cbm_pipeline_storage_admit(cbm_pipeline_t *p, cbm_pipeline_storage_checkpoint_t checkpoint, + const char *final_path, const char *staging_path) { + if (!p || !cbm_index_policy_storage_enabled(&p->resource_policy)) { + return 0; + } + cbm_index_storage_sample_t sample; + cbm_index_resource_t failed_resource = CBM_INDEX_RESOURCE_NONE; + bool measured; +#ifdef CBM_ENABLE_TEST_SEAMS + if (p->storage_probe) { + measured = p->storage_probe(checkpoint, final_path, staging_path, &sample, &failed_resource, + p->storage_probe_context); + } else +#endif + { + measured = storage_probe_default(&p->resource_policy, checkpoint, final_path, staging_path, + &sample, &failed_resource); + } + if (!measured) { + const cbm_index_limit_u64_t *limit = + storage_limit_for_resource(&p->resource_policy, failed_resource); + p->resource_violation = (cbm_index_resource_violation_t){ + .resource = failed_resource, + .limit = limit && limit->enabled ? limit->value : 0, + .probe_failed = true, + }; + return CBM_PIPELINE_RESOURCE_LIMIT; + } + if (!cbm_index_policy_check_storage(&p->resource_policy, &sample, &p->resource_violation)) { + return CBM_PIPELINE_RESOURCE_LIMIT; + } + return 0; +} + static void cleanup_staging_db(const char *path) { if (!path) { return; @@ -2399,16 +2623,41 @@ static bool ensure_db_parent(const char *path) { return ok; } +static bool staging_token_valid(const char *token) { + size_t length = token ? strlen(token) : 0; + if (length < 6U || length >= CBM_SZ_64) { + return false; + } + for (size_t index = 0; index < length; index++) { + unsigned char ch = (unsigned char)token[index]; + if (!((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || + ch == '-' || ch == '_')) { + return false; + } + } + return true; +} + static char *create_staging_path(const char *final_path) { if (!final_path) { return NULL; } - static const char suffix[] = ".stage.XXXXXX"; + char suffix[CBM_SZ_128]; + const char *stage_token = getenv("CBM_INDEX_STAGE_TOKEN"); + if (!staging_token_valid(stage_token)) { + stage_token = NULL; + } + int suffix_length = stage_token && stage_token[0] + ? snprintf(suffix, sizeof(suffix), ".stage.%s.XXXXXX", stage_token) + : snprintf(suffix, sizeof(suffix), ".stage.XXXXXX"); + if (suffix_length <= 0 || (size_t)suffix_length >= sizeof(suffix)) { + return NULL; + } size_t final_len = strlen(final_path); - if (final_len > SIZE_MAX - sizeof(suffix)) { + if (final_len > SIZE_MAX - (size_t)suffix_length - 1U) { return NULL; } - size_t path_size = final_len + sizeof(suffix); + size_t path_size = final_len + (size_t)suffix_length + 1U; #ifdef _WIN32 /* The Windows cbm_mkstemp compatibility contract may expand a /tmp/ * prefix in-place and copies through a 4 KiB scratch path. Give it that @@ -2423,7 +2672,7 @@ static char *create_staging_path(const char *final_path) { return NULL; } memcpy(path, final_path, final_len); - memcpy(path + final_len, suffix, sizeof(suffix)); + memcpy(path + final_len, suffix, (size_t)suffix_length + 1U); int fd = cbm_mkstemp(path); if (fd < 0) { free(path); @@ -2542,11 +2791,18 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { if (!p) { return CBM_NOT_FOUND; } + p->resource_violation = (cbm_index_resource_violation_t){0}; char *final_path = resolve_db_path(p); if (!final_path || !ensure_db_parent(final_path)) { free(final_path); return CBM_NOT_FOUND; } + int storage_rc = + cbm_pipeline_storage_admit(p, CBM_PIPELINE_STORAGE_PREFLIGHT, final_path, NULL); + if (storage_rc != 0) { + free(final_path); + return storage_rc; + } struct stat final_st; bool final_existed = stat(final_path, &final_st) == 0; char *staging_path = create_staging_path(final_path); @@ -2589,6 +2845,14 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { free(final_path); return rc; } + storage_rc = + cbm_pipeline_storage_admit(p, CBM_PIPELINE_STORAGE_GROWTH, final_path, staging_path); + if (storage_rc != 0) { + cleanup_staging_db(staging_path); + free(staging_path); + free(final_path); + return storage_rc; + } if (check_cancel(p)) { cleanup_staging_db(staging_path); free(staging_path); @@ -2620,6 +2884,14 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { free(final_path); return CBM_PIPELINE_PERSIST_FAILED; } + storage_rc = + cbm_pipeline_storage_admit(p, CBM_PIPELINE_STORAGE_PREPUBLISH, final_path, staging_path); + if (storage_rc != 0) { + cleanup_staging_db(staging_path); + free(staging_path); + free(final_path); + return storage_rc; + } cbm_replacement_prepare_t prepared = {0}; int prepare_rc = diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index c62b847db..2502846b7 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -62,6 +62,21 @@ void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, const cbm_index_resourc void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p, cbm_index_resource_violation_t *violation); +typedef enum { + CBM_PIPELINE_STORAGE_PREFLIGHT = 0, + CBM_PIPELINE_STORAGE_GROWTH, + CBM_PIPELINE_STORAGE_PREPUBLISH, +} cbm_pipeline_storage_checkpoint_t; +#ifdef CBM_ENABLE_TEST_SEAMS +typedef bool (*cbm_pipeline_storage_probe_fn)(cbm_pipeline_storage_checkpoint_t checkpoint, + const char *final_db_path, + const char *staging_db_path, + cbm_index_storage_sample_t *sample, + cbm_index_resource_t *failed_resource, void *context); +void cbm_pipeline_set_storage_probe_for_testing(cbm_pipeline_t *p, + cbm_pipeline_storage_probe_fn probe, void *context); +#endif + /* Free a pipeline and all its internal state. NULL-safe. */ void cbm_pipeline_free(cbm_pipeline_t *p); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index c311bbd00..eac94ac94 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -708,6 +708,9 @@ typedef struct { bool surfaces_in_place; } cbm_pipeline_generation_t; +int cbm_pipeline_storage_admit(cbm_pipeline_t *p, cbm_pipeline_storage_checkpoint_t checkpoint, + const char *final_path, const char *staging_path); + /* Serialize and fully populate a sibling staging database, then atomically * replace final_db_path. The old generation is untouched on every failure or * cancellation observed before the rename commit point. */ diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index 5518c565f..e804ba7bd 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1400,6 +1400,7 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem atomic_init(&worker->cancelled, false); worker->result.exit_code = -1; if (worker->attempt < APP_FAKE_MAX_ATTEMPTS) { + worker->result.resource_violation = context->resource_violations[worker->attempt]; (void)snprintf(context->args_json[worker->attempt], APP_FAKE_ARGS_CAP, "%s", args_json ? args_json : ""); atomic_store(&context->args_captured[worker->attempt], true); @@ -1431,8 +1432,8 @@ static bool app_wait_for_atomic_bool(atomic_bool *value, bool expected); static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, int attempt, const char *max_files, const char *max_source_mb, - const char *max_rss_mb, - const char *max_duration_seconds) { + const char *max_rss_mb, const char *max_duration_seconds, + const char *cache_max_mb, const char *min_free_disk_mb) { if (!context || attempt < 0 || attempt >= APP_FAKE_MAX_ATTEMPTS) { return false; } @@ -1455,12 +1456,21 @@ static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, in yyjson_val *duration = policy && yyjson_is_obj(policy) ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS) : NULL; - bool equal = files && bytes && rss && duration && yyjson_is_str(files) && + yyjson_val *cache = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_CACHE_MAX_MB) + : NULL; + yyjson_val *free_disk = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB) + : NULL; + bool equal = files && bytes && rss && duration && cache && free_disk && yyjson_is_str(files) && yyjson_is_str(bytes) && yyjson_is_str(rss) && yyjson_is_str(duration) && + yyjson_is_str(cache) && yyjson_is_str(free_disk) && strcmp(yyjson_get_str(files), max_files) == 0 && strcmp(yyjson_get_str(bytes), max_source_mb) == 0 && strcmp(yyjson_get_str(rss), max_rss_mb) == 0 && - strcmp(yyjson_get_str(duration), max_duration_seconds) == 0; + strcmp(yyjson_get_str(duration), max_duration_seconds) == 0 && + strcmp(yyjson_get_str(cache), cache_max_mb) == 0 && + strcmp(yyjson_get_str(free_disk), min_free_disk_mb) == 0; yyjson_doc_free(document); return equal; } @@ -2074,7 +2084,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { app_wait_for_subscribers(application, project, 1) && app_wait_for_atomic_int(&fake.starts, 1); bool auto_policy_propagated = - first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4", "64", "7"); + first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4", "64", "7", "off", "off"); bool second_initialized = app_test_initialize_profile(&callbacks, sessions[1], root, CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); bool coalesced = first_owned && second_initialized && project && @@ -2161,7 +2171,9 @@ TEST(daemon_application_programmatic_index_injects_resource_policy) { stored_config && cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 && cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0 && cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7") == 0; + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_CACHE_MAX_MB, "8") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, "9") == 0; app_fake_worker_context_t fake; app_fake_worker_context_init(&fake); @@ -2182,7 +2194,7 @@ TEST(daemon_application_programmatic_index_injects_resource_policy) { int index_rc = application ? cbm_daemon_application_index(application, "policy-programmatic", root) : -1; bool policy_propagated = - index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6", "64", "7"); + index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6", "64", "7", "8", "9"); bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); cbm_daemon_application_free(application); @@ -2275,6 +2287,41 @@ TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { PASS(); } +TEST(daemon_application_worker_storage_resource_failure_is_structured_and_not_retried) { + app_watch_race_fixture_t fixture; + bool ready = app_watch_race_fixture_init(&fixture, 77); + atomic_store(&fixture.fake.scripted, true); + fixture.fake.outcomes[0] = CBM_PROC_KILLED; + fixture.fake.resource_violations[0] = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_TASK_TEMP_BYTES, .observed = 2048, .limit = 1024}; + + char args[APP_TEST_PATH_CAP + 32]; + (void)snprintf(args, sizeof(args), "{\"repo_path\":\"%s\"}", fixture.root); + uint8_t *request = NULL; + uint32_t request_length = 0; + uint8_t *response = NULL; + uint32_t response_length = 0; + bool encoded = + ready && app_test_tool_request("index_repository", args, &request, &request_length); + bool completed = encoded && app_test_request(&fixture.callbacks, fixture.session, request, + request_length, &response, &response_length) == + CBM_DAEMON_RUNTIME_APPLICATION_OK; + bool structured = + completed && response && strstr((char *)response, "resource_limit_exceeded") && + strstr((char *)response, "task_temp_bytes") && strstr((char *)response, "storage"); + int starts = atomic_load(&fixture.fake.starts); + free(request); + free(response); + bool cleaned = app_watch_race_fixture_finish(&fixture); + + ASSERT_TRUE(ready); + ASSERT_TRUE(completed); + ASSERT_TRUE(structured); + ASSERT_EQ(starts, 1); + ASSERT_TRUE(cleaned); + PASS(); +} + TEST(daemon_application_auto_index_honors_tracked_file_limit) { app_env_backup_t cache_environment; bool cache_saved = app_env_backup_capture(&cache_environment, "CBM_CACHE_DIR"); @@ -5285,6 +5332,7 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions); RUN_TEST(daemon_application_programmatic_index_injects_resource_policy); RUN_TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried); + RUN_TEST(daemon_application_worker_storage_resource_failure_is_structured_and_not_retried); RUN_TEST(daemon_application_auto_index_honors_tracked_file_limit); RUN_TEST(daemon_application_auto_index_file_count_handles_literal_metacharacter_path); RUN_TEST(daemon_application_auto_index_file_count_supports_non_git_roots); diff --git a/tests/test_index_policy.c b/tests/test_index_policy.c index b33d64cdb..dc362f6b5 100644 --- a/tests/test_index_policy.c +++ b/tests/test_index_policy.c @@ -3,6 +3,7 @@ #include "cli/cli.h" #include "foundation/compat.h" +#include "foundation/constants.h" #include "foundation/index_policy.h" #include "mcp/index_supervisor.h" #include "mcp/mcp.h" @@ -23,13 +24,21 @@ TEST(index_policy_defaults_are_disabled) { ASSERT_FALSE(policy.max_source_bytes.enabled); ASSERT_FALSE(policy.max_rss_bytes.enabled); ASSERT_FALSE(policy.max_duration_ms.enabled); + ASSERT_FALSE(policy.max_cache_bytes.enabled); + ASSERT_FALSE(policy.min_free_disk_bytes.enabled); + ASSERT_FALSE(policy.max_final_db_bytes.enabled); + ASSERT_FALSE(policy.max_staging_bytes.enabled); + ASSERT_FALSE(policy.max_task_temp_bytes.enabled); ASSERT_FALSE(cbm_index_policy_enabled(&policy)); ASSERT_FALSE(cbm_index_policy_discovery_enabled(&policy)); ASSERT_FALSE(cbm_index_policy_worker_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_storage_enabled(&policy)); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_FILES), "off"); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_SOURCE_MB), "off"); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_RSS_MB), "off"); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_DURATION_SECONDS), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_CACHE_MAX_MB), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MIN_FREE_DISK_MB), "off"); PASS(); } @@ -98,6 +107,70 @@ TEST(index_policy_worker_limits_validate_and_convert_units) { PASS(); } +TEST(index_policy_storage_limits_validate_and_convert_units) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + char value[64]; + + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_CACHE_MAX_MB, "131072", error, + sizeof(error))); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, "4096", error, + sizeof(error))); + ASSERT_EQ(policy.max_cache_bytes.value, UINT64_C(131072) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.min_free_disk_bytes.value, UINT64_C(4096) * CBM_INDEX_MIB_BYTES); + ASSERT_TRUE(cbm_index_policy_storage_enabled(&policy)); + ASSERT_TRUE( + cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_CACHE_MAX_MB, value, sizeof(value))); + ASSERT_STR_EQ(value, "131072"); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_CACHE_MAX_MB, "off", error, sizeof(error))); + ASSERT_FALSE(policy.max_cache_bytes.enabled); + PASS(); +} + +TEST(index_policy_storage_projection_boundaries_are_overflow_safe) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_cache_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 100}; + policy.min_free_disk_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 40}; + policy.max_final_db_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 60}; + policy.max_staging_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 70}; + policy.max_task_temp_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 80}; + cbm_index_storage_sample_t sample = { + .current_cache_bytes = 130, + .replaceable_old_bytes = 50, + .operation_bytes = 20, + .free_disk_bytes = 40, + .final_db_bytes = 60, + .staging_bytes = 70, + .task_temp_bytes = 80, + }; + cbm_index_resource_violation_t violation = {0}; + + ASSERT_TRUE(cbm_index_policy_check_storage(&policy, &sample, &violation)); + sample.current_cache_bytes++; + ASSERT_FALSE(cbm_index_policy_check_storage(&policy, &sample, &violation)); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_CACHE_BYTES); + ASSERT_EQ(violation.observed, 101); + ASSERT_EQ(violation.limit, 100); + sample.current_cache_bytes--; + sample.free_disk_bytes--; + ASSERT_FALSE(cbm_index_policy_check_storage(&policy, &sample, &violation)); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FREE_DISK_BYTES); + ASSERT_EQ(violation.observed, 39); + sample = (cbm_index_storage_sample_t){ + .current_cache_bytes = UINT64_MAX, + .replaceable_old_bytes = 0, + .operation_bytes = UINT64_MAX, + .free_disk_bytes = UINT64_MAX, + }; + ASSERT_FALSE(cbm_index_policy_check_storage(&policy, &sample, &violation)); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_CACHE_BYTES); + ASSERT_EQ(violation.observed, UINT64_MAX); + PASS(); +} + TEST(index_policy_invalid_value_is_rejected_atomically) { static const char *const invalid[] = {"", "0", "-1", "1MB", "1 ", "+1", "10000001", "18446744073709551616"}; @@ -149,6 +222,9 @@ TEST(index_policy_format_round_trips_public_values) { TEST(index_policy_violation_metadata_is_stable) { ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_FILES), "files"); ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_SOURCE_BYTES), "source_bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_CACHE_BYTES), "cache_bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_FREE_DISK_BYTES), "free_disk_bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_STAGING_BYTES), "staging_bytes"); ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_FILES), "files"); ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_SOURCE_BYTES), "bytes"); ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_RSS_BYTES), "rss_bytes"); @@ -163,6 +239,10 @@ TEST(index_policy_violation_metadata_is_stable) { CBM_INDEX_CONFIG_MAX_RSS_MB); ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_DURATION_MS), CBM_INDEX_CONFIG_MAX_DURATION_SECONDS); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_CACHE_BYTES), + CBM_INDEX_CONFIG_CACHE_MAX_MB); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_FREE_DISK_BYTES), + CBM_INDEX_CONFIG_MIN_FREE_DISK_MB); PASS(); } @@ -180,11 +260,15 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "3"), 0); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64"), 0); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_CACHE_MAX_MB, "17"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, "5"), 0); ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); ASSERT_EQ(policy.max_files.value, 9); ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(3) * CBM_INDEX_MIB_BYTES); ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(64) * CBM_INDEX_MIB_BYTES); ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(7000)); + ASSERT_EQ(policy.max_cache_bytes.value, UINT64_C(17) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.min_free_disk_bytes.value, UINT64_C(5) * CBM_INDEX_MIB_BYTES); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "corrupt"), 0); ASSERT_FALSE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); @@ -200,6 +284,8 @@ TEST(index_policy_cli_lists_all_operator_keys) { bool bytes_found = false; bool rss_found = false; bool duration_found = false; + bool cache_found = false; + bool free_found = false; for (size_t index = 0; index < cbm_cli_config_key_count_for_testing(); index++) { const char *key = cbm_cli_config_key_at_for_testing(index); files_found = files_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0); @@ -207,11 +293,15 @@ TEST(index_policy_cli_lists_all_operator_keys) { rss_found = rss_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_RSS_MB) == 0); duration_found = duration_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS) == 0); + cache_found = cache_found || (key && strcmp(key, CBM_INDEX_CONFIG_CACHE_MAX_MB) == 0); + free_found = free_found || (key && strcmp(key, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB) == 0); } ASSERT_TRUE(files_found); ASSERT_TRUE(bytes_found); ASSERT_TRUE(rss_found); ASSERT_TRUE(duration_found); + ASSERT_TRUE(cache_found); + ASSERT_TRUE(free_found); PASS(); } @@ -454,11 +544,92 @@ TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index) { PASS(); } +TEST(index_policy_storage_limit_preserves_old_index_and_unrelated_cache_files) { + char *repo = th_mktempdir("cbm_index_storage_repo"); + char *cache = th_mktempdir("cbm_index_storage_cache"); + ASSERT_NOT_NULL(repo); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int original(void) { return 1; }\n"), 0); + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + cbm_config_t *config = cbm_config_open(cache); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + if (server && config) { + cbm_mcp_server_set_config(server, config); + } + char args[CBM_SZ_4K]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"StoragePolicyFixture\",\"mode\":\"fast\"}", + repo); + char *first = server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + bool first_ok = first && strstr(first, "\\\"status\\\":\\\"indexed\\\""); + free(first); + + char unrelated[CBM_SZ_4K]; + (void)snprintf(unrelated, sizeof(unrelated), "%s/unrelated.bin", cache); + FILE *large = cbm_fopen(unrelated, "wb"); + char block[4096] = {0}; + bool wrote_large = large != NULL; + for (int index = 0; wrote_large && index < 512; index++) { + wrote_large = fwrite(block, 1, sizeof(block), large) == sizeof(block); + } + if (large) { + wrote_large = fclose(large) == 0 && wrote_large; + } + bool configured = + config && cbm_config_set(config, CBM_INDEX_CONFIG_CACHE_MAX_MB, "1") == 0 && + th_write_file(TH_PATH(repo, "main.c"), "int changed(void) { return 2; }\n") == 0; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"StoragePolicyFixture\",\"mode\":\"fast\"," + "\"_cbm_index_policy\":{\"index_max_files\":\"off\"," + "\"index_max_source_mb\":\"off\",\"index_cache_max_mb\":\"off\"," + "\"index_min_free_disk_mb\":\"off\"}}", + repo); + char *limited = configured && wrote_large && server + ? cbm_mcp_handle_tool(server, "index_repository", args) + : NULL; + bool contract_ok = limited && strstr(limited, "resource_limit_exceeded") && + strstr(limited, "\\\"stage\\\":\\\"storage\\\"") && + strstr(limited, "\\\"resource\\\":\\\"cache_bytes\\\"") && + strstr(limited, "\\\"limit\\\":1048576") && + strstr(limited, "\\\"unit\\\":\\\"bytes\\\"") && + strstr(limited, "\\\"serving_index_preserved\\\":true"); + free(limited); + int64_t unrelated_size = cbm_file_size(unrelated); + char db_path[CBM_SZ_4K]; + (void)snprintf(db_path, sizeof(db_path), "%s/StoragePolicyFixture.db", cache); + cbm_store_t *store = cbm_store_open_path_query(db_path); + bool old_queryable = store && cbm_store_count_nodes(store, "StoragePolicyFixture") > 0; + cbm_store_close(store); + + cbm_mcp_server_free(server); + cbm_config_close(config); + th_cleanup(repo); + th_cleanup(cache); + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); + + ASSERT_TRUE(first_ok); + ASSERT_TRUE(wrote_large); + ASSERT_TRUE(configured); + ASSERT_TRUE(contract_ok); + ASSERT_EQ(unrelated_size, (int64_t)sizeof(block) * 512); + ASSERT_TRUE(old_queryable); + PASS(); +} + SUITE(index_policy) { RUN_TEST(index_policy_defaults_are_disabled); RUN_TEST(index_policy_file_limit_accepts_off_and_exact_range); RUN_TEST(index_policy_source_limit_converts_mib_without_overflow); RUN_TEST(index_policy_worker_limits_validate_and_convert_units); + RUN_TEST(index_policy_storage_limits_validate_and_convert_units); + RUN_TEST(index_policy_storage_projection_boundaries_are_overflow_safe); RUN_TEST(index_policy_invalid_value_is_rejected_atomically); RUN_TEST(index_policy_format_round_trips_public_values); RUN_TEST(index_policy_violation_metadata_is_stable); @@ -468,4 +639,5 @@ SUITE(index_policy) { RUN_TEST(index_policy_cli_set_reports_a_failed_write); RUN_TEST(index_policy_worker_rejects_missing_parent_policy); RUN_TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index); + RUN_TEST(index_policy_storage_limit_preserves_old_index_and_unrelated_cache_files); } diff --git a/tests/test_index_supervisor.c b/tests/test_index_supervisor.c index b061f0dd1..87028d471 100644 --- a/tests/test_index_supervisor.c +++ b/tests/test_index_supervisor.c @@ -121,6 +121,24 @@ static bool index_supervisor_test_append_log(const char *path, const char *line) return fclose(file) == 0 && written; } +typedef struct { + uint64_t bytes; + bool succeeds; + int calls; +} index_supervisor_task_temp_fake_t; + +static bool index_supervisor_task_temp_fake(const char *final_db_path, const char *log_path, + const char *response_path, uint64_t *bytes_out, + void *context) { + index_supervisor_task_temp_fake_t *fake = context; + fake->calls++; + if (!final_db_path || !log_path || !response_path || !fake->succeeds) { + return false; + } + *bytes_out = fake->bytes; + return true; +} + static bool index_supervisor_test_append_terminal_backlog(const char *path) { FILE *file = cbm_fopen(path, "ab"); if (!file) { @@ -236,18 +254,21 @@ TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar) { "/tmp/m", CBM_INDEX_WORKER_QUARANTINE_ARG, "/tmp/q", + CBM_INDEX_WORKER_STAGE_TOKEN_ARG, + "task123", NULL, }; cbm_index_worker_invocation_t invocation; - ASSERT_EQ(cbm_index_worker_parse_process_argv(16, valid, &invocation), + ASSERT_EQ(cbm_index_worker_parse_process_argv(18, valid, &invocation), CBM_INDEX_WORKER_ARGV_VALID); - ASSERT_EQ(cbm_daemon_process_role(16, valid), CBM_DAEMON_PROCESS_WORKER); + ASSERT_EQ(cbm_daemon_process_role(18, valid), CBM_DAEMON_PROCESS_WORKER); ASSERT_STR_EQ(invocation.args_json, valid[6]); ASSERT_STR_EQ(invocation.response_out, valid[8]); ASSERT_EQ(invocation.memory_budget_bytes, 1024); ASSERT_TRUE(invocation.single_thread); ASSERT_STR_EQ(invocation.marker_file, valid[13]); ASSERT_STR_EQ(invocation.quarantine_file, valid[15]); + ASSERT_STR_EQ(invocation.stage_token, valid[17]); char *missing_build[] = {"test-runner", "cli", "--index-worker", "index_repository", "{}", "--response-out", @@ -311,6 +332,18 @@ TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar) { CBM_INDEX_WORKER_MEMORY_BUDGET_ARG, "184467440737095516160", NULL}; + char *invalid_stage_token[] = {"test-runner", + "cli", + "--index-worker", + CBM_INDEX_WORKER_BUILD_ARG, + (char *)captured, + "index_repository", + "{}", + "--response-out", + "/tmp/r", + CBM_INDEX_WORKER_STAGE_TOKEN_ARG, + "../escape", + NULL}; char *user_value[] = {"test-runner", "cli", "search_code", "--query", "--index-worker", NULL}; ASSERT_EQ(cbm_index_worker_parse_process_argv(7, missing_build, &invocation), CBM_INDEX_WORKER_ARGV_INVALID); @@ -327,6 +360,9 @@ TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar) { ASSERT_EQ(cbm_index_worker_parse_process_argv(11, overflow_budget, &invocation), CBM_INDEX_WORKER_ARGV_INVALID); ASSERT_EQ(cbm_daemon_process_role(11, overflow_budget), CBM_DAEMON_PROCESS_INVALID); + ASSERT_EQ(cbm_index_worker_parse_process_argv(11, invalid_stage_token, &invocation), + CBM_INDEX_WORKER_ARGV_INVALID); + ASSERT_EQ(cbm_daemon_process_role(11, invalid_stage_token), CBM_DAEMON_PROCESS_INVALID); ASSERT_EQ(cbm_index_worker_parse_process_argv(5, user_value, &invocation), CBM_INDEX_WORKER_ARGV_INVALID); ASSERT_EQ(cbm_daemon_process_role(5, user_value), CBM_DAEMON_PROCESS_INVALID); @@ -1112,6 +1148,42 @@ TEST(index_supervisor_three_failed_rss_probes_fail_closed) { PASS(); } +TEST(index_supervisor_task_temp_limit_terminates_as_resource_failure) { + char *root = th_mktempdir("cbm_supervisor_task_temp"); + ASSERT_NOT_NULL(root); + char db_path[INDEX_SUPERVISOR_TEST_PATH_CAP]; + (void)snprintf(db_path, sizeof(db_path), "%s/index.db", root); + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_task_temp_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + index_supervisor_task_temp_fake_t fake = {.bytes = 2, .succeeds = true}; + cbm_index_supervisor_set_task_temp_hook_for_testing(index_supervisor_task_temp_fake, &fake); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = + cbm_index_worker_start_with_storage_policy("{\"__cbm_test_worker\":\"clean\"}", 0, &policy, + db_path, false, NULL, NULL, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool cancelled = result ? result->cancellation_requested : true; + cbm_index_resource_violation_t violation = + result ? result->resource_violation : (cbm_index_resource_violation_t){0}; + cbm_index_supervisor_reset_task_temp_hook_for_testing(); + if (handle) { + cbm_index_worker_destroy(handle); + } + th_cleanup(root); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(terminal); + ASSERT_FALSE(cancelled); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_TASK_TEMP_BYTES); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + ASSERT_TRUE(fake.calls > 0); + PASS(); +} + SUITE(index_supervisor) { RUN_TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar); RUN_TEST(index_supervisor_async_jobs_are_isolated_cancellable_and_terminal_cached); @@ -1126,4 +1198,5 @@ SUITE(index_supervisor) { RUN_TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled); RUN_TEST(index_supervisor_cancel_precedes_resource_probe); RUN_TEST(index_supervisor_three_failed_rss_probes_fail_closed); + RUN_TEST(index_supervisor_task_temp_limit_terminates_as_resource_failure); } diff --git a/tests/test_main.c b/tests/test_main.c index b38cc912b..1562a9650 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -341,7 +341,7 @@ static int tf_maybe_run_index_worker(int argc, char **argv) { free(worker_repo_path); cbm_index_set_worker_role_options(true, invocation.response_out, invocation.single_thread, invocation.marker_file, invocation.quarantine_file, - invocation.memory_budget_bytes); + invocation.memory_budget_bytes, invocation.stage_token); cbm_mem_init_with_cap(0.5, invocation.memory_budget_bytes); tf_index_worker_probe(invocation.args_json, invocation.response_out); cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 37c0b9374..9472565dc 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -9603,13 +9603,13 @@ TEST(index_supervisor_unsafe_clean_is_never_fallback_or_recovery) { result.supervision_failed = false; result.tree_quiesced = true; result.outcome = CBM_PROC_KILLED; - result.resource_violation.resource = CBM_INDEX_RESOURCE_RSS_BYTES; + result.resource_violation = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_TASK_TEMP_BYTES, .observed = 2048, .limit = 1024}; ASSERT_EQ(cbm_mcp_supervised_result_disposition(0, &result), CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE); result.tree_quiesced = false; ASSERT_EQ(cbm_mcp_supervised_result_disposition(0, &result), CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL); - result.resource_violation = (cbm_index_resource_violation_t){0}; result.tree_quiesced = true; result.outcome = CBM_PROC_CRASH; @@ -9717,6 +9717,26 @@ TEST(index_worker_resource_limit_is_trusted_structured_and_not_retried) { #endif } +TEST(index_supervisor_resource_probe_failure_response_is_structured) { + cbm_index_worker_result_t worker_result = { + .resource_violation = + { + .resource = CBM_INDEX_RESOURCE_CACHE_BYTES, + .limit = 1024, + .probe_failed = true, + }, + }; + char *response = + cbm_mcp_index_worker_resource_response("{\"repo_path\":\"/tmp/missing\"}", &worker_result); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "\"code\":\"resource_probe_failed\"")); + ASSERT_NOT_NULL(strstr(response, "\"stage\":\"storage\"")); + ASSERT_NOT_NULL(strstr(response, "\"resource\":\"cache_bytes\"")); + ASSERT_NOT_NULL(strstr(response, "\"retryable\":true")); + free(response); + PASS(); +} + /* Child-side check: index a tiny fixture and verify it ran IN-PROCESS. * Distinct exit codes so the parent can report the exact failure mode. */ enum { @@ -10647,6 +10667,57 @@ TEST(mcp_auto_watch_false_skips_watcher_on_connect) { PASS(); } +TEST(mcp_inprocess_autoindex_inherits_resource_policy) { + char *root = th_mktempdir("cbm-autoindex-policy-root"); + char *cache = th_mktempdir("cbm-autoindex-policy-cache"); + ASSERT_NOT_NULL(root); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(root, "first.c"), "int first(void) { return 1; }\n"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "second.c"), "int second(void) { return 2; }\n"), 0); + + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + char saved_cwd[CBM_SZ_4K]; + bool cwd_saved = cbm_getcwd(saved_cwd, sizeof(saved_cwd)) != NULL; + bool environment_ready = + cwd_saved && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0 && cbm_chdir(root) == 0; + cbm_config_t *config = environment_ready ? cbm_config_open(cache) : NULL; + bool configured = config && cbm_config_set(config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && + cbm_config_set(config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && + cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "1") == 0; + char *project = configured ? cbm_project_name_from_path(root) : NULL; + char db_path[CBM_SZ_4K] = {0}; + if (project) { + (void)snprintf(db_path, sizeof(db_path), "%s/%s.db", cache, project); + } + + cbm_mcp_server_t *server = configured && project ? cbm_mcp_server_new(NULL) : NULL; + if (server) { + cbm_mcp_server_set_config(server, config); + char *response = cbm_mcp_server_handle( + server, "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"initialize\",\"params\":{}}"); + free(response); + cbm_mcp_server_free(server); /* joins the in-process auto-index thread */ + } + bool limit_preserved_no_index = project && !cbm_file_exists(db_path); + + cbm_config_close(config); + if (cwd_saved) { + (void)cbm_chdir(saved_cwd); + } + restore_cache_dir(saved_cache_copy); + free(saved_cache_copy); + free(project); + th_cleanup(root); + th_cleanup(cache); + + ASSERT_TRUE(environment_ready); + ASSERT_TRUE(configured); + ASSERT_NOT_NULL(server); + ASSERT_TRUE(limit_preserved_no_index); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ * #853 — auto_watch=false must ALSO gate the SUPERVISED fresh-index * watcher registration (keystone × #849 merge interaction) @@ -11509,6 +11580,7 @@ SUITE(mcp) { RUN_TEST(index_repository_cli_name_override_issue823); RUN_TEST(index_supervisor_unsafe_clean_is_never_fallback_or_recovery); RUN_TEST(index_worker_resource_limit_is_trusted_structured_and_not_retried); + RUN_TEST(index_supervisor_resource_probe_failure_response_is_structured); RUN_TEST(index_supervisor_gate_requires_marked_host_issue845); RUN_TEST(index_supervisor_start_failure_is_fail_closed_in_real_host); RUN_TEST(index_bg_paths_route_through_supervisor_issue832); @@ -11579,6 +11651,7 @@ SUITE(mcp) { /* auto_watch gate (distilled from PR #625) */ RUN_TEST(mcp_auto_watch_default_registers_watcher_on_connect); RUN_TEST(mcp_auto_watch_false_skips_watcher_on_connect); + RUN_TEST(mcp_inprocess_autoindex_inherits_resource_policy); RUN_TEST(mcp_auto_watch_false_skips_supervised_autoindex_issue853); } diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 5ef810908..2a9a5cc2c 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -179,6 +179,212 @@ TEST(pipeline_discovery_limit_returns_exact_violation_without_publishing) { PASS(); } +typedef struct { + const char *target_db; + cbm_pipeline_storage_checkpoint_t fail_at; + cbm_index_resource_t failed_resource; + cbm_index_storage_sample_t samples[3]; + bool sample_set[3]; + int calls[3]; + int unexpected_paths; +} pipeline_storage_probe_fake_t; + +static bool pipeline_storage_probe_fake(cbm_pipeline_storage_checkpoint_t checkpoint, + const char *final_db_path, const char *staging_db_path, + cbm_index_storage_sample_t *sample, + cbm_index_resource_t *failed_resource, void *context) { + (void)staging_db_path; + pipeline_storage_probe_fake_t *fake = (pipeline_storage_probe_fake_t *)context; + fake->calls[checkpoint]++; + if (fake->target_db && (!final_db_path || strcmp(fake->target_db, final_db_path) != 0)) { + fake->unexpected_paths++; + } + *sample = (cbm_index_storage_sample_t){ + .current_cache_bytes = 0, + .free_disk_bytes = UINT64_MAX, + }; + if (fake->sample_set[checkpoint]) { + *sample = fake->samples[checkpoint]; + } + if (fake->failed_resource != CBM_INDEX_RESOURCE_NONE && checkpoint == fake->fail_at && + fake->target_db && final_db_path && strcmp(fake->target_db, final_db_path) == 0) { + *failed_resource = fake->failed_resource; + return false; + } + return true; +} + +static int pipeline_stage_artifact_count(const char *directory) { + cbm_dir_t *dir = cbm_opendir(directory); + if (!dir) { + return -1; + } + int count = 0; + cbm_dirent_t *entry; + while ((entry = cbm_readdir(dir)) != NULL) { + if (strstr(entry->name, ".stage.")) { + count++; + } + } + cbm_closedir(dir); + return count; +} + +TEST(pipeline_disabled_storage_policy_performs_no_probes) { + char *repo = th_mktempdir("cbm_pipeline_storage_off"); + ASSERT_NOT_NULL(repo); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int main(void) { return 0; }\n"), 0); + char db_path[CBM_SZ_4K]; + (void)snprintf(db_path, sizeof(db_path), "%s/index.db", repo); + cbm_pipeline_t *pipeline = cbm_pipeline_new(repo, db_path, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + pipeline_storage_probe_fake_t fake = {.target_db = db_path, + .fail_at = CBM_PIPELINE_STORAGE_PREFLIGHT, + .failed_resource = CBM_INDEX_RESOURCE_FREE_DISK_BYTES}; + cbm_pipeline_set_storage_probe_for_testing(pipeline, pipeline_storage_probe_fake, &fake); + + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + ASSERT_EQ(fake.calls[CBM_PIPELINE_STORAGE_PREFLIGHT], 0); + ASSERT_EQ(fake.calls[CBM_PIPELINE_STORAGE_GROWTH], 0); + ASSERT_EQ(fake.calls[CBM_PIPELINE_STORAGE_PREPUBLISH], 0); + cbm_pipeline_free(pipeline); + th_cleanup(repo); + PASS(); +} + +TEST(pipeline_storage_probe_failure_preserves_old_db_and_cleans_stage) { + char *repo = th_mktempdir("cbm_pipeline_storage_probe"); + ASSERT_NOT_NULL(repo); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int original(void) { return 1; }\n"), 0); + char db_path[CBM_SZ_4K]; + (void)snprintf(db_path, sizeof(db_path), "%s/index.db", repo); + cbm_pipeline_t *first = cbm_pipeline_new(repo, db_path, CBM_MODE_FAST); + ASSERT_NOT_NULL(first); + ASSERT_EQ(cbm_pipeline_run(first), 0); + const char *project = cbm_pipeline_project_name(first); + char *project_copy = project ? strdup(project) : NULL; + cbm_pipeline_free(first); + ASSERT_NOT_NULL(project_copy); + + cbm_store_t *before = cbm_store_open_path_query(db_path); + cbm_project_t before_project = {0}; + ASSERT_NOT_NULL(before); + ASSERT_EQ(cbm_store_get_project(before, project_copy, &before_project), CBM_STORE_OK); + char *indexed_at = before_project.indexed_at ? strdup(before_project.indexed_at) : NULL; + cbm_project_free_fields(&before_project); + cbm_store_close(before); + ASSERT_NOT_NULL(indexed_at); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int changed(void) { return 2; }\n"), 0); + + cbm_pipeline_t *second = cbm_pipeline_new(repo, db_path, CBM_MODE_FAST); + ASSERT_NOT_NULL(second); + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.min_free_disk_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_pipeline_set_resource_policy(second, &policy); + pipeline_storage_probe_fake_t fake = {.target_db = db_path, + .fail_at = CBM_PIPELINE_STORAGE_PREPUBLISH, + .failed_resource = CBM_INDEX_RESOURCE_FREE_DISK_BYTES}; + cbm_pipeline_set_storage_probe_for_testing(second, pipeline_storage_probe_fake, &fake); + int rc = cbm_pipeline_run(second); + cbm_index_resource_violation_t violation = {0}; + cbm_pipeline_get_resource_violation(second, &violation); + cbm_pipeline_free(second); + + cbm_store_t *after = cbm_store_open_path_query(db_path); + cbm_project_t after_project = {0}; + bool preserved = after && + cbm_store_get_project(after, project_copy, &after_project) == CBM_STORE_OK && + after_project.indexed_at && strcmp(indexed_at, after_project.indexed_at) == 0; + cbm_project_free_fields(&after_project); + cbm_store_close(after); + int stage_count = pipeline_stage_artifact_count(repo); + free(indexed_at); + free(project_copy); + th_cleanup(repo); + + ASSERT_EQ(rc, CBM_PIPELINE_RESOURCE_LIMIT); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FREE_DISK_BYTES); + ASSERT_TRUE(violation.probe_failed); + ASSERT_TRUE(fake.calls[CBM_PIPELINE_STORAGE_PREPUBLISH] > 0); + ASSERT_TRUE(preserved); + ASSERT_EQ(stage_count, 0); + PASS(); +} + +TEST(pipeline_full_and_incremental_storage_limits_report_identical_failure) { + char *full_repo = th_mktempdir("cbm_pipeline_storage_full"); + char *incremental_repo = th_mktempdir("cbm_pipeline_storage_incremental"); + ASSERT_NOT_NULL(full_repo); + ASSERT_NOT_NULL(incremental_repo); + ASSERT_EQ(th_write_file(TH_PATH(full_repo, "main.c"), "int full(void) { return 1; }\n"), 0); + ASSERT_EQ( + th_write_file(TH_PATH(incremental_repo, "main.c"), "int incremental(void) { return 1; }\n"), + 0); + char full_db[CBM_SZ_4K]; + char incremental_db[CBM_SZ_4K]; + (void)snprintf(full_db, sizeof(full_db), "%s/index.db", full_repo); + (void)snprintf(incremental_db, sizeof(incremental_db), "%s/index.db", incremental_repo); + + cbm_pipeline_t *seed = cbm_pipeline_new(incremental_repo, incremental_db, CBM_MODE_FAST); + ASSERT_NOT_NULL(seed); + ASSERT_EQ(cbm_pipeline_run(seed), 0); + cbm_pipeline_free(seed); + ASSERT_EQ( + th_write_file(TH_PATH(incremental_repo, "main.c"), "int incremental(void) { return 2; }\n"), + 0); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_staging_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 100}; + pipeline_storage_probe_fake_t full_fake = {.target_db = full_db}; + full_fake.sample_set[CBM_PIPELINE_STORAGE_GROWTH] = true; + full_fake.samples[CBM_PIPELINE_STORAGE_GROWTH] = + (cbm_index_storage_sample_t){.free_disk_bytes = UINT64_MAX, .staging_bytes = 101}; + pipeline_storage_probe_fake_t incremental_fake = full_fake; + incremental_fake.target_db = incremental_db; + + cbm_pipeline_t *full = cbm_pipeline_new(full_repo, full_db, CBM_MODE_FAST); + cbm_pipeline_t *incremental = cbm_pipeline_new(incremental_repo, incremental_db, CBM_MODE_FAST); + ASSERT_NOT_NULL(full); + ASSERT_NOT_NULL(incremental); + cbm_pipeline_set_resource_policy(full, &policy); + cbm_pipeline_set_resource_policy(incremental, &policy); + cbm_pipeline_set_storage_probe_for_testing(full, pipeline_storage_probe_fake, &full_fake); + cbm_pipeline_set_storage_probe_for_testing(incremental, pipeline_storage_probe_fake, + &incremental_fake); + int full_rc = cbm_pipeline_run(full); + int incremental_rc = cbm_pipeline_run(incremental); + cbm_index_resource_violation_t full_violation = {0}; + cbm_index_resource_violation_t incremental_violation = {0}; + cbm_pipeline_get_resource_violation(full, &full_violation); + cbm_pipeline_get_resource_violation(incremental, &incremental_violation); + cbm_pipeline_free(full); + cbm_pipeline_free(incremental); + + ASSERT_EQ(full_rc, CBM_PIPELINE_RESOURCE_LIMIT); + ASSERT_EQ(incremental_rc, CBM_PIPELINE_RESOURCE_LIMIT); + ASSERT_EQ(full_violation.resource, CBM_INDEX_RESOURCE_STAGING_BYTES); + ASSERT_EQ(incremental_violation.resource, full_violation.resource); + ASSERT_EQ(incremental_violation.observed, full_violation.observed); + ASSERT_EQ(incremental_violation.limit, full_violation.limit); + ASSERT_STR_EQ(cbm_index_resource_unit(incremental_violation.resource), + cbm_index_resource_unit(full_violation.resource)); + ASSERT_EQ(full_fake.unexpected_paths, 0); + ASSERT_EQ(incremental_fake.unexpected_paths, 0); + ASSERT_EQ(full_fake.calls[CBM_PIPELINE_STORAGE_PREFLIGHT], 1); + ASSERT_EQ(full_fake.calls[CBM_PIPELINE_STORAGE_GROWTH], 1); + ASSERT_EQ(full_fake.calls[CBM_PIPELINE_STORAGE_PREPUBLISH], 0); + ASSERT_EQ(incremental_fake.calls[CBM_PIPELINE_STORAGE_PREFLIGHT], 1); + ASSERT_EQ(incremental_fake.calls[CBM_PIPELINE_STORAGE_GROWTH], 1); + ASSERT_EQ(incremental_fake.calls[CBM_PIPELINE_STORAGE_PREPUBLISH], 0); + ASSERT_EQ(pipeline_stage_artifact_count(full_repo), 0); + ASSERT_EQ(pipeline_stage_artifact_count(incremental_repo), 0); + th_cleanup(full_repo); + th_cleanup(incremental_repo); + PASS(); +} + /* ── Focused: file-backed store persistence ─────────────────────── */ TEST(store_file_persistence) { @@ -12018,6 +12224,9 @@ SUITE(pipeline) { RUN_TEST(pipeline_cancel_null); RUN_TEST(pipeline_run_null); RUN_TEST(pipeline_discovery_limit_returns_exact_violation_without_publishing); + RUN_TEST(pipeline_disabled_storage_policy_performs_no_probes); + RUN_TEST(pipeline_storage_probe_failure_preserves_old_db_and_cleans_stage); + RUN_TEST(pipeline_full_and_incremental_storage_limits_report_identical_failure); RUN_TEST(pipeline_late_source_limit_preserves_previous_generation); /* Extraction back-pressure */ RUN_TEST(pipeline_backpressure_futile_nap_disengages); diff --git a/tests/test_platform.c b/tests/test_platform.c index 1b55df699..c5ceb2d0c 100644 --- a/tests/test_platform.c +++ b/tests/test_platform.c @@ -2,6 +2,7 @@ * test_platform.c — RED phase tests for foundation/platform. */ #include "test_framework.h" +#include "test_helpers.h" #include "../src/foundation/compat.h" /* cbm_setenv / cbm_unsetenv (Windows-portable) */ #include "../src/foundation/compat_fs.h" #include "../src/foundation/constants.h" @@ -292,6 +293,93 @@ TEST(platform_file_size) { PASS(); } +TEST(platform_storage_measurement_uses_logical_bytes_without_following_links) { + char *root = th_mktempdir("cbm_storage_measure"); + ASSERT_NOT_NULL(root); + ASSERT_TRUE(cbm_mkdir_p(TH_PATH(root, "nested"), 0700)); + ASSERT_EQ(th_write_file(TH_PATH(root, "first.bin"), "12345"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "nested/second.bin"), "1234567"), 0); +#ifndef _WIN32 + ASSERT_EQ(symlink("/dev/null", TH_PATH(root, "external-link")), 0); +#endif + uint64_t bytes = 0; + uint64_t free_bytes = 0; + ASSERT_TRUE(cbm_directory_size_bytes(root, &bytes)); + ASSERT_EQ(bytes, 12); + ASSERT_TRUE(cbm_filesystem_free_bytes(root, &free_bytes)); + ASSERT_GT(free_bytes, 0); + th_cleanup(root); + PASS(); +} + +TEST(platform_db_artifact_measurement_saturates_and_counts_sidecars) { + char *root = th_mktempdir("cbm_storage_db_artifacts"); + ASSERT_NOT_NULL(root); + char db[CBM_SZ_4K]; + (void)snprintf(db, sizeof(db), "%s/project.db", root); + ASSERT_EQ(th_write_file(db, "12345"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "project.db-wal"), "1234567"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "project.db-shm"), "123"), 0); + uint64_t bytes = 0; + ASSERT_TRUE(cbm_db_artifact_bytes(db, &bytes)); + ASSERT_EQ(bytes, 15); + ASSERT_EQ(th_write_file(TH_PATH(root, "project.db.stage.task123.ABCDEF"), "12345678901"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "project.db.stage.task123.ABCDEF-wal"), "1234567890123"), + 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "project.db.stage.other456.ABCDEF"), "unrelated"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "worker.log"), "12"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "worker.response"), "123"), 0); + ASSERT_TRUE(cbm_index_task_temp_bytes(db, "task123", TH_PATH(root, "worker.log"), + TH_PATH(root, "worker.response"), &bytes)); + ASSERT_EQ(bytes, 29); + ASSERT_TRUE(cbm_index_staging_cleanup(db, "task123")); + ASSERT_FALSE(cbm_file_exists(TH_PATH(root, "project.db.stage.task123.ABCDEF"))); + ASSERT_FALSE(cbm_file_exists(TH_PATH(root, "project.db.stage.task123.ABCDEF-wal"))); + ASSERT_TRUE(cbm_file_exists(TH_PATH(root, "project.db.stage.other456.ABCDEF"))); + ASSERT_TRUE(cbm_file_exists(db)); + ASSERT_TRUE(cbm_file_exists(TH_PATH(root, "worker.log"))); + th_cleanup(root); + PASS(); +} + +#ifndef _WIN32 +TEST(platform_task_temp_measurement_supports_relative_database_paths) { + char *root = th_mktempdir("cbm_task_temp_relative"); + ASSERT_NOT_NULL(root); + char saved_cwd[CBM_SZ_4K]; + ASSERT_NOT_NULL(getcwd(saved_cwd, sizeof(saved_cwd))); + ASSERT_EQ(chdir(root), 0); + + int write_rc = th_write_file("project.db.stage.task123.ABCDEF", "12345"); + int wal_rc = th_write_file("project.db.stage.task123.ABCDEF-wal", "123456"); + int other_rc = th_write_file("project.db.stage.other456.ABCDEF", "unrelated"); + int log_rc = th_write_file("worker.log", "123"); + int response_rc = th_write_file("worker.response", "1234"); + uint64_t bytes = 0; + bool measured = + cbm_index_task_temp_bytes("project.db", "task123", "worker.log", "worker.response", &bytes); + bool cleaned = cbm_index_staging_cleanup("project.db", "task123"); + bool own_stage_removed = !cbm_file_exists("project.db.stage.task123.ABCDEF") && + !cbm_file_exists("project.db.stage.task123.ABCDEF-wal"); + bool other_stage_preserved = cbm_file_exists("project.db.stage.other456.ABCDEF"); + int restore_rc = chdir(saved_cwd); + th_cleanup(root); + + ASSERT_EQ(write_rc, 0); + ASSERT_EQ(wal_rc, 0); + ASSERT_EQ(other_rc, 0); + ASSERT_EQ(log_rc, 0); + ASSERT_EQ(response_rc, 0); + ASSERT_TRUE(measured); + ASSERT_EQ(bytes, 18); + ASSERT_TRUE(cleaned); + ASSERT_TRUE(own_stage_removed); + ASSERT_TRUE(other_stage_preserved); + ASSERT_EQ(restore_rc, 0); + PASS(); +} +#endif + TEST(platform_mmap) { /* mmap this test file and verify first bytes */ size_t sz = 0; @@ -696,6 +784,11 @@ SUITE(platform) { RUN_TEST(platform_file_exists); RUN_TEST(platform_is_dir); RUN_TEST(platform_file_size); + RUN_TEST(platform_storage_measurement_uses_logical_bytes_without_following_links); + RUN_TEST(platform_db_artifact_measurement_saturates_and_counts_sidecars); +#ifndef _WIN32 + RUN_TEST(platform_task_temp_measurement_supports_relative_database_paths); +#endif RUN_TEST(platform_mmap); RUN_TEST(platform_mmap_nonexistent); RUN_TEST(platform_path_helpers_use_per_thread_storage); From 8a3b8d48fa262a8dd8bd09c1a52df3421d84435e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Wed, 19 Aug 2026 14:39:38 +0800 Subject: [PATCH 4/6] feat(index): add opt-in index resource profiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six independent keys are an honest interface and a poor default. An operator who only wants indexing to stay within reason has to learn all six, choose a value for each, and keep them consistent as the machine changes. Add index_resource_profile, accepting off, balanced or strict. Balanced guards the host: it lets an index use what the machine can spare and stops a runaway worker before the host is exhausted. Strict holds indexing to the daemon's own budget, so a repository that needs more fails fast and attributed instead of finishing at the host's expense. Any individual key, including an explicit off, replaces that dimension of the selected profile, so a profile is a baseline rather than a lock. The balanced worker ceiling is derived from detected host memory instead of being tabled. Large-repository indexing peaks in the tens of gigabytes, so any round number low enough to feel safe would reject repositories that index successfully today. When host memory cannot be read, balanced falls back to a fixed floor. Profiles also bound directory count, entry count, traversal depth and discovery time. These have no individual keys for the same reason as the storage internals: they are only meaningful inside a composed decision. The resolved profile travels through the same trusted worker contract as the individual limits, so every entry point enforces the same decision. That contract now also carries the resolved profile and the override mask, so the shell fixture standing in for the supervisor sends both. The coverage that proves a forged policy cannot fork a second job asserts that an explicit request joins the auto-index job already running for the same root, and that assertion found the join broken on Windows. A project root reaches the daemon through the platform canonicalizer, which answers in backslash form there, while every tool handler normalizes the separators of the path it canonicalizes. One directory therefore had two spellings, and they met in comparisons that are exact: whether an index request may join the running job, and whether a watch is still live for that root. Both were wrong on Windows, in opposite directions, so aligning either one alone moved the failure rather than removing it. The session context now stores the normalized spelling, and the daemon canonicalizes project roots through a single function that normalizes on the way out. Only the storage side is directly assertable off Windows, since separators are folded on every platform; the watch-liveness side is held by the existing watcher-ownership tests, which is where the second half of this surfaced. Signed-off-by: 刘冲 --- README.md | 12 +- docs/CONFIGURATION.md | 9 +- docs/INDEX_RESOURCE_LIMITS.md | 51 ++++++++ scripts/test-runtime.sh | 9 +- src/cli/cli.c | 24 ++-- src/daemon/application.c | 55 ++++++++- src/discover/discover.c | 87 +++++++++++++- src/discover/discover.h | 5 + src/foundation/index_policy.c | 204 ++++++++++++++++++++++++++++++- src/foundation/index_policy.h | 29 ++++- src/mcp/mcp.c | 46 +++++-- tests/test_daemon.c | 20 ++++ tests/test_daemon_application.c | 44 ++++++- tests/test_discover.c | 182 ++++++++++++++++++++++++++++ tests/test_index_policy.c | 205 ++++++++++++++++++++++++++++++++ 15 files changed, 938 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 93312dca0..2a2ab98c4 100644 --- a/README.md +++ b/README.md @@ -667,6 +667,7 @@ codebase-memory-mcp config list # show all settings codebase-memory-mcp config set auto_index true # auto-index on session start codebase-memory-mcp config set auto_index_limit 50000 # max files for auto-index codebase-memory-mcp config set auto_watch false # don't register background git watcher (default: true) +codebase-memory-mcp config set index_resource_profile balanced # opt-in composed resource limits codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit codebase-memory-mcp config set index_max_rss_mb 8192 # optional worker-tree current RSS limit @@ -676,10 +677,13 @@ codebase-memory-mcp config set index_min_free_disk_mb 4096 # optional free-space codebase-memory-mcp config reset auto_index # reset to default ``` -The six index resource settings default to `off`. Exceeding one fails the -complete index attempt rather than publishing a partial graph; an existing -serving index is preserved. Storage probes also fail closed when an enabled -measurement cannot be completed. See +`index_resource_profile` defaults to `off` and accepts `off`, `balanced`, or +`strict`. The six individual resource settings also default to `off`; an +explicit value, including `off`, replaces that dimension of a selected +profile. Exceeding an effective limit fails the complete index attempt rather +than publishing a partial graph, and an existing serving index is preserved. +Storage probes also fail closed when an enabled measurement cannot be +completed. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). ### Environment Variables diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 6538ce3c0..aa5bb0474 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -86,6 +86,7 @@ Current keys: |---|---|---| | `auto_index` | `false` | Automatically index new projects when an MCP session starts. | | `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. | +| `index_resource_profile` | `off` | Composed index resource baseline: `off`, `balanced`, or `strict`. | | `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | | `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | | `index_max_rss_mb` | `off` | Optional maximum current RSS in MiB for the complete contained index-worker process tree (`64..1048576`). | @@ -93,8 +94,12 @@ Current keys: | `index_cache_max_mb` | `off` | Optional maximum projected cache size in MiB after publication. | | `index_min_free_disk_mb` | `off` | Optional minimum free MiB reserved on the cache filesystem while indexing. | -The six index resource settings are independent and disabled by default. -They apply to explicit indexing, automatic indexing, and watcher re-indexing, +The profile and all six individual resource settings are disabled by default. +An individual stored value replaces its profile dimension; storing `off` +disables only that dimension. Profile-only limits also bound traversed +directories, directory entries, depth, discovery time, final database size, +staging size, and task temporary size. Effective limits apply to explicit +indexing, automatic indexing, and watcher re-indexing, but not to `cross-repo-intelligence`, which does not scan or publish repository source indexes. Equality is allowed; exceeding a setting fails the complete index request and preserves any previously serving database. Worker RSS covers diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index 68fce77f8..be43cff56 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -4,6 +4,51 @@ Index resource limits are optional operator controls for repositories whose discovery breadth or worker runtime is not known in advance. They are disabled by default so existing large-repository workloads retain their current behavior. +## Resource profiles + +`index_resource_profile` accepts `off`, `balanced`, or `strict` and defaults to +`off`. A profile supplies a baseline; each explicitly stored individual +resource key then replaces its dimension. An explicit `off` disables only that +dimension. + +| Dimension | `balanced` | `strict` | +|---|---:|---:| +| Accepted files | 500,000 | 100,000 | +| Traversed directories | 250,000 | 20,000 | +| Directory entries | 2,000,000 | 500,000 | +| Directory depth | 128 | 64 | +| Accepted source | 65,536 MiB | 4,096 MiB | +| Discovery duration | 1,800 seconds | 30 seconds | +| Worker duration | 7,200 seconds | 3,600 seconds | +| Final database | 65,536 MiB | 16,384 MiB | +| Staging artifacts | 81,920 MiB | 20,480 MiB | +| Task temporary artifacts | 98,304 MiB | 24,576 MiB | +| Projected cache | 131,072 MiB | 32,768 MiB | +| Free-disk reserve | 4,096 MiB | 4,096 MiB | + +The two profiles answer different questions about worker memory, so their RSS +ceilings are derived rather than tabled. + +Balanced protects the host. Its ceiling is 80% of detected host memory, which +leaves an index free to use everything this machine can spare and still stops a +runaway worker before the host is exhausted. A fixed number is deliberately not +used here: large-repository indexing peaks in the tens of gigabytes, so any +round ceiling low enough to feel safe would reject repositories that index +successfully with the profile off. When host memory cannot be read, balanced +falls back to the higher of twice the worker soft budget or 8,192 MiB. + +Strict protects the budget. Its ceiling is the lower of the worker soft budget +and 8,192 MiB, which is the point of the profile: a repository that needs more +memory than the daemon budgets for itself fails fast with an attributed +`rss_bytes` violation instead of completing at the host's expense. On a large +repository this is expected to fail, and the failure names the limit and the +observed value so the operator can choose `balanced`, raise +`index_max_rss_mb`, or leave the profile `off`. + +Both derived values are held inside the same 64 to 1,048,576 MiB range that +`index_max_rss_mb` accepts, and detected host memory can lower but never raise +an explicitly configured `index_max_rss_mb`. + ## Discovery settings | Key | Default | Accepted value | Protects | @@ -63,6 +108,12 @@ rules, filename and suffix filters, language detection, and the existing per-file size rule. `index_max_source_mb` sums the filesystem sizes of that same accepted set. +Profile directory counting includes the request root and each non-skipped +directory admitted for traversal. Entry counting happens before ignore, +language, and file-size filtering. Root depth is zero. The monotonic discovery +deadline is checked before opening a directory and before processing each +entry. Equality is allowed for every dimension. + Equality is allowed. The first file or byte that makes an observed value greater than its limit stops discovery. CBM discards the partial file list and does not publish a partial graph as a complete index. diff --git a/scripts/test-runtime.sh b/scripts/test-runtime.sh index b7e3b885a..8978680a6 100644 --- a/scripts/test-runtime.sh +++ b/scripts/test-runtime.sh @@ -109,10 +109,13 @@ _cbm_test_runtime_daemon() { # worker in argv; a worker that finds no complete policy refuses to start rather # than index unbounded. A shell test that spawns `cli --index-worker` itself # stands in for the supervisor and owes the worker the same object. Mirrors -# cbm_mcp_index_policy_add_to_args: every key in cbm_index_policy_key_at, and -# nothing else. +# cbm_mcp_index_policy_add_to_args: the resolved profile, the override mask +# that says which keys the operator set by hand, every key in +# cbm_index_policy_key_at, and nothing else. cbm_test_index_worker_policy_json() { - printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"' + printf '%s' '"_cbm_index_policy":{"index_resource_profile":"off"' + printf '%s' ',"_cbm_index_override_mask":0' + printf '%s' ',"index_max_files":"off","index_max_source_mb":"off"' printf '%s' ',"index_max_rss_mb":"off","index_max_duration_seconds":"off"' printf '%s' ',"index_cache_max_mb":"off","index_min_free_disk_mb":"off"}' } diff --git a/src/cli/cli.c b/src/cli/cli.c index 9ae0a819c..9a7b5e47b 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -21,6 +21,7 @@ #include "foundation/platform.h" #include "foundation/constants.h" #include "foundation/log.h" +#include "foundation/mem.h" #include "foundation/sha256.h" #include "cli/client_adapter.h" #include "mcp/mcp.h" // cbm_mcp_tool_input_schema — CLI flag parser + per-tool --help @@ -6743,13 +6744,19 @@ bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t return false; } cbm_index_policy_init(policy); + const char *profile = cbm_config_get(cfg, CBM_INDEX_CONFIG_RESOURCE_PROFILE, "off"); + if (!cbm_index_policy_set_profile(policy, profile, error, error_size)) { + return false; + } for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { const char *key = cbm_index_policy_key_at(index); - const char *value = cbm_config_get(cfg, key, cbm_index_policy_default_value(key)); - if (!cbm_index_policy_set(policy, key, value, error, error_size)) { + const char *value = cbm_config_get(cfg, key, NULL); + if (value && !cbm_index_policy_set(policy, key, value, error, error_size)) { return false; } } + cbm_system_info_t system = cbm_system_info(); + cbm_index_policy_finalize(policy, (uint64_t)system.total_ram, (uint64_t)cbm_mem_budget()); return true; } @@ -6775,6 +6782,7 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_CONFIG_UI_LANG, "auto", "Pin graph UI language: en, zh, or auto"}, {CBM_CONFIG_UI_ENABLED, "false", "Serve the graph UI on a loopback HTTP port"}, {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"}, + {CBM_INDEX_CONFIG_RESOURCE_PROFILE, "off", "Index resource profile: off, balanced, or strict"}, {CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"}, {CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"}, {CBM_INDEX_CONFIG_MAX_RSS_MB, "off", "Max worker process-tree RSS MiB, or off"}, @@ -6808,19 +6816,17 @@ static bool config_key_is_ui(const char *key) { } static bool config_key_is_index_policy(const char *key) { - for (size_t index = 0; key && index < cbm_index_policy_key_count(); index++) { - if (strcmp(key, cbm_index_policy_key_at(index)) == 0) { - return true; - } - } - return false; + return cbm_index_policy_is_config_key(key); } static int config_index_policy_write(cbm_config_t *config, const char *key, const char *value) { cbm_index_resource_policy_t candidate; cbm_index_policy_init(&candidate); char error[CLI_BUF_256]; - if (!cbm_index_policy_set(&candidate, key, value, error, sizeof(error))) { + bool valid = strcmp(key, CBM_INDEX_CONFIG_RESOURCE_PROFILE) == 0 + ? cbm_index_policy_set_profile(&candidate, value, error, sizeof(error)) + : cbm_index_policy_set(&candidate, key, value, error, sizeof(error)); + if (!valid) { (void)fprintf(stderr, "error: %s\n", error); return CLI_ERR; } diff --git a/src/daemon/application.c b/src/daemon/application.c index 3dc14161b..cf7a43c3a 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -595,6 +595,20 @@ static void application_tmp_unlock(void) { atomic_flag_clear_explicit(&g_application_tmp_lock, memory_order_release); } +/* The daemon's only spelling of a project root. Tool handlers normalize the + * separators of every path they canonicalize, so a root left in the platform's + * native form is a second name for one directory — and the two names then meet + * in comparisons that are exact: whether a watch is still live for this root, + * and whether an index request may join the job already running for it. The + * forms differ on Windows, which is where both comparisons were wrong. */ +static bool application_canonical_root(const char *path, char *out, size_t out_size) { + if (!path || !out || out_size == 0 || !cbm_canonical_path(path, out, out_size)) { + return false; + } + cbm_normalize_path_sep(out); + return true; +} + static bool application_cache_dir(char out[APPLICATION_PATH_CAP]) { char configured[APPLICATION_PATH_CAP] = {0}; if (cbm_safe_getenv("CBM_CACHE_DIR", configured, sizeof(configured), NULL) && configured[0]) { @@ -1935,9 +1949,32 @@ static bool application_index_args_add_policy(cbm_daemon_application_t *applicat cbm_log_error("daemon.index.policy", "error", error); return false; } + cbm_system_info_t system = cbm_system_info(); + cbm_index_policy_finalize(&policy, (uint64_t)system.total_ram, + application ? (uint64_t)application->worker_memory_budget_bytes : 0); return cbm_mcp_index_policy_add_to_args(document, root, &policy); } +static char *application_index_args_replace_policy(cbm_daemon_application_t *application, + const char *args_json) { + yyjson_doc *source = args_json ? yyjson_read(args_json, strlen(args_json), 0) : NULL; + yyjson_mut_doc *document = source ? yyjson_doc_mut_copy(source, NULL) : NULL; + yyjson_doc_free(source); + yyjson_mut_val *root = document ? yyjson_mut_doc_get_root(document) : NULL; + if (!root || !yyjson_mut_is_obj(root)) { + yyjson_mut_doc_free(document); + return NULL; + } + while (yyjson_mut_obj_get(root, "_cbm_index_policy")) { + (void)yyjson_mut_obj_remove_key(root, "_cbm_index_policy"); + } + char *rewritten = application_index_args_add_policy(application, document, root) + ? yyjson_mut_write(document, 0, NULL) + : NULL; + yyjson_mut_doc_free(document); + return rewritten; +} + static char *application_auto_index_args(cbm_daemon_application_t *application, const char *root_path) { yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); @@ -2218,14 +2255,19 @@ static char *application_index_execute(void *context, const char *root_path, if (!session || !root_path || !args_json) { return NULL; } - char *project_key = application_index_project_key(root_path, args_json); + char *trusted_args = application_index_args_replace_policy(session->application, args_json); + if (!trusted_args) { + return cbm_mcp_text_result("failed to resolve daemon index resource policy", true); + } + char *project_key = application_index_project_key(root_path, trusted_args); if (!project_key) { + free(trusted_args); return cbm_mcp_text_result("failed to derive index project identity", true); } application_job_subscribe_status_t subscribe_status = APPLICATION_JOB_SUBSCRIBE_UNAVAILABLE; cbm_daemon_application_job_t *job = NULL; for (;;) { - job = application_job_subscribe(session->application, project_key, root_path, args_json, + job = application_job_subscribe(session->application, project_key, root_path, trusted_args, &subscribe_status); if (job || (subscribe_status != APPLICATION_JOB_SUBSCRIBE_BUSY && subscribe_status != APPLICATION_JOB_SUBSCRIBE_CANCELLING)) { @@ -2244,11 +2286,13 @@ static char *application_index_execute(void *context, const char *root_path, cbm_mutex_unlock(&session->application->mutex); if (queued_cancelled) { free(project_key); + free(trusted_args); return cbm_mcp_text_result("index operation cancelled for this session", true); } cbm_usleep(APPLICATION_JOB_POLL_US); } free(project_key); + free(trusted_args); if (!job) { const char *message = "daemon index coordinator is stopping or unavailable"; if (subscribe_status == APPLICATION_JOB_SUBSCRIBE_OPTIONS_CONFLICT) { @@ -2360,9 +2404,10 @@ static cbm_daemon_runtime_application_status_t application_set_context( } char canonical_root[APPLICATION_PATH_CAP] = {0}; char canonical_allowed[APPLICATION_PATH_CAP] = {0}; - bool canonical = cbm_canonical_path(root, canonical_root, sizeof(canonical_root)); + bool canonical = application_canonical_root(root, canonical_root, sizeof(canonical_root)); if (canonical && allowed_present) { - canonical = cbm_canonical_path(allowed, canonical_allowed, sizeof(canonical_allowed)); + canonical = + application_canonical_root(allowed, canonical_allowed, sizeof(canonical_allowed)); } struct stat root_status; canonical = @@ -3380,7 +3425,7 @@ static int application_background_index(cbm_daemon_application_t *application, } char canonical_root[APPLICATION_PATH_CAP]; struct stat root_status; - if (!cbm_canonical_path(root_path, canonical_root, sizeof(canonical_root)) || + if (!application_canonical_root(root_path, canonical_root, sizeof(canonical_root)) || stat(canonical_root, &root_status) != 0 || !S_ISDIR(root_status.st_mode)) { return -1; } diff --git a/src/discover/discover.c b/src/discover/discover.c index 94cceaeb1..e029b654e 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -28,6 +28,25 @@ int cbm_gitignore_match_result(const cbm_gitignore_t *gi, const char *rel_path, bool is_dir); +#ifdef CBM_ENABLE_TEST_SEAMS +static cbm_discover_now_ms_fn g_discover_now_ms; +static void *g_discover_now_ms_context; + +void cbm_discover_set_now_ms_for_testing(cbm_discover_now_ms_fn now_ms, void *context) { + g_discover_now_ms = now_ms; + g_discover_now_ms_context = context; +} +#endif + +static uint64_t discover_now_ms(void) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_discover_now_ms) { + return g_discover_now_ms(g_discover_now_ms_context); + } +#endif + return cbm_now_ms(); +} + /* ── Hardcoded always-skip directories ──────────────────────────── */ static const char *ALWAYS_SKIP_DIRS[] = { @@ -417,6 +436,9 @@ typedef struct { uint64_t deadline_ms; const cbm_index_resource_policy_t *resource_policy; cbm_index_resource_violation_t *resource_violation; + uint64_t resource_started_ms; + uint64_t directories; + uint64_t entries; uint64_t source_files; uint64_t source_bytes; bool count_only; @@ -459,9 +481,21 @@ static bool file_list_should_stop(file_list_t *fl) { if (!fl) { return true; } - if (!fl->failed && fl->deadline_ms != 0 && cbm_now_ms() >= fl->deadline_ms) { + bool needs_now = fl->deadline_ms != 0 || + (fl->resource_policy && fl->resource_policy->discovery_deadline_ms.enabled); + uint64_t now = needs_now ? discover_now_ms() : 0; + if (!fl->failed && fl->deadline_ms != 0 && now >= fl->deadline_ms) { fl->failed = true; } + if (!fl->failed && fl->resource_policy && fl->resource_policy->discovery_deadline_ms.enabled) { + uint64_t elapsed = + now >= fl->resource_started_ms ? now - fl->resource_started_ms : UINT64_C(0); + uint64_t limit = fl->resource_policy->discovery_deadline_ms.value; + if (elapsed > limit) { + file_list_resource_violation(fl, CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS, elapsed, + limit); + } + } return fl->failed || fl->limit_exceeded; } @@ -845,6 +879,7 @@ static void walk_dir_process_file(const char *abs_path, const char *rel_path, co typedef struct { char dir[CBM_SZ_4K]; char prefix[CBM_SZ_4K]; + uint64_t depth; cbm_gitignore_t *local_gi; /* nested .gitignore for this subtree */ char local_gi_prefix[CBM_SZ_4K]; /* rel_prefix when local_gi was loaded */ } walk_frame_t; @@ -898,6 +933,7 @@ static void walk_push_subdir(walk_stack_t *ws, const char *abs_path, const char return; } slot->local_gi = parent->local_gi; + slot->depth = parent->depth + 1U; int local_prefix_length = snprintf(slot->local_gi_prefix, CBM_SZ_4K, "%s", parent->local_gi_prefix); if (local_prefix_length < 0 || local_prefix_length >= CBM_SZ_4K) { @@ -913,6 +949,18 @@ static void walk_dir_process_entry(cbm_dirent_t *entry, const walk_frame_t *fram const cbm_gitignore_t *global_gi, const cbm_gitignore_t *cbmignore, walk_stack_t *ws, file_list_t *out) { + if (out->resource_policy && out->resource_policy->max_entries.enabled) { + uint64_t limit = out->resource_policy->max_entries.value; + if (out->entries >= limit) { + uint64_t observed = out->entries == UINT64_MAX ? UINT64_MAX : out->entries + 1U; + file_list_resource_violation(out, CBM_INDEX_RESOURCE_ENTRIES, observed, limit); + return; + } + } + if (out->entries < UINT64_MAX) { + out->entries++; + } + char abs_path[CBM_SZ_4K]; char rel_path[CBM_SZ_4K]; int absolute_length = snprintf(abs_path, sizeof(abs_path), "%s/%s", frame->dir, entry->name); @@ -940,6 +988,24 @@ static void walk_dir_process_entry(cbm_dirent_t *entry, const walk_frame_t *fram if (!dir_is_cache_tree(abs_path) && !should_skip_directory(entry->name, rel_path, opts, gitignore, global_gi, cbmignore, frame->local_gi, frame->local_gi_prefix)) { + uint64_t child_depth = frame->depth == UINT64_MAX ? UINT64_MAX : frame->depth + 1U; + if (out->resource_policy && out->resource_policy->max_depth.enabled && + child_depth > out->resource_policy->max_depth.value) { + file_list_resource_violation(out, CBM_INDEX_RESOURCE_DEPTH, child_depth, + out->resource_policy->max_depth.value); + return; + } + if (out->resource_policy && out->resource_policy->max_directories.enabled && + out->directories >= out->resource_policy->max_directories.value) { + uint64_t observed = + out->directories == UINT64_MAX ? UINT64_MAX : out->directories + 1U; + file_list_resource_violation(out, CBM_INDEX_RESOURCE_DIRECTORIES, observed, + out->resource_policy->max_directories.value); + return; + } + if (out->directories < UINT64_MAX) { + out->directories++; + } walk_push_subdir(ws, abs_path, rel_path, frame, out); } else { /* Record the excluded subtree root so callers can report it (#411). */ @@ -1022,8 +1088,12 @@ static void walk_dir(const char *dir_path, const char *rel_prefix, const cbm_dis continue; } - cbm_dirent_t *entry; - while (!file_list_should_stop(out) && (entry = cbm_readdir(d)) != NULL) { + while (!file_list_should_stop(out)) { + cbm_dirent_t *entry = cbm_readdir(d); + if (!entry) { + (void)file_list_should_stop(out); + break; + } walk_dir_process_entry(entry, &frame, opts, gitignore, global_gi, cbmignore, &ws, out); } cbm_closedir(d); @@ -1250,12 +1320,23 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc .deadline_ms = count_only ? deadline_ms : 0, .resource_policy = opts ? opts->resource_policy : NULL, .resource_violation = opts ? opts->resource_violation : NULL, + .resource_started_ms = + opts && opts->resource_policy && opts->resource_policy->discovery_deadline_ms.enabled + ? discover_now_ms() + : 0, + .directories = 1, .count_only = count_only, .collect_excluded = !count_only && excluded_out != NULL, .collect_ignored = !count_only && ignored_out != NULL, }; + if (fl.resource_policy && fl.resource_policy->max_directories.enabled && + fl.resource_policy->max_directories.value < 1U) { + file_list_resource_violation(&fl, CBM_INDEX_RESOURCE_DIRECTORIES, 1, + fl.resource_policy->max_directories.value); + } walk_cache_dir_snapshot(); walk_dir(repo_path, "", opts, gitignore, global_gi, cbmignore, &fl); + (void)file_list_should_stop(&fl); /* Cleanup */ cbm_gitignore_free(gitignore); diff --git a/src/discover/discover.h b/src/discover/discover.h index 59a5ed7a4..6741c15fd 100644 --- a/src/discover/discover.h +++ b/src/discover/discover.h @@ -145,6 +145,11 @@ typedef enum { CBM_DISCOVER_LIMIT_EXCEEDED = 1, } cbm_discover_status_t; +#ifdef CBM_ENABLE_TEST_SEAMS +typedef uint64_t (*cbm_discover_now_ms_fn)(void *context); +void cbm_discover_set_now_ms_for_testing(cbm_discover_now_ms_fn now_ms, void *context); +#endif + /* Walk a repository directory tree and discover all source files. * Applies hardcoded filters, gitignore patterns, and language detection. * Returns 0 on success, -1 on error. diff --git a/src/foundation/index_policy.c b/src/foundation/index_policy.c index 9c0f22e9c..6b9cab1ca 100644 --- a/src/foundation/index_policy.c +++ b/src/foundation/index_policy.c @@ -27,6 +27,61 @@ static const index_policy_metadata_t INDEX_POLICY_METADATA[] = { 1, CBM_INDEX_MAX_STORAGE_MB_VALUE, CBM_INDEX_MIB_BYTES}, }; +static cbm_index_limit_u64_t limit_value(uint64_t value) { + return (cbm_index_limit_u64_t){.enabled = true, .value = value}; +} + +static uint64_t mib(uint64_t value) { + return value * CBM_INDEX_MIB_BYTES; +} + +static void apply_profile_baseline(cbm_index_resource_policy_t *policy, + cbm_index_resource_profile_t profile) { + policy->max_files = (cbm_index_limit_u64_t){0}; + policy->max_directories = (cbm_index_limit_u64_t){0}; + policy->max_entries = (cbm_index_limit_u64_t){0}; + policy->max_depth = (cbm_index_limit_u64_t){0}; + policy->max_source_bytes = (cbm_index_limit_u64_t){0}; + policy->discovery_deadline_ms = (cbm_index_limit_u64_t){0}; + policy->max_rss_bytes = (cbm_index_limit_u64_t){0}; + policy->max_duration_ms = (cbm_index_limit_u64_t){0}; + policy->max_cache_bytes = (cbm_index_limit_u64_t){0}; + policy->min_free_disk_bytes = (cbm_index_limit_u64_t){0}; + policy->max_final_db_bytes = (cbm_index_limit_u64_t){0}; + policy->max_staging_bytes = (cbm_index_limit_u64_t){0}; + policy->max_task_temp_bytes = (cbm_index_limit_u64_t){0}; + policy->profile = profile; + if (profile == CBM_INDEX_PROFILE_BALANCED) { + policy->max_files = limit_value(UINT64_C(500000)); + policy->max_directories = limit_value(UINT64_C(250000)); + policy->max_entries = limit_value(UINT64_C(2000000)); + policy->max_depth = limit_value(UINT64_C(128)); + policy->max_source_bytes = limit_value(mib(UINT64_C(65536))); + policy->discovery_deadline_ms = limit_value(UINT64_C(1800000)); + policy->max_rss_bytes = limit_value(mib(UINT64_C(8192))); + policy->max_duration_ms = limit_value(UINT64_C(7200000)); + policy->max_final_db_bytes = limit_value(mib(UINT64_C(65536))); + policy->max_staging_bytes = limit_value(mib(UINT64_C(81920))); + policy->max_task_temp_bytes = limit_value(mib(UINT64_C(98304))); + policy->max_cache_bytes = limit_value(mib(UINT64_C(131072))); + policy->min_free_disk_bytes = limit_value(mib(UINT64_C(4096))); + } else if (profile == CBM_INDEX_PROFILE_STRICT) { + policy->max_files = limit_value(UINT64_C(100000)); + policy->max_directories = limit_value(UINT64_C(20000)); + policy->max_entries = limit_value(UINT64_C(500000)); + policy->max_depth = limit_value(UINT64_C(64)); + policy->max_source_bytes = limit_value(mib(UINT64_C(4096))); + policy->discovery_deadline_ms = limit_value(UINT64_C(30000)); + policy->max_rss_bytes = limit_value(mib(UINT64_C(8192))); + policy->max_duration_ms = limit_value(UINT64_C(3600000)); + policy->max_final_db_bytes = limit_value(mib(UINT64_C(16384))); + policy->max_staging_bytes = limit_value(mib(UINT64_C(20480))); + policy->max_task_temp_bytes = limit_value(mib(UINT64_C(24576))); + policy->max_cache_bytes = limit_value(mib(UINT64_C(32768))); + policy->min_free_disk_bytes = limit_value(mib(UINT64_C(4096))); + } +} + static void set_error(char *error, size_t error_size, const char *key, uint64_t minimum, uint64_t maximum) { if (error && error_size > 0) { @@ -64,13 +119,138 @@ void cbm_index_policy_init(cbm_index_resource_policy_t *policy) { } } +bool cbm_index_policy_set_profile(cbm_index_resource_policy_t *policy, const char *value, + char *error, size_t error_size) { + if (error && error_size > 0) { + error[0] = '\0'; + } + cbm_index_resource_profile_t profile; + if (value && strcmp(value, "off") == 0) { + profile = CBM_INDEX_PROFILE_OFF; + } else if (value && strcmp(value, "balanced") == 0) { + profile = CBM_INDEX_PROFILE_BALANCED; + } else if (value && strcmp(value, "strict") == 0) { + profile = CBM_INDEX_PROFILE_STRICT; + } else { + if (error && error_size > 0) { + (void)snprintf(error, error_size, "%s must be off, balanced, or strict", + CBM_INDEX_CONFIG_RESOURCE_PROFILE); + } + return false; + } + if (!policy) { + return false; + } + + cbm_index_resource_policy_t candidate = *policy; + cbm_index_limit_u64_t + overrides[sizeof(INDEX_POLICY_METADATA) / sizeof(INDEX_POLICY_METADATA[0])]; + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + const cbm_index_limit_u64_t *source = + (const cbm_index_limit_u64_t *)((const unsigned char *)policy + + INDEX_POLICY_METADATA[index].field_offset); + overrides[index] = *source; + } + apply_profile_baseline(&candidate, profile); + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if ((candidate.override_mask & (UINT64_C(1) << index)) == 0) { + continue; + } + cbm_index_limit_u64_t *target = + (cbm_index_limit_u64_t *)((unsigned char *)&candidate + + INDEX_POLICY_METADATA[index].field_offset); + *target = overrides[index]; + } + *policy = candidate; + return true; +} + +void cbm_index_policy_finalize(cbm_index_resource_policy_t *policy, uint64_t host_memory_bytes, + uint64_t soft_budget_bytes) { + if (!policy) { + return; + } + uint64_t host_cap = host_memory_bytes > 0 + ? (host_memory_bytes / 5U) * 4U + ((host_memory_bytes % 5U) * 4U) / 5U + : 0; + const uint64_t rss_override_bit = UINT64_C(1) << 2U; + if ((policy->override_mask & rss_override_bit) == 0) { + if (policy->profile == CBM_INDEX_PROFILE_BALANCED) { + /* Balanced guards the host, not the workload. Any fixed ceiling + * below what this machine can actually finish would reject indexes + * that succeed today — the recorded large-repository runs peak far + * above any round number worth hard-coding. The host share is the + * only honest balanced answer; the fixed floor is a fallback for + * hosts whose memory cannot be read. */ + uint64_t doubled = + soft_budget_bytes > UINT64_MAX / 2U ? UINT64_MAX : soft_budget_bytes * 2U; + uint64_t fallback = doubled > mib(UINT64_C(8192)) ? doubled : mib(UINT64_C(8192)); + policy->max_rss_bytes = limit_value(host_cap > 0 ? host_cap : fallback); + } else if (policy->profile == CBM_INDEX_PROFILE_STRICT) { + /* Strict holds the worker to the daemon's own soft budget, so a + * repository larger than that budget fails fast and attributed + * instead of being finished at the host's expense. */ + uint64_t baseline = soft_budget_bytes > 0 ? soft_budget_bytes : mib(UINT64_C(8192)); + if (baseline > mib(UINT64_C(8192))) { + baseline = mib(UINT64_C(8192)); + } + policy->max_rss_bytes = limit_value(baseline); + } + } + if (policy->max_rss_bytes.enabled && host_cap > 0 && policy->max_rss_bytes.value > host_cap) { + policy->max_rss_bytes.value = host_cap; + } + if (policy->max_rss_bytes.enabled) { + uint64_t minimum = mib(CBM_INDEX_MIN_RSS_MB_VALUE); + uint64_t maximum = mib(CBM_INDEX_MAX_RSS_MB_VALUE); + if (policy->max_rss_bytes.value < minimum) { + policy->max_rss_bytes.value = minimum; + } else if (policy->max_rss_bytes.value > maximum) { + policy->max_rss_bytes.value = maximum; + } + } +} + +const char *cbm_index_policy_profile_name(const cbm_index_resource_policy_t *policy) { + if (!policy || policy->profile == CBM_INDEX_PROFILE_OFF) { + return "off"; + } + return policy->profile == CBM_INDEX_PROFILE_BALANCED ? "balanced" : "strict"; +} + +const char *cbm_index_policy_source_name(const cbm_index_resource_policy_t *policy) { + bool profile = policy && policy->profile != CBM_INDEX_PROFILE_OFF; + bool override = policy && policy->override_mask != 0; + if (profile && override) { + return "profile+override"; + } + if (profile) { + return "profile"; + } + return override ? "override" : "off"; +} + +bool cbm_index_policy_is_config_key(const char *key) { + if (key && strcmp(key, CBM_INDEX_CONFIG_RESOURCE_PROFILE) == 0) { + return true; + } + for (size_t index = 0; key && index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { + return true; + } + } + return false; +} + bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy) { return cbm_index_policy_discovery_enabled(policy) || cbm_index_policy_worker_enabled(policy) || cbm_index_policy_storage_enabled(policy); } bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy) { - return policy && (policy->max_files.enabled || policy->max_source_bytes.enabled); + return policy && (policy->max_files.enabled || policy->max_directories.enabled || + policy->max_entries.enabled || policy->max_depth.enabled || + policy->max_source_bytes.enabled || policy->discovery_deadline_ms.enabled); } bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy) { @@ -185,6 +365,7 @@ bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, candidate.value = parsed * metadata->multiplier; } *target = candidate; + policy->override_mask |= UINT64_C(1) << (size_t)(metadata - INDEX_POLICY_METADATA); return true; } @@ -232,6 +413,14 @@ const char *cbm_index_resource_name(cbm_index_resource_t resource) { return "staging_bytes"; case CBM_INDEX_RESOURCE_TASK_TEMP_BYTES: return "task_temp_bytes"; + case CBM_INDEX_RESOURCE_DIRECTORIES: + return "directories"; + case CBM_INDEX_RESOURCE_ENTRIES: + return "entries"; + case CBM_INDEX_RESOURCE_DEPTH: + return "depth"; + case CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS: + return "discovery_duration_ms"; case CBM_INDEX_RESOURCE_NONE: default: return "unknown"; @@ -245,6 +434,13 @@ const char *cbm_index_resource_unit(cbm_index_resource_t resource) { if (resource == CBM_INDEX_RESOURCE_DURATION_MS) { return "milliseconds"; } + if (resource == CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS) { + return "milliseconds"; + } + if (resource == CBM_INDEX_RESOURCE_DIRECTORIES || resource == CBM_INDEX_RESOURCE_ENTRIES || + resource == CBM_INDEX_RESOURCE_DEPTH) { + return "count"; + } return "bytes"; } @@ -265,7 +461,11 @@ const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { case CBM_INDEX_RESOURCE_FINAL_DB_BYTES: case CBM_INDEX_RESOURCE_STAGING_BYTES: case CBM_INDEX_RESOURCE_TASK_TEMP_BYTES: - return "index_resource_profile"; + case CBM_INDEX_RESOURCE_DIRECTORIES: + case CBM_INDEX_RESOURCE_ENTRIES: + case CBM_INDEX_RESOURCE_DEPTH: + case CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS: + return CBM_INDEX_CONFIG_RESOURCE_PROFILE; case CBM_INDEX_RESOURCE_NONE: default: return "index_resource_limit"; diff --git a/src/foundation/index_policy.h b/src/foundation/index_policy.h index bac059f2e..85d1234b0 100644 --- a/src/foundation/index_policy.h +++ b/src/foundation/index_policy.h @@ -11,6 +11,7 @@ #define CBM_INDEX_CONFIG_MAX_DURATION_SECONDS "index_max_duration_seconds" #define CBM_INDEX_CONFIG_CACHE_MAX_MB "index_cache_max_mb" #define CBM_INDEX_CONFIG_MIN_FREE_DISK_MB "index_min_free_disk_mb" +#define CBM_INDEX_CONFIG_RESOURCE_PROFILE "index_resource_profile" #define CBM_INDEX_MAX_FILES_VALUE UINT64_C(10000000) #define CBM_INDEX_MAX_SOURCE_MB_VALUE UINT64_C(1048576) @@ -25,17 +26,26 @@ typedef struct { uint64_t value; } cbm_index_limit_u64_t; +typedef enum { + CBM_INDEX_PROFILE_OFF = 0, + CBM_INDEX_PROFILE_BALANCED, + CBM_INDEX_PROFILE_STRICT, +} cbm_index_resource_profile_t; + typedef struct { + cbm_index_resource_profile_t profile; + uint64_t override_mask; cbm_index_limit_u64_t max_files; + cbm_index_limit_u64_t max_directories; + cbm_index_limit_u64_t max_entries; + cbm_index_limit_u64_t max_depth; cbm_index_limit_u64_t max_source_bytes; + cbm_index_limit_u64_t discovery_deadline_ms; cbm_index_limit_u64_t max_rss_bytes; cbm_index_limit_u64_t max_duration_ms; cbm_index_limit_u64_t max_cache_bytes; cbm_index_limit_u64_t min_free_disk_bytes; - /* Internal storage dimensions with no public config key. Nothing enables - * them on its own; they exist so that a single composed decision can bound - * the database, the staging artifacts and the task temporary directory - * together, which no individual key can express. */ + /* Profile-only dimensions. They are deliberately not public config keys. */ cbm_index_limit_u64_t max_final_db_bytes; cbm_index_limit_u64_t max_staging_bytes; cbm_index_limit_u64_t max_task_temp_bytes; @@ -52,6 +62,10 @@ typedef enum { CBM_INDEX_RESOURCE_FINAL_DB_BYTES, CBM_INDEX_RESOURCE_STAGING_BYTES, CBM_INDEX_RESOURCE_TASK_TEMP_BYTES, + CBM_INDEX_RESOURCE_DIRECTORIES, + CBM_INDEX_RESOURCE_ENTRIES, + CBM_INDEX_RESOURCE_DEPTH, + CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS, } cbm_index_resource_t; typedef struct { @@ -72,6 +86,13 @@ typedef struct { } cbm_index_storage_sample_t; void cbm_index_policy_init(cbm_index_resource_policy_t *policy); +bool cbm_index_policy_set_profile(cbm_index_resource_policy_t *policy, const char *value, + char *error, size_t error_size); +void cbm_index_policy_finalize(cbm_index_resource_policy_t *policy, uint64_t host_memory_bytes, + uint64_t soft_budget_bytes); +const char *cbm_index_policy_profile_name(const cbm_index_resource_policy_t *policy); +const char *cbm_index_policy_source_name(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_is_config_key(const char *key); bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy); bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy); bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 887848c2b..1a663ddb2 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -1652,7 +1652,17 @@ bool cbm_mcp_server_set_session_context(cbm_mcp_server_t *srv, const char *sessi return false; } - char *project = cbm_project_name_from_path(session_root); + /* Every tool handler normalizes the separators of the path it canonicalizes, + * so a root kept in the platform's native spelling is a second name for one + * directory. The two names meet when an explicit index_repository request is + * compared against the auto-index job started from this root: on Windows the + * backslash form never matched the request's slash form, and the duplicate + * was refused as an options conflict instead of joining the running job. */ + char normalized_root[sizeof(srv->session_root)]; + snprintf(normalized_root, sizeof(normalized_root), "%s", session_root); + cbm_normalize_path_sep(normalized_root); + + char *project = cbm_project_name_from_path(normalized_root); if (!project || project[0] == '\0' || strlen(project) >= sizeof(srv->session_project)) { free(project); return false; @@ -1663,8 +1673,9 @@ bool cbm_mcp_server_set_session_context(cbm_mcp_server_t *srv, const char *sessi free(project); return false; } + cbm_normalize_path_sep(allowed_copy); - snprintf(srv->session_root, sizeof(srv->session_root), "%s", session_root); + snprintf(srv->session_root, sizeof(srv->session_root), "%s", normalized_root); snprintf(srv->session_project, sizeof(srv->session_project), "%s", project); free(project); @@ -7825,20 +7836,32 @@ bool cbm_mcp_index_policy_from_internal_args(const char *args, cbm_index_resourc yyjson_val *encoded = root && yyjson_is_obj(root) ? yyjson_obj_get(root, "_cbm_index_policy") : NULL; if (!encoded || !yyjson_is_obj(encoded) || - yyjson_obj_size(encoded) != cbm_index_policy_key_count()) { + yyjson_obj_size(encoded) != cbm_index_policy_key_count() + 2U) { yyjson_doc_free(doc); (void)snprintf(error, error_size, "missing or incomplete trusted worker policy"); return false; } cbm_index_policy_init(policy); - bool valid = true; + yyjson_val *profile = yyjson_obj_get(encoded, CBM_INDEX_CONFIG_RESOURCE_PROFILE); + yyjson_val *override_mask = yyjson_obj_get(encoded, "_cbm_index_override_mask"); + bool valid = profile && yyjson_is_str(profile) && override_mask && + yyjson_is_uint(override_mask) && + cbm_index_policy_set_profile(policy, yyjson_get_str(profile), error, error_size); for (size_t index = 0; valid && index < cbm_index_policy_key_count(); index++) { const char *key = cbm_index_policy_key_at(index); yyjson_val *value = yyjson_obj_get(encoded, key); valid = value && yyjson_is_str(value) && cbm_index_policy_set(policy, key, yyjson_get_str(value), error, error_size); } + if (valid) { + uint64_t known_mask = cbm_index_policy_key_count() >= 64U + ? UINT64_MAX + : (UINT64_C(1) << cbm_index_policy_key_count()) - 1U; + uint64_t decoded_mask = yyjson_get_uint(override_mask); + valid = (decoded_mask & ~known_mask) == 0; + policy->override_mask = decoded_mask; + } yyjson_doc_free(doc); if (!valid && error && error_size > 0 && error[0] == '\0') { (void)snprintf(error, error_size, "invalid trusted worker policy"); @@ -7883,7 +7906,11 @@ static bool load_index_policy(cbm_mcp_server_t *srv, const char *args, bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, const cbm_index_resource_policy_t *policy) { yyjson_mut_val *encoded = yyjson_mut_obj(doc); - bool valid = encoded != NULL; + bool valid = encoded != NULL && + yyjson_mut_obj_add_strcpy(doc, encoded, CBM_INDEX_CONFIG_RESOURCE_PROFILE, + cbm_index_policy_profile_name(policy)) && + yyjson_mut_obj_add_uint(doc, encoded, "_cbm_index_override_mask", + policy ? policy->override_mask : 0); for (size_t index = 0; valid && index < cbm_index_policy_key_count(); index++) { const char *key = cbm_index_policy_key_at(index); char value[CBM_SZ_64]; @@ -8518,8 +8545,13 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { } else if (rc == CBM_PIPELINE_RESOURCE_LIMIT && resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { const char *config_key = cbm_index_resource_config_key(resource_violation.resource); - bool discovery_resource = resource_violation.resource == CBM_INDEX_RESOURCE_FILES || - resource_violation.resource == CBM_INDEX_RESOURCE_SOURCE_BYTES; + bool discovery_resource = + resource_violation.resource == CBM_INDEX_RESOURCE_FILES || + resource_violation.resource == CBM_INDEX_RESOURCE_SOURCE_BYTES || + resource_violation.resource == CBM_INDEX_RESOURCE_DIRECTORIES || + resource_violation.resource == CBM_INDEX_RESOURCE_ENTRIES || + resource_violation.resource == CBM_INDEX_RESOURCE_DEPTH || + resource_violation.resource == CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS; char message[CBM_SZ_256]; (void)snprintf(message, sizeof(message), resource_violation.probe_failed ? "Index resource probe failed for %s" diff --git a/tests/test_daemon.c b/tests/test_daemon.c index fb0d0095a..0edbe19eb 100644 --- a/tests/test_daemon.c +++ b/tests/test_daemon.c @@ -467,6 +467,25 @@ TEST(daemon_sessions_keep_distinct_roots_and_allowed_root_policy) { PASS(); } +/* A daemon canonicalizes the client's root through the platform API, which on + * Windows answers in backslash form, while every tool handler normalizes the + * paths it canonicalizes. Keeping the native spelling gave one directory two + * names, and an explicit index_repository request was then refused as an + * options conflict instead of joining the auto-index job already running for + * that root. The separators are folded on every platform, so this holds + * wherever the test runs. */ +TEST(daemon_session_context_keeps_one_spelling_of_a_root) { + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + + ASSERT_TRUE(cbm_mcp_server_set_session_context(srv, "C:\\repos\\cbm", "C:\\repos")); + ASSERT_STR_EQ(cbm_mcp_server_session_root(srv), "C:/repos/cbm"); + ASSERT_STR_EQ(cbm_mcp_server_allowed_root(srv), "C:/repos"); + + cbm_mcp_server_free(srv); + PASS(); +} + SUITE(daemon) { RUN_TEST(daemon_client_ids_are_connection_bound); RUN_TEST(daemon_shared_job_survives_until_final_subscriber_disconnects); @@ -480,4 +499,5 @@ SUITE(daemon) { RUN_TEST(daemon_bridge_rejects_embedded_nul_body); RUN_TEST(daemon_bridge_rejects_oversized_headers); RUN_TEST(daemon_sessions_keep_distinct_roots_and_allowed_root_policy); + RUN_TEST(daemon_session_context_keeps_one_spelling_of_a_root); } diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index e804ba7bd..e715c46c1 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -2035,10 +2035,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { bool config_ready = stored_config && cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "3") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "4") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7") == 0; + cbm_config_set(stored_config, CBM_INDEX_CONFIG_RESOURCE_PROFILE, "strict") == 0; char canonical_root[APP_TEST_PATH_CAP] = {0}; bool canonical = dirs_ok && cbm_canonical_path(root, canonical_root, sizeof(canonical_root)); char *project = canonical ? cbm_project_name_from_path(canonical_root) : NULL; @@ -2057,6 +2054,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { .config = stored_config, .worker_ops = &worker_ops, .update_ops = &update_ops, + .aggregate_memory_budget_bytes = UINT64_C(128) * CBM_INDEX_MIB_BYTES, }; cbm_daemon_application_t *application = config_ready ? cbm_daemon_application_new(&config) : NULL; @@ -2084,7 +2082,41 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { app_wait_for_subscribers(application, project, 1) && app_wait_for_atomic_int(&fake.starts, 1); bool auto_policy_propagated = - first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4", "64", "7", "off", "off"); + first_owned && + app_fake_worker_policy_equals(&fake, 0, "100000", "4096", "64", "3600", "32768", "4096"); + char explicit_args[APP_TEST_PATH_CAP + 64]; + (void)snprintf(explicit_args, sizeof(explicit_args), "{\"repo_path\":\"%s\"}", root); + uint8_t *explicit_tool = NULL; + uint32_t explicit_tool_length = 0; + bool explicit_encoded = app_test_tool_request("index_repository", explicit_args, &explicit_tool, + &explicit_tool_length); + app_request_thread_t explicit_request = { + .callbacks = callbacks, + .session = sessions[3], + .request = explicit_tool, + .request_length = explicit_tool_length, + }; + cbm_thread_t explicit_thread; + bool explicit_started = + explicit_encoded && + cbm_thread_create(&explicit_thread, 0, app_request_thread, &explicit_request) == 0; + bool explicit_coalesced = explicit_started && project && + app_wait_for_subscribers(application, project, 2) && + atomic_load(&fake.starts) == 1; + if (sessions[3]) { + callbacks.session_cancel(callbacks.context, sessions[3]); + } + if (explicit_started) { + (void)cbm_thread_join(&explicit_thread); + } + free(explicit_request.response); + free(explicit_tool); + if (sessions[3]) { + callbacks.session_close(callbacks.context, sessions[3]); + sessions[3] = NULL; + } + bool explicit_detached = + explicit_coalesced && project && app_wait_for_subscribers(application, project, 1); bool second_initialized = app_test_initialize_profile(&callbacks, sessions[1], root, CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); bool coalesced = first_owned && second_initialized && project && @@ -2147,6 +2179,8 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { ASSERT_TRUE(first_initialized); ASSERT_TRUE(first_owned); ASSERT_TRUE(auto_policy_propagated); + ASSERT_TRUE(explicit_coalesced); + ASSERT_TRUE(explicit_detached); ASSERT_TRUE(second_initialized); ASSERT_TRUE(coalesced); ASSERT_TRUE(restricted_disconnect_kept_job); diff --git a/tests/test_discover.c b/tests/test_discover.c index ac1ada5e9..0d04a01b3 100644 --- a/tests/test_discover.c +++ b/tests/test_discover.c @@ -13,6 +13,21 @@ typedef struct { char *xdg_config_home; } git_env_snapshot_t; +#ifdef CBM_ENABLE_TEST_SEAMS +typedef struct { + uint64_t first; + uint64_t second; + uint64_t later; + int calls; +} discover_clock_fake_t; + +static uint64_t discover_clock_fake(void *context) { + discover_clock_fake_t *fake = context; + int call = fake->calls++; + return call == 0 ? fake->first : call == 1 ? fake->second : fake->later; +} +#endif + static git_env_snapshot_t save_git_env(void) { git_env_snapshot_t snapshot = {0}; const char *home = getenv("HOME"); @@ -585,6 +600,169 @@ TEST(discover_resource_source_bytes_allows_equality_and_rejects_one_more_byte) { PASS(); } +TEST(discover_profile_directory_limit_counts_root_and_admitted_directories) { + char *base = th_mktempdir("cbm_disc_policy_dirs"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "first/source.c"), "int first;\n"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_directories = (cbm_index_limit_u64_t){.enabled = true, .value = 2}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK); + ASSERT_EQ(count, 1); + cbm_discover_free(files, count); + + th_write_file(TH_PATH(base, "second/source.py"), "second = 2\n"); + files = NULL; + count = 99; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_DIRECTORIES); + ASSERT_EQ(violation.observed, 3); + ASSERT_EQ(violation.limit, 2); + th_cleanup(base); + PASS(); +} + +TEST(discover_profile_entry_limit_counts_before_ignore_and_language_filters) { + char *base = th_mktempdir("cbm_disc_policy_entries"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, ".gitignore"), "ignored.c\n"); + th_write_file(TH_PATH(base, "ignored.c"), "int ignored;\n"); + th_write_file(TH_PATH(base, "asset.png"), "not source\n"); + th_write_file(TH_PATH(base, "accepted.c"), "int accepted;\n"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_entries = (cbm_index_limit_u64_t){.enabled = true, .value = 4}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK); + ASSERT_EQ(count, 1); + cbm_discover_free(files, count); + + policy.max_entries.value = 3; + files = NULL; + count = 99; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_ENTRIES); + ASSERT_EQ(violation.observed, 4); + ASSERT_EQ(violation.limit, 3); + th_cleanup(base); + PASS(); +} + +TEST(discover_profile_depth_limit_allows_equality_and_rejects_child) { + char *base = th_mktempdir("cbm_disc_policy_depth"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "one/two/source.c"), "int source;\n"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_depth = (cbm_index_limit_u64_t){.enabled = true, .value = 2}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK); + ASSERT_EQ(count, 1); + cbm_discover_free(files, count); + + policy.max_depth.value = 1; + files = NULL; + count = 99; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_DEPTH); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + + th_cleanup(base); + PASS(); +} + +TEST(discover_profile_deadline_uses_monotonic_test_clock) { +#ifndef CBM_ENABLE_TEST_SEAMS + SKIP_PLATFORM("discovery clock seam unavailable"); +#else + char *base = th_mktempdir("cbm_disc_policy_deadline"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "source.c"), "int source;\n"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.discovery_deadline_ms = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + discover_clock_fake_t clock = {.first = 1000, .second = 1001, .later = 1002}; + cbm_discover_set_now_ms_for_testing(discover_clock_fake, &clock); + cbm_file_info_t *files = NULL; + int count = 99; + cbm_discover_status_t status = cbm_discover(base, &opts, &files, &count); + cbm_discover_set_now_ms_for_testing(NULL, NULL); + + ASSERT_EQ(status, CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + + char empty[4096]; + (void)snprintf(empty, sizeof(empty), "%s/empty", base); + ASSERT_EQ(th_mkdir_p(empty), 0); + violation = (cbm_index_resource_violation_t){0}; + clock = (discover_clock_fake_t){.first = 2000, .second = 2001, .later = 2002}; + cbm_discover_set_now_ms_for_testing(discover_clock_fake, &clock); + status = cbm_discover(empty, &opts, &files, &count); + cbm_discover_set_now_ms_for_testing(NULL, NULL); + ASSERT_EQ(status, CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + + violation = (cbm_index_resource_violation_t){0}; + clock = (discover_clock_fake_t){.first = 3000, .second = 3001, .later = 3001}; + cbm_discover_set_now_ms_for_testing(discover_clock_fake, &clock); + status = cbm_discover(empty, &opts, &files, &count); + cbm_discover_set_now_ms_for_testing(NULL, NULL); + ASSERT_EQ(status, CBM_DISCOVER_OK); + ASSERT_EQ(count, 0); + ASSERT(files == NULL); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_NONE); + th_cleanup(base); + PASS(); +#endif +} + TEST(discover_resource_file_budget_excludes_existing_oversized_skip) { char *base = th_mktempdir("cbm_disc_policy_oversized"); ASSERT(base != NULL); @@ -1927,6 +2105,10 @@ SUITE(discover) { RUN_TEST(discover_resource_policy_off_matches_legacy_discovery); RUN_TEST(discover_resource_file_limit_is_exact_and_counts_only_accepted_sources); RUN_TEST(discover_resource_source_bytes_allows_equality_and_rejects_one_more_byte); + RUN_TEST(discover_profile_directory_limit_counts_root_and_admitted_directories); + RUN_TEST(discover_profile_entry_limit_counts_before_ignore_and_language_filters); + RUN_TEST(discover_profile_depth_limit_allows_equality_and_rejects_child); + RUN_TEST(discover_profile_deadline_uses_monotonic_test_clock); RUN_TEST(discover_resource_file_budget_excludes_existing_oversized_skip); RUN_TEST(discover_skips_git_dir); RUN_TEST(discover_with_gitignore); diff --git a/tests/test_index_policy.c b/tests/test_index_policy.c index dc362f6b5..71f8240c7 100644 --- a/tests/test_index_policy.c +++ b/tests/test_index_policy.c @@ -7,7 +7,9 @@ #include "foundation/index_policy.h" #include "mcp/index_supervisor.h" #include "mcp/mcp.h" +#include "mcp/mcp_internal.h" #include "store/store.h" +#include #include #include @@ -20,8 +22,14 @@ TEST(index_policy_defaults_are_disabled) { cbm_index_resource_policy_t policy; cbm_index_policy_init(&policy); + ASSERT_STR_EQ(cbm_index_policy_profile_name(&policy), "off"); + ASSERT_STR_EQ(cbm_index_policy_source_name(&policy), "off"); ASSERT_FALSE(policy.max_files.enabled); + ASSERT_FALSE(policy.max_directories.enabled); + ASSERT_FALSE(policy.max_entries.enabled); + ASSERT_FALSE(policy.max_depth.enabled); ASSERT_FALSE(policy.max_source_bytes.enabled); + ASSERT_FALSE(policy.discovery_deadline_ms.enabled); ASSERT_FALSE(policy.max_rss_bytes.enabled); ASSERT_FALSE(policy.max_duration_ms.enabled); ASSERT_FALSE(policy.max_cache_bytes.enabled); @@ -42,6 +50,140 @@ TEST(index_policy_defaults_are_disabled) { PASS(); } +TEST(index_policy_balanced_profile_expands_exact_baseline) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "balanced", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(20) * 1024 * CBM_INDEX_MIB_BYTES, + UINT64_C(4) * 1024 * CBM_INDEX_MIB_BYTES); + + ASSERT_STR_EQ(cbm_index_policy_profile_name(&policy), "balanced"); + ASSERT_STR_EQ(cbm_index_policy_source_name(&policy), "profile"); + ASSERT_EQ(policy.max_files.value, UINT64_C(500000)); + ASSERT_EQ(policy.max_directories.value, UINT64_C(250000)); + ASSERT_EQ(policy.max_entries.value, UINT64_C(2000000)); + ASSERT_EQ(policy.max_depth.value, UINT64_C(128)); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(65536) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.discovery_deadline_ms.value, UINT64_C(1800000)); + /* Four fifths of a 20 GiB host. */ + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(16) * 1024 * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(7200000)); + ASSERT_EQ(policy.max_final_db_bytes.value, UINT64_C(65536) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_staging_bytes.value, UINT64_C(81920) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_task_temp_bytes.value, UINT64_C(98304) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_cache_bytes.value, UINT64_C(131072) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.min_free_disk_bytes.value, UINT64_C(4096) * CBM_INDEX_MIB_BYTES); + PASS(); +} + +/* A balanced ceiling must never sit below what the host can actually finish. + * The large-repository runs this project records as reference workloads peak + * in the tens of gigabytes, so a fixed balanced ceiling would reject indexes + * that complete today with the profile off. */ +TEST(index_policy_balanced_rss_follows_the_host_share) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "balanced", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(40) * 1024 * CBM_INDEX_MIB_BYTES, + UINT64_C(4) * 1024 * CBM_INDEX_MIB_BYTES); + ASSERT_TRUE(policy.max_rss_bytes.enabled); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(32) * 1024 * CBM_INDEX_MIB_BYTES); + + /* A small host still receives a real ceiling below its own memory. */ + cbm_index_policy_init(&policy); + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "balanced", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(10) * 1024 * CBM_INDEX_MIB_BYTES, + UINT64_C(2) * 1024 * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(8) * 1024 * CBM_INDEX_MIB_BYTES); + + /* Host memory that cannot be read falls back to the fixed floor. */ + cbm_index_policy_init(&policy); + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "balanced", error, sizeof(error))); + cbm_index_policy_finalize(&policy, 0, UINT64_C(6) * 1024 * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(12) * 1024 * CBM_INDEX_MIB_BYTES); + PASS(); +} + +TEST(index_policy_strict_profile_expands_exact_baseline) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "strict", error, sizeof(error))); + cbm_index_policy_finalize(&policy, 0, UINT64_C(3) * 1024 * CBM_INDEX_MIB_BYTES); + + ASSERT_STR_EQ(cbm_index_policy_profile_name(&policy), "strict"); + ASSERT_EQ(policy.max_files.value, UINT64_C(100000)); + ASSERT_EQ(policy.max_directories.value, UINT64_C(20000)); + ASSERT_EQ(policy.max_entries.value, UINT64_C(500000)); + ASSERT_EQ(policy.max_depth.value, UINT64_C(64)); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(4096) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.discovery_deadline_ms.value, UINT64_C(30000)); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(3072) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(3600000)); + ASSERT_EQ(policy.max_final_db_bytes.value, UINT64_C(16384) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_staging_bytes.value, UINT64_C(20480) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_task_temp_bytes.value, UINT64_C(24576) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_cache_bytes.value, UINT64_C(32768) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.min_free_disk_bytes.value, UINT64_C(4096) * CBM_INDEX_MIB_BYTES); + PASS(); +} + +TEST(index_policy_explicit_override_and_off_replace_profile_dimensions) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "balanced", error, sizeof(error))); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "42", error, sizeof(error))); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "off", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(1024) * CBM_INDEX_MIB_BYTES, + UINT64_C(512) * CBM_INDEX_MIB_BYTES); + + ASSERT_EQ(policy.max_files.value, UINT64_C(42)); + ASSERT_FALSE(policy.max_rss_bytes.enabled); + ASSERT_TRUE(policy.max_entries.enabled); + ASSERT_STR_EQ(cbm_index_policy_source_name(&policy), "profile+override"); + PASS(); +} + +TEST(index_policy_host_clamp_only_tightens_explicit_rss) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "4096", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(2048) * CBM_INDEX_MIB_BYTES, 0); + ASSERT_EQ(policy.max_rss_bytes.value, + (UINT64_C(2048) * CBM_INDEX_MIB_BYTES * UINT64_C(4)) / UINT64_C(5)); + + cbm_index_policy_init(&policy); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "512", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(2048) * CBM_INDEX_MIB_BYTES, 0); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(512) * CBM_INDEX_MIB_BYTES); + PASS(); +} + +TEST(index_policy_profile_validation_is_atomic) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "strict", error, sizeof(error))); + cbm_index_resource_policy_t before = policy; + ASSERT_FALSE(cbm_index_policy_set_profile(&policy, "STRICT", error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + ASSERT_TRUE(strstr(error, CBM_INDEX_CONFIG_RESOURCE_PROFILE) != NULL); + PASS(); +} + TEST(index_policy_file_limit_accepts_off_and_exact_range) { cbm_index_resource_policy_t policy; cbm_index_policy_init(&policy); @@ -256,6 +398,8 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); ASSERT_FALSE(cbm_index_policy_enabled(&policy)); + ASSERT_STR_EQ(cbm_index_policy_profile_name(&policy), "off"); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_RESOURCE_PROFILE, "strict"), 0); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "9"), 0); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "3"), 0); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64"), 0); @@ -264,6 +408,8 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, "5"), 0); ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); ASSERT_EQ(policy.max_files.value, 9); + ASSERT_STR_EQ(cbm_index_policy_profile_name(&policy), "strict"); + ASSERT_EQ(policy.max_directories.value, UINT64_C(20000)); ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(3) * CBM_INDEX_MIB_BYTES); ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(64) * CBM_INDEX_MIB_BYTES); ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(7000)); @@ -280,6 +426,7 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { } TEST(index_policy_cli_lists_all_operator_keys) { + bool profile_found = false; bool files_found = false; bool bytes_found = false; bool rss_found = false; @@ -288,6 +435,8 @@ TEST(index_policy_cli_lists_all_operator_keys) { bool free_found = false; for (size_t index = 0; index < cbm_cli_config_key_count_for_testing(); index++) { const char *key = cbm_cli_config_key_at_for_testing(index); + profile_found = + profile_found || (key && strcmp(key, CBM_INDEX_CONFIG_RESOURCE_PROFILE) == 0); files_found = files_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0); bytes_found = bytes_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); rss_found = rss_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_RSS_MB) == 0); @@ -296,6 +445,7 @@ TEST(index_policy_cli_lists_all_operator_keys) { cache_found = cache_found || (key && strcmp(key, CBM_INDEX_CONFIG_CACHE_MAX_MB) == 0); free_found = free_found || (key && strcmp(key, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB) == 0); } + ASSERT_TRUE(profile_found); ASSERT_TRUE(files_found); ASSERT_TRUE(bytes_found); ASSERT_TRUE(rss_found); @@ -314,6 +464,8 @@ TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite) { char *set_valid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "7"}; char *set_invalid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "0"}; + char *set_profile[] = {"set", CBM_INDEX_CONFIG_RESOURCE_PROFILE, "balanced"}; + char *set_invalid_profile[] = {"set", CBM_INDEX_CONFIG_RESOURCE_PROFILE, "BALANCED"}; char *reset[] = {"reset", CBM_INDEX_CONFIG_MAX_FILES}; int valid_rc = cbm_cmd_config(3, set_valid); cbm_config_t *config = cbm_config_open(cache); @@ -323,6 +475,14 @@ TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite) { int invalid_rc = cbm_cmd_config(3, set_invalid); stored = config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "missing") : "missing"; bool invalid_preserved = strcmp(stored, "7") == 0; + int profile_rc = cbm_cmd_config(3, set_profile); + stored = + config ? cbm_config_get(config, CBM_INDEX_CONFIG_RESOURCE_PROFILE, "missing") : "missing"; + bool profile_stored = strcmp(stored, "balanced") == 0; + int invalid_profile_rc = cbm_cmd_config(3, set_invalid_profile); + stored = + config ? cbm_config_get(config, CBM_INDEX_CONFIG_RESOURCE_PROFILE, "missing") : "missing"; + bool invalid_profile_preserved = strcmp(stored, "balanced") == 0; int reset_rc = cbm_cmd_config(2, reset); stored = config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "off") : "missing"; bool reset_to_default = strcmp(stored, "off") == 0; @@ -340,6 +500,10 @@ TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite) { ASSERT_TRUE(valid_stored); ASSERT_TRUE(invalid_rc != 0); ASSERT_TRUE(invalid_preserved); + ASSERT_EQ(profile_rc, 0); + ASSERT_TRUE(profile_stored); + ASSERT_TRUE(invalid_profile_rc != 0); + ASSERT_TRUE(invalid_profile_preserved); ASSERT_EQ(reset_rc, 0); ASSERT_TRUE(reset_to_default); PASS(); @@ -443,6 +607,40 @@ TEST(index_policy_worker_rejects_missing_parent_policy) { PASS(); } +TEST(index_policy_worker_contract_preserves_profile_only_dimensions) { + cbm_index_resource_policy_t parent; + cbm_index_policy_init(&parent); + char error[256]; + ASSERT_TRUE(cbm_index_policy_set_profile(&parent, "strict", error, sizeof(error))); + cbm_index_policy_finalize(&parent, UINT64_C(16) * 1024 * CBM_INDEX_MIB_BYTES, + UINT64_C(32) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(parent.max_rss_bytes.value, CBM_INDEX_MIN_RSS_MB_VALUE * CBM_INDEX_MIB_BYTES); + + yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_TRUE(cbm_mcp_index_policy_add_to_args(document, root, &parent)); + char *args = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(args); + + cbm_index_resource_policy_t worker; + ASSERT_TRUE(cbm_mcp_index_policy_from_internal_args(args, &worker, error, sizeof(error))); + ASSERT_STR_EQ(cbm_index_policy_profile_name(&worker), "strict"); + ASSERT_STR_EQ(cbm_index_policy_source_name(&worker), "profile"); + ASSERT_EQ(worker.max_directories.value, parent.max_directories.value); + ASSERT_EQ(worker.max_entries.value, parent.max_entries.value); + ASSERT_EQ(worker.max_depth.value, parent.max_depth.value); + ASSERT_EQ(worker.discovery_deadline_ms.value, parent.discovery_deadline_ms.value); + ASSERT_EQ(worker.max_final_db_bytes.value, parent.max_final_db_bytes.value); + ASSERT_EQ(worker.max_staging_bytes.value, parent.max_staging_bytes.value); + ASSERT_EQ(worker.max_task_temp_bytes.value, parent.max_task_temp_bytes.value); + ASSERT_EQ(worker.max_rss_bytes.value, parent.max_rss_bytes.value); + + free(args); + yyjson_mut_doc_free(document); + PASS(); +} + TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index) { char *repo = th_mktempdir("cbm_index_policy_mcp_repo"); char *cache = th_mktempdir("cbm_index_policy_mcp_cache"); @@ -625,6 +823,12 @@ TEST(index_policy_storage_limit_preserves_old_index_and_unrelated_cache_files) { SUITE(index_policy) { RUN_TEST(index_policy_defaults_are_disabled); + RUN_TEST(index_policy_balanced_profile_expands_exact_baseline); + RUN_TEST(index_policy_balanced_rss_follows_the_host_share); + RUN_TEST(index_policy_strict_profile_expands_exact_baseline); + RUN_TEST(index_policy_explicit_override_and_off_replace_profile_dimensions); + RUN_TEST(index_policy_host_clamp_only_tightens_explicit_rss); + RUN_TEST(index_policy_profile_validation_is_atomic); RUN_TEST(index_policy_file_limit_accepts_off_and_exact_range); RUN_TEST(index_policy_source_limit_converts_mib_without_overflow); RUN_TEST(index_policy_worker_limits_validate_and_convert_units); @@ -638,6 +842,7 @@ SUITE(index_policy) { RUN_TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite); RUN_TEST(index_policy_cli_set_reports_a_failed_write); RUN_TEST(index_policy_worker_rejects_missing_parent_policy); + RUN_TEST(index_policy_worker_contract_preserves_profile_only_dimensions); RUN_TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index); RUN_TEST(index_policy_storage_limit_preserves_old_index_and_unrelated_cache_files); } From e5eb0b7cae723ac0cc688e480ef479df14ba32a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Wed, 19 Aug 2026 14:39:38 +0800 Subject: [PATCH 5/6] feat(index): record and report the latest index attempt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed index is currently a log line. A watcher-triggered rebuild that dies on a resource limit leaves the previously published graph in service and nothing durable behind it, so index_status reports a healthy generation and the operator has no way to learn that it is no longer the tree. Record one attempt per project under the cache directory, updated under a file lock across queued, running and terminal states, carrying the origin, timestamps, the effective profile source and the resource failure when there is one. index_status returns it as last_index_attempt, and delete_project removes a matching record when no database was published. Freshness is reported alongside it and is deliberately conservative. A generation is comparable only when matching clean Git snapshots were observed before and after indexing, and it is fresh only while the worktree is still clean at that HEAD. A different clean HEAD is stale. Non-Git roots, dirty trees, a tree that changed during indexing, a failed probe and a corrupt record are all unknown, because a dirty tree is evidence of neither freshness nor staleness. Daemon startup fails attempts whose worker no longer exists. Liveness is judged by process identity rather than the pid alone, so a recycled pid cannot keep a dead attempt alive, and an unavailable start time falls back to plain liveness rather than declaring the worker gone. Projects with no record keep the previous response shape. Signed-off-by: 刘冲 --- README.md | 4 +- docs/CONFIGURATION.md | 9 + docs/INDEX_RESOURCE_LIMITS.md | 19 + src/daemon/application.c | 153 ++++++ src/git/git_context.c | 59 ++ src/git/git_context.h | 8 + src/mcp/mcp.c | 927 +++++++++++++++++++++++++++++++- src/mcp/mcp_internal.h | 22 + tests/test_daemon_application.c | 123 ++++- tests/test_git_context.c | 33 ++ tests/test_index_policy.c | 14 +- tests/test_mcp.c | 346 ++++++++++++ 12 files changed, 1706 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 2a2ab98c4..72ac3814c 100644 --- a/README.md +++ b/README.md @@ -682,8 +682,8 @@ codebase-memory-mcp config reset auto_index # reset to default explicit value, including `off`, replaces that dimension of a selected profile. Exceeding an effective limit fails the complete index attempt rather than publishing a partial graph, and an existing serving index is preserved. -Storage probes also fail closed when an enabled measurement cannot be -completed. See +`index_status` reports the latest physical indexing attempt and Git-based +freshness after the first recorded attempt. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). ### Environment Variables diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index aa5bb0474..08e7ce387 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -10,6 +10,7 @@ This page documents the configuration files that `codebase-memory-mcp` reads or | Per-project custom extension mapping | `{repo_root}/.codebase-memory.json` | JSON | Overrides conflicting global `extra_extensions` entries. | | CLI-managed runtime settings | `${CBM_CACHE_DIR:-~/.cache/codebase-memory-mcp}/_config.db` | SQLite | Written by `codebase-memory-mcp config set/reset`. | | UI settings | `${CBM_CACHE_DIR:-~/.cache/codebase-memory-mcp}/config.json` | JSON | Stores `ui_enabled` and `ui_port`. | +| Latest indexing attempt | `${CBM_CACHE_DIR:-~/.cache/codebase-memory-mcp}/status/.json` | JSON | Owner-private, atomically replaced status used by `index_status`. | | Daemon operation log | `${CBM_CACHE_DIR:-~/.cache/codebase-memory-mcp}/logs/cbm-daemon.log` | Structured log | Durable daemon lifecycle, watcher/indexing, UI, resource, and error events. | | Admission conflict log | `${CBM_CACHE_DIR:-~/.cache/codebase-memory-mcp}/logs/daemon-conflicts.ndjson` | NDJSON | Exact-build, ABI, and canonical-cache conflicts. | | Activation log | `${CBM_CACHE_DIR:-~/.cache/codebase-memory-mcp}/logs/activation-events.ndjson` | NDJSON | Install/update/uninstall activation progress and outcomes. | @@ -109,6 +110,14 @@ timeout. See [Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and error-response details. +After the first physical indexing attempt for a project, `index_status` adds +`last_index_attempt` and `freshness`. Freshness is `fresh` only when a clean +Git snapshot observed both before and after indexing still matches the current +clean `HEAD`. A different clean `HEAD` is `stale`, while non-Git, dirty, +changed-during-index, missing, or unreadable snapshots are `unknown`. +`delete_project` also removes a matching attempt record when no database was +published. + ## 3. UI Settings The optional built-in graph UI stores its settings in: diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index be43cff56..20438c493 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -170,6 +170,25 @@ Both cases preserve the old serving database. Staging files created by a terminated supervised worker are tagged with a private task token and removed after its process tree is quiescent; cleanup cannot match another attempt. +## Attempt visibility and freshness + +The latest physical attempt for each project is atomically stored in an +owner-private file below `${CBM_CACHE_DIR}/status/`. It records the +`explicit`, `auto`, or `watcher` origin; `queued`, `running`, `completed`, +`failed`, or `cancelled` state; effective profile source; timestamps; and +stable resource failure details when applicable. Daemon startup changes +abandoned `queued` or `running` records to `failed` with +`failure_code=worker_lost`. + +After a record exists, `index_status` returns `last_index_attempt` and +`freshness`. A generation is comparable only when matching clean Git snapshots +were observed before and after indexing, and it is `fresh` only while the +current worktree remains clean at that `HEAD`. A different clean `HEAD`, or a +failed watcher rebuild after an observed change, is `stale`. Non-Git roots, +dirty or changed-during-index snapshots, failed Git probes, and corrupt records +are `unknown`. Dirty state is never proof of freshness or staleness. Projects +with no attempt record retain the previous response shape. + ## Trust and compatibility Limits are read from the CLI-managed `_config.db`; they are not MCP request diff --git a/src/daemon/application.c b/src/daemon/application.c index cf7a43c3a..56ae5b86b 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -146,6 +146,8 @@ struct cbm_daemon_application_job { bool cancelled; bool cancel_requested; bool supervision_failed; + bool attempt_recorded; + char attempt_id[33]; cbm_daemon_application_job_t *next; }; @@ -1212,6 +1214,63 @@ static void application_record_cancelled(cbm_index_worker_result_t *last_result, last_log[0] = '\0'; } +static cbm_index_resource_t application_resource_from_name(const char *name) { + for (int resource = CBM_INDEX_RESOURCE_FILES; + resource <= CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS; resource++) { + if (name && strcmp(name, cbm_index_resource_name((cbm_index_resource_t)resource)) == 0) { + return (cbm_index_resource_t)resource; + } + } + return CBM_INDEX_RESOURCE_NONE; +} + +static bool application_response_error_metadata(const char *response, char *code, size_t code_size, + cbm_index_resource_violation_t *violation) { + if (code && code_size > 0) { + code[0] = '\0'; + } + if (violation) { + *violation = (cbm_index_resource_violation_t){0}; + } + yyjson_doc *outer = response ? yyjson_read(response, strlen(response), 0) : NULL; + yyjson_val *outer_root = outer ? yyjson_doc_get_root(outer) : NULL; + yyjson_val *is_error = + outer_root && yyjson_is_obj(outer_root) ? yyjson_obj_get(outer_root, "isError") : NULL; + yyjson_val *content = + outer_root && yyjson_is_obj(outer_root) ? yyjson_obj_get(outer_root, "content") : NULL; + yyjson_val *first = content && yyjson_is_arr(content) ? yyjson_arr_get_first(content) : NULL; + yyjson_val *text = first && yyjson_is_obj(first) ? yyjson_obj_get(first, "text") : NULL; + yyjson_doc *inner = text && yyjson_is_str(text) + ? yyjson_read(yyjson_get_str(text), yyjson_get_len(text), 0) + : NULL; + yyjson_val *inner_root = inner ? yyjson_doc_get_root(inner) : NULL; + yyjson_val *inner_status = + inner_root && yyjson_is_obj(inner_root) ? yyjson_obj_get(inner_root, "status") : NULL; + bool failed = (is_error && yyjson_is_bool(is_error) && yyjson_get_bool(is_error)) || + (inner_status && yyjson_is_str(inner_status) && + strcmp(yyjson_get_str(inner_status), "error") == 0); + if (failed && inner_root && yyjson_is_obj(inner_root)) { + yyjson_val *failure_code = yyjson_obj_get(inner_root, "code"); + if (code && code_size > 0 && failure_code && yyjson_is_str(failure_code)) { + (void)snprintf(code, code_size, "%s", yyjson_get_str(failure_code)); + } + yyjson_val *resource = yyjson_obj_get(inner_root, "resource"); + yyjson_val *observed = yyjson_obj_get(inner_root, "observed"); + yyjson_val *limit = yyjson_obj_get(inner_root, "limit"); + if (violation && resource && yyjson_is_str(resource)) { + violation->resource = application_resource_from_name(yyjson_get_str(resource)); + violation->probe_failed = code && strcmp(code, "resource_probe_failed") == 0; + if (observed && yyjson_is_uint(observed) && limit && yyjson_is_uint(limit)) { + violation->observed = yyjson_get_uint(observed); + violation->limit = yyjson_get_uint(limit); + } + } + } + yyjson_doc_free(inner); + yyjson_doc_free(outer); + return failed; +} + static bool application_result_is_attributable_failure( const application_attempt_t *attempt, cbm_mcp_supervised_result_disposition_t disposition) { return disposition == CBM_MCP_SUPERVISED_RESULT_CONTAINED_FAILURE && attempt->has_result && @@ -1227,6 +1286,7 @@ typedef enum { typedef struct { char *response; bool successful; + bool response_is_error; bool unsafe_terminal; bool supervision_failed; cbm_index_worker_result_t last_result; @@ -1256,12 +1316,17 @@ static application_attempt_decision_t application_consume_attempt( execution->response = attempt->result.response; attempt->result.response = NULL; execution->successful = execution->response != NULL; + execution->response_is_error = + execution->response && + application_response_error_metadata(execution->response, NULL, 0, NULL); application_attempt_free(attempt); return APPLICATION_ATTEMPT_DECISION_SUCCESS; } if (disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { execution->response = cbm_mcp_index_worker_resource_response(job->args_json, &attempt->result); + execution->successful = execution->response != NULL; + execution->response_is_error = execution->response != NULL; application_attempt_free(attempt); return APPLICATION_ATTEMPT_DECISION_STOP; } @@ -1461,6 +1526,46 @@ static void application_job_publish(cbm_daemon_application_job_t *job, execution->have_last_result ? &execution->last_result : NULL, execution->last_log[0] ? execution->last_log : NULL); } + if (job->attempt_recorded) { + bool cancelled = + execution->have_last_result && execution->last_result.cancellation_requested; + bool attempt_completed = execution->successful && !execution->response_is_error; + const char *state = attempt_completed ? "completed" : cancelled ? "cancelled" : "failed"; + const char *failure_code = NULL; + const cbm_index_resource_violation_t *violation = NULL; + char response_failure_code[CBM_SZ_128] = {0}; + cbm_index_resource_violation_t response_violation = {0}; + if (!attempt_completed && !cancelled) { + if (application_response_error_metadata(execution->response, response_failure_code, + sizeof(response_failure_code), + &response_violation)) { + failure_code = response_failure_code[0] ? response_failure_code : "index_failed"; + if (response_violation.resource != CBM_INDEX_RESOURCE_NONE) { + violation = &response_violation; + } + } else if (execution->have_last_result && + execution->last_result.resource_violation.resource != + CBM_INDEX_RESOURCE_NONE) { + violation = &execution->last_result.resource_violation; + failure_code = + violation->probe_failed ? "resource_probe_failed" : "resource_limit_exceeded"; + } else if (execution->supervision_failed || execution->unsafe_terminal) { + failure_code = "supervision_failed"; + } else if (!execution->have_last_result || + execution->last_result.outcome == CBM_PROC_SPAWN_FAILED) { + failure_code = "worker_start_failed"; + } else { + failure_code = "index_failed"; + } + } else if (cancelled) { + failure_code = "cancelled"; + } + if (!cbm_mcp_index_attempt_transition(job->project_key, job->root_path, job->attempt_id, + state, failure_code, violation, attempt_completed)) { + cbm_log_warn("daemon.index.attempt_record", "project", job->project_key, "state", + "terminal_write_failed"); + } + } cbm_daemon_application_t *application = job->application; cbm_mutex_lock(&application->mutex); job->response = execution->response; @@ -1508,6 +1613,12 @@ static void *application_job_thread(void *opaque) { if (!application_job_wait_for_mutations(job)) { application_job_execution_cancel(&execution); } else { + if (job->attempt_recorded && + !cbm_mcp_index_attempt_transition(job->project_key, job->root_path, job->attempt_id, + "running", NULL, NULL, false)) { + cbm_log_warn("daemon.index.attempt_record", "project", job->project_key, "state", + "running_write_failed"); + } application_attempt_t attempt; application_attempt_status_t status = application_job_run_attempt(job, NULL, NULL, &attempt); @@ -1556,6 +1667,18 @@ static char *application_index_project_key(const char *root_path, const char *ar return key; } +static const char *application_index_origin(const char *args_json, char **owned_origin) { + *owned_origin = cbm_mcp_get_string_arg(args_json, "_cbm_index_origin"); + if (*owned_origin && + (strcmp(*owned_origin, "auto") == 0 || strcmp(*owned_origin, "watcher") == 0 || + strcmp(*owned_origin, "explicit") == 0)) { + return *owned_origin; + } + free(*owned_origin); + *owned_origin = NULL; + return "explicit"; +} + static size_t application_active_job_count_locked(cbm_daemon_application_t *application) { size_t count = 0; for (cbm_daemon_application_job_t *job = application->jobs; job; job = job->next) { @@ -1587,6 +1710,7 @@ static bool application_index_args_normalize_defaults(yyjson_mut_val *root) { if (name && (!yyjson_mut_is_str(name) || yyjson_mut_get_len(name) == 0)) { (void)yyjson_mut_obj_remove_key(root, "name"); } + (void)yyjson_mut_obj_remove_key(root, "_cbm_index_origin"); return true; } @@ -1619,6 +1743,9 @@ static cbm_daemon_application_job_t *application_job_subscribe_locked( if (application->stopping) { return NULL; } + char *requested_origin = cbm_mcp_get_string_arg(args_json, "_cbm_index_origin"); + bool watcher_request = requested_origin && strcmp(requested_origin, "watcher") == 0; + free(requested_origin); cbm_daemon_application_job_t *job = application_find_active_job_locked(application, project_key); if (job) { @@ -1631,6 +1758,9 @@ static cbm_daemon_application_job_t *application_job_subscribe_locked( return NULL; } job->subscribers++; + if (watcher_request && job->attempt_recorded) { + (void)cbm_mcp_index_attempt_mark_watcher_change(job->project_key, job->attempt_id); + } *status_out = APPLICATION_JOB_SUBSCRIBE_OK; return job; } @@ -1656,6 +1786,20 @@ static cbm_daemon_application_job_t *application_job_subscribe_locked( } job->application = application; job->subscribers = 1; + cbm_index_resource_policy_t policy; + char policy_error[CBM_SZ_256] = {0}; + char *owned_origin = NULL; + const char *origin = application_index_origin(args_json, &owned_origin); + if (cbm_mcp_index_policy_from_internal_args(args_json, &policy, policy_error, + sizeof(policy_error))) { + job->attempt_recorded = + cbm_mcp_index_attempt_begin(project_key, root_path, origin, &policy, + strcmp(origin, "watcher") == 0, job->attempt_id); + } + free(owned_origin); + if (!job->attempt_recorded) { + cbm_log_warn("daemon.index.attempt_record", "project", project_key, "state", "unavailable"); + } job->next = application->jobs; application->jobs = job; if (application_job_thread_create(&job->thread, job) == 0) { @@ -1666,6 +1810,11 @@ static cbm_daemon_application_job_t *application_job_subscribe_locked( * reservation back synchronously and let background callers retry. */ application->jobs = job->next; job->next = NULL; + if (job->attempt_recorded) { + (void)cbm_mcp_index_attempt_transition(job->project_key, job->root_path, + job->attempt_id, "failed", "worker_start_failed", + NULL, false); + } application_job_free(job); *status_out = APPLICATION_JOB_SUBSCRIBE_UNAVAILABLE; cbm_log_warn("daemon.index.thread_start_failed", "action", "retry"); @@ -1985,6 +2134,7 @@ static char *application_auto_index_args(cbm_daemon_application_t *application, } yyjson_mut_doc_set_root(document, root); char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) && + yyjson_mut_obj_add_str(document, root, "_cbm_index_origin", "auto") && application_index_args_add_policy(application, document, root) ? yyjson_mut_write(document, 0, NULL) : NULL; @@ -3013,6 +3163,7 @@ cbm_daemon_application_t *cbm_daemon_application_new( application->watcher, application_watcher_mutation_begin, application_watcher_mutation_end, application_watcher_project_pruned, application); } + cbm_mcp_index_attempt_recover_abandoned(); return application; } @@ -3437,6 +3588,8 @@ static int application_background_index(cbm_daemon_application_t *application, } yyjson_mut_doc_set_root(document, root); bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root) && + yyjson_mut_obj_add_str(document, root, "_cbm_index_origin", + require_live_watch ? "watcher" : "explicit") && application_index_args_add_policy(application, document, root); char *default_project = cbm_project_name_from_path(canonical_root); bool custom_project = diff --git a/src/git/git_context.c b/src/git/git_context.c index f739c46e6..a170a2554 100644 --- a/src/git/git_context.c +++ b/src/git/git_context.c @@ -86,6 +86,35 @@ static int git_capture(const char *repo_path, const char *git_args, char **out) return *out ? 0 : CBM_NOT_FOUND; } +static int git_dirty_status(const char *repo_path, bool *dirty_out) { + if (!dirty_out || !repo_path || !git_validate_repo_path(repo_path)) { + return CBM_NOT_FOUND; + } + *dirty_out = false; +#ifdef _WIN32 + const char *null_dev = "NUL"; +#else + const char *null_dev = "/dev/null"; +#endif + char cmd[GIT_CMD_MAX]; + int n = snprintf( + cmd, sizeof(cmd), + "git --no-optional-locks -C \"%s\" status --porcelain --untracked-files=normal 2>%s", + repo_path, null_dev); + if (n < 0 || n >= (int)sizeof(cmd)) { + return CBM_NOT_FOUND; + } + FILE *fp = cbm_popen(cmd, "r"); + if (!fp) { + return CBM_NOT_FOUND; + } + char buffer[CBM_SZ_4K]; + while (fread(buffer, 1, sizeof(buffer), fp) > 0) { + *dirty_out = true; + } + return cbm_pclose(fp) == 0 ? 0 : CBM_NOT_FOUND; +} + static bool path_is_absolute(const char *path) { if (!path || !path[0]) { return false; @@ -239,6 +268,36 @@ void cbm_git_context_free(cbm_git_context_t *ctx) { memset(ctx, 0, sizeof(*ctx)); } +int cbm_git_snapshot_read(const char *path, cbm_git_snapshot_t *out) { + if (!out) { + return CBM_NOT_FOUND; + } + memset(out, 0, sizeof(*out)); + char *head_sha = NULL; + int status = git_capture(path, "rev-parse --verify HEAD", &head_sha); + if (status != 0 || !head_sha || !head_sha[0]) { + free(head_sha); + return status == 0 ? CBM_NOT_FOUND : status; + } + bool dirty = false; + if (git_dirty_status(path, &dirty) != 0) { + free(head_sha); + return CBM_NOT_FOUND; + } + out->head_sha = head_sha; + out->available = true; + out->dirty = dirty; + return 0; +} + +void cbm_git_snapshot_free(cbm_git_snapshot_t *snapshot) { + if (!snapshot) { + return; + } + free(snapshot->head_sha); + memset(snapshot, 0, sizeof(*snapshot)); +} + int cbm_git_context_resolve(const char *path, cbm_git_context_t *out) { if (!out) { return CBM_NOT_FOUND; diff --git a/src/git/git_context.h b/src/git/git_context.h index 876309eb6..79042cdbb 100644 --- a/src/git/git_context.h +++ b/src/git/git_context.h @@ -19,9 +19,17 @@ typedef struct { char *base_sha; } cbm_git_context_t; +typedef struct { + bool available; + bool dirty; + char *head_sha; +} cbm_git_snapshot_t; + int cbm_git_context_resolve(const char *path, cbm_git_context_t *out); void cbm_git_context_free(cbm_git_context_t *ctx); char *cbm_git_context_branch_qn(const char *project_name, const cbm_git_context_t *ctx); int cbm_git_context_props_json(const cbm_git_context_t *ctx, char *buf, int buf_size); +int cbm_git_snapshot_read(const char *path, cbm_git_snapshot_t *out); +void cbm_git_snapshot_free(cbm_git_snapshot_t *snapshot); #endif diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 1a663ddb2..dc38a9c40 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -61,6 +61,7 @@ enum { #include "foundation/log.h" #include "foundation/limits.h" #include "foundation/subprocess.h" +#include "foundation/secure_random.h" #include "mcp/index_supervisor.h" #include "mcp/compact_out.h" #include "foundation/str_util.h" @@ -80,12 +81,22 @@ enum { * path — see search_scratch_open. Mirrors config_toml_edit.c's toml_fdopen. */ #define mcp_fdopen _fdopen #define mcp_close _close +#define mcp_fsync _commit #else #include #include #include +#include +#ifndef _WIN32 +#include +#endif +#ifdef __APPLE__ +#include +#include +#endif #define mcp_fdopen fdopen #define mcp_close close +#define mcp_fsync fsync #endif #include #include @@ -4550,10 +4561,807 @@ static char *handle_check_index_coverage(cbm_mcp_server_t *srv, const char *args return result; } +static bool index_attempt_paths(const char *project, bool create_directory, char *directory, + size_t directory_size, char *record, size_t record_size) { + if (!cbm_validate_project_name(project) || !directory || directory_size == 0 || !record || + record_size == 0) { + return false; + } + int directory_length = + snprintf(directory, directory_size, "%s/status", cbm_resolve_cache_dir()); + if (directory_length <= 0 || (size_t)directory_length >= directory_size || + (create_directory && !cbm_mkdir_p(directory, 0700))) { + return false; + } + int record_length = snprintf(record, record_size, "%s/%s.json", directory, project); + return record_length > 0 && (size_t)record_length < record_size; +} + +typedef struct { +#ifdef _WIN32 + HANDLE handle; + OVERLAPPED overlapped; +#else + int descriptor; +#endif +} index_attempt_lock_t; + +static bool index_attempt_lock_acquire(index_attempt_lock_t *lock, const char *project) { + if (!lock || !project) { + return false; + } + memset(lock, 0, sizeof(*lock)); + char directory[CBM_SZ_4K]; + char record[CBM_SZ_4K]; + if (!index_attempt_paths(project, true, directory, sizeof(directory), record, sizeof(record))) { + return false; + } + char path[CBM_SZ_4K]; + int path_length = snprintf(path, sizeof(path), "%s", record); + if (path_length <= 5 || (size_t)path_length >= sizeof(path)) { + return false; + } + memcpy(path + path_length - 5, ".lock", 6); +#ifdef _WIN32 + wchar_t *wide_path = cbm_utf8_to_wide(path); + if (!wide_path) { + return false; + } + lock->handle = + CreateFileW(wide_path, GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, + NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_HIDDEN | FILE_FLAG_OPEN_REPARSE_POINT, NULL); + free(wide_path); + if (lock->handle == INVALID_HANDLE_VALUE) { + lock->handle = NULL; + return false; + } + if (!LockFileEx(lock->handle, LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &lock->overlapped)) { + (void)CloseHandle(lock->handle); + lock->handle = NULL; + return false; + } + return true; +#else + int flags = O_CREAT | O_RDWR; +#ifdef O_CLOEXEC + flags |= O_CLOEXEC; +#endif +#ifdef O_NOFOLLOW + flags |= O_NOFOLLOW; +#endif + lock->descriptor = open(path, flags, 0600); + if (lock->descriptor < 0) { + return false; + } + (void)fchmod(lock->descriptor, 0600); + if (flock(lock->descriptor, LOCK_EX) != 0) { + (void)close(lock->descriptor); + lock->descriptor = -1; + return false; + } + return true; +#endif +} + +static void index_attempt_lock_release(index_attempt_lock_t *lock) { + if (!lock) { + return; + } +#ifdef _WIN32 + if (lock->handle) { + (void)UnlockFileEx(lock->handle, 0, 1, 0, &lock->overlapped); + (void)CloseHandle(lock->handle); + lock->handle = NULL; + } +#else + if (lock->descriptor >= 0) { + (void)flock(lock->descriptor, LOCK_UN); + (void)close(lock->descriptor); + lock->descriptor = -1; + } +#endif +} + +static yyjson_doc *index_attempt_read(const char *project, bool *exists_out) { + if (exists_out) { + *exists_out = false; + } + char directory[CBM_SZ_4K]; + char record[CBM_SZ_4K]; + if (!index_attempt_paths(project, false, directory, sizeof(directory), record, + sizeof(record))) { + return NULL; + } + if (access(record, F_OK) != 0) { + return NULL; + } + if (exists_out) { + *exists_out = true; + } + yyjson_doc *document = yyjson_read_file(record, 0, NULL, NULL); + yyjson_val *root = document ? yyjson_doc_get_root(document) : NULL; + yyjson_val *schema = + root && yyjson_is_obj(root) ? yyjson_obj_get(root, "schema_version") : NULL; + yyjson_val *stored_project = + root && yyjson_is_obj(root) ? yyjson_obj_get(root, "project") : NULL; + yyjson_val *attempt_id = + root && yyjson_is_obj(root) ? yyjson_obj_get(root, "attempt_id") : NULL; + yyjson_val *state = root && yyjson_is_obj(root) ? yyjson_obj_get(root, "state") : NULL; + if (!schema || !yyjson_is_uint(schema) || yyjson_get_uint(schema) != 1U || !stored_project || + !yyjson_is_str(stored_project) || strcmp(yyjson_get_str(stored_project), project) != 0 || + !attempt_id || !yyjson_is_str(attempt_id) || yyjson_get_len(attempt_id) != 32U || !state || + !yyjson_is_str(state)) { + yyjson_doc_free(document); + return NULL; + } + return document; +} + +static bool index_attempt_put_string(yyjson_mut_doc *document, yyjson_mut_val *object, + const char *key, const char *value) { + yyjson_mut_val *json_key = yyjson_mut_strcpy(document, key); + yyjson_mut_val *json_value = yyjson_mut_strcpy(document, value ? value : ""); + return json_key && json_value && yyjson_mut_obj_put(object, json_key, json_value); +} + +static bool index_attempt_put_bool(yyjson_mut_doc *document, yyjson_mut_val *object, + const char *key, bool value) { + yyjson_mut_val *json_key = yyjson_mut_strcpy(document, key); + yyjson_mut_val *json_value = yyjson_mut_bool(document, value); + return json_key && json_value && yyjson_mut_obj_put(object, json_key, json_value); +} + +static bool index_attempt_put_uint(yyjson_mut_doc *document, yyjson_mut_val *object, + const char *key, uint64_t value) { + yyjson_mut_val *json_key = yyjson_mut_strcpy(document, key); + yyjson_mut_val *json_value = yyjson_mut_uint(document, value); + return json_key && json_value && yyjson_mut_obj_put(object, json_key, json_value); +} + +static bool index_attempt_put_value(yyjson_mut_doc *document, yyjson_mut_val *object, + const char *key, yyjson_mut_val *value) { + yyjson_mut_val *json_key = yyjson_mut_strcpy(document, key); + return json_key && value && yyjson_mut_obj_put(object, json_key, value); +} + +static bool index_attempt_now(char out[32]) { + time_t now = time(NULL); + struct tm utc; + return cbm_gmtime_r(&now, &utc) && strftime(out, 32, "%Y-%m-%dT%H:%M:%SZ", &utc) > 0; +} + +/* A false `available` means only "no trustworthy start time", never "the + * process is gone"; callers must fall back to plain liveness before deciding + * an attempt was abandoned. */ +typedef struct { + bool available; + uint64_t token; +} index_attempt_process_start_t; + +static void index_attempt_process_start(uint64_t process_id, index_attempt_process_start_t *start) { + if (!start) { + return; + } + start->available = false; + start->token = 0; + if (process_id == 0) { + return; + } +#ifdef _WIN32 + if (process_id > UINT32_MAX) { + return; + } + HANDLE process = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, (DWORD)process_id); + if (!process) { + return; + } + FILETIME created; + FILETIME exited; + FILETIME kernel; + FILETIME user; + DWORD exit_code = 0; + if (GetProcessTimes(process, &created, &exited, &kernel, &user) != 0 && + GetExitCodeProcess(process, &exit_code) != 0 && exit_code == STILL_ACTIVE) { + start->available = true; + start->token = ((uint64_t)created.dwHighDateTime << 32U) | created.dwLowDateTime; + } + (void)CloseHandle(process); +#elif defined(__APPLE__) + if (process_id > INT_MAX) { + return; + } + struct proc_bsdinfo info; + memset(&info, 0, sizeof(info)); + if (proc_pidinfo((int)process_id, PROC_PIDTBSDINFO, 0, &info, sizeof(info)) != + (int)sizeof(info) || + info.pbi_pid != (uint32_t)process_id || info.pbi_status == SZOMB) { + return; + } + start->available = true; + start->token = info.pbi_start_tvsec > UINT64_MAX / UINT64_C(1000000) + ? UINT64_MAX + : info.pbi_start_tvsec * UINT64_C(1000000) + info.pbi_start_tvusec; +#elif defined(__linux__) + if (process_id > INT_MAX) { + return; + } + char path[64]; + int length = snprintf(path, sizeof(path), "/proc/%llu/stat", (unsigned long long)process_id); + if (length <= 0 || (size_t)length >= sizeof(path)) { + return; + } + FILE *file = fopen(path, "rb"); + char buffer[CBM_SZ_4K]; + bool read = file && fgets(buffer, sizeof(buffer), file) != NULL; + if (file) { + (void)fclose(file); + } + char *fields = read ? strrchr(buffer, ')') : NULL; + if (!fields || fields[1] != ' ') { + return; + } + fields += 2; + char *save = NULL; + char *field = strtok_r(fields, " ", &save); + if (!field || field[0] == 'Z') { + return; + } + for (int index = 0; field && index < 19; index++) { + field = strtok_r(NULL, " ", &save); + } + if (!field) { + return; + } + char *end = NULL; + unsigned long long parsed = strtoull(field, &end, 10); + if (end == field || (*end != '\0' && *end != '\n')) { + return; + } + start->available = true; + start->token = (uint64_t)parsed; +#else + (void)process_id; +#endif +} + +static bool index_attempt_generate_id(char out[33]) { + unsigned char bytes[16]; + if (!cbm_secure_random(bytes, sizeof(bytes))) { + return false; + } + static const char HEX[] = "0123456789abcdef"; + for (size_t index = 0; index < sizeof(bytes); index++) { + out[index * 2U] = HEX[bytes[index] >> 4U]; + out[index * 2U + 1U] = HEX[bytes[index] & 0x0FU]; + } + out[32] = '\0'; + return true; +} + +static bool index_attempt_write(const char *project, yyjson_mut_doc *document) { + char directory[CBM_SZ_4K]; + char record[CBM_SZ_4K]; + if (!index_attempt_paths(project, true, directory, sizeof(directory), record, sizeof(record))) { + return false; + } + char temporary[CBM_SZ_4K]; + int temporary_length = + snprintf(temporary, sizeof(temporary), "%s/.%s.XXXXXX", directory, project); + if (temporary_length <= 0 || (size_t)temporary_length >= sizeof(temporary)) { + return false; + } + int descriptor = cbm_mkstemp(temporary); + if (descriptor < 0) { + return false; + } + char *json = yyjson_mut_write(document, 0, NULL); + FILE *file = json ? mcp_fdopen(descriptor, "wb") : NULL; + bool written = false; + if (file) { + size_t length = strlen(json); + bool content_written = fwrite(json, 1, length, file) == length; + bool flushed = fflush(file) == 0; + bool synced = flushed && mcp_fsync(cbm_fileno(file)) == 0; + bool closed = fclose(file) == 0; + written = content_written && flushed && synced && closed; + } else { + (void)mcp_close(descriptor); + } + free(json); + if (!written || cbm_rename_replace(temporary, record) != 0) { + (void)cbm_unlink(temporary); + return false; + } + return true; +} + +static void index_attempt_copy_generation(yyjson_mut_doc *document, yyjson_mut_val *target, + yyjson_val *previous) { + static const char *KEYS[] = { + "generation_snapshot_available", "generation_commit", "generation_dirty", + "serviceable_generation", "watcher_observed_change", + }; + for (size_t index = 0; previous && index < sizeof(KEYS) / sizeof(KEYS[0]); index++) { + yyjson_val *value = yyjson_obj_get(previous, KEYS[index]); + yyjson_mut_val *copy = value ? yyjson_val_mut_copy(document, value) : NULL; + if (copy) { + yyjson_mut_obj_add_val(document, target, KEYS[index], copy); + } + } +} + +bool cbm_mcp_index_attempt_begin(const char *project, const char *repo_path, const char *origin, + const cbm_index_resource_policy_t *policy, + bool watcher_observed_change, char attempt_id[33]) { + if (!project || !repo_path || !repo_path[0] || !policy || !attempt_id || + (!origin || (strcmp(origin, "explicit") != 0 && strcmp(origin, "auto") != 0 && + strcmp(origin, "watcher") != 0)) || + !index_attempt_generate_id(attempt_id)) { + return false; + } + index_attempt_lock_t lock; + if (!index_attempt_lock_acquire(&lock, project)) { + return false; + } + yyjson_doc *previous_document = index_attempt_read(project, NULL); + yyjson_val *previous = previous_document ? yyjson_doc_get_root(previous_document) : NULL; + yyjson_val *previous_watcher = previous && yyjson_is_obj(previous) + ? yyjson_obj_get(previous, "watcher_observed_change") + : NULL; + yyjson_val *previous_state = + previous && yyjson_is_obj(previous) ? yyjson_obj_get(previous, "state") : NULL; + bool unresolved_watcher_change = previous_watcher && yyjson_is_bool(previous_watcher) && + yyjson_get_bool(previous_watcher) && previous_state && + yyjson_is_str(previous_state) && + strcmp(yyjson_get_str(previous_state), "completed") != 0; + + char queued_at[32]; + uint64_t owner_pid = (uint64_t)getpid(); + index_attempt_process_start_t owner_start; + index_attempt_process_start(owner_pid, &owner_start); + yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = document ? yyjson_mut_obj(document) : NULL; + bool valid = document && root && index_attempt_now(queued_at); + if (valid) { + yyjson_mut_doc_set_root(document, root); + valid = index_attempt_put_uint(document, root, "schema_version", 1U) && + index_attempt_put_string(document, root, "project", project) && + index_attempt_put_string(document, root, "repo_path", repo_path) && + index_attempt_put_string(document, root, "attempt_id", attempt_id) && + index_attempt_put_uint(document, root, "owner_pid", owner_pid) && + index_attempt_put_string(document, root, "origin", origin) && + index_attempt_put_string(document, root, "state", "queued") && + index_attempt_put_string(document, root, "profile", + cbm_index_policy_profile_name(policy)) && + index_attempt_put_string(document, root, "policy_source", + cbm_index_policy_source_name(policy)) && + index_attempt_put_string(document, root, "queued_at", queued_at); + if (valid && owner_start.available) { + valid = index_attempt_put_uint(document, root, "owner_start_token", owner_start.token); + } + } + if (valid) { + index_attempt_copy_generation(document, root, previous); + valid = index_attempt_put_bool(document, root, "watcher_observed_change", + watcher_observed_change || unresolved_watcher_change) && + index_attempt_write(project, document); + } + yyjson_mut_doc_free(document); + yyjson_doc_free(previous_document); + index_attempt_lock_release(&lock); + return valid; +} + +static bool index_attempt_transition_allowed(const char *from, const char *to) { + if (!from || !to) { + return false; + } + if (strcmp(from, "queued") == 0) { + return strcmp(to, "running") == 0 || strcmp(to, "completed") == 0 || + strcmp(to, "failed") == 0 || strcmp(to, "cancelled") == 0; + } + return strcmp(from, "running") == 0 && + (strcmp(to, "completed") == 0 || strcmp(to, "failed") == 0 || + strcmp(to, "cancelled") == 0); +} + +static bool index_attempt_capture_start_snapshot(yyjson_mut_doc *document, yyjson_mut_val *root, + const char *repo_path) { + cbm_git_snapshot_t snapshot = {0}; + bool snapshot_available = + cbm_git_snapshot_read(repo_path, &snapshot) == 0 && snapshot.available; + bool valid = + index_attempt_put_bool(document, root, "attempt_snapshot_available", snapshot_available); + if (valid && snapshot_available) { + valid = index_attempt_put_string(document, root, "attempt_commit", snapshot.head_sha) && + index_attempt_put_bool(document, root, "attempt_dirty", snapshot.dirty); + } + cbm_git_snapshot_free(&snapshot); + return valid; +} + +static bool index_attempt_capture_generation(yyjson_mut_doc *document, yyjson_mut_val *root, + const char *project, const char *repo_path) { + yyjson_mut_val *attempt_available = yyjson_mut_obj_get(root, "attempt_snapshot_available"); + yyjson_mut_val *attempt_commit = yyjson_mut_obj_get(root, "attempt_commit"); + yyjson_mut_val *attempt_dirty = yyjson_mut_obj_get(root, "attempt_dirty"); + cbm_git_snapshot_t completed = {0}; + bool completed_available = + cbm_git_snapshot_read(repo_path, &completed) == 0 && completed.available; + char database_path[CBM_SZ_1K]; + project_db_path(project, database_path, sizeof(database_path)); + cbm_store_t *store = database_path[0] ? cbm_store_open_path_query(database_path) : NULL; + cbm_project_t stored = {0}; + bool serviceable = store && cbm_store_get_project(store, project, &stored) == CBM_STORE_OK && + stored.indexed_at; + bool available = attempt_available && yyjson_mut_is_bool(attempt_available) && + yyjson_mut_get_bool(attempt_available) && attempt_commit && + yyjson_mut_is_str(attempt_commit) && attempt_dirty && + yyjson_mut_is_bool(attempt_dirty) && !yyjson_mut_get_bool(attempt_dirty) && + completed_available && !completed.dirty && serviceable && + strcmp(yyjson_mut_get_str(attempt_commit), completed.head_sha) == 0; + bool valid = index_attempt_put_bool(document, root, "generation_snapshot_available", available); + if (available) { + valid = valid && + index_attempt_put_string(document, root, "generation_commit", completed.head_sha) && + index_attempt_put_bool(document, root, "generation_dirty", false); + } else { + (void)yyjson_mut_obj_remove_key(root, "generation_commit"); + (void)yyjson_mut_obj_remove_key(root, "generation_dirty"); + } + cbm_git_snapshot_free(&completed); + + if (serviceable) { + valid = valid && index_attempt_put_string(document, root, "serviceable_generation", + stored.indexed_at); + } + cbm_project_free_fields(&stored); + if (store) { + cbm_store_close(store); + } + return valid && index_attempt_put_bool(document, root, "watcher_observed_change", false); +} + +bool cbm_mcp_index_attempt_transition(const char *project, const char *repo_path, + const char *attempt_id, const char *state, + const char *failure_code, + const cbm_index_resource_violation_t *violation, + bool generation_completed) { + index_attempt_lock_t lock; + if (!index_attempt_lock_acquire(&lock, project)) { + return false; + } + yyjson_doc *source = index_attempt_read(project, NULL); + yyjson_val *source_root = source ? yyjson_doc_get_root(source) : NULL; + yyjson_val *stored_id = source_root && yyjson_is_obj(source_root) + ? yyjson_obj_get(source_root, "attempt_id") + : NULL; + yyjson_val *stored_state = + source_root && yyjson_is_obj(source_root) ? yyjson_obj_get(source_root, "state") : NULL; + if (!source_root || !stored_id || !yyjson_is_str(stored_id) || + strcmp(yyjson_get_str(stored_id), attempt_id ? attempt_id : "") != 0 || !stored_state || + !yyjson_is_str(stored_state) || + !index_attempt_transition_allowed(yyjson_get_str(stored_state), state)) { + yyjson_doc_free(source); + index_attempt_lock_release(&lock); + return false; + } + + yyjson_mut_doc *document = yyjson_doc_mut_copy(source, NULL); + yyjson_mut_val *root = document ? yyjson_mut_doc_get_root(document) : NULL; + char timestamp[32]; + bool terminal = strcmp(state, "completed") == 0 || strcmp(state, "failed") == 0 || + strcmp(state, "cancelled") == 0; + bool valid = root && index_attempt_now(timestamp) && + index_attempt_put_string(document, root, "state", state); + if (valid && strcmp(state, "running") == 0) { + valid = index_attempt_put_string(document, root, "started_at", timestamp) && + index_attempt_capture_start_snapshot(document, root, repo_path); + } else if (valid && terminal) { + valid = index_attempt_put_string(document, root, "finished_at", timestamp); + } + if (valid && failure_code && failure_code[0]) { + valid = index_attempt_put_string(document, root, "failure_code", failure_code); + } else if (valid && terminal) { + (void)yyjson_mut_obj_remove_key(root, "failure_code"); + } + if (valid && violation && violation->resource != CBM_INDEX_RESOURCE_NONE) { + yyjson_mut_val *resource = yyjson_mut_obj(document); + valid = resource && + index_attempt_put_string(document, resource, "resource", + cbm_index_resource_name(violation->resource)) && + index_attempt_put_bool(document, resource, "probe_failed", violation->probe_failed); + if (valid && !violation->probe_failed) { + valid = index_attempt_put_uint(document, resource, "observed", violation->observed) && + index_attempt_put_uint(document, resource, "limit", violation->limit) && + index_attempt_put_string(document, resource, "unit", + cbm_index_resource_unit(violation->resource)); + } + if (valid) { + valid = index_attempt_put_value(document, root, "resource_failure", resource); + } + } + if (valid && generation_completed) { + valid = index_attempt_capture_generation(document, root, project, repo_path); + } + if (valid) { + valid = index_attempt_write(project, document); + } + yyjson_mut_doc_free(document); + yyjson_doc_free(source); + index_attempt_lock_release(&lock); + return valid; +} + +bool cbm_mcp_index_attempt_mark_watcher_change(const char *project, const char *attempt_id) { + index_attempt_lock_t lock; + if (!index_attempt_lock_acquire(&lock, project)) { + return false; + } + yyjson_doc *source = index_attempt_read(project, NULL); + yyjson_val *source_root = source ? yyjson_doc_get_root(source) : NULL; + yyjson_val *stored_id = source_root && yyjson_is_obj(source_root) + ? yyjson_obj_get(source_root, "attempt_id") + : NULL; + if (!stored_id || !yyjson_is_str(stored_id) || + strcmp(yyjson_get_str(stored_id), attempt_id ? attempt_id : "") != 0) { + yyjson_doc_free(source); + index_attempt_lock_release(&lock); + return false; + } + yyjson_mut_doc *document = yyjson_doc_mut_copy(source, NULL); + yyjson_mut_val *root = document ? yyjson_mut_doc_get_root(document) : NULL; + bool valid = root && index_attempt_put_bool(document, root, "watcher_observed_change", true) && + index_attempt_write(project, document); + yyjson_mut_doc_free(document); + yyjson_doc_free(source); + index_attempt_lock_release(&lock); + return valid; +} + +static cbm_index_resource_t index_attempt_resource_from_name(const char *name) { + for (int resource = CBM_INDEX_RESOURCE_FILES; + resource <= CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS; resource++) { + if (name && strcmp(name, cbm_index_resource_name((cbm_index_resource_t)resource)) == 0) { + return (cbm_index_resource_t)resource; + } + } + return CBM_INDEX_RESOURCE_NONE; +} + +static void index_attempt_finish_response(const char *project, const char *repo_path, + const char *attempt_id, const char *response) { + yyjson_doc *outer = response ? yyjson_read(response, strlen(response), 0) : NULL; + yyjson_val *outer_root = outer ? yyjson_doc_get_root(outer) : NULL; + yyjson_val *is_error = + outer_root && yyjson_is_obj(outer_root) ? yyjson_obj_get(outer_root, "isError") : NULL; + yyjson_val *content = + outer_root && yyjson_is_obj(outer_root) ? yyjson_obj_get(outer_root, "content") : NULL; + yyjson_val *first = content && yyjson_is_arr(content) ? yyjson_arr_get_first(content) : NULL; + yyjson_val *text = first && yyjson_is_obj(first) ? yyjson_obj_get(first, "text") : NULL; + yyjson_doc *inner = text && yyjson_is_str(text) + ? yyjson_read(yyjson_get_str(text), yyjson_get_len(text), 0) + : NULL; + yyjson_val *inner_root = inner ? yyjson_doc_get_root(inner) : NULL; + yyjson_val *status = + inner_root && yyjson_is_obj(inner_root) ? yyjson_obj_get(inner_root, "status") : NULL; + bool successful = (!is_error || !yyjson_is_bool(is_error) || !yyjson_get_bool(is_error)) && + status && yyjson_is_str(status) && + (strcmp(yyjson_get_str(status), "indexed") == 0 || + strcmp(yyjson_get_str(status), "degraded") == 0); + const char *failure_code = successful ? NULL : "index_failed"; + yyjson_val *code = + inner_root && yyjson_is_obj(inner_root) ? yyjson_obj_get(inner_root, "code") : NULL; + if (!successful && code && yyjson_is_str(code) && yyjson_get_len(code) > 0) { + failure_code = yyjson_get_str(code); + } + cbm_index_resource_violation_t violation = {0}; + yyjson_val *resource = + inner_root && yyjson_is_obj(inner_root) ? yyjson_obj_get(inner_root, "resource") : NULL; + yyjson_val *observed = + inner_root && yyjson_is_obj(inner_root) ? yyjson_obj_get(inner_root, "observed") : NULL; + yyjson_val *limit = + inner_root && yyjson_is_obj(inner_root) ? yyjson_obj_get(inner_root, "limit") : NULL; + if (resource && yyjson_is_str(resource)) { + violation.resource = index_attempt_resource_from_name(yyjson_get_str(resource)); + violation.probe_failed = failure_code && strcmp(failure_code, "resource_probe_failed") == 0; + if (observed && yyjson_is_uint(observed) && limit && yyjson_is_uint(limit)) { + violation.observed = yyjson_get_uint(observed); + violation.limit = yyjson_get_uint(limit); + } + } + (void)cbm_mcp_index_attempt_transition( + project, repo_path, attempt_id, successful ? "completed" : "failed", failure_code, + violation.resource != CBM_INDEX_RESOURCE_NONE ? &violation : NULL, successful); + yyjson_doc_free(inner); + yyjson_doc_free(outer); +} + +cbm_index_attempt_read_status_t cbm_mcp_index_attempt_add_status(yyjson_mut_doc *document, + yyjson_mut_val *root, + const char *project, + const char *current_root_path) { + bool exists = false; + yyjson_doc *attempt = index_attempt_read(project, &exists); + if (!exists) { + return CBM_INDEX_ATTEMPT_NONE; + } + if (!attempt) { + yyjson_mut_obj_add_null(document, root, "last_index_attempt"); + yyjson_mut_obj_add_str(document, root, "freshness", "unknown"); + return CBM_INDEX_ATTEMPT_CORRUPT; + } + yyjson_val *attempt_root = yyjson_doc_get_root(attempt); + yyjson_mut_val *copy = yyjson_val_mut_copy(document, attempt_root); + if (!copy || !yyjson_mut_obj_add_val(document, root, "last_index_attempt", copy)) { + yyjson_doc_free(attempt); + return CBM_INDEX_ATTEMPT_CORRUPT; + } + + const char *freshness = "unknown"; + yyjson_val *watcher_change = yyjson_obj_get(attempt_root, "watcher_observed_change"); + yyjson_val *state = yyjson_obj_get(attempt_root, "state"); + bool watcher_stale = watcher_change && yyjson_is_bool(watcher_change) && + yyjson_get_bool(watcher_change) && state && yyjson_is_str(state) && + strcmp(yyjson_get_str(state), "completed") != 0; + yyjson_val *available = yyjson_obj_get(attempt_root, "generation_snapshot_available"); + yyjson_val *generation_commit = yyjson_obj_get(attempt_root, "generation_commit"); + yyjson_val *generation_dirty = yyjson_obj_get(attempt_root, "generation_dirty"); + if (watcher_stale) { + freshness = "stale"; + } else if (available && yyjson_is_bool(available) && yyjson_get_bool(available) && + generation_commit && yyjson_is_str(generation_commit) && generation_dirty && + yyjson_is_bool(generation_dirty) && !yyjson_get_bool(generation_dirty) && + current_root_path && current_root_path[0]) { + cbm_git_snapshot_t current = {0}; + if (cbm_git_snapshot_read(current_root_path, ¤t) == 0 && current.available) { + if (!current.dirty) { + freshness = strcmp(current.head_sha, yyjson_get_str(generation_commit)) == 0 + ? "fresh" + : "stale"; + } + } + cbm_git_snapshot_free(¤t); + } + yyjson_mut_obj_add_str(document, root, "freshness", freshness); + yyjson_doc_free(attempt); + return CBM_INDEX_ATTEMPT_AVAILABLE; +} + +bool cbm_mcp_index_attempt_remove(const char *project) { + index_attempt_lock_t lock; + if (!index_attempt_lock_acquire(&lock, project)) { + return false; + } + char directory[CBM_SZ_4K]; + char record[CBM_SZ_4K]; + if (!index_attempt_paths(project, false, directory, sizeof(directory), record, + sizeof(record))) { + index_attempt_lock_release(&lock); + return false; + } + bool removed = access(record, F_OK) != 0 || cbm_unlink(record) == 0; + index_attempt_lock_release(&lock); + return removed; +} + +void cbm_mcp_index_attempt_recover_abandoned(void) { + char directory[CBM_SZ_4K]; + char unused[CBM_SZ_4K]; + if (!index_attempt_paths("recovery-probe", false, directory, sizeof(directory), unused, + sizeof(unused))) { + return; + } + cbm_dir_t *status_directory = cbm_opendir(directory); + if (!status_directory) { + return; + } + cbm_dirent_t *entry; + while ((entry = cbm_readdir(status_directory)) != NULL) { + size_t length = strlen(entry->name); + if (length <= 5U || strcmp(entry->name + length - 5U, ".json") != 0) { + continue; + } + char project[CBM_SZ_1K]; + if (length - 5U >= sizeof(project)) { + continue; + } + memcpy(project, entry->name, length - 5U); + project[length - 5U] = '\0'; + if (!cbm_validate_project_name(project)) { + continue; + } + bool exists = false; + yyjson_doc *document = index_attempt_read(project, &exists); + yyjson_val *root = document ? yyjson_doc_get_root(document) : NULL; + yyjson_val *state = root ? yyjson_obj_get(root, "state") : NULL; + yyjson_val *attempt_id = root ? yyjson_obj_get(root, "attempt_id") : NULL; + yyjson_val *repo_path = root ? yyjson_obj_get(root, "repo_path") : NULL; + yyjson_val *owner_pid = root ? yyjson_obj_get(root, "owner_pid") : NULL; + yyjson_val *owner_start_token = root ? yyjson_obj_get(root, "owner_start_token") : NULL; + bool abandoned = state && yyjson_is_str(state) && + (strcmp(yyjson_get_str(state), "queued") == 0 || + strcmp(yyjson_get_str(state), "running") == 0); + if (abandoned && owner_pid && yyjson_is_uint(owner_pid)) { + uint64_t process_id = yyjson_get_uint(owner_pid); + bool same_process = false; + if (owner_start_token && yyjson_is_uint(owner_start_token)) { + index_attempt_process_start_t current; + index_attempt_process_start(process_id, ¤t); + same_process = + current.available && current.token == yyjson_get_uint(owner_start_token); +#ifdef _WIN32 + if (!current.available && GetLastError() == ERROR_ACCESS_DENIED) { + same_process = true; + } +#else + if (!current.available && process_id > 0 && process_id <= (uint64_t)INT_MAX && + kill((pid_t)process_id, 0) != 0 && errno == EPERM) { + same_process = true; + } +#endif + } else { +#ifdef _WIN32 + HANDLE process = process_id > 0 && process_id <= UINT32_MAX + ? OpenProcess(SYNCHRONIZE, FALSE, (DWORD)process_id) + : NULL; + if (process) { + same_process = WaitForSingleObject(process, 0) == WAIT_TIMEOUT; + CloseHandle(process); + } else if (GetLastError() == ERROR_ACCESS_DENIED) { + same_process = true; + } +#else + same_process = process_id > 0 && process_id <= (uint64_t)INT_MAX && + (kill((pid_t)process_id, 0) == 0 || errno == EPERM); +#endif + } + if (same_process) { + abandoned = false; + } + } + if (abandoned && attempt_id && yyjson_is_str(attempt_id) && repo_path && + yyjson_is_str(repo_path)) { + char id[33]; + (void)snprintf(id, sizeof(id), "%s", yyjson_get_str(attempt_id)); + char path[CBM_SZ_4K]; + (void)snprintf(path, sizeof(path), "%s", yyjson_get_str(repo_path)); + yyjson_doc_free(document); + document = NULL; + (void)cbm_mcp_index_attempt_transition(project, path, id, "failed", "worker_lost", NULL, + false); + } + yyjson_doc_free(document); + } + cbm_closedir(status_directory); +} + static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) { char *project = get_project_arg(args); cbm_store_t *store = resolve_store(srv, project); - REQUIRE_STORE(store, project); + if (!store) { + yyjson_mut_doc *attempt_document = yyjson_mut_doc_new(NULL); + yyjson_mut_val *attempt_root = attempt_document ? yyjson_mut_obj(attempt_document) : NULL; + cbm_index_attempt_read_status_t attempt_status = + attempt_root && project + ? cbm_mcp_index_attempt_add_status(attempt_document, attempt_root, project, NULL) + : CBM_INDEX_ATTEMPT_NONE; + if (attempt_status != CBM_INDEX_ATTEMPT_NONE) { + yyjson_mut_doc_set_root(attempt_document, attempt_root); + yyjson_mut_obj_add_strcpy(attempt_document, attempt_root, "project", project); + yyjson_mut_obj_add_int(attempt_document, attempt_root, "nodes", 0); + yyjson_mut_obj_add_int(attempt_document, attempt_root, "edges", 0); + yyjson_mut_obj_add_str(attempt_document, attempt_root, "status", "unavailable"); + char *json = yy_doc_to_str(attempt_document); + yyjson_mut_doc_free(attempt_document); + free(project); + char *result = cbm_mcp_text_result(json, false); + free(json); + return result; + } + yyjson_mut_doc_free(attempt_document); + REQUIRE_STORE(store, project); + } /* The git context block (worktree/shadow path variants) only matters when * debugging index-location issues — gate it so the common status call * stays lean. */ @@ -4577,6 +5385,7 @@ static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) { if (verbose) { add_git_context_json(doc, root, proj_info.root_path); } + (void)cbm_mcp_index_attempt_add_status(doc, root, project, proj_info.root_path); safe_str_free(&proj_info.name); safe_str_free(&proj_info.indexed_at); safe_str_free(&proj_info.root_path); @@ -4645,13 +5454,29 @@ static char *handle_delete_project(cbm_mcp_server_t *srv, const char *args) { (void)cbm_unlink(shm); if (rc == 0) { status = "deleted"; + if (!cbm_mcp_index_attempt_remove(name)) { + status = "delete_failed"; + error_detail = "project deleted but attempt status cleanup failed"; + is_error = true; + } } else { status = "delete_failed"; error_detail = strerror(errno); is_error = true; } } else { - is_error = true; + bool attempt_exists = false; + yyjson_doc *attempt = index_attempt_read(name, &attempt_exists); + yyjson_doc_free(attempt); + if (attempt_exists && cbm_mcp_index_attempt_remove(name)) { + status = "deleted"; + } else { + is_error = true; + if (attempt_exists) { + status = "delete_failed"; + error_detail = "attempt status cleanup failed"; + } + } } cbm_pipeline_unlock(); @@ -8158,7 +8983,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, * run it through index_run_supervised. Shared by the session auto-index (srv * present → its cached store is invalidated) and the watcher re-index (srv NULL). * Returns the worker's response string (caller frees) or NULL to degrade. */ -static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_path) { +static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_path, + const char *origin) { if (!root_path || !root_path[0]) { return NULL; } @@ -8172,6 +8998,7 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p yyjson_mut_val *root = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root); if (!yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path) || + !yyjson_mut_obj_add_strcpy(doc, root, "_cbm_index_origin", origin) || !cbm_mcp_index_policy_add_to_args(doc, root, &policy)) { yyjson_mut_doc_free(doc); return NULL; @@ -8181,7 +9008,20 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p if (!args) { return NULL; } + char *project = cbm_project_name_from_path(root_path); + char attempt_id[33]; + bool attempt_recorded = + project && cbm_mcp_index_attempt_begin(project, root_path, origin, &policy, + strcmp(origin, "watcher") == 0, attempt_id); + if (attempt_recorded) { + (void)cbm_mcp_index_attempt_transition(project, root_path, attempt_id, "running", NULL, + NULL, false); + } char *resp = index_run_supervised(srv, args, &policy); + if (attempt_recorded) { + index_attempt_finish_response(project, root_path, attempt_id, resp); + } + free(project); free(args); return resp; } @@ -8189,7 +9029,7 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p /* Public entry (see mcp.h): the watcher re-index in main.c has no MCP server, so * it reaches the supervised runner through this srv-less wrapper. */ char *cbm_mcp_index_run_supervised_path(const char *root_path) { - return index_run_supervised_path(NULL, root_path); + return index_run_supervised_path(NULL, root_path, "watcher"); } bool cbm_path_within_root(const char *root_path, const char *abs_path); /* defined below */ @@ -8249,7 +9089,11 @@ static char *index_args_with_repo_path(const char *args, const char *canonical_r while (yyjson_mut_obj_get(copy_root, "_cbm_index_policy")) { (void)yyjson_mut_obj_remove_key(copy_root, "_cbm_index_policy"); } + while (yyjson_mut_obj_get(copy_root, "_cbm_index_origin")) { + (void)yyjson_mut_obj_remove_key(copy_root, "_cbm_index_origin"); + } if (!yyjson_mut_obj_add_strcpy(copy, copy_root, "repo_path", canonical_repo_path) || + !yyjson_mut_obj_add_str(copy, copy_root, "_cbm_index_origin", "explicit") || !cbm_mcp_index_policy_add_to_args(copy, copy_root, policy)) { yyjson_mut_doc_free(copy); return NULL; @@ -8408,7 +9252,24 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { free(name_override); return cbm_mcp_text_result("failed to prepare supervised index request", true); } + char *owned_origin = cbm_mcp_get_string_arg(worker_args, "_cbm_index_origin"); + const char *origin = owned_origin && (strcmp(owned_origin, "auto") == 0 || + strcmp(owned_origin, "watcher") == 0) + ? owned_origin + : "explicit"; + char attempt_id[33]; + bool attempt_recorded = + cbm_mcp_index_attempt_begin(mutation_project, repo_path, origin, &resource_policy, + strcmp(origin, "watcher") == 0, attempt_id); + if (attempt_recorded) { + (void)cbm_mcp_index_attempt_transition(mutation_project, repo_path, attempt_id, + "running", NULL, NULL, false); + } char *supervised = index_run_supervised(srv, worker_args, &resource_policy); + if (attempt_recorded) { + index_attempt_finish_response(mutation_project, repo_path, attempt_id, supervised); + } + free(owned_origin); free(worker_args); if (supervised) { free(mutation_project); @@ -8453,9 +9314,17 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { free(mode_str); bool persistence = cbm_mcp_get_bool_arg(args, "persistence"); + char attempt_id[33]; + bool attempt_recorded = !cbm_index_worker_active() && + cbm_mcp_index_attempt_begin(mutation_project, repo_path, "explicit", + &resource_policy, false, attempt_id); cbm_pipeline_t *p = cbm_pipeline_new(repo_path, NULL, mode); if (!p) { + if (attempt_recorded) { + (void)cbm_mcp_index_attempt_transition(mutation_project, repo_path, attempt_id, + "failed", "index_failed", NULL, false); + } mcp_project_mutation_end(srv, mutation_project); free(mutation_project); free(name_override); @@ -8463,6 +9332,10 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { return cbm_mcp_text_result("failed to create pipeline", true); } if (name_override && name_override[0] && !cbm_pipeline_set_project_name(p, name_override)) { + if (attempt_recorded) { + (void)cbm_mcp_index_attempt_transition(mutation_project, repo_path, attempt_id, + "failed", "index_failed", NULL, false); + } cbm_pipeline_free(p); mcp_project_mutation_end(srv, mutation_project); free(mutation_project); @@ -8490,6 +9363,11 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { free(srv->current_project); srv->current_project = NULL; + if (attempt_recorded) { + (void)cbm_mcp_index_attempt_transition(project_name, repo_path, attempt_id, "running", NULL, + NULL, false); + } + /* Serialize pipeline runs to prevent concurrent writes. * Track active pipeline so signal handler and notifications/cancelled * can cancel it mid-run. */ @@ -8514,6 +9392,18 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { cbm_pipeline_get_file_errors(p, &file_errors, &file_error_count); cbm_index_resource_violation_t resource_violation = {0}; cbm_pipeline_get_resource_violation(p, &resource_violation); + if (attempt_recorded) { + const char *failure_code = + rc == 0 ? NULL + : resource_violation.resource != CBM_INDEX_RESOURCE_NONE + ? (resource_violation.probe_failed ? "resource_probe_failed" + : "resource_limit_exceeded") + : "index_failed"; + (void)cbm_mcp_index_attempt_transition( + project_name, repo_path, attempt_id, rc == 0 ? "completed" : "failed", failure_code, + resource_violation.resource != CBM_INDEX_RESOURCE_NONE ? &resource_violation : NULL, + rc == 0); + } cbm_mem_collect(); /* return mimalloc pages to OS after large indexing */ @@ -11784,7 +12674,7 @@ static void *autoindex_thread(void *arg) { * 100% of that memory back to the OS every cycle. In a marked host this is a * safety boundary: preparation/start failure stops the operation. */ if (cbm_index_supervisor_should_wrap()) { - char *resp = index_run_supervised_path(srv, srv->session_root); + char *resp = index_run_supervised_path(srv, srv->session_root, "auto"); if (resp) { free(resp); cbm_log_info("autoindex.done", "project", srv->session_project, "mode", "supervised"); @@ -11806,18 +12696,45 @@ static void *autoindex_thread(void *arg) { cbm_log_warn("autoindex.err", "msg", "resource_policy_load_failed", "error", policy_error); return NULL; } + char attempt_id[33]; + bool attempt_recorded = cbm_mcp_index_attempt_begin( + srv->session_project, srv->session_root, "auto", &resource_policy, false, attempt_id); cbm_pipeline_t *p = cbm_pipeline_new(srv->session_root, NULL, CBM_MODE_FULL); if (!p) { + if (attempt_recorded) { + (void)cbm_mcp_index_attempt_transition(srv->session_project, srv->session_root, + attempt_id, "failed", "index_failed", NULL, + false); + } cbm_log_warn("autoindex.err", "msg", "pipeline_create_failed"); return NULL; } cbm_pipeline_set_resource_policy(p, &resource_policy); + if (attempt_recorded) { + (void)cbm_mcp_index_attempt_transition(srv->session_project, srv->session_root, attempt_id, + "running", NULL, NULL, false); + } /* Block until any concurrent pipeline finishes */ cbm_pipeline_lock(); int rc = cbm_pipeline_run(p); cbm_pipeline_unlock(); + cbm_index_resource_violation_t resource_violation = {0}; + cbm_pipeline_get_resource_violation(p, &resource_violation); + if (attempt_recorded) { + const char *failure_code = + rc == 0 ? NULL + : resource_violation.resource != CBM_INDEX_RESOURCE_NONE + ? (resource_violation.probe_failed ? "resource_probe_failed" + : "resource_limit_exceeded") + : "index_failed"; + (void)cbm_mcp_index_attempt_transition( + srv->session_project, srv->session_root, attempt_id, rc == 0 ? "completed" : "failed", + failure_code, + resource_violation.resource != CBM_INDEX_RESOURCE_NONE ? &resource_violation : NULL, + rc == 0); + } cbm_pipeline_free(p); cbm_mem_collect(); /* return mimalloc pages to OS after indexing (in-process only) */ diff --git a/src/mcp/mcp_internal.h b/src/mcp/mcp_internal.h index b42b43315..46ee7265e 100644 --- a/src/mcp/mcp_internal.h +++ b/src/mcp/mcp_internal.h @@ -38,6 +38,28 @@ bool cbm_mcp_index_task_db_path(const char *args, char *path_out, size_t path_si char *cbm_mcp_index_worker_resource_response(const char *args, const cbm_index_worker_result_t *worker_result); +typedef enum { + CBM_INDEX_ATTEMPT_NONE = 0, + CBM_INDEX_ATTEMPT_AVAILABLE, + CBM_INDEX_ATTEMPT_CORRUPT, +} cbm_index_attempt_read_status_t; + +bool cbm_mcp_index_attempt_begin(const char *project, const char *repo_path, const char *origin, + const cbm_index_resource_policy_t *policy, + bool watcher_observed_change, char attempt_id[33]); +bool cbm_mcp_index_attempt_transition(const char *project, const char *repo_path, + const char *attempt_id, const char *state, + const char *failure_code, + const cbm_index_resource_violation_t *violation, + bool generation_completed); +bool cbm_mcp_index_attempt_mark_watcher_change(const char *project, const char *attempt_id); +cbm_index_attempt_read_status_t cbm_mcp_index_attempt_add_status(yyjson_mut_doc *document, + yyjson_mut_val *root, + const char *project, + const char *current_root_path); +bool cbm_mcp_index_attempt_remove(const char *project); +void cbm_mcp_index_attempt_recover_abandoned(void); + enum { CBM_MCP_DEFAULT_AUTO_INDEX_LIMIT = 50000 }; /* Count indexable files with the pipeline's native full-mode discovery policy, diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index e715c46c1..a89b316c0 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -2241,8 +2241,81 @@ TEST(daemon_application_programmatic_index_injects_resource_policy) { PASS(); } +TEST(daemon_application_controlled_tool_error_completes_background_observation) { + char *root_created = th_mktempdir("cbm_app_controlled_error_root"); + char *root = root_created ? strdup(root_created) : NULL; + char *cache_created = th_mktempdir("cbm_app_controlled_error_cache"); + char *cache = cache_created ? strdup(cache_created) : NULL; + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + bool cache_env = cache && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0; + cbm_config_t *stored_config = cache ? cbm_config_open(cache) : NULL; + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + atomic_store(&fake.scripted, true); + fake.outcomes[0] = CBM_PROC_CLEAN; + fake.responses[0] = "{\"content\":[{\"type\":\"text\",\"text\":\"{" + "\\\"status\\\":\\\"error\\\"," + "\\\"code\\\":\\\"resource_limit_exceeded\\\"," + "\\\"stage\\\":\\\"discovery\\\"," + "\\\"resource\\\":\\\"entries\\\"," + "\\\"observed\\\":11," + "\\\"limit\\\":10," + "\\\"unit\\\":\\\"entries\\\"}\"}],\"isError\":true}"; + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = { + .config = stored_config, + .worker_ops = &worker_ops, + }; + cbm_daemon_application_t *application = + cache_env && stored_config ? cbm_daemon_application_new(&config) : NULL; + int index_rc = application && root + ? cbm_daemon_application_index(application, "ControlledFailureFixture", root) + : -1; + char attempt_path[APP_TEST_PATH_CAP]; + char attempt_record[APP_TEST_PATH_CAP] = {0}; + (void)snprintf(attempt_path, sizeof(attempt_path), "%s/status/ControlledFailureFixture.json", + cache ? cache : ""); + app_fake_worker_read_file(attempt_path, attempt_record, sizeof(attempt_record)); + bool attempt_failed = strstr(attempt_record, "\"state\":\"failed\"") && + strstr(attempt_record, "\"failure_code\":\"resource_limit_exceeded\"") && + strstr(attempt_record, "\"resource\":\"entries\""); + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + int starts = atomic_load(&fake.starts); + + cbm_daemon_application_free(application); + cbm_config_close(stored_config); + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); + th_cleanup(root); + th_cleanup(cache); + free(root); + free(cache); + + ASSERT_EQ(index_rc, 0); + ASSERT_TRUE(attempt_failed); + ASSERT_EQ(starts, 1); + ASSERT_TRUE(stopped); + PASS(); +} + TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { - char *cache = th_mktempdir("cbm_app_worker_resource_cache"); + char *cache_created = th_mktempdir("cbm_app_worker_resource_cache"); + char *cache = cache_created ? strdup(cache_created) : NULL; + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + bool cache_env = cache && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0; cbm_config_t *stored_config = cache ? cbm_config_open(cache) : NULL; app_fake_worker_context_t fake; app_fake_worker_context_init(&fake); @@ -2276,7 +2349,7 @@ TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { (void)snprintf(args, sizeof(args), "{\"repo_path\":\"%s\",\"name\":\"DaemonWorkerResourceFixture\"}", root ? root : ""); - bool setup = cache && stored_config && application && session && root && + bool setup = cache_env && stored_config && application && session && root && app_test_context_request(root, root, &context, &context_length) && app_test_tool_request("index_repository", args, &tool, &tool_length); uint8_t *response = NULL; @@ -2298,6 +2371,17 @@ TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { strstr((char *)response, "\\\"observed\\\":7001") && strstr((char *)response, "\\\"limit\\\":7000") && strstr((char *)response, "\\\"unit\\\":\\\"milliseconds\\\""); + char attempt_path[APP_TEST_PATH_CAP]; + char attempt_record[APP_TEST_PATH_CAP] = {0}; + (void)snprintf(attempt_path, sizeof(attempt_path), "%s/status/DaemonWorkerResourceFixture.json", + cache ? cache : ""); + app_fake_worker_read_file(attempt_path, attempt_record, sizeof(attempt_record)); + bool attempt_visible = strstr(attempt_record, "\"origin\":\"explicit\"") && + strstr(attempt_record, "\"state\":\"failed\"") && + strstr(attempt_record, "\"failure_code\":\"resource_limit_exceeded\"") && + strstr(attempt_record, "\"resource\":\"duration_ms\"") && + strstr(attempt_record, "\"observed\":7001") && + strstr(attempt_record, "\"limit\":7000"); if (session) { callbacks.session_close(callbacks.context, session); } @@ -2309,12 +2393,20 @@ TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { free(response); free(context); free(tool); + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); th_cleanup(root); th_cleanup(cache); + free(cache); ASSERT_TRUE(setup); ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_OK); ASSERT_TRUE(structured); + ASSERT_TRUE(attempt_visible); ASSERT_EQ(starts, 1); ASSERT_EQ(destroys, 1); ASSERT_TRUE(stopped); @@ -4956,6 +5048,11 @@ TEST(daemon_application_cancellation_between_recovery_attempts_stops_retry) { } TEST(daemon_application_thread_start_failure_rolls_back_job_reservation) { + char *cache_created = th_mktempdir("cbm_app_thread_start_cache"); + char *cache = cache_created ? strdup(cache_created) : NULL; + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + bool cache_ready = cache && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0; app_fake_worker_context_t fake; app_fake_worker_context_init(&fake); atomic_store(&fake.allow_completion, true); @@ -4968,7 +5065,8 @@ TEST(daemon_application_thread_start_failure_rolls_back_job_reservation) { .destroy = app_fake_worker_destroy, }; cbm_daemon_application_config_t config = {.worker_ops = &worker_ops}; - cbm_daemon_application_t *application = cbm_daemon_application_new(&config); + cbm_daemon_application_t *application = + cache_ready ? cbm_daemon_application_new(&config) : NULL; char root[APP_TEST_PATH_CAP]; (void)snprintf(root, sizeof(root), "%s/cbm-app-thread-start-XXXXXX", cbm_tmpdir()); bool root_ok = cbm_mkdtemp(root) != NULL; @@ -4981,17 +5079,35 @@ TEST(daemon_application_thread_start_failure_rolls_back_job_reservation) { size_t active_after_failure = application ? cbm_daemon_application_active_jobs(application) : 1; size_t subscribers_after_failure = application ? cbm_daemon_application_job_subscribers(application, "thread-start") : 1; + char attempt_path[APP_TEST_PATH_CAP]; + char attempt_record[APP_TEST_PATH_CAP] = {0}; + (void)snprintf(attempt_path, sizeof(attempt_path), "%s/status/thread-start.json", + cache ? cache : ""); + app_fake_worker_read_file(attempt_path, attempt_record, sizeof(attempt_record)); + bool failure_recorded = strstr(attempt_record, "\"state\":\"failed\"") && + strstr(attempt_record, "\"failure_code\":\"worker_start_failed\"") && + strstr(attempt_record, "\"origin\":\"explicit\""); int retried = application && root_ok ? cbm_daemon_application_index(application, "thread-start", root) : -1; bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); cbm_daemon_application_free(application); + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); (void)cbm_rmdir(root); + th_cleanup(cache); + free(cache); + ASSERT_TRUE(cache_ready); ASSERT_TRUE(root_ok); ASSERT_LT(failed, 0); ASSERT_EQ(active_after_failure, 0); ASSERT_EQ(subscribers_after_failure, 0); + ASSERT_TRUE(failure_recorded); ASSERT_EQ(retried, 0); ASSERT_EQ(atomic_load(&fake.starts), 1); ASSERT_EQ(atomic_load(&fake.destroys), 1); @@ -5365,6 +5481,7 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_prune_clears_logical_watch_for_reregistration); RUN_TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions); RUN_TEST(daemon_application_programmatic_index_injects_resource_policy); + RUN_TEST(daemon_application_controlled_tool_error_completes_background_observation); RUN_TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried); RUN_TEST(daemon_application_worker_storage_resource_failure_is_structured_and_not_retried); RUN_TEST(daemon_application_auto_index_honors_tracked_file_limit); diff --git a/tests/test_git_context.c b/tests/test_git_context.c index a384651a5..6c58c4459 100644 --- a/tests/test_git_context.c +++ b/tests/test_git_context.c @@ -232,10 +232,43 @@ TEST(canonical_root_linked_worktree) { #endif /* _WIN32 */ } +TEST(git_snapshot_reports_head_and_dirty_state_read_only) { +#ifdef _WIN32 + SKIP_PLATFORM("git snapshot test not supported on Windows CI"); +#else + char *tmp = th_mktempdir("cbm_git_snapshot"); + ASSERT_NOT_NULL(tmp); + if (make_git_repo(tmp) != 0) { + th_rmtree(tmp); + SKIP_PLATFORM("git not available to init a repo"); + } + + cbm_git_snapshot_t clean = {0}; + ASSERT_EQ(cbm_git_snapshot_read(tmp, &clean), 0); + ASSERT_TRUE(clean.available); + ASSERT_NOT_NULL(clean.head_sha); + ASSERT_EQ(strlen(clean.head_sha), 40); + ASSERT_FALSE(clean.dirty); + + ASSERT_EQ(th_write_file(TH_PATH(tmp, "untracked.c"), "int untracked;\n"), 0); + cbm_git_snapshot_t dirty = {0}; + ASSERT_EQ(cbm_git_snapshot_read(tmp, &dirty), 0); + ASSERT_TRUE(dirty.available); + ASSERT_STR_EQ(dirty.head_sha, clean.head_sha); + ASSERT_TRUE(dirty.dirty); + + cbm_git_snapshot_free(&dirty); + cbm_git_snapshot_free(&clean); + th_rmtree(tmp); + PASS(); +#endif +} + /* ── Suite ──────────────────────────────────────────────────────── */ SUITE(git_context) { RUN_TEST(canonical_root_repo_root); RUN_TEST(canonical_root_subdir); RUN_TEST(canonical_root_linked_worktree); + RUN_TEST(git_snapshot_reports_head_and_dirty_state_read_only); } diff --git a/tests/test_index_policy.c b/tests/test_index_policy.c index 71f8240c7..2cc6b5130 100644 --- a/tests/test_index_policy.c +++ b/tests/test_index_policy.c @@ -683,7 +683,8 @@ TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index) { th_write_file(TH_PATH(repo, "second.py"), "def second():\n return 2\n") == 0; (void)snprintf(args, sizeof(args), "{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\"," - "\"mode\":\"fast\",\"_cbm_index_policy\":{" + "\"mode\":\"fast\",\"_cbm_index_origin\":\"watcher\"," + "\"_cbm_index_policy\":{" "\"index_max_files\":\"off\",\"index_max_source_mb\":\"off\"}}", repo); char *limited_response = @@ -697,6 +698,16 @@ TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index) { strstr(limited_response, "\\\"retryable\\\":true") && strstr(limited_response, "\\\"serving_index_preserved\\\":true"); free(limited_response); + char attempt_path[2048]; + (void)snprintf(attempt_path, sizeof(attempt_path), "%s/status/ResourcePolicyFixture.json", + cache); + yyjson_doc *attempt_record = yyjson_read_file(attempt_path, 0, NULL, NULL); + yyjson_val *attempt_root = attempt_record ? yyjson_doc_get_root(attempt_record) : NULL; + yyjson_val *attempt_origin = + attempt_root && yyjson_is_obj(attempt_root) ? yyjson_obj_get(attempt_root, "origin") : NULL; + bool origin_not_forged = attempt_origin && yyjson_is_str(attempt_origin) && + strcmp(yyjson_get_str(attempt_origin), "explicit") == 0; + yyjson_doc_free(attempt_record); cbm_store_t *after_store = cbm_store_open_path_query(db_path); cbm_project_t after_project = {0}; @@ -737,6 +748,7 @@ TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index) { ASSERT_TRUE(first_indexed); ASSERT_TRUE(configured); ASSERT_TRUE(contract_ok); + ASSERT_TRUE(origin_not_forged); ASSERT_TRUE(generation_preserved); ASSERT_TRUE(cross_repo_unaffected); PASS(); diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 9472565dc..fadc037b4 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -655,6 +655,41 @@ TEST(tree_cell_sanitizes_control_and_invalid_utf8) { PASS(); } +TEST(index_attempt_only_project_can_be_deleted) { + char *cache = th_mktempdir("cbm_attempt_only_delete"); + ASSERT_NOT_NULL(cache); + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_copy = saved_cache ? strdup(saved_cache) : NULL; + ASSERT_EQ(cbm_setenv("CBM_CACHE_DIR", cache, 1), 0); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char attempt_id[33]; + ASSERT_TRUE(cbm_mcp_index_attempt_begin("AttemptOnlyFixture", cache, "explicit", &policy, false, + attempt_id)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptOnlyFixture", cache, attempt_id, "failed", + "worker_start_failed", NULL, false)); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(server); + char *response = + cbm_mcp_handle_tool(server, "delete_project", "{\"project\":\"AttemptOnlyFixture\"}"); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "\\\"status\\\":\\\"deleted\\\"")); + free(response); + cbm_mcp_server_free(server); + + yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "AttemptOnlyFixture", cache), + CBM_INDEX_ATTEMPT_NONE); + yyjson_mut_doc_free(document); + restore_cache_dir(saved_copy); + free(saved_copy); + th_cleanup(cache); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ * JSON-RPC PARSING * ══════════════════════════════════════════════════════════════════ */ @@ -2893,6 +2928,276 @@ TEST(tool_index_status_includes_git_metadata) { PASS(); } +TEST(index_attempt_record_rejects_stale_transition_and_recovers_abandoned_job) { + char *cache = th_mktempdir("cbm_attempt_record"); + ASSERT_NOT_NULL(cache); + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_copy = saved_cache ? strdup(saved_cache) : NULL; + ASSERT_EQ(cbm_setenv("CBM_CACHE_DIR", cache, 1), 0); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[128]; + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "strict", error, sizeof(error))); + char attempt_id[33]; + ASSERT_TRUE(cbm_mcp_index_attempt_begin("AttemptFixture", cache, "explicit", &policy, false, + attempt_id)); + ASSERT_EQ(strlen(attempt_id), 32); + ASSERT_FALSE(cbm_mcp_index_attempt_transition( + "AttemptFixture", cache, "00000000000000000000000000000000", "running", NULL, NULL, false)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, attempt_id, "running", + NULL, NULL, false)); + char replacement_attempt_id[33]; + ASSERT_TRUE(cbm_mcp_index_attempt_begin("AttemptFixture", cache, "auto", &policy, false, + replacement_attempt_id)); + ASSERT_FALSE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, attempt_id, "failed", + "index_failed", NULL, false)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, replacement_attempt_id, + "running", NULL, NULL, false)); + + char attempt_path[512]; + (void)snprintf(attempt_path, sizeof(attempt_path), "%s/status/AttemptFixture.json", cache); + yyjson_read_err read_error; + yyjson_doc *stored = yyjson_read_file(attempt_path, 0, NULL, &read_error); + ASSERT_NOT_NULL(stored); + yyjson_mut_doc *abandoned = yyjson_doc_mut_copy(stored, NULL); + yyjson_doc_free(stored); + ASSERT_NOT_NULL(abandoned); + yyjson_mut_val *abandoned_root = yyjson_mut_doc_get_root(abandoned); + ASSERT_TRUE(yyjson_mut_obj_put(abandoned_root, + yyjson_mut_strcpy(abandoned, "owner_start_token"), + yyjson_mut_uint(abandoned, 0))); + char *abandoned_json = yyjson_mut_write(abandoned, 0, NULL); + ASSERT_NOT_NULL(abandoned_json); + ASSERT_EQ(th_write_file(attempt_path, abandoned_json), 0); + free(abandoned_json); + yyjson_mut_doc_free(abandoned); + + cbm_mcp_index_attempt_recover_abandoned(); + yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "AttemptFixture", cache), + CBM_INDEX_ATTEMPT_AVAILABLE); + char *json = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(json); + ASSERT_NOT_NULL(strstr(json, "\"state\":\"failed\"")); + ASSERT_NOT_NULL(strstr(json, "\"failure_code\":\"worker_lost\"")); + ASSERT_NOT_NULL(strstr(json, "\"profile\":\"strict\"")); + ASSERT_NOT_NULL(strstr(json, "\"freshness\":\"unknown\"")); + free(json); + yyjson_mut_doc_free(document); + + char probe_attempt_id[33]; + ASSERT_TRUE(cbm_mcp_index_attempt_begin("AttemptFixture", cache, "auto", &policy, false, + probe_attempt_id)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, probe_attempt_id, + "running", NULL, NULL, false)); + cbm_index_resource_violation_t probe_failure = { + .resource = CBM_INDEX_RESOURCE_RSS_BYTES, + .probe_failed = true, + }; + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, probe_attempt_id, + "failed", "resource_probe_failed", &probe_failure, + false)); + document = yyjson_mut_doc_new(NULL); + root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "AttemptFixture", cache), + CBM_INDEX_ATTEMPT_AVAILABLE); + json = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(json); + ASSERT_NOT_NULL(strstr(json, "\"failure_code\":\"resource_probe_failed\"")); + ASSERT_NOT_NULL(strstr(json, "\"resource\":\"rss_bytes\"")); + ASSERT_NOT_NULL(strstr(json, "\"probe_failed\":true")); + ASSERT_NULL(strstr(json, "\"observed\"")); + free(json); + yyjson_mut_doc_free(document); + + char watcher_attempt_id[33]; + ASSERT_TRUE(cbm_mcp_index_attempt_begin("AttemptFixture", cache, "watcher", &policy, true, + watcher_attempt_id)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, watcher_attempt_id, + "running", NULL, NULL, false)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, watcher_attempt_id, + "failed", "index_failed", NULL, false)); + char retry_attempt_id[33]; + ASSERT_TRUE(cbm_mcp_index_attempt_begin("AttemptFixture", cache, "explicit", &policy, false, + retry_attempt_id)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, retry_attempt_id, + "running", NULL, NULL, false)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, retry_attempt_id, + "failed", "index_failed", NULL, false)); + document = yyjson_mut_doc_new(NULL); + root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "AttemptFixture", cache), + CBM_INDEX_ATTEMPT_AVAILABLE); + json = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(json); + ASSERT_NOT_NULL(strstr(json, "\"origin\":\"explicit\"")); + ASSERT_NOT_NULL(strstr(json, "\"watcher_observed_change\":true")); + ASSERT_NOT_NULL(strstr(json, "\"freshness\":\"stale\"")); + free(json); + yyjson_mut_doc_free(document); + + ASSERT_TRUE(cbm_mcp_index_attempt_remove("AttemptFixture")); + restore_cache_dir(saved_copy); + free(saved_copy); + th_cleanup(cache); + PASS(); +} + +TEST(index_attempt_freshness_requires_same_clean_git_snapshot) { + char *cache_created = th_mktempdir("cbm_attempt_fresh_cache"); + char *cache = cache_created ? strdup(cache_created) : NULL; + char *repo_created = th_mktempdir("cbm_attempt_fresh_repo"); + char *repo = repo_created ? strdup(repo_created) : NULL; + ASSERT_NOT_NULL(cache); + ASSERT_NOT_NULL(repo); + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_copy = saved_cache ? strdup(saved_cache) : NULL; + ASSERT_EQ(cbm_setenv("CBM_CACHE_DIR", cache, 1), 0); + + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int main(void) { return 0; }\n"), 0); + const char *init[] = {"-c", "init.defaultBranch=main", "init", "-q", NULL}; + const char *add[] = {"add", "main.c", NULL}; + const char *commit[] = {"-c", "user.name=Test", + "-c", "user.email=test@example.com", + "-c", "commit.gpgsign=false", + "commit", "-q", + "-m", "init", + NULL}; + if (mcp_test_git(repo, init) != 0 || mcp_test_git(repo, add) != 0 || + mcp_test_git(repo, commit) != 0) { + restore_cache_dir(saved_copy); + free(saved_copy); + th_cleanup(cache); + th_cleanup(repo); + free(cache); + free(repo); + SKIP_PLATFORM("git fixture unavailable"); + } + ASSERT_EQ(cbm_unlink(TH_PATH(repo, ".cbm-empty-gitconfig")), 0); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char db_path[512]; + (void)snprintf(db_path, sizeof(db_path), "%s/FreshFixture.db", cache); + ASSERT_TRUE(mcp_make_valid_project_store_at(db_path, "FreshFixture", repo)); + char attempt_id[33]; + ASSERT_TRUE( + cbm_mcp_index_attempt_begin("FreshFixture", repo, "explicit", &policy, false, attempt_id)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("FreshFixture", repo, attempt_id, "running", NULL, + NULL, false)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("FreshFixture", repo, attempt_id, "completed", + NULL, NULL, true)); + + yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "FreshFixture", repo), + CBM_INDEX_ATTEMPT_AVAILABLE); + char *json = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(json); + ASSERT_NOT_NULL(strstr(json, "\"freshness\":\"fresh\"")); + free(json); + yyjson_mut_doc_free(document); + + char failed_attempt_id[33]; + ASSERT_TRUE(cbm_mcp_index_attempt_begin("FreshFixture", repo, "explicit", &policy, false, + failed_attempt_id)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("FreshFixture", repo, failed_attempt_id, "running", + NULL, NULL, false)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("FreshFixture", repo, failed_attempt_id, "failed", + "index_failed", NULL, false)); + document = yyjson_mut_doc_new(NULL); + root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "FreshFixture", repo), + CBM_INDEX_ATTEMPT_AVAILABLE); + json = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(json); + ASSERT_NOT_NULL(strstr(json, "\"state\":\"failed\"")); + ASSERT_NOT_NULL(strstr(json, "\"freshness\":\"fresh\"")); + free(json); + yyjson_mut_doc_free(document); + + ASSERT_EQ(th_write_file(TH_PATH(repo, "untracked.c"), "int dirty;\n"), 0); + document = yyjson_mut_doc_new(NULL); + root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "FreshFixture", repo), + CBM_INDEX_ATTEMPT_AVAILABLE); + json = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(json); + ASSERT_NOT_NULL(strstr(json, "\"freshness\":\"unknown\"")); + free(json); + yyjson_mut_doc_free(document); + + ASSERT_EQ(cbm_unlink(TH_PATH(repo, "untracked.c")), 0); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int main(void) { return 1; }\n"), 0); + if (mcp_test_git(repo, add) != 0 || mcp_test_git(repo, commit) != 0) { + restore_cache_dir(saved_copy); + free(saved_copy); + th_cleanup(cache); + th_cleanup(repo); + free(cache); + free(repo); + SKIP_PLATFORM("git fixture update unavailable"); + } + ASSERT_EQ(cbm_unlink(TH_PATH(repo, ".cbm-empty-gitconfig")), 0); + document = yyjson_mut_doc_new(NULL); + root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "FreshFixture", repo), + CBM_INDEX_ATTEMPT_AVAILABLE); + json = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(json); + ASSERT_NOT_NULL(strstr(json, "\"freshness\":\"stale\"")); + free(json); + yyjson_mut_doc_free(document); + + char changed_attempt_id[33]; + ASSERT_TRUE(cbm_mcp_index_attempt_begin("FreshFixture", repo, "explicit", &policy, false, + changed_attempt_id)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("FreshFixture", repo, changed_attempt_id, + "running", NULL, NULL, false)); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int main(void) { return 2; }\n"), 0); + if (mcp_test_git(repo, add) != 0 || mcp_test_git(repo, commit) != 0) { + restore_cache_dir(saved_copy); + free(saved_copy); + th_cleanup(cache); + th_cleanup(repo); + free(cache); + free(repo); + SKIP_PLATFORM("git fixture update unavailable"); + } + ASSERT_EQ(cbm_unlink(TH_PATH(repo, ".cbm-empty-gitconfig")), 0); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("FreshFixture", repo, changed_attempt_id, + "completed", NULL, NULL, true)); + document = yyjson_mut_doc_new(NULL); + root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "FreshFixture", repo), + CBM_INDEX_ATTEMPT_AVAILABLE); + json = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(json); + ASSERT_NOT_NULL(strstr(json, "\"generation_snapshot_available\":false")); + ASSERT_NOT_NULL(strstr(json, "\"freshness\":\"unknown\"")); + free(json); + yyjson_mut_doc_free(document); + + ASSERT_TRUE(cbm_mcp_index_attempt_remove("FreshFixture")); + restore_cache_dir(saved_copy); + free(saved_copy); + th_cleanup(cache); + th_cleanup(repo); + free(cache); + free(repo); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ * TOOL HANDLERS WITH DATA * ══════════════════════════════════════════════════════════════════ */ @@ -10021,6 +10326,7 @@ enum { IDX832_NULL_RESP = 52, /* supervised entry degraded to NULL */ IDX832_NOT_INDEXED = 53, /* response/store lacks the indexed Function node */ IDX832_SERVER_FAIL = 54, + IDX832_ATTEMPT_MISSING = 55, }; #ifndef _WIN32 /* helper used only by the POSIX fork harness below */ @@ -10055,6 +10361,24 @@ static int idx832_supervised_route_check(const char *repo_dir) { /* Store-level proof the worker child did real work: the Function node it wrote * must be queryable from a fresh server reading the DB the child produced. */ char *project = cbm_project_name_from_path(repo_dir); + const char *cache = cbm_resolve_cache_dir(); + char attempt_path[1024]; + int attempt_length = project && cache ? snprintf(attempt_path, sizeof(attempt_path), + "%s/status/%s.json", cache, project) + : -1; + yyjson_doc *attempt = attempt_length > 0 && (size_t)attempt_length < sizeof(attempt_path) + ? yyjson_read_file(attempt_path, 0, NULL, NULL) + : NULL; + yyjson_val *attempt_root = attempt ? yyjson_doc_get_root(attempt) : NULL; + yyjson_val *attempt_origin = + attempt_root && yyjson_is_obj(attempt_root) ? yyjson_obj_get(attempt_root, "origin") : NULL; + bool watcher_attempt = attempt_origin && yyjson_is_str(attempt_origin) && + strcmp(yyjson_get_str(attempt_origin), "watcher") == 0; + yyjson_doc_free(attempt); + if (!watcher_attempt) { + free(project); + return IDX832_ATTEMPT_MISSING; + } cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); if (!srv) { free(project); @@ -10748,6 +11072,7 @@ enum { IDX853_NO_SPAWN = 62, /* spawn_count unchanged → supervised path not exercised */ IDX853_SETUP_FAIL = 63, /* config/watcher/server/cwd setup failed */ IDX853_BAD_COUNT = 64, /* unexpected watch_count (<0 or >1) */ + IDX853_ATTEMPT_MISSING = 65, /* supervised auto path omitted its attempt record */ }; #ifndef _WIN32 /* helper used only by the POSIX fork harness below */ @@ -10806,9 +11131,27 @@ static int idx853_supervised_autowatch_check(const char *repo_dir, const char *c int spawns_after = cbm_index_supervisor_spawn_count(); int watch_count = cbm_watcher_watch_count(watcher); + char *project = cbm_project_name_from_path(repo_dir); + char attempt_path[1024]; + int attempt_length = project ? snprintf(attempt_path, sizeof(attempt_path), + "%s/status/%s.json", cache_dir, project) + : -1; + yyjson_doc *attempt = attempt_length > 0 && (size_t)attempt_length < sizeof(attempt_path) + ? yyjson_read_file(attempt_path, 0, NULL, NULL) + : NULL; + yyjson_val *attempt_root = attempt ? yyjson_doc_get_root(attempt) : NULL; + yyjson_val *attempt_origin = attempt_root && yyjson_is_obj(attempt_root) + ? yyjson_obj_get(attempt_root, "origin") + : NULL; + bool attempt_recorded = attempt_origin && yyjson_is_str(attempt_origin) && + strcmp(yyjson_get_str(attempt_origin), "auto") == 0; + yyjson_doc_free(attempt); + free(project); if (spawns_after == spawns_before) { code = IDX853_NO_SPAWN; /* supervised branch never ran — not a valid probe */ + } else if (!attempt_recorded) { + code = IDX853_ATTEMPT_MISSING; } else if (watch_count == 1) { code = IDX853_WATCHER_REGISTERED; /* the discriminating RED assertion */ } else if (watch_count == 0) { @@ -11513,6 +11856,9 @@ SUITE(mcp) { RUN_TEST(tool_check_index_coverage_requires_source_when_file_metadata_changed); RUN_TEST(tool_check_index_coverage_surfaces_lookup_errors); RUN_TEST(tool_index_status_includes_git_metadata); + RUN_TEST(index_attempt_record_rejects_stale_transition_and_recovers_abandoned_job); + RUN_TEST(index_attempt_freshness_requires_same_clean_git_snapshot); + RUN_TEST(index_attempt_only_project_can_be_deleted); /* Tool handlers with validation */ RUN_TEST(tool_trace_call_path_not_found); From b27372cf807ba083e95989fb8e33071c214af777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Wed, 19 Aug 2026 14:39:38 +0800 Subject: [PATCH 6/6] feat(mcp): warn on answers served from a failed rebuild MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A recorded attempt is only visible to whoever thinks to ask for it. An agent that calls search_graph after a background rebuild failed gets a confident answer from a graph that no longer matches the tree, and nothing in that response says so. While the recorded attempt is failed or cancelled, attach a warning to the answers served from that project's graph, naming the origin, the recorded finish time and the limit that ended the attempt. A JSON payload gains a stale_index_warning field so a structured reader cannot miss it; text and tree payloads gain a trailing content block. The answer itself keeps its position and its content. index_status and check_index_coverage already report freshness themselves and index_repository is the remedy, so none of them is annotated. A queued or running rebuild is not yet a failure and stays silent, a completed rebuild clears the warning, and every failure inside the warning path is silent: a warning that cannot be produced must never damage an answer that is fine. Signed-off-by: 刘冲 --- README.md | 4 +- docs/CONFIGURATION.md | 5 + docs/INDEX_RESOURCE_LIMITS.md | 14 +++ src/mcp/mcp.c | 173 ++++++++++++++++++++++++++++++++++ tests/test_mcp.c | 111 ++++++++++++++++++++++ 5 files changed, 306 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 72ac3814c..ca7798a1e 100644 --- a/README.md +++ b/README.md @@ -683,7 +683,9 @@ explicit value, including `off`, replaces that dimension of a selected profile. Exceeding an effective limit fails the complete index attempt rather than publishing a partial graph, and an existing serving index is preserved. `index_status` reports the latest physical indexing attempt and Git-based -freshness after the first recorded attempt. See +freshness after the first recorded attempt, and while that attempt is failed or +cancelled the graph-answering tools attach a stale-index warning so a rebuild +that failed in the background cannot go unnoticed. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). ### Environment Variables diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 08e7ce387..e1360e766 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -118,6 +118,11 @@ changed-during-index, missing, or unreadable snapshots are `unknown`. `delete_project` also removes a matching attempt record when no database was published. +While the recorded attempt is `failed` or `cancelled`, the graph-answering +tools also attach a stale-index warning to their results, so a rebuild that +failed in the background cannot leave a caller trusting an older graph in +silence. A completed rebuild clears it. + ## 3. UI Settings The optional built-in graph UI stores its settings in: diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index 20438c493..6bef7818f 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -189,6 +189,20 @@ dirty or changed-during-index snapshots, failed Git probes, and corrupt records are `unknown`. Dirty state is never proof of freshness or staleness. Projects with no attempt record retain the previous response shape. +A background rebuild that fails reaches only the logs and `index_status`, which +leaves a caller querying an older graph with nothing to warn it. Once a record +is in a `failed` or `cancelled` state, the next answer served from that +project's graph carries a stale-index warning naming the origin, the recorded +finish time, and the limit that ended the attempt. The warning is attached to +`search_graph`, `query_graph`, `trace_path`, `trace_call_path`, +`get_architecture`, `get_code_snippet`, and `search_code`; `index_status` and +`check_index_coverage` already report freshness themselves, and +`index_repository` is the remedy. A JSON payload receives a +`stale_index_warning` field so a structured reader cannot miss it, and a text +or tree payload receives a trailing content block; the answer itself keeps its +position and content. A queued or running rebuild is not yet a failure and +produces no warning, and a completed rebuild clears it. + ## Trust and compatibility Limits are read from the CLI-managed `_config.db`; they are not MCP request diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index dc38a9c40..56442683c 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -5228,6 +5228,81 @@ cbm_index_attempt_read_status_t cbm_mcp_index_attempt_add_status(yyjson_mut_doc return CBM_INDEX_ATTEMPT_AVAILABLE; } +/* Describe why the recorded attempt left the caller on an older graph. The + * violation detail is optional: an attempt can fail before any resource is + * attributed, and the warning still has to say something useful. */ +static void index_attempt_stale_cause(yyjson_val *root, char *out, size_t out_size) { + out[0] = '\0'; + yyjson_val *failure = yyjson_obj_get(root, "resource_failure"); + yyjson_val *resource = + failure && yyjson_is_obj(failure) ? yyjson_obj_get(failure, "resource") : NULL; + const char *name = resource && yyjson_is_str(resource) ? yyjson_get_str(resource) : NULL; + if (!name) { + yyjson_val *code = yyjson_obj_get(root, "failure_code"); + if (code && yyjson_is_str(code) && yyjson_get_len(code) > 0) { + snprintf(out, out_size, " (%s)", yyjson_get_str(code)); + } + return; + } + const char *key = cbm_index_resource_config_key(index_attempt_resource_from_name(name)); + yyjson_val *probe_failed = yyjson_obj_get(failure, "probe_failed"); + yyjson_val *observed = yyjson_obj_get(failure, "observed"); + yyjson_val *limit = yyjson_obj_get(failure, "limit"); + yyjson_val *unit = yyjson_obj_get(failure, "unit"); + if (probe_failed && yyjson_is_bool(probe_failed) && yyjson_get_bool(probe_failed)) { + snprintf(out, out_size, + " because the %s measurement could not be taken and the %s limit is enforced " + "fail-closed", + name, key); + return; + } + if (observed && yyjson_is_uint(observed) && limit && yyjson_is_uint(limit)) { + snprintf(out, out_size, " because %s reached %llu %s against the %s limit of %llu", name, + (unsigned long long)yyjson_get_uint(observed), + unit && yyjson_is_str(unit) ? yyjson_get_str(unit) : "units", key, + (unsigned long long)yyjson_get_uint(limit)); + return; + } + snprintf(out, out_size, " on the %s limit", key); +} + +/* Fires only on a recorded terminal failure, so it costs one small record read + * per answered query and never probes git or the filesystem. A queued or + * running rebuild stays silent here and is reported by index_status. */ +static bool index_attempt_stale_warning(const char *project, char *out, size_t out_size) { + if (!project || !project[0] || !out || out_size == 0) { + return false; + } + yyjson_doc *attempt = index_attempt_read(project, NULL); + yyjson_val *root = attempt ? yyjson_doc_get_root(attempt) : NULL; + if (!root || !yyjson_is_obj(root)) { + yyjson_doc_free(attempt); + return false; + } + yyjson_val *state = yyjson_obj_get(root, "state"); + const char *state_name = state && yyjson_is_str(state) ? yyjson_get_str(state) : ""; + bool cancelled = strcmp(state_name, "cancelled") == 0; + if (strcmp(state_name, "failed") != 0 && !cancelled) { + yyjson_doc_free(attempt); + return false; + } + yyjson_val *origin = yyjson_obj_get(root, "origin"); + yyjson_val *finished = yyjson_obj_get(root, "finished_at"); + char cause[CBM_SZ_256]; + index_attempt_stale_cause(root, cause, sizeof(cause)); + snprintf(out, out_size, + "Stale index warning: the last %s index attempt for project '%s' %s%s%s%s. These " + "results come from the previously published graph and may not match the current " + "working tree. Call index_status for the recorded attempt, then index_repository to " + "rebuild once the cause is addressed.", + origin && yyjson_is_str(origin) ? yyjson_get_str(origin) : "recorded", project, + cancelled ? "was cancelled" : "failed", + finished && yyjson_is_str(finished) ? " at " : "", + finished && yyjson_is_str(finished) ? yyjson_get_str(finished) : "", cause); + yyjson_doc_free(attempt); + return out[0] != '\0'; +} + bool cbm_mcp_index_attempt_remove(const char *project) { index_attempt_lock_t lock; if (!index_attempt_lock_acquire(&lock, project)) { @@ -12577,6 +12652,103 @@ static void release_request_store(cbm_mcp_server_t *srv) { cbm_mem_collect(); } +/* Attach one stale-graph warning to a finished tool result. A JSON payload + * gains a `stale_index_warning` field so the signal survives structured + * parsing; text and tree payloads gain a trailing content block instead. The + * first content item is never displaced, because hook_augment and the smoke + * tests read it as the payload. */ +static void mcp_result_attach_stale_warning(char **result_io, const char *warning) { + yyjson_doc *parsed = yyjson_read(*result_io, strlen(*result_io), 0); + yyjson_mut_doc *document = parsed ? yyjson_mut_doc_new(NULL) : NULL; + yyjson_mut_val *root = + document ? yyjson_val_mut_copy(document, yyjson_doc_get_root(parsed)) : NULL; + yyjson_doc_free(parsed); + yyjson_mut_val *failed = + root && yyjson_mut_is_obj(root) ? yyjson_mut_obj_get(root, "isError") : NULL; + yyjson_mut_val *content = + root && yyjson_mut_is_obj(root) ? yyjson_mut_obj_get(root, "content") : NULL; + if (!content || !yyjson_mut_is_arr(content) || + (failed && yyjson_mut_is_bool(failed) && yyjson_mut_get_bool(failed))) { + yyjson_mut_doc_free(document); + return; + } + yyjson_mut_doc_set_root(document, root); + yyjson_mut_val *item = yyjson_mut_arr_get(content, 0); + yyjson_mut_val *text = + item && yyjson_mut_is_obj(item) ? yyjson_mut_obj_get(item, "text") : NULL; + const char *payload = text && yyjson_mut_is_str(text) ? yyjson_mut_get_str(text) : NULL; + yyjson_doc *payload_document = payload ? yyjson_read(payload, strlen(payload), 0) : NULL; + yyjson_val *payload_root = payload_document ? yyjson_doc_get_root(payload_document) : NULL; + bool attached = false; + if (payload_root && yyjson_is_obj(payload_root)) { + yyjson_mut_doc *rewritten = yyjson_doc_mut_copy(payload_document, NULL); + yyjson_mut_val *rewritten_root = rewritten ? yyjson_mut_doc_get_root(rewritten) : NULL; + char *rendered = rewritten_root && yyjson_mut_obj_add_strcpy(rewritten, rewritten_root, + "stale_index_warning", warning) + ? yyjson_mut_write(rewritten, YYJSON_WRITE_ALLOW_INVALID_UNICODE, NULL) + : NULL; + yyjson_mut_doc_free(rewritten); + if (rendered) { + (void)yyjson_mut_obj_remove_key(item, "text"); + attached = yyjson_mut_obj_add_strcpy(document, item, "text", rendered); + yyjson_mut_val *structured = yyjson_mut_obj_get(root, "structuredContent"); + if (attached && structured && yyjson_mut_is_obj(structured)) { + attached = + yyjson_mut_obj_add_strcpy(document, structured, "stale_index_warning", warning); + } + free(rendered); + } + } else { + yyjson_mut_val *extra = yyjson_mut_obj(document); + attached = extra && yyjson_mut_obj_add_str(document, extra, "type", "text") && + yyjson_mut_obj_add_strcpy(document, extra, "text", warning) && + yyjson_mut_arr_append(content, extra); + } + yyjson_doc_free(payload_document); + char *replacement = + attached ? yyjson_mut_write(document, YYJSON_WRITE_ALLOW_INVALID_UNICODE, NULL) : NULL; + yyjson_mut_doc_free(document); + if (replacement) { + free(*result_io); + *result_io = replacement; + } +} + +/* Tools that answer from the published graph. A stale graph makes their + * answers quietly wrong, which is exactly the case the warning exists for. + * index_status and check_index_coverage already report freshness themselves, + * and index_repository is the remedy, so none of them are annotated. */ +static bool mcp_tool_answers_from_graph(const char *tool_name) { + static const char *GRAPH_TOOLS[] = { + "search_graph", "query_graph", "trace_path", "trace_call_path", + "get_architecture", "get_code_snippet", "search_code", + }; + for (size_t index = 0; tool_name && index < sizeof(GRAPH_TOOLS) / sizeof(GRAPH_TOOLS[0]); + index++) { + if (strcmp(tool_name, GRAPH_TOOLS[index]) == 0) { + return true; + } + } + return false; +} + +/* A background rebuild that fails only reaches the logs and index_status, + * which leaves a caller querying an older graph with no visible signal. Say so + * on the next answer served from that graph. Every failure here is silent: + * a warning that cannot be produced must never turn a good answer into one. */ +static void mcp_result_warn_if_stale(const char *tool_name, const char *args_json, + char **result_io) { + if (!args_json || !result_io || !*result_io || !mcp_tool_answers_from_graph(tool_name)) { + return; + } + char *project = get_project_arg(args_json); + char warning[CBM_SZ_1K]; + if (project && index_attempt_stale_warning(project, warning, sizeof(warning))) { + mcp_result_attach_stale_warning(result_io, warning); + } + free(project); +} + char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const char *args_json) { /* Phase marks bracket the WHOLE request with no unlabelled gap, so growth * cannot hide between them (CBM_MEM_PHASES=1; see foundation/mem.h). The @@ -12591,6 +12763,7 @@ char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const ch } cbm_mem_phase_mark("request.dispatch_tool"); char *result = dispatch_tool(srv, tool_name, args_json); + mcp_result_warn_if_stale(tool_name, args_json, &result); cbm_mem_phase_mark("request.scope_end"); if (srv) { cbm_mcp_server_request_scope_end(srv); diff --git a/tests/test_mcp.c b/tests/test_mcp.c index fadc037b4..d370a6077 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -3048,6 +3048,116 @@ TEST(index_attempt_record_rejects_stale_transition_and_recovers_abandoned_job) { PASS(); } +/* A rebuild that dies on a resource limit leaves the previously published + * graph in service. Without a signal on the answers themselves, the caller + * keeps trusting a graph that no longer matches the tree. */ +TEST(stale_index_warning_reaches_graph_answers) { + char *cache = th_mktempdir("cbm_stale_warning"); + ASSERT_NOT_NULL(cache); + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_copy = saved_cache ? strdup(saved_cache) : NULL; + ASSERT_EQ(cbm_setenv("CBM_CACHE_DIR", cache, 1), 0); + + const char *project = "StaleWarningFixture"; + char db_path[700]; + snprintf(db_path, sizeof(db_path), "%s/%s.db", cache, project); + cbm_store_t *setup = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(setup); + ASSERT_EQ(cbm_store_upsert_project(setup, project, cache), CBM_STORE_OK); + cbm_node_t node = {.project = project, + .label = "Function", + .name = "StaleProbe", + .qualified_name = "stale.StaleProbe", + .file_path = "mod.c", + .start_line = 1, + .end_line = 2}; + ASSERT_GT(cbm_store_upsert_node(setup, &node), 0); + cbm_store_close(setup); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[128]; + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "strict", error, sizeof(error))); + char attempt_id[33]; + ASSERT_TRUE(cbm_mcp_index_attempt_begin(project, cache, "watcher", &policy, true, attempt_id)); + ASSERT_TRUE( + cbm_mcp_index_attempt_transition(project, cache, attempt_id, "running", NULL, NULL, false)); + cbm_index_resource_violation_t violation = { + .resource = CBM_INDEX_RESOURCE_FILES, + .observed = 120000, + .limit = 100000, + }; + ASSERT_TRUE(cbm_mcp_index_attempt_transition(project, cache, attempt_id, "failed", + "resource_limit_exceeded", &violation, false)); + + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + char json_args[512]; + snprintf(json_args, sizeof(json_args), + "{\"project\":\"%s\",\"name_pattern\":\".*StaleProbe.*\",\"format\":\"json\"}", + project); + char *resp = cbm_mcp_handle_tool(srv, "search_graph", json_args); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "\"isError\":false")); + char *inner = extract_text_content(resp); + ASSERT_NOT_NULL(inner); + /* The answer keeps its place at content[0]; the warning rides along in the + * same object so a structured reader cannot miss it. */ + ASSERT_NOT_NULL(strstr(inner, "StaleProbe")); + ASSERT_NOT_NULL(strstr(inner, "\"stale_index_warning\"")); + ASSERT_NOT_NULL(strstr(inner, "index_max_files")); + ASSERT_NOT_NULL(strstr(inner, "120000")); + ASSERT_NOT_NULL(strstr(inner, "watcher")); + free(inner); + free(resp); + + char default_args[512]; + snprintf(default_args, sizeof(default_args), + "{\"project\":\"%s\",\"name_pattern\":\".*StaleProbe.*\"}", project); + resp = cbm_mcp_handle_tool(srv, "search_graph", default_args); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "Stale index warning")); + inner = extract_text_content(resp); + ASSERT_NOT_NULL(inner); + ASSERT_NOT_NULL(strstr(inner, "StaleProbe")); + free(inner); + free(resp); + + /* index_status reports the attempt in full and needs no second copy. */ + char status_args[512]; + snprintf(status_args, sizeof(status_args), "{\"project\":\"%s\"}", project); + resp = cbm_mcp_handle_tool(srv, "index_status", status_args); + ASSERT_NOT_NULL(resp); + ASSERT_NULL(strstr(resp, "stale_index_warning")); + free(resp); + + /* A rebuild that is merely in flight is not yet a failure. */ + char inflight_id[33]; + ASSERT_TRUE( + cbm_mcp_index_attempt_begin(project, cache, "explicit", &policy, false, inflight_id)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition(project, cache, inflight_id, "running", NULL, NULL, + false)); + resp = cbm_mcp_handle_tool(srv, "search_graph", json_args); + ASSERT_NOT_NULL(resp); + ASSERT_NULL(strstr(resp, "stale_index_warning")); + free(resp); + + ASSERT_TRUE(cbm_mcp_index_attempt_transition(project, cache, inflight_id, "completed", NULL, + NULL, true)); + resp = cbm_mcp_handle_tool(srv, "search_graph", json_args); + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "StaleProbe")); + ASSERT_NULL(strstr(resp, "stale_index_warning")); + free(resp); + + cbm_mcp_server_free(srv); + ASSERT_TRUE(cbm_mcp_index_attempt_remove(project)); + restore_cache_dir(saved_copy); + free(saved_copy); + th_cleanup(cache); + PASS(); +} + TEST(index_attempt_freshness_requires_same_clean_git_snapshot) { char *cache_created = th_mktempdir("cbm_attempt_fresh_cache"); char *cache = cache_created ? strdup(cache_created) : NULL; @@ -11857,6 +11967,7 @@ SUITE(mcp) { RUN_TEST(tool_check_index_coverage_surfaces_lookup_errors); RUN_TEST(tool_index_status_includes_git_metadata); RUN_TEST(index_attempt_record_rejects_stale_transition_and_recovers_abandoned_job); + RUN_TEST(stale_index_warning_reaches_graph_answers); RUN_TEST(index_attempt_freshness_requires_same_clean_git_snapshot); RUN_TEST(index_attempt_only_project_can_be_deleted);