diff --git a/Makefile.cbm b/Makefile.cbm index bf51a5a8c..7368467c5 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -224,6 +224,7 @@ FOUNDATION_SRCS = \ src/foundation/profile.c \ src/foundation/dump_verify.c \ src/foundation/limits.c \ + src/foundation/index_policy.c \ src/foundation/subprocess.c \ src/foundation/sha256.c \ src/foundation/secure_random.c \ @@ -502,6 +503,7 @@ TEST_FOUNDATION_SRCS = \ tests/test_str_intern.c \ tests/test_log.c \ tests/test_str_util.c \ + tests/test_index_policy.c \ tests/test_workspace.c \ tests/test_platform.c \ tests/test_diagnostics.c \ diff --git a/README.md b/README.md index ca39ef418..2a2ab98c4 100644 --- a/README.md +++ b/README.md @@ -667,9 +667,25 @@ codebase-memory-mcp config list # show all settings codebase-memory-mcp config set auto_index true # auto-index on session start codebase-memory-mcp config set auto_index_limit 50000 # max files for auto-index codebase-memory-mcp config set auto_watch false # don't register background git watcher (default: true) +codebase-memory-mcp config set index_resource_profile balanced # opt-in composed resource limits +codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit +codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit +codebase-memory-mcp config set index_max_rss_mb 8192 # optional worker-tree current RSS limit +codebase-memory-mcp config set index_max_duration_seconds 3600 # optional total worker duration +codebase-memory-mcp config set index_cache_max_mb 32768 # optional projected cache-size limit +codebase-memory-mcp config set index_min_free_disk_mb 4096 # optional free-space reserve codebase-memory-mcp config reset auto_index # reset to default ``` +`index_resource_profile` defaults to `off` and accepts `off`, `balanced`, or +`strict`. The six individual resource settings also default to `off`; an +explicit value, including `off`, replaces that dimension of a selected +profile. Exceeding an effective limit fails the complete index attempt rather +than publishing a partial graph, and an existing serving index is preserved. +Storage probes also fail closed when an enabled measurement cannot be +completed. See +[Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). + ### Environment Variables | Variable | Default | Description | diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 0ee8058d1..aa5bb0474 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -86,6 +86,28 @@ Current keys: |---|---|---| | `auto_index` | `false` | Automatically index new projects when an MCP session starts. | | `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. | +| `index_resource_profile` | `off` | Composed index resource baseline: `off`, `balanced`, or `strict`. | +| `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | +| `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | +| `index_max_rss_mb` | `off` | Optional maximum current RSS in MiB for the complete contained index-worker process tree (`64..1048576`). | +| `index_max_duration_seconds` | `off` | Optional maximum total worker duration in seconds (`1..86400`). | +| `index_cache_max_mb` | `off` | Optional maximum projected cache size in MiB after publication. | +| `index_min_free_disk_mb` | `off` | Optional minimum free MiB reserved on the cache filesystem while indexing. | + +The profile and all six individual resource settings are disabled by default. +An individual stored value replaces its profile dimension; storing `off` +disables only that dimension. Profile-only limits also bound traversed +directories, directory entries, depth, discovery time, final database size, +staging size, and task temporary size. Effective limits apply to explicit +indexing, automatic indexing, and watcher re-indexing, +but not to `cross-repo-intelligence`, which does not scan or publish repository +source indexes. Equality is allowed; exceeding a setting fails the complete +index request and preserves any previously serving database. Worker RSS covers +descendants and is not the same as the internal `CBM_MEM_BUDGET_MB` allocation +budget. Total duration is independent of the existing 15-minute no-log-progress +timeout. See +[Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and +error-response details. ## 3. UI Settings diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md new file mode 100644 index 000000000..be43cff56 --- /dev/null +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -0,0 +1,184 @@ +# Index resource limits + +Index resource limits are optional operator controls for repositories whose +discovery breadth or worker runtime is not known in advance. They are disabled +by default so existing large-repository workloads retain their current behavior. + +## Resource profiles + +`index_resource_profile` accepts `off`, `balanced`, or `strict` and defaults to +`off`. A profile supplies a baseline; each explicitly stored individual +resource key then replaces its dimension. An explicit `off` disables only that +dimension. + +| Dimension | `balanced` | `strict` | +|---|---:|---:| +| Accepted files | 500,000 | 100,000 | +| Traversed directories | 250,000 | 20,000 | +| Directory entries | 2,000,000 | 500,000 | +| Directory depth | 128 | 64 | +| Accepted source | 65,536 MiB | 4,096 MiB | +| Discovery duration | 1,800 seconds | 30 seconds | +| Worker duration | 7,200 seconds | 3,600 seconds | +| Final database | 65,536 MiB | 16,384 MiB | +| Staging artifacts | 81,920 MiB | 20,480 MiB | +| Task temporary artifacts | 98,304 MiB | 24,576 MiB | +| Projected cache | 131,072 MiB | 32,768 MiB | +| Free-disk reserve | 4,096 MiB | 4,096 MiB | + +The two profiles answer different questions about worker memory, so their RSS +ceilings are derived rather than tabled. + +Balanced protects the host. Its ceiling is 80% of detected host memory, which +leaves an index free to use everything this machine can spare and still stops a +runaway worker before the host is exhausted. A fixed number is deliberately not +used here: large-repository indexing peaks in the tens of gigabytes, so any +round ceiling low enough to feel safe would reject repositories that index +successfully with the profile off. When host memory cannot be read, balanced +falls back to the higher of twice the worker soft budget or 8,192 MiB. + +Strict protects the budget. Its ceiling is the lower of the worker soft budget +and 8,192 MiB, which is the point of the profile: a repository that needs more +memory than the daemon budgets for itself fails fast with an attributed +`rss_bytes` violation instead of completing at the host's expense. On a large +repository this is expected to fail, and the failure names the limit and the +observed value so the operator can choose `balanced`, raise +`index_max_rss_mb`, or leave the profile `off`. + +Both derived values are held inside the same 64 to 1,048,576 MiB range that +`index_max_rss_mb` accepts, and detected host memory can lower but never raise +an explicitly configured `index_max_rss_mb`. + +## Discovery settings + +| Key | Default | Accepted value | Protects | +|---|---:|---:|---| +| `index_max_files` | `off` | `off` or `1..10000000` | Accepted source-file count | +| `index_max_source_mb` | `off` | `off` or `1..1048576` | Accepted source-file bytes | + +Set or reset them with the normal configuration command: + +```bash +codebase-memory-mcp config set index_max_files 250000 +codebase-memory-mcp config set index_max_source_mb 16384 +codebase-memory-mcp config reset index_max_files +``` + +Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero, +negative values, suffixes, trailing characters, and values outside the stated +ranges are rejected without changing the stored value. + +## Worker settings + +| Key | Default | Accepted value | Protects | +|---|---:|---:|---| +| `index_max_rss_mb` | `off` | `off` or `64..1048576` | Current RSS of the complete worker process tree | +| `index_max_duration_seconds` | `off` | `off` or `1..86400` | Total worker wall-clock duration | + +```bash +codebase-memory-mcp config set index_max_rss_mb 8192 +codebase-memory-mcp config set index_max_duration_seconds 3600 +``` + +RSS is the current resident memory of the contained worker and every descendant, +not the worker's allocation budget and not peak memory. This hard watchdog is +separate from the internal `CBM_MEM_BUDGET_MB` soft budget. The supervisor +samples RSS at most once every 250 milliseconds so the watchdog does not turn +full process-table enumeration into a busy loop. + +Duration uses a monotonic clock from successful spawn. It is independent of the +existing 15-minute quiet timeout: continuous log progress does not reset total +duration, while the quiet timeout continues to identify a worker that stops +making progress. + +Equality is allowed. The first RSS or elapsed-duration observation above its +limit starts the existing graceful-to-force process-tree shutdown. CBM reports +terminal only after the tree is quiescent or a bounded containment failure is +explicitly surfaced. Resource termination is not retried and does not +quarantine a source file. + +If RSS is enabled and three consecutive probes cannot obtain any trustworthy +tree measurement while the root worker is still running, CBM fails closed with +`code=resource_probe_failed`. + +## Counting and failure semantics + +`index_max_files` counts a file only after it passes directory pruning, ignore +rules, filename and suffix filters, language detection, and the existing +per-file size rule. `index_max_source_mb` sums the filesystem sizes of that same +accepted set. + +Profile directory counting includes the request root and each non-skipped +directory admitted for traversal. Entry counting happens before ignore, +language, and file-size filtering. Root depth is zero. The monotonic discovery +deadline is checked before opening a directory and before processing each +entry. Equality is allowed for every dimension. + +Equality is allowed. The first file or byte that makes an observed value greater +than its limit stops discovery. CBM discards the partial file list and does not +publish a partial graph as a complete index. + +For an explicit MCP request the error payload contains: + +```json +{ + "status": "error", + "code": "resource_limit_exceeded", + "stage": "discovery", + "resource": "files", + "observed": 250001, + "limit": 250000, + "unit": "files", + "retryable": true, + "serving_index_preserved": true, + "message": "Index discovery exceeded index_max_files" +} +``` + +The previous database remains available because publication occurs only after a +complete discovery and successful staged build. If no previous database exists, +`serving_index_preserved` is false. + +Worker limit failures use the same shape with `stage=worker`, +`resource=rss_bytes` and `unit=bytes`, or `resource=duration_ms` and +`unit=milliseconds`. RSS measurement failures use `code=resource_probe_failed` +and omit `observed`, `limit`, and `unit` because no trustworthy observation was +available. + +## Storage settings + +| Key | Default | Accepted value | Protects | +|---|---:|---:|---| +| `index_cache_max_mb` | `off` | `off` or `1..1048576` | Projected cache bytes after replacement | +| `index_min_free_disk_mb` | `off` | `off` or `1..1048576` | Free bytes reserved on the cache filesystem | + +Set or reset these keys through the same `config set` and `config reset` +commands. With both keys `off`, indexing does not scan the cache tree or probe +filesystem capacity. + +Projected cache usage is the current cache size, minus the old project +database and SQLite sidecars only when that generation is confirmed valid and +replaceable, plus the current operation's staging artifacts. Other projects +and unrelated files always count toward the limit and are never evicted. + +Free space is checked before staging, after the staged build completes, and +immediately before atomic publication. Equality is allowed. An enabled +measurement that cannot be completed fails closed with +`code: "resource_probe_failed"` and +`stage: "storage"`. A limit breach uses `code: "resource_limit_exceeded"`. +Both cases preserve the old serving database. Staging files created by a +terminated supervised worker are tagged with a private task token and removed +after its process tree is quiescent; cleanup cannot match another attempt. + +## Trust and compatibility + +Limits are read from the CLI-managed `_config.db`; they are not MCP request +arguments. A supervised parent replaces any caller-supplied internal policy +before spawning its worker, and the worker rejects a missing or incomplete +parent policy. + +These settings do not replace or increase `auto_index_limit`, change the 512 MiB +single-file cap, alter workspace-root authorization, or affect +`cross-repo-intelligence`. With all settings `off`, discovery follows the +existing path, the supervisor performs no periodic RSS probe or total-duration +termination, and the pipeline performs no cache-tree or free-space probe. diff --git a/scripts/test-runtime.sh b/scripts/test-runtime.sh index cb69cadfb..8978680a6 100644 --- a/scripts/test-runtime.sh +++ b/scripts/test-runtime.sh @@ -105,6 +105,21 @@ _cbm_test_runtime_daemon() { CBM_CACHE_DIR="$_CBM_TEST_RUNTIME_PRODUCT_CACHE" "$1" daemon "$2" } +# The supervisor resolves one resource policy per index and hands it to the +# worker in argv; a worker that finds no complete policy refuses to start rather +# than index unbounded. A shell test that spawns `cli --index-worker` itself +# stands in for the supervisor and owes the worker the same object. Mirrors +# cbm_mcp_index_policy_add_to_args: the resolved profile, the override mask +# that says which keys the operator set by hand, every key in +# cbm_index_policy_key_at, and nothing else. +cbm_test_index_worker_policy_json() { + printf '%s' '"_cbm_index_policy":{"index_resource_profile":"off"' + printf '%s' ',"_cbm_index_override_mask":0' + printf '%s' ',"index_max_files":"off","index_max_source_mb":"off"' + printf '%s' ',"index_max_rss_mb":"off","index_max_duration_seconds":"off"' + printf '%s' ',"index_cache_max_mb":"off","index_min_free_disk_mb":"off"}' +} + cbm_test_runtime_cleanup() { local binary="${1:-}" root="$_CBM_TEST_RUNTIME_CREATED_ROOT" local name="${_CBM_TEST_RUNTIME_CREATED_ROOT##*/}" runtime_entry="" active=0 diff --git a/src/cli/cli.c b/src/cli/cli.c index d981af266..be1677470 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -21,6 +21,7 @@ #include "foundation/platform.h" #include "foundation/constants.h" #include "foundation/log.h" +#include "foundation/mem.h" #include "foundation/sha256.h" #include "cli/client_adapter.h" #include "mcp/mcp.h" // cbm_mcp_tool_input_schema — CLI flag parser + per-tool --help @@ -6734,6 +6735,31 @@ int cbm_config_delete(cbm_config_t *cfg, const char *key) { return rc; } +bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy, + char *error, size_t error_size) { + if (!cfg || !policy) { + if (error && error_size > 0) { + (void)snprintf(error, error_size, "index resource configuration is unavailable"); + } + return false; + } + cbm_index_policy_init(policy); + const char *profile = cbm_config_get(cfg, CBM_INDEX_CONFIG_RESOURCE_PROFILE, "off"); + if (!cbm_index_policy_set_profile(policy, profile, error, error_size)) { + return false; + } + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + const char *key = cbm_index_policy_key_at(index); + const char *value = cbm_config_get(cfg, key, NULL); + if (value && !cbm_index_policy_set(policy, key, value, error, error_size)) { + return false; + } + } + cbm_system_info_t system = cbm_system_info(); + cbm_index_policy_finalize(policy, (uint64_t)system.total_ram, (uint64_t)cbm_mem_budget()); + return true; +} + /* ── Config CLI subcommand ────────────────────────────────────── */ /* THE config-key table. list, get, help, and key validation all read this one @@ -6756,6 +6782,14 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_CONFIG_UI_LANG, "auto", "Pin graph UI language: en, zh, or auto"}, {CBM_CONFIG_UI_ENABLED, "false", "Serve the graph UI on a loopback HTTP port"}, {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"}, + {CBM_INDEX_CONFIG_RESOURCE_PROFILE, "off", "Index resource profile: off, balanced, or strict"}, + {CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"}, + {CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"}, + {CBM_INDEX_CONFIG_MAX_RSS_MB, "off", "Max worker process-tree RSS MiB, or off"}, + {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "off", "Max worker duration in seconds, or off"}, + {CBM_INDEX_CONFIG_CACHE_MAX_MB, "off", "Max projected cache MiB after publish, or off"}, + {CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, "off", + "Minimum free cache-filesystem MiB during indexing, or off"}, }; /* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing @@ -6781,6 +6815,24 @@ static bool config_key_is_ui(const char *key) { return key && (strcmp(key, CBM_CONFIG_UI_ENABLED) == 0 || strcmp(key, CBM_CONFIG_UI_PORT) == 0); } +static bool config_key_is_index_policy(const char *key) { + return cbm_index_policy_is_config_key(key); +} + +static int config_index_policy_write(cbm_config_t *config, const char *key, const char *value) { + cbm_index_resource_policy_t candidate; + cbm_index_policy_init(&candidate); + char error[CLI_BUF_256]; + bool valid = strcmp(key, CBM_INDEX_CONFIG_RESOURCE_PROFILE) == 0 + ? cbm_index_policy_set_profile(&candidate, value, error, sizeof(error)) + : cbm_index_policy_set(&candidate, key, value, error, sizeof(error)); + if (!valid) { + (void)fprintf(stderr, "error: %s\n", error); + return CLI_ERR; + } + return cbm_config_set(config, key, value); +} + static void config_ui_read(const char *key, char *out, size_t out_sz) { cbm_ui_config_t ui; cbm_ui_config_load(&ui); @@ -6918,10 +6970,16 @@ int cbm_cmd_config(int argc, char **argv) { rc = CLI_TRUE; } } else { - if (cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]) == 0) { + int set_rc = + config_key_is_index_policy(argv[CLI_SKIP_ONE]) + ? config_index_policy_write(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]) + : cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]); + if (set_rc == 0) { printf("%s = %s\n", argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]); } else { - (void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]); + if (!config_key_is_index_policy(argv[CLI_SKIP_ONE])) { + (void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]); + } rc = CLI_TRUE; } } diff --git a/src/cli/cli.h b/src/cli/cli.h index c2121a940..56769c42a 100644 --- a/src/cli/cli.h +++ b/src/cli/cli.h @@ -13,6 +13,8 @@ #include #include +#include "foundation/index_policy.h" + typedef struct cbm_mcp_server cbm_mcp_server_t; /* ── Version ──────────────────────────────────────────────────── */ @@ -417,6 +419,11 @@ int cbm_config_set(cbm_config_t *cfg, const char *key, const char *value); /* Delete a config key. Returns 0 on success. */ int cbm_config_delete(cbm_config_t *cfg, const char *key); +/* Load and validate the operator-controlled discovery policy. Invalid stored + * values fail closed instead of silently disabling a guard. */ +bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy, + char *error, size_t error_size); + /* Well-known config keys */ #define CBM_CONFIG_AUTO_INDEX "auto_index" #define CBM_CONFIG_AUTO_INDEX_LIMIT "auto_index_limit" diff --git a/src/daemon/application.c b/src/daemon/application.c index 7e4a35be7..cf7a43c3a 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -207,7 +207,8 @@ static bool application_unique_recovery_file(char out[APPLICATION_PATH_CAP], con static bool application_update_reap(cbm_daemon_application_t *application, bool wait, uint32_t timeout_ms); static void *application_job_thread(void *opaque); -static char *application_auto_index_args(const char *root_path); +static char *application_auto_index_args(cbm_daemon_application_t *application, + const char *root_path); static cbm_daemon_application_job_t *application_job_subscribe_locked( cbm_daemon_application_t *application, const char *project_key, const char *root_path, const char *args_json, application_job_subscribe_status_t *status_out); @@ -303,9 +304,21 @@ static int application_worker_start_default(void *context, const char *args_json const char *quarantine_file, cbm_daemon_application_worker_t *worker_out) { (void)context; + cbm_index_resource_policy_t resource_policy; + char error[CBM_SZ_256] = {0}; + char task_db_path[CBM_SZ_1K]; + if (!cbm_mcp_index_policy_from_internal_args(args_json, &resource_policy, error, + sizeof(error)) || + !cbm_mcp_index_task_db_path(args_json, task_db_path, sizeof(task_db_path))) { + cbm_log_error("daemon.index.worker_policy", "error", + error[0] ? error : "could not resolve worker index path"); + *worker_out = NULL; + return -1; + } cbm_index_worker_handle_t *worker = NULL; - int result = cbm_index_worker_start(args_json, memory_budget_bytes, false, marker_file, - quarantine_file, &worker); + int result = cbm_index_worker_start_with_storage_policy( + args_json, memory_budget_bytes, &resource_policy, task_db_path, false, marker_file, + quarantine_file, NULL, NULL, &worker); *worker_out = worker; return result; } @@ -582,6 +595,20 @@ static void application_tmp_unlock(void) { atomic_flag_clear_explicit(&g_application_tmp_lock, memory_order_release); } +/* The daemon's only spelling of a project root. Tool handlers normalize the + * separators of every path they canonicalize, so a root left in the platform's + * native form is a second name for one directory — and the two names then meet + * in comparisons that are exact: whether a watch is still live for this root, + * and whether an index request may join the job already running for it. The + * forms differ on Windows, which is where both comparisons were wrong. */ +static bool application_canonical_root(const char *path, char *out, size_t out_size) { + if (!path || !out || out_size == 0 || !cbm_canonical_path(path, out, out_size)) { + return false; + } + cbm_normalize_path_sep(out); + return true; +} + static bool application_cache_dir(char out[APPLICATION_PATH_CAP]) { char configured[APPLICATION_PATH_CAP] = {0}; if (cbm_safe_getenv("CBM_CACHE_DIR", configured, sizeof(configured), NULL) && configured[0]) { @@ -1232,6 +1259,12 @@ static application_attempt_decision_t application_consume_attempt( application_attempt_free(attempt); return APPLICATION_ATTEMPT_DECISION_SUCCESS; } + if (disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + execution->response = + cbm_mcp_index_worker_resource_response(job->args_json, &attempt->result); + application_attempt_free(attempt); + return APPLICATION_ATTEMPT_DECISION_STOP; + } if (disposition == CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL) { execution->unsafe_terminal = true; execution->supervision_failed = @@ -1399,7 +1432,7 @@ static void application_auto_index_retry_pending_locked(cbm_daemon_application_t application_refresh_watch_locked(session); continue; } - char *args = application_auto_index_args(root_path); + char *args = application_auto_index_args(application, root_path); if (!args) { continue; } @@ -1900,7 +1933,50 @@ static bool application_update_reap(cbm_daemon_application_t *application, bool } } -static char *application_auto_index_args(const char *root_path) { +static bool application_index_args_add_policy(cbm_daemon_application_t *application, + yyjson_mut_doc *document, yyjson_mut_val *root) { + cbm_config_t *owned_config = NULL; + cbm_config_t *config = application ? application->config : NULL; + if (!config) { + owned_config = cbm_config_open(cbm_resolve_cache_dir()); + config = owned_config; + } + cbm_index_resource_policy_t policy; + char error[CBM_SZ_256] = {0}; + bool loaded = cbm_config_load_index_policy(config, &policy, error, sizeof(error)); + cbm_config_close(owned_config); + if (!loaded) { + cbm_log_error("daemon.index.policy", "error", error); + return false; + } + cbm_system_info_t system = cbm_system_info(); + cbm_index_policy_finalize(&policy, (uint64_t)system.total_ram, + application ? (uint64_t)application->worker_memory_budget_bytes : 0); + return cbm_mcp_index_policy_add_to_args(document, root, &policy); +} + +static char *application_index_args_replace_policy(cbm_daemon_application_t *application, + const char *args_json) { + yyjson_doc *source = args_json ? yyjson_read(args_json, strlen(args_json), 0) : NULL; + yyjson_mut_doc *document = source ? yyjson_doc_mut_copy(source, NULL) : NULL; + yyjson_doc_free(source); + yyjson_mut_val *root = document ? yyjson_mut_doc_get_root(document) : NULL; + if (!root || !yyjson_mut_is_obj(root)) { + yyjson_mut_doc_free(document); + return NULL; + } + while (yyjson_mut_obj_get(root, "_cbm_index_policy")) { + (void)yyjson_mut_obj_remove_key(root, "_cbm_index_policy"); + } + char *rewritten = application_index_args_add_policy(application, document, root) + ? yyjson_mut_write(document, 0, NULL) + : NULL; + yyjson_mut_doc_free(document); + return rewritten; +} + +static char *application_auto_index_args(cbm_daemon_application_t *application, + const char *root_path) { yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); yyjson_mut_val *root = document ? yyjson_mut_obj(document) : NULL; if (!document || !root) { @@ -1908,7 +1984,8 @@ static char *application_auto_index_args(const char *root_path) { return NULL; } yyjson_mut_doc_set_root(document, root); - char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) + char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) && + application_index_args_add_policy(application, document, root) ? yyjson_mut_write(document, 0, NULL) : NULL; yyjson_mut_doc_free(document); @@ -1951,7 +2028,7 @@ static void application_background_initialize_impl(cbm_daemon_application_sessio files); } bool args_required = auto_index_candidate && within_auto_index_limit; - char *args = args_required ? application_auto_index_args(root_path) : NULL; + char *args = args_required ? application_auto_index_args(application, root_path) : NULL; application_jobs_reap_completed(application); cbm_mutex_lock(&application->mutex); if (application->stopping || application_request_cancelled_locked(session)) { @@ -2178,14 +2255,19 @@ static char *application_index_execute(void *context, const char *root_path, if (!session || !root_path || !args_json) { return NULL; } - char *project_key = application_index_project_key(root_path, args_json); + char *trusted_args = application_index_args_replace_policy(session->application, args_json); + if (!trusted_args) { + return cbm_mcp_text_result("failed to resolve daemon index resource policy", true); + } + char *project_key = application_index_project_key(root_path, trusted_args); if (!project_key) { + free(trusted_args); return cbm_mcp_text_result("failed to derive index project identity", true); } application_job_subscribe_status_t subscribe_status = APPLICATION_JOB_SUBSCRIBE_UNAVAILABLE; cbm_daemon_application_job_t *job = NULL; for (;;) { - job = application_job_subscribe(session->application, project_key, root_path, args_json, + job = application_job_subscribe(session->application, project_key, root_path, trusted_args, &subscribe_status); if (job || (subscribe_status != APPLICATION_JOB_SUBSCRIBE_BUSY && subscribe_status != APPLICATION_JOB_SUBSCRIBE_CANCELLING)) { @@ -2204,11 +2286,13 @@ static char *application_index_execute(void *context, const char *root_path, cbm_mutex_unlock(&session->application->mutex); if (queued_cancelled) { free(project_key); + free(trusted_args); return cbm_mcp_text_result("index operation cancelled for this session", true); } cbm_usleep(APPLICATION_JOB_POLL_US); } free(project_key); + free(trusted_args); if (!job) { const char *message = "daemon index coordinator is stopping or unavailable"; if (subscribe_status == APPLICATION_JOB_SUBSCRIBE_OPTIONS_CONFLICT) { @@ -2320,9 +2404,10 @@ static cbm_daemon_runtime_application_status_t application_set_context( } char canonical_root[APPLICATION_PATH_CAP] = {0}; char canonical_allowed[APPLICATION_PATH_CAP] = {0}; - bool canonical = cbm_canonical_path(root, canonical_root, sizeof(canonical_root)); + bool canonical = application_canonical_root(root, canonical_root, sizeof(canonical_root)); if (canonical && allowed_present) { - canonical = cbm_canonical_path(allowed, canonical_allowed, sizeof(canonical_allowed)); + canonical = + application_canonical_root(allowed, canonical_allowed, sizeof(canonical_allowed)); } struct stat root_status; canonical = @@ -3340,7 +3425,7 @@ static int application_background_index(cbm_daemon_application_t *application, } char canonical_root[APPLICATION_PATH_CAP]; struct stat root_status; - if (!cbm_canonical_path(root_path, canonical_root, sizeof(canonical_root)) || + if (!application_canonical_root(root_path, canonical_root, sizeof(canonical_root)) || stat(canonical_root, &root_status) != 0 || !S_ISDIR(root_status.st_mode)) { return -1; } @@ -3351,7 +3436,8 @@ static int application_background_index(cbm_daemon_application_t *application, return -1; } yyjson_mut_doc_set_root(document, root); - bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root); + bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root) && + application_index_args_add_policy(application, document, root); char *default_project = cbm_project_name_from_path(canonical_root); bool custom_project = project_name[0] && (!default_project || strcmp(default_project, project_name) != 0); diff --git a/src/daemon/bootstrap.c b/src/daemon/bootstrap.c index ee61261d3..c3fecbf77 100644 --- a/src/daemon/bootstrap.c +++ b/src/daemon/bootstrap.c @@ -95,6 +95,21 @@ static bool bootstrap_worker_budget_valid(const char *text) { return value > 0; } +static bool bootstrap_worker_stage_token_valid(const char *token) { + size_t length = token ? strlen(token) : 0; + if (length < 6U || length >= 64U) { + return false; + } + for (size_t index = 0; index < length; index++) { + unsigned char ch = (unsigned char)token[index]; + if (!((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || + ch == '-' || ch == '_')) { + return false; + } + } + return true; +} + /* Keep the bootstrap role boundary exact and fail closed before any client or * worker state is initialized. index_supervisor owns the matching builder and * performs the captured-build comparison after this syntax-only classification. */ @@ -129,6 +144,12 @@ static bool bootstrap_worker_argv_exact(int argc, char *const argv[]) { } next += 2; } + if (next < argc && bootstrap_arg_is(argv[next], "--index-worker-stage-token")) { + if (next + 1 >= argc || !bootstrap_worker_stage_token_valid(argv[next + 1])) { + return false; + } + next += 2; + } return next == argc; } diff --git a/src/discover/discover.c b/src/discover/discover.c index fe1e4e19d..e029b654e 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -13,6 +13,7 @@ #include "foundation/constants.h" #include "foundation/compat_fs.h" +#include "foundation/limits.h" #include "foundation/workspace.h" #include "foundation/platform.h" #ifdef _WIN32 @@ -27,6 +28,25 @@ int cbm_gitignore_match_result(const cbm_gitignore_t *gi, const char *rel_path, bool is_dir); +#ifdef CBM_ENABLE_TEST_SEAMS +static cbm_discover_now_ms_fn g_discover_now_ms; +static void *g_discover_now_ms_context; + +void cbm_discover_set_now_ms_for_testing(cbm_discover_now_ms_fn now_ms, void *context) { + g_discover_now_ms = now_ms; + g_discover_now_ms_context = context; +} +#endif + +static uint64_t discover_now_ms(void) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_discover_now_ms) { + return g_discover_now_ms(g_discover_now_ms_context); + } +#endif + return cbm_now_ms(); +} + /* ── Hardcoded always-skip directories ──────────────────────────── */ static const char *ALWAYS_SKIP_DIRS[] = { @@ -414,6 +434,13 @@ typedef struct { int capacity; int max_files; uint64_t deadline_ms; + const cbm_index_resource_policy_t *resource_policy; + cbm_index_resource_violation_t *resource_violation; + uint64_t resource_started_ms; + uint64_t directories; + uint64_t entries; + uint64_t source_files; + uint64_t source_bytes; bool count_only; bool collect_excluded; bool limit_exceeded; @@ -435,13 +462,40 @@ typedef struct { int ignored_total; } file_list_t; +static void file_list_resource_violation(file_list_t *fl, cbm_index_resource_t resource, + uint64_t observed, uint64_t limit) { + if (!fl || fl->limit_exceeded || fl->failed) { + return; + } + fl->limit_exceeded = true; + if (fl->resource_violation) { + *fl->resource_violation = (cbm_index_resource_violation_t){ + .resource = resource, + .observed = observed, + .limit = limit, + }; + } +} + static bool file_list_should_stop(file_list_t *fl) { if (!fl) { return true; } - if (!fl->failed && fl->deadline_ms != 0 && cbm_now_ms() >= fl->deadline_ms) { + bool needs_now = fl->deadline_ms != 0 || + (fl->resource_policy && fl->resource_policy->discovery_deadline_ms.enabled); + uint64_t now = needs_now ? discover_now_ms() : 0; + if (!fl->failed && fl->deadline_ms != 0 && now >= fl->deadline_ms) { fl->failed = true; } + if (!fl->failed && fl->resource_policy && fl->resource_policy->discovery_deadline_ms.enabled) { + uint64_t elapsed = + now >= fl->resource_started_ms ? now - fl->resource_started_ms : UINT64_C(0); + uint64_t limit = fl->resource_policy->discovery_deadline_ms.value; + if (elapsed > limit) { + file_list_resource_violation(fl, CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS, elapsed, + limit); + } + } return fl->failed || fl->limit_exceeded; } @@ -504,6 +558,28 @@ static void fl_add(file_list_t *fl, const char *abs_path, const char *rel_path, fl->limit_exceeded = true; return; } + uint64_t source_size = size > 0 ? (uint64_t)size : 0; + bool resource_accepted = fl->resource_policy && source_size <= (uint64_t)cbm_max_file_bytes(); + if (resource_accepted) { + if (fl->resource_policy->max_files.enabled && + fl->source_files >= fl->resource_policy->max_files.value) { + file_list_resource_violation(fl, CBM_INDEX_RESOURCE_FILES, fl->source_files + 1U, + fl->resource_policy->max_files.value); + return; + } + if (fl->resource_policy->max_source_bytes.enabled) { + uint64_t limit = fl->resource_policy->max_source_bytes.value; + if (source_size > limit || fl->source_bytes > limit - source_size) { + uint64_t observed = source_size > UINT64_MAX - fl->source_bytes + ? UINT64_MAX + : fl->source_bytes + source_size; + file_list_resource_violation(fl, CBM_INDEX_RESOURCE_SOURCE_BYTES, observed, limit); + return; + } + fl->source_bytes += source_size; + } + fl->source_files++; + } if (fl->count_only) { fl->count++; return; @@ -803,6 +879,7 @@ static void walk_dir_process_file(const char *abs_path, const char *rel_path, co typedef struct { char dir[CBM_SZ_4K]; char prefix[CBM_SZ_4K]; + uint64_t depth; cbm_gitignore_t *local_gi; /* nested .gitignore for this subtree */ char local_gi_prefix[CBM_SZ_4K]; /* rel_prefix when local_gi was loaded */ } walk_frame_t; @@ -856,6 +933,7 @@ static void walk_push_subdir(walk_stack_t *ws, const char *abs_path, const char return; } slot->local_gi = parent->local_gi; + slot->depth = parent->depth + 1U; int local_prefix_length = snprintf(slot->local_gi_prefix, CBM_SZ_4K, "%s", parent->local_gi_prefix); if (local_prefix_length < 0 || local_prefix_length >= CBM_SZ_4K) { @@ -871,6 +949,18 @@ static void walk_dir_process_entry(cbm_dirent_t *entry, const walk_frame_t *fram const cbm_gitignore_t *global_gi, const cbm_gitignore_t *cbmignore, walk_stack_t *ws, file_list_t *out) { + if (out->resource_policy && out->resource_policy->max_entries.enabled) { + uint64_t limit = out->resource_policy->max_entries.value; + if (out->entries >= limit) { + uint64_t observed = out->entries == UINT64_MAX ? UINT64_MAX : out->entries + 1U; + file_list_resource_violation(out, CBM_INDEX_RESOURCE_ENTRIES, observed, limit); + return; + } + } + if (out->entries < UINT64_MAX) { + out->entries++; + } + char abs_path[CBM_SZ_4K]; char rel_path[CBM_SZ_4K]; int absolute_length = snprintf(abs_path, sizeof(abs_path), "%s/%s", frame->dir, entry->name); @@ -898,6 +988,24 @@ static void walk_dir_process_entry(cbm_dirent_t *entry, const walk_frame_t *fram if (!dir_is_cache_tree(abs_path) && !should_skip_directory(entry->name, rel_path, opts, gitignore, global_gi, cbmignore, frame->local_gi, frame->local_gi_prefix)) { + uint64_t child_depth = frame->depth == UINT64_MAX ? UINT64_MAX : frame->depth + 1U; + if (out->resource_policy && out->resource_policy->max_depth.enabled && + child_depth > out->resource_policy->max_depth.value) { + file_list_resource_violation(out, CBM_INDEX_RESOURCE_DEPTH, child_depth, + out->resource_policy->max_depth.value); + return; + } + if (out->resource_policy && out->resource_policy->max_directories.enabled && + out->directories >= out->resource_policy->max_directories.value) { + uint64_t observed = + out->directories == UINT64_MAX ? UINT64_MAX : out->directories + 1U; + file_list_resource_violation(out, CBM_INDEX_RESOURCE_DIRECTORIES, observed, + out->resource_policy->max_directories.value); + return; + } + if (out->directories < UINT64_MAX) { + out->directories++; + } walk_push_subdir(ws, abs_path, rel_path, frame, out); } else { /* Record the excluded subtree root so callers can report it (#411). */ @@ -980,8 +1088,12 @@ static void walk_dir(const char *dir_path, const char *rel_prefix, const cbm_dis continue; } - cbm_dirent_t *entry; - while (!file_list_should_stop(out) && (entry = cbm_readdir(d)) != NULL) { + while (!file_list_should_stop(out)) { + cbm_dirent_t *entry = cbm_readdir(d); + if (!entry) { + (void)file_list_should_stop(out); + break; + } walk_dir_process_entry(entry, &frame, opts, gitignore, global_gi, cbmignore, &ws, out); } cbm_closedir(d); @@ -1129,6 +1241,9 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc if (ignored_total_out) { *ignored_total_out = 0; } + if (opts && opts->resource_violation) { + *opts->resource_violation = (cbm_index_resource_violation_t){0}; + } if (!repo_path || !out || !count || (count_only && max_files < 0)) { return CBM_DISCOVER_ERROR; } @@ -1203,12 +1318,25 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc file_list_t fl = { .max_files = count_only ? max_files : -1, .deadline_ms = count_only ? deadline_ms : 0, + .resource_policy = opts ? opts->resource_policy : NULL, + .resource_violation = opts ? opts->resource_violation : NULL, + .resource_started_ms = + opts && opts->resource_policy && opts->resource_policy->discovery_deadline_ms.enabled + ? discover_now_ms() + : 0, + .directories = 1, .count_only = count_only, .collect_excluded = !count_only && excluded_out != NULL, .collect_ignored = !count_only && ignored_out != NULL, }; + if (fl.resource_policy && fl.resource_policy->max_directories.enabled && + fl.resource_policy->max_directories.value < 1U) { + file_list_resource_violation(&fl, CBM_INDEX_RESOURCE_DIRECTORIES, 1, + fl.resource_policy->max_directories.value); + } walk_cache_dir_snapshot(); walk_dir(repo_path, "", opts, gitignore, global_gi, cbmignore, &fl); + (void)file_list_should_stop(&fl); /* Cleanup */ cbm_gitignore_free(gitignore); @@ -1225,11 +1353,11 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc } return fl.limit_exceeded ? CBM_DISCOVER_LIMIT_EXCEEDED : CBM_DISCOVER_OK; } - if (fl.failed) { + if (fl.failed || fl.limit_exceeded) { cbm_discover_free(fl.files, fl.count); cbm_discover_free_excluded(fl.excluded, fl.excluded_count); cbm_discover_free_ignored(fl.ignored, fl.ignored_count); - return CBM_DISCOVER_ERROR; + return fl.limit_exceeded ? CBM_DISCOVER_LIMIT_EXCEEDED : CBM_DISCOVER_ERROR; } *out = fl.files; diff --git a/src/discover/discover.h b/src/discover/discover.h index afb60fa36..6741c15fd 100644 --- a/src/discover/discover.h +++ b/src/discover/discover.h @@ -17,6 +17,7 @@ /* Use the existing CBMLanguage enum from extraction layer */ #include "cbm.h" +#include "foundation/index_policy.h" /* ── Language detection ──────────────────────────────────────────── */ @@ -131,9 +132,11 @@ typedef struct { } cbm_file_info_t; typedef struct { - cbm_index_mode_t mode; /* CBM_MODE_FULL or CBM_MODE_FAST */ - const char *ignore_file; /* path to .cbmignore file, or NULL */ - int64_t max_file_size; /* 0 = no limit */ + cbm_index_mode_t mode; /* discovery filtering mode */ + const char *ignore_file; /* .cbmignore path, or NULL */ + int64_t max_file_size; /* 0 = no per-file limit */ + const cbm_index_resource_policy_t *resource_policy; /* NULL = no resource limits */ + cbm_index_resource_violation_t *resource_violation; /* optional exact diagnostic */ } cbm_discover_opts_t; typedef enum { @@ -142,6 +145,11 @@ typedef enum { CBM_DISCOVER_LIMIT_EXCEEDED = 1, } cbm_discover_status_t; +#ifdef CBM_ENABLE_TEST_SEAMS +typedef uint64_t (*cbm_discover_now_ms_fn)(void *context); +void cbm_discover_set_now_ms_for_testing(cbm_discover_now_ms_fn now_ms, void *context); +#endif + /* Walk a repository directory tree and discover all source files. * Applies hardcoded filters, gitignore patterns, and language detection. * Returns 0 on success, -1 on error. diff --git a/src/foundation/compat_fs.c b/src/foundation/compat_fs.c index 2610c421f..2b82156e0 100644 --- a/src/foundation/compat_fs.c +++ b/src/foundation/compat_fs.c @@ -121,19 +121,23 @@ cbm_dirent_t *cbm_readdir(cbm_dir_t *d) { return &d->entry; } -int cbm_path_info_utf8(const char *path, cbm_path_info_t *out) { +cbm_path_probe_status_t cbm_path_probe_info_utf8(const char *path, cbm_path_info_t *out) { if (!path || !out) { - return CBM_NOT_FOUND; + return CBM_PATH_PROBE_ERROR; } wchar_t *wpath = cbm_path_to_wide(path); if (!wpath) { - return CBM_NOT_FOUND; + return CBM_PATH_PROBE_ERROR; } WIN32_FILE_ATTRIBUTE_DATA data; BOOL ok = GetFileAttributesExW(wpath, GetFileExInfoStandard, &data); free(wpath); if (!ok) { - return CBM_NOT_FOUND; + DWORD error = GetLastError(); + return error == ERROR_FILE_NOT_FOUND || error == ERROR_PATH_NOT_FOUND || + error == ERROR_INVALID_NAME + ? CBM_PATH_PROBE_NOT_FOUND + : CBM_PATH_PROBE_ERROR; } memset(out, 0, sizeof(*out)); out->is_directory = (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; @@ -145,6 +149,9 @@ int cbm_path_info_utf8(const char *path, cbm_path_info_t *out) { * reports all four halves as assigned-but-never-read. This form says the * same thing without the union, so the checker needs no exception. */ uint64_t file_size = ((uint64_t)data.nFileSizeHigh << 32) | (uint64_t)data.nFileSizeLow; + if (file_size > INT64_MAX) { + return CBM_PATH_PROBE_ERROR; + } out->size = (int64_t)file_size; uint64_t written = ((uint64_t)data.ftLastWriteTime.dwHighDateTime << 32) | (uint64_t)data.ftLastWriteTime.dwLowDateTime; @@ -154,7 +161,29 @@ int cbm_path_info_utf8(const char *path, cbm_path_info_t *out) { written >= windows_to_unix_ticks ? (int64_t)((written - windows_to_unix_ticks) * NANOSECONDS_PER_WINDOWS_TICK) : 0; - return 0; + return CBM_PATH_PROBE_OK; +} + +int cbm_path_info_utf8(const char *path, cbm_path_info_t *out) { + return cbm_path_probe_info_utf8(path, out) == CBM_PATH_PROBE_OK ? 0 : CBM_NOT_FOUND; +} + +bool cbm_filesystem_free_bytes(const char *path, uint64_t *bytes_out) { + if (!path || !bytes_out) { + return false; + } + wchar_t *wpath = cbm_path_to_wide(path); + if (!wpath) { + return false; + } + ULARGE_INTEGER available; + BOOL ok = GetDiskFreeSpaceExW(wpath, &available, NULL, NULL); + free(wpath); + if (!ok) { + return false; + } + *bytes_out = available.QuadPart; + return true; } void cbm_closedir(cbm_dir_t *d) { @@ -728,6 +757,7 @@ int cbm_exec_no_shell(const char *const *argv) { #include #include #include +#include #include #include @@ -793,13 +823,14 @@ cbm_dirent_t *cbm_readdir(cbm_dir_t *d) { return NULL; } -int cbm_path_info_utf8(const char *path, cbm_path_info_t *out) { +cbm_path_probe_status_t cbm_path_probe_info_utf8(const char *path, cbm_path_info_t *out) { if (!path || !out) { - return CBM_NOT_FOUND; + return CBM_PATH_PROBE_ERROR; } struct stat state; if (lstat(path, &state) != 0) { - return CBM_NOT_FOUND; + return errno == ENOENT || errno == ENOTDIR ? CBM_PATH_PROBE_NOT_FOUND + : CBM_PATH_PROBE_ERROR; } memset(out, 0, sizeof(*out)); out->is_regular = S_ISREG(state.st_mode); @@ -813,7 +844,28 @@ int cbm_path_info_utf8(const char *path, cbm_path_info_t *out) { out->mtime_ns = ((int64_t)state.st_mtim.tv_sec * INT64_C(1000000000)) + (int64_t)state.st_mtim.tv_nsec; #endif - return 0; + return CBM_PATH_PROBE_OK; +} + +int cbm_path_info_utf8(const char *path, cbm_path_info_t *out) { + return cbm_path_probe_info_utf8(path, out) == CBM_PATH_PROBE_OK ? 0 : CBM_NOT_FOUND; +} + +bool cbm_filesystem_free_bytes(const char *path, uint64_t *bytes_out) { + if (!path || !bytes_out) { + return false; + } + struct statvfs status; + if (statvfs(path, &status) != 0) { + return false; + } + uint64_t blocks = (uint64_t)status.f_bavail; + uint64_t fragment = status.f_frsize != 0 ? (uint64_t)status.f_frsize : (uint64_t)status.f_bsize; + if (fragment == 0) { + return false; + } + *bytes_out = blocks > UINT64_MAX / fragment ? UINT64_MAX : blocks * fragment; + return true; } void cbm_closedir(cbm_dir_t *d) { @@ -953,6 +1005,282 @@ int cbm_exec_no_shell(const char *const *argv) { #endif /* _WIN32 */ +static uint64_t fs_add_saturated(uint64_t left, uint64_t right) { + return left > UINT64_MAX - right ? UINT64_MAX : left + right; +} + +static bool directory_size_recursive(const char *path, unsigned int depth, uint64_t *total) { + enum { CBM_DIRECTORY_SCAN_MAX_DEPTH = 1024 }; + if (!path || !total || depth > CBM_DIRECTORY_SCAN_MAX_DEPTH) { + return false; + } + cbm_dir_t *directory = cbm_opendir(path); + if (!directory) { + return false; + } + bool ok = true; + cbm_dirent_t *entry; + while (ok && (entry = cbm_readdir(directory)) != NULL) { + size_t path_len = strlen(path); + size_t name_len = strlen(entry->name); + bool separator_needed = + path_len > 0 && path[path_len - 1] != '/' && path[path_len - 1] != '\\'; + if (path_len > SIZE_MAX - name_len - (separator_needed ? 2U : 1U)) { + ok = false; + break; + } + size_t child_size = path_len + name_len + (separator_needed ? 2U : 1U); + char *child = (char *)malloc(child_size); + if (!child) { + ok = false; + break; + } + int written = + snprintf(child, child_size, separator_needed ? "%s/%s" : "%s%s", path, entry->name); + cbm_path_info_t info; + cbm_path_probe_status_t status = written > 0 && (size_t)written < child_size + ? cbm_path_probe_info_utf8(child, &info) + : CBM_PATH_PROBE_ERROR; + if (status == CBM_PATH_PROBE_ERROR) { + ok = false; + } else if (status == CBM_PATH_PROBE_OK && !info.is_symlink && info.is_regular) { + if (info.size < 0) { + ok = false; + } else { + *total = fs_add_saturated(*total, (uint64_t)info.size); + } + } else if (status == CBM_PATH_PROBE_OK && !info.is_symlink && info.is_directory) { + ok = directory_size_recursive(child, depth + 1U, total); + } + free(child); + } + cbm_closedir(directory); + return ok; +} + +bool cbm_directory_size_bytes(const char *path, uint64_t *bytes_out) { + if (!path || !bytes_out) { + return false; + } + *bytes_out = 0; + return directory_size_recursive(path, 0, bytes_out); +} + +bool cbm_db_artifact_bytes(const char *db_path, uint64_t *bytes_out) { + if (!db_path || !bytes_out) { + return false; + } + *bytes_out = 0; + static const char *const suffixes[] = {"", "-wal", "-shm", "-journal"}; + size_t base_len = strlen(db_path); + for (size_t index = 0; index < sizeof(suffixes) / sizeof(suffixes[0]); index++) { + size_t suffix_len = strlen(suffixes[index]); + if (base_len > SIZE_MAX - suffix_len - 1U) { + return false; + } + size_t path_size = base_len + suffix_len + 1U; + char *path = (char *)malloc(path_size); + if (!path) { + return false; + } + (void)snprintf(path, path_size, "%s%s", db_path, suffixes[index]); + cbm_path_info_t info; + cbm_path_probe_status_t status = cbm_path_probe_info_utf8(path, &info); + free(path); + if (status == CBM_PATH_PROBE_NOT_FOUND) { + continue; + } + if (status != CBM_PATH_PROBE_OK || !info.is_regular || info.is_symlink || info.size < 0) { + return false; + } + *bytes_out = fs_add_saturated(*bytes_out, (uint64_t)info.size); + } + return true; +} + +static bool fs_optional_regular_bytes(const char *path, uint64_t *total) { + if (!path || !path[0]) { + return true; + } + cbm_path_info_t info; + cbm_path_probe_status_t status = cbm_path_probe_info_utf8(path, &info); + if (status == CBM_PATH_PROBE_NOT_FOUND) { + return true; + } + if (status != CBM_PATH_PROBE_OK || !info.is_regular || info.is_symlink || info.size < 0) { + return false; + } + *total = fs_add_saturated(*total, (uint64_t)info.size); + return true; +} + +bool cbm_index_task_temp_bytes(const char *final_db_path, const char *stage_token, + const char *log_path, const char *response_path, + uint64_t *bytes_out) { + if (!final_db_path || !final_db_path[0] || !stage_token || !stage_token[0] || !bytes_out) { + return false; + } + *bytes_out = 0; + if (!fs_optional_regular_bytes(log_path, bytes_out) || + !fs_optional_regular_bytes(response_path, bytes_out)) { + return false; + } + const char *basename = strrchr(final_db_path, '/'); +#ifdef _WIN32 + const char *backslash = strrchr(final_db_path, '\\'); + if (backslash && (!basename || backslash > basename)) { + basename = backslash; + } +#endif + basename = basename ? basename + 1 : final_db_path; + size_t directory_len = (size_t)(basename - final_db_path); + char *directory = NULL; + if (directory_len == 0) { + directory = strdup("."); + directory_len = 1; + } else { + directory = (char *)malloc(directory_len + 1U); + if (directory) { + memcpy(directory, final_db_path, directory_len); + while (directory_len > 1U && (directory[directory_len - 1U] == '/' || + directory[directory_len - 1U] == '\\')) { + directory_len--; + } + directory[directory_len] = '\0'; + } + } + size_t basename_len = strlen(basename); + size_t token_len = strlen(stage_token); + static const char stage_marker[] = ".stage."; + if (!directory || token_len > SIZE_MAX - sizeof(stage_marker) - 1U || + basename_len > SIZE_MAX - token_len - sizeof(stage_marker) - 1U) { + free(directory); + return false; + } + size_t prefix_size = basename_len + (sizeof(stage_marker) - 1U) + token_len + 2U; + char *prefix = (char *)malloc(prefix_size); + if (!prefix) { + free(directory); + return false; + } + (void)snprintf(prefix, prefix_size, "%s%s%s.", basename, stage_marker, stage_token); + cbm_dir_t *dir = cbm_opendir(directory); + if (!dir) { + free(prefix); + free(directory); + return false; + } + bool ok = true; + cbm_dirent_t *entry; + while (ok && (entry = cbm_readdir(dir)) != NULL) { + if (strncmp(entry->name, prefix, prefix_size - 1U) != 0) { + continue; + } + size_t name_len = strlen(entry->name); + if (directory_len > SIZE_MAX - name_len - 2U) { + ok = false; + break; + } + size_t path_size = directory_len + name_len + 2U; + char *path = (char *)malloc(path_size); + if (!path) { + ok = false; + break; + } + (void)snprintf(path, path_size, "%s/%s", directory, entry->name); + ok = fs_optional_regular_bytes(path, bytes_out); + free(path); + } + cbm_closedir(dir); + free(prefix); + free(directory); + return ok; +} + +bool cbm_index_staging_cleanup(const char *final_db_path, const char *stage_token) { + if (!final_db_path || !final_db_path[0] || !stage_token || !stage_token[0]) { + return false; + } + const char *basename = strrchr(final_db_path, '/'); +#ifdef _WIN32 + const char *backslash = strrchr(final_db_path, '\\'); + if (backslash && (!basename || backslash > basename)) { + basename = backslash; + } +#endif + basename = basename ? basename + 1 : final_db_path; + size_t directory_len = (size_t)(basename - final_db_path); + char *directory = NULL; + if (directory_len == 0) { + directory = strdup("."); + directory_len = 1; + } else { + directory = (char *)malloc(directory_len + 1U); + if (directory) { + memcpy(directory, final_db_path, directory_len); + while (directory_len > 1U && (directory[directory_len - 1U] == '/' || + directory[directory_len - 1U] == '\\')) { + directory_len--; + } + directory[directory_len] = '\0'; + } + } + static const char stage_marker[] = ".stage."; + size_t basename_len = strlen(basename); + size_t token_len = strlen(stage_token); + if (!directory || token_len > SIZE_MAX - sizeof(stage_marker) - 1U || + basename_len > SIZE_MAX - token_len - sizeof(stage_marker) - 1U) { + free(directory); + return false; + } + size_t prefix_size = basename_len + (sizeof(stage_marker) - 1U) + token_len + 2U; + char *prefix = (char *)malloc(prefix_size); + if (!prefix) { + free(directory); + return false; + } + (void)snprintf(prefix, prefix_size, "%s%s%s.", basename, stage_marker, stage_token); + cbm_dir_t *dir = cbm_opendir(directory); + if (!dir) { + free(prefix); + free(directory); + return false; + } + bool ok = true; + cbm_dirent_t *entry; + while ((entry = cbm_readdir(dir)) != NULL) { + if (strncmp(entry->name, prefix, prefix_size - 1U) != 0) { + continue; + } + size_t name_len = strlen(entry->name); + if (directory_len > SIZE_MAX - name_len - 2U) { + ok = false; + continue; + } + size_t path_size = directory_len + name_len + 2U; + char *path = (char *)malloc(path_size); + if (!path) { + ok = false; + continue; + } + (void)snprintf(path, path_size, "%s/%s", directory, entry->name); + cbm_path_info_t info; + cbm_path_probe_status_t status = cbm_path_probe_info_utf8(path, &info); + if (status == CBM_PATH_PROBE_OK && !info.is_symlink && info.is_regular) { + if (cbm_unlink(path) != 0 && errno != ENOENT) { + ok = false; + } + } else if (status != CBM_PATH_PROBE_NOT_FOUND) { + ok = false; + } + free(path); + } + cbm_closedir(dir); + free(prefix); + free(directory); + return ok; +} + /* Canonicalize an EXISTING path (collapse `..`, resolve links/junctions): * realpath on POSIX; a final path queried from an opened handle on Windows. * The previous Windows callers used the ANSI CRT (_access/_fullpath) on UTF-8 diff --git a/src/foundation/compat_fs.h b/src/foundation/compat_fs.h index 240ebceae..51250e7d3 100644 --- a/src/foundation/compat_fs.h +++ b/src/foundation/compat_fs.h @@ -36,8 +36,34 @@ typedef struct { int64_t mtime_ns; } cbm_path_info_t; +typedef enum { + CBM_PATH_PROBE_ERROR = -1, + CBM_PATH_PROBE_OK = 0, + CBM_PATH_PROBE_NOT_FOUND = 1, +} cbm_path_probe_status_t; + /* Returns 0 on success and -1 when the path cannot be inspected. */ int cbm_path_info_utf8(const char *path, cbm_path_info_t *out); +cbm_path_probe_status_t cbm_path_probe_info_utf8(const char *path, cbm_path_info_t *out); + +/* Measure logical regular-file bytes beneath path without following symlinks + * or Windows reparse points. Saturates at UINT64_MAX. */ +bool cbm_directory_size_bytes(const char *path, uint64_t *bytes_out); + +/* Return bytes available to the current process on the filesystem containing + * an existing path (statvfs / GetDiskFreeSpaceExW). */ +bool cbm_filesystem_free_bytes(const char *path, uint64_t *bytes_out); + +/* Sum a SQLite main file and its -wal/-shm/-journal sidecars when present. + * Missing artifacts contribute zero; inspection errors fail the measurement. */ +bool cbm_db_artifact_bytes(const char *db_path, uint64_t *bytes_out); + +/* Sum this indexing attempt's explicit worker log/response plus every regular + * sibling whose name starts with ".stage.". */ +bool cbm_index_task_temp_bytes(const char *final_db_path, const char *stage_token, + const char *log_path, const char *response_path, + uint64_t *bytes_out); +bool cbm_index_staging_cleanup(const char *final_db_path, const char *stage_token); /* Open a directory for iteration. Returns NULL on error. */ cbm_dir_t *cbm_opendir(const char *path); diff --git a/src/foundation/index_policy.c b/src/foundation/index_policy.c new file mode 100644 index 000000000..6b9cab1ca --- /dev/null +++ b/src/foundation/index_policy.c @@ -0,0 +1,473 @@ +#include "foundation/index_policy.h" + +#include +#include +#include + +typedef struct { + const char *key; + size_t field_offset; + uint64_t minimum; + uint64_t maximum; + uint64_t multiplier; +} index_policy_metadata_t; + +static const index_policy_metadata_t INDEX_POLICY_METADATA[] = { + {CBM_INDEX_CONFIG_MAX_FILES, offsetof(cbm_index_resource_policy_t, max_files), 1, + CBM_INDEX_MAX_FILES_VALUE, 1}, + {CBM_INDEX_CONFIG_MAX_SOURCE_MB, offsetof(cbm_index_resource_policy_t, max_source_bytes), 1, + CBM_INDEX_MAX_SOURCE_MB_VALUE, CBM_INDEX_MIB_BYTES}, + {CBM_INDEX_CONFIG_MAX_RSS_MB, offsetof(cbm_index_resource_policy_t, max_rss_bytes), + CBM_INDEX_MIN_RSS_MB_VALUE, CBM_INDEX_MAX_RSS_MB_VALUE, CBM_INDEX_MIB_BYTES}, + {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, offsetof(cbm_index_resource_policy_t, max_duration_ms), + 1, CBM_INDEX_MAX_DURATION_SECONDS_VALUE, UINT64_C(1000)}, + {CBM_INDEX_CONFIG_CACHE_MAX_MB, offsetof(cbm_index_resource_policy_t, max_cache_bytes), 1, + CBM_INDEX_MAX_STORAGE_MB_VALUE, CBM_INDEX_MIB_BYTES}, + {CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, offsetof(cbm_index_resource_policy_t, min_free_disk_bytes), + 1, CBM_INDEX_MAX_STORAGE_MB_VALUE, CBM_INDEX_MIB_BYTES}, +}; + +static cbm_index_limit_u64_t limit_value(uint64_t value) { + return (cbm_index_limit_u64_t){.enabled = true, .value = value}; +} + +static uint64_t mib(uint64_t value) { + return value * CBM_INDEX_MIB_BYTES; +} + +static void apply_profile_baseline(cbm_index_resource_policy_t *policy, + cbm_index_resource_profile_t profile) { + policy->max_files = (cbm_index_limit_u64_t){0}; + policy->max_directories = (cbm_index_limit_u64_t){0}; + policy->max_entries = (cbm_index_limit_u64_t){0}; + policy->max_depth = (cbm_index_limit_u64_t){0}; + policy->max_source_bytes = (cbm_index_limit_u64_t){0}; + policy->discovery_deadline_ms = (cbm_index_limit_u64_t){0}; + policy->max_rss_bytes = (cbm_index_limit_u64_t){0}; + policy->max_duration_ms = (cbm_index_limit_u64_t){0}; + policy->max_cache_bytes = (cbm_index_limit_u64_t){0}; + policy->min_free_disk_bytes = (cbm_index_limit_u64_t){0}; + policy->max_final_db_bytes = (cbm_index_limit_u64_t){0}; + policy->max_staging_bytes = (cbm_index_limit_u64_t){0}; + policy->max_task_temp_bytes = (cbm_index_limit_u64_t){0}; + policy->profile = profile; + if (profile == CBM_INDEX_PROFILE_BALANCED) { + policy->max_files = limit_value(UINT64_C(500000)); + policy->max_directories = limit_value(UINT64_C(250000)); + policy->max_entries = limit_value(UINT64_C(2000000)); + policy->max_depth = limit_value(UINT64_C(128)); + policy->max_source_bytes = limit_value(mib(UINT64_C(65536))); + policy->discovery_deadline_ms = limit_value(UINT64_C(1800000)); + policy->max_rss_bytes = limit_value(mib(UINT64_C(8192))); + policy->max_duration_ms = limit_value(UINT64_C(7200000)); + policy->max_final_db_bytes = limit_value(mib(UINT64_C(65536))); + policy->max_staging_bytes = limit_value(mib(UINT64_C(81920))); + policy->max_task_temp_bytes = limit_value(mib(UINT64_C(98304))); + policy->max_cache_bytes = limit_value(mib(UINT64_C(131072))); + policy->min_free_disk_bytes = limit_value(mib(UINT64_C(4096))); + } else if (profile == CBM_INDEX_PROFILE_STRICT) { + policy->max_files = limit_value(UINT64_C(100000)); + policy->max_directories = limit_value(UINT64_C(20000)); + policy->max_entries = limit_value(UINT64_C(500000)); + policy->max_depth = limit_value(UINT64_C(64)); + policy->max_source_bytes = limit_value(mib(UINT64_C(4096))); + policy->discovery_deadline_ms = limit_value(UINT64_C(30000)); + policy->max_rss_bytes = limit_value(mib(UINT64_C(8192))); + policy->max_duration_ms = limit_value(UINT64_C(3600000)); + policy->max_final_db_bytes = limit_value(mib(UINT64_C(16384))); + policy->max_staging_bytes = limit_value(mib(UINT64_C(20480))); + policy->max_task_temp_bytes = limit_value(mib(UINT64_C(24576))); + policy->max_cache_bytes = limit_value(mib(UINT64_C(32768))); + policy->min_free_disk_bytes = limit_value(mib(UINT64_C(4096))); + } +} + +static void set_error(char *error, size_t error_size, const char *key, uint64_t minimum, + uint64_t maximum) { + if (error && error_size > 0) { + (void)snprintf(error, error_size, "%s must be off or an integer from %llu to %llu", key, + (unsigned long long)minimum, (unsigned long long)maximum); + } +} + +static bool parse_bounded_uint64(const char *value, uint64_t minimum, uint64_t maximum, + uint64_t *parsed) { + if (!value || !value[0] || !parsed) { + return false; + } + uint64_t result = 0; + for (const unsigned char *cursor = (const unsigned char *)value; *cursor; cursor++) { + if (*cursor < '0' || *cursor > '9') { + return false; + } + uint64_t digit = (uint64_t)(*cursor - '0'); + if (result > (maximum - digit) / 10U) { + return false; + } + result = result * 10U + digit; + } + if (result < minimum) { + return false; + } + *parsed = result; + return true; +} + +void cbm_index_policy_init(cbm_index_resource_policy_t *policy) { + if (policy) { + *policy = (cbm_index_resource_policy_t){0}; + } +} + +bool cbm_index_policy_set_profile(cbm_index_resource_policy_t *policy, const char *value, + char *error, size_t error_size) { + if (error && error_size > 0) { + error[0] = '\0'; + } + cbm_index_resource_profile_t profile; + if (value && strcmp(value, "off") == 0) { + profile = CBM_INDEX_PROFILE_OFF; + } else if (value && strcmp(value, "balanced") == 0) { + profile = CBM_INDEX_PROFILE_BALANCED; + } else if (value && strcmp(value, "strict") == 0) { + profile = CBM_INDEX_PROFILE_STRICT; + } else { + if (error && error_size > 0) { + (void)snprintf(error, error_size, "%s must be off, balanced, or strict", + CBM_INDEX_CONFIG_RESOURCE_PROFILE); + } + return false; + } + if (!policy) { + return false; + } + + cbm_index_resource_policy_t candidate = *policy; + cbm_index_limit_u64_t + overrides[sizeof(INDEX_POLICY_METADATA) / sizeof(INDEX_POLICY_METADATA[0])]; + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + const cbm_index_limit_u64_t *source = + (const cbm_index_limit_u64_t *)((const unsigned char *)policy + + INDEX_POLICY_METADATA[index].field_offset); + overrides[index] = *source; + } + apply_profile_baseline(&candidate, profile); + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if ((candidate.override_mask & (UINT64_C(1) << index)) == 0) { + continue; + } + cbm_index_limit_u64_t *target = + (cbm_index_limit_u64_t *)((unsigned char *)&candidate + + INDEX_POLICY_METADATA[index].field_offset); + *target = overrides[index]; + } + *policy = candidate; + return true; +} + +void cbm_index_policy_finalize(cbm_index_resource_policy_t *policy, uint64_t host_memory_bytes, + uint64_t soft_budget_bytes) { + if (!policy) { + return; + } + uint64_t host_cap = host_memory_bytes > 0 + ? (host_memory_bytes / 5U) * 4U + ((host_memory_bytes % 5U) * 4U) / 5U + : 0; + const uint64_t rss_override_bit = UINT64_C(1) << 2U; + if ((policy->override_mask & rss_override_bit) == 0) { + if (policy->profile == CBM_INDEX_PROFILE_BALANCED) { + /* Balanced guards the host, not the workload. Any fixed ceiling + * below what this machine can actually finish would reject indexes + * that succeed today — the recorded large-repository runs peak far + * above any round number worth hard-coding. The host share is the + * only honest balanced answer; the fixed floor is a fallback for + * hosts whose memory cannot be read. */ + uint64_t doubled = + soft_budget_bytes > UINT64_MAX / 2U ? UINT64_MAX : soft_budget_bytes * 2U; + uint64_t fallback = doubled > mib(UINT64_C(8192)) ? doubled : mib(UINT64_C(8192)); + policy->max_rss_bytes = limit_value(host_cap > 0 ? host_cap : fallback); + } else if (policy->profile == CBM_INDEX_PROFILE_STRICT) { + /* Strict holds the worker to the daemon's own soft budget, so a + * repository larger than that budget fails fast and attributed + * instead of being finished at the host's expense. */ + uint64_t baseline = soft_budget_bytes > 0 ? soft_budget_bytes : mib(UINT64_C(8192)); + if (baseline > mib(UINT64_C(8192))) { + baseline = mib(UINT64_C(8192)); + } + policy->max_rss_bytes = limit_value(baseline); + } + } + if (policy->max_rss_bytes.enabled && host_cap > 0 && policy->max_rss_bytes.value > host_cap) { + policy->max_rss_bytes.value = host_cap; + } + if (policy->max_rss_bytes.enabled) { + uint64_t minimum = mib(CBM_INDEX_MIN_RSS_MB_VALUE); + uint64_t maximum = mib(CBM_INDEX_MAX_RSS_MB_VALUE); + if (policy->max_rss_bytes.value < minimum) { + policy->max_rss_bytes.value = minimum; + } else if (policy->max_rss_bytes.value > maximum) { + policy->max_rss_bytes.value = maximum; + } + } +} + +const char *cbm_index_policy_profile_name(const cbm_index_resource_policy_t *policy) { + if (!policy || policy->profile == CBM_INDEX_PROFILE_OFF) { + return "off"; + } + return policy->profile == CBM_INDEX_PROFILE_BALANCED ? "balanced" : "strict"; +} + +const char *cbm_index_policy_source_name(const cbm_index_resource_policy_t *policy) { + bool profile = policy && policy->profile != CBM_INDEX_PROFILE_OFF; + bool override = policy && policy->override_mask != 0; + if (profile && override) { + return "profile+override"; + } + if (profile) { + return "profile"; + } + return override ? "override" : "off"; +} + +bool cbm_index_policy_is_config_key(const char *key) { + if (key && strcmp(key, CBM_INDEX_CONFIG_RESOURCE_PROFILE) == 0) { + return true; + } + for (size_t index = 0; key && index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { + return true; + } + } + return false; +} + +bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy) { + return cbm_index_policy_discovery_enabled(policy) || cbm_index_policy_worker_enabled(policy) || + cbm_index_policy_storage_enabled(policy); +} + +bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy) { + return policy && (policy->max_files.enabled || policy->max_directories.enabled || + policy->max_entries.enabled || policy->max_depth.enabled || + policy->max_source_bytes.enabled || policy->discovery_deadline_ms.enabled); +} + +bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy) { + return policy && (policy->max_rss_bytes.enabled || policy->max_duration_ms.enabled); +} + +bool cbm_index_policy_storage_enabled(const cbm_index_resource_policy_t *policy) { + return policy && (policy->max_cache_bytes.enabled || policy->min_free_disk_bytes.enabled || + policy->max_final_db_bytes.enabled || policy->max_staging_bytes.enabled || + policy->max_task_temp_bytes.enabled); +} + +static uint64_t add_saturated(uint64_t left, uint64_t right) { + return left > UINT64_MAX - right ? UINT64_MAX : left + right; +} + +static bool storage_within_max(cbm_index_resource_t resource, uint64_t observed, + const cbm_index_limit_u64_t *limit, + cbm_index_resource_violation_t *violation) { + if (!limit->enabled || observed <= limit->value) { + return true; + } + *violation = (cbm_index_resource_violation_t){ + .resource = resource, + .observed = observed, + .limit = limit->value, + }; + return false; +} + +bool cbm_index_policy_check_storage(const cbm_index_resource_policy_t *policy, + const cbm_index_storage_sample_t *sample, + cbm_index_resource_violation_t *violation) { + if (!policy || !sample || !violation) { + return false; + } + *violation = (cbm_index_resource_violation_t){0}; + uint64_t remaining_cache = sample->current_cache_bytes > sample->replaceable_old_bytes + ? sample->current_cache_bytes - sample->replaceable_old_bytes + : 0; + uint64_t projected_cache = add_saturated(remaining_cache, sample->operation_bytes); + if (!storage_within_max(CBM_INDEX_RESOURCE_CACHE_BYTES, projected_cache, + &policy->max_cache_bytes, violation)) { + return false; + } + if (policy->min_free_disk_bytes.enabled && + sample->free_disk_bytes < policy->min_free_disk_bytes.value) { + *violation = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_FREE_DISK_BYTES, + .observed = sample->free_disk_bytes, + .limit = policy->min_free_disk_bytes.value, + }; + return false; + } + return storage_within_max(CBM_INDEX_RESOURCE_FINAL_DB_BYTES, sample->final_db_bytes, + &policy->max_final_db_bytes, violation) && + storage_within_max(CBM_INDEX_RESOURCE_STAGING_BYTES, sample->staging_bytes, + &policy->max_staging_bytes, violation) && + storage_within_max(CBM_INDEX_RESOURCE_TASK_TEMP_BYTES, sample->task_temp_bytes, + &policy->max_task_temp_bytes, violation); +} + +size_t cbm_index_policy_key_count(void) { + return sizeof(INDEX_POLICY_METADATA) / sizeof(INDEX_POLICY_METADATA[0]); +} + +const char *cbm_index_policy_key_at(size_t index) { + return index < cbm_index_policy_key_count() ? INDEX_POLICY_METADATA[index].key : NULL; +} + +const char *cbm_index_policy_default_value(const char *key) { + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if (key && strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { + return "off"; + } + } + return NULL; +} + +bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, const char *value, + char *error, size_t error_size) { + if (error && error_size > 0) { + error[0] = '\0'; + } + if (!policy || !key || !value) { + set_error(error, error_size, key ? key : "index resource limit", 0, 0); + return false; + } + + const index_policy_metadata_t *metadata = NULL; + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { + metadata = &INDEX_POLICY_METADATA[index]; + break; + } + } + if (!metadata) { + set_error(error, error_size, key, 0, 0); + return false; + } + cbm_index_limit_u64_t *target = + (cbm_index_limit_u64_t *)((unsigned char *)policy + metadata->field_offset); + + cbm_index_limit_u64_t candidate = {0}; + if (strcmp(value, "off") != 0) { + uint64_t parsed = 0; + if (!parse_bounded_uint64(value, metadata->minimum, metadata->maximum, &parsed)) { + set_error(error, error_size, key, metadata->minimum, metadata->maximum); + return false; + } + candidate.enabled = true; + candidate.value = parsed * metadata->multiplier; + } + *target = candidate; + policy->override_mask |= UINT64_C(1) << (size_t)(metadata - INDEX_POLICY_METADATA); + return true; +} + +bool cbm_index_policy_format(const cbm_index_resource_policy_t *policy, const char *key, char *out, + size_t out_size) { + if (!policy || !key || !out || out_size == 0) { + return false; + } + const index_policy_metadata_t *metadata = NULL; + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { + metadata = &INDEX_POLICY_METADATA[index]; + break; + } + } + if (!metadata) { + return false; + } + const cbm_index_limit_u64_t *limit = + (const cbm_index_limit_u64_t *)((const unsigned char *)policy + metadata->field_offset); + int length = limit->enabled + ? snprintf(out, out_size, "%llu", + (unsigned long long)(limit->value / metadata->multiplier)) + : snprintf(out, out_size, "off"); + return length >= 0 && (size_t)length < out_size; +} + +const char *cbm_index_resource_name(cbm_index_resource_t resource) { + switch (resource) { + case CBM_INDEX_RESOURCE_FILES: + return "files"; + case CBM_INDEX_RESOURCE_SOURCE_BYTES: + return "source_bytes"; + case CBM_INDEX_RESOURCE_RSS_BYTES: + return "rss_bytes"; + case CBM_INDEX_RESOURCE_DURATION_MS: + return "duration_ms"; + case CBM_INDEX_RESOURCE_CACHE_BYTES: + return "cache_bytes"; + case CBM_INDEX_RESOURCE_FREE_DISK_BYTES: + return "free_disk_bytes"; + case CBM_INDEX_RESOURCE_FINAL_DB_BYTES: + return "final_db_bytes"; + case CBM_INDEX_RESOURCE_STAGING_BYTES: + return "staging_bytes"; + case CBM_INDEX_RESOURCE_TASK_TEMP_BYTES: + return "task_temp_bytes"; + case CBM_INDEX_RESOURCE_DIRECTORIES: + return "directories"; + case CBM_INDEX_RESOURCE_ENTRIES: + return "entries"; + case CBM_INDEX_RESOURCE_DEPTH: + return "depth"; + case CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS: + return "discovery_duration_ms"; + case CBM_INDEX_RESOURCE_NONE: + default: + return "unknown"; + } +} + +const char *cbm_index_resource_unit(cbm_index_resource_t resource) { + if (resource == CBM_INDEX_RESOURCE_FILES) { + return "files"; + } + if (resource == CBM_INDEX_RESOURCE_DURATION_MS) { + return "milliseconds"; + } + if (resource == CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS) { + return "milliseconds"; + } + if (resource == CBM_INDEX_RESOURCE_DIRECTORIES || resource == CBM_INDEX_RESOURCE_ENTRIES || + resource == CBM_INDEX_RESOURCE_DEPTH) { + return "count"; + } + return "bytes"; +} + +const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { + switch (resource) { + case CBM_INDEX_RESOURCE_FILES: + return CBM_INDEX_CONFIG_MAX_FILES; + case CBM_INDEX_RESOURCE_SOURCE_BYTES: + return CBM_INDEX_CONFIG_MAX_SOURCE_MB; + case CBM_INDEX_RESOURCE_RSS_BYTES: + return CBM_INDEX_CONFIG_MAX_RSS_MB; + case CBM_INDEX_RESOURCE_DURATION_MS: + return CBM_INDEX_CONFIG_MAX_DURATION_SECONDS; + case CBM_INDEX_RESOURCE_CACHE_BYTES: + return CBM_INDEX_CONFIG_CACHE_MAX_MB; + case CBM_INDEX_RESOURCE_FREE_DISK_BYTES: + return CBM_INDEX_CONFIG_MIN_FREE_DISK_MB; + case CBM_INDEX_RESOURCE_FINAL_DB_BYTES: + case CBM_INDEX_RESOURCE_STAGING_BYTES: + case CBM_INDEX_RESOURCE_TASK_TEMP_BYTES: + case CBM_INDEX_RESOURCE_DIRECTORIES: + case CBM_INDEX_RESOURCE_ENTRIES: + case CBM_INDEX_RESOURCE_DEPTH: + case CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS: + return CBM_INDEX_CONFIG_RESOURCE_PROFILE; + case CBM_INDEX_RESOURCE_NONE: + default: + return "index_resource_limit"; + } +} diff --git a/src/foundation/index_policy.h b/src/foundation/index_policy.h new file mode 100644 index 000000000..85d1234b0 --- /dev/null +++ b/src/foundation/index_policy.h @@ -0,0 +1,117 @@ +#ifndef CBM_INDEX_POLICY_H +#define CBM_INDEX_POLICY_H + +#include +#include +#include + +#define CBM_INDEX_CONFIG_MAX_FILES "index_max_files" +#define CBM_INDEX_CONFIG_MAX_SOURCE_MB "index_max_source_mb" +#define CBM_INDEX_CONFIG_MAX_RSS_MB "index_max_rss_mb" +#define CBM_INDEX_CONFIG_MAX_DURATION_SECONDS "index_max_duration_seconds" +#define CBM_INDEX_CONFIG_CACHE_MAX_MB "index_cache_max_mb" +#define CBM_INDEX_CONFIG_MIN_FREE_DISK_MB "index_min_free_disk_mb" +#define CBM_INDEX_CONFIG_RESOURCE_PROFILE "index_resource_profile" + +#define CBM_INDEX_MAX_FILES_VALUE UINT64_C(10000000) +#define CBM_INDEX_MAX_SOURCE_MB_VALUE UINT64_C(1048576) +#define CBM_INDEX_MIN_RSS_MB_VALUE UINT64_C(64) +#define CBM_INDEX_MAX_RSS_MB_VALUE UINT64_C(1048576) +#define CBM_INDEX_MAX_DURATION_SECONDS_VALUE UINT64_C(86400) +#define CBM_INDEX_MAX_STORAGE_MB_VALUE UINT64_C(1048576) +#define CBM_INDEX_MIB_BYTES UINT64_C(1048576) + +typedef struct { + bool enabled; + uint64_t value; +} cbm_index_limit_u64_t; + +typedef enum { + CBM_INDEX_PROFILE_OFF = 0, + CBM_INDEX_PROFILE_BALANCED, + CBM_INDEX_PROFILE_STRICT, +} cbm_index_resource_profile_t; + +typedef struct { + cbm_index_resource_profile_t profile; + uint64_t override_mask; + cbm_index_limit_u64_t max_files; + cbm_index_limit_u64_t max_directories; + cbm_index_limit_u64_t max_entries; + cbm_index_limit_u64_t max_depth; + cbm_index_limit_u64_t max_source_bytes; + cbm_index_limit_u64_t discovery_deadline_ms; + cbm_index_limit_u64_t max_rss_bytes; + cbm_index_limit_u64_t max_duration_ms; + cbm_index_limit_u64_t max_cache_bytes; + cbm_index_limit_u64_t min_free_disk_bytes; + /* Profile-only dimensions. They are deliberately not public config keys. */ + cbm_index_limit_u64_t max_final_db_bytes; + cbm_index_limit_u64_t max_staging_bytes; + cbm_index_limit_u64_t max_task_temp_bytes; +} cbm_index_resource_policy_t; + +typedef enum { + CBM_INDEX_RESOURCE_NONE = 0, + CBM_INDEX_RESOURCE_FILES, + CBM_INDEX_RESOURCE_SOURCE_BYTES, + CBM_INDEX_RESOURCE_RSS_BYTES, + CBM_INDEX_RESOURCE_DURATION_MS, + CBM_INDEX_RESOURCE_CACHE_BYTES, + CBM_INDEX_RESOURCE_FREE_DISK_BYTES, + CBM_INDEX_RESOURCE_FINAL_DB_BYTES, + CBM_INDEX_RESOURCE_STAGING_BYTES, + CBM_INDEX_RESOURCE_TASK_TEMP_BYTES, + CBM_INDEX_RESOURCE_DIRECTORIES, + CBM_INDEX_RESOURCE_ENTRIES, + CBM_INDEX_RESOURCE_DEPTH, + CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS, +} cbm_index_resource_t; + +typedef struct { + cbm_index_resource_t resource; + uint64_t observed; + uint64_t limit; + bool probe_failed; +} cbm_index_resource_violation_t; + +typedef struct { + uint64_t current_cache_bytes; + uint64_t replaceable_old_bytes; + uint64_t operation_bytes; + uint64_t free_disk_bytes; + uint64_t final_db_bytes; + uint64_t staging_bytes; + uint64_t task_temp_bytes; +} cbm_index_storage_sample_t; + +void cbm_index_policy_init(cbm_index_resource_policy_t *policy); +bool cbm_index_policy_set_profile(cbm_index_resource_policy_t *policy, const char *value, + char *error, size_t error_size); +void cbm_index_policy_finalize(cbm_index_resource_policy_t *policy, uint64_t host_memory_bytes, + uint64_t soft_budget_bytes); +const char *cbm_index_policy_profile_name(const cbm_index_resource_policy_t *policy); +const char *cbm_index_policy_source_name(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_is_config_key(const char *key); +bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_storage_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_check_storage(const cbm_index_resource_policy_t *policy, + const cbm_index_storage_sample_t *sample, + cbm_index_resource_violation_t *violation); + +size_t cbm_index_policy_key_count(void); +const char *cbm_index_policy_key_at(size_t index); +const char *cbm_index_policy_default_value(const char *key); + +bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, const char *value, + char *error, size_t error_size); +bool cbm_index_policy_format(const cbm_index_resource_policy_t *policy, const char *key, char *out, + size_t out_size); + +const char *cbm_index_resource_name(cbm_index_resource_t resource); +const char *cbm_index_resource_unit(cbm_index_resource_t resource); +const char *cbm_index_resource_config_key(cbm_index_resource_t resource); + +#endif diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index 9174391f4..f6be6b93d 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -11,6 +11,7 @@ #include "platform.h" /* cbm_now_ms */ #include "sanitized.h" /* CBM_SANITIZED — spawn-retry budget */ +#include #include #include #include @@ -18,14 +19,17 @@ #ifdef _WIN32 #include +#include #include "win_utf8.h" /* cbm_utf8_to_wide — spawn the worker with a wide command line so a * non-ASCII repo path survives CreateProcess (#423/#20) */ #include /* free */ #else +#include #include #include #include #ifdef __APPLE__ +#include #include extern char **environ; #endif @@ -436,6 +440,287 @@ struct cbm_subprocess { #endif }; +static void cbm_rss_add_saturated(uint64_t *total, uint64_t value) { + if (value > UINT64_MAX - *total) { + *total = UINT64_MAX; + } else { + *total += value; + } +} + +#ifdef CBM_ENABLE_TEST_SEAMS +uint64_t cbm_subprocess_rss_sum_for_testing(const uint64_t *values, size_t count) { + uint64_t total = 0; + for (size_t index = 0; values && index < count; index++) { + cbm_rss_add_saturated(&total, values[index]); + } + return total; +} +#endif + +#ifdef _WIN32 +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + DWORD capacity = 32; + JOBOBJECT_BASIC_PROCESS_ID_LIST *processes = NULL; + for (;;) { + size_t bytes = sizeof(*processes) + (size_t)(capacity - 1) * sizeof(ULONG_PTR); + if (bytes > UINT32_MAX) { + free(processes); + return CBM_PROC_TREE_RSS_ERROR; + } + JOBOBJECT_BASIC_PROCESS_ID_LIST *grown = realloc(processes, bytes); + if (!grown) { + free(processes); + return CBM_PROC_TREE_RSS_ERROR; + } + processes = grown; + ZeroMemory(processes, bytes); + processes->NumberOfAssignedProcesses = capacity; + if (QueryInformationJobObject(process->job, JobObjectBasicProcessIdList, processes, + (DWORD)bytes, NULL)) { + break; + } + if (GetLastError() != ERROR_MORE_DATA || capacity > (UINT32_MAX / 2U)) { + free(processes); + return CBM_PROC_TREE_RSS_ERROR; + } + capacity *= 2U; + } + + uint64_t total = 0; + DWORD measured = 0; + bool root_failed = false; + for (DWORD index = 0; index < processes->NumberOfProcessIdsInList; index++) { + DWORD pid = (DWORD)processes->ProcessIdList[index]; + HANDLE member = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid); + if (!member) { + root_failed = root_failed || pid == process->process_id; + continue; + } + PROCESS_MEMORY_COUNTERS memory; + ZeroMemory(&memory, sizeof(memory)); + if (GetProcessMemoryInfo(member, &memory, sizeof(memory))) { + cbm_rss_add_saturated(&total, (uint64_t)memory.WorkingSetSize); + measured++; + } else if (pid == process->process_id) { + root_failed = true; + } + CloseHandle(member); + } + DWORD listed = processes->NumberOfProcessIdsInList; + free(processes); + if (root_failed || (measured == 0 && listed > 0)) { + return CBM_PROC_TREE_RSS_ERROR; + } + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#elif defined(__APPLE__) +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + int capacity = proc_listallpids(NULL, 0); + if (capacity <= 0 || capacity > INT_MAX / (int)sizeof(pid_t) - 64) { + return CBM_PROC_TREE_RSS_ERROR; + } + capacity += 64; + pid_t *pids = NULL; + int count = 0; + for (;;) { + pid_t *grown = realloc(pids, (size_t)capacity * sizeof(*pids)); + if (!grown) { + free(pids); + return CBM_PROC_TREE_RSS_ERROR; + } + pids = grown; + count = proc_listallpids(pids, capacity * (int)sizeof(*pids)); + if (count <= 0) { + free(pids); + return CBM_PROC_TREE_RSS_ERROR; + } + if (count < capacity) { + break; + } + if (capacity > INT_MAX / (int)sizeof(*pids) / 2) { + free(pids); + return CBM_PROC_TREE_RSS_ERROR; + } + capacity *= 2; + } + + uint64_t total = 0; + int measured = 0; + bool root_failed = false; + for (int index = 0; index < count; index++) { + struct proc_bsdinfo info; + if (pids[index] <= 0) { + continue; + } + if (proc_pidinfo(pids[index], PROC_PIDTBSDINFO, 0, &info, sizeof(info)) != + (int)sizeof(info)) { + root_failed = root_failed || pids[index] == process->pid; + continue; + } + if ((pid_t)info.pbi_pgid != process->pgid) { + continue; + } + struct rusage_info_v2 usage; + if (proc_pid_rusage(pids[index], RUSAGE_INFO_V2, (rusage_info_t *)&usage) != 0) { + root_failed = root_failed || pids[index] == process->pid; + continue; + } + cbm_rss_add_saturated(&total, usage.ri_resident_size); + measured++; + } + free(pids); + if (root_failed) { + return CBM_PROC_TREE_RSS_ERROR; + } + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#else +static bool cbm_proc_pid_name(const char *name) { + if (!name || !name[0]) { + return false; + } + for (const unsigned char *cursor = (const unsigned char *)name; *cursor; cursor++) { + if (*cursor < '0' || *cursor > '9') { + return false; + } + } + return true; +} + +static bool cbm_linux_proc_stat(const char *pid_name, pid_t *group, int64_t *rss_pages) { + char path[64]; + int length = snprintf(path, sizeof(path), "/proc/%s/stat", pid_name); + if (length <= 0 || length >= (int)sizeof(path)) { + return false; + } + FILE *stat_file = cbm_fopen(path, "rb"); + if (!stat_file) { + return false; + } + char stat_line[4096]; + bool read = fgets(stat_line, sizeof(stat_line), stat_file) != NULL; + (void)fclose(stat_file); + char *command_end = read ? strrchr(stat_line, ')') : NULL; + if (!command_end || command_end[1] != ' ') { + return false; + } + + char *save = NULL; + char *token = strtok_r(command_end + 2, " ", &save); + int field = 3; + bool have_group = false; + bool have_rss = false; + while (token) { + if (field == 5 || field == 24) { + char *end = NULL; + long long value = strtoll(token, &end, 10); + if (!end || *end != '\0') { + return false; + } + if (field == 5) { + *group = (pid_t)value; + have_group = true; + } else { + *rss_pages = (int64_t)value; + have_rss = true; + break; + } + } + token = strtok_r(NULL, " ", &save); + field++; + } + return have_group && have_rss; +} + +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + DIR *proc = opendir("/proc"); + if (!proc) { + return CBM_PROC_TREE_RSS_ERROR; + } + long page_size = sysconf(_SC_PAGESIZE); + if (page_size <= 0) { + (void)closedir(proc); + return CBM_PROC_TREE_RSS_ERROR; + } + + uint64_t total = 0; + int measured = 0; + bool root_failed = false; + struct dirent *entry; + while ((entry = readdir(proc)) != NULL) { + if (!cbm_proc_pid_name(entry->d_name)) { + continue; + } + pid_t group = 0; + int64_t pages = 0; + bool root_entry = strtol(entry->d_name, NULL, 10) == (long)process->pid; + if (!cbm_linux_proc_stat(entry->d_name, &group, &pages)) { + root_failed = root_failed || root_entry; + continue; + } + if (group != process->pgid || pages < 0) { + continue; + } + uint64_t page_count = (uint64_t)pages; + uint64_t bytes = page_count > UINT64_MAX / (uint64_t)page_size + ? UINT64_MAX + : page_count * (uint64_t)page_size; + cbm_rss_add_saturated(&total, bytes); + measured++; + } + (void)closedir(proc); + if (root_failed) { + return CBM_PROC_TREE_RSS_ERROR; + } + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#endif + +cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_bytes(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + if (!process || !rss_bytes) { + return CBM_PROC_TREE_RSS_ERROR; + } + *rss_bytes = 0; + return cbm_subprocess_tree_rss_platform(process, rss_bytes); +} + +bool cbm_subprocess_root_running(const cbm_subprocess_t *process) { + if (!process || process->root_reaped) { + return false; + } + int lifecycle = atomic_load_explicit(&process->lifecycle, memory_order_acquire); + return lifecycle == CBM_SUBPROCESS_ACTIVE || lifecycle == CBM_SUBPROCESS_CANCEL_REQUESTED; +} + +bool cbm_subprocess_termination_pending(const cbm_subprocess_t *process) { + return process && process->termination_started; +} + +bool cbm_subprocess_supervision_active(const cbm_subprocess_t *process) { + if (!process) { + return false; + } + int lifecycle = atomic_load_explicit(&process->lifecycle, memory_order_acquire); + return lifecycle == CBM_SUBPROCESS_ACTIVE || lifecycle == CBM_SUBPROCESS_CANCEL_REQUESTED; +} + static void cbm_subprocess_result_init(cbm_proc_result_t *result) { result->outcome = CBM_PROC_SPAWN_FAILED; result->exit_code = -1; diff --git a/src/foundation/subprocess.h b/src/foundation/subprocess.h index 62592c592..07998603c 100644 --- a/src/foundation/subprocess.h +++ b/src/foundation/subprocess.h @@ -22,6 +22,7 @@ #include #include /* size_t (cbm_build_win_cmdline) */ +#include /* How a supervised child ended. */ typedef enum { @@ -87,6 +88,12 @@ typedef enum { CBM_PROC_POLL_TERMINAL = 1 } cbm_proc_poll_t; +typedef enum { + CBM_PROC_TREE_RSS_ERROR = -1, + CBM_PROC_TREE_RSS_EMPTY = 0, + CBM_PROC_TREE_RSS_OK = 1, +} cbm_proc_tree_rss_status_t; + /* Spawn opts->bin and return immediately with a supervisor handle. On success, * *out owns the process until a terminal poll followed by destroy. Spawn copies * the option strings/argv it needs after return; log_ud remains caller-owned until @@ -117,6 +124,16 @@ int cbm_subprocess_spawn(const cbm_proc_opts_t *opts, cbm_subprocess_t **out); * tree when cancel_grace_ms elapses. Callers must keep polling to make progress. */ cbm_proc_poll_t cbm_subprocess_poll(cbm_subprocess_t *process, cbm_proc_result_t *out); +/* Read the current resident-set size of the complete contained process tree. + * OK returns an overflow-safe byte total, EMPTY means the owned tree currently + * has no observable members, and ERROR means no trustworthy measurement could + * be obtained. Individual processes that exit during enumeration are ignored. */ +cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_bytes(cbm_subprocess_t *process, + uint64_t *rss_bytes); +bool cbm_subprocess_root_running(const cbm_subprocess_t *process); +bool cbm_subprocess_termination_pending(const cbm_subprocess_t *process); +bool cbm_subprocess_supervision_active(const cbm_subprocess_t *process); + /* Record an explicit cancellation request without waiting. Safe to repeat and * safe to call from a cancellation thread while one owner thread polls. The * owner must stop cancellation producers before destroying the handle. true @@ -183,6 +200,7 @@ bool cbm_build_win_cmd_payload(char *buf, size_t cap, const char *cmd_executable * of hoping a loaded machine reproduces it. Test builds only. */ void cbm_subprocess_force_spawn_eagain_for_testing(int attempts); int cbm_subprocess_pending_spawn_eagain_for_testing(void); +uint64_t cbm_subprocess_rss_sum_for_testing(const uint64_t *values, size_t count); #endif #endif /* CBM_SUBPROCESS_H */ diff --git a/src/main.c b/src/main.c index 288dec357..e7f22e60a 100644 --- a/src/main.c +++ b/src/main.c @@ -687,9 +687,11 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo const char *worker_marker = cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_MARKER_ARG); const char *worker_quarantine = cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_QUARANTINE_ARG); + const char *worker_stage_token = + cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_STAGE_TOKEN_ARG); cbm_index_set_worker_role_options(index_worker, response_out, worker_single_thread, worker_marker, worker_quarantine, - cbm_index_worker_memory_budget_bytes()); + cbm_index_worker_memory_budget_bytes(), worker_stage_token); if (argc < MAIN_MIN_ARGC) { (void)fprintf(stderr, CLI_USAGE); @@ -2718,7 +2720,7 @@ int main(int argc, char **argv) { } cbm_index_set_worker_role_options(true, invocation.response_out, invocation.single_thread, invocation.marker_file, invocation.quarantine_file, - invocation.memory_budget_bytes); + invocation.memory_budget_bytes, invocation.stage_token); #ifndef _WIN32 /* Split into three ordered steps rather than one condition, because the * ORDER is load-bearing and the middle step only exists in test builds: diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index b5257dfc1..c485b9a75 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -67,7 +67,7 @@ static void worker_set_local_env(const char *name, const char *value) { void cbm_index_set_worker_role_options(bool is_worker, const char *response_out, bool single_thread, const char *marker_file, const char *quarantine_file, - size_t memory_budget_bytes) { + size_t memory_budget_bytes, const char *stage_token) { cbm_index_set_worker_role(is_worker, response_out); g_worker_memory_budget_bytes = is_worker ? memory_budget_bytes : 0; if (!is_worker) { @@ -76,6 +76,7 @@ void cbm_index_set_worker_role_options(bool is_worker, const char *response_out, worker_set_local_env("CBM_INDEX_SINGLE_THREAD", single_thread ? "1" : NULL); worker_set_local_env("CBM_INDEX_MARKER_FILE", marker_file); worker_set_local_env("CBM_INDEX_QUARANTINE_FILE", quarantine_file); + worker_set_local_env("CBM_INDEX_STAGE_TOKEN", stage_token); } bool cbm_index_worker_active(void) { @@ -161,6 +162,21 @@ static bool worker_fingerprint_valid(const char *fingerprint) { return true; } +static bool worker_stage_token_valid(const char *token) { + size_t length = token ? strlen(token) : 0; + if (length < 6U || length >= CBM_SZ_64) { + return false; + } + for (size_t index = 0; index < length; index++) { + unsigned char ch = (unsigned char)token[index]; + if (!((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || + ch == '-' || ch == '_')) { + return false; + } + } + return true; +} + static bool worker_parse_positive_size(const char *text, size_t *value_out) { if (!text || !text[0] || !value_out) { return false; @@ -238,6 +254,13 @@ cbm_index_worker_argv_status_t cbm_index_worker_parse_process_argv( parsed.quarantine_file = argv[next + 1]; next += 2; } + if (next < argc && argv[next] && strcmp(argv[next], CBM_INDEX_WORKER_STAGE_TOKEN_ARG) == 0) { + if (next + 1 >= argc || !worker_stage_token_valid(argv[next + 1])) { + return CBM_INDEX_WORKER_ARGV_INVALID; + } + parsed.stage_token = argv[next + 1]; + next += 2; + } if (next != argc) { return CBM_INDEX_WORKER_ARGV_INVALID; } @@ -386,12 +409,74 @@ static char *slurp_worker_response(const char *path, worker_response_read_status enum { INDEX_WORKER_PATH_CAP = CBM_SZ_4K, - INDEX_WORKER_ARGV_CAP = 17, + INDEX_WORKER_ARGV_CAP = 19, INDEX_WORKER_SYNC_POLL_NS = 10000000, INDEX_WORKER_RELAY_LINES_PER_POLL = 64, INDEX_WORKER_RELAY_BYTES_PER_POLL = 64 * 1024, + INDEX_WORKER_RSS_PROBE_FAILURE_LIMIT = 3, + INDEX_WORKER_RSS_PROBE_INTERVAL_MS = 250, + INDEX_WORKER_TASK_TEMP_PROBE_INTERVAL_MS = 250, }; +typedef enum { + INDEX_WORKER_TERMINATION_NONE = 0, + INDEX_WORKER_TERMINATION_CANCEL_PENDING, + INDEX_WORKER_TERMINATION_CANCEL, + INDEX_WORKER_TERMINATION_RESOURCE, +} index_worker_termination_reason_t; + +#ifdef CBM_ENABLE_TEST_SEAMS +static cbm_index_supervisor_clock_fn g_resource_clock; +static cbm_index_supervisor_rss_fn g_resource_rss; +static void *g_resource_hook_context; +static cbm_index_supervisor_task_temp_fn g_task_temp_hook; +static void *g_task_temp_hook_context; + +void cbm_index_supervisor_set_resource_hooks_for_testing(cbm_index_supervisor_clock_fn clock_fn, + cbm_index_supervisor_rss_fn rss_fn, + void *context) { + g_resource_clock = clock_fn; + g_resource_rss = rss_fn; + g_resource_hook_context = context; +} + +void cbm_index_supervisor_reset_resource_hooks_for_testing(void) { + g_resource_clock = NULL; + g_resource_rss = NULL; + g_resource_hook_context = NULL; +} + +void cbm_index_supervisor_set_task_temp_hook_for_testing( + cbm_index_supervisor_task_temp_fn task_temp_fn, void *context) { + g_task_temp_hook = task_temp_fn; + g_task_temp_hook_context = context; +} + +void cbm_index_supervisor_reset_task_temp_hook_for_testing(void) { + g_task_temp_hook = NULL; + g_task_temp_hook_context = NULL; +} +#endif + +static uint64_t worker_resource_now_ms(void) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_resource_clock) { + return g_resource_clock(g_resource_hook_context); + } +#endif + return cbm_now_ms(); +} + +static cbm_proc_tree_rss_status_t worker_resource_rss(cbm_subprocess_t *process, + uint64_t *rss_bytes) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_resource_rss) { + return g_resource_rss(process, rss_bytes, g_resource_hook_context); + } +#endif + return cbm_subprocess_tree_rss_bytes(process, rss_bytes); +} + struct cbm_index_worker_handle { cbm_subprocess_t *process; char response_path[INDEX_WORKER_PATH_CAP]; @@ -401,6 +486,16 @@ struct cbm_index_worker_handle { long relay_tail_pos; bool process_terminal; cbm_proc_result_t process_result; + cbm_index_resource_policy_t resource_policy; + char task_db_path[INDEX_WORKER_PATH_CAP]; + char stage_token[CBM_SZ_64]; + uint64_t started_ms; + uint64_t last_rss_probe_ms; + uint64_t last_task_temp_probe_ms; + unsigned int rss_probe_failures; + bool rss_probe_started; + bool task_temp_probe_started; + atomic_int termination_reason; atomic_bool terminal; cbm_index_worker_result_t result; }; @@ -578,7 +673,8 @@ static bool worker_unique_file(char *out, size_t out_size, const char *kind) { static bool worker_result_succeeded(const cbm_index_worker_result_t *result) { return result && result->outcome == CBM_PROC_CLEAN && !result->cancellation_requested && - result->tree_quiesced && !result->supervision_failed; + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed && result->tree_quiesced && !result->supervision_failed; } static void worker_terminal_log(cbm_index_worker_handle_t *handle) { @@ -594,6 +690,13 @@ static void worker_terminal_log(cbm_index_worker_handle_t *handle) { } else if (handle->result.supervision_failed || !handle->result.tree_quiesced) { cbm_log_error("index.supervisor.containment_failed", "outcome", cbm_proc_outcome_str(handle->result.outcome), "log", handle->log_path); + } else if (handle->result.resource_probe_failed) { + cbm_log_error("index.supervisor.resource_probe_failed", "resource", "rss_bytes", "log", + handle->log_path); + } else if (handle->result.resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { + cbm_log_warn("index.supervisor.resource_limit", "resource", + cbm_index_resource_name(handle->result.resource_violation.resource), "log", + handle->log_path); } else if (handle->result.cancellation_requested) { cbm_log_warn("index.supervisor.worker_cancelled", "outcome", cbm_proc_outcome_str(handle->result.outcome), "log", handle->log_path); @@ -608,10 +711,125 @@ static void worker_terminal_log(cbm_index_worker_handle_t *handle) { } } -int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_bytes, - bool single_thread, const char *marker_file, - const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, cbm_index_worker_handle_t **handle_out) { +static void worker_request_resource_termination(cbm_index_worker_handle_t *handle, + cbm_index_resource_t resource, uint64_t observed, + uint64_t limit, bool probe_failed) { + int expected = INDEX_WORKER_TERMINATION_NONE; + if (!atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + INDEX_WORKER_TERMINATION_RESOURCE, + memory_order_acq_rel, memory_order_acquire)) { + return; + } + handle->result.resource_violation = (cbm_index_resource_violation_t){ + .resource = resource, .observed = observed, .limit = limit, .probe_failed = probe_failed}; + handle->result.resource_probe_failed = probe_failed; + if (!cbm_subprocess_request_cancel(handle->process)) { + handle->result.resource_violation = (cbm_index_resource_violation_t){0}; + handle->result.resource_probe_failed = false; + expected = INDEX_WORKER_TERMINATION_RESOURCE; + (void)atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + INDEX_WORKER_TERMINATION_NONE, + memory_order_acq_rel, memory_order_acquire); + } +} + +static bool worker_check_resource_limits(cbm_index_worker_handle_t *handle) { + if (atomic_load_explicit(&handle->termination_reason, memory_order_acquire) != + INDEX_WORKER_TERMINATION_NONE || + cbm_subprocess_termination_pending(handle->process) || + !cbm_subprocess_supervision_active(handle->process)) { + return false; + } + uint64_t now = 0; + bool have_now = false; + bool rss_probe_failed = false; + if (handle->resource_policy.max_rss_bytes.enabled) { + now = worker_resource_now_ms(); + have_now = true; + bool probe_due = !handle->rss_probe_started || now < handle->last_rss_probe_ms || + now - handle->last_rss_probe_ms >= INDEX_WORKER_RSS_PROBE_INTERVAL_MS; + if (!probe_due) { + goto duration_check; + } + handle->rss_probe_started = true; + handle->last_rss_probe_ms = now; + uint64_t rss_bytes = 0; + cbm_proc_tree_rss_status_t status = worker_resource_rss(handle->process, &rss_bytes); + if (status == CBM_PROC_TREE_RSS_OK) { + handle->rss_probe_failures = 0; + if (rss_bytes > handle->resource_policy.max_rss_bytes.value) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_RSS_BYTES, rss_bytes, + handle->resource_policy.max_rss_bytes.value, + false); + return false; + } + } else if (status == CBM_PROC_TREE_RSS_ERROR) { + handle->rss_probe_failures++; + if (handle->rss_probe_failures >= INDEX_WORKER_RSS_PROBE_FAILURE_LIMIT) { + rss_probe_failed = true; + } + } else { + handle->rss_probe_failures = 0; + } + } +duration_check: + if (handle->resource_policy.max_duration_ms.enabled) { + if (!have_now) { + now = worker_resource_now_ms(); + } + uint64_t elapsed = now >= handle->started_ms ? now - handle->started_ms : 0; + if (elapsed > handle->resource_policy.max_duration_ms.value) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_DURATION_MS, elapsed, + handle->resource_policy.max_duration_ms.value, + false); + } + } + return rss_probe_failed; +} + +static bool worker_task_temp_bytes(cbm_index_worker_handle_t *handle, uint64_t *bytes_out) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_task_temp_hook) { + return g_task_temp_hook(handle->task_db_path, handle->log_path, handle->response_path, + bytes_out, g_task_temp_hook_context); + } +#endif + return cbm_index_task_temp_bytes(handle->task_db_path, handle->stage_token, handle->log_path, + handle->response_path, bytes_out); +} + +static void worker_check_task_temp_limit(cbm_index_worker_handle_t *handle) { + const cbm_index_limit_u64_t *limit = &handle->resource_policy.max_task_temp_bytes; + if (!limit->enabled || + atomic_load_explicit(&handle->termination_reason, memory_order_acquire) != + INDEX_WORKER_TERMINATION_NONE || + cbm_subprocess_termination_pending(handle->process) || + !cbm_subprocess_supervision_active(handle->process)) { + return; + } + uint64_t now = worker_resource_now_ms(); + bool probe_due = + !handle->task_temp_probe_started || now < handle->last_task_temp_probe_ms || + now - handle->last_task_temp_probe_ms >= INDEX_WORKER_TASK_TEMP_PROBE_INTERVAL_MS; + if (!probe_due) { + return; + } + handle->task_temp_probe_started = true; + handle->last_task_temp_probe_ms = now; + uint64_t observed = 0; + bool measured = worker_task_temp_bytes(handle, &observed); + if (!measured || observed > limit->value) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_TASK_TEMP_BYTES, observed, + limit->value, !measured); + } +} + +static int worker_start_internal(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + const char *task_db_path, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + cbm_index_worker_handle_t **handle_out) { if (handle_out) { *handle_out = NULL; } @@ -643,6 +861,20 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ if (!handle) { return -1; } + cbm_index_policy_init(&handle->resource_policy); + if (resource_policy) { + handle->resource_policy = *resource_policy; + } + if (handle->resource_policy.max_task_temp_bytes.enabled) { + int written = task_db_path ? snprintf(handle->task_db_path, sizeof(handle->task_db_path), + "%s", task_db_path) + : -1; + if (written <= 0 || written >= (int)sizeof(handle->task_db_path)) { + free(handle); + return -1; + } + } + atomic_init(&handle->termination_reason, INDEX_WORKER_TERMINATION_NONE); atomic_init(&handle->terminal, false); handle->log_callback = log_callback; handle->log_context = log_context; @@ -654,6 +886,17 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ free(handle); return -1; } + const char *random_suffix = strrchr(handle->response_path, '-'); + int token_written = random_suffix ? snprintf(handle->stage_token, sizeof(handle->stage_token), + "%ld%s", (long)worker_getpid(), random_suffix) + : -1; + if (token_written <= 0 || token_written >= (int)sizeof(handle->stage_token) || + !worker_stage_token_valid(handle->stage_token)) { + (void)cbm_unlink(handle->response_path); + (void)cbm_unlink(handle->log_path); + free(handle); + return -1; + } const char *argv[INDEX_WORKER_ARGV_CAP]; size_t argc = 0; @@ -683,6 +926,8 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ argv[argc++] = CBM_INDEX_WORKER_QUARANTINE_ARG; argv[argc++] = quarantine_file; } + argv[argc++] = CBM_INDEX_WORKER_STAGE_TOKEN_ARG; + argv[argc++] = handle->stage_token; argv[argc] = NULL; cbm_proc_opts_t options = {0}; @@ -703,15 +948,48 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ cbm_log_error("index.supervisor.spawn_failed", "action", "fail_closed"); return -1; } + if (handle->resource_policy.max_duration_ms.enabled) { + handle->started_ms = worker_resource_now_ms(); + } *handle_out = handle; return 0; } +int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_bytes, + bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, NULL, NULL, single_thread, + marker_file, quarantine_file, log_callback, log_context, + handle_out); +} + int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out) { - return cbm_index_worker_start_with_log(args_json, memory_budget_bytes, single_thread, - marker_file, quarantine_file, NULL, NULL, handle_out); + return worker_start_internal(args_json, memory_budget_bytes, NULL, NULL, single_thread, + marker_file, quarantine_file, NULL, NULL, handle_out); +} + +int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, + cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, resource_policy, NULL, + single_thread, marker_file, quarantine_file, NULL, NULL, + handle_out); +} + +int cbm_index_worker_start_with_storage_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + const char *task_db_path, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, resource_policy, task_db_path, + single_thread, marker_file, quarantine_file, log_callback, + log_context, handle_out); } cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, @@ -728,10 +1006,17 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, } bool relay_caught_up = true; if (!handle->process_terminal) { + bool rss_probe_failed = worker_check_resource_limits(handle); + worker_check_task_temp_limit(handle); cbm_proc_result_t process_result; cbm_proc_poll_t state = cbm_subprocess_poll(handle->process, &process_result); relay_caught_up = worker_relay_log(handle); if (state == CBM_PROC_POLL_RUNNING) { + if (rss_probe_failed && cbm_subprocess_root_running(handle->process)) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_RSS_BYTES, 0, + handle->resource_policy.max_rss_bytes.value, + true); + } return CBM_INDEX_WORKER_POLL_RUNNING; } if (state != CBM_PROC_POLL_TERMINAL) { @@ -752,7 +1037,12 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, handle->result.outcome = process_result->outcome; handle->result.exit_code = process_result->exit_code; handle->result.term_signal = process_result->term_signal; - handle->result.cancellation_requested = process_result->cancellation_requested; + int termination_reason = + atomic_load_explicit(&handle->termination_reason, memory_order_acquire); + handle->result.cancellation_requested = + termination_reason == INDEX_WORKER_TERMINATION_CANCEL || + (termination_reason == INDEX_WORKER_TERMINATION_CANCEL_PENDING && + process_result->cancellation_requested); handle->result.forced = process_result->forced; handle->result.tree_quiesced = process_result->tree_quiesced; handle->result.supervision_failed = process_result->supervision_failed; @@ -767,6 +1057,14 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, handle->result.exit_code = -1; } } + if (termination_reason == INDEX_WORKER_TERMINATION_RESOURCE && + handle->result.resource_violation.resource == CBM_INDEX_RESOURCE_TASK_TEMP_BYTES) { + if (!handle->result.tree_quiesced || handle->result.supervision_failed) { + cbm_log_warn("index.supervisor.staging_cleanup_deferred", "path", handle->task_db_path); + } else if (!cbm_index_staging_cleanup(handle->task_db_path, handle->stage_token)) { + cbm_log_warn("index.supervisor.staging_cleanup_failed", "path", handle->task_db_path); + } + } (void)cbm_unlink(handle->response_path); worker_terminal_log(handle); atomic_store_explicit(&handle->terminal, true, memory_order_release); @@ -775,8 +1073,36 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, } bool cbm_index_worker_request_cancel(cbm_index_worker_handle_t *handle) { - return handle && !atomic_load_explicit(&handle->terminal, memory_order_acquire) && - cbm_subprocess_request_cancel(handle->process); + if (!handle || atomic_load_explicit(&handle->terminal, memory_order_acquire)) { + return false; + } + int expected = INDEX_WORKER_TERMINATION_NONE; + bool claimed = atomic_compare_exchange_strong_explicit( + &handle->termination_reason, &expected, INDEX_WORKER_TERMINATION_CANCEL_PENDING, + memory_order_acq_rel, memory_order_acquire); + if (!claimed) { + if (expected == INDEX_WORKER_TERMINATION_CANCEL) { + return true; + } + if (expected != INDEX_WORKER_TERMINATION_CANCEL_PENDING) { + return false; + } + bool accepted = cbm_subprocess_request_cancel(handle->process); + if (accepted) { + expected = INDEX_WORKER_TERMINATION_CANCEL_PENDING; + (void)atomic_compare_exchange_strong_explicit( + &handle->termination_reason, &expected, INDEX_WORKER_TERMINATION_CANCEL, + memory_order_acq_rel, memory_order_acquire); + } + return accepted; + } + bool accepted = cbm_subprocess_request_cancel(handle->process); + expected = INDEX_WORKER_TERMINATION_CANCEL_PENDING; + (void)atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + accepted ? INDEX_WORKER_TERMINATION_CANCEL + : INDEX_WORKER_TERMINATION_NONE, + memory_order_acq_rel, memory_order_acquire); + return accepted; } const char *cbm_index_worker_response_path(const cbm_index_worker_handle_t *handle) { @@ -801,18 +1127,21 @@ void cbm_index_worker_destroy(cbm_index_worker_handle_t *handle) { free(handle); } -int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_thread, - const char *marker_file, const char *quarantine_file, - cbm_proc_log_cb log_callback, void *log_context, - const atomic_int *cancel_requested, - cbm_index_worker_result_t *result) { +static int worker_spawn_internal(const char *args_json, + const cbm_index_resource_policy_t *resource_policy, + const char *task_db_path, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + const atomic_int *cancel_requested, + cbm_index_worker_result_t *result) { if (!result) { return -1; } worker_result_init(result); cbm_index_worker_handle_t *handle = NULL; - if (cbm_index_worker_start_with_log(args_json, 0, single_thread, marker_file, quarantine_file, - log_callback, log_context, &handle) != 0) { + if (worker_start_internal(args_json, 0, resource_policy, task_db_path, single_thread, + marker_file, quarantine_file, log_callback, log_context, + &handle) != 0) { return -1; } const cbm_index_worker_result_t *cached = NULL; @@ -838,6 +1167,34 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th return 0; } +int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + const atomic_int *cancel_requested, + cbm_index_worker_result_t *result) { + return worker_spawn_internal(args_json, NULL, NULL, single_thread, marker_file, quarantine_file, + log_callback, log_context, cancel_requested, result); +} + +int cbm_index_spawn_worker_with_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, + const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result) { + return worker_spawn_internal(args_json, resource_policy, NULL, single_thread, marker_file, + quarantine_file, log_callback, log_context, cancel_requested, + result); +} + +int cbm_index_spawn_worker_with_storage_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, + const char *task_db_path, bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, + const atomic_int *cancel_requested, cbm_index_worker_result_t *result) { + return worker_spawn_internal(args_json, resource_policy, task_db_path, single_thread, + marker_file, quarantine_file, log_callback, log_context, + cancel_requested, result); +} + int cbm_index_spawn_worker_with_log(const char *args_json, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, diff --git a/src/mcp/index_supervisor.h b/src/mcp/index_supervisor.h index 31259df90..e8a3ef7db 100644 --- a/src/mcp/index_supervisor.h +++ b/src/mcp/index_supervisor.h @@ -22,8 +22,10 @@ #include #include +#include #include +#include "foundation/index_policy.h" #include "foundation/subprocess.h" /* cbm_proc_outcome_t */ /* Worker-role state, set once from the CLI arg parser (main.c) when this process @@ -40,11 +42,12 @@ void cbm_index_set_worker_role(bool is_worker, const char *response_out); #define CBM_INDEX_WORKER_QUARANTINE_ARG "--index-worker-quarantine" #define CBM_INDEX_WORKER_MEMORY_BUDGET_ARG "--index-worker-memory-budget-bytes" #define CBM_INDEX_WORKER_BUILD_ARG "--index-worker-build" +#define CBM_INDEX_WORKER_STAGE_TOKEN_ARG "--index-worker-stage-token" #define CBM_INDEX_WORKER_BUILD_FINGERPRINT_LENGTH 64U #define CBM_INDEX_WORKER_BUILD_FINGERPRINT_SIZE 65U void cbm_index_set_worker_role_options(bool is_worker, const char *response_out, bool single_thread, const char *marker_file, const char *quarantine_file, - size_t memory_budget_bytes); + size_t memory_budget_bytes, const char *stage_token); bool cbm_index_worker_active(void); const char *cbm_index_worker_response_out(void); size_t cbm_index_worker_memory_budget_bytes(void); @@ -84,6 +87,7 @@ typedef struct { const char *marker_file; const char *quarantine_file; size_t memory_budget_bytes; + const char *stage_token; } cbm_index_worker_invocation_t; typedef enum { @@ -138,8 +142,10 @@ typedef struct { bool tree_quiesced; bool supervision_failed; bool response_rejected; /* clean worker exceeded the bounded response protocol */ - char *response; /* worker result only after a contained, uncancelled CLEAN exit; - * borrowed for async polls, caller-owned from the sync wrapper */ + cbm_index_resource_violation_t resource_violation; + bool resource_probe_failed; + char *response; /* worker result only after a contained, uncancelled CLEAN exit; + * borrowed for async polls, caller-owned from the sync wrapper */ } cbm_index_worker_result_t; /* Daemon-owned, nonblocking supervisor for one contained worker process tree. */ @@ -157,6 +163,17 @@ typedef enum { int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out); +int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, + cbm_index_worker_handle_t **handle_out); +int cbm_index_worker_start_with_storage_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + const char *task_db_path, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + cbm_index_worker_handle_t **handle_out); /* Request-scoped variant used by interactive local CLI calls. The callback is * invoked by the owner thread while it polls the contained worker; log_context @@ -220,7 +237,33 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th cbm_proc_log_cb log_callback, void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result); +int cbm_index_spawn_worker_with_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, + const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result); +int cbm_index_spawn_worker_with_storage_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, + const char *task_db_path, bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, + const atomic_int *cancel_requested, cbm_index_worker_result_t *result); void cbm_index_worker_result_free(cbm_index_worker_result_t *result); +#ifdef CBM_ENABLE_TEST_SEAMS +typedef uint64_t (*cbm_index_supervisor_clock_fn)(void *context); +typedef cbm_proc_tree_rss_status_t (*cbm_index_supervisor_rss_fn)(cbm_subprocess_t *process, + uint64_t *rss_bytes, + void *context); +void cbm_index_supervisor_set_resource_hooks_for_testing(cbm_index_supervisor_clock_fn clock_fn, + cbm_index_supervisor_rss_fn rss_fn, + void *context); +void cbm_index_supervisor_reset_resource_hooks_for_testing(void); +typedef bool (*cbm_index_supervisor_task_temp_fn)(const char *final_db_path, const char *log_path, + const char *response_path, uint64_t *bytes_out, + void *context); +void cbm_index_supervisor_set_task_temp_hook_for_testing( + cbm_index_supervisor_task_temp_fn task_temp_fn, void *context); +void cbm_index_supervisor_reset_task_temp_hook_for_testing(void); +#endif + #endif /* CBM_INDEX_SUPERVISOR_H */ diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 618d04469..1a663ddb2 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -1652,7 +1652,17 @@ bool cbm_mcp_server_set_session_context(cbm_mcp_server_t *srv, const char *sessi return false; } - char *project = cbm_project_name_from_path(session_root); + /* Every tool handler normalizes the separators of the path it canonicalizes, + * so a root kept in the platform's native spelling is a second name for one + * directory. The two names meet when an explicit index_repository request is + * compared against the auto-index job started from this root: on Windows the + * backslash form never matched the request's slash form, and the duplicate + * was refused as an options conflict instead of joining the running job. */ + char normalized_root[sizeof(srv->session_root)]; + snprintf(normalized_root, sizeof(normalized_root), "%s", session_root); + cbm_normalize_path_sep(normalized_root); + + char *project = cbm_project_name_from_path(normalized_root); if (!project || project[0] == '\0' || strlen(project) >= sizeof(srv->session_project)) { free(project); return false; @@ -1663,8 +1673,9 @@ bool cbm_mcp_server_set_session_context(cbm_mcp_server_t *srv, const char *sessi free(project); return false; } + cbm_normalize_path_sep(allowed_copy); - snprintf(srv->session_root, sizeof(srv->session_root), "%s", session_root); + snprintf(srv->session_root, sizeof(srv->session_root), "%s", normalized_root); snprintf(srv->session_project, sizeof(srv->session_project), "%s", project); free(project); @@ -7543,10 +7554,14 @@ static bool build_index_success_response(cbm_mcp_server_t *srv, yyjson_mut_doc * return degraded; } +static bool project_db_is_servable(const char *project, const char *db_path); + /* Build the response for a worker that crashed/hung/failed without producing a * result. The crash is already contained (this process survived); we report it * rather than dying. Precise skip-and-continue (quarantine the culprit, index the * rest) is layered on in the probe stage. */ +static bool project_db_is_servable(const char *project, const char *db_path); + static char *build_worker_failure_response(const char *args, cbm_proc_outcome_t outcome) { char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); @@ -7608,6 +7623,59 @@ static char *build_worker_unsafe_terminal_response(const char *args, cbm_proc_ou return response; } +char *cbm_mcp_index_worker_resource_response(const char *args, + const cbm_index_worker_result_t *worker_result) { + if (!worker_result || worker_result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE) { + return NULL; + } + char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); + char *name_override = cbm_mcp_get_string_arg(args, "name"); + char *project_name = + cbm_project_name_from_path(name_override && name_override[0] ? name_override : repo_path); + char db_path[CBM_SZ_1K] = {0}; + if (project_name) { + project_db_path(project_name, db_path, sizeof(db_path)); + } + + const cbm_index_resource_violation_t *violation = &worker_result->resource_violation; + bool probe_failed = worker_result->resource_probe_failed || violation->probe_failed; + bool worker_stage = violation->resource == CBM_INDEX_RESOURCE_RSS_BYTES || + violation->resource == CBM_INDEX_RESOURCE_DURATION_MS; + const char *config_key = cbm_index_resource_config_key(violation->resource); + char message[CBM_SZ_256]; + (void)snprintf(message, sizeof(message), + probe_failed ? "Index resource measurement failed for %s" : "Index exceeded %s", + config_key); + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root); + yyjson_mut_obj_add_str(doc, root, "status", "error"); + yyjson_mut_obj_add_str(doc, root, "code", + probe_failed ? "resource_probe_failed" : "resource_limit_exceeded"); + yyjson_mut_obj_add_str(doc, root, "stage", worker_stage ? "worker" : "storage"); + yyjson_mut_obj_add_str(doc, root, "resource", cbm_index_resource_name(violation->resource)); + if (!probe_failed) { + yyjson_mut_obj_add_uint(doc, root, "observed", violation->observed); + yyjson_mut_obj_add_uint(doc, root, "limit", violation->limit); + yyjson_mut_obj_add_str(doc, root, "unit", cbm_index_resource_unit(violation->resource)); + } + yyjson_mut_obj_add_bool(doc, root, "retryable", true); + yyjson_mut_obj_add_bool(doc, root, "serving_index_preserved", + project_name && project_db_is_servable(project_name, db_path)); + yyjson_mut_obj_add_strcpy(doc, root, "message", message); + if (repo_path) { + yyjson_mut_obj_add_strcpy(doc, root, "repo_path", repo_path); + } + char *json = yy_doc_to_str(doc); + yyjson_mut_doc_free(doc); + free(project_name); + free(name_override); + free(repo_path); + char *response = cbm_mcp_text_result(json, true); + free(json); + return response; +} + /* Drop the cached store so the next query reopens whatever the worker wrote (each * worker is a fresh process that deletes + recreates the .db). NULL-safe: the * background watcher path (main.c) has no MCP server / cached store — the child @@ -7751,6 +7819,9 @@ cbm_mcp_supervised_result_disposition_t cbm_mcp_supervised_result_disposition( worker_result->supervision_failed) { return CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL; } + if (worker_result->resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { + return CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE; + } if (worker_result->outcome == CBM_PROC_CLEAN) { return worker_result->response ? CBM_MCP_SUPERVISED_RESULT_SUCCESS : CBM_MCP_SUPERVISED_RESULT_FALLBACK; @@ -7758,6 +7829,97 @@ cbm_mcp_supervised_result_disposition_t cbm_mcp_supervised_result_disposition( return CBM_MCP_SUPERVISED_RESULT_CONTAINED_FAILURE; } +bool cbm_mcp_index_policy_from_internal_args(const char *args, cbm_index_resource_policy_t *policy, + char *error, size_t error_size) { + yyjson_doc *doc = args ? yyjson_read(args, strlen(args), 0) : NULL; + yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; + yyjson_val *encoded = + root && yyjson_is_obj(root) ? yyjson_obj_get(root, "_cbm_index_policy") : NULL; + if (!encoded || !yyjson_is_obj(encoded) || + yyjson_obj_size(encoded) != cbm_index_policy_key_count() + 2U) { + yyjson_doc_free(doc); + (void)snprintf(error, error_size, "missing or incomplete trusted worker policy"); + return false; + } + + cbm_index_policy_init(policy); + yyjson_val *profile = yyjson_obj_get(encoded, CBM_INDEX_CONFIG_RESOURCE_PROFILE); + yyjson_val *override_mask = yyjson_obj_get(encoded, "_cbm_index_override_mask"); + bool valid = profile && yyjson_is_str(profile) && override_mask && + yyjson_is_uint(override_mask) && + cbm_index_policy_set_profile(policy, yyjson_get_str(profile), error, error_size); + for (size_t index = 0; valid && index < cbm_index_policy_key_count(); index++) { + const char *key = cbm_index_policy_key_at(index); + yyjson_val *value = yyjson_obj_get(encoded, key); + valid = value && yyjson_is_str(value) && + cbm_index_policy_set(policy, key, yyjson_get_str(value), error, error_size); + } + if (valid) { + uint64_t known_mask = cbm_index_policy_key_count() >= 64U + ? UINT64_MAX + : (UINT64_C(1) << cbm_index_policy_key_count()) - 1U; + uint64_t decoded_mask = yyjson_get_uint(override_mask); + valid = (decoded_mask & ~known_mask) == 0; + policy->override_mask = decoded_mask; + } + yyjson_doc_free(doc); + if (!valid && error && error_size > 0 && error[0] == '\0') { + (void)snprintf(error, error_size, "invalid trusted worker policy"); + } + return valid; +} + +bool cbm_mcp_index_task_db_path(const char *args, char *path_out, size_t path_size) { + if (!path_out || path_size == 0) { + return false; + } + path_out[0] = '\0'; + char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); + char *name_override = cbm_mcp_get_string_arg(args, "name"); + char *project = + cbm_project_name_from_path(name_override && name_override[0] ? name_override : repo_path); + char cache_path[CBM_SZ_1K]; + cache_dir(cache_path, sizeof(cache_path)); + int written = project ? snprintf(path_out, path_size, "%s/%s.db", cache_path, project) : -1; + free(project); + free(name_override); + free(repo_path); + return written > 0 && (size_t)written < path_size; +} + +static bool load_index_policy(cbm_mcp_server_t *srv, const char *args, + cbm_index_resource_policy_t *policy, char *error, size_t error_size) { + if (cbm_index_worker_active()) { + return cbm_mcp_index_policy_from_internal_args(args, policy, error, error_size); + } + cbm_config_t *owned_config = NULL; + cbm_config_t *config = srv ? srv->config : NULL; + if (!config) { + owned_config = cbm_config_open(cbm_resolve_cache_dir()); + config = owned_config; + } + bool loaded = cbm_config_load_index_policy(config, policy, error, error_size); + cbm_config_close(owned_config); + return loaded; +} + +bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, + const cbm_index_resource_policy_t *policy) { + yyjson_mut_val *encoded = yyjson_mut_obj(doc); + bool valid = encoded != NULL && + yyjson_mut_obj_add_strcpy(doc, encoded, CBM_INDEX_CONFIG_RESOURCE_PROFILE, + cbm_index_policy_profile_name(policy)) && + yyjson_mut_obj_add_uint(doc, encoded, "_cbm_index_override_mask", + policy ? policy->override_mask : 0); + for (size_t index = 0; valid && index < cbm_index_policy_key_count(); index++) { + const char *key = cbm_index_policy_key_at(index); + char value[CBM_SZ_64]; + valid = cbm_index_policy_format(policy, key, value, sizeof(value)) && + yyjson_mut_obj_add_strcpy(doc, encoded, key, value); + } + return valid && yyjson_mut_obj_add_val(doc, root, "_cbm_index_policy", encoded); +} + /* Run index_repository in a supervised worker subprocess with skip-and-continue * (Stage 3c). Returns the response string (caller frees): * - the worker's own response on a clean first run (the common path); @@ -7769,14 +7931,20 @@ cbm_mcp_supervised_result_disposition_t cbm_mcp_supervised_result_disposition( * - a contained-failure response only if even that cannot produce a clean run. * A physical CBM host never falls back to its in-process pipeline: an initial * start/protocol failure is returned as an explicit error response. */ -static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { +static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, + const cbm_index_resource_policy_t *resource_policy) { invalidate_cached_store(srv); + char task_db_path[CBM_SZ_1K]; + if (!resource_policy || !cbm_mcp_index_task_db_path(args, task_db_path, sizeof(task_db_path))) { + return cbm_mcp_text_result("could not resolve worker index path", true); + } /* First attempt: normal parallel run. */ cbm_index_worker_result_t wr; - int rc = cbm_index_spawn_worker_with_log_cancel( - args, false, NULL, NULL, srv ? srv->index_log_callback : NULL, - srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wr); + int rc = cbm_index_spawn_worker_with_storage_policy_log_cancel( + args, resource_policy, task_db_path, false, NULL, NULL, + srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, + srv ? &srv->pipeline_cancel_requested : NULL, &wr); cbm_mcp_supervised_result_disposition_t disposition = cbm_mcp_supervised_result_disposition(rc, &wr); @@ -7793,6 +7961,12 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { invalidate_cached_store(srv); return failure; } + if (disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + char *failure = cbm_mcp_index_worker_resource_response(args, &wr); + cbm_index_worker_result_free(&wr); + invalidate_cached_store(srv); + return failure; + } if (disposition == CBM_MCP_SUPERVISED_RESULT_SUCCESS) { /* Clean exit → transfer the worker's response (the common path). */ char *resp = wr.response; /* transfer ownership to caller (may be NULL) */ @@ -7801,7 +7975,6 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { invalidate_cached_store(srv); return resp; } - /* Crash / hang / nonzero exit → skip-and-continue recovery. Re-run the * worker PARALLEL (there are no sequential production runs) with the * per-file marker JOURNAL armed; after each failed run the journal's @@ -7849,10 +8022,11 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { bool terminal_cancelled = false; for (int i = 0; i < cap; i++) { cbm_index_worker_result_t wr2; - int rc2 = cbm_index_spawn_worker_with_log_cancel( - args, /*single_thread=*/false, marker_path, quarantine_path, - srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, - srv ? &srv->pipeline_cancel_requested : NULL, &wr2); + int rc2 = cbm_index_spawn_worker_with_storage_policy_log_cancel( + args, resource_policy, task_db_path, /*single_thread=*/false, marker_path, + quarantine_path, srv ? srv->index_log_callback : NULL, + srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, + &wr2); cbm_mcp_supervised_result_disposition_t recovery_disposition = cbm_mcp_supervised_result_disposition(rc2, &wr2); if (recovery_disposition == CBM_MCP_SUPERVISED_RESULT_FALLBACK) { @@ -7867,6 +8041,11 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { cbm_index_worker_result_free(&wr2); break; } + if (recovery_disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + resp = cbm_mcp_index_worker_resource_response(args, &wr2); + cbm_index_worker_result_free(&wr2); + break; + } if (recovery_disposition == CBM_MCP_SUPERVISED_RESULT_SUCCESS) { resp = wr2.response; /* transfer ownership to caller */ wr2.response = NULL; @@ -7940,8 +8119,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { * so it cannot itself hang. Rare given monotonic progress. */ if (!resp && !unsafe_terminal && quarantined > 0) { cbm_index_worker_result_t wrp; - int rcp = cbm_index_spawn_worker_with_log_cancel( - args, /*single_thread=*/false, NULL, quarantine_path, + int rcp = cbm_index_spawn_worker_with_storage_policy_log_cancel( + args, resource_policy, task_db_path, /*single_thread=*/false, NULL, quarantine_path, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wrp); cbm_mcp_supervised_result_disposition_t partial_disposition = @@ -7957,6 +8136,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { last_outcome = wrp.outcome; unsafe_terminal = true; terminal_cancelled = wrp.cancellation_requested; + } else if (partial_disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + resp = cbm_mcp_index_worker_resource_response(args, &wrp); } cbm_index_worker_result_free(&wrp); } @@ -7981,16 +8162,26 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p if (!root_path || !root_path[0]) { return NULL; } + cbm_index_resource_policy_t policy; + char policy_error[CBM_SZ_256] = {0}; + if (!load_index_policy(srv, NULL, &policy, policy_error, sizeof(policy_error))) { + cbm_log_error("index.policy", "error", policy_error); + return NULL; + } yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); yyjson_mut_val *root = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root); - yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path); + if (!yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path) || + !cbm_mcp_index_policy_add_to_args(doc, root, &policy)) { + yyjson_mut_doc_free(doc); + return NULL; + } char *args = yy_doc_to_str(doc); yyjson_mut_doc_free(doc); if (!args) { return NULL; } - char *resp = index_run_supervised(srv, args); + char *resp = index_run_supervised(srv, args, &policy); free(args); return resp; } @@ -8033,8 +8224,9 @@ static bool resolve_session_repo_path(cbm_mcp_server_t *srv, char **repo_path) { /* Preserve every index option while replacing all caller-supplied repo_path * keys with the one canonical path that was actually authorized. */ -static char *index_args_with_repo_path(const char *args, const char *canonical_repo_path) { - if (!args || !canonical_repo_path) { +static char *index_args_with_repo_path(const char *args, const char *canonical_repo_path, + const cbm_index_resource_policy_t *policy) { + if (!args || !canonical_repo_path || !policy) { return NULL; } yyjson_doc *source = yyjson_read(args, strlen(args), 0); @@ -8051,8 +8243,14 @@ static char *index_args_with_repo_path(const char *args, const char *canonical_r yyjson_mut_doc_free(copy); return NULL; } - (void)yyjson_mut_obj_remove_key(copy_root, "repo_path"); - if (!yyjson_mut_obj_add_strcpy(copy, copy_root, "repo_path", canonical_repo_path)) { + while (yyjson_mut_obj_get(copy_root, "repo_path")) { + (void)yyjson_mut_obj_remove_key(copy_root, "repo_path"); + } + while (yyjson_mut_obj_get(copy_root, "_cbm_index_policy")) { + (void)yyjson_mut_obj_remove_key(copy_root, "_cbm_index_policy"); + } + if (!yyjson_mut_obj_add_strcpy(copy, copy_root, "repo_path", canonical_repo_path) || + !cbm_mcp_index_policy_add_to_args(copy, copy_root, policy)) { yyjson_mut_doc_free(copy); return NULL; } @@ -8091,6 +8289,19 @@ static char *resolved_repo_path_from_project_arg(const char *args) { return root_path; } +static bool project_db_is_servable(const char *project, const char *db_path) { + cbm_store_t *store = db_path && db_path[0] ? cbm_store_open_path_query(db_path) : NULL; + if (!store) { + return false; + } + cbm_project_t stored_project = {0}; + bool servable = cbm_store_get_project(store, project, &stored_project) == CBM_STORE_OK && + stored_project.root_path && stored_project.root_path[0]; + cbm_project_free_fields(&stored_project); + cbm_store_close(store); + return servable; +} + static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); char *mode_str = cbm_mcp_get_string_arg(args, "mode"); @@ -8145,10 +8356,19 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { return result; } + cbm_index_resource_policy_t resource_policy; + char policy_error[CBM_SZ_256] = {0}; + if (!load_index_policy(srv, args, &resource_policy, policy_error, sizeof(policy_error))) { + free(mode_str); + free(name_override); + free(repo_path); + return cbm_mcp_text_result(policy_error, true); + } + /* A daemon session delegates the one physical write to its shared job * registry only after path canonicalization and workspace authorization. */ if (srv->index_executor) { - char *worker_args = index_args_with_repo_path(args, repo_path); + char *worker_args = index_args_with_repo_path(args, repo_path, &resource_policy); char *coordinated = worker_args ? srv->index_executor(srv->index_executor_context, repo_path, worker_args) : NULL; @@ -8180,7 +8400,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { * installs the same guard before running the in-process pipeline. A marked * host fails closed if preparation or worker startup cannot complete. */ if (cbm_index_supervisor_should_wrap()) { - char *worker_args = index_args_with_repo_path(args, repo_path); + char *worker_args = index_args_with_repo_path(args, repo_path, &resource_policy); if (!worker_args) { free(mutation_project); free(repo_path); @@ -8188,7 +8408,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { free(name_override); return cbm_mcp_text_result("failed to prepare supervised index request", true); } - char *supervised = index_run_supervised(srv, worker_args); + char *supervised = index_run_supervised(srv, worker_args, &resource_policy); free(worker_args); if (supervised) { free(mutation_project); @@ -8252,11 +8472,15 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { } free(name_override); cbm_pipeline_set_persistence(p, persistence); + cbm_pipeline_set_resource_policy(p, &resource_policy); char *project_name = heap_strdup(cbm_pipeline_project_name(p)); /* Bootstrap from artifact if no local DB exists */ try_artifact_bootstrap(project_name, repo_path); + char serving_db_path[CBM_SZ_1K]; + project_db_path(project_name, serving_db_path, sizeof(serving_db_path)); + bool serving_index_was_servable = project_db_is_servable(project_name, serving_db_path); /* Close cached store — pipeline will delete + recreate the .db file */ if (srv->owns_store && srv->store) { @@ -8288,6 +8512,8 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { cbm_file_error_t *file_errors = NULL; int file_error_count = 0; cbm_pipeline_get_file_errors(p, &file_errors, &file_error_count); + cbm_index_resource_violation_t resource_violation = {0}; + cbm_pipeline_get_resource_violation(p, &resource_violation); cbm_mem_collect(); /* return mimalloc pages to OS after large indexing */ @@ -8316,6 +8542,37 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { srv, doc, root, project_name, repo_path, persistence, p, excluded_dirs, excluded_count, file_errors, file_error_count, has_logfile ? logfile_path : NULL); yyjson_mut_obj_add_str(doc, root, "status", degraded ? "degraded" : "indexed"); + } else if (rc == CBM_PIPELINE_RESOURCE_LIMIT && + resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { + const char *config_key = cbm_index_resource_config_key(resource_violation.resource); + bool discovery_resource = + resource_violation.resource == CBM_INDEX_RESOURCE_FILES || + resource_violation.resource == CBM_INDEX_RESOURCE_SOURCE_BYTES || + resource_violation.resource == CBM_INDEX_RESOURCE_DIRECTORIES || + resource_violation.resource == CBM_INDEX_RESOURCE_ENTRIES || + resource_violation.resource == CBM_INDEX_RESOURCE_DEPTH || + resource_violation.resource == CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS; + char message[CBM_SZ_256]; + (void)snprintf(message, sizeof(message), + resource_violation.probe_failed ? "Index resource probe failed for %s" + : "Index exceeded %s", + config_key); + yyjson_mut_obj_add_str(doc, root, "status", "error"); + yyjson_mut_obj_add_str(doc, root, "code", + resource_violation.probe_failed ? "resource_probe_failed" + : "resource_limit_exceeded"); + yyjson_mut_obj_add_str(doc, root, "stage", discovery_resource ? "discovery" : "storage"); + yyjson_mut_obj_add_str(doc, root, "resource", + cbm_index_resource_name(resource_violation.resource)); + yyjson_mut_obj_add_uint(doc, root, "observed", resource_violation.observed); + yyjson_mut_obj_add_uint(doc, root, "limit", resource_violation.limit); + yyjson_mut_obj_add_str(doc, root, "unit", + cbm_index_resource_unit(resource_violation.resource)); + yyjson_mut_obj_add_bool(doc, root, "retryable", true); + yyjson_mut_obj_add_bool(doc, root, "serving_index_preserved", + serving_index_was_servable && + project_db_is_servable(project_name, serving_db_path)); + yyjson_mut_obj_add_strcpy(doc, root, "message", message); } else { yyjson_mut_obj_add_str(doc, root, "status", "error"); yyjson_mut_obj_add_str(doc, root, "hint", @@ -11543,11 +11800,18 @@ static void *autoindex_thread(void *arg) { return NULL; } + cbm_index_resource_policy_t resource_policy; + char policy_error[CBM_SZ_256] = {0}; + if (!load_index_policy(srv, NULL, &resource_policy, policy_error, sizeof(policy_error))) { + cbm_log_warn("autoindex.err", "msg", "resource_policy_load_failed", "error", policy_error); + return NULL; + } cbm_pipeline_t *p = cbm_pipeline_new(srv->session_root, NULL, CBM_MODE_FULL); if (!p) { cbm_log_warn("autoindex.err", "msg", "pipeline_create_failed"); return NULL; } + cbm_pipeline_set_resource_policy(p, &resource_policy); /* Block until any concurrent pipeline finishes */ cbm_pipeline_lock(); diff --git a/src/mcp/mcp.h b/src/mcp/mcp.h index a70b6372b..320f3dda0 100644 --- a/src/mcp/mcp.h +++ b/src/mcp/mcp.h @@ -221,6 +221,7 @@ char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const ch typedef enum { CBM_MCP_SUPERVISED_RESULT_FALLBACK = 0, CBM_MCP_SUPERVISED_RESULT_SUCCESS, + CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE, CBM_MCP_SUPERVISED_RESULT_CONTAINED_FAILURE, CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL, } cbm_mcp_supervised_result_disposition_t; diff --git a/src/mcp/mcp_internal.h b/src/mcp/mcp_internal.h index d9321c595..b42b43315 100644 --- a/src/mcp/mcp_internal.h +++ b/src/mcp/mcp_internal.h @@ -1,10 +1,13 @@ #ifndef CBM_MCP_INTERNAL_H #define CBM_MCP_INTERNAL_H +#include "foundation/index_policy.h" #include "mcp/mcp.h" #include "pipeline/pipeline.h" /* cbm_changed_hunk_t */ #include "store/store.h" /* cbm_node_t */ +#include + /* White-box fault injection for deterministic cross-platform quarantine * safety tests. This header is internal and is not part of the MCP API. */ typedef bool (*cbm_mcp_quarantine_test_hook_fn)(void *context, const char *step); @@ -25,6 +28,16 @@ bool cbm_mcp_server_release_pristine_memory_store(cbm_mcp_server_t *srv); * On success replaces and frees *response_io; on failure it is unchanged. */ bool cbm_mcp_jsonrpc_response_prepend_notice(char **response_io, const char *notice); +/* Encode the complete trusted policy on an internal worker request. Callers + * must remove any untrusted field with the same name before invoking this. */ +bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, + const cbm_index_resource_policy_t *policy); +bool cbm_mcp_index_policy_from_internal_args(const char *args, cbm_index_resource_policy_t *policy, + char *error, size_t error_size); +bool cbm_mcp_index_task_db_path(const char *args, char *path_out, size_t path_size); +char *cbm_mcp_index_worker_resource_response(const char *args, + const cbm_index_worker_result_t *worker_result); + enum { CBM_MCP_DEFAULT_AUTO_INDEX_LIMIT = 50000 }; /* Count indexable files with the pipeline's native full-mode discovery policy, diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 65ac75183..0e479558c 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -156,6 +156,12 @@ struct cbm_pipeline { atomic_int cancelled_storage; atomic_int *cancelled; bool persistence; /* write .codebase-memory/graph.db.zst after indexing */ + cbm_index_resource_policy_t resource_policy; + cbm_index_resource_violation_t resource_violation; +#ifdef CBM_ENABLE_TEST_SEAMS + cbm_pipeline_storage_probe_fn storage_probe; + void *storage_probe_context; +#endif /* Indexing state (set during run) */ cbm_gbuf_t *gbuf; @@ -304,6 +310,31 @@ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled) { } } +void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, + const cbm_index_resource_policy_t *policy) { + if (p && policy) { + p->resource_policy = *policy; + } +} + +void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p, + cbm_index_resource_violation_t *violation) { + if (violation) { + *violation = p ? p->resource_violation : (cbm_index_resource_violation_t){0}; + } +} + +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_pipeline_set_storage_probe_for_testing(cbm_pipeline_t *p, + cbm_pipeline_storage_probe_fn probe, + void *context) { + if (p) { + p->storage_probe = probe; + p->storage_probe_context = context; + } +} +#endif + bool cbm_pipeline_set_project_name(cbm_pipeline_t *p, const char *name) { if (!p || !name || !name[0]) { return false; @@ -413,6 +444,15 @@ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p) { return p ? p->repo_path : NULL; } +const cbm_index_resource_policy_t *cbm_pipeline_resource_policy(const cbm_pipeline_t *p) { + return p && cbm_index_policy_discovery_enabled(&p->resource_policy) ? &p->resource_policy + : NULL; +} + +cbm_index_resource_violation_t *cbm_pipeline_resource_violation(cbm_pipeline_t *p) { + return p ? &p->resource_violation : NULL; +} + atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p) { return p ? p->cancelled : NULL; } @@ -1671,7 +1711,6 @@ int cbm_pipeline_publish_generation(const cbm_pipeline_generation_t *generation) if (generation->cancelled && atomic_load(generation->cancelled)) { return CBM_PIPELINE_ABORT_PRESERVE_DB; } - /* The staging name must be unpredictable and created exclusively. It used * to be ".stage..", which any other process can compute * in advance; this path is then unlinked and written, so in a @@ -1916,15 +1955,17 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas #if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API cbm_pipeline_persist_test_run_before_final_manifest(); #endif - if (cbm_pipeline_build_fresh_semantic_manifest(p->project_name, p->repo_path, p->mode, - &manifest, &manifest_count) != 0) { + int manifest_rc = + cbm_pipeline_build_fresh_semantic_manifest(p, p->project_name, &manifest, &manifest_count); + if (manifest_rc != 0) { cbm_log_error("pipeline.err", "phase", "semantic_manifest"); /* db_path and db_dir are this function's strdups; the success tail and * the publish-failure return release them, and these two aborts must * too -- LSan caught exactly these paths leaking both strings. */ free(db_dir); free(db_path); - return CBM_PIPELINE_ABORT_PRESERVE_DB; + return manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT + : CBM_PIPELINE_ABORT_PRESERVE_DB; } if (!cbm_pipeline_semantic_manifests_equal(baseline_manifest, baseline_count, manifest, manifest_count)) { @@ -2178,10 +2219,14 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { /* Phase 1: Discover files */ CBM_PROF_START(t_discover); + p->resource_violation = (cbm_index_resource_violation_t){0}; cbm_discover_opts_t opts = { .mode = p->requested_mode, .ignore_file = NULL, .max_file_size = 0, + .resource_policy = + cbm_index_policy_discovery_enabled(&p->resource_policy) ? &p->resource_policy : NULL, + .resource_violation = &p->resource_violation, }; cbm_file_info_t *files = NULL; int file_count = 0; @@ -2206,7 +2251,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { cbm_log_info("pipeline.discover", "files", itoa_buf(file_count), "elapsed_ms", itoa_buf((int)elapsed_ms(t0))); if (rc != 0 || check_cancel(p)) { - rc = CBM_NOT_FOUND; + rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT : CBM_NOT_FOUND; goto cleanup; } @@ -2214,14 +2259,15 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { * bytes drive exact no-op comparison and are checked against a fresh * rediscovery immediately before any replacement is published. */ rc = mode_promoted - ? cbm_pipeline_build_fresh_semantic_manifest(p->project_name, p->repo_path, p->mode, - &baseline_manifest, &baseline_count) + ? cbm_pipeline_build_fresh_semantic_manifest(p, p->project_name, &baseline_manifest, + &baseline_count) : cbm_pipeline_build_semantic_manifest(p->project_name, p->repo_path, files, file_count, p->excluded_dirs, p->excluded_count, &p->git_ctx, p->userconfig, &baseline_manifest, &baseline_count); if (rc != 0) { - rc = CBM_PIPELINE_ABORT_PRESERVE_DB; + rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT + : CBM_PIPELINE_ABORT_PRESERVE_DB; goto cleanup; } @@ -2265,7 +2311,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { cbm_log_info("pipeline.rediscover", "requested_mode", pipeline_mode_name(p->requested_mode), "effective_mode", pipeline_mode_name(p->mode), "files", itoa_buf(file_count)); if (rc != 0 || check_cancel(p)) { - rc = CBM_NOT_FOUND; + rc = rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT : CBM_NOT_FOUND; goto cleanup; } } @@ -2334,6 +2380,216 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { return rc; } +static const cbm_index_limit_u64_t *storage_limit_for_resource( + const cbm_index_resource_policy_t *policy, cbm_index_resource_t resource) { + if (!policy) { + return NULL; + } + switch (resource) { + case CBM_INDEX_RESOURCE_CACHE_BYTES: + return &policy->max_cache_bytes; + case CBM_INDEX_RESOURCE_FREE_DISK_BYTES: + return &policy->min_free_disk_bytes; + case CBM_INDEX_RESOURCE_FINAL_DB_BYTES: + return &policy->max_final_db_bytes; + case CBM_INDEX_RESOURCE_STAGING_BYTES: + return &policy->max_staging_bytes; + case CBM_INDEX_RESOURCE_TASK_TEMP_BYTES: + return &policy->max_task_temp_bytes; + default: + return NULL; + } +} + +static cbm_index_resource_t storage_enabled_probe_resource( + const cbm_index_resource_policy_t *policy) { + if (policy->max_cache_bytes.enabled) { + return CBM_INDEX_RESOURCE_CACHE_BYTES; + } + if (policy->min_free_disk_bytes.enabled) { + return CBM_INDEX_RESOURCE_FREE_DISK_BYTES; + } + if (policy->max_staging_bytes.enabled) { + return CBM_INDEX_RESOURCE_STAGING_BYTES; + } + if (policy->max_task_temp_bytes.enabled) { + return CBM_INDEX_RESOURCE_TASK_TEMP_BYTES; + } + return policy->max_final_db_bytes.enabled ? CBM_INDEX_RESOURCE_FINAL_DB_BYTES + : CBM_INDEX_RESOURCE_NONE; +} + +static char *storage_root_from_final_path(const char *final_path) { + if (!final_path) { + return NULL; + } + char *root = strdup(final_path); + if (!root) { + return NULL; + } + char *slash = strrchr(root, '/'); +#ifdef _WIN32 + char *backslash = strrchr(root, '\\'); + if (backslash && (!slash || backslash > slash)) { + slash = backslash; + } +#endif + if (!slash) { + free(root); + return strdup("."); + } + if (slash == root) { + slash[1] = '\0'; + } else { + *slash = '\0'; + } + return root; +} + +static bool storage_regular_file_bytes(const char *path, uint64_t *bytes_out) { + if (!path || !bytes_out) { + return false; + } + cbm_path_info_t info; + cbm_path_probe_status_t status = cbm_path_probe_info_utf8(path, &info); + if (status == CBM_PATH_PROBE_NOT_FOUND) { + *bytes_out = 0; + return true; + } + if (status != CBM_PATH_PROBE_OK || !info.is_regular || info.is_symlink || info.size < 0) { + return false; + } + *bytes_out = (uint64_t)info.size; + return true; +} + +static bool storage_replaceable_old_bytes(const char *final_path, uint64_t *bytes_out) { + if (!final_path || !bytes_out) { + return false; + } + *bytes_out = 0; + cbm_path_info_t info; + cbm_path_probe_status_t status = cbm_path_probe_info_utf8(final_path, &info); + if (status == CBM_PATH_PROBE_NOT_FOUND) { + return true; + } + if (status != CBM_PATH_PROBE_OK || !info.is_regular || info.is_symlink) { + return false; + } + cbm_store_t *store = cbm_store_open_path_query(final_path); + if (!store) { + /* A file that cannot be opened is not proven replaceable. Keep its + * bytes in the projection rather than blocking repair of a confirmed + * corrupt/non-database destination. */ + return true; + } + cbm_integrity_verdict_t verdict = cbm_store_check_integrity_verdict(store); + cbm_store_close(store); + if (verdict == CBM_INTEGRITY_TRANSIENT) { + return false; + } + /* A generation that cannot be proven healthy remains in current_cache but is + * not deducted: publication may quarantine it instead of replacing it. */ + return verdict != CBM_INTEGRITY_OK || cbm_db_artifact_bytes(final_path, bytes_out); +} + +static bool storage_probe_default(const cbm_index_resource_policy_t *policy, + cbm_pipeline_storage_checkpoint_t checkpoint, + const char *final_path, const char *staging_path, + cbm_index_storage_sample_t *sample, + cbm_index_resource_t *failed_resource) { + *sample = (cbm_index_storage_sample_t){0}; + *failed_resource = CBM_INDEX_RESOURCE_NONE; + char *storage_root = NULL; + if (policy->max_cache_bytes.enabled || policy->min_free_disk_bytes.enabled) { + storage_root = storage_root_from_final_path(final_path); + if (!storage_root) { + *failed_resource = storage_enabled_probe_resource(policy); + return false; + } + } + uint64_t operation_bytes = 0; + bool need_operation = + staging_path && + (policy->max_cache_bytes.enabled || policy->max_staging_bytes.enabled || + policy->max_task_temp_bytes.enabled || + (checkpoint == CBM_PIPELINE_STORAGE_PREPUBLISH && policy->max_final_db_bytes.enabled)); + if (need_operation && !cbm_db_artifact_bytes(staging_path, &operation_bytes)) { + *failed_resource = + policy->max_staging_bytes.enabled + ? CBM_INDEX_RESOURCE_STAGING_BYTES + : (policy->max_task_temp_bytes.enabled + ? CBM_INDEX_RESOURCE_TASK_TEMP_BYTES + : (policy->max_final_db_bytes.enabled ? CBM_INDEX_RESOURCE_FINAL_DB_BYTES + : CBM_INDEX_RESOURCE_CACHE_BYTES)); + free(storage_root); + return false; + } + if (policy->max_cache_bytes.enabled) { + uint64_t cache_with_operation = 0; + if (!cbm_directory_size_bytes(storage_root, &cache_with_operation) || + !storage_replaceable_old_bytes(final_path, &sample->replaceable_old_bytes)) { + *failed_resource = CBM_INDEX_RESOURCE_CACHE_BYTES; + free(storage_root); + return false; + } + sample->current_cache_bytes = + cache_with_operation > operation_bytes ? cache_with_operation - operation_bytes : 0; + sample->operation_bytes = operation_bytes; + } + if (policy->min_free_disk_bytes.enabled && + !cbm_filesystem_free_bytes(storage_root, &sample->free_disk_bytes)) { + *failed_resource = CBM_INDEX_RESOURCE_FREE_DISK_BYTES; + free(storage_root); + return false; + } + free(storage_root); + if (checkpoint != CBM_PIPELINE_STORAGE_PREFLIGHT) { + sample->staging_bytes = operation_bytes; + sample->task_temp_bytes = operation_bytes; + } + if (checkpoint == CBM_PIPELINE_STORAGE_PREPUBLISH && policy->max_final_db_bytes.enabled && + !storage_regular_file_bytes(staging_path, &sample->final_db_bytes)) { + *failed_resource = CBM_INDEX_RESOURCE_FINAL_DB_BYTES; + return false; + } + return true; +} + +int cbm_pipeline_storage_admit(cbm_pipeline_t *p, cbm_pipeline_storage_checkpoint_t checkpoint, + const char *final_path, const char *staging_path) { + if (!p || !cbm_index_policy_storage_enabled(&p->resource_policy)) { + return 0; + } + cbm_index_storage_sample_t sample; + cbm_index_resource_t failed_resource = CBM_INDEX_RESOURCE_NONE; + bool measured; +#ifdef CBM_ENABLE_TEST_SEAMS + if (p->storage_probe) { + measured = p->storage_probe(checkpoint, final_path, staging_path, &sample, &failed_resource, + p->storage_probe_context); + } else +#endif + { + measured = storage_probe_default(&p->resource_policy, checkpoint, final_path, staging_path, + &sample, &failed_resource); + } + if (!measured) { + const cbm_index_limit_u64_t *limit = + storage_limit_for_resource(&p->resource_policy, failed_resource); + p->resource_violation = (cbm_index_resource_violation_t){ + .resource = failed_resource, + .limit = limit && limit->enabled ? limit->value : 0, + .probe_failed = true, + }; + return CBM_PIPELINE_RESOURCE_LIMIT; + } + if (!cbm_index_policy_check_storage(&p->resource_policy, &sample, &p->resource_violation)) { + return CBM_PIPELINE_RESOURCE_LIMIT; + } + return 0; +} + static void cleanup_staging_db(const char *path) { if (!path) { return; @@ -2367,16 +2623,41 @@ static bool ensure_db_parent(const char *path) { return ok; } +static bool staging_token_valid(const char *token) { + size_t length = token ? strlen(token) : 0; + if (length < 6U || length >= CBM_SZ_64) { + return false; + } + for (size_t index = 0; index < length; index++) { + unsigned char ch = (unsigned char)token[index]; + if (!((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || + ch == '-' || ch == '_')) { + return false; + } + } + return true; +} + static char *create_staging_path(const char *final_path) { if (!final_path) { return NULL; } - static const char suffix[] = ".stage.XXXXXX"; + char suffix[CBM_SZ_128]; + const char *stage_token = getenv("CBM_INDEX_STAGE_TOKEN"); + if (!staging_token_valid(stage_token)) { + stage_token = NULL; + } + int suffix_length = stage_token && stage_token[0] + ? snprintf(suffix, sizeof(suffix), ".stage.%s.XXXXXX", stage_token) + : snprintf(suffix, sizeof(suffix), ".stage.XXXXXX"); + if (suffix_length <= 0 || (size_t)suffix_length >= sizeof(suffix)) { + return NULL; + } size_t final_len = strlen(final_path); - if (final_len > SIZE_MAX - sizeof(suffix)) { + if (final_len > SIZE_MAX - (size_t)suffix_length - 1U) { return NULL; } - size_t path_size = final_len + sizeof(suffix); + size_t path_size = final_len + (size_t)suffix_length + 1U; #ifdef _WIN32 /* The Windows cbm_mkstemp compatibility contract may expand a /tmp/ * prefix in-place and copies through a 4 KiB scratch path. Give it that @@ -2391,7 +2672,7 @@ static char *create_staging_path(const char *final_path) { return NULL; } memcpy(path, final_path, final_len); - memcpy(path + final_len, suffix, sizeof(suffix)); + memcpy(path + final_len, suffix, (size_t)suffix_length + 1U); int fd = cbm_mkstemp(path); if (fd < 0) { free(path); @@ -2510,11 +2791,18 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { if (!p) { return CBM_NOT_FOUND; } + p->resource_violation = (cbm_index_resource_violation_t){0}; char *final_path = resolve_db_path(p); if (!final_path || !ensure_db_parent(final_path)) { free(final_path); return CBM_NOT_FOUND; } + int storage_rc = + cbm_pipeline_storage_admit(p, CBM_PIPELINE_STORAGE_PREFLIGHT, final_path, NULL); + if (storage_rc != 0) { + free(final_path); + return storage_rc; + } struct stat final_st; bool final_existed = stat(final_path, &final_st) == 0; char *staging_path = create_staging_path(final_path); @@ -2557,6 +2845,14 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { free(final_path); return rc; } + storage_rc = + cbm_pipeline_storage_admit(p, CBM_PIPELINE_STORAGE_GROWTH, final_path, staging_path); + if (storage_rc != 0) { + cleanup_staging_db(staging_path); + free(staging_path); + free(final_path); + return storage_rc; + } if (check_cancel(p)) { cleanup_staging_db(staging_path); free(staging_path); @@ -2588,6 +2884,14 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { free(final_path); return CBM_PIPELINE_PERSIST_FAILED; } + storage_rc = + cbm_pipeline_storage_admit(p, CBM_PIPELINE_STORAGE_PREPUBLISH, final_path, staging_path); + if (storage_rc != 0) { + cleanup_staging_db(staging_path); + free(staging_path); + free(final_path); + return storage_rc; + } cbm_replacement_prepare_t prepared = {0}; int prepare_rc = diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 5cde0c0f0..2502846b7 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -30,6 +30,8 @@ typedef struct cbm_gbuf cbm_gbuf_t; typedef struct cbm_pipeline cbm_pipeline_t; +#define CBM_PIPELINE_RESOURCE_LIMIT (-5) + /* ── Index mode ─────────────────────────────────────────────────── */ #ifndef CBM_INDEX_MODE_T_DEFINED @@ -53,6 +55,28 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path, cbm * When enabled, the pipeline writes a compressed artifact after indexing. */ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled); +/* Apply a validated discovery resource policy. The value is copied. */ +void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, const cbm_index_resource_policy_t *policy); + +/* Copy the exact discovery violation from the most recent run. */ +void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p, + cbm_index_resource_violation_t *violation); + +typedef enum { + CBM_PIPELINE_STORAGE_PREFLIGHT = 0, + CBM_PIPELINE_STORAGE_GROWTH, + CBM_PIPELINE_STORAGE_PREPUBLISH, +} cbm_pipeline_storage_checkpoint_t; +#ifdef CBM_ENABLE_TEST_SEAMS +typedef bool (*cbm_pipeline_storage_probe_fn)(cbm_pipeline_storage_checkpoint_t checkpoint, + const char *final_db_path, + const char *staging_db_path, + cbm_index_storage_sample_t *sample, + cbm_index_resource_t *failed_resource, void *context); +void cbm_pipeline_set_storage_probe_for_testing(cbm_pipeline_t *p, + cbm_pipeline_storage_probe_fn probe, void *context); +#endif + /* Free a pipeline and all its internal state. NULL-safe. */ void cbm_pipeline_free(cbm_pipeline_t *p); diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index d3c5247f5..d0db3fbd0 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -609,17 +609,20 @@ bool cbm_pipeline_semantic_manifests_equal(const cbm_file_hash_t *left, int left return equal; } -int cbm_pipeline_build_fresh_semantic_manifest(const char *project, const char *repo_path, int mode, +int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *project, cbm_file_hash_t **out, int *out_count) { - if (!project || !repo_path || !out || !out_count) { + const char *repo_path = cbm_pipeline_repo_path(p); + if (!p || !project || !repo_path || !out || !out_count) { return CBM_NOT_FOUND; } *out = NULL; *out_count = 0; cbm_discover_opts_t opts = { - .mode = (cbm_index_mode_t)mode, + .mode = (cbm_index_mode_t)cbm_pipeline_get_mode(p), .ignore_file = NULL, .max_file_size = 0, + .resource_policy = cbm_pipeline_resource_policy(p), + .resource_violation = cbm_pipeline_resource_violation(p), }; cbm_file_info_t *fresh_files = NULL; int fresh_file_count = 0; @@ -2224,13 +2227,13 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char #if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API cbm_pipeline_persist_test_run_before_final_manifest(); #endif - if (cbm_pipeline_build_fresh_semantic_manifest(project, cbm_pipeline_repo_path(p), - cbm_pipeline_get_mode(p), &manifest, - &manifest_count) != 0 || - !cbm_pipeline_semantic_manifests_equal(baseline_manifest, baseline_count, manifest, - manifest_count)) { + int manifest_rc = + cbm_pipeline_build_fresh_semantic_manifest(p, project, &manifest, &manifest_count); + if (manifest_rc != 0 || !cbm_pipeline_semantic_manifests_equal( + baseline_manifest, baseline_count, manifest, manifest_count)) { cbm_log_warn("delta.abort", "reason", "semantic_inputs_changed"); - result = CBM_PIPELINE_ABORT_PRESERVE_DB; + result = manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT + : CBM_PIPELINE_ABORT_PRESERVE_DB; goto out; } @@ -2816,8 +2819,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil #if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API cbm_pipeline_persist_test_run_before_final_manifest(); #endif - int manifest_rc = cbm_pipeline_build_fresh_semantic_manifest( - project, cbm_pipeline_repo_path(p), cbm_pipeline_get_mode(p), &manifest, &manifest_count); + int manifest_rc = + cbm_pipeline_build_fresh_semantic_manifest(p, project, &manifest, &manifest_count); if (manifest_rc != 0 || !cbm_pipeline_semantic_manifests_equal( baseline_manifest, baseline_count, manifest, manifest_count)) { cbm_log_warn("incremental.abort", "reason", "semantic_inputs_changed"); @@ -2827,7 +2830,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil free_mode_skipped(mode_skipped, mode_skipped_count); free(saved_adr); cbm_gbuf_free(existing); - return CBM_PIPELINE_ABORT_PRESERVE_DB; + return manifest_rc == CBM_DISCOVER_LIMIT_EXCEEDED ? CBM_PIPELINE_RESOURCE_LIMIT + : CBM_PIPELINE_ABORT_PRESERVE_DB; } /* Step 7: atomically publish the complete staged generation. */ diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 754fa4ffa..eac94ac94 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -678,7 +678,7 @@ bool cbm_pipeline_semantic_manifests_equal(const cbm_file_hash_t *left, int left const cbm_file_hash_t *right, int right_count); /* Re-run discovery and hash its exact semantic inputs. Used at the publication * boundary so late additions/deletions cannot escape a frozen file list. */ -int cbm_pipeline_build_fresh_semantic_manifest(const char *project, const char *repo_path, int mode, +int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *project, cbm_file_hash_t **out, int *out_count); /* Compatibility contract persisted in coverage metadata. Increment when a @@ -708,6 +708,9 @@ typedef struct { bool surfaces_in_place; } cbm_pipeline_generation_t; +int cbm_pipeline_storage_admit(cbm_pipeline_t *p, cbm_pipeline_storage_checkpoint_t checkpoint, + const char *final_path, const char *staging_path); + /* Serialize and fully populate a sibling staging database, then atomically * replace final_db_path. The old generation is untouched on every failure or * cancellation observed before the rename commit point. */ @@ -760,6 +763,8 @@ void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *row /* Pipeline accessors for incremental use */ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p); +const cbm_index_resource_policy_t *cbm_pipeline_resource_policy(const cbm_pipeline_t *p); +cbm_index_resource_violation_t *cbm_pipeline_resource_violation(cbm_pipeline_t *p); atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p); /* Record committed graph size (#334 gate axis) from the incremental path, * which cannot see the opaque cbm_pipeline struct. Call before the dump. */ diff --git a/tests/test_daemon.c b/tests/test_daemon.c index fb0d0095a..0edbe19eb 100644 --- a/tests/test_daemon.c +++ b/tests/test_daemon.c @@ -467,6 +467,25 @@ TEST(daemon_sessions_keep_distinct_roots_and_allowed_root_policy) { PASS(); } +/* A daemon canonicalizes the client's root through the platform API, which on + * Windows answers in backslash form, while every tool handler normalizes the + * paths it canonicalizes. Keeping the native spelling gave one directory two + * names, and an explicit index_repository request was then refused as an + * options conflict instead of joining the auto-index job already running for + * that root. The separators are folded on every platform, so this holds + * wherever the test runs. */ +TEST(daemon_session_context_keeps_one_spelling_of_a_root) { + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + + ASSERT_TRUE(cbm_mcp_server_set_session_context(srv, "C:\\repos\\cbm", "C:\\repos")); + ASSERT_STR_EQ(cbm_mcp_server_session_root(srv), "C:/repos/cbm"); + ASSERT_STR_EQ(cbm_mcp_server_allowed_root(srv), "C:/repos"); + + cbm_mcp_server_free(srv); + PASS(); +} + SUITE(daemon) { RUN_TEST(daemon_client_ids_are_connection_bound); RUN_TEST(daemon_shared_job_survives_until_final_subscriber_disconnects); @@ -480,4 +499,5 @@ SUITE(daemon) { RUN_TEST(daemon_bridge_rejects_embedded_nul_body); RUN_TEST(daemon_bridge_rejects_oversized_headers); RUN_TEST(daemon_sessions_keep_distinct_roots_and_allowed_root_policy); + RUN_TEST(daemon_session_context_keeps_one_spelling_of_a_root); } diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index c639ace2f..e715c46c1 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1308,6 +1308,7 @@ TEST(daemon_application_prune_clears_logical_watch_for_reregistration) { } enum { APP_FAKE_MAX_ATTEMPTS = 16 }; +enum { APP_FAKE_ARGS_CAP = 2048 }; typedef struct { atomic_int starts; @@ -1323,11 +1324,18 @@ typedef struct { cbm_project_lock_manager_t *project_locks; bool project_lock_busy_is_error; cbm_proc_outcome_t outcomes[APP_FAKE_MAX_ATTEMPTS]; + cbm_index_resource_violation_t resource_violations[APP_FAKE_MAX_ATTEMPTS]; + bool resource_probe_failures[APP_FAKE_MAX_ATTEMPTS]; const char *responses[APP_FAKE_MAX_ATTEMPTS]; const char *marker_payloads[APP_FAKE_MAX_ATTEMPTS]; char marker_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; char quarantine_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; char quarantine_seen[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; + char args_json[APP_FAKE_MAX_ATTEMPTS][APP_FAKE_ARGS_CAP]; + /* `starts` is incremented before the slot is filled, so waiting on it says + * a worker began, not that its arguments are readable. Each slot publishes + * itself instead, which is the edge a reader on another thread needs. */ + atomic_bool args_captured[APP_FAKE_MAX_ATTEMPTS]; size_t memory_budgets[APP_FAKE_MAX_ATTEMPTS]; } app_fake_worker_context_t; @@ -1392,6 +1400,10 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem atomic_init(&worker->cancelled, false); worker->result.exit_code = -1; if (worker->attempt < APP_FAKE_MAX_ATTEMPTS) { + worker->result.resource_violation = context->resource_violations[worker->attempt]; + (void)snprintf(context->args_json[worker->attempt], APP_FAKE_ARGS_CAP, "%s", + args_json ? args_json : ""); + atomic_store(&context->args_captured[worker->attempt], true); context->memory_budgets[worker->attempt] = memory_budget_bytes; if (marker_file) { (void)snprintf(context->marker_paths[worker->attempt], APP_TEST_PATH_CAP, "%s", @@ -1416,6 +1428,53 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem return 0; } +static bool app_wait_for_atomic_bool(atomic_bool *value, bool expected); + +static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, int attempt, + const char *max_files, const char *max_source_mb, + const char *max_rss_mb, const char *max_duration_seconds, + const char *cache_max_mb, const char *min_free_disk_mb) { + if (!context || attempt < 0 || attempt >= APP_FAKE_MAX_ATTEMPTS) { + return false; + } + if (!app_wait_for_atomic_bool(&context->args_captured[attempt], true)) { + return false; + } + const char *args = context->args_json[attempt]; + yyjson_doc *document = yyjson_read(args, strlen(args), 0); + yyjson_val *root = document ? yyjson_doc_get_root(document) : NULL; + yyjson_val *policy = + root && yyjson_is_obj(root) ? yyjson_obj_get(root, "_cbm_index_policy") : NULL; + yyjson_val *files = + policy && yyjson_is_obj(policy) ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_FILES) : NULL; + yyjson_val *bytes = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB) + : NULL; + yyjson_val *rss = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_RSS_MB) + : NULL; + yyjson_val *duration = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS) + : NULL; + yyjson_val *cache = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_CACHE_MAX_MB) + : NULL; + yyjson_val *free_disk = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB) + : NULL; + bool equal = files && bytes && rss && duration && cache && free_disk && yyjson_is_str(files) && + yyjson_is_str(bytes) && yyjson_is_str(rss) && yyjson_is_str(duration) && + yyjson_is_str(cache) && yyjson_is_str(free_disk) && + strcmp(yyjson_get_str(files), max_files) == 0 && + strcmp(yyjson_get_str(bytes), max_source_mb) == 0 && + strcmp(yyjson_get_str(rss), max_rss_mb) == 0 && + strcmp(yyjson_get_str(duration), max_duration_seconds) == 0 && + strcmp(yyjson_get_str(cache), cache_max_mb) == 0 && + strcmp(yyjson_get_str(free_disk), min_free_disk_mb) == 0; + yyjson_doc_free(document); + return equal; +} + static cbm_index_worker_poll_t app_fake_worker_poll(void *opaque, cbm_daemon_application_worker_t handle, const cbm_index_worker_result_t **result_out) { @@ -1453,6 +1512,12 @@ static cbm_index_worker_poll_t app_fake_worker_poll(void *opaque, worker->result.outcome = outcome; worker->result.exit_code = outcome == CBM_PROC_CLEAN ? 0 : -1; worker->result.tree_quiesced = true; + if (worker->attempt < APP_FAKE_MAX_ATTEMPTS) { + worker->result.resource_violation = + worker->context->resource_violations[worker->attempt]; + worker->result.resource_probe_failed = + worker->context->resource_probe_failures[worker->attempt]; + } const char *response = worker->attempt < APP_FAKE_MAX_ATTEMPTS && worker->context->responses[worker->attempt] ? worker->context->responses[worker->attempt] @@ -1967,9 +2032,10 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { bool dirs_ok = cbm_mkdtemp(root) != NULL && cbm_mkdtemp(cache) != NULL; bool cache_set = dirs_ok && cache_saved && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0; cbm_config_t *stored_config = cache_set ? cbm_config_open(cache) : NULL; - bool config_ready = stored_config && - cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && - cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0; + bool config_ready = + stored_config && cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && + cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_RESOURCE_PROFILE, "strict") == 0; char canonical_root[APP_TEST_PATH_CAP] = {0}; bool canonical = dirs_ok && cbm_canonical_path(root, canonical_root, sizeof(canonical_root)); char *project = canonical ? cbm_project_name_from_path(canonical_root) : NULL; @@ -1988,6 +2054,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { .config = stored_config, .worker_ops = &worker_ops, .update_ops = &update_ops, + .aggregate_memory_budget_bytes = UINT64_C(128) * CBM_INDEX_MIB_BYTES, }; cbm_daemon_application_t *application = config_ready ? cbm_daemon_application_new(&config) : NULL; @@ -2014,6 +2081,42 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { bool first_owned = first_initialized && project && app_wait_for_subscribers(application, project, 1) && app_wait_for_atomic_int(&fake.starts, 1); + bool auto_policy_propagated = + first_owned && + app_fake_worker_policy_equals(&fake, 0, "100000", "4096", "64", "3600", "32768", "4096"); + char explicit_args[APP_TEST_PATH_CAP + 64]; + (void)snprintf(explicit_args, sizeof(explicit_args), "{\"repo_path\":\"%s\"}", root); + uint8_t *explicit_tool = NULL; + uint32_t explicit_tool_length = 0; + bool explicit_encoded = app_test_tool_request("index_repository", explicit_args, &explicit_tool, + &explicit_tool_length); + app_request_thread_t explicit_request = { + .callbacks = callbacks, + .session = sessions[3], + .request = explicit_tool, + .request_length = explicit_tool_length, + }; + cbm_thread_t explicit_thread; + bool explicit_started = + explicit_encoded && + cbm_thread_create(&explicit_thread, 0, app_request_thread, &explicit_request) == 0; + bool explicit_coalesced = explicit_started && project && + app_wait_for_subscribers(application, project, 2) && + atomic_load(&fake.starts) == 1; + if (sessions[3]) { + callbacks.session_cancel(callbacks.context, sessions[3]); + } + if (explicit_started) { + (void)cbm_thread_join(&explicit_thread); + } + free(explicit_request.response); + free(explicit_tool); + if (sessions[3]) { + callbacks.session_close(callbacks.context, sessions[3]); + sessions[3] = NULL; + } + bool explicit_detached = + explicit_coalesced && project && app_wait_for_subscribers(application, project, 1); bool second_initialized = app_test_initialize_profile(&callbacks, sessions[1], root, CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); bool coalesced = first_owned && second_initialized && project && @@ -2075,6 +2178,9 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { ASSERT_TRUE(restricted_started_nothing); ASSERT_TRUE(first_initialized); ASSERT_TRUE(first_owned); + ASSERT_TRUE(auto_policy_propagated); + ASSERT_TRUE(explicit_coalesced); + ASSERT_TRUE(explicit_detached); ASSERT_TRUE(second_initialized); ASSERT_TRUE(coalesced); ASSERT_TRUE(restricted_disconnect_kept_job); @@ -2089,6 +2195,167 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { PASS(); } +TEST(daemon_application_programmatic_index_injects_resource_policy) { + char *root = th_mktempdir("cbm_app_policy_root"); + char *cache = th_mktempdir("cbm_app_policy_cache"); + ASSERT_NOT_NULL(root); + ASSERT_NOT_NULL(cache); + cbm_config_t *stored_config = cbm_config_open(cache); + bool configured = + stored_config && cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_CACHE_MAX_MB, "8") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, "9") == 0; + + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + atomic_store(&fake.allow_completion, true); + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = { + .config = stored_config, + .worker_ops = &worker_ops, + }; + cbm_daemon_application_t *application = configured ? cbm_daemon_application_new(&config) : NULL; + int index_rc = + application ? cbm_daemon_application_index(application, "policy-programmatic", root) : -1; + bool policy_propagated = + index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6", "64", "7", "8", "9"); + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + + cbm_daemon_application_free(application); + cbm_config_close(stored_config); + th_cleanup(root); + th_cleanup(cache); + ASSERT_TRUE(configured); + ASSERT_TRUE(policy_propagated); + ASSERT_TRUE(stopped); + PASS(); +} + +TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { + char *cache = th_mktempdir("cbm_app_worker_resource_cache"); + cbm_config_t *stored_config = cache ? cbm_config_open(cache) : NULL; + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + atomic_store(&fake.scripted, true); + fake.outcomes[0] = CBM_PROC_KILLED; + fake.resource_violations[0] = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_DURATION_MS, .observed = 7001, .limit = 7000}; + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = { + .config = stored_config, + .worker_ops = &worker_ops, + }; + cbm_daemon_application_t *application = + stored_config ? cbm_daemon_application_new(&config) : NULL; + cbm_daemon_runtime_application_callbacks_t callbacks = + cbm_daemon_application_runtime_callbacks(application); + cbm_daemon_runtime_application_session_t *session = app_test_open(&callbacks, 4188); + char *root = th_mktempdir("cbm_app_worker_resource"); + uint8_t *context = NULL; + uint32_t context_length = 0; + uint8_t *tool = NULL; + uint32_t tool_length = 0; + char args[APP_TEST_PATH_CAP + 96]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"DaemonWorkerResourceFixture\"}", + root ? root : ""); + bool setup = cache && stored_config && application && session && root && + app_test_context_request(root, root, &context, &context_length) && + app_test_tool_request("index_repository", args, &tool, &tool_length); + uint8_t *response = NULL; + uint32_t response_length = 0; + if (setup) { + uint8_t *context_response = NULL; + uint32_t context_response_length = 0; + setup = app_test_request(&callbacks, session, context, context_length, &context_response, + &context_response_length) == CBM_DAEMON_RUNTIME_APPLICATION_OK; + free(context_response); + } + cbm_daemon_runtime_application_status_t status = + setup + ? app_test_request(&callbacks, session, tool, tool_length, &response, &response_length) + : CBM_DAEMON_RUNTIME_APPLICATION_TRANSPORT_ERROR; + bool structured = response && strstr((char *)response, "resource_limit_exceeded") && + strstr((char *)response, "\\\"stage\\\":\\\"worker\\\"") && + strstr((char *)response, "\\\"resource\\\":\\\"duration_ms\\\"") && + strstr((char *)response, "\\\"observed\\\":7001") && + strstr((char *)response, "\\\"limit\\\":7000") && + strstr((char *)response, "\\\"unit\\\":\\\"milliseconds\\\""); + if (session) { + callbacks.session_close(callbacks.context, session); + } + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + int starts = atomic_load(&fake.starts); + int destroys = atomic_load(&fake.destroys); + cbm_daemon_application_free(application); + cbm_config_close(stored_config); + free(response); + free(context); + free(tool); + th_cleanup(root); + th_cleanup(cache); + + ASSERT_TRUE(setup); + ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_OK); + ASSERT_TRUE(structured); + ASSERT_EQ(starts, 1); + ASSERT_EQ(destroys, 1); + ASSERT_TRUE(stopped); + PASS(); +} + +TEST(daemon_application_worker_storage_resource_failure_is_structured_and_not_retried) { + app_watch_race_fixture_t fixture; + bool ready = app_watch_race_fixture_init(&fixture, 77); + atomic_store(&fixture.fake.scripted, true); + fixture.fake.outcomes[0] = CBM_PROC_KILLED; + fixture.fake.resource_violations[0] = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_TASK_TEMP_BYTES, .observed = 2048, .limit = 1024}; + + char args[APP_TEST_PATH_CAP + 32]; + (void)snprintf(args, sizeof(args), "{\"repo_path\":\"%s\"}", fixture.root); + uint8_t *request = NULL; + uint32_t request_length = 0; + uint8_t *response = NULL; + uint32_t response_length = 0; + bool encoded = + ready && app_test_tool_request("index_repository", args, &request, &request_length); + bool completed = encoded && app_test_request(&fixture.callbacks, fixture.session, request, + request_length, &response, &response_length) == + CBM_DAEMON_RUNTIME_APPLICATION_OK; + bool structured = + completed && response && strstr((char *)response, "resource_limit_exceeded") && + strstr((char *)response, "task_temp_bytes") && strstr((char *)response, "storage"); + int starts = atomic_load(&fixture.fake.starts); + free(request); + free(response); + bool cleaned = app_watch_race_fixture_finish(&fixture); + + ASSERT_TRUE(ready); + ASSERT_TRUE(completed); + ASSERT_TRUE(structured); + ASSERT_EQ(starts, 1); + ASSERT_TRUE(cleaned); + PASS(); +} + TEST(daemon_application_auto_index_honors_tracked_file_limit) { app_env_backup_t cache_environment; bool cache_saved = app_env_backup_capture(&cache_environment, "CBM_CACHE_DIR"); @@ -5097,6 +5364,9 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_free_releases_live_watch_once); RUN_TEST(daemon_application_prune_clears_logical_watch_for_reregistration); RUN_TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions); + RUN_TEST(daemon_application_programmatic_index_injects_resource_policy); + RUN_TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried); + RUN_TEST(daemon_application_worker_storage_resource_failure_is_structured_and_not_retried); RUN_TEST(daemon_application_auto_index_honors_tracked_file_limit); RUN_TEST(daemon_application_auto_index_file_count_handles_literal_metacharacter_path); RUN_TEST(daemon_application_auto_index_file_count_supports_non_git_roots); diff --git a/tests/test_discover.c b/tests/test_discover.c index 7e0007b11..0d04a01b3 100644 --- a/tests/test_discover.c +++ b/tests/test_discover.c @@ -13,6 +13,21 @@ typedef struct { char *xdg_config_home; } git_env_snapshot_t; +#ifdef CBM_ENABLE_TEST_SEAMS +typedef struct { + uint64_t first; + uint64_t second; + uint64_t later; + int calls; +} discover_clock_fake_t; + +static uint64_t discover_clock_fake(void *context) { + discover_clock_fake_t *fake = context; + int call = fake->calls++; + return call == 0 ? fake->first : call == 1 ? fake->second : fake->later; +} +#endif + static git_env_snapshot_t save_git_env(void) { git_env_snapshot_t snapshot = {0}; const char *home = getenv("HOME"); @@ -475,6 +490,326 @@ TEST(discover_bounded_count_matches_shebang_discovery) { PASS(); } +TEST(discover_resource_policy_off_matches_legacy_discovery) { + char *base = th_mktempdir("cbm_disc_policy_off"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "src/first.c"), "int first;\n"); + th_write_file(TH_PATH(base, "src/second.py"), "second = 2\n"); + th_write_file(TH_PATH(base, "src/ignored.png"), "not source\n"); + + cbm_file_info_t *legacy_files = NULL; + int legacy_count = 0; + cbm_discover_opts_t legacy_opts = {.mode = CBM_MODE_FULL}; + ASSERT_EQ(cbm_discover(base, &legacy_opts, &legacy_files, &legacy_count), CBM_DISCOVER_OK); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + cbm_index_resource_violation_t violation = {0}; + cbm_file_info_t *policy_files = NULL; + int policy_count = 0; + cbm_discover_opts_t policy_opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_discover_status_t status = cbm_discover(base, &policy_opts, &policy_files, &policy_count); + + ASSERT_EQ(status, CBM_DISCOVER_OK); + ASSERT_EQ(policy_count, legacy_count); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_NONE); + for (int index = 0; index < legacy_count; index++) { + ASSERT_STR_EQ(policy_files[index].rel_path, legacy_files[index].rel_path); + ASSERT_EQ(policy_files[index].size, legacy_files[index].size); + } + + cbm_discover_free(legacy_files, legacy_count); + cbm_discover_free(policy_files, policy_count); + th_cleanup(base); + PASS(); +} + +TEST(discover_resource_file_limit_is_exact_and_counts_only_accepted_sources) { + char *base = th_mktempdir("cbm_disc_policy_files"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, ".gitignore"), "ignored.c\n"); + th_write_file(TH_PATH(base, "accepted.c"), "int accepted;\n"); + th_write_file(TH_PATH(base, "ignored.c"), "int ignored;\n"); + th_write_file(TH_PATH(base, "unsupported.png"), "not source\n"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK); + ASSERT_EQ(count, 1); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_NONE); + cbm_discover_free(files, count); + + th_write_file(TH_PATH(base, "second.py"), "second = 2\n"); + files = NULL; + count = 99; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + + th_cleanup(base); + PASS(); +} + +TEST(discover_resource_source_bytes_allows_equality_and_rejects_one_more_byte) { + char *base = th_mktempdir("cbm_disc_policy_bytes"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "exact.c"), "1234567"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_source_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 7}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK); + ASSERT_EQ(count, 1); + cbm_discover_free(files, count); + + th_write_file(TH_PATH(base, "plus.py"), "x"); + files = NULL; + count = 99; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_SOURCE_BYTES); + ASSERT_EQ(violation.observed, 8); + ASSERT_EQ(violation.limit, 7); + + th_cleanup(base); + PASS(); +} + +TEST(discover_profile_directory_limit_counts_root_and_admitted_directories) { + char *base = th_mktempdir("cbm_disc_policy_dirs"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "first/source.c"), "int first;\n"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_directories = (cbm_index_limit_u64_t){.enabled = true, .value = 2}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK); + ASSERT_EQ(count, 1); + cbm_discover_free(files, count); + + th_write_file(TH_PATH(base, "second/source.py"), "second = 2\n"); + files = NULL; + count = 99; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_DIRECTORIES); + ASSERT_EQ(violation.observed, 3); + ASSERT_EQ(violation.limit, 2); + th_cleanup(base); + PASS(); +} + +TEST(discover_profile_entry_limit_counts_before_ignore_and_language_filters) { + char *base = th_mktempdir("cbm_disc_policy_entries"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, ".gitignore"), "ignored.c\n"); + th_write_file(TH_PATH(base, "ignored.c"), "int ignored;\n"); + th_write_file(TH_PATH(base, "asset.png"), "not source\n"); + th_write_file(TH_PATH(base, "accepted.c"), "int accepted;\n"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_entries = (cbm_index_limit_u64_t){.enabled = true, .value = 4}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK); + ASSERT_EQ(count, 1); + cbm_discover_free(files, count); + + policy.max_entries.value = 3; + files = NULL; + count = 99; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_ENTRIES); + ASSERT_EQ(violation.observed, 4); + ASSERT_EQ(violation.limit, 3); + th_cleanup(base); + PASS(); +} + +TEST(discover_profile_depth_limit_allows_equality_and_rejects_child) { + char *base = th_mktempdir("cbm_disc_policy_depth"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "one/two/source.c"), "int source;\n"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_depth = (cbm_index_limit_u64_t){.enabled = true, .value = 2}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK); + ASSERT_EQ(count, 1); + cbm_discover_free(files, count); + + policy.max_depth.value = 1; + files = NULL; + count = 99; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_DEPTH); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + + th_cleanup(base); + PASS(); +} + +TEST(discover_profile_deadline_uses_monotonic_test_clock) { +#ifndef CBM_ENABLE_TEST_SEAMS + SKIP_PLATFORM("discovery clock seam unavailable"); +#else + char *base = th_mktempdir("cbm_disc_policy_deadline"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "source.c"), "int source;\n"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.discovery_deadline_ms = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + discover_clock_fake_t clock = {.first = 1000, .second = 1001, .later = 1002}; + cbm_discover_set_now_ms_for_testing(discover_clock_fake, &clock); + cbm_file_info_t *files = NULL; + int count = 99; + cbm_discover_status_t status = cbm_discover(base, &opts, &files, &count); + cbm_discover_set_now_ms_for_testing(NULL, NULL); + + ASSERT_EQ(status, CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + + char empty[4096]; + (void)snprintf(empty, sizeof(empty), "%s/empty", base); + ASSERT_EQ(th_mkdir_p(empty), 0); + violation = (cbm_index_resource_violation_t){0}; + clock = (discover_clock_fake_t){.first = 2000, .second = 2001, .later = 2002}; + cbm_discover_set_now_ms_for_testing(discover_clock_fake, &clock); + status = cbm_discover(empty, &opts, &files, &count); + cbm_discover_set_now_ms_for_testing(NULL, NULL); + ASSERT_EQ(status, CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + + violation = (cbm_index_resource_violation_t){0}; + clock = (discover_clock_fake_t){.first = 3000, .second = 3001, .later = 3001}; + cbm_discover_set_now_ms_for_testing(discover_clock_fake, &clock); + status = cbm_discover(empty, &opts, &files, &count); + cbm_discover_set_now_ms_for_testing(NULL, NULL); + ASSERT_EQ(status, CBM_DISCOVER_OK); + ASSERT_EQ(count, 0); + ASSERT(files == NULL); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_NONE); + th_cleanup(base); + PASS(); +#endif +} + +TEST(discover_resource_file_budget_excludes_existing_oversized_skip) { + char *base = th_mktempdir("cbm_disc_policy_oversized"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "accepted.c"), "x"); + th_write_file(TH_PATH(base, "oversized.py"), "123"); + const char *saved_limit = getenv("CBM_MAX_FILE_BYTES"); + char *saved_limit_copy = saved_limit ? strdup(saved_limit) : NULL; + cbm_setenv("CBM_MAX_FILE_BYTES", "2", 1); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + cbm_discover_status_t initial_status = cbm_discover(base, &opts, &files, &count); + bool oversized_did_not_consume_budget = initial_status == CBM_DISCOVER_OK && count == 2 && + violation.resource == CBM_INDEX_RESOURCE_NONE; + cbm_discover_free(files, count); + + th_write_file(TH_PATH(base, "second.c"), "y"); + files = NULL; + count = 0; + cbm_discover_status_t exceeded_status = cbm_discover(base, &opts, &files, &count); + bool accepted_sources_exceeded = exceeded_status == CBM_DISCOVER_LIMIT_EXCEEDED && + files == NULL && count == 0 && + violation.resource == CBM_INDEX_RESOURCE_FILES && + violation.observed == 2 && violation.limit == 1; + + if (saved_limit_copy) { + cbm_setenv("CBM_MAX_FILE_BYTES", saved_limit_copy, 1); + } else { + cbm_unsetenv("CBM_MAX_FILE_BYTES"); + } + free(saved_limit_copy); + th_cleanup(base); + + ASSERT_TRUE(oversized_did_not_consume_budget); + ASSERT_TRUE(accepted_sources_exceeded); + PASS(); +} + TEST(discover_skips_git_dir) { char *base = th_mktempdir("cbm_disc_git"); ASSERT(base != NULL); @@ -1767,6 +2102,14 @@ SUITE(discover) { RUN_TEST(discover_bounded_count_is_allocation_free_and_limit_exact); RUN_TEST(discover_bounded_count_fails_closed_after_deadline); RUN_TEST(discover_bounded_count_matches_shebang_discovery); + RUN_TEST(discover_resource_policy_off_matches_legacy_discovery); + RUN_TEST(discover_resource_file_limit_is_exact_and_counts_only_accepted_sources); + RUN_TEST(discover_resource_source_bytes_allows_equality_and_rejects_one_more_byte); + RUN_TEST(discover_profile_directory_limit_counts_root_and_admitted_directories); + RUN_TEST(discover_profile_entry_limit_counts_before_ignore_and_language_filters); + RUN_TEST(discover_profile_depth_limit_allows_equality_and_rejects_child); + RUN_TEST(discover_profile_deadline_uses_monotonic_test_clock); + RUN_TEST(discover_resource_file_budget_excludes_existing_oversized_skip); RUN_TEST(discover_skips_git_dir); RUN_TEST(discover_with_gitignore); RUN_TEST(discover_with_global_xdg_ignore); diff --git a/tests/test_index_policy.c b/tests/test_index_policy.c new file mode 100644 index 000000000..cbd918649 --- /dev/null +++ b/tests/test_index_policy.c @@ -0,0 +1,764 @@ +#include "test_framework.h" +#include "test_helpers.h" + +#include "cli/cli.h" +#include "foundation/compat.h" +#include "foundation/constants.h" +#include "foundation/index_policy.h" +#include "mcp/index_supervisor.h" +#include "mcp/mcp.h" +#include "mcp/mcp_internal.h" +#include "store/store.h" +#include + +#include +#include + +TEST(index_policy_defaults_are_disabled) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + + ASSERT_STR_EQ(cbm_index_policy_profile_name(&policy), "off"); + ASSERT_STR_EQ(cbm_index_policy_source_name(&policy), "off"); + ASSERT_FALSE(policy.max_files.enabled); + ASSERT_FALSE(policy.max_directories.enabled); + ASSERT_FALSE(policy.max_entries.enabled); + ASSERT_FALSE(policy.max_depth.enabled); + ASSERT_FALSE(policy.max_source_bytes.enabled); + ASSERT_FALSE(policy.discovery_deadline_ms.enabled); + ASSERT_FALSE(policy.max_rss_bytes.enabled); + ASSERT_FALSE(policy.max_duration_ms.enabled); + ASSERT_FALSE(policy.max_cache_bytes.enabled); + ASSERT_FALSE(policy.min_free_disk_bytes.enabled); + ASSERT_FALSE(policy.max_final_db_bytes.enabled); + ASSERT_FALSE(policy.max_staging_bytes.enabled); + ASSERT_FALSE(policy.max_task_temp_bytes.enabled); + ASSERT_FALSE(cbm_index_policy_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_discovery_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_worker_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_storage_enabled(&policy)); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_FILES), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_SOURCE_MB), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_RSS_MB), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_DURATION_SECONDS), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_CACHE_MAX_MB), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MIN_FREE_DISK_MB), "off"); + PASS(); +} + +TEST(index_policy_balanced_profile_expands_exact_baseline) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "balanced", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(20) * 1024 * CBM_INDEX_MIB_BYTES, + UINT64_C(4) * 1024 * CBM_INDEX_MIB_BYTES); + + ASSERT_STR_EQ(cbm_index_policy_profile_name(&policy), "balanced"); + ASSERT_STR_EQ(cbm_index_policy_source_name(&policy), "profile"); + ASSERT_EQ(policy.max_files.value, UINT64_C(500000)); + ASSERT_EQ(policy.max_directories.value, UINT64_C(250000)); + ASSERT_EQ(policy.max_entries.value, UINT64_C(2000000)); + ASSERT_EQ(policy.max_depth.value, UINT64_C(128)); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(65536) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.discovery_deadline_ms.value, UINT64_C(1800000)); + /* Four fifths of a 20 GiB host. */ + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(16) * 1024 * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(7200000)); + ASSERT_EQ(policy.max_final_db_bytes.value, UINT64_C(65536) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_staging_bytes.value, UINT64_C(81920) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_task_temp_bytes.value, UINT64_C(98304) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_cache_bytes.value, UINT64_C(131072) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.min_free_disk_bytes.value, UINT64_C(4096) * CBM_INDEX_MIB_BYTES); + PASS(); +} + +/* A balanced ceiling must never sit below what the host can actually finish. + * The large-repository runs this project records as reference workloads peak + * in the tens of gigabytes, so a fixed balanced ceiling would reject indexes + * that complete today with the profile off. */ +TEST(index_policy_balanced_rss_follows_the_host_share) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "balanced", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(40) * 1024 * CBM_INDEX_MIB_BYTES, + UINT64_C(4) * 1024 * CBM_INDEX_MIB_BYTES); + ASSERT_TRUE(policy.max_rss_bytes.enabled); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(32) * 1024 * CBM_INDEX_MIB_BYTES); + + /* A small host still receives a real ceiling below its own memory. */ + cbm_index_policy_init(&policy); + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "balanced", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(10) * 1024 * CBM_INDEX_MIB_BYTES, + UINT64_C(2) * 1024 * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(8) * 1024 * CBM_INDEX_MIB_BYTES); + + /* Host memory that cannot be read falls back to the fixed floor. */ + cbm_index_policy_init(&policy); + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "balanced", error, sizeof(error))); + cbm_index_policy_finalize(&policy, 0, UINT64_C(6) * 1024 * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(12) * 1024 * CBM_INDEX_MIB_BYTES); + PASS(); +} + +TEST(index_policy_strict_profile_expands_exact_baseline) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "strict", error, sizeof(error))); + cbm_index_policy_finalize(&policy, 0, UINT64_C(3) * 1024 * CBM_INDEX_MIB_BYTES); + + ASSERT_STR_EQ(cbm_index_policy_profile_name(&policy), "strict"); + ASSERT_EQ(policy.max_files.value, UINT64_C(100000)); + ASSERT_EQ(policy.max_directories.value, UINT64_C(20000)); + ASSERT_EQ(policy.max_entries.value, UINT64_C(500000)); + ASSERT_EQ(policy.max_depth.value, UINT64_C(64)); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(4096) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.discovery_deadline_ms.value, UINT64_C(30000)); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(3072) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(3600000)); + ASSERT_EQ(policy.max_final_db_bytes.value, UINT64_C(16384) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_staging_bytes.value, UINT64_C(20480) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_task_temp_bytes.value, UINT64_C(24576) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_cache_bytes.value, UINT64_C(32768) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.min_free_disk_bytes.value, UINT64_C(4096) * CBM_INDEX_MIB_BYTES); + PASS(); +} + +TEST(index_policy_explicit_override_and_off_replace_profile_dimensions) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "balanced", error, sizeof(error))); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "42", error, sizeof(error))); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "off", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(1024) * CBM_INDEX_MIB_BYTES, + UINT64_C(512) * CBM_INDEX_MIB_BYTES); + + ASSERT_EQ(policy.max_files.value, UINT64_C(42)); + ASSERT_FALSE(policy.max_rss_bytes.enabled); + ASSERT_TRUE(policy.max_entries.enabled); + ASSERT_STR_EQ(cbm_index_policy_source_name(&policy), "profile+override"); + PASS(); +} + +TEST(index_policy_host_clamp_only_tightens_explicit_rss) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "4096", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(2048) * CBM_INDEX_MIB_BYTES, 0); + ASSERT_EQ(policy.max_rss_bytes.value, + (UINT64_C(2048) * CBM_INDEX_MIB_BYTES * UINT64_C(4)) / UINT64_C(5)); + + cbm_index_policy_init(&policy); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "512", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(2048) * CBM_INDEX_MIB_BYTES, 0); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(512) * CBM_INDEX_MIB_BYTES); + PASS(); +} + +TEST(index_policy_profile_validation_is_atomic) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "strict", error, sizeof(error))); + cbm_index_resource_policy_t before = policy; + ASSERT_FALSE(cbm_index_policy_set_profile(&policy, "STRICT", error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + ASSERT_TRUE(strstr(error, CBM_INDEX_CONFIG_RESOURCE_PROFILE) != NULL); + PASS(); +} + +TEST(index_policy_file_limit_accepts_off_and_exact_range) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "1", error, sizeof(error))); + ASSERT_TRUE(policy.max_files.enabled); + ASSERT_EQ(policy.max_files.value, 1); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "10000000", error, + sizeof(error))); + ASSERT_EQ(policy.max_files.value, 10000000); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "off", error, sizeof(error))); + ASSERT_FALSE(policy.max_files.enabled); + PASS(); +} + +TEST(index_policy_source_limit_converts_mib_without_overflow) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "1", error, sizeof(error))); + ASSERT_TRUE(policy.max_source_bytes.enabled); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(1024) * UINT64_C(1024)); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "1048576", error, + sizeof(error))); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(1048576) * UINT64_C(1024) * UINT64_C(1024)); + PASS(); +} + +TEST(index_policy_worker_limits_validate_and_convert_units) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "64", error, sizeof(error))); + ASSERT_TRUE(cbm_index_policy_enabled(&policy)); + ASSERT_TRUE(cbm_index_policy_worker_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_discovery_enabled(&policy)); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(64) * CBM_INDEX_MIB_BYTES); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "1048576", error, + sizeof(error))); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(1048576) * CBM_INDEX_MIB_BYTES); + cbm_index_resource_policy_t before = policy; + ASSERT_FALSE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "63", error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "1", error, + sizeof(error))); + ASSERT_EQ(policy.max_duration_ms.value, 1000); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "86400", error, + sizeof(error))); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(86400000)); + before = policy; + ASSERT_FALSE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "86401", + error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + PASS(); +} + +TEST(index_policy_storage_limits_validate_and_convert_units) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + char value[64]; + + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_CACHE_MAX_MB, "131072", error, + sizeof(error))); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, "4096", error, + sizeof(error))); + ASSERT_EQ(policy.max_cache_bytes.value, UINT64_C(131072) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.min_free_disk_bytes.value, UINT64_C(4096) * CBM_INDEX_MIB_BYTES); + ASSERT_TRUE(cbm_index_policy_storage_enabled(&policy)); + ASSERT_TRUE( + cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_CACHE_MAX_MB, value, sizeof(value))); + ASSERT_STR_EQ(value, "131072"); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_CACHE_MAX_MB, "off", error, sizeof(error))); + ASSERT_FALSE(policy.max_cache_bytes.enabled); + PASS(); +} + +TEST(index_policy_storage_projection_boundaries_are_overflow_safe) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_cache_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 100}; + policy.min_free_disk_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 40}; + policy.max_final_db_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 60}; + policy.max_staging_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 70}; + policy.max_task_temp_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 80}; + cbm_index_storage_sample_t sample = { + .current_cache_bytes = 130, + .replaceable_old_bytes = 50, + .operation_bytes = 20, + .free_disk_bytes = 40, + .final_db_bytes = 60, + .staging_bytes = 70, + .task_temp_bytes = 80, + }; + cbm_index_resource_violation_t violation = {0}; + + ASSERT_TRUE(cbm_index_policy_check_storage(&policy, &sample, &violation)); + sample.current_cache_bytes++; + ASSERT_FALSE(cbm_index_policy_check_storage(&policy, &sample, &violation)); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_CACHE_BYTES); + ASSERT_EQ(violation.observed, 101); + ASSERT_EQ(violation.limit, 100); + sample.current_cache_bytes--; + sample.free_disk_bytes--; + ASSERT_FALSE(cbm_index_policy_check_storage(&policy, &sample, &violation)); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FREE_DISK_BYTES); + ASSERT_EQ(violation.observed, 39); + sample = (cbm_index_storage_sample_t){ + .current_cache_bytes = UINT64_MAX, + .replaceable_old_bytes = 0, + .operation_bytes = UINT64_MAX, + .free_disk_bytes = UINT64_MAX, + }; + ASSERT_FALSE(cbm_index_policy_check_storage(&policy, &sample, &violation)); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_CACHE_BYTES); + ASSERT_EQ(violation.observed, UINT64_MAX); + PASS(); +} + +TEST(index_policy_invalid_value_is_rejected_atomically) { + static const char *const invalid[] = {"", "0", "-1", "1MB", + "1 ", "+1", "10000001", "18446744073709551616"}; + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "42", error, sizeof(error))); + + for (size_t index = 0; index < sizeof(invalid) / sizeof(invalid[0]); index++) { + cbm_index_resource_policy_t before = policy; + ASSERT_FALSE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, invalid[index], + error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + ASSERT_TRUE(error[0] != '\0'); + } + ASSERT_FALSE(cbm_index_policy_set(&policy, "index_unknown_limit", "1", error, sizeof(error))); + PASS(); +} + +TEST(index_policy_format_round_trips_public_values) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + char value[64]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "321", error, sizeof(error))); + ASSERT_TRUE(cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_FILES, value, sizeof(value))); + ASSERT_STR_EQ(value, "321"); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "7", error, sizeof(error))); + ASSERT_TRUE( + cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, value, sizeof(value))); + ASSERT_STR_EQ(value, "7"); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "64", error, sizeof(error))); + ASSERT_TRUE( + cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, value, sizeof(value))); + ASSERT_STR_EQ(value, "64"); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "9", error, + sizeof(error))); + ASSERT_TRUE(cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, value, + sizeof(value))); + ASSERT_STR_EQ(value, "9"); + PASS(); +} + +TEST(index_policy_violation_metadata_is_stable) { + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_FILES), "files"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_SOURCE_BYTES), "source_bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_CACHE_BYTES), "cache_bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_FREE_DISK_BYTES), "free_disk_bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_STAGING_BYTES), "staging_bytes"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_FILES), "files"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_SOURCE_BYTES), "bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_RSS_BYTES), "rss_bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_DURATION_MS), "duration_ms"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_RSS_BYTES), "bytes"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_DURATION_MS), "milliseconds"); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_FILES), + CBM_INDEX_CONFIG_MAX_FILES); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_SOURCE_BYTES), + CBM_INDEX_CONFIG_MAX_SOURCE_MB); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_RSS_BYTES), + CBM_INDEX_CONFIG_MAX_RSS_MB); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_DURATION_MS), + CBM_INDEX_CONFIG_MAX_DURATION_SECONDS); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_CACHE_BYTES), + CBM_INDEX_CONFIG_CACHE_MAX_MB); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_FREE_DISK_BYTES), + CBM_INDEX_CONFIG_MIN_FREE_DISK_MB); + PASS(); +} + +TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { + char *cache = th_mktempdir("cbm_index_policy_config"); + ASSERT_NOT_NULL(cache); + cbm_config_t *config = cbm_config_open(cache); + ASSERT_NOT_NULL(config); + cbm_index_resource_policy_t policy; + char error[256]; + + ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); + ASSERT_FALSE(cbm_index_policy_enabled(&policy)); + ASSERT_STR_EQ(cbm_index_policy_profile_name(&policy), "off"); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_RESOURCE_PROFILE, "strict"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "9"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "3"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_CACHE_MAX_MB, "17"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, "5"), 0); + ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); + ASSERT_EQ(policy.max_files.value, 9); + ASSERT_STR_EQ(cbm_index_policy_profile_name(&policy), "strict"); + ASSERT_EQ(policy.max_directories.value, UINT64_C(20000)); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(3) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(64) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(7000)); + ASSERT_EQ(policy.max_cache_bytes.value, UINT64_C(17) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.min_free_disk_bytes.value, UINT64_C(5) * CBM_INDEX_MIB_BYTES); + + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "corrupt"), 0); + ASSERT_FALSE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); + ASSERT_TRUE(strstr(error, CBM_INDEX_CONFIG_MAX_FILES) != NULL); + + cbm_config_close(config); + th_cleanup(cache); + PASS(); +} + +TEST(index_policy_cli_lists_all_operator_keys) { + bool profile_found = false; + bool files_found = false; + bool bytes_found = false; + bool rss_found = false; + bool duration_found = false; + bool cache_found = false; + bool free_found = false; + for (size_t index = 0; index < cbm_cli_config_key_count_for_testing(); index++) { + const char *key = cbm_cli_config_key_at_for_testing(index); + profile_found = + profile_found || (key && strcmp(key, CBM_INDEX_CONFIG_RESOURCE_PROFILE) == 0); + files_found = files_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0); + bytes_found = bytes_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); + rss_found = rss_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_RSS_MB) == 0); + duration_found = + duration_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS) == 0); + cache_found = cache_found || (key && strcmp(key, CBM_INDEX_CONFIG_CACHE_MAX_MB) == 0); + free_found = free_found || (key && strcmp(key, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB) == 0); + } + ASSERT_TRUE(profile_found); + ASSERT_TRUE(files_found); + ASSERT_TRUE(bytes_found); + ASSERT_TRUE(rss_found); + ASSERT_TRUE(duration_found); + ASSERT_TRUE(cache_found); + ASSERT_TRUE(free_found); + PASS(); +} + +TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite) { + char *cache = th_mktempdir("cbm_index_policy_cli"); + ASSERT_NOT_NULL(cache); + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + + char *set_valid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "7"}; + char *set_invalid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "0"}; + char *set_profile[] = {"set", CBM_INDEX_CONFIG_RESOURCE_PROFILE, "balanced"}; + char *set_invalid_profile[] = {"set", CBM_INDEX_CONFIG_RESOURCE_PROFILE, "BALANCED"}; + char *reset[] = {"reset", CBM_INDEX_CONFIG_MAX_FILES}; + int valid_rc = cbm_cmd_config(3, set_valid); + cbm_config_t *config = cbm_config_open(cache); + const char *stored = + config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "missing") : "missing"; + bool valid_stored = strcmp(stored, "7") == 0; + int invalid_rc = cbm_cmd_config(3, set_invalid); + stored = config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "missing") : "missing"; + bool invalid_preserved = strcmp(stored, "7") == 0; + int profile_rc = cbm_cmd_config(3, set_profile); + stored = + config ? cbm_config_get(config, CBM_INDEX_CONFIG_RESOURCE_PROFILE, "missing") : "missing"; + bool profile_stored = strcmp(stored, "balanced") == 0; + int invalid_profile_rc = cbm_cmd_config(3, set_invalid_profile); + stored = + config ? cbm_config_get(config, CBM_INDEX_CONFIG_RESOURCE_PROFILE, "missing") : "missing"; + bool invalid_profile_preserved = strcmp(stored, "balanced") == 0; + int reset_rc = cbm_cmd_config(2, reset); + stored = config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "off") : "missing"; + bool reset_to_default = strcmp(stored, "off") == 0; + cbm_config_close(config); + + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); + th_cleanup(cache); + + ASSERT_EQ(valid_rc, 0); + ASSERT_TRUE(valid_stored); + ASSERT_TRUE(invalid_rc != 0); + ASSERT_TRUE(invalid_preserved); + ASSERT_EQ(profile_rc, 0); + ASSERT_TRUE(profile_stored); + ASSERT_TRUE(invalid_profile_rc != 0); + ASSERT_TRUE(invalid_profile_preserved); + ASSERT_EQ(reset_rc, 0); + ASSERT_TRUE(reset_to_default); + PASS(); +} + +TEST(index_policy_worker_rejects_missing_parent_policy) { + char *repo = th_mktempdir("cbm_index_policy_worker"); + ASSERT_NOT_NULL(repo); + char args[1024]; + (void)snprintf(args, sizeof(args), "{\"repo_path\":\"%s\",\"mode\":\"fast\"}", repo); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + + cbm_index_set_worker_role(true, NULL); + char *response = server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + cbm_index_set_worker_role(false, NULL); + bool rejected = response && strstr(response, "missing or incomplete trusted worker policy"); + + free(response); + cbm_mcp_server_free(server); + th_cleanup(repo); + ASSERT_TRUE(rejected); + PASS(); +} + +TEST(index_policy_worker_contract_preserves_profile_only_dimensions) { + cbm_index_resource_policy_t parent; + cbm_index_policy_init(&parent); + char error[256]; + ASSERT_TRUE(cbm_index_policy_set_profile(&parent, "strict", error, sizeof(error))); + cbm_index_policy_finalize(&parent, UINT64_C(16) * 1024 * CBM_INDEX_MIB_BYTES, + UINT64_C(32) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(parent.max_rss_bytes.value, CBM_INDEX_MIN_RSS_MB_VALUE * CBM_INDEX_MIB_BYTES); + + yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_TRUE(cbm_mcp_index_policy_add_to_args(document, root, &parent)); + char *args = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(args); + + cbm_index_resource_policy_t worker; + ASSERT_TRUE(cbm_mcp_index_policy_from_internal_args(args, &worker, error, sizeof(error))); + ASSERT_STR_EQ(cbm_index_policy_profile_name(&worker), "strict"); + ASSERT_STR_EQ(cbm_index_policy_source_name(&worker), "profile"); + ASSERT_EQ(worker.max_directories.value, parent.max_directories.value); + ASSERT_EQ(worker.max_entries.value, parent.max_entries.value); + ASSERT_EQ(worker.max_depth.value, parent.max_depth.value); + ASSERT_EQ(worker.discovery_deadline_ms.value, parent.discovery_deadline_ms.value); + ASSERT_EQ(worker.max_final_db_bytes.value, parent.max_final_db_bytes.value); + ASSERT_EQ(worker.max_staging_bytes.value, parent.max_staging_bytes.value); + ASSERT_EQ(worker.max_task_temp_bytes.value, parent.max_task_temp_bytes.value); + ASSERT_EQ(worker.max_rss_bytes.value, parent.max_rss_bytes.value); + + free(args); + yyjson_mut_doc_free(document); + PASS(); +} + +TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index) { + char *repo = th_mktempdir("cbm_index_policy_mcp_repo"); + char *cache = th_mktempdir("cbm_index_policy_mcp_cache"); + ASSERT_NOT_NULL(repo); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first(void) { return 1; }\n"), 0); + + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + cbm_config_t *config = cbm_config_open(cache); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + if (server && config) { + cbm_mcp_server_set_config(server, config); + } + + char args[2048]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\"," + "\"mode\":\"fast\"}", + repo); + char *first_response = + server && config ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + bool first_indexed = first_response && strstr(first_response, "\\\"status\\\":\\\"indexed\\\""); + free(first_response); + + char db_path[2048]; + (void)snprintf(db_path, sizeof(db_path), "%s/ResourcePolicyFixture.db", cache); + cbm_store_t *before_store = cbm_store_open_path_query(db_path); + cbm_project_t before_project = {0}; + bool before_read = before_store && cbm_store_get_project(before_store, "ResourcePolicyFixture", + &before_project) == CBM_STORE_OK; + char *indexed_at_before = + before_read && before_project.indexed_at ? strdup(before_project.indexed_at) : NULL; + cbm_project_free_fields(&before_project); + cbm_store_close(before_store); + + bool configured = + config && cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "1") == 0 && + th_write_file(TH_PATH(repo, "second.py"), "def second():\n return 2\n") == 0; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\"," + "\"mode\":\"fast\",\"_cbm_index_policy\":{" + "\"index_max_files\":\"off\",\"index_max_source_mb\":\"off\"}}", + repo); + char *limited_response = + configured && server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + bool contract_ok = limited_response && strstr(limited_response, "resource_limit_exceeded") && + strstr(limited_response, "\\\"stage\\\":\\\"discovery\\\"") && + strstr(limited_response, "\\\"resource\\\":\\\"files\\\"") && + strstr(limited_response, "\\\"observed\\\":2") && + strstr(limited_response, "\\\"limit\\\":1") && + strstr(limited_response, "\\\"unit\\\":\\\"files\\\"") && + strstr(limited_response, "\\\"retryable\\\":true") && + strstr(limited_response, "\\\"serving_index_preserved\\\":true"); + free(limited_response); + + cbm_store_t *after_store = cbm_store_open_path_query(db_path); + cbm_project_t after_project = {0}; + bool after_read = after_store && cbm_store_get_project(after_store, "ResourcePolicyFixture", + &after_project) == CBM_STORE_OK; + bool generation_preserved = indexed_at_before && after_read && after_project.indexed_at && + strcmp(indexed_at_before, after_project.indexed_at) == 0; + cbm_project_free_fields(&after_project); + cbm_store_close(after_store); + free(indexed_at_before); + + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\"," + "\"mode\":\"cross-repo-intelligence\"}", + repo); + char *cross_response = server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + bool cross_repo_unaffected = + cross_response && strstr(cross_response, "resource_limit_exceeded") == NULL; + free(cross_response); + + cbm_mcp_server_free(server); + cbm_config_close(config); + (void)cbm_unlink(db_path); + char sidecar[2100]; + (void)snprintf(sidecar, sizeof(sidecar), "%s-wal", db_path); + (void)cbm_unlink(sidecar); + (void)snprintf(sidecar, sizeof(sidecar), "%s-shm", db_path); + (void)cbm_unlink(sidecar); + th_cleanup(repo); + th_cleanup(cache); + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); + + ASSERT_TRUE(first_indexed); + ASSERT_TRUE(configured); + ASSERT_TRUE(contract_ok); + ASSERT_TRUE(generation_preserved); + ASSERT_TRUE(cross_repo_unaffected); + PASS(); +} + +TEST(index_policy_storage_limit_preserves_old_index_and_unrelated_cache_files) { + char *repo = th_mktempdir("cbm_index_storage_repo"); + char *cache = th_mktempdir("cbm_index_storage_cache"); + ASSERT_NOT_NULL(repo); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int original(void) { return 1; }\n"), 0); + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + cbm_config_t *config = cbm_config_open(cache); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + if (server && config) { + cbm_mcp_server_set_config(server, config); + } + char args[CBM_SZ_4K]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"StoragePolicyFixture\",\"mode\":\"fast\"}", + repo); + char *first = server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + bool first_ok = first && strstr(first, "\\\"status\\\":\\\"indexed\\\""); + free(first); + + char unrelated[CBM_SZ_4K]; + (void)snprintf(unrelated, sizeof(unrelated), "%s/unrelated.bin", cache); + FILE *large = cbm_fopen(unrelated, "wb"); + char block[4096] = {0}; + bool wrote_large = large != NULL; + for (int index = 0; wrote_large && index < 512; index++) { + wrote_large = fwrite(block, 1, sizeof(block), large) == sizeof(block); + } + if (large) { + wrote_large = fclose(large) == 0 && wrote_large; + } + bool configured = + config && cbm_config_set(config, CBM_INDEX_CONFIG_CACHE_MAX_MB, "1") == 0 && + th_write_file(TH_PATH(repo, "main.c"), "int changed(void) { return 2; }\n") == 0; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"StoragePolicyFixture\",\"mode\":\"fast\"," + "\"_cbm_index_policy\":{\"index_max_files\":\"off\"," + "\"index_max_source_mb\":\"off\",\"index_cache_max_mb\":\"off\"," + "\"index_min_free_disk_mb\":\"off\"}}", + repo); + char *limited = configured && wrote_large && server + ? cbm_mcp_handle_tool(server, "index_repository", args) + : NULL; + bool contract_ok = limited && strstr(limited, "resource_limit_exceeded") && + strstr(limited, "\\\"stage\\\":\\\"storage\\\"") && + strstr(limited, "\\\"resource\\\":\\\"cache_bytes\\\"") && + strstr(limited, "\\\"limit\\\":1048576") && + strstr(limited, "\\\"unit\\\":\\\"bytes\\\"") && + strstr(limited, "\\\"serving_index_preserved\\\":true"); + free(limited); + int64_t unrelated_size = cbm_file_size(unrelated); + char db_path[CBM_SZ_4K]; + (void)snprintf(db_path, sizeof(db_path), "%s/StoragePolicyFixture.db", cache); + cbm_store_t *store = cbm_store_open_path_query(db_path); + bool old_queryable = store && cbm_store_count_nodes(store, "StoragePolicyFixture") > 0; + cbm_store_close(store); + + cbm_mcp_server_free(server); + cbm_config_close(config); + th_cleanup(repo); + th_cleanup(cache); + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); + + ASSERT_TRUE(first_ok); + ASSERT_TRUE(wrote_large); + ASSERT_TRUE(configured); + ASSERT_TRUE(contract_ok); + ASSERT_EQ(unrelated_size, (int64_t)sizeof(block) * 512); + ASSERT_TRUE(old_queryable); + PASS(); +} + +SUITE(index_policy) { + RUN_TEST(index_policy_defaults_are_disabled); + RUN_TEST(index_policy_balanced_profile_expands_exact_baseline); + RUN_TEST(index_policy_balanced_rss_follows_the_host_share); + RUN_TEST(index_policy_strict_profile_expands_exact_baseline); + RUN_TEST(index_policy_explicit_override_and_off_replace_profile_dimensions); + RUN_TEST(index_policy_host_clamp_only_tightens_explicit_rss); + RUN_TEST(index_policy_profile_validation_is_atomic); + RUN_TEST(index_policy_file_limit_accepts_off_and_exact_range); + RUN_TEST(index_policy_source_limit_converts_mib_without_overflow); + RUN_TEST(index_policy_worker_limits_validate_and_convert_units); + RUN_TEST(index_policy_storage_limits_validate_and_convert_units); + RUN_TEST(index_policy_storage_projection_boundaries_are_overflow_safe); + RUN_TEST(index_policy_invalid_value_is_rejected_atomically); + RUN_TEST(index_policy_format_round_trips_public_values); + RUN_TEST(index_policy_violation_metadata_is_stable); + RUN_TEST(index_policy_config_loads_defaults_values_and_rejects_corruption); + RUN_TEST(index_policy_cli_lists_all_operator_keys); + RUN_TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite); + RUN_TEST(index_policy_worker_rejects_missing_parent_policy); + RUN_TEST(index_policy_worker_contract_preserves_profile_only_dimensions); + RUN_TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index); + RUN_TEST(index_policy_storage_limit_preserves_old_index_and_unrelated_cache_files); +} diff --git a/tests/test_index_supervisor.c b/tests/test_index_supervisor.c index c67f61fc4..87028d471 100644 --- a/tests/test_index_supervisor.c +++ b/tests/test_index_supervisor.c @@ -121,6 +121,24 @@ static bool index_supervisor_test_append_log(const char *path, const char *line) return fclose(file) == 0 && written; } +typedef struct { + uint64_t bytes; + bool succeeds; + int calls; +} index_supervisor_task_temp_fake_t; + +static bool index_supervisor_task_temp_fake(const char *final_db_path, const char *log_path, + const char *response_path, uint64_t *bytes_out, + void *context) { + index_supervisor_task_temp_fake_t *fake = context; + fake->calls++; + if (!final_db_path || !log_path || !response_path || !fake->succeeds) { + return false; + } + *bytes_out = fake->bytes; + return true; +} + static bool index_supervisor_test_append_terminal_backlog(const char *path) { FILE *file = cbm_fopen(path, "ab"); if (!file) { @@ -236,18 +254,21 @@ TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar) { "/tmp/m", CBM_INDEX_WORKER_QUARANTINE_ARG, "/tmp/q", + CBM_INDEX_WORKER_STAGE_TOKEN_ARG, + "task123", NULL, }; cbm_index_worker_invocation_t invocation; - ASSERT_EQ(cbm_index_worker_parse_process_argv(16, valid, &invocation), + ASSERT_EQ(cbm_index_worker_parse_process_argv(18, valid, &invocation), CBM_INDEX_WORKER_ARGV_VALID); - ASSERT_EQ(cbm_daemon_process_role(16, valid), CBM_DAEMON_PROCESS_WORKER); + ASSERT_EQ(cbm_daemon_process_role(18, valid), CBM_DAEMON_PROCESS_WORKER); ASSERT_STR_EQ(invocation.args_json, valid[6]); ASSERT_STR_EQ(invocation.response_out, valid[8]); ASSERT_EQ(invocation.memory_budget_bytes, 1024); ASSERT_TRUE(invocation.single_thread); ASSERT_STR_EQ(invocation.marker_file, valid[13]); ASSERT_STR_EQ(invocation.quarantine_file, valid[15]); + ASSERT_STR_EQ(invocation.stage_token, valid[17]); char *missing_build[] = {"test-runner", "cli", "--index-worker", "index_repository", "{}", "--response-out", @@ -311,6 +332,18 @@ TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar) { CBM_INDEX_WORKER_MEMORY_BUDGET_ARG, "184467440737095516160", NULL}; + char *invalid_stage_token[] = {"test-runner", + "cli", + "--index-worker", + CBM_INDEX_WORKER_BUILD_ARG, + (char *)captured, + "index_repository", + "{}", + "--response-out", + "/tmp/r", + CBM_INDEX_WORKER_STAGE_TOKEN_ARG, + "../escape", + NULL}; char *user_value[] = {"test-runner", "cli", "search_code", "--query", "--index-worker", NULL}; ASSERT_EQ(cbm_index_worker_parse_process_argv(7, missing_build, &invocation), CBM_INDEX_WORKER_ARGV_INVALID); @@ -327,6 +360,9 @@ TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar) { ASSERT_EQ(cbm_index_worker_parse_process_argv(11, overflow_budget, &invocation), CBM_INDEX_WORKER_ARGV_INVALID); ASSERT_EQ(cbm_daemon_process_role(11, overflow_budget), CBM_DAEMON_PROCESS_INVALID); + ASSERT_EQ(cbm_index_worker_parse_process_argv(11, invalid_stage_token, &invocation), + CBM_INDEX_WORKER_ARGV_INVALID); + ASSERT_EQ(cbm_daemon_process_role(11, invalid_stage_token), CBM_DAEMON_PROCESS_INVALID); ASSERT_EQ(cbm_index_worker_parse_process_argv(5, user_value, &invocation), CBM_INDEX_WORKER_ARGV_INVALID); ASSERT_EQ(cbm_daemon_process_role(5, user_value), CBM_DAEMON_PROCESS_INVALID); @@ -849,6 +885,305 @@ TEST(index_supervisor_killed_worker_log_is_never_empty_and_names_the_run) { PASS(); } +typedef struct { + uint64_t now_ms; + uint64_t rss_values[4]; + int rss_value_count; + int rss_calls; + int clock_calls; + cbm_proc_tree_rss_status_t rss_status; +} index_supervisor_resource_fake_t; + +static uint64_t index_supervisor_fake_clock(void *context) { + index_supervisor_resource_fake_t *fake = context; + fake->clock_calls++; + return fake->now_ms; +} + +static cbm_proc_tree_rss_status_t index_supervisor_fake_rss(cbm_subprocess_t *process, + uint64_t *rss_bytes, void *context) { + (void)process; + index_supervisor_resource_fake_t *fake = context; + int value_index = + fake->rss_calls < fake->rss_value_count ? fake->rss_calls : fake->rss_value_count - 1; + fake->rss_calls++; + if (fake->rss_status == CBM_PROC_TREE_RSS_OK && rss_bytes && value_index >= 0) { + *rss_bytes = fake->rss_values[value_index]; + } + return fake->rss_status; +} + +static cbm_index_resource_policy_t index_supervisor_test_worker_policy(uint64_t rss_bytes, + uint64_t duration_ms) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_rss_bytes = (cbm_index_limit_u64_t){.enabled = rss_bytes > 0, .value = rss_bytes}; + policy.max_duration_ms = + (cbm_index_limit_u64_t){.enabled = duration_ms > 0, .value = duration_ms}; + return policy; +} + +TEST(index_supervisor_disabled_limits_do_not_probe) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {UINT64_MAX}, + .rss_value_count = 1, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"clean\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + cbm_index_resource_t resource = + result ? result->resource_violation.resource : CBM_INDEX_RESOURCE_RSS_BYTES; + bool probe_failed = result && result->resource_probe_failed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(terminal); + ASSERT_EQ(fake.rss_calls, 0); + ASSERT_EQ(fake.clock_calls, 0); + ASSERT_EQ(resource, CBM_INDEX_RESOURCE_NONE); + ASSERT_FALSE(probe_failed); + PASS(); +} + +TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {limit, limit + 1}, + .rss_value_count = 2, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t equal_state = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms += 250; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + const cbm_index_worker_result_t *cached = NULL; + bool cached_terminal = + terminal && cbm_index_worker_poll(handle, &cached) == CBM_INDEX_WORKER_POLL_TERMINAL && + cached == result; + bool limited = terminal && result && + result->resource_violation.resource == CBM_INDEX_RESOURCE_RSS_BYTES && + result->resource_violation.observed == limit + 1 && + result->resource_violation.limit == limit && !result->cancellation_requested && + result->tree_quiesced && !result->supervision_failed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(equal_state, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(limited); + ASSERT_TRUE(cached_terminal); + PASS(); +} + +TEST(index_supervisor_duration_is_total_time_not_quiet_timeout) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_EMPTY, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + fake.now_ms = 1100; + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t equal_state = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms = 1101; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool limited = terminal && result && + result->resource_violation.resource == CBM_INDEX_RESOURCE_DURATION_MS && + result->resource_violation.observed == 1001 && + result->resource_violation.limit == 1000 && result->outcome != CBM_PROC_HANG && + result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(equal_state, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(limited); + PASS(); +} + +TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled) { + const char *saved_timeout = getenv("CBM_INDEX_WORKER_TIMEOUT_S"); + char *saved_timeout_copy = saved_timeout ? cbm_strdup(saved_timeout) : NULL; + (void)cbm_setenv("CBM_INDEX_WORKER_TIMEOUT_S", "1", 1); + cbm_index_supervisor_reset_resource_hooks_for_testing(); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 60000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool remained_hang = terminal && result && result->outcome == CBM_PROC_HANG && + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + index_supervisor_test_restore_env("CBM_INDEX_WORKER_TIMEOUT_S", saved_timeout_copy); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(terminal); + ASSERT_TRUE(remained_hang); + PASS(); +} + +TEST(index_supervisor_cancel_precedes_resource_probe) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {limit + 1}, + .rss_value_count = 1, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 1000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + fake.now_ms = 2000; + bool cancel_accepted = handle && cbm_index_worker_request_cancel(handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool cancelled = terminal && result && result->cancellation_requested && + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed && result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(cancel_accepted); + ASSERT_TRUE(cancelled); + ASSERT_EQ(fake.rss_calls, 0); + PASS(); +} + +TEST(index_supervisor_three_failed_rss_probes_fail_closed) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_ERROR, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t first = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + cbm_index_worker_poll_t throttled = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + int calls_before_interval = fake.rss_calls; + fake.now_ms += 250; + cbm_index_worker_poll_t second = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms += 250; + cbm_index_worker_poll_t third = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool failed_closed = terminal && result && result->resource_probe_failed && + result->resource_violation.resource == CBM_INDEX_RESOURCE_RSS_BYTES && + !result->cancellation_requested && result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(first, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(throttled, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(calls_before_interval, 1); + ASSERT_EQ(second, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(third, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(failed_closed); + ASSERT_EQ(fake.rss_calls, 3); + PASS(); +} + +TEST(index_supervisor_task_temp_limit_terminates_as_resource_failure) { + char *root = th_mktempdir("cbm_supervisor_task_temp"); + ASSERT_NOT_NULL(root); + char db_path[INDEX_SUPERVISOR_TEST_PATH_CAP]; + (void)snprintf(db_path, sizeof(db_path), "%s/index.db", root); + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_task_temp_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + index_supervisor_task_temp_fake_t fake = {.bytes = 2, .succeeds = true}; + cbm_index_supervisor_set_task_temp_hook_for_testing(index_supervisor_task_temp_fake, &fake); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = + cbm_index_worker_start_with_storage_policy("{\"__cbm_test_worker\":\"clean\"}", 0, &policy, + db_path, false, NULL, NULL, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool cancelled = result ? result->cancellation_requested : true; + cbm_index_resource_violation_t violation = + result ? result->resource_violation : (cbm_index_resource_violation_t){0}; + cbm_index_supervisor_reset_task_temp_hook_for_testing(); + if (handle) { + cbm_index_worker_destroy(handle); + } + th_cleanup(root); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(terminal); + ASSERT_FALSE(cancelled); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_TASK_TEMP_BYTES); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + ASSERT_TRUE(fake.calls > 0); + PASS(); +} + SUITE(index_supervisor) { RUN_TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar); RUN_TEST(index_supervisor_async_jobs_are_isolated_cancellable_and_terminal_cached); @@ -857,4 +1192,11 @@ SUITE(index_supervisor) { RUN_TEST(index_supervisor_drains_terminal_backlog_into_request_progress_callback); RUN_TEST(index_supervisor_oversized_response_is_contained_and_log_is_retained); RUN_TEST(index_supervisor_killed_worker_log_is_never_empty_and_names_the_run); + RUN_TEST(index_supervisor_disabled_limits_do_not_probe); + RUN_TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree); + RUN_TEST(index_supervisor_duration_is_total_time_not_quiet_timeout); + RUN_TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled); + RUN_TEST(index_supervisor_cancel_precedes_resource_probe); + RUN_TEST(index_supervisor_three_failed_rss_probes_fail_closed); + RUN_TEST(index_supervisor_task_temp_limit_terminates_as_resource_failure); } diff --git a/tests/test_main.c b/tests/test_main.c index ba6f26d45..1562a9650 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -341,7 +341,7 @@ static int tf_maybe_run_index_worker(int argc, char **argv) { free(worker_repo_path); cbm_index_set_worker_role_options(true, invocation.response_out, invocation.single_thread, invocation.marker_file, invocation.quarantine_file, - invocation.memory_budget_bytes); + invocation.memory_budget_bytes, invocation.stage_token); cbm_mem_init_with_cap(0.5, invocation.memory_budget_bytes); tf_index_worker_probe(invocation.args_json, invocation.response_out); cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); @@ -677,6 +677,7 @@ extern void suite_dyn_array(void); extern void suite_str_intern(void); extern void suite_log(void); extern void suite_str_util(void); +extern void suite_index_policy(void); extern void suite_workspace(void); extern void suite_platform(void); extern void suite_diagnostics(void); @@ -924,6 +925,7 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(str_intern); RUN_SELECTED_SUITE(log); RUN_SELECTED_SUITE(str_util); + RUN_SELECTED_SUITE(index_policy); RUN_SELECTED_SUITE(workspace); RUN_SELECTED_SUITE(platform); RUN_SELECTED_SUITE(diagnostics); diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 278d33e68..9472565dc 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -9601,6 +9601,17 @@ TEST(index_supervisor_unsafe_clean_is_never_fallback_or_recovery) { CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL); result.supervision_failed = false; + result.tree_quiesced = true; + result.outcome = CBM_PROC_KILLED; + result.resource_violation = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_TASK_TEMP_BYTES, .observed = 2048, .limit = 1024}; + ASSERT_EQ(cbm_mcp_supervised_result_disposition(0, &result), + CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE); + result.tree_quiesced = false; + ASSERT_EQ(cbm_mcp_supervised_result_disposition(0, &result), + CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL); + result.resource_violation = (cbm_index_resource_violation_t){0}; + result.tree_quiesced = true; result.outcome = CBM_PROC_CRASH; result.response = NULL; ASSERT_EQ(cbm_mcp_supervised_result_disposition(0, &result), @@ -9610,6 +9621,122 @@ TEST(index_supervisor_unsafe_clean_is_never_fallback_or_recovery) { PASS(); } +#ifndef _WIN32 +typedef struct { + int calls; +} index_worker_resource_clock_t; + +static uint64_t index_worker_resource_clock(void *context) { + index_worker_resource_clock_t *clock = context; + clock->calls++; + return clock->calls == 1 ? 100 : 1101; +} + +enum { + IDXRESOURCE_OK = 0, + IDXRESOURCE_SETUP = 71, + IDXRESOURCE_NO_RESPONSE = 72, + IDXRESOURCE_BAD_CONTRACT = 73, + IDXRESOURCE_RETRIED = 74, +}; + +static int index_worker_resource_response_check(const char *repo, const char *cache) { + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + cbm_unsetenv("CBM_INDEX_SUPERVISOR"); + cbm_index_supervisor_mark_host(); + cbm_config_t *config = cbm_config_open(cache); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + if (!config || !server || + cbm_config_set(config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "1") != 0) { + cbm_mcp_server_free(server); + cbm_config_close(config); + return IDXRESOURCE_SETUP; + } + cbm_mcp_server_set_config(server, config); + index_worker_resource_clock_t clock = {0}; + cbm_index_supervisor_set_resource_hooks_for_testing(index_worker_resource_clock, NULL, &clock); + int before = cbm_index_supervisor_spawn_count(); + char args[CBM_SZ_2K]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"mode\":\"fast\"," + "\"_cbm_index_policy\":{\"index_max_files\":\"off\"," + "\"index_max_source_mb\":\"off\",\"index_max_rss_mb\":\"off\"," + "\"index_max_duration_seconds\":\"off\"}}", + repo); + char *response = cbm_mcp_handle_tool(server, "index_repository", args); + int after = cbm_index_supervisor_spawn_count(); + cbm_index_supervisor_reset_resource_hooks_for_testing(); + int result = IDXRESOURCE_OK; + if (!response) { + result = IDXRESOURCE_NO_RESPONSE; + } else if (!strstr(response, "resource_limit_exceeded") || + !strstr(response, "\\\"stage\\\":\\\"worker\\\"") || + !strstr(response, "\\\"resource\\\":\\\"duration_ms\\\"") || + !strstr(response, "\\\"observed\\\":1001") || + !strstr(response, "\\\"limit\\\":1000") || + !strstr(response, "\\\"unit\\\":\\\"milliseconds\\\"")) { + result = IDXRESOURCE_BAD_CONTRACT; + } else if (after - before != 1) { + result = IDXRESOURCE_RETRIED; + } + free(response); + cbm_mcp_server_free(server); + cbm_config_close(config); + return result; +} +#endif + +TEST(index_worker_resource_limit_is_trusted_structured_and_not_retried) { +#ifdef _WIN32 + SKIP_PLATFORM("fork-isolated MCP host harness; supervisor state machine is cross-platform"); +#else + char *repo = th_mktempdir("cbm_worker_resource_repo"); + char *cache = th_mktempdir("cbm_worker_resource_cache"); + ASSERT_NOT_NULL(repo); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int main(void) { return 0; }\n"), 0); + fflush(NULL); + pid_t pid = fork(); + if (pid == 0) { + alarm(60); + _exit(index_worker_resource_response_check(repo, cache)); + } + int status = 0; + bool waited = waitpid(pid, &status, 0) == pid; + int exit_code = waited && WIFEXITED(status) ? WEXITSTATUS(status) : -1; + int signal_code = waited && WIFSIGNALED(status) ? WTERMSIG(status) : 0; + th_cleanup(repo); + th_cleanup(cache); + if (exit_code != IDXRESOURCE_OK) { + printf(" worker resource child exit=%d signal=%d\n", exit_code, signal_code); + } + ASSERT_TRUE(waited); + ASSERT_EQ(signal_code, 0); + ASSERT_EQ(exit_code, IDXRESOURCE_OK); + PASS(); +#endif +} + +TEST(index_supervisor_resource_probe_failure_response_is_structured) { + cbm_index_worker_result_t worker_result = { + .resource_violation = + { + .resource = CBM_INDEX_RESOURCE_CACHE_BYTES, + .limit = 1024, + .probe_failed = true, + }, + }; + char *response = + cbm_mcp_index_worker_resource_response("{\"repo_path\":\"/tmp/missing\"}", &worker_result); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "\"code\":\"resource_probe_failed\"")); + ASSERT_NOT_NULL(strstr(response, "\"stage\":\"storage\"")); + ASSERT_NOT_NULL(strstr(response, "\"resource\":\"cache_bytes\"")); + ASSERT_NOT_NULL(strstr(response, "\"retryable\":true")); + free(response); + PASS(); +} + /* Child-side check: index a tiny fixture and verify it ran IN-PROCESS. * Distinct exit codes so the parent can report the exact failure mode. */ enum { @@ -10540,6 +10667,57 @@ TEST(mcp_auto_watch_false_skips_watcher_on_connect) { PASS(); } +TEST(mcp_inprocess_autoindex_inherits_resource_policy) { + char *root = th_mktempdir("cbm-autoindex-policy-root"); + char *cache = th_mktempdir("cbm-autoindex-policy-cache"); + ASSERT_NOT_NULL(root); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(root, "first.c"), "int first(void) { return 1; }\n"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "second.c"), "int second(void) { return 2; }\n"), 0); + + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + char saved_cwd[CBM_SZ_4K]; + bool cwd_saved = cbm_getcwd(saved_cwd, sizeof(saved_cwd)) != NULL; + bool environment_ready = + cwd_saved && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0 && cbm_chdir(root) == 0; + cbm_config_t *config = environment_ready ? cbm_config_open(cache) : NULL; + bool configured = config && cbm_config_set(config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && + cbm_config_set(config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && + cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "1") == 0; + char *project = configured ? cbm_project_name_from_path(root) : NULL; + char db_path[CBM_SZ_4K] = {0}; + if (project) { + (void)snprintf(db_path, sizeof(db_path), "%s/%s.db", cache, project); + } + + cbm_mcp_server_t *server = configured && project ? cbm_mcp_server_new(NULL) : NULL; + if (server) { + cbm_mcp_server_set_config(server, config); + char *response = cbm_mcp_server_handle( + server, "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"initialize\",\"params\":{}}"); + free(response); + cbm_mcp_server_free(server); /* joins the in-process auto-index thread */ + } + bool limit_preserved_no_index = project && !cbm_file_exists(db_path); + + cbm_config_close(config); + if (cwd_saved) { + (void)cbm_chdir(saved_cwd); + } + restore_cache_dir(saved_cache_copy); + free(saved_cache_copy); + free(project); + th_cleanup(root); + th_cleanup(cache); + + ASSERT_TRUE(environment_ready); + ASSERT_TRUE(configured); + ASSERT_NOT_NULL(server); + ASSERT_TRUE(limit_preserved_no_index); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ * #853 — auto_watch=false must ALSO gate the SUPERVISED fresh-index * watcher registration (keystone × #849 merge interaction) @@ -11401,6 +11579,8 @@ SUITE(mcp) { RUN_TEST(index_repository_supervisor_uses_canonical_session_path); RUN_TEST(index_repository_cli_name_override_issue823); RUN_TEST(index_supervisor_unsafe_clean_is_never_fallback_or_recovery); + RUN_TEST(index_worker_resource_limit_is_trusted_structured_and_not_retried); + RUN_TEST(index_supervisor_resource_probe_failure_response_is_structured); RUN_TEST(index_supervisor_gate_requires_marked_host_issue845); RUN_TEST(index_supervisor_start_failure_is_fail_closed_in_real_host); RUN_TEST(index_bg_paths_route_through_supervisor_issue832); @@ -11471,6 +11651,7 @@ SUITE(mcp) { /* auto_watch gate (distilled from PR #625) */ RUN_TEST(mcp_auto_watch_default_registers_watcher_on_connect); RUN_TEST(mcp_auto_watch_false_skips_watcher_on_connect); + RUN_TEST(mcp_inprocess_autoindex_inherits_resource_policy); RUN_TEST(mcp_auto_watch_false_skips_supervised_autoindex_issue853); } diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 25878a8b3..2a9a5cc2c 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -151,6 +151,240 @@ TEST(pipeline_run_null) { PASS(); } +TEST(pipeline_discovery_limit_returns_exact_violation_without_publishing) { + char *repo = th_mktempdir("cbm_pipeline_discovery_limit"); + ASSERT_NOT_NULL(repo); + ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first;\n"), 0); + ASSERT_EQ(th_write_file(TH_PATH(repo, "second.py"), "second = 2\n"), 0); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/index.db", repo); + cbm_pipeline_t *pipeline = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(pipeline); + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_pipeline_set_resource_policy(pipeline, &policy); + + ASSERT_EQ(cbm_pipeline_run(pipeline), CBM_PIPELINE_RESOURCE_LIMIT); + cbm_index_resource_violation_t violation = {0}; + cbm_pipeline_get_resource_violation(pipeline, &violation); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + ASSERT_FALSE(cbm_file_exists(db_path)); + + cbm_pipeline_free(pipeline); + th_cleanup(repo); + PASS(); +} + +typedef struct { + const char *target_db; + cbm_pipeline_storage_checkpoint_t fail_at; + cbm_index_resource_t failed_resource; + cbm_index_storage_sample_t samples[3]; + bool sample_set[3]; + int calls[3]; + int unexpected_paths; +} pipeline_storage_probe_fake_t; + +static bool pipeline_storage_probe_fake(cbm_pipeline_storage_checkpoint_t checkpoint, + const char *final_db_path, const char *staging_db_path, + cbm_index_storage_sample_t *sample, + cbm_index_resource_t *failed_resource, void *context) { + (void)staging_db_path; + pipeline_storage_probe_fake_t *fake = (pipeline_storage_probe_fake_t *)context; + fake->calls[checkpoint]++; + if (fake->target_db && (!final_db_path || strcmp(fake->target_db, final_db_path) != 0)) { + fake->unexpected_paths++; + } + *sample = (cbm_index_storage_sample_t){ + .current_cache_bytes = 0, + .free_disk_bytes = UINT64_MAX, + }; + if (fake->sample_set[checkpoint]) { + *sample = fake->samples[checkpoint]; + } + if (fake->failed_resource != CBM_INDEX_RESOURCE_NONE && checkpoint == fake->fail_at && + fake->target_db && final_db_path && strcmp(fake->target_db, final_db_path) == 0) { + *failed_resource = fake->failed_resource; + return false; + } + return true; +} + +static int pipeline_stage_artifact_count(const char *directory) { + cbm_dir_t *dir = cbm_opendir(directory); + if (!dir) { + return -1; + } + int count = 0; + cbm_dirent_t *entry; + while ((entry = cbm_readdir(dir)) != NULL) { + if (strstr(entry->name, ".stage.")) { + count++; + } + } + cbm_closedir(dir); + return count; +} + +TEST(pipeline_disabled_storage_policy_performs_no_probes) { + char *repo = th_mktempdir("cbm_pipeline_storage_off"); + ASSERT_NOT_NULL(repo); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int main(void) { return 0; }\n"), 0); + char db_path[CBM_SZ_4K]; + (void)snprintf(db_path, sizeof(db_path), "%s/index.db", repo); + cbm_pipeline_t *pipeline = cbm_pipeline_new(repo, db_path, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + pipeline_storage_probe_fake_t fake = {.target_db = db_path, + .fail_at = CBM_PIPELINE_STORAGE_PREFLIGHT, + .failed_resource = CBM_INDEX_RESOURCE_FREE_DISK_BYTES}; + cbm_pipeline_set_storage_probe_for_testing(pipeline, pipeline_storage_probe_fake, &fake); + + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + ASSERT_EQ(fake.calls[CBM_PIPELINE_STORAGE_PREFLIGHT], 0); + ASSERT_EQ(fake.calls[CBM_PIPELINE_STORAGE_GROWTH], 0); + ASSERT_EQ(fake.calls[CBM_PIPELINE_STORAGE_PREPUBLISH], 0); + cbm_pipeline_free(pipeline); + th_cleanup(repo); + PASS(); +} + +TEST(pipeline_storage_probe_failure_preserves_old_db_and_cleans_stage) { + char *repo = th_mktempdir("cbm_pipeline_storage_probe"); + ASSERT_NOT_NULL(repo); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int original(void) { return 1; }\n"), 0); + char db_path[CBM_SZ_4K]; + (void)snprintf(db_path, sizeof(db_path), "%s/index.db", repo); + cbm_pipeline_t *first = cbm_pipeline_new(repo, db_path, CBM_MODE_FAST); + ASSERT_NOT_NULL(first); + ASSERT_EQ(cbm_pipeline_run(first), 0); + const char *project = cbm_pipeline_project_name(first); + char *project_copy = project ? strdup(project) : NULL; + cbm_pipeline_free(first); + ASSERT_NOT_NULL(project_copy); + + cbm_store_t *before = cbm_store_open_path_query(db_path); + cbm_project_t before_project = {0}; + ASSERT_NOT_NULL(before); + ASSERT_EQ(cbm_store_get_project(before, project_copy, &before_project), CBM_STORE_OK); + char *indexed_at = before_project.indexed_at ? strdup(before_project.indexed_at) : NULL; + cbm_project_free_fields(&before_project); + cbm_store_close(before); + ASSERT_NOT_NULL(indexed_at); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int changed(void) { return 2; }\n"), 0); + + cbm_pipeline_t *second = cbm_pipeline_new(repo, db_path, CBM_MODE_FAST); + ASSERT_NOT_NULL(second); + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.min_free_disk_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_pipeline_set_resource_policy(second, &policy); + pipeline_storage_probe_fake_t fake = {.target_db = db_path, + .fail_at = CBM_PIPELINE_STORAGE_PREPUBLISH, + .failed_resource = CBM_INDEX_RESOURCE_FREE_DISK_BYTES}; + cbm_pipeline_set_storage_probe_for_testing(second, pipeline_storage_probe_fake, &fake); + int rc = cbm_pipeline_run(second); + cbm_index_resource_violation_t violation = {0}; + cbm_pipeline_get_resource_violation(second, &violation); + cbm_pipeline_free(second); + + cbm_store_t *after = cbm_store_open_path_query(db_path); + cbm_project_t after_project = {0}; + bool preserved = after && + cbm_store_get_project(after, project_copy, &after_project) == CBM_STORE_OK && + after_project.indexed_at && strcmp(indexed_at, after_project.indexed_at) == 0; + cbm_project_free_fields(&after_project); + cbm_store_close(after); + int stage_count = pipeline_stage_artifact_count(repo); + free(indexed_at); + free(project_copy); + th_cleanup(repo); + + ASSERT_EQ(rc, CBM_PIPELINE_RESOURCE_LIMIT); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FREE_DISK_BYTES); + ASSERT_TRUE(violation.probe_failed); + ASSERT_TRUE(fake.calls[CBM_PIPELINE_STORAGE_PREPUBLISH] > 0); + ASSERT_TRUE(preserved); + ASSERT_EQ(stage_count, 0); + PASS(); +} + +TEST(pipeline_full_and_incremental_storage_limits_report_identical_failure) { + char *full_repo = th_mktempdir("cbm_pipeline_storage_full"); + char *incremental_repo = th_mktempdir("cbm_pipeline_storage_incremental"); + ASSERT_NOT_NULL(full_repo); + ASSERT_NOT_NULL(incremental_repo); + ASSERT_EQ(th_write_file(TH_PATH(full_repo, "main.c"), "int full(void) { return 1; }\n"), 0); + ASSERT_EQ( + th_write_file(TH_PATH(incremental_repo, "main.c"), "int incremental(void) { return 1; }\n"), + 0); + char full_db[CBM_SZ_4K]; + char incremental_db[CBM_SZ_4K]; + (void)snprintf(full_db, sizeof(full_db), "%s/index.db", full_repo); + (void)snprintf(incremental_db, sizeof(incremental_db), "%s/index.db", incremental_repo); + + cbm_pipeline_t *seed = cbm_pipeline_new(incremental_repo, incremental_db, CBM_MODE_FAST); + ASSERT_NOT_NULL(seed); + ASSERT_EQ(cbm_pipeline_run(seed), 0); + cbm_pipeline_free(seed); + ASSERT_EQ( + th_write_file(TH_PATH(incremental_repo, "main.c"), "int incremental(void) { return 2; }\n"), + 0); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_staging_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 100}; + pipeline_storage_probe_fake_t full_fake = {.target_db = full_db}; + full_fake.sample_set[CBM_PIPELINE_STORAGE_GROWTH] = true; + full_fake.samples[CBM_PIPELINE_STORAGE_GROWTH] = + (cbm_index_storage_sample_t){.free_disk_bytes = UINT64_MAX, .staging_bytes = 101}; + pipeline_storage_probe_fake_t incremental_fake = full_fake; + incremental_fake.target_db = incremental_db; + + cbm_pipeline_t *full = cbm_pipeline_new(full_repo, full_db, CBM_MODE_FAST); + cbm_pipeline_t *incremental = cbm_pipeline_new(incremental_repo, incremental_db, CBM_MODE_FAST); + ASSERT_NOT_NULL(full); + ASSERT_NOT_NULL(incremental); + cbm_pipeline_set_resource_policy(full, &policy); + cbm_pipeline_set_resource_policy(incremental, &policy); + cbm_pipeline_set_storage_probe_for_testing(full, pipeline_storage_probe_fake, &full_fake); + cbm_pipeline_set_storage_probe_for_testing(incremental, pipeline_storage_probe_fake, + &incremental_fake); + int full_rc = cbm_pipeline_run(full); + int incremental_rc = cbm_pipeline_run(incremental); + cbm_index_resource_violation_t full_violation = {0}; + cbm_index_resource_violation_t incremental_violation = {0}; + cbm_pipeline_get_resource_violation(full, &full_violation); + cbm_pipeline_get_resource_violation(incremental, &incremental_violation); + cbm_pipeline_free(full); + cbm_pipeline_free(incremental); + + ASSERT_EQ(full_rc, CBM_PIPELINE_RESOURCE_LIMIT); + ASSERT_EQ(incremental_rc, CBM_PIPELINE_RESOURCE_LIMIT); + ASSERT_EQ(full_violation.resource, CBM_INDEX_RESOURCE_STAGING_BYTES); + ASSERT_EQ(incremental_violation.resource, full_violation.resource); + ASSERT_EQ(incremental_violation.observed, full_violation.observed); + ASSERT_EQ(incremental_violation.limit, full_violation.limit); + ASSERT_STR_EQ(cbm_index_resource_unit(incremental_violation.resource), + cbm_index_resource_unit(full_violation.resource)); + ASSERT_EQ(full_fake.unexpected_paths, 0); + ASSERT_EQ(incremental_fake.unexpected_paths, 0); + ASSERT_EQ(full_fake.calls[CBM_PIPELINE_STORAGE_PREFLIGHT], 1); + ASSERT_EQ(full_fake.calls[CBM_PIPELINE_STORAGE_GROWTH], 1); + ASSERT_EQ(full_fake.calls[CBM_PIPELINE_STORAGE_PREPUBLISH], 0); + ASSERT_EQ(incremental_fake.calls[CBM_PIPELINE_STORAGE_PREFLIGHT], 1); + ASSERT_EQ(incremental_fake.calls[CBM_PIPELINE_STORAGE_GROWTH], 1); + ASSERT_EQ(incremental_fake.calls[CBM_PIPELINE_STORAGE_PREPUBLISH], 0); + ASSERT_EQ(pipeline_stage_artifact_count(full_repo), 0); + ASSERT_EQ(pipeline_stage_artifact_count(incremental_repo), 0); + th_cleanup(full_repo); + th_cleanup(incremental_repo); + PASS(); +} + /* ── Focused: file-backed store persistence ─────────────────────── */ TEST(store_file_persistence) { @@ -2845,6 +3079,44 @@ static void mutate_semantic_input_before_final_manifest(void *userdata) { mutation->write_rc = th_write_file(mutation->path, mutation->replacement); } +TEST(pipeline_late_source_limit_preserves_previous_generation) { + char *repo = th_mktempdir("cbm_pipeline_late_limit"); + ASSERT_NOT_NULL(repo); + ASSERT_EQ(th_write_file(TH_PATH(repo, "first.c"), "int first;\n"), 0); + char db_path[512]; + char late_path[512]; + snprintf(db_path, sizeof(db_path), "%s/index.db", repo); + snprintf(late_path, sizeof(late_path), "%s/late.py", repo); + + manifest_race_mutation_t mutation = { + .path = late_path, + .replacement = "late = 2\n", + .write_rc = -1, + }; + cbm_pipeline_incremental_test_before_final_manifest_once( + mutate_semantic_input_before_final_manifest, &mutation); + cbm_pipeline_t *pipeline = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(pipeline); + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_files = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_pipeline_set_resource_policy(pipeline, &policy); + int rc = cbm_pipeline_run(pipeline); + cbm_index_resource_violation_t violation = {0}; + cbm_pipeline_get_resource_violation(pipeline, &violation); + cbm_pipeline_free(pipeline); + cbm_pipeline_incremental_test_reset_faults(); + + ASSERT_EQ(mutation.write_rc, 0); + ASSERT_EQ(rc, CBM_PIPELINE_RESOURCE_LIMIT); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FILES); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + ASSERT_FALSE(cbm_file_exists(db_path)); + th_cleanup(repo); + PASS(); +} + /* A graph and its exact manifest are one generation. If source bytes change * after extraction but before publication, the mixed generation must be * rejected and the previous live DB preserved. */ @@ -11951,6 +12223,11 @@ SUITE(pipeline) { RUN_TEST(pipeline_cancel); RUN_TEST(pipeline_cancel_null); RUN_TEST(pipeline_run_null); + RUN_TEST(pipeline_discovery_limit_returns_exact_violation_without_publishing); + RUN_TEST(pipeline_disabled_storage_policy_performs_no_probes); + RUN_TEST(pipeline_storage_probe_failure_preserves_old_db_and_cleans_stage); + RUN_TEST(pipeline_full_and_incremental_storage_limits_report_identical_failure); + RUN_TEST(pipeline_late_source_limit_preserves_previous_generation); /* Extraction back-pressure */ RUN_TEST(pipeline_backpressure_futile_nap_disengages); /* Sequential cross-LSP shared registry (ms-typescript quadratic) */ diff --git a/tests/test_platform.c b/tests/test_platform.c index 1b55df699..c5ceb2d0c 100644 --- a/tests/test_platform.c +++ b/tests/test_platform.c @@ -2,6 +2,7 @@ * test_platform.c — RED phase tests for foundation/platform. */ #include "test_framework.h" +#include "test_helpers.h" #include "../src/foundation/compat.h" /* cbm_setenv / cbm_unsetenv (Windows-portable) */ #include "../src/foundation/compat_fs.h" #include "../src/foundation/constants.h" @@ -292,6 +293,93 @@ TEST(platform_file_size) { PASS(); } +TEST(platform_storage_measurement_uses_logical_bytes_without_following_links) { + char *root = th_mktempdir("cbm_storage_measure"); + ASSERT_NOT_NULL(root); + ASSERT_TRUE(cbm_mkdir_p(TH_PATH(root, "nested"), 0700)); + ASSERT_EQ(th_write_file(TH_PATH(root, "first.bin"), "12345"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "nested/second.bin"), "1234567"), 0); +#ifndef _WIN32 + ASSERT_EQ(symlink("/dev/null", TH_PATH(root, "external-link")), 0); +#endif + uint64_t bytes = 0; + uint64_t free_bytes = 0; + ASSERT_TRUE(cbm_directory_size_bytes(root, &bytes)); + ASSERT_EQ(bytes, 12); + ASSERT_TRUE(cbm_filesystem_free_bytes(root, &free_bytes)); + ASSERT_GT(free_bytes, 0); + th_cleanup(root); + PASS(); +} + +TEST(platform_db_artifact_measurement_saturates_and_counts_sidecars) { + char *root = th_mktempdir("cbm_storage_db_artifacts"); + ASSERT_NOT_NULL(root); + char db[CBM_SZ_4K]; + (void)snprintf(db, sizeof(db), "%s/project.db", root); + ASSERT_EQ(th_write_file(db, "12345"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "project.db-wal"), "1234567"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "project.db-shm"), "123"), 0); + uint64_t bytes = 0; + ASSERT_TRUE(cbm_db_artifact_bytes(db, &bytes)); + ASSERT_EQ(bytes, 15); + ASSERT_EQ(th_write_file(TH_PATH(root, "project.db.stage.task123.ABCDEF"), "12345678901"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "project.db.stage.task123.ABCDEF-wal"), "1234567890123"), + 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "project.db.stage.other456.ABCDEF"), "unrelated"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "worker.log"), "12"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "worker.response"), "123"), 0); + ASSERT_TRUE(cbm_index_task_temp_bytes(db, "task123", TH_PATH(root, "worker.log"), + TH_PATH(root, "worker.response"), &bytes)); + ASSERT_EQ(bytes, 29); + ASSERT_TRUE(cbm_index_staging_cleanup(db, "task123")); + ASSERT_FALSE(cbm_file_exists(TH_PATH(root, "project.db.stage.task123.ABCDEF"))); + ASSERT_FALSE(cbm_file_exists(TH_PATH(root, "project.db.stage.task123.ABCDEF-wal"))); + ASSERT_TRUE(cbm_file_exists(TH_PATH(root, "project.db.stage.other456.ABCDEF"))); + ASSERT_TRUE(cbm_file_exists(db)); + ASSERT_TRUE(cbm_file_exists(TH_PATH(root, "worker.log"))); + th_cleanup(root); + PASS(); +} + +#ifndef _WIN32 +TEST(platform_task_temp_measurement_supports_relative_database_paths) { + char *root = th_mktempdir("cbm_task_temp_relative"); + ASSERT_NOT_NULL(root); + char saved_cwd[CBM_SZ_4K]; + ASSERT_NOT_NULL(getcwd(saved_cwd, sizeof(saved_cwd))); + ASSERT_EQ(chdir(root), 0); + + int write_rc = th_write_file("project.db.stage.task123.ABCDEF", "12345"); + int wal_rc = th_write_file("project.db.stage.task123.ABCDEF-wal", "123456"); + int other_rc = th_write_file("project.db.stage.other456.ABCDEF", "unrelated"); + int log_rc = th_write_file("worker.log", "123"); + int response_rc = th_write_file("worker.response", "1234"); + uint64_t bytes = 0; + bool measured = + cbm_index_task_temp_bytes("project.db", "task123", "worker.log", "worker.response", &bytes); + bool cleaned = cbm_index_staging_cleanup("project.db", "task123"); + bool own_stage_removed = !cbm_file_exists("project.db.stage.task123.ABCDEF") && + !cbm_file_exists("project.db.stage.task123.ABCDEF-wal"); + bool other_stage_preserved = cbm_file_exists("project.db.stage.other456.ABCDEF"); + int restore_rc = chdir(saved_cwd); + th_cleanup(root); + + ASSERT_EQ(write_rc, 0); + ASSERT_EQ(wal_rc, 0); + ASSERT_EQ(other_rc, 0); + ASSERT_EQ(log_rc, 0); + ASSERT_EQ(response_rc, 0); + ASSERT_TRUE(measured); + ASSERT_EQ(bytes, 18); + ASSERT_TRUE(cleaned); + ASSERT_TRUE(own_stage_removed); + ASSERT_TRUE(other_stage_preserved); + ASSERT_EQ(restore_rc, 0); + PASS(); +} +#endif + TEST(platform_mmap) { /* mmap this test file and verify first bytes */ size_t sz = 0; @@ -696,6 +784,11 @@ SUITE(platform) { RUN_TEST(platform_file_exists); RUN_TEST(platform_is_dir); RUN_TEST(platform_file_size); + RUN_TEST(platform_storage_measurement_uses_logical_bytes_without_following_links); + RUN_TEST(platform_db_artifact_measurement_saturates_and_counts_sidecars); +#ifndef _WIN32 + RUN_TEST(platform_task_temp_measurement_supports_relative_database_paths); +#endif RUN_TEST(platform_mmap); RUN_TEST(platform_mmap_nonexistent); RUN_TEST(platform_path_helpers_use_per_thread_storage); diff --git a/tests/test_subprocess.c b/tests/test_subprocess.c index 30fd1537d..82caff354 100644 --- a/tests/test_subprocess.c +++ b/tests/test_subprocess.c @@ -82,6 +82,12 @@ TEST(subprocess_outcome_str) { PASS(); } +TEST(subprocess_tree_rss_sum_saturates_on_overflow) { + const uint64_t values[] = {UINT64_MAX - 4, 4, 1}; + ASSERT_EQ(cbm_subprocess_rss_sum_for_testing(values, 3), UINT64_MAX); + PASS(); +} + /* ── Layer 2: real spawn/reap (POSIX) ─────────────────────────────────────── */ #ifndef _WIN32 @@ -430,6 +436,45 @@ TEST(subprocess_spawn_returns_while_child_is_running) { #endif } +TEST(subprocess_tree_rss_measures_contained_descendants) { +#ifdef _WIN32 + SKIP_PLATFORM("native Job Object process-tree RSS runs on Windows CI"); +#else + char pid_path[64]; + ASSERT_TRUE(make_tree_pid_path(pid_path)); + cbm_subprocess_t *process = NULL; + ASSERT_EQ(spawn_ignoring_tree(pid_path, 0, 100, &process), 0); + pid_t parent_pid = 0; + pid_t grandchild_pid = 0; + bool ready = wait_for_tree_pids(pid_path, process, &parent_pid, &grandchild_pid, 3000); + uint64_t rss_bytes = 0; + cbm_proc_tree_rss_status_t rss_status = + ready ? cbm_subprocess_tree_rss_bytes(process, &rss_bytes) : CBM_PROC_TREE_RSS_ERROR; + bool cancel_accepted = cbm_subprocess_request_cancel(process); + cbm_proc_result_t result; + bool terminal = poll_until_terminal(process, 5000, &result); + uint64_t final_rss = UINT64_MAX; + cbm_proc_tree_rss_status_t final_rss_status = + terminal ? cbm_subprocess_tree_rss_bytes(process, &final_rss) : CBM_PROC_TREE_RSS_ERROR; + if (terminal) { + cbm_subprocess_destroy(process); + } else { + force_probe_cleanup(parent_pid, grandchild_pid); + } + (void)unlink(pid_path); + + ASSERT_TRUE(ready); + ASSERT_EQ(rss_status, CBM_PROC_TREE_RSS_OK); + ASSERT_TRUE(rss_bytes > 0); + ASSERT_TRUE(cancel_accepted); + ASSERT_TRUE(terminal); + ASSERT_TRUE(result.tree_quiesced); + ASSERT_EQ(final_rss_status, CBM_PROC_TREE_RSS_EMPTY); + ASSERT_EQ(final_rss, 0); + PASS(); +#endif +} + TEST(subprocess_natural_completion_is_cached_across_polls) { #ifdef _WIN32 SKIP_PLATFORM("POSIX /bin/sh completion-cache probe; native Job Object coverage pending"); @@ -1027,6 +1072,7 @@ SUITE(subprocess) { RUN_TEST(subprocess_classify_non_fault_signal_is_killed); RUN_TEST(subprocess_classify_timeout_dominates); RUN_TEST(subprocess_outcome_str); + RUN_TEST(subprocess_tree_rss_sum_saturates_on_overflow); RUN_TEST(subprocess_run_clean); RUN_TEST(subprocess_run_exit_nonzero); RUN_TEST(subprocess_run_resolves_literal_binary_name_from_path); @@ -1037,6 +1083,7 @@ SUITE(subprocess) { RUN_TEST(subprocess_run_spawn_failure); RUN_TEST(subprocess_run_null_bin_rejected); RUN_TEST(subprocess_spawn_returns_while_child_is_running); + RUN_TEST(subprocess_tree_rss_measures_contained_descendants); RUN_TEST(subprocess_natural_completion_is_cached_across_polls); RUN_TEST(subprocess_cancel_is_idempotent_and_kills_ignoring_tree); RUN_TEST(subprocess_quiet_timeout_kills_ignoring_tree); diff --git a/tests/test_worker_error_response.sh b/tests/test_worker_error_response.sh index 33e2db115..500c57ff7 100755 --- a/tests/test_worker_error_response.sh +++ b/tests/test_worker_error_response.sh @@ -48,7 +48,7 @@ trap cleanup EXIT missing="${tmpdir}/repository-does-not-exist" response="${tmpdir}/worker.response" -args="{\"repo_path\":\"${missing}\",\"mode\":\"fast\"}" +args="{\"repo_path\":\"${missing}\",\"mode\":\"fast\",$(cbm_test_index_worker_policy_json)}" if ! CBM_CACHE_DIR="${tmpdir}/cache-worker" \ "${BINARY}" cli --index-worker \ diff --git a/tests/test_worker_watchdog.sh b/tests/test_worker_watchdog.sh index 7a52603a0..e38be10bf 100755 --- a/tests/test_worker_watchdog.sh +++ b/tests/test_worker_watchdog.sh @@ -109,7 +109,7 @@ SH chmod +x "${tmpdir}/wrapper.sh" CBM_BINARY="${BINARY}" BUILD_FINGERPRINT="${BUILD_FINGERPRINT}" TMPDIR_PATH="${tmpdir}" \ - ARGS_JSON="{\"repo_path\":\"${tmpdir}/repo\"}" \ + ARGS_JSON="{\"repo_path\":\"${tmpdir}/repo\",$(cbm_test_index_worker_policy_json)}" \ CBM_TEST_HANG_ON=hang_me \ CBM_TEST_WORKER_DESCENDANT_PID_FILE="${tmpdir}/descendant.pid" \ "${tmpdir}/wrapper.sh" &