feat(index): enforce opt-in storage resource limits

Makefile.cbm

@@ -224,6 +224,7 @@ FOUNDATION_SRCS = \
     src/foundation/profile.c \
     src/foundation/dump_verify.c \
     src/foundation/limits.c \
+    src/foundation/index_policy.c \
     src/foundation/subprocess.c \
     src/foundation/sha256.c \
     src/foundation/secure_random.c \
@@ -502,6 +503,7 @@ TEST_FOUNDATION_SRCS = \
     tests/test_str_intern.c \
     tests/test_log.c \
     tests/test_str_util.c \
+    tests/test_index_policy.c \
     tests/test_workspace.c \
     tests/test_platform.c \
     tests/test_diagnostics.c \

README.md

@@ -667,9 +667,21 @@ codebase-memory-mcp config list                          # show all settings
 codebase-memory-mcp config set auto_index true           # auto-index on session start
 codebase-memory-mcp config set auto_index_limit 50000    # max files for auto-index
 codebase-memory-mcp config set auto_watch false          # don't register background git watcher (default: true)
+codebase-memory-mcp config set index_max_files 250000    # optional per-index source-file limit
+codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit
+codebase-memory-mcp config set index_max_rss_mb 8192     # optional worker-tree current RSS limit
+codebase-memory-mcp config set index_max_duration_seconds 3600 # optional total worker duration
+codebase-memory-mcp config set index_cache_max_mb 32768  # optional projected cache-size limit
+codebase-memory-mcp config set index_min_free_disk_mb 4096 # optional free-space reserve
 codebase-memory-mcp config reset auto_index              # reset to default
 ```
 
+The six index resource settings default to `off`. Exceeding one fails the
+complete index attempt rather than publishing a partial graph; an existing
+serving index is preserved. Storage probes also fail closed when an enabled
+measurement cannot be completed. See
+[Index resource limits](docs/INDEX_RESOURCE_LIMITS.md).
+
 ### Environment Variables
 
 | Variable | Default | Description |

docs/CONFIGURATION.md

@@ -86,6 +86,23 @@ Current keys:
 |---|---|---|
 | `auto_index` | `false` | Automatically index new projects when an MCP session starts. |
 | `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. |
+| `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. |
+| `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. |
+| `index_max_rss_mb` | `off` | Optional maximum current RSS in MiB for the complete contained index-worker process tree (`64..1048576`). |
+| `index_max_duration_seconds` | `off` | Optional maximum total worker duration in seconds (`1..86400`). |
+| `index_cache_max_mb` | `off` | Optional maximum projected cache size in MiB after publication. |
+| `index_min_free_disk_mb` | `off` | Optional minimum free MiB reserved on the cache filesystem while indexing. |
+
+The six index resource settings are independent and disabled by default.
+They apply to explicit indexing, automatic indexing, and watcher re-indexing,
+but not to `cross-repo-intelligence`, which does not scan or publish repository
+source indexes. Equality is allowed; exceeding a setting fails the complete
+index request and preserves any previously serving database. Worker RSS covers
+descendants and is not the same as the internal `CBM_MEM_BUDGET_MB` allocation
+budget. Total duration is independent of the existing 15-minute no-log-progress
+timeout. See
+[Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and
+error-response details.
 
 ## 3. UI Settings
 

docs/INDEX_RESOURCE_LIMITS.md

@@ -0,0 +1,133 @@
+# Index resource limits
+
+Index resource limits are optional operator controls for repositories whose
+discovery breadth or worker runtime is not known in advance. They are disabled
+by default so existing large-repository workloads retain their current behavior.
+
+## Discovery settings
+
+| Key | Default | Accepted value | Protects |
+|---|---:|---:|---|
+| `index_max_files` | `off` | `off` or `1..10000000` | Accepted source-file count |
+| `index_max_source_mb` | `off` | `off` or `1..1048576` | Accepted source-file bytes |
+
+Set or reset them with the normal configuration command:
+
+```bash
+codebase-memory-mcp config set index_max_files 250000
+codebase-memory-mcp config set index_max_source_mb 16384
+codebase-memory-mcp config reset index_max_files
+```
+
+Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero,
+negative values, suffixes, trailing characters, and values outside the stated
+ranges are rejected without changing the stored value.
+
+## Worker settings
+
+| Key | Default | Accepted value | Protects |
+|---|---:|---:|---|
+| `index_max_rss_mb` | `off` | `off` or `64..1048576` | Current RSS of the complete worker process tree |
+| `index_max_duration_seconds` | `off` | `off` or `1..86400` | Total worker wall-clock duration |
+
+```bash
+codebase-memory-mcp config set index_max_rss_mb 8192
+codebase-memory-mcp config set index_max_duration_seconds 3600
+```
+
+RSS is the current resident memory of the contained worker and every descendant,
+not the worker's allocation budget and not peak memory. This hard watchdog is
+separate from the internal `CBM_MEM_BUDGET_MB` soft budget. The supervisor
+samples RSS at most once every 250 milliseconds so the watchdog does not turn
+full process-table enumeration into a busy loop.
+
+Duration uses a monotonic clock from successful spawn. It is independent of the
+existing 15-minute quiet timeout: continuous log progress does not reset total
+duration, while the quiet timeout continues to identify a worker that stops
+making progress.
+
+Equality is allowed. The first RSS or elapsed-duration observation above its
+limit starts the existing graceful-to-force process-tree shutdown. CBM reports
+terminal only after the tree is quiescent or a bounded containment failure is
+explicitly surfaced. Resource termination is not retried and does not
+quarantine a source file.
+
+If RSS is enabled and three consecutive probes cannot obtain any trustworthy
+tree measurement while the root worker is still running, CBM fails closed with
+`code=resource_probe_failed`.
+
+## Counting and failure semantics
+
+`index_max_files` counts a file only after it passes directory pruning, ignore
+rules, filename and suffix filters, language detection, and the existing
+per-file size rule. `index_max_source_mb` sums the filesystem sizes of that same
+accepted set.
+
+Equality is allowed. The first file or byte that makes an observed value greater
+than its limit stops discovery. CBM discards the partial file list and does not
+publish a partial graph as a complete index.
+
+For an explicit MCP request the error payload contains:
+
+```json
+{
+  "status": "error",
+  "code": "resource_limit_exceeded",
+  "stage": "discovery",
+  "resource": "files",
+  "observed": 250001,
+  "limit": 250000,
+  "unit": "files",
+  "retryable": true,
+  "serving_index_preserved": true,
+  "message": "Index discovery exceeded index_max_files"
+}
+```
+
+The previous database remains available because publication occurs only after a
+complete discovery and successful staged build. If no previous database exists,
+`serving_index_preserved` is false.
+
+Worker limit failures use the same shape with `stage=worker`,
+`resource=rss_bytes` and `unit=bytes`, or `resource=duration_ms` and
+`unit=milliseconds`. RSS measurement failures use `code=resource_probe_failed`
+and omit `observed`, `limit`, and `unit` because no trustworthy observation was
+available.
+
+## Storage settings
+
+| Key | Default | Accepted value | Protects |
+|---|---:|---:|---|
+| `index_cache_max_mb` | `off` | `off` or `1..1048576` | Projected cache bytes after replacement |
+| `index_min_free_disk_mb` | `off` | `off` or `1..1048576` | Free bytes reserved on the cache filesystem |
+
+Set or reset these keys through the same `config set` and `config reset`
+commands. With both keys `off`, indexing does not scan the cache tree or probe
+filesystem capacity.
+
+Projected cache usage is the current cache size, minus the old project
+database and SQLite sidecars only when that generation is confirmed valid and
+replaceable, plus the current operation's staging artifacts. Other projects
+and unrelated files always count toward the limit and are never evicted.
+
+Free space is checked before staging, after the staged build completes, and
+immediately before atomic publication. Equality is allowed. An enabled
+measurement that cannot be completed fails closed with
+`code: "resource_probe_failed"` and
+`stage: "storage"`. A limit breach uses `code: "resource_limit_exceeded"`.
+Both cases preserve the old serving database. Staging files created by a
+terminated supervised worker are tagged with a private task token and removed
+after its process tree is quiescent; cleanup cannot match another attempt.
+
+## Trust and compatibility
+
+Limits are read from the CLI-managed `_config.db`; they are not MCP request
+arguments. A supervised parent replaces any caller-supplied internal policy
+before spawning its worker, and the worker rejects a missing or incomplete
+parent policy.
+
+These settings do not replace or increase `auto_index_limit`, change the 512 MiB
+single-file cap, alter workspace-root authorization, or affect
+`cross-repo-intelligence`. With all settings `off`, discovery follows the
+existing path, the supervisor performs no periodic RSS probe or total-duration
+termination, and the pipeline performs no cache-tree or free-space probe.

scripts/test-runtime.sh

@@ -105,6 +105,18 @@ _cbm_test_runtime_daemon() {
         CBM_CACHE_DIR="$_CBM_TEST_RUNTIME_PRODUCT_CACHE" "$1" daemon "$2"
 }
 
+# The supervisor resolves one resource policy per index and hands it to the
+# worker in argv; a worker that finds no complete policy refuses to start rather
+# than index unbounded. A shell test that spawns `cli --index-worker` itself
+# stands in for the supervisor and owes the worker the same object. Mirrors
+# cbm_mcp_index_policy_add_to_args: every key in cbm_index_policy_key_at, and
+# nothing else.
+cbm_test_index_worker_policy_json() {
+    printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"'
+    printf '%s' ',"index_max_rss_mb":"off","index_max_duration_seconds":"off"'
+    printf '%s' ',"index_cache_max_mb":"off","index_min_free_disk_mb":"off"}'
+}
+
 cbm_test_runtime_cleanup() {
     local binary="${1:-}" root="$_CBM_TEST_RUNTIME_CREATED_ROOT"
     local name="${_CBM_TEST_RUNTIME_CREATED_ROOT##*/}" runtime_entry="" active=0

src/cli/cli.c

@@ -6734,6 +6734,25 @@ int cbm_config_delete(cbm_config_t *cfg, const char *key) {
     return rc;
 }
 
+bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy,
+                                  char *error, size_t error_size) {
+    if (!cfg || !policy) {
+        if (error && error_size > 0) {
+            (void)snprintf(error, error_size, "index resource configuration is unavailable");
+        }
+        return false;
+    }
+    cbm_index_policy_init(policy);
+    for (size_t index = 0; index < cbm_index_policy_key_count(); index++) {
+        const char *key = cbm_index_policy_key_at(index);
+        const char *value = cbm_config_get(cfg, key, cbm_index_policy_default_value(key));
+        if (!cbm_index_policy_set(policy, key, value, error, error_size)) {
+            return false;
+        }
+    }
+    return true;
+}
+
 /* ── Config CLI subcommand ────────────────────────────────────── */
 
 /* THE config-key table. list, get, help, and key validation all read this one
@@ -6756,6 +6775,13 @@ static const config_key_def_t CONFIG_KEYS[] = {
     {CBM_CONFIG_UI_LANG, "auto", "Pin graph UI language: en, zh, or auto"},
     {CBM_CONFIG_UI_ENABLED, "false", "Serve the graph UI on a loopback HTTP port"},
     {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"},
+    {CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"},
+    {CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"},
+    {CBM_INDEX_CONFIG_MAX_RSS_MB, "off", "Max worker process-tree RSS MiB, or off"},
+    {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "off", "Max worker duration in seconds, or off"},
+    {CBM_INDEX_CONFIG_CACHE_MAX_MB, "off", "Max projected cache MiB after publish, or off"},
+    {CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, "off",
+     "Minimum free cache-filesystem MiB during indexing, or off"},
 };
 
 /* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing
@@ -6781,6 +6807,26 @@ static bool config_key_is_ui(const char *key) {
     return key && (strcmp(key, CBM_CONFIG_UI_ENABLED) == 0 || strcmp(key, CBM_CONFIG_UI_PORT) == 0);
 }
 
+static bool config_key_is_index_policy(const char *key) {
+    for (size_t index = 0; key && index < cbm_index_policy_key_count(); index++) {
+        if (strcmp(key, cbm_index_policy_key_at(index)) == 0) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static int config_index_policy_write(cbm_config_t *config, const char *key, const char *value) {
+    cbm_index_resource_policy_t candidate;
+    cbm_index_policy_init(&candidate);
+    char error[CLI_BUF_256];
+    if (!cbm_index_policy_set(&candidate, key, value, error, sizeof(error))) {
+        (void)fprintf(stderr, "error: %s\n", error);
+        return CLI_ERR;
+    }
+    return cbm_config_set(config, key, value);
+}
+
 static void config_ui_read(const char *key, char *out, size_t out_sz) {
     cbm_ui_config_t ui;
     cbm_ui_config_load(&ui);
@@ -6918,10 +6964,16 @@ int cbm_cmd_config(int argc, char **argv) {
                 rc = CLI_TRUE;
             }
         } else {
-            if (cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]) == 0) {
+            int set_rc =
+                config_key_is_index_policy(argv[CLI_SKIP_ONE])
+                    ? config_index_policy_write(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN])
+                    : cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]);
+            if (set_rc == 0) {
                 printf("%s = %s\n", argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]);
             } else {
-                (void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]);
+                if (!config_key_is_index_policy(argv[CLI_SKIP_ONE])) {
+                    (void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]);
+                }
                 rc = CLI_TRUE;
             }
         }

src/cli/cli.h

@@ -13,6 +13,8 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include "foundation/index_policy.h"
+
 typedef struct cbm_mcp_server cbm_mcp_server_t;
 
 /* ── Version ──────────────────────────────────────────────────── */
@@ -417,6 +419,11 @@ int cbm_config_set(cbm_config_t *cfg, const char *key, const char *value);
 /* Delete a config key. Returns 0 on success. */
 int cbm_config_delete(cbm_config_t *cfg, const char *key);
 
+/* Load and validate the operator-controlled discovery policy. Invalid stored
+ * values fail closed instead of silently disabling a guard. */
+bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy,
+                                  char *error, size_t error_size);
+
 /* Well-known config keys */
 #define CBM_CONFIG_AUTO_INDEX "auto_index"
 #define CBM_CONFIG_AUTO_INDEX_LIMIT "auto_index_limit"