diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index 5a44d4019..b2136a832 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -143,7 +143,7 @@ CI watching must be bounded. Do not loop indefinitely. ## Status Contract -- `status --json` and related JSON/MCP payloads currently expose the trust fields documented in `README.md` and `DEVELOPER_GUIDE.md`, including `fold_ready`, `fold_ready_reason`, `graph_table_available`, `graph_data_current`, `index_complete`, `index_incomplete_reasons`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `workspace_verified_head_sha`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `head_freshness`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `unknown_extension_group_count`, `unknown_extension_groups_truncated`, `unknown_extension_group_limit`, `unknown_extension_group_omitted_count`, `unknown_extension_guidance`, `extractors`, `git_executable`, `github_cli_executable`, `path_case_sensitive`, `data_dir`, `data_dir_source`, `data_dir_mode`, `db_file_mode`, `database_permission_policy`, `database_permission_diagnostics`, `mac_profile`, `mac_profile_diagnostics`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`, WAL checkpoint diagnostics (`read_only_fallback`, `wal_checkpoint_attempted`, `wal_checkpoint_succeeded`, `wal_checkpoint_skipped_reason`, `wal_checkpoint_failure_reason`, `wal_checkpoint_busy`, `wal_checkpoint_log_page_count`, `wal_checkpoint_checkpointed_page_count`, `wal_checkpoint_remaining_page_count`, `read_only_immutable_fallback`, `wal_stale_snapshot_risk`, `wal_stale_snapshot_reason`), `symbol_kinds`, `symbols_by_language`, status kind cap metadata (`symbol_kind_limit`, `symbol_kind_name_limit`, `symbol_kind_total_count`, `symbol_kind_omitted_count`, `symbol_kind_names_truncated`, `symbols_by_language_kind_total_counts`, `symbols_by_language_kind_omitted_counts`, `symbols_by_language_kind_names_truncated`), `process`, `last_index_run`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`, `status_metadata_diagnostics`, `last_workspace_freshened_at`, `hooks`, `hook_diagnostics`, `trust_overrides`, MCP-only `mcp_session`, `mcp.rate_limit.bucket_limit`, `mcp.rate_limit.bucket_limit_rejection_count`, and the `status --check`-only `stale_after_seconds` / `index_age_seconds` threshold audit fields and `repair_commands`. +- `status --json` and related JSON/MCP payloads currently expose the trust fields documented in `README.md` and `DEVELOPER_GUIDE.md`, including `fold_ready`, `fold_ready_reason`, `graph_table_available`, `graph_data_current`, `index_complete`, `index_incomplete_reasons`, `symbol_kind_filter_provenance_available`, `symbol_kind_filter`, `symbols_dropped_by_kind_filter`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `workspace_verified_head_sha`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `head_freshness`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `unknown_extension_group_count`, `unknown_extension_groups_truncated`, `unknown_extension_group_limit`, `unknown_extension_group_omitted_count`, `unknown_extension_guidance`, `extractors`, `git_executable`, `github_cli_executable`, `path_case_sensitive`, `data_dir`, `data_dir_source`, `data_dir_mode`, `db_file_mode`, `database_permission_policy`, `database_permission_diagnostics`, `mac_profile`, `mac_profile_diagnostics`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`, WAL checkpoint diagnostics (`read_only_fallback`, `wal_checkpoint_attempted`, `wal_checkpoint_succeeded`, `wal_checkpoint_skipped_reason`, `wal_checkpoint_failure_reason`, `wal_checkpoint_busy`, `wal_checkpoint_log_page_count`, `wal_checkpoint_checkpointed_page_count`, `wal_checkpoint_remaining_page_count`, `read_only_immutable_fallback`, `wal_stale_snapshot_risk`, `wal_stale_snapshot_reason`), `symbol_kinds`, `symbols_by_language`, status kind cap metadata (`symbol_kind_limit`, `symbol_kind_name_limit`, `symbol_kind_total_count`, `symbol_kind_omitted_count`, `symbol_kind_names_truncated`, `symbols_by_language_kind_total_counts`, `symbols_by_language_kind_omitted_counts`, `symbols_by_language_kind_names_truncated`), `process`, `last_index_run`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`, `status_metadata_diagnostics`, `last_workspace_freshened_at`, `hooks`, `hook_diagnostics`, `trust_overrides`, MCP-only `mcp_session`, `mcp.rate_limit.bucket_limit`, `mcp.rate_limit.bucket_limit_rejection_count`, and the `status --check`-only `stale_after_seconds` / `index_age_seconds` threshold audit fields and `repair_commands`. - Persisted status subdocuments for reference-extraction cap hits, rebuild reclaim, and failed-file diagnostics use a 512 KiB UTF-8 input limit and maximum JSON depth 16 before semantic validation. Preserve the shared 50-item writer/reader limits, nested reason and string budgets documented in `README.md` and `DEVELOPER_GUIDE.md`, omission of only the rejected subdocument, and `status_metadata_diagnostics` reasons `raw_size_exceeded`, `invalid_json`, and `semantic_validation_failed`. Human first-failure and recovery-hint fields must pass through the bounded control-flattening formatter while accepted structured JSON remains unchanged. - `status --check` repair actions are structured by `name`, `action`, `args`, `mutation_class`, `safety_class`, and `safety_notes`. Preserve compatibility `reason` as the first trigger and ordered `reasons` as the complete trigger set. Deduplicate only exact structured identities; different targets, options, actions, mutation classes, or safety semantics must remain distinct. JSON and human output must use the same deduplicated order. Human output must preserve platform-aware shell quoting, visibly escape control characters to keep each repair action on one diagnostic line, and leave structured JSON `args` unchanged. - `maintenance_guidance.fts_optimization` is the shared, read-only recommendation contract for status, explain, optimize preview, and optimize execution. Keep `recommended`, `action`, `reason`, `threshold_writes`, `observed_writes`, and `state` synchronized; stale or unavailable snapshots must not recommend mutation. @@ -157,7 +157,7 @@ CI watching must be bounded. Do not loop indefinitely. - `hotspot_family_degraded_reason` currently uses `hotspot_family_support_not_indexed`, `hotspot_family_metadata_stale`, `hotspot_family_disabled_at_index_time`, `partial_family_key_population`, and `hotspot_family_marker_fingerprint_incomplete`; the incomplete marker fingerprint code means marker traversal hit safety caps and should stay synchronized with README / developer-guide recovery notes. - `issues_table_available` reports physical `file_issues` table presence only. `file_issues_data_current` reports whether the table is also stamped current for the active index generation. - `graph_table_available` reports a queryable persisted reference generation, while `graph_data_current`, `reference_graph_complete`, and `index_complete` report current-generation coverage. Reference extraction is bounded at 50,000 lookup symbols, 20,000 lookup lines, 512 names per line, and 20,000 container candidates; `reference_extraction_limits`, `reference_graph_incomplete_reasons`, and `reference_extraction_cap_hits` publish cap state, and `last_index_run.reference_extraction_cap_hits` snapshots it per run. Cap hits persist per file and propagate degraded, non-authoritative absence semantics to callers, callees, deps, and impact. Indexed Crystal, Groovy, Tcl, Prolog, or `ambiguous_pl` rows with a missing or stale extractor stamp add `dynamic_reference_graph_contract_stale` to `reference_graph_incomplete_reasons` and keep graph readiness false until a normal index refresh rewrites them. A per-file extraction failure keeps successful graph rows queryable, stamps completeness false with bounded `last_failed_or_partial_index_run.file_errors`, and returns exit `11` unless `index --allow-partial` explicitly opts into exit `0`. While such file failures remain unresolved, a later scoped update automatically uses the normal incremental full-scan path so unrelated targets cannot clear the failure and successful recovery can restore every workspace-wide readiness contract without `--rebuild`. -- Successful CLI full/update indexing, immediate status/workspace status, and MCP indexing/status must derive `index_complete`, `index_incomplete_reasons`, `reference_graph_complete`, and `reference_graph_incomplete_reasons` from the same persisted-readiness snapshot. Symbols-only runs and persisted file-size, symbol-count, reference-count, extractor-failure, or reference-cap evidence make the generation incomplete. Legacy databases keep the complete compatibility default only when persisted rows do not prove an omission. +- Successful CLI full/update indexing, immediate status/workspace status, and MCP indexing/status must derive `index_complete`, `index_incomplete_reasons`, `reference_graph_complete`, `reference_graph_incomplete_reasons`, `symbol_kind_filter_provenance_available`, `symbol_kind_filter`, and `symbols_dropped_by_kind_filter` from the same persisted-readiness snapshot. Symbols-only runs and persisted file-size, symbol-count, reference-count, extractor-failure, reference-cap, or active symbol-kind policy evidence make the generation incomplete. Active filters use `symbol_kind_filter_coverage_limited` even at zero drops; legacy databases without a policy stamp remain readable but use `symbol_kind_filter_provenance_unavailable`, and scoped updates remain blocked until a current whole-workspace refresh stamps explicit provenance. Trust per-file filter counts only when both the audit-generation marker and permanent `SymbolKindFilterAuditStorageContractFlag` are present; successful current writers stamp that user-version downgrade guard and ordinary readiness clearing preserves it. Negative-result authority warnings apply to symbol and graph queries, including row-only JSON forms; file, text-search, and validation counts must not inherit symbol-filter degradation. - `index_writer_version` records the `cdidx` version that last wrote to the DB (stamped into `codeindex_meta` as `cdidx_writer_version` on every full scan, update, and MCP index). `index_newer_than_reader` flips to `true` whenever any persisted numeric contract stamp in `codeindex_meta` (or unknown `PRAGMA user_version` readiness bits) exceeds the current binary's compiled maximum, so an older CLI re-opening a DB written by a newer CLI degrades loudly with an audit trail instead of silently dropping back to text-search fallbacks. `index_newer_than_reader_reason` enumerates the specific newer-than-reader stamps. - `status` separates latest-write provenance (`indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`) from whole-workspace verification (`workspace_verified_head_sha`) and the full-scan compatibility baseline (`indexed_head_commit`). Every successful full or partial run updates the latest-write triple; only a full scan or a Git-scoped refresh reconciled from the prior verified baseline advances whole-workspace verification. Scoped mutations persist their affected paths until that reconciled refresh revisits them, including net-zero Git ranges; incomplete pending-path coverage fails closed until a normal full scan succeeds. `worktree_head_changed`, status, map, and MCP use that verified value, falling back conservatively to `indexed_head_commit` for legacy DBs. `head_freshness.state=fresh` requires `status --check` to match the workspace, `fresh_but_incomplete` keeps matching-workspace freshness distinct from incomplete extraction coverage, and `state=head_current` only means the runtime HEAD matches the `indexed_head` selected by `indexed_head_source`. - `status` also surfaces unsupported-language scan coverage via `unknown_extension_file_count`, stamped by successful full-repository index runs (`cdidx index ` and MCP `index_project`) as the number of readable, text-like, non-indexed files whose filename or extension does not map to a known language. Extensionless files without a recognized shebang or a token-delimited first-line `#compdef` zsh directive are included as ``; binary, oversized, unreadable, explicitly ignored, and internal index-artifact files are excluded. Current scans also stamp a diagnostics contract version and `unknown_extension_files` as a path sample bounded by `unknown_extension_file_path_limit` items and the string-list decoded-character budget, `unknown_extension_files_truncated` when more paths existed than were emitted for either bound, and `unknown_extension_file_path_limit` as the item cap rather than a guarantee that that many paths are returned. Newer scans also expose `unknown_extension_extension_counts`, `unknown_extension_category_counts`, and `unknown_extension_groups`; groups classify common non-code buckets such as repository metadata, licenses, binary assets, configuration, structural metadata, and language-support candidates, and include `recommended_action` values of `ignore_configuration`, `first_class_structural_extraction`, or `language_support`. These fields are omitted on legacy DBs, when the persisted diagnostics version is absent or unsupported, or before a current full scan has stamped them. Guidance for actionable extensionless files starts with adding a recognized shebang, adding a first-line `#compdef` directive for zsh completion functions, or renaming the file with a supported extension. diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 725209698..d01b0a08b 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -29,13 +29,14 @@ must refuse writes that could silently discard newer data. | `4` | `fold_ready` | Folded-name columns are current for Unicode-aware exact-name matching. | | `8` | hotspot reference aggregate storage contract | The database uses `hotspot_reference_counts`; this permanent downgrade guard is preserved while other readiness bits are cleared. | | `16` | hotspot reference aggregate readiness | `hotspot_reference_counts` is synchronized with raw reference rows. Writers clear this bit before reference mutations and restore it only after updating the aggregate. | +| `32` | symbol-kind filter audit storage contract | The per-file `symbols_dropped_by_kind_filter` audit belongs to the persisted filter policy. This permanent downgrade guard blocks older writers that cannot maintain the column or audit generation. | -The storage-contract bit is a downgrade guard: binaries that predate the -maintained aggregate see an unknown bit and refuse write-capable opens instead -of leaving hotspot counts stale. It remains set while the separate readiness -bit is transiently cleared during reference mutations. Current query readers -fall back to raw reference rows when readiness is absent; older query-only -readers may still use their normal forward-compatibility degradation behavior. +Storage-contract bits are downgrade guards: binaries that predate the +maintained aggregate or per-file symbol-filter audit see an unknown bit and +refuse write-capable opens instead of leaving evidence stale. They remain set +while transient readiness bits are cleared. Current query readers fall back to +raw reference rows when aggregate readiness is absent; older query-only readers +may still use their normal forward-compatibility degradation behavior. Additional per-feature contract versions live in `codeindex_meta`, including folded-key metadata, C# symbol-name and metadata-target versions, SQL graph @@ -61,6 +62,20 @@ than a new schema bit. Current readers aggregate those rows into legacy database without inspectable issue state degrades rather than claiming complete graph coverage. +Symbol-kind indexing policy uses normalized `index_symbol_kind_filter` metadata, +the successful-generation `index_symbol_kind_filter_audit_version` marker, and +the additive per-file `files.symbols_dropped_by_kind_filter` audit column. +Current writable opens add the column in place. An active legacy generation +without the audit marker omits the count until a full incremental scan +re-extracts every file; scoped updates are rejected instead of mixing audit +generations. A successful current index also stamps user-version bit `32`, so +pre-change binaries refuse later writes that would update the policy without +maintaining its per-file evidence. Read-only legacy DBs without the column remain readable and report +`symbol_kind_filter_provenance_unavailable` when the policy stamp is also +missing. Any active persisted policy reports `symbol_kind_filter_coverage_limited` +and keeps negative symbol/graph results non-authoritative. Rebuild unfiltered to +restore full coverage; the additive column itself does not require a rebuild. + ## Version Skew Behavior Use `cdidx status --json` or `cdidx status --check --json` before relying on a @@ -118,10 +133,11 @@ degrade しなければなりません。古い binary が新しい database を | `4` | `fold_ready` | Unicode-aware exact-name matching 用の folded-name column が最新。 | | `8` | hotspot reference aggregate storage contract | database が `hotspot_reference_counts` を使用することを示す永続 downgrade guard。他の readiness bit のクリア時にも保持される。 | | `16` | hotspot reference aggregate readiness | `hotspot_reference_counts` と raw reference row が同期済み。writer は reference の変更前にこの bit をクリアし、aggregate 更新後だけ復元する。 | +| `32` | symbol-kind filter audit storage contract | file ごとの `symbols_dropped_by_kind_filter` audit が永続 filter policy に属することを示す。column / audit 世代を維持できない旧 writer を拒否する永続 downgrade guard。 | -storage-contract bit は downgrade guard です。maintained aggregate 導入前の binary は -この未知 bit を検知し、hotspot count を stale にする write-capable open を拒否します。 -reference 更新中に別の readiness bit が一時的にクリアされても、この bit は保持されます。 +storage-contract bit は downgrade guard です。maintained aggregate または file ごとの +symbol-filter audit 導入前の binary は未知 bit を検知し、証拠を stale にする +write-capable open を拒否します。一時的な readiness bit のクリア中も保持されます。 現行の query reader は readiness が無い場合に raw reference row へフォールバックし、 旧 query-only reader は通常の forward-compatibility degradation を継続できます。 @@ -147,6 +163,19 @@ reference-extraction cap hit は新しい schema bit ではなく、既存の fi へ集約して `reference_graph_complete=false` にします。issue state を確認できない legacy database は complete graph coverage を主張せず degraded になります。 +symbol-kind indexing policy は正規化済み `index_symbol_kind_filter` metadata、成功世代の +`index_symbol_kind_filter_audit_version` marker、additive な file ごとの +`files.symbols_dropped_by_kind_filter` audit column を使います。現行の writable open は列を +in-place で追加します。audit marker の無い active な legacy generation は、全 file を再抽出する +full incremental scan まで count を省略し、scoped update は audit 世代を混在させず拒否します。 +現行 binary の index 成功時には user-version bit `32` も stamp されるため、policy だけを +更新して file ごとの証拠を維持できない旧 binary は以後の write を拒否します。 +列を持たない read-only legacy DB も読み取り可能で、policy stamp も無い場合は +`symbol_kind_filter_provenance_unavailable` を報告します。active な永続 policy は +`symbol_kind_filter_coverage_limited` を報告し、symbol/graph の否定結果を +non-authoritative に保ちます。full coverage の復元には filter なし rebuild を使いますが、 +additive column 自体のために rebuild する必要はありません。 + ## Version skew 時の動作 binary upgrade / downgrade をまたいで database を使う前に、 diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index db4603545..527a77223 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1411,12 +1411,22 @@ Index-generation completeness is computed by one persisted-readiness reader and reused by the successful full/update index response, immediate status and workspace status, and MCP indexing/status responses. Persisted omission evidence from symbols-only runs, `file_too_large`, `symbol_count_exceeded`, -`reference_count_exceeded`, extractor failures, and reference safety caps makes +`reference_count_exceeded`, extractor failures, reference safety caps, and an +active persisted symbol-kind policy makes `index_complete=false` with stable `index_incomplete_reasons`. `reference_graph_complete` additionally requires an available, current graph -generation and repeats graph-specific stable reasons. A legacy database without -the completeness metadata keeps the compatibility default unless its persisted -rows prove that work was omitted. +generation and repeats graph-specific stable reasons. The normalized +`index_symbol_kind_filter` metadata, the successful-generation +`index_symbol_kind_filter_audit_version` marker, and per-file +`files.symbols_dropped_by_kind_filter` facts are shared by CLI index/status, +workspace health, and MCP indexing/status. An active policy always adds +`symbol_kind_filter_coverage_limited`, including zero-drop runs. A legacy DB +without policy provenance remains readable but adds +`symbol_kind_filter_provenance_unavailable`; this conservative fallback differs +from older completeness metadata, whose compatibility default remains unless +persisted rows prove that work was omitted. An active legacy generation without +the audit marker withholds its aggregate until a whole-workspace refresh has +restamped every per-file fact. Reference extraction publishes its fixed safety limits through CLI `languages --json` / `status --json` and the corresponding MCP responses: @@ -1478,7 +1488,9 @@ Current stable codes and triggers: | `graph_table_available=false` | `symbol_references` is missing or not graph-ready | `cdidx index ` | | `symbols_only_graph_omitted` | the last symbols-only generation intentionally omitted reference-graph rows | run `cdidx index ` without `--symbols-only` | | `reference_graph_complete=false` | the graph generation is unavailable/stale, a symbols-only run omitted it, or persisted file/extractor/cap evidence makes the index generation incomplete | address the reported stable reasons, then run `cdidx index ` | -| `index_complete=false` | a symbols-only run or persisted file-size, symbol-count, reference-count, extractor-failure, or safety-cap evidence proves that indexing work was omitted | address `index_incomplete_reasons`, then run `cdidx index ` | +| `index_complete=false` | persisted omission evidence, an active symbol-kind policy, or unavailable legacy policy provenance means full-generation absence authority is unavailable | address `index_incomplete_reasons`, then rerun indexing; rebuild unfiltered for full coverage | +| `symbol_kind_filter_coverage_limited` | persisted include/exclude policy intentionally limits symbol coverage, including zero-drop runs | `cdidx index --rebuild` without the symbol-kind filter | +| `symbol_kind_filter_provenance_unavailable` | a legacy DB has no persisted policy stamp | rebuild with a current binary; negative symbol/graph results remain non-authoritative until then | | `issues_table_available=false` | `file_issues` is missing or not issue-ready | `cdidx index ` | | `csharp_symbol_name_ready=false` | C# canonical symbol-name stamps are stale | `cdidx index ` | | `csharp_metadata_target_ready=false` | C# metadata-target stamps are stale | `cdidx index ` | @@ -5548,11 +5560,18 @@ index generation の completeness は単一の persisted-readiness reader で計 成功した full/update index response、直後の status / workspace status、MCP の indexing/status response で再利用します。symbols-only run、`file_too_large`、 `symbol_count_exceeded`、`reference_count_exceeded`、extractor failure、 -reference safety cap の永続化済み省略証拠がある場合は +reference safety cap の永続化済み省略証拠、または active な永続 symbol-kind policy がある場合は `index_complete=false` となり、安定した `index_incomplete_reasons` を返します。 `reference_graph_complete` はさらに利用可能かつ current な graph generation を要求し、 -graph 固有の安定した理由を返します。completeness metadata を持たない legacy database は、 -永続化済み row が処理の省略を証明しない限り compatibility default を維持します。 +graph 固有の安定した理由を返します。正規化済み `index_symbol_kind_filter` metadata、成功世代の +`index_symbol_kind_filter_audit_version` marker、file ごとの +`files.symbols_dropped_by_kind_filter` fact は CLI index/status、workspace health、 +MCP indexing/status で共有します。active policy は除外数0でも必ず +`symbol_kind_filter_coverage_limited` を追加します。policy provenance を持たない legacy DB は +読み取り可能なまま `symbol_kind_filter_provenance_unavailable` を追加します。この保守的な +fallback は従来の completeness metadata とは別で、後者は永続 row が処理の省略を証明しない限り +compatibility default を維持します。audit marker の無い active な legacy generation は、 +全 workspace refresh が file ごとの fact をすべて再 stamp するまで aggregate を省略します。 reference extraction の固定 safety limit は lookup symbol 50,000件、lookup line 20,000行、1行あたりの name 512件、container candidate 20,000件で、CLI の @@ -5639,7 +5658,9 @@ alternative action を同じ場所へ追加してください。 | `graph_table_available=false` | `symbol_references` が無い、または graph-ready ではない | `cdidx index ` | | `symbols_only_graph_omitted` | 直前の symbols-only generation が reference-graph row を意図的に省略した | `--symbols-only` を付けずに `cdidx index ` を実行 | | `reference_graph_complete=false` | graph generation が unavailable/stale、symbols-only run で省略、または永続化済み file/extractor/cap 証拠により index generation が incomplete | 報告された安定理由に対処してから `cdidx index ` | -| `index_complete=false` | symbols-only run、または永続化済みの file-size / symbol-count / reference-count / extractor-failure / safety-cap 証拠により indexing work の省略が判明 | `index_incomplete_reasons` に対処してから `cdidx index ` | +| `index_complete=false` | 永続化済み省略証拠、active symbol-kind policy、または legacy policy provenance 不在により generation 全体の不在 authority が利用できない | `index_incomplete_reasons` に対処して再 index。full coverage が必要なら filter なしで rebuild | +| `symbol_kind_filter_coverage_limited` | 永続 include/exclude policy が、除外数0の場合も含めて symbol coverage を意図的に制限 | symbol-kind filter なしで `cdidx index --rebuild` | +| `symbol_kind_filter_provenance_unavailable` | legacy DB に永続 policy stamp が無い | 現行 binary で rebuild。完了まで symbol/graph の否定結果は non-authoritative | | `issues_table_available=false` | `file_issues` が無い、または issue-ready ではない | `cdidx index ` | | `csharp_symbol_name_ready=false` | C# canonical symbol-name stamp が stale | `cdidx index ` | | `csharp_metadata_target_ready=false` | C# metadata-target stamp が stale | `cdidx index ` | diff --git a/README.md b/README.md index f039ae2f6..0166bfe43 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ visible here as a compact compatibility index. | Field group | Fields | |---|---| -| Readiness and graph trust | `fold_ready`, `fold_ready_reason`, `graph_table_available`, `graph_data_current`, `reference_extraction_limits`, `reference_graph_complete`, `reference_graph_incomplete_reasons`, `reference_extraction_cap_hits`, `index_complete`, `index_incomplete_reasons`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`. | +| Readiness and graph trust | `fold_ready`, `fold_ready_reason`, `graph_table_available`, `graph_data_current`, `reference_extraction_limits`, `reference_graph_complete`, `reference_graph_incomplete_reasons`, `reference_extraction_cap_hits`, `index_complete`, `index_incomplete_reasons`, `symbol_kind_filter_provenance_available`, `symbol_kind_filter`, `symbols_dropped_by_kind_filter`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`. | | Language readiness | `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`. | | Workspace and HEAD freshness | `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `head_freshness`. | | Workspace-check path samples | `workspace_check.changed_files`, `workspace_check.missing_files`, `workspace_check.outside_sparse_cone_files`, `workspace_check.unindexed_files`, `workspace_check.unverifiable_files`, and `workspace_check.scan_errors`, each paired with authoritative `*_count`, `*_truncated`, `*_path_limit`, and `*_omitted_count` fields. | @@ -428,7 +428,7 @@ field group を表に残します。 | field group | field | |---|---| -| readiness / graph trust | `fold_ready`、`fold_ready_reason`、`graph_table_available`、`graph_data_current`、`reference_extraction_limits`、`reference_graph_complete`、`reference_graph_incomplete_reasons`、`reference_extraction_cap_hits`、`index_complete`、`index_incomplete_reasons`、`issues_table_available`、`file_issues_data_current`、`migration_in_progress`、`sql_graph_contract_ready`、`sql_graph_contract_degraded_reason`。 | +| readiness / graph trust | `fold_ready`、`fold_ready_reason`、`graph_table_available`、`graph_data_current`、`reference_extraction_limits`、`reference_graph_complete`、`reference_graph_incomplete_reasons`、`reference_extraction_cap_hits`、`index_complete`、`index_incomplete_reasons`、`symbol_kind_filter_provenance_available`、`symbol_kind_filter`、`symbols_dropped_by_kind_filter`、`issues_table_available`、`file_issues_data_current`、`migration_in_progress`、`sql_graph_contract_ready`、`sql_graph_contract_degraded_reason`。 | | language readiness | `hotspot_family_ready`、`hotspot_family_degraded_reason`、`language_readiness`、`csharp_symbol_name_ready`、`csharp_metadata_target_ready`、`csharp_metadata_target_degraded_reason`。 | | workspace / HEAD freshness | `indexed_head_commit`、`worktree_head_changed`、`indexed_head_sha`、`indexed_head_branch`、`indexed_head_timestamp`、`commits_ahead_of_indexed_head`、`head_freshness`。 | | workspace-check の path sample | `workspace_check.changed_files`、`workspace_check.missing_files`、`workspace_check.outside_sparse_cone_files`、`workspace_check.unindexed_files`、`workspace_check.unverifiable_files`、`workspace_check.scan_errors`。各一覧には authoritative な `*_count`、`*_truncated`、`*_path_limit`、`*_omitted_count` が対応します。 | diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 351c4785d..c00a6cdba 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -2834,7 +2834,7 @@ and duplicate targets. cdidx scans your project directory, applies the built-in skip lists plus user `.gitignore` / `.cdidxignore` rules, skips Windows Hidden/System paths before language detection, splits each remaining source file into overlapping chunks, and stores everything in a SQLite database with FTS5 full-text search. In each directory, `.gitignore` is loaded before `.cdidxignore`; later rules are additive, so a `!` pattern in `.cdidxignore` can re-include a path ignored earlier by `.gitignore` in the same directory scope. Incremental mode (default) first purges database entries for files that no longer exist on disk, then checks each file's last-modified timestamp against the database — only files whose timestamp exactly matches are skipped, and any difference (newer or older) triggers re-indexing. Newly appeared files are indexed as new entries. The same path filter is reused for scoped `--files` / `--commits` refreshes, commit-based refreshes automatically switch to a full scan when ignore files changed, and Git-managed workspaces follow the repository's `core.ignorecase` setting when evaluating ignore rules. This means re-indexing after a branch switch only processes the files that actually differ unless ignore rules themselves changed. -At index time, `--include-symbol-kind` keeps only matching symbol kinds and `--exclude-symbol-kind` drops matching symbol kinds before rows are written to `symbols`. Values are comma-separated and case-insensitive. If both filters are present, include is applied first and exclude wins for overlapping kinds. The resolved policy is included in index JSON as `symbol_kind_filter`, and the summary reports `symbols_dropped_by_kind_filter`. Dry-run JSON applies the same policy to its DB-backed `estimated_table_mutations.symbols` estimate and exposes both `symbol_kind_filter` and the estimated `symbols_dropped_by_kind_filter`; reference estimates remain unchanged because symbol-kind filters do not filter reference rows. +At index time, `--include-symbol-kind` keeps only matching symbol kinds and `--exclude-symbol-kind` drops matching symbol kinds before rows are written to `symbols`. Values are comma-separated and case-insensitive. If both filters are present, include is applied first and exclude wins for overlapping kinds. The normalized policy is persisted for the current generation, and each file records its dropped-symbol count. Index, `status --json`, workspace status, and MCP responses expose `symbol_kind_filter_provenance_available`, `symbol_kind_filter`, and the generation-wide `symbols_dropped_by_kind_filter`. Any active policy makes `index_complete=false`, `graph_data_current=false`, and `reference_graph_complete=false` with `symbol_kind_filter_coverage_limited`, even when no symbol happened to be dropped, because negative symbol and graph results do not describe full repository coverage. A legacy DB without policy provenance stays readable but uses the conservative `symbol_kind_filter_provenance_unavailable` fallback; scoped updates are rejected until a whole-workspace refresh stamps current provenance. Row-only JSON array and NDJSON symbol queries emit the same negative-result authority warning on stderr, while file, text-search, and validation counts remain authoritative because symbol-kind filtering does not affect those datasets. Dry-run JSON applies the same policy to its DB-backed `estimated_table_mutations.symbols` estimate; reference estimates remain unchanged because symbol-kind filters do not filter reference rows. ### Incremental update reliability @@ -6496,7 +6496,7 @@ directory link を full scanner と同じ depth-first 順で辿り、解決済 cdidxはプロジェクトディレクトリを走査し、組み込みのスキップ対象とユーザーの `.gitignore` / `.cdidxignore` を適用し、Windows の Hidden/System パスを言語検出前にスキップしたうえで、各ソースファイルを重複を持つチャンクに分割し、FTS5全文検索付きのSQLiteデータベースに格納します。同じディレクトリでは `.gitignore` を先に読み、`.cdidxignore` を後から読むため、後の `.cdidxignore` ルールは加算的に適用され、`!` パターンで同じディレクトリスコープの `.gitignore` 除外を再包含できます。インクリメンタルモード（デフォルト）では各ファイルの最終更新タイムスタンプをDB内の値と比較し、完全一致するファイルのみスキップします。タイムスタンプが異なれば（新しくても古くても）再インデックスされるため、ブランチ切り替え後も正確にインデックスが更新されます。`--files` / `--commits` の部分更新も同じパスフィルタを再利用し、commit 側で ignore ファイルが変わったときは自動でフルスキャンへ切り替わります。Git 管理下の ignore 判定は OS 固定ではなく `core.ignorecase` を参照し、`**` も Git の path-form globstar だけを特別扱いするため、差分更新でも Git と同じ範囲で ignore されます。つまり ignore ルール自体が変わらない限り、差分再インデックスは実際に変わったファイルだけに比例します。 -index 時には `--include-symbol-kind` で一致する kind だけを保持し、`--exclude-symbol-kind` で一致する kind を `symbols` に書き込む前に除外できます。値はカンマ区切りで、大文字小文字は区別しません。両方を指定した場合は include を先に適用し、重複した kind では exclude が優先されます。解決済み policy は index JSON の `symbol_kind_filter` に入り、summary には `symbols_dropped_by_kind_filter` が出ます。dry-run JSON も同じ policy を DB ベースの `estimated_table_mutations.symbols` 推計へ適用し、`symbol_kind_filter` と推計値の `symbols_dropped_by_kind_filter` を出力します。symbol-kind filter は reference 行を絞り込まないため、reference の推計値は変わりません。 +index 時には `--include-symbol-kind` で一致する kind だけを保持し、`--exclude-symbol-kind` で一致する kind を `symbols` に書き込む前に除外できます。値はカンマ区切りで、大文字小文字は区別しません。両方を指定した場合は include を先に適用し、重複した kind では exclude が優先されます。正規化済み policy は current generation 用に永続化され、各 file に除外 symbol 数が記録されます。index、`status --json`、workspace status、MCP response は `symbol_kind_filter_provenance_available`、`symbol_kind_filter`、generation 全体の `symbols_dropped_by_kind_filter` を返します。active な policy があれば、実際の除外数が0でも repository 全体の否定結果を表さないため、`symbol_kind_filter_coverage_limited` とともに `index_complete=false`、`graph_data_current=false`、`reference_graph_complete=false` になります。policy provenance を持たない legacy DB は読み取り可能なまま保守的な `symbol_kind_filter_provenance_unavailable` fallback を使い、workspace 全体の refresh で現行 provenance を stamp するまでは scoped update を拒否します。row-only JSON array / NDJSON の symbol query も0件時の authority warning を stderr に出力しますが、symbol-kind filter の影響を受けない file・text-search・validation count は authoritative のままです。dry-run JSON も同じ policy を DB ベースの `estimated_table_mutations.symbols` 推計へ適用します。symbol-kind filter は reference 行を絞り込まないため、reference の推計値は変わりません。 ### インクリメンタル更新の信頼性 diff --git a/changelog.d/unreleased/5224.fixed.md b/changelog.d/unreleased/5224.fixed.md new file mode 100644 index 000000000..cdce86fdd --- /dev/null +++ b/changelog.d/unreleased/5224.fixed.md @@ -0,0 +1,22 @@ +--- +category: fixed +issues: + - 5224 +affected: + - src/CodeIndex/Database/DbReader.IndexGenerationReadiness.cs + - src/CodeIndex/Database/DbWriter.Files.cs + - src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs + - src/CodeIndex/Cli/WorkspaceCommandRunner.cs + - src/CodeIndex/Mcp/McpToolHandlers.Graph.cs + - USER_GUIDE.md + - DEVELOPER_GUIDE.md + - COMPATIBILITY.md +--- + +## English + +- **Symbol-kind-filtered indexes now persist and publish their coverage limits (#5224)** — CLI, workspace, and MCP readiness now share the normalized include/exclude policy and per-file dropped-symbol facts, protected from older writers by a permanent storage-contract bit. Reserved delimiters and controls are rejected, and formatted or row-only JSON zero-result symbol queries retain an authority warning without degrading unrelated file, text-search, or validation counts. Filtered generations and legacy databases without policy provenance no longer report negative symbol or graph results as authoritative; scoped updates require a whole-workspace provenance refresh. + +## 日本語 + +- **symbol-kind filter を使った index が coverage 制限を永続化・公開するようになりました (#5224)** — CLI、workspace、MCP の readiness は正規化済み include/exclude policy と file ごとの除外 symbol 数を共有し、永続 storage-contract bit で旧 writer から保護します。予約区切り文字と control 文字を拒否し、整形済みまたは row-only JSON の symbol 0 件 query でも authority warning を維持しつつ、影響を受けない file・text-search・validation count は degraded にしません。filter 済み generation と policy provenance のない legacy DB は、symbol / graph の否定結果を authoritative として報告せず、scoped update の前に workspace 全体の provenance refresh を要求します。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs index 545bc31da..880e54c0b 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs @@ -86,14 +86,16 @@ private static int RunDryRun( || (dbSnapshot.SymbolKindFilterSignature == null && !options.SymbolKindFilter.IsActive); if (IsUpdateMode(options) - && !scopedUpdateSymbolKindFilterMatchesPrior) + && (!scopedUpdateSymbolKindFilterMatchesPrior + || (options.SymbolKindFilter.IsActive + && !dbSnapshot.SymbolKindFilterAuditCurrent))) { return WriteCommandError( options.Json, jsonOptions, - "symbol-kind filter policy cannot change during a scoped update because existing files would keep symbols from the prior index policy", + "symbol-kind filter policy cannot change and its per-file audit generation must be current during a scoped update because untouched files would retain incompatible evidence", CommandExitCodes.UsageError, - "Run a full index refresh without --files, --commits, or --changed-between when changing --include-symbol-kind or --exclude-symbol-kind.", + "Run a full index refresh without --files, --commits, or --changed-between to establish one symbol-kind policy and audit generation.", CommandErrorCodes.UsageError); } if (options.MemoryTrace) @@ -1702,6 +1704,8 @@ private static bool IsDryRunReuseAllowed( snapshot.SymbolKindFilterSignature, options.SymbolKindFilter.Signature, StringComparison.Ordinal) + || (options.SymbolKindFilter.IsActive + && !snapshot.SymbolKindFilterAuditCurrent) || !DryRunExtractorContractsMatchCurrent(snapshot, language)) { return false; @@ -1885,6 +1889,12 @@ private static DryRunDbSnapshot ReadDryRunDbSnapshot( if (!DryRunTableExists(connection, "files")) return DryRunDbSnapshot.Empty; + using var userVersionCommand = connection.CreateCommand(); + userVersionCommand.CommandText = "PRAGMA user_version"; + var userVersion = SqliteCommandPolicy.ReadInt32Scalar( + userVersionCommand, + "dry-run user_version"); + var metadata = DryRunReadMetadata(connection); metadata.TryGetValue( DbContext.IndexedProjectRootMetaKey, @@ -2089,6 +2099,7 @@ FROM files f hasSymbolReferences, hasReferenceLines, hasFileIssues, + userVersion, hasCSharpWorkspaceEvidence, ReadFailed: false); } @@ -2335,12 +2346,19 @@ private sealed record DryRunDbSnapshot( bool SymbolReferencesAvailable, bool ReferenceLinesAvailable, bool FileIssuesAvailable, + int UserVersion, bool CSharpWorkspaceEvidenceAvailable, bool ReadFailed) { internal string? SymbolKindFilterSignature => GetMeta(SymbolKindFilterMetaKey); + internal bool SymbolKindFilterAuditCurrent => string.Equals( + GetMeta(SymbolKindFilterAuditVersionMetaKey), + DbContext.SymbolKindFilterAuditVersion, + StringComparison.Ordinal) + && (UserVersion & DbContext.SymbolKindFilterAuditStorageContractFlag) != 0; + internal bool SymbolsOnlyGraphOmitted => string.Equals( GetMeta(DbContext.SymbolsOnlyGraphOmittedMetaKey), "true", @@ -2366,6 +2384,7 @@ internal bool? CSharpStaticInterfaceSourceEvidence false, false, false, + 0, false, ReadFailed: false); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.FilePersistence.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.FilePersistence.cs index 6fa046a79..eefbdff84 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.FilePersistence.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.FilePersistence.cs @@ -216,6 +216,7 @@ private static FullScanFilePersistenceResult PersistFullScanFile( item.Content, familyScopeKey); symbolsDroppedByKindFilter = options.SymbolKindFilter.Apply(mutableSymbols); + writer.SetSymbolsDroppedByKindFilter(fileId, symbolsDroppedByKindFilter); symbols = (IReadOnlyList)mutableSymbols; if (symbols.Count > options.MaxSymbolsPerFile) { diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs index 5e75b4447..6a42a140c 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs @@ -182,6 +182,10 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) SymbolsDroppedByKindFilter = output.SymbolsDroppedByKindFilter, }, SymbolKindFilter = output.Options.SymbolKindFilter.ToJsonResult(), + SymbolKindFilterProvenanceAvailable = + persistedReadinessAfter.SymbolKindFilterPolicy.ProvenanceAvailable, + SymbolsDroppedByKindFilter = + persistedReadinessAfter.SymbolKindFilterPolicy.SymbolsDropped, GraphTableAvailable = persistedReadinessAfter.GraphTableAvailable, GraphDataCurrent = persistedReadinessAfter.GraphDataCurrent, IndexComplete = persistedReadinessAfter.IndexComplete, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.SnapshotFailure.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.SnapshotFailure.cs index b59811600..a96e3f1d1 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.SnapshotFailure.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.SnapshotFailure.cs @@ -100,6 +100,7 @@ private static int WriteFullScanSnapshotFailure( warningCount++; } + var persistedSymbolKindFilterPolicy = signalReader.GetPersistedSymbolKindFilterPolicy(); if (failure.Options.Json) { CommandOutputWriter.WriteLine(JsonSerializer.Serialize(new IndexFullScanJsonResult @@ -132,6 +133,9 @@ private static int WriteFullScanSnapshotFailure( SymbolsDroppedByKindFilter = failure.SymbolsDroppedByKindFilter, }, SymbolKindFilter = failure.Options.SymbolKindFilter.ToJsonResult(), + SymbolKindFilterProvenanceAvailable = + persistedSymbolKindFilterPolicy.ProvenanceAvailable, + SymbolsDroppedByKindFilter = persistedSymbolKindFilterPolicy.SymbolsDropped, GraphTableAvailable = graphTableAvailable, GraphDataCurrent = false, IndexComplete = false, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 0231a337e..7e2f143cf 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -58,6 +58,7 @@ private static int RunFullScan( string? priorIndexedHeadCommit, string? currentHeadCommit, string? priorSymbolKindFilterSignature, + bool priorSymbolKindFilterAuditCurrent, string? initialCwd, List? indexRunDiagnostics, bool showNextSteps, @@ -90,7 +91,8 @@ private static int RunFullScan( var symbolKindFilterMatchesPrior = string.Equals( priorSymbolKindFilterSignature, options.SymbolKindFilter.Signature, - StringComparison.Ordinal); + StringComparison.Ordinal) + && (!options.SymbolKindFilter.IsActive || priorSymbolKindFilterAuditCurrent); var priorFilterRetainedCSharpContractMembers = SymbolKindFilter.SignatureRetainsCSharpStaticInterfaceContractMembers( priorSymbolKindFilterSignature); @@ -525,6 +527,12 @@ bool FullScanJavaScriptTypeScriptConfigChanged() writer.RecoverInterruptedFtsBulkLoadIfNeeded(cancellationToken); writer.MarkBatchInProgress(); writer.ClearReadyFlags(); + if (!symbolKindFilterMatchesPrior) + { + writer.SetMetaValues( + (SymbolKindFilterMetaKey, null), + (SymbolKindFilterAuditVersionMetaKey, null)); + } writer.ClearHotspotFamilyReady(); writer.ClearSqlGraphContractReady(); writer.SetMeta(DbContext.CSharpSymbolNameContractVersionMetaKey, null); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.FileLoop.SerialPersistence.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.FileLoop.SerialPersistence.cs index 4ab9f24eb..2904a75d9 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.FileLoop.SerialPersistence.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.FileLoop.SerialPersistence.cs @@ -150,6 +150,7 @@ private UpdateFilePersistenceResult PersistSerialUpdateFile( loaded.Content, familyScopeKey); symbolsDroppedByKindFilter = options.SymbolKindFilter.Apply(symbols); + writer.SetSymbolsDroppedByKindFilter(fileId, symbolsDroppedByKindFilter); if (symbols.Count > options.MaxSymbolsPerFile) { var issue = BuildSymbolCountExceededIssue( diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.Output.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.Output.cs index d88264ebe..b99fb2bb9 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.Output.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.Output.cs @@ -104,6 +104,10 @@ private static int WriteUpdateFinalOutput(UpdateFinalOutputContext output) FtsMergeRan = output.FtsMergeRan, }, SymbolKindFilter = output.Options.SymbolKindFilter.ToJsonResult(), + SymbolKindFilterProvenanceAvailable = + persistedReadinessAfter.SymbolKindFilterPolicy.ProvenanceAvailable, + SymbolsDroppedByKindFilter = + persistedReadinessAfter.SymbolKindFilterPolicy.SymbolsDropped, GraphTableAvailable = persistedReadinessAfter.GraphTableAvailable, GraphDataCurrent = persistedReadinessAfter.GraphDataCurrent, IndexComplete = persistedReadinessAfter.IndexComplete, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.SnapshotFailure.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.SnapshotFailure.cs index 4a9588f09..2b7c45add 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.SnapshotFailure.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.SnapshotFailure.cs @@ -59,6 +59,7 @@ private static int WriteUpdateSnapshotFailure( var foldReady = (failure.PriorReadiness & DbContext.FoldReadyFlag) != 0; var memoryTimeline = BuildMemoryTimeline(failure.MemorySamples); + var persistedSymbolKindFilterPolicy = signalReader.GetPersistedSymbolKindFilterPolicy(); if (failure.Options.Json) { CommandOutputWriter.WriteLine(JsonSerializer.Serialize(new IndexUpdateJsonResult @@ -81,6 +82,9 @@ private static int WriteUpdateSnapshotFailure( FtsMergeRan = false, }, SymbolKindFilter = failure.Options.SymbolKindFilter.ToJsonResult(), + SymbolKindFilterProvenanceAvailable = + persistedSymbolKindFilterPolicy.ProvenanceAvailable, + SymbolsDroppedByKindFilter = persistedSymbolKindFilterPolicy.SymbolsDropped, GraphTableAvailable = graphTableAvailable, GraphDataCurrent = false, IndexComplete = false, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index 3a975b020..103980852 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -52,6 +52,7 @@ private static int RunUpdateMode( bool priorWorkspaceVerificationPendingPathsComplete, string? currentHeadCommit, string? priorSymbolKindFilterSignature, + bool priorSymbolKindFilterAuditCurrent, string? initialCwd, List? indexRunDiagnostics, CancellationToken cancellationToken) @@ -69,18 +70,6 @@ private static int RunUpdateMode( priorSymbolKindFilterSignature, options.SymbolKindFilter.Signature, StringComparison.Ordinal); - var scopedUpdateSymbolKindFilterMatchesPrior = symbolKindFilterMatchesPrior - || (priorSymbolKindFilterSignature == null && !options.SymbolKindFilter.IsActive); - if (!scopedUpdateSymbolKindFilterMatchesPrior) - { - return WriteCommandError( - options.Json, - jsonOptions, - "symbol-kind filter policy cannot change during a scoped update because existing files would keep symbols from the prior index policy", - CommandExitCodes.UsageError, - "Run a full index refresh without --files, --commits, or --changed-between when changing --include-symbol-kind or --exclude-symbol-kind.", - CommandErrorCodes.UsageError); - } var priorFilterRetainedCSharpContractMembers = SymbolKindFilter.SignatureRetainsCSharpStaticInterfaceContractMembers( priorSymbolKindFilterSignature); @@ -172,6 +161,7 @@ private static int RunUpdateMode( priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, + priorSymbolKindFilterAuditCurrent, initialCwd, indexRunDiagnostics, showNextSteps: false, @@ -180,6 +170,25 @@ private static int RunUpdateMode( forceExtractorRefresh: extractorConfigurationChanged || ambiguousLanguageProjectMarkerChanged); } + // Missing policy provenance is safe only when there are no existing file rows to retain. + // An explicitly unfiltered policy does not depend on per-file dropped-symbol counts, but + // an active filter still requires the current audit generation and downgrade guard. + // policy provenance 不明でも既存 file 行が無ければ保持対象はない。明示的な + // unfiltered policy は file ごとの drop 数に依存しないが、active filter は現 audit 世代と + // downgrade guard を必須とする。 + if (writer.HasIndexedFiles() + && (!symbolKindFilterMatchesPrior + || (options.SymbolKindFilter.IsActive && !priorSymbolKindFilterAuditCurrent))) + { + return WriteCommandError( + options.Json, + jsonOptions, + "symbol-kind filter policy cannot change and its per-file audit generation must be current during a scoped update because untouched files would retain incompatible evidence", + CommandExitCodes.UsageError, + "Run a full index refresh without --files, --commits, or --changed-between to establish one symbol-kind policy and audit generation.", + CommandErrorCodes.UsageError); + } + if (!options.Json && !options.Quiet) CommandOutputWriter.WriteLine($"Updating {ConsoleUi.Counted(targetPaths.Count, "file")}..."); int updated = 0, removed = 0, skipped = 0, warnings = options.OptionWarnings.Count, errors = 0; diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index 861fde354..2fb7910b3 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -30,7 +30,8 @@ public static partial class IndexCommandRunner internal const int MaxGeneratedCodePatternCsvLength = 32_768; internal const int MaxGeneratedCodePatternCount = 128; internal const int MaxGitExcludeBytes = 256 * 1024; - internal const string SymbolKindFilterMetaKey = "index_symbol_kind_filter"; + internal const string SymbolKindFilterMetaKey = DbContext.SymbolKindFilterMetaKey; + internal const string SymbolKindFilterAuditVersionMetaKey = DbContext.SymbolKindFilterAuditVersionMetaKey; private const int MaxIndexRunDiagnosticLength = 512; private const int ScanCheckpointVersion = 1; private const string ScanCheckpointFileName = "scan-checkpoint.json"; @@ -44,15 +45,23 @@ internal static void StampWriterVersionAndSymbolKindFilter( string? writerVersion, string symbolKindFilterSignature) { + // Stamp an old-writer-visible user_version guard in the same successful write scope as + // the policy and audit generation. Pre-#5224 writers then reject writable opens instead + // of preserving a current marker beside stale per-file counts. + // policy / audit 世代と同じ成功 write scope で旧 writer 可視の guard を立てる。 + writer.MarkSymbolKindFilterAuditStorageContract(); if (string.IsNullOrWhiteSpace(writerVersion)) { - writer.SetMeta(SymbolKindFilterMetaKey, symbolKindFilterSignature); + writer.SetMetaValues( + (SymbolKindFilterMetaKey, symbolKindFilterSignature), + (SymbolKindFilterAuditVersionMetaKey, DbContext.SymbolKindFilterAuditVersion)); return; } writer.SetMetaValues( (DbContext.CdidxWriterVersionMetaKey, writerVersion), - (SymbolKindFilterMetaKey, symbolKindFilterSignature)); + (SymbolKindFilterMetaKey, symbolKindFilterSignature), + (SymbolKindFilterAuditVersionMetaKey, DbContext.SymbolKindFilterAuditVersion)); } private sealed record ScanCheckpoint( @@ -490,6 +499,7 @@ private static int RunInitialIndex(IndexRunExecutionContext context) DbContext.SymbolsOnlyGraphOmittedMetaKey, DbContext.IndexedProjectRootMetaKey, SymbolKindFilterMetaKey, + SymbolKindFilterAuditVersionMetaKey, DbContext.IndexedHeadCommitMetaKey, DbContext.WorkspaceVerifiedHeadShaMetaKey, DbContext.WorkspaceVerificationPendingPathsMetaKey, @@ -525,6 +535,11 @@ reason is "file_index_error" var priorHotspotFamilyMarkerFingerprints = GetHotspotFamilyMetaSnapshot(db, DbContext.GetHotspotFamilyMarkerFingerprintMetaKey); var priorIndexedProjectRoot = PriorMeta(DbContext.IndexedProjectRootMetaKey); var priorSymbolKindFilterSignature = PriorMeta(SymbolKindFilterMetaKey); + var priorSymbolKindFilterAuditCurrent = string.Equals( + PriorMeta(SymbolKindFilterAuditVersionMetaKey), + DbContext.SymbolKindFilterAuditVersion, + StringComparison.Ordinal) + && (priorReadiness & DbContext.SymbolKindFilterAuditStorageContractFlag) != 0; // Captured BEFORE `--rebuild` drops the DB so an incremental run can warn the user when // the worktree's HEAD has moved since the previously indexed snapshot. The same value // is read at `status` time (without `--check`) to surface a worktree branch / HEAD @@ -576,8 +591,8 @@ reason is "file_index_error" var projectRoot = Path.GetFullPath(options.ProjectPath!); initialExitCode = isUpdateMode - ? RunUpdateMode(db, writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorReadiness, priorIndexComplete, priorScopedUpdateRequiresFullScan, priorSymbolsOnlyGraphOmitted, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHdlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints!, priorIndexedProjectRoot, priorIndexedHeadCommit, priorWorkspaceVerifiedHead, priorWorkspaceVerificationPendingPaths, priorWorkspaceVerificationPendingPathsComplete, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexRunDiagnostics, indexCancellation.Token) - : RunFullScan(db, writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorReadiness, priorIndexComplete, priorSymbolsOnlyGraphOmitted, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHdlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexRunDiagnostics, showNextSteps: !databaseExistedBeforeIndex, indexCancellation.Token); + ? RunUpdateMode(db, writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorReadiness, priorIndexComplete, priorScopedUpdateRequiresFullScan, priorSymbolsOnlyGraphOmitted, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHdlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints!, priorIndexedProjectRoot, priorIndexedHeadCommit, priorWorkspaceVerifiedHead, priorWorkspaceVerificationPendingPaths, priorWorkspaceVerificationPendingPathsComplete, currentHeadCommit, priorSymbolKindFilterSignature, priorSymbolKindFilterAuditCurrent, initialCwd, indexRunDiagnostics, indexCancellation.Token) + : RunFullScan(db, writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorReadiness, priorIndexComplete, priorSymbolsOnlyGraphOmitted, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHdlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, priorSymbolKindFilterAuditCurrent, initialCwd, indexRunDiagnostics, showNextSteps: !databaseExistedBeforeIndex, indexCancellation.Token); if (initialExitCode == CommandExitCodes.Success) { try @@ -739,6 +754,7 @@ public sealed class IndexCommandOptions public sealed class SymbolKindFilter { + private const int MaxPersistedSignatureLength = 32_768; private static readonly string[] CSharpStaticInterfaceContractMemberKinds = ["function", "operator", "property"]; @@ -765,6 +781,30 @@ private SymbolKindFilter(IReadOnlyList include, IReadOnlyList ex public static SymbolKindFilter Create(IEnumerable include, IEnumerable exclude, string? parseError) { + static string? Validate(IEnumerable values) + { + foreach (var value in values) + { + if (value.Contains(';', StringComparison.Ordinal) + || value.Contains(',', StringComparison.Ordinal)) + { + return "symbol kinds cannot contain the reserved ',' or ';' policy delimiters"; + } + + if (value.Any(char.IsControl)) + return "symbol kinds cannot contain control characters"; + } + + return null; + } + + var includeValues = include.ToArray(); + var excludeValues = exclude.ToArray(); + parseError ??= Validate(includeValues) ?? Validate(excludeValues); + + if (parseError != null) + return new SymbolKindFilter([], [], parseError); + static IReadOnlyList Normalize(IEnumerable values) => values .Select(value => value.Trim()) @@ -773,7 +813,45 @@ static IReadOnlyList Normalize(IEnumerable values) .OrderBy(value => value, StringComparer.OrdinalIgnoreCase) .ToArray(); - return new SymbolKindFilter(Normalize(include), Normalize(exclude), parseError); + return new SymbolKindFilter( + Normalize(includeValues), + Normalize(excludeValues), + parseError); + } + + internal static bool TryParsePersistedSignature(string? signature, out SymbolKindFilter filter) + { + filter = Empty; + const string includePrefix = "include="; + const string excludeSeparator = ";exclude="; + if (string.IsNullOrEmpty(signature) + || signature.Length > MaxPersistedSignatureLength + || !signature.StartsWith(includePrefix, StringComparison.Ordinal)) + { + return false; + } + + var separatorIndex = signature.IndexOf(excludeSeparator, StringComparison.Ordinal); + if (separatorIndex < includePrefix.Length + || signature.IndexOf(excludeSeparator, separatorIndex + excludeSeparator.Length, StringComparison.Ordinal) >= 0) + { + return false; + } + + var includeText = signature[includePrefix.Length..separatorIndex]; + var excludeText = signature[(separatorIndex + excludeSeparator.Length)..]; + if (includeText.Contains(';', StringComparison.Ordinal) + || excludeText.Contains(';', StringComparison.Ordinal)) + { + return false; + } + + filter = Create( + includeText.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries), + excludeText.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries), + parseError: null); + return filter.ParseError == null + && string.Equals(filter.Signature, signature, StringComparison.Ordinal); } internal static bool SignatureRetainsCSharpStaticInterfaceContractMembers(string? signature) diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index 6646315a2..dde3bbcec 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -1117,6 +1117,8 @@ internal sealed class IndexUpdateJsonResult : IVersionedJsonResult public string Mode { get; init; } = string.Empty; public IndexUpdateSummaryJsonResult Summary { get; init; } = new(); public IndexSymbolKindFilterJsonResult SymbolKindFilter { get; init; } = new(); + public bool SymbolKindFilterProvenanceAvailable { get; init; } + public long? SymbolsDroppedByKindFilter { get; init; } public bool GraphTableAvailable { get; init; } public bool GraphDataCurrent { get; init; } public bool IndexComplete { get; init; } @@ -1169,6 +1171,8 @@ internal sealed class IndexFullScanJsonResult : IVersionedJsonResult public bool UnknownExtensionFileCountLowerBound { get; init; } public string? UnknownExtensionGuidance { get; init; } public IndexSymbolKindFilterJsonResult SymbolKindFilter { get; init; } = new(); + public bool SymbolKindFilterProvenanceAvailable { get; init; } + public long? SymbolsDroppedByKindFilter { get; init; } public bool GraphTableAvailable { get; init; } public bool GraphDataCurrent { get; init; } public bool IndexComplete { get; init; } diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Definition.cs b/src/CodeIndex/Cli/QueryCommandRunner.Definition.cs index 6e96c5544..f7f554639 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Definition.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Definition.cs @@ -102,7 +102,8 @@ public static int RunDefinition(string[] cmdArgs, JsonSerializerOptions jsonOpti queryOptions: options, extraFields: options.GroupPartials ? payload => AddLogicalPartialCountJsonFields(payload, logicalCount: 0, physicalCount: 0, physicalFileCount: 0) - : null); + : null, + includeIndexGenerationAuthority: true); var writeExitCode = WriteJsonPayloadWithOptionalByteLimit( zeroPayload, options, @@ -116,6 +117,7 @@ public static int RunDefinition(string[] cmdArgs, JsonSerializerOptions jsonOpti else { Console.WriteLine("0"); + WriteIndexGenerationAuthorityWarningIfNeeded(reader); } return CommandExitCodes.Success; } @@ -181,21 +183,25 @@ public static int RunDefinition(string[] cmdArgs, JsonSerializerOptions jsonOpti WriteExactSymbolWarningIfNeeded(exact, options.Json, exactSignal, reader, options); if (results.Count == 0) { - if (options.Json && TryWriteEmptyFormattedResult(options, jsonOptions)) + if (options.Json && TryWriteEmptyFormattedResult( + options, + jsonOptions, + authorityReader: reader)) return ZeroResultExitCode(options); if (options.Json) { const string hint = "Check the symbol spelling or narrow/adjust --kind, --lang, and --path filters."; - var notFoundJson = JsonSerializer.Serialize( + var notFoundJson = JsonSerializer.SerializeToNode( new CommandErrorJsonResult( "error", BuildZeroResultLine("No definitions found", options), hint, CommandErrorCodes.QueryNotFound, Category: "not_found"), - CliJsonSerializerContextFactory.Create(jsonOptions).CommandErrorJsonResult); + CliJsonSerializerContextFactory.Create(jsonOptions).CommandErrorJsonResult)!.AsObject(); + AddIndexGenerationAuthorityJsonFields(notFoundJson, reader, jsonOptions); var writeExitCode = WriteJsonObjectWithOptionalByteLimit( - notFoundJson, + notFoundJson.ToJsonString(EnsureJsonNodeSerializerOptions(jsonOptions)), options, "definition not-found response", "Increase --max-json-bytes to allow the structured not-found response.", @@ -212,6 +218,7 @@ public static int RunDefinition(string[] cmdArgs, JsonSerializerOptions jsonOpti WriteKindHint(options.Kind, reader); WriteLangHint(options.Lang, reader); WriteZeroResultHints(options, reader, "Try 'search' for full-text matches instead of symbol lookup."); + WriteIndexGenerationAuthorityWarningIfNeeded(reader); } return ZeroResultExitCode(options); } @@ -342,8 +349,24 @@ public static int RunGoto(string[] cmdArgs, JsonSerializerOptions jsonOptions) var results = reader.GetDefinitions(options.Query, limit, options.Kind, options.Lang, includeBody: false, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters, groupPartials: !all); if (results.Count == 0) { + if (options.Json) + { + var payload = JsonSerializer.SerializeToNode( + new CommandErrorJsonResult( + "error", + BuildZeroResultLine("No definitions found", options), + "Check the symbol spelling or narrow/adjust --kind, --lang, and --path filters.", + CommandErrorCodes.QueryNotFound, + Category: "not_found"), + CliJsonSerializerContextFactory.Create(jsonOptions).CommandErrorJsonResult)!.AsObject(); + AddIndexGenerationAuthorityJsonFields(payload, reader, jsonOptions); + Console.WriteLine(payload.ToJsonString(EnsureJsonNodeSerializerOptions(jsonOptions))); + return CommandExitCodes.NotFound; + } + + WriteIndexGenerationAuthorityWarningIfNeeded(reader); return CommandErrorWriter.WriteJsonOrHuman( - options.Json, + false, jsonOptions, BuildZeroResultLine("No definitions found", options), CommandExitCodes.NotFound, diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs b/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs index cfa7e9e2a..497effc7a 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs @@ -139,7 +139,8 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions queryOptions: options, extraFields: options.GroupPartials ? json => AddLogicalPartialCountJsonFields(json, logicalCount: 0, physicalCount: 0, physicalFileCount: 0) - : null); + : null, + includeIndexGenerationAuthority: true); return WriteJsonPayloadWithOptionalByteLimit( payload, options, @@ -250,7 +251,10 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions if (results.Count == 0) { if (IsDiscoveryNdjson(options)) + { + WriteIndexGenerationAuthorityWarningIfNeeded(reader); return ZeroResultExitCode(options); + } if (ShouldWriteBoundedDiscoveryJsonPayload(options)) { var payloadExitCode = WriteBoundedDiscoveryJsonPayload( @@ -269,10 +273,16 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions if (options.OutputFormat == OutputFormatJson) { if (options.JsonOutputFormat == JsonOutputFormatArray) + { + WriteIndexGenerationAuthorityWarningIfNeeded(reader); WriteDiscoveryJsonArray(results, rowFactory, rowExactSignal, jsonOptions); + } return ZeroResultExitCode(options); } - if (TryWriteEmptyFormattedResult(options, jsonOptions)) + if (TryWriteEmptyFormattedResult( + options, + jsonOptions, + authorityReader: reader)) return ZeroResultExitCode(options); if (!options.Json) { @@ -282,6 +292,7 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions WriteLangHint(options.Lang, reader); WriteSymbolExtractionCapabilityHint(options.Lang, reader); WriteZeroResultHints(options, reader); + WriteIndexGenerationAuthorityWarningIfNeeded(reader); } return ZeroResultExitCode(options); } @@ -1079,6 +1090,8 @@ private static JsonObject BuildBoundedDiscoveryPayload( reader.GeneratedFileFilterAvailable); } AddFreshnessHint(payload, reader); + if (totalCount == 0) + AddIndexGenerationAuthorityJsonFields(payload, reader, jsonOptions); return payload; } diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Graph.cs b/src/CodeIndex/Cli/QueryCommandRunner.Graph.cs index cedd6b4a6..e7324761a 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Graph.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Graph.cs @@ -136,7 +136,8 @@ public static int RunReferences(string[] cmdArgs, JsonSerializerOptions jsonOpti if (options.Json && TryWriteEmptyFormattedResult( options, jsonOptions, - extraFields: payload => AddGraphIdentityJsonFields(payload, identityMetadata, jsonOptions))) + extraFields: payload => AddGraphIdentityJsonFields(payload, identityMetadata, jsonOptions), + authorityReader: reader)) return ZeroResultExitCode(options); if (options.Json) WriteGraphZeroJsonResult(reader, "references", jsonOptions, graphAvailable: reader._hasReferencesTable, exact ? exactSignal : (ExactQuerySignal?)null, exactZeroHint, queryOptions: options, extraFields: payload => @@ -379,7 +380,8 @@ public static int RunCallers(string[] cmdArgs, JsonSerializerOptions jsonOptions if (options.Json && TryWriteEmptyFormattedResult( options, jsonOptions, - extraFields: payload => AddGraphIdentityJsonFields(payload, identityMetadata, jsonOptions))) + extraFields: payload => AddGraphIdentityJsonFields(payload, identityMetadata, jsonOptions), + authorityReader: reader)) return ZeroResultExitCode(options); if (options.Json) WriteGraphZeroJsonResult(reader, "callers", jsonOptions, graphAvailable: reader._hasReferencesTable, exact ? exactSignal : (ExactQuerySignal?)null, exactZeroHint, queryOptions: options, extraFields: payload => @@ -605,7 +607,8 @@ public static int RunCallees(string[] cmdArgs, JsonSerializerOptions jsonOptions if (options.Json && TryWriteEmptyFormattedResult( options, jsonOptions, - extraFields: payload => AddGraphIdentityJsonFields(payload, identityMetadata, jsonOptions))) + extraFields: payload => AddGraphIdentityJsonFields(payload, identityMetadata, jsonOptions), + authorityReader: reader)) return ZeroResultExitCode(options); if (options.Json) WriteGraphZeroJsonResult(reader, "callees", jsonOptions, graphAvailable: reader._hasReferencesTable, exact ? exactSignal : (ExactQuerySignal?)null, exactZeroHint, queryOptions: options, extraFields: payload => diff --git a/src/CodeIndex/Cli/QueryCommandRunner.GraphOutput.cs b/src/CodeIndex/Cli/QueryCommandRunner.GraphOutput.cs index c017a5797..48d4ce92a 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.GraphOutput.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.GraphOutput.cs @@ -243,6 +243,7 @@ private static void WriteGraphCountResult(DbReader reader, int count, int files, queryOptions: options, graphTableAvailable: graphAvailable, degraded: !graphAvailable, + includeIndexGenerationAuthority: true, deferAuthority: true); AddGraphSupportOverrideFields(payload, graphSupportOverride); if (options.Exact || options.ExactName) diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Locations.cs b/src/CodeIndex/Cli/QueryCommandRunner.Locations.cs index d94302383..0f971d714 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Locations.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Locations.cs @@ -150,16 +150,43 @@ private static bool TryWriteEmptyFormattedResult( QueryCommandOptions options, JsonSerializerOptions jsonOptions, JsonObject? sarifRunProperties = null, - Action? extraFields = null) + Action? extraFields = null, + DbReader? authorityReader = null) { + var handled = options.OutputFormat is OutputFormatCount + or OutputFormatCompact + or OutputFormatCsv + or OutputFormatTsv + or OutputFormatLsp + or OutputFormatQf + or OutputFormatSarif; + if (!handled) + return false; + + Action? formattedExtraFields = extraFields; + if (authorityReader != null) + { + formattedExtraFields = payload => + { + extraFields?.Invoke(payload); + AddIndexGenerationAuthorityJsonFields(payload, authorityReader, jsonOptions); + }; + // Array, delimited, quickfix, and SARIF schemas cannot carry the shared authority + // object without breaking their output contract. stderr remains a separate terminal + // signal, while count/compact also receive the structured fields above. + // array / delimited / quickfix / SARIF は schema を壊さず authority object を持てない + // ため stderr warning を併用し、count / compact には structured field も加える。 + WriteIndexGenerationAuthorityWarningIfNeeded(authorityReader); + } + if (options.OutputFormat == OutputFormatCount) { - WriteFormattedCount(0, jsonOptions, extraFields); + WriteFormattedCount(0, jsonOptions, formattedExtraFields); return true; } if (options.OutputFormat == OutputFormatCompact) { - WriteCompactLocations([], options, jsonOptions, extraFields); + WriteCompactLocations([], options, jsonOptions, formattedExtraFields); return true; } if (options.OutputFormat == OutputFormatCsv || options.OutputFormat == OutputFormatTsv) diff --git a/src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs b/src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs index 1d433393b..490623273 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs @@ -495,6 +495,7 @@ private static JsonObject BuildCountJsonPayload( FtsQueryDiagnostics? ftsQueryDiagnostics = null, SearchQueryHint? exactSubstringHint = null, Action? extraFields = null, + bool includeIndexGenerationAuthority = false, bool deferAuthority = false) { var payload = new JsonObject @@ -524,6 +525,8 @@ private static JsonObject BuildCountJsonPayload( if (exactSubstringHint != null) payload["exact_substring_hint"] = BuildSearchQueryHintJson(exactSubstringHint); extraFields?.Invoke(payload); + if (count == 0 && includeIndexGenerationAuthority) + AddIndexGenerationAuthorityJsonFields(payload, reader, jsonOptions); AddCountEnvelopeJsonFields(payload, reader, jsonOptions, queryOptions, deferAuthority); return payload; } @@ -612,11 +615,60 @@ private static JsonObject BuildJsonZeroResultPayload( if (queryOptions != null) payload["query_context"] = BuildQueryContextJson(queryOptions, jsonOptions); extraFields?.Invoke(payload); + AddIndexGenerationAuthorityJsonFields(payload, reader, jsonOptions); AddFreshnessHint(payload, reader); return payload; } + private static void AddIndexGenerationAuthorityJsonFields( + JsonObject payload, + DbReader reader, + JsonSerializerOptions jsonOptions) + { + var completion = reader.GetPersistedIndexCompletion(); + if (completion.IndexComplete) + return; + + var policy = completion.SymbolKindFilterPolicy; + payload["index_complete"] = completion.IndexComplete; + payload["symbol_kind_filter_provenance_available"] = policy.ProvenanceAvailable; + if (policy.ProvenanceAvailable) + { + payload["symbol_kind_filter"] = new JsonObject + { + ["include"] = JsonSerializer.SerializeToNode( + policy.Include.ToList(), + CliJsonSerializerContextFactory.Create(jsonOptions).ListString), + ["exclude"] = JsonSerializer.SerializeToNode( + policy.Exclude.ToList(), + CliJsonSerializerContextFactory.Create(jsonOptions).ListString), + }; + } + if (policy.SymbolsDropped.HasValue) + payload["symbols_dropped_by_kind_filter"] = policy.SymbolsDropped.Value; + payload["index_incomplete_reasons"] = JsonSerializer.SerializeToNode( + completion.IndexIncompleteReasons.ToList(), + CliJsonSerializerContextFactory.Create(jsonOptions).ListString); + payload["degraded"] = true; + payload["authoritative_count"] = false; + payload["index_generation_warning"] = + "The persisted index generation is coverage-limited; negative symbol and graph results are not authoritative."; + } + + private static void WriteIndexGenerationAuthorityWarningIfNeeded(DbReader reader) + { + var completion = reader.GetPersistedIndexCompletion(); + if (completion.IndexComplete) + return; + + var reasons = completion.IndexIncompleteReasons.Count == 0 + ? DegradationReasonCodes.IndexIncomplete + : string.Join(", ", completion.IndexIncompleteReasons.Take(4)); + CommandErrorWriter.WriteStderr( + $"WARN: index generation is coverage-limited ({reasons}); negative symbol and graph results are not authoritative."); + } + private static JsonObject BuildSearchQueryHintJson(SearchQueryHint hint) => new() { ["reason"] = hint.Reason, diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Status.cs b/src/CodeIndex/Cli/QueryCommandRunner.Status.cs index ebf39cd56..4781a82ce 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Status.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Status.cs @@ -317,6 +317,14 @@ public static int RunStatus( // #1546: case-sensitivity を診断用に明示する。 if (status.PathCaseSensitive != null) Console.WriteLine(ConsoleUi.FormatSummaryLine("FS Case", status.PathCaseSensitive == true ? "case-sensitive" : "case-insensitive")); + var symbolKindPolicy = status.SymbolKindFilter; + var policyText = symbolKindPolicy == null + ? "unavailable (legacy generation)" + : ConsoleUi.FormatBoundedValue( + $"include={string.Join(",", symbolKindPolicy.Include)};exclude={string.Join(",", symbolKindPolicy.Exclude)}"); + Console.WriteLine(ConsoleUi.FormatSummaryLine("Kind policy", policyText)); + if (status.SymbolsDroppedByKindFilter.HasValue) + Console.WriteLine(ConsoleUi.FormatSummaryLine("Kind drops", $"{status.SymbolsDroppedByKindFilter.Value:N0}")); WriteStatusReadinessSummary(status, options); if (status.WorktreeHeadChanged == true) Console.WriteLine(ConsoleUi.FormatSummaryLine("WARN", $"worktree HEAD changed since the workspace was verified ({ShortSha(verifiedHead)} -> {ShortSha(status.GitHead)}). Run `{BuildReindexRepairCommand(status.ProjectRoot, options.DbPath, options.DbPathExplicit)}` to refresh the index for the current branch.")); @@ -893,7 +901,11 @@ private static List BuildStatusReadinessDegradations if (status.MigrationInProgress) result.Add(BuildStatusReadinessDegradation("migration_in_progress", DegradationReasonCodes.MigrationInProgress, options, status)); if (!status.IndexComplete) - result.Add(BuildStatusReadinessDegradation("index_complete", DegradationReasonCodes.IndexIncomplete, options, status)); + result.Add(BuildStatusReadinessDegradation( + "index_complete", + GetIndexGenerationDegradationRootCause(status), + options, + status)); if (!status.GraphTableAvailable) result.Add(BuildStatusReadinessDegradation("graph_table_available", DegradationReasonCodes.GraphTableMissing, options, status)); if (!status.ReferenceGraphComplete) @@ -919,6 +931,26 @@ private static List BuildStatusReadinessDegradations return result; } + private static string GetIndexGenerationDegradationRootCause(StatusResult status) + { + var reasons = status.IndexIncompleteReasons ?? []; + if (reasons.Contains( + DegradationReasonCodes.SymbolKindFilterProvenanceUnavailable, + StringComparer.Ordinal)) + { + return DegradationReasonCodes.SymbolKindFilterProvenanceUnavailable; + } + + if (reasons.Contains( + DegradationReasonCodes.SymbolKindFilterCoverageLimited, + StringComparer.Ordinal)) + { + return DegradationReasonCodes.SymbolKindFilterCoverageLimited; + } + + return DegradationReasonCodes.IndexIncomplete; + } + private static string GetReferenceGraphDegradationRootCause(StatusResult status) { if (status.ReferenceGraphIncompleteReasons?.Contains( diff --git a/src/CodeIndex/Cli/QueryCommandRunner.StatusFields.cs b/src/CodeIndex/Cli/QueryCommandRunner.StatusFields.cs index f05cac630..ca8e6619c 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.StatusFields.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.StatusFields.cs @@ -69,9 +69,27 @@ private sealed record StatusFieldExplanation( new( "index_complete", "Index generation completeness", - "the latest persisted index generation completed every candidate file.", - "one or more files failed while successful files and their graph rows were committed.", - "Fix the structured per-file failure under `last_failed_or_partial_index_run.file_errors`, then rerun the same index command; a rebuild is not required."), + "the latest persisted index generation completed every candidate file without a symbol-kind coverage filter.", + "one or more files failed, a persisted symbol-kind filter intentionally limited coverage, or legacy filter provenance is unavailable.", + "Inspect `index_incomplete_reasons`: fix structured file failures and rerun the same command, or rebuild without `--include-symbol-kind` / `--exclude-symbol-kind` when authoritative negative symbol and graph results are required."), + new( + "symbol_kind_filter_provenance_available", + "Persisted symbol-kind policy provenance", + "the normalized include/exclude policy for this persisted generation is available.", + "the DB predates persisted policy provenance, so negative symbol and graph results are conservatively non-authoritative.", + "Run `cdidx index --rebuild` with a current binary to stamp an explicit unfiltered or filtered policy."), + new( + "symbol_kind_filter", + "Persisted symbol-kind policy", + "the object publishes normalized `include` and `exclude` lists; empty lists mean an unfiltered generation.", + "a non-empty list intentionally limits generation coverage even when `symbols_dropped_by_kind_filter` is zero.", + "Rebuild without the symbol-kind options when full symbol and graph absence authority is required."), + new( + "symbols_dropped_by_kind_filter", + "Persisted symbol-kind drop count", + "zero means the persisted per-file audit facts recorded no dropped symbols for this generation.", + "a positive value counts symbols intentionally removed by the persisted kind policy; absence means the legacy DB has no per-file audit column.", + "Use the count for bounded audit evidence and inspect `symbol_kind_filter`; rebuild unfiltered for authoritative negative queries."), new( "sql_graph_contract_ready", "SQL graph contract", diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Validate.cs b/src/CodeIndex/Cli/QueryCommandRunner.Validate.cs index c6fe2f33c..1f0cc571c 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Validate.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Validate.cs @@ -1,5 +1,6 @@ using System.Text.Json; using System.Text.Json.Nodes; +using CodeIndex.Database; using CodeIndex.Models; namespace CodeIndex.Cli; @@ -72,6 +73,11 @@ public static int RunValidate(string[] cmdArgs, JsonSerializerOptions jsonOption var severityFilterAvailable = reader.HasIssueSeverityColumn; var requestedFiltersAvailable = options.Severity == null || severityFilterAvailable; var indexCompletion = reader.GetPersistedIndexCompletion(); + var validationCoverageComplete = indexCompletion.IndexComplete + || (indexCompletion.IndexIncompleteReasons.Count > 0 + && indexCompletion.IndexIncompleteReasons.All(reason => reason is + DbReader.SymbolKindFilterCoverageLimitedReason or + DbReader.SymbolKindFilterProvenanceUnavailableReason)); var payload = BuildCountJsonPayload( reader, jsonOptions, @@ -80,7 +86,7 @@ public static int RunValidate(string[] cmdArgs, JsonSerializerOptions jsonOption degraded: !issuesTableAvailable || !fileIssuesDataCurrent || !requestedFiltersAvailable - || !indexCompletion.IndexComplete, + || !validationCoverageComplete, extraFields: countPayload => { // Kept for compatibility with the pre-envelope count shape (#4908). diff --git a/src/CodeIndex/Cli/WorkspaceCommandRunner.cs b/src/CodeIndex/Cli/WorkspaceCommandRunner.cs index 46ec9906f..521d82813 100644 --- a/src/CodeIndex/Cli/WorkspaceCommandRunner.cs +++ b/src/CodeIndex/Cli/WorkspaceCommandRunner.cs @@ -422,7 +422,21 @@ private static WorkspaceMemberIndexHealth ProbeMemberHealth( ReferenceGraphComplete: snapshot.ReferenceGraphComplete, IndexComplete: snapshot.IndexComplete, GraphReady: graphReady, - IndexNewerThanReader: true); + IndexNewerThanReader: true, + IndexIncompleteReasons: snapshot.IndexComplete + ? null + : snapshot.IndexIncompleteReasons, + SymbolKindFilterProvenanceAvailable: + snapshot.SymbolKindFilterPolicy.ProvenanceAvailable, + SymbolKindFilter: snapshot.SymbolKindFilterPolicy.ProvenanceAvailable + ? new IndexSymbolKindFilterJsonResult + { + Include = snapshot.SymbolKindFilterPolicy.Include, + Exclude = snapshot.SymbolKindFilterPolicy.Exclude, + } + : null, + SymbolsDroppedByKindFilter: + snapshot.SymbolKindFilterPolicy.SymbolsDropped); } var freshness = IndexFreshnessChecker.Check( @@ -483,7 +497,21 @@ private static WorkspaceMemberIndexHealth ProbeMemberHealth( ReferenceGraphComplete: snapshot.ReferenceGraphComplete, IndexComplete: snapshot.IndexComplete, GraphReady: graphReady, - IndexNewerThanReader: false); + IndexNewerThanReader: false, + IndexIncompleteReasons: snapshot.IndexComplete + ? null + : snapshot.IndexIncompleteReasons, + SymbolKindFilterProvenanceAvailable: + snapshot.SymbolKindFilterPolicy.ProvenanceAvailable, + SymbolKindFilter: snapshot.SymbolKindFilterPolicy.ProvenanceAvailable + ? new IndexSymbolKindFilterJsonResult + { + Include = snapshot.SymbolKindFilterPolicy.Include, + Exclude = snapshot.SymbolKindFilterPolicy.Exclude, + } + : null, + SymbolsDroppedByKindFilter: + snapshot.SymbolKindFilterPolicy.SymbolsDropped); } catch (Exception ex) when (IsMemberHealthProbeFailure(ex)) { diff --git a/src/CodeIndex/Cli/WorkspaceManifest.cs b/src/CodeIndex/Cli/WorkspaceManifest.cs index f11166f52..37d08c793 100644 --- a/src/CodeIndex/Cli/WorkspaceManifest.cs +++ b/src/CodeIndex/Cli/WorkspaceManifest.cs @@ -46,7 +46,15 @@ internal sealed record WorkspaceMemberIndexHealth( bool? ReferenceGraphComplete = null, bool? IndexComplete = null, bool? GraphReady = null, - bool? IndexNewerThanReader = null); + bool? IndexNewerThanReader = null, + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + IReadOnlyList? IndexIncompleteReasons = null, + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + bool? SymbolKindFilterProvenanceAvailable = null, + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + IndexSymbolKindFilterJsonResult? SymbolKindFilter = null, + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + long? SymbolsDroppedByKindFilter = null); internal sealed record WorkspaceMemberHealthSummary( int MemberCount, diff --git a/src/CodeIndex/Database/DbContext.ReadMigrations.cs b/src/CodeIndex/Database/DbContext.ReadMigrations.cs index 162e4b829..6c0fcdfe3 100644 --- a/src/CodeIndex/Database/DbContext.ReadMigrations.cs +++ b/src/CodeIndex/Database/DbContext.ReadMigrations.cs @@ -215,6 +215,8 @@ PRIMARY KEY(reference_id, symbol_id) yield return ("EnsureColumn files.checksum", () => EnsureColumn("files", "checksum", "TEXT")); yield return ("EnsureColumn files.modified", () => EnsureColumn("files", "modified", "DATETIME")); yield return ("EnsureColumn files.indexed_at", () => EnsureColumn("files", "indexed_at", "DATETIME")); + yield return ($"EnsureColumn files.{SymbolsDroppedByKindFilterColumn}", + () => EnsureColumn("files", SymbolsDroppedByKindFilterColumn, "INTEGER NOT NULL DEFAULT 0")); yield return ("EnsureColumn symbols.start_line", () => EnsureColumn("symbols", "start_line", "INTEGER")); yield return ("EnsureColumn symbols.end_line", () => EnsureColumn("symbols", "end_line", "INTEGER")); yield return ("EnsureColumn symbols.body_start_line", () => EnsureColumn("symbols", "body_start_line", "INTEGER")); diff --git a/src/CodeIndex/Database/DbContext.SchemaInitialization.cs b/src/CodeIndex/Database/DbContext.SchemaInitialization.cs index cadabecac..d40a7e1aa 100644 --- a/src/CodeIndex/Database/DbContext.SchemaInitialization.cs +++ b/src/CodeIndex/Database/DbContext.SchemaInitialization.cs @@ -76,6 +76,7 @@ private void InitializeSchemaInOwnedTransaction(string legacyAlterTable) private bool EnsureCoreSchemaTables() { + var filesTableExisted = TableExists("files"); // Files table / ファイルテーブル Execute(@" CREATE TABLE IF NOT EXISTS files ( @@ -87,6 +88,7 @@ CREATE TABLE IF NOT EXISTS files ( checksum TEXT, modified DATETIME, generated INTEGER NOT NULL DEFAULT 0, + symbols_dropped_by_kind_filter INTEGER NOT NULL DEFAULT 0, indexed_at DATETIME DEFAULT CURRENT_TIMESTAMP )"); @@ -193,6 +195,16 @@ CREATE TABLE IF NOT EXISTS codeindex_meta ( key TEXT PRIMARY KEY NOT NULL, value TEXT )"); + if (!filesTableExisted) + { + Execute($""" + INSERT INTO codeindex_meta(key, value) + VALUES + ('{SymbolKindFilterMetaKey}', 'include=;exclude='), + ('{SymbolKindFilterAuditVersionMetaKey}', '{SymbolKindFilterAuditVersion}') + ON CONFLICT(key) DO NOTHING + """); + } NormalizeCodeIndexMetaKeys(); return backfillHotspotReferenceCounts; } @@ -205,6 +217,7 @@ private void MigrateCoreTableColumns() EnsureColumn("files", "modified", "DATETIME"); EnsureColumn("files", "generated", "INTEGER NOT NULL DEFAULT 0"); EnsureColumn("files", "indexed_at", "DATETIME"); + EnsureColumn("files", SymbolsDroppedByKindFilterColumn, "INTEGER NOT NULL DEFAULT 0"); EnsureColumn("symbols", "start_line", "INTEGER"); EnsureColumn("symbols", "sub_kind", "TEXT"); EnsureColumn("symbols", "start_column", "INTEGER"); diff --git a/src/CodeIndex/Database/DbContext.SchemaMetadata.cs b/src/CodeIndex/Database/DbContext.SchemaMetadata.cs index df5a4c314..ff93a07b8 100644 --- a/src/CodeIndex/Database/DbContext.SchemaMetadata.cs +++ b/src/CodeIndex/Database/DbContext.SchemaMetadata.cs @@ -35,8 +35,15 @@ public partial class DbContext : IDisposable public const int HotspotReferenceAggregateReadyFlag = 16; public const int HotspotReferenceAggregateFlags = HotspotReferenceAggregateStorageContractFlag | HotspotReferenceAggregateReadyFlag; + // bit 5 is a permanent downgrade guard for the per-file symbol-kind audit column. Binaries + // that predate this bit reject writable opens instead of changing the filter signature while + // leaving the audit generation marker and per-file counts stale. + // bit 5 は file ごとの symbol-kind audit 列を旧 writer から守る永続 downgrade guard。 + public const int SymbolKindFilterAuditStorageContractFlag = 32; + public const int PreservedIndexStorageContractFlags = + HotspotReferenceAggregateFlags | SymbolKindFilterAuditStorageContractFlag; public const int CurrentSchemaVersion = - GraphReadyFlag | IssuesReadyFlag | FoldReadyFlag | HotspotReferenceAggregateFlags; // 31 + GraphReadyFlag | IssuesReadyFlag | FoldReadyFlag | PreservedIndexStorageContractFlags; // 63 public const int CodeIndexMetaSchemaVersion = 1; public const string CodeIndexMetaSchemaVersionMetaKey = "codeindex_meta_schema_version"; // Query-semantic readiness for hotspot family grouping. Stored in codeindex_meta instead of @@ -180,6 +187,10 @@ public static string GetDynamicReferenceGraphContractVersionMetaKey(string lang) public const string LastFailedIndexRunFileErrorsMetaKey = "last_failed_index_run_file_errors_json"; public const string IndexCompletenessMetaKey = "index_completeness"; public const string IndexIncompleteReasonsMetaKey = "index_incomplete_reasons_json"; + public const string SymbolKindFilterMetaKey = "index_symbol_kind_filter"; + public const string SymbolKindFilterAuditVersionMetaKey = "index_symbol_kind_filter_audit_version"; + public const string SymbolKindFilterAuditVersion = "1"; + public const string SymbolsDroppedByKindFilterColumn = "symbols_dropped_by_kind_filter"; // Issue #1585: count of files seen by the most recent successful full-repository scan // whose filename or extension did not map to a known language. This is a scan coverage // signal, not an indexed-file count, and is omitted by readers until a current index pass @@ -350,8 +361,8 @@ private void MarkHotspotReferenceAggregateReady() // index 開始時にビットをクリア。途中で落ちた場合は縮退状態のまま残す。 public void ClearReadyFlags() { - var aggregateContractBits = GetUserVersion() & HotspotReferenceAggregateFlags; - Execute($"PRAGMA user_version = {aggregateContractBits}"); + var preservedContractBits = GetUserVersion() & PreservedIndexStorageContractFlags; + Execute($"PRAGMA user_version = {preservedContractBits}"); } /// diff --git a/src/CodeIndex/Database/DbReader.IndexGenerationReadiness.cs b/src/CodeIndex/Database/DbReader.IndexGenerationReadiness.cs index 333e37286..180d41069 100644 --- a/src/CodeIndex/Database/DbReader.IndexGenerationReadiness.cs +++ b/src/CodeIndex/Database/DbReader.IndexGenerationReadiness.cs @@ -1,3 +1,4 @@ +using CodeIndex.Cli; using CodeIndex.Indexer; using CodeIndex.Models; using Microsoft.Data.Sqlite; @@ -7,7 +8,17 @@ namespace CodeIndex.Database; internal sealed record PersistedIndexCompletion( bool IndexComplete, IReadOnlyList IndexIncompleteReasons, - bool MigrationInProgress); + bool MigrationInProgress, + PersistedSymbolKindFilterPolicy SymbolKindFilterPolicy); + +internal sealed record PersistedSymbolKindFilterPolicy( + bool ProvenanceAvailable, + IReadOnlyList Include, + IReadOnlyList Exclude, + long? SymbolsDropped) +{ + internal bool IsActive => Include.Count > 0 || Exclude.Count > 0; +} internal sealed record PersistedIndexGenerationReadiness( bool GraphTableAvailable, @@ -17,7 +28,8 @@ internal sealed record PersistedIndexGenerationReadiness( bool ReferenceGraphComplete, IReadOnlyList ReferenceGraphIncompleteReasons, ReferenceExtractionCapHitSummary ReferenceExtractionCapHits, - bool MigrationInProgress); + bool MigrationInProgress, + PersistedSymbolKindFilterPolicy SymbolKindFilterPolicy); public partial class DbReader { @@ -25,6 +37,10 @@ public partial class DbReader internal const string SymbolsOnlyReferenceGraphIncompleteReason = DegradationReasonCodes.SymbolsOnlyGraphOmitted; internal const string BatchInProgressIncompleteReason = "batch_in_progress"; + internal const string SymbolKindFilterCoverageLimitedReason = + DegradationReasonCodes.SymbolKindFilterCoverageLimited; + internal const string SymbolKindFilterProvenanceUnavailableReason = + DegradationReasonCodes.SymbolKindFilterProvenanceUnavailable; private static readonly string[] IndexOmissionIssueKinds = [ @@ -95,7 +111,8 @@ internal PersistedIndexGenerationReadiness GetPersistedIndexGenerationReadiness( referenceGraphComplete, referenceGraphIncompleteReasons, capHits, - migrationInProgress); + migrationInProgress, + indexCompletion.SymbolKindFilterPolicy); } internal PersistedIndexCompletion GetPersistedIndexCompletion( @@ -109,6 +126,19 @@ internal PersistedIndexCompletion GetPersistedIndexCompletion( _hasIssuesPhysicalTable, ParseMetaBool(TryGetMetaStringInternal(DbContext.SymbolsOnlyGraphOmittedMetaKey)) == true, transaction)); + var symbolKindFilterPolicy = GetPersistedSymbolKindFilterPolicy(transaction); + if (!symbolKindFilterPolicy.ProvenanceAvailable) + { + AddDistinctReason( + indexIncompleteReasons, + SymbolKindFilterProvenanceUnavailableReason); + } + else if (symbolKindFilterPolicy.IsActive) + { + AddDistinctReason( + indexIncompleteReasons, + SymbolKindFilterCoverageLimitedReason); + } var migrationInProgress = string.Equals( TryGetMetaStringInternal(DbContext.BatchInProgressMetaKey), "true", @@ -129,7 +159,55 @@ internal PersistedIndexCompletion GetPersistedIndexCompletion( return new PersistedIndexCompletion( indexComplete, indexIncompleteReasons, - migrationInProgress); + migrationInProgress, + symbolKindFilterPolicy); + } + + internal PersistedSymbolKindFilterPolicy GetPersistedSymbolKindFilterPolicy( + SqliteTransaction? transaction = null) + { + var signature = TryGetMetaStringInternal(DbContext.SymbolKindFilterMetaKey); + var provenanceAvailable = SymbolKindFilter.TryParsePersistedSignature( + signature, + out var filter); + var auditCurrent = string.Equals( + TryGetMetaStringInternal(DbContext.SymbolKindFilterAuditVersionMetaKey), + DbContext.SymbolKindFilterAuditVersion, + StringComparison.Ordinal) + && (_userVersion & DbContext.SymbolKindFilterAuditStorageContractFlag) != 0; + long? symbolsDropped = null; + if (provenanceAvailable + && (!filter.IsActive || auditCurrent) + && _fileColumns.Contains(DbContext.SymbolsDroppedByKindFilterColumn)) + { + using var command = _conn.CreateCommand(); + command.Transaction = transaction; + command.CommandText = $""" + SELECT COALESCE(SUM( + CASE + WHEN {DbContext.SymbolsDroppedByKindFilterColumn} BETWEEN 0 AND 2147483647 + THEN {DbContext.SymbolsDroppedByKindFilterColumn} + ELSE 0 + END), 0) + FROM files + """; + symbolsDropped = Convert.ToInt64( + command.ExecuteScalar(), + System.Globalization.CultureInfo.InvariantCulture); + } + else if (provenanceAvailable && !filter.IsActive) + { + // An unfiltered generation cannot have policy drops, even when the per-file + // audit column predates the opened DB. + // filter 無し generation なら、per-file audit 列が無い旧 DB でも policy drop は 0。 + symbolsDropped = 0; + } + + return new PersistedSymbolKindFilterPolicy( + provenanceAvailable, + provenanceAvailable ? filter.Include : [], + provenanceAvailable ? filter.Exclude : [], + symbolsDropped); } internal static IReadOnlyList ReadPersistedIndexOmissionReasons( diff --git a/src/CodeIndex/Database/DbReader.Status.cs b/src/CodeIndex/Database/DbReader.Status.cs index 1fcccc116..05d936db9 100644 --- a/src/CodeIndex/Database/DbReader.Status.cs +++ b/src/CodeIndex/Database/DbReader.Status.cs @@ -284,6 +284,15 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind IndexIncompleteReasons = persistedReadiness.IndexComplete ? null : persistedReadiness.IndexIncompleteReasons.ToList(), + SymbolKindFilterProvenanceAvailable = persistedReadiness.SymbolKindFilterPolicy.ProvenanceAvailable, + SymbolKindFilter = persistedReadiness.SymbolKindFilterPolicy.ProvenanceAvailable + ? new StatusSymbolKindFilter + { + Include = persistedReadiness.SymbolKindFilterPolicy.Include, + Exclude = persistedReadiness.SymbolKindFilterPolicy.Exclude, + } + : null, + SymbolsDroppedByKindFilter = persistedReadiness.SymbolKindFilterPolicy.SymbolsDropped, HotspotFamilyReady = hotspotFamilySignal.Ready, HotspotFamilyDegradedReason = hotspotFamilySignal.DegradedReason, LanguageReadiness = languageReadiness.Count > 0 ? languageReadiness : null, diff --git a/src/CodeIndex/Database/DbReader.WorkspaceHealth.cs b/src/CodeIndex/Database/DbReader.WorkspaceHealth.cs index 6f416facf..a806f9db0 100644 --- a/src/CodeIndex/Database/DbReader.WorkspaceHealth.cs +++ b/src/CodeIndex/Database/DbReader.WorkspaceHealth.cs @@ -7,7 +7,9 @@ internal sealed record WorkspaceIndexHealthSnapshot( bool GraphDataCurrent, bool ReferenceGraphComplete, bool IndexComplete, - bool IndexNewerThanReader); + bool IndexNewerThanReader, + IReadOnlyList IndexIncompleteReasons, + PersistedSymbolKindFilterPolicy SymbolKindFilterPolicy); public partial class DbReader { @@ -34,6 +36,8 @@ internal WorkspaceIndexHealthSnapshot GetWorkspaceIndexHealth() persistedReadiness.GraphDataCurrent, persistedReadiness.ReferenceGraphComplete, persistedReadiness.IndexComplete, - _indexNewerThanReader); + _indexNewerThanReader, + persistedReadiness.IndexIncompleteReasons, + persistedReadiness.SymbolKindFilterPolicy); }); } diff --git a/src/CodeIndex/Database/DbReader.cs b/src/CodeIndex/Database/DbReader.cs index 05d3ca101..a9e54485a 100644 --- a/src/CodeIndex/Database/DbReader.cs +++ b/src/CodeIndex/Database/DbReader.cs @@ -77,6 +77,7 @@ public partial class DbReader : IDisposable private readonly IReadOnlyList _databasePermissionDiagnostics; private readonly DbSchemaCache? _schemaCache; private readonly CancellationToken _cancellation; + private readonly int _userVersion; private readonly IReadOnlySet _fileColumns; private readonly IReadOnlySet _symbolColumns; private readonly IReadOnlySet _referenceColumns; @@ -621,6 +622,7 @@ private DbReader( var raw = v.ExecuteScalar(); userVersion = raw is long l ? (int)l : (raw is int i ? i : 0); } + _userVersion = userVersion; _hasChunksTable = HasTable("chunks"); _hasReferencesTable = HasTable("symbol_references") && (userVersion & DbContext.GraphReadyFlag) != 0; _hasHotspotReferenceCountsTable = HasTable(HotspotReferenceAggregateSql.TableName) diff --git a/src/CodeIndex/Database/DbWriter.Counts.cs b/src/CodeIndex/Database/DbWriter.Counts.cs index c92db2292..73a1b20f0 100644 --- a/src/CodeIndex/Database/DbWriter.Counts.cs +++ b/src/CodeIndex/Database/DbWriter.Counts.cs @@ -16,6 +16,13 @@ public partial class DbWriter return (files, chunks, symbols, references); } + /// + /// Return whether a scoped update could leave existing file rows untouched. + /// scoped update が既存 file 行を未更新のまま残し得るかを返す。 + /// + internal bool HasIndexedFiles() + => ExecuteScalar("SELECT EXISTS(SELECT 1 FROM files LIMIT 1)") != 0; + private long ExecuteScalar(string sql) { using var cmd = _conn.CreateCommand(); diff --git a/src/CodeIndex/Database/DbWriter.Files.cs b/src/CodeIndex/Database/DbWriter.Files.cs index f8168abc2..171c04b38 100644 --- a/src/CodeIndex/Database/DbWriter.Files.cs +++ b/src/CodeIndex/Database/DbWriter.Files.cs @@ -149,6 +149,7 @@ ON CONFLICT(path) DO UPDATE SET checksum = excluded.checksum, modified = excluded.modified, generated = excluded.generated, + symbols_dropped_by_kind_filter = 0, indexed_at = CURRENT_TIMESTAMP RETURNING id", static c => @@ -195,6 +196,35 @@ ON CONFLICT(path) DO UPDATE SET return fileId; } + /// + /// Persist the number of symbols removed by the active kind policy for one indexed file. + /// 現在の kind policy により除外された symbol 数を index 済み file 単位で保存する。 + /// + public void SetSymbolsDroppedByKindFilter(long fileId, int count) + { + if (count < 0) + throw new ArgumentOutOfRangeException(nameof(count)); + + var command = RentCommand( + $"UPDATE files SET {DbContext.SymbolsDroppedByKindFilterColumn} = @count WHERE id = @fileId", + static c => + { + c.Parameters.Add("@count", SqliteType.Integer); + c.Parameters.Add("@fileId", SqliteType.Integer); + }); + try + { + command.Parameters["@count"].Value = count; + command.Parameters["@fileId"].Value = fileId; + if (command.ExecuteNonQuery() != 1) + throw new InvalidOperationException("Could not persist the symbol-kind filter drop count for the indexed file."); + } + finally + { + ReleaseCommand(command); + } + } + /// /// Insert a new file record and return its ID. /// Use only when the caller knows the path cannot already exist in the diff --git a/src/CodeIndex/Database/DbWriter.ReadyFlags.cs b/src/CodeIndex/Database/DbWriter.ReadyFlags.cs index 10eef0fea..361b04b8c 100644 --- a/src/CodeIndex/Database/DbWriter.ReadyFlags.cs +++ b/src/CodeIndex/Database/DbWriter.ReadyFlags.cs @@ -11,6 +11,7 @@ public partial class DbWriter // bit 2 (FoldReadyFlag) — name_folded columns populated for Unicode --exact (#86) // bit 3 (HotspotReferenceAggregateStorageContractFlag) — permanent downgrade guard // bit 4 (HotspotReferenceAggregateReadyFlag) — maintained aggregate synchronized + // bit 5 (SymbolKindFilterAuditStorageContractFlag) — permanent per-file audit guard // CLI and MCP full-scan indexing set graph + fold; CLI additionally sets issues (MCP // now persists file_issues too after bdbb2bd, so both can stamp it). The index runner // ClearReadyFlags() first so partial / aborted runs demote trust until a successful @@ -19,6 +20,8 @@ public partial class DbWriter // CLI / MCP 共に full-scan で graph + fold を立てる。fold は部分更新では立てない。 public void MarkGraphReady() => SetReadyBit(DbContext.GraphReadyFlag); public void MarkIssuesReady() => SetReadyBit(DbContext.IssuesReadyFlag); + public void MarkSymbolKindFilterAuditStorageContract() + => SetReadyBit(DbContext.SymbolKindFilterAuditStorageContractFlag); /// /// Stamp FoldReadyFlag AND write the current plus the @@ -128,8 +131,8 @@ public void ClearReadyFlags() read.CommandText = "PRAGMA user_version"; var raw = read.ExecuteScalar(); var current = raw is long l ? (int)l : (raw is int i ? i : 0); - var aggregateContractBits = current & DbContext.HotspotReferenceAggregateFlags; - Execute($"PRAGMA user_version = {aggregateContractBits}", _activeTransaction); + var preservedContractBits = current & DbContext.PreservedIndexStorageContractFlags; + Execute($"PRAGMA user_version = {preservedContractBits}", _activeTransaction); } private bool ClearHotspotReferenceAggregateReady() diff --git a/src/CodeIndex/Database/DegradationReasonCodes.cs b/src/CodeIndex/Database/DegradationReasonCodes.cs index 49c743043..33924d9c2 100644 --- a/src/CodeIndex/Database/DegradationReasonCodes.cs +++ b/src/CodeIndex/Database/DegradationReasonCodes.cs @@ -24,6 +24,8 @@ public static class DegradationReasonCodes public const string GraphTableMissing = "graph_table_available=false"; public const string GraphDataNotCurrent = "graph_data_current=false"; public const string IndexIncomplete = "index_complete=false"; + public const string SymbolKindFilterCoverageLimited = "symbol_kind_filter_coverage_limited"; + public const string SymbolKindFilterProvenanceUnavailable = "symbol_kind_filter_provenance_unavailable"; public const string ReferenceGraphIncomplete = "reference_graph_complete=false"; public const string SymbolsOnlyGraphOmitted = "symbols_only_graph_omitted"; public const string ReferenceExtractionCapStateUnavailable = "reference_extraction_cap_state_unavailable"; @@ -55,6 +57,8 @@ public static class DegradationReasonCodes GraphTableMissing, GraphDataNotCurrent, IndexIncomplete, + SymbolKindFilterCoverageLimited, + SymbolKindFilterProvenanceUnavailable, ReferenceGraphIncomplete, SymbolsOnlyGraphOmitted, ReferenceExtractionCapStateUnavailable, @@ -187,6 +191,16 @@ private static DegradationReasonMetadata CreateMetadata(string code) "Required input or extraction work was omitted or failed, so persisted rows cover only a partial index generation.", "Run `cdidx status --json`, address `index_incomplete_reasons`, then rerun the same index command; a rebuild is normally not required.", "Use `cdidx status --json` and inspect `last_failed_or_partial_index_run.file_errors` for extractor failures or the stable omission reasons for capped and symbols-only runs."), + SymbolKindFilterCoverageLimited => new( + code, + "The persisted generation intentionally filtered symbol kinds, so negative symbol and graph results are not authoritative.", + "Run `cdidx index --rebuild` without `--include-symbol-kind` or `--exclude-symbol-kind` for full coverage.", + "Run `cdidx status --json` to audit `symbol_kind_filter` and `symbols_dropped_by_kind_filter` while retaining the limited index."), + SymbolKindFilterProvenanceUnavailable => new( + code, + "This legacy DB does not record the symbol-kind policy used for its persisted generation, so negative symbol and graph results are conservatively non-authoritative.", + "Run `cdidx index --rebuild` with a current cdidx binary to stamp explicit policy provenance.", + "Run `cdidx status --json` and treat negative results as degraded until the DB can be rebuilt."), ReferenceGraphIncomplete => new( code, "Reference extraction hit one or more hard safety caps, so missing callers, callees, dependencies, or impact edges are not authoritative absences.", diff --git a/src/CodeIndex/Mcp/McpToolCatalog.cs b/src/CodeIndex/Mcp/McpToolCatalog.cs index 89aedce1d..0c91db6ab 100644 --- a/src/CodeIndex/Mcp/McpToolCatalog.cs +++ b/src/CodeIndex/Mcp/McpToolCatalog.cs @@ -724,8 +724,8 @@ private static JsonObject[] CreateIndexMaintenanceTools() ["maxSymbolsPerFile"] = new JsonObject { ["type"] = "integer", ["description"] = "Skip symbol/reference indexing for files that produce more symbols than this limit, matching CLI --max-symbols-per-file.", ["default"] = IndexCommandRunner.DefaultMaxSymbolsPerFile, ["minimum"] = 1, ["maximum"] = IndexCommandRunner.MaxSymbolsPerFileLimit }, ["maxReferencesPerFile"] = new JsonObject { ["type"] = "integer", ["description"] = "Skip references for files that produce more references than this limit, matching CLI --max-references-per-file.", ["default"] = IndexCommandRunner.DefaultMaxReferencesPerFile, ["minimum"] = 1, ["maximum"] = IndexCommandRunner.MaxReferencesPerFileLimit }, ["followSymlinks"] = new JsonObject { ["type"] = "string", ["enum"] = new JsonArray { "none", "internal", "all" }, ["description"] = "Directory and file symlink policy matching CLI --follow-symlinks.", ["default"] = "none" }, - ["includeSymbolKind"] = new JsonObject { ["oneOf"] = new JsonArray { new JsonObject { ["type"] = "string" }, new JsonObject { ["type"] = "array", ["items"] = new JsonObject { ["type"] = "string" } } }, ["description"] = "Only index symbols with these kinds. Accepts a value, comma-separated string, or array." }, - ["excludeSymbolKind"] = new JsonObject { ["oneOf"] = new JsonArray { new JsonObject { ["type"] = "string" }, new JsonObject { ["type"] = "array", ["items"] = new JsonObject { ["type"] = "string" } } }, ["description"] = "Drop symbols with these kinds before indexing. Accepts a value, comma-separated string, or array." }, + ["includeSymbolKind"] = new JsonObject { ["oneOf"] = new JsonArray { new JsonObject { ["type"] = "string" }, new JsonObject { ["type"] = "array", ["items"] = new JsonObject { ["type"] = "string" } } }, ["description"] = "Only index symbols with these kinds. Accepts a value, comma-separated string, or array. The normalized policy and per-file drop counts are persisted, and the resulting generation is coverage-limited for negative symbol/graph queries." }, + ["excludeSymbolKind"] = new JsonObject { ["oneOf"] = new JsonArray { new JsonObject { ["type"] = "string" }, new JsonObject { ["type"] = "array", ["items"] = new JsonObject { ["type"] = "string" } } }, ["description"] = "Drop symbols with these kinds before indexing. Accepts a value, comma-separated string, or array. The normalized policy and per-file drop counts are persisted, and the resulting generation is coverage-limited for negative symbol/graph queries." }, ["memoryTrace"] = new JsonObject { ["type"] = "boolean", ["description"] = "Include lightweight MCP memory samples and duration diagnostics in the response.", ["default"] = false }, ["parallelism"] = new JsonObject { ["type"] = "integer", ["description"] = "CLI compatibility knob. MCP index currently runs serially and reports effective_parallelism=1 instead of silently using this value.", ["minimum"] = 1, ["maximum"] = IndexCommandRunner.MaxIndexParallelism }, ["commits"] = new JsonObject { ["oneOf"] = new JsonArray { new JsonObject { ["type"] = "string" }, new JsonObject { ["type"] = "array", ["items"] = new JsonObject { ["type"] = "string" } } }, ["description"] = "CLI compatibility scope. Commit-scoped MCP indexing is not supported; non-dry runs reject it explicitly." }, diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Graph.cs b/src/CodeIndex/Mcp/McpToolHandlers.Graph.cs index ca329362d..41ff6f856 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Graph.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Graph.cs @@ -1111,6 +1111,25 @@ private void AddIndexGenerationReadinessSignal( payload["graph_table_available"] = readiness.GraphTableAvailable; payload["graph_data_current"] = readiness.GraphDataCurrent; payload["index_complete"] = readiness.IndexComplete; + payload["symbol_kind_filter_provenance_available"] = + readiness.SymbolKindFilterPolicy.ProvenanceAvailable; + if (readiness.SymbolKindFilterPolicy.ProvenanceAvailable) + { + payload["symbol_kind_filter"] = new JsonObject + { + ["include"] = JsonSerializer.SerializeToNode( + readiness.SymbolKindFilterPolicy.Include.ToList(), + _jsonOptions), + ["exclude"] = JsonSerializer.SerializeToNode( + readiness.SymbolKindFilterPolicy.Exclude.ToList(), + _jsonOptions), + }; + } + if (readiness.SymbolKindFilterPolicy.SymbolsDropped.HasValue) + { + payload["symbols_dropped_by_kind_filter"] = + readiness.SymbolKindFilterPolicy.SymbolsDropped.Value; + } if (!readiness.IndexComplete) { payload["index_incomplete_reasons"] = JsonSerializer.SerializeToNode( diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs index 3270aa4cb..a462d8db0 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs @@ -140,7 +140,8 @@ bool IsPathAuthorized(string path) var symbolKindFilterMatchesPrior = string.Equals( indexSnapshot.SymbolKindFilterSignature, symbolKindFilter.Signature, - StringComparison.Ordinal); + StringComparison.Ordinal) + && (!symbolKindFilter.IsActive || indexSnapshot.SymbolKindFilterAuditCurrent); var priorFilterRetainedCSharpContractMembers = SymbolKindFilter.SignatureRetainsCSharpStaticInterfaceContractMembers( indexSnapshot.SymbolKindFilterSignature); @@ -206,7 +207,9 @@ void MarkSymbolKindFilterMetaIncompleteOnce() { if (symbolKindFilterMetaMarkedIncomplete) return; - writer.SetMeta(IndexCommandRunner.SymbolKindFilterMetaKey, null); + writer.SetMetaValues( + (IndexCommandRunner.SymbolKindFilterMetaKey, null), + (IndexCommandRunner.SymbolKindFilterAuditVersionMetaKey, null)); symbolKindFilterMetaMarkedIncomplete = true; } @@ -761,6 +764,16 @@ bool CanReuseCSharpPrepassTargetWithoutRead(CSharpStaticInterfacePrepass.FileTar db.DropAll(); db.InitializeSchema(); writer = new DbWriter(db); + // DropAll intentionally retains codeindex_meta, and InitializeSchema seeds a new + // files table with an unfiltered default. Clear both provenance keys immediately at + // the destructive boundary so cancellation or a first-file failure cannot expose the + // prior policy (or the seeded default) as evidence for the partial rebuilt DB. + // destructive rebuild 境界で provenance を即時消去し、最初の file 前の中断でも + // 旧 policy / seeded default を partial DB の証拠として公開しない。 + writer.SetMetaValues( + (IndexCommandRunner.SymbolKindFilterMetaKey, null), + (IndexCommandRunner.SymbolKindFilterAuditVersionMetaKey, null)); + symbolKindFilterMetaMarkedIncomplete = true; } writer.SetMeta( DbContext.WorkspaceVerificationPendingPathsCompleteMetaKey, diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.ExtractedFilePersistence.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.ExtractedFilePersistence.cs index 517630108..c99da062d 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.ExtractedFilePersistence.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.ExtractedFilePersistence.cs @@ -41,7 +41,9 @@ private static void PersistExtractedMcpIndexFile( symbols, loaded.Content, familyScopeKey); - session.SymbolsDroppedByKindFilter += context.SymbolKindFilter.Apply(symbols); + var symbolsDroppedByKindFilter = context.SymbolKindFilter.Apply(symbols); + context.Writer.SetSymbolsDroppedByKindFilter(fileId, symbolsDroppedByKindFilter); + session.SymbolsDroppedByKindFilter += symbolsDroppedByKindFilter; var committed = PersistMcpIndexSymbolsAndReferences( context, diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Snapshots.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Snapshots.cs index 5c877e83f..000f7f81f 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Snapshots.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Snapshots.cs @@ -21,10 +21,12 @@ private sealed class IndexDatabaseSnapshot public required Dictionary HotspotFamilyMarkerFingerprints { get; init; } public string? IndexedProjectRoot { get; init; } public string? SymbolKindFilterSignature { get; init; } + public bool SymbolKindFilterAuditCurrent { get; init; } } private static IndexDatabaseSnapshot CaptureIndexDatabaseSnapshot(DbContext db) { + var readiness = db.GetUserVersion(); var csharpMetadataTargetVersionMetaKey = DbContext.GetMetadataTargetVersionMetaKey("csharp"); var meta = db.GetMetaStrings( [ @@ -39,6 +41,7 @@ private static IndexDatabaseSnapshot CaptureIndexDatabaseSnapshot(DbContext db) DbContext.IndexCompletenessMetaKey, DbContext.IndexedProjectRootMetaKey, IndexCommandRunner.SymbolKindFilterMetaKey, + IndexCommandRunner.SymbolKindFilterAuditVersionMetaKey, ]); return new IndexDatabaseSnapshot @@ -63,7 +66,7 @@ private static IndexDatabaseSnapshot CaptureIndexDatabaseSnapshot(DbContext db) meta[DbContext.IndexCompletenessMetaKey], "complete", StringComparison.OrdinalIgnoreCase), - Readiness = db.GetUserVersion(), + Readiness = readiness, HotspotFamilyVersions = GetHotspotFamilyMetaSnapshot( db, DbContext.GetHotspotFamilyVersionMetaKey), @@ -72,6 +75,11 @@ private static IndexDatabaseSnapshot CaptureIndexDatabaseSnapshot(DbContext db) DbContext.GetHotspotFamilyMarkerFingerprintMetaKey), IndexedProjectRoot = meta[DbContext.IndexedProjectRootMetaKey], SymbolKindFilterSignature = meta[IndexCommandRunner.SymbolKindFilterMetaKey], + SymbolKindFilterAuditCurrent = string.Equals( + meta[IndexCommandRunner.SymbolKindFilterAuditVersionMetaKey], + DbContext.SymbolKindFilterAuditVersion, + StringComparison.Ordinal) + && (readiness & DbContext.SymbolKindFilterAuditStorageContractFlag) != 0, }; } } diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index 300bffbcc..ee0cca25e 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -1514,6 +1514,14 @@ public sealed class StatusDatabaseSizeAttribution public List? TopObjects { get; set; } } +public sealed class StatusSymbolKindFilter +{ + [JsonPropertyName("include")] + public IReadOnlyList Include { get; set; } = []; + [JsonPropertyName("exclude")] + public IReadOnlyList Exclude { get; set; } = []; +} + public class StatusResult { internal const string SqliteConnectionPolicyJsonFieldName = "sqlite_connection_policy"; @@ -1861,6 +1869,22 @@ public class StatusResult [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? IndexIncompleteReasons { get; set; } /// + /// True when the normalized include/exclude policy used for the persisted generation is + /// available. False is the conservative legacy fallback: negative symbol and graph results + /// are not authoritative until a current index pass stamps the policy. + /// 永続 generation に適用した正規化済み include/exclude policy が利用可能なら true。 + /// false は旧 DB 向けの保守的 fallback で、現行 index が policy を stamp するまで + /// symbol / graph の否定結果を authoritative とみなさない。 + /// + [JsonPropertyName("symbol_kind_filter_provenance_available")] + public bool SymbolKindFilterProvenanceAvailable { get; set; } + [JsonPropertyName("symbol_kind_filter")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public StatusSymbolKindFilter? SymbolKindFilter { get; set; } + [JsonPropertyName("symbols_dropped_by_kind_filter")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? SymbolsDroppedByKindFilter { get; set; } + /// /// True when authoritative cross-file hotspot-family grouping metadata is current for every /// marker-capable language currently indexed in this DB. False means `hotspots` can still /// run, but duplicate-name families may be conservatively degraded. The degraded reason diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index 93400b613..12b720d1a 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -440,7 +440,7 @@ public void Run_FullScanJson_ProjectMarkerBudgetWarningIncludesTruncatedWarning( } [Fact] - public void Run_FullScan_ExcludeSymbolKindDropsMatchingSymbols() + public void Run_FullScan_ExcludeSymbolKindDropsMatchingSymbols_Issue5224() { var projectRoot = CreateTempProject(); try @@ -458,11 +458,365 @@ def helper(): Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal("success", json.GetProperty("status").GetString()); Assert.Equal(1, json.GetProperty("summary").GetProperty("symbols_dropped_by_kind_filter").GetInt32()); + Assert.Equal(1, json.GetProperty("symbols_dropped_by_kind_filter").GetInt64()); + Assert.True(json.GetProperty("symbol_kind_filter_provenance_available").GetBoolean()); Assert.Equal(["function"], json.GetProperty("symbol_kind_filter").GetProperty("exclude").EnumerateArray().Select(value => value.GetString()).ToArray()); + Assert.False(json.GetProperty("index_complete").GetBoolean()); + Assert.Contains( + DbReader.SymbolKindFilterCoverageLimitedReason, + ReadCompletenessReasons(json, "index_incomplete_reasons")); + Assert.False(json.GetProperty("reference_graph_complete").GetBoolean()); - var counts = ReadSymbolKindCounts(Path.Combine(projectRoot, ".cdidx", "codeindex.db")); + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using (var verificationDb = new DbContext(DbOpenIntent.QueryOnly, dbPath)) + using (var userVersionCommand = verificationDb.Connection.CreateCommand()) + { + userVersionCommand.CommandText = "PRAGMA user_version"; + var userVersion = (long)userVersionCommand.ExecuteScalar()!; + Assert.NotEqual(0, userVersion & DbContext.SymbolKindFilterAuditStorageContractFlag); + } + var counts = ReadSymbolKindCounts(dbPath); Assert.True(counts.GetValueOrDefault("class") > 0); Assert.False(counts.ContainsKey("function")); + + var (statusExitCode, status) = RunStatusAndCaptureJson(["--db", dbPath, "--json"]); + Assert.Equal(CommandExitCodes.Success, statusExitCode); + Assert.False(status.GetProperty("index_complete").GetBoolean()); + Assert.False(status.GetProperty("graph_data_current").GetBoolean()); + Assert.Equal(1, status.GetProperty("symbols_dropped_by_kind_filter").GetInt64()); + Assert.True(status.GetProperty("symbol_kind_filter_provenance_available").GetBoolean()); + Assert.Equal( + ["function"], + status.GetProperty("symbol_kind_filter").GetProperty("exclude") + .EnumerateArray().Select(value => value.GetString()).ToArray()); + + var (definitionExitCode, definitionStdout, definitionStderr) = + RunDefinitionAndCaptureIssue5224(["helper", "--db", dbPath, "--json"]); + Assert.Equal(CommandExitCodes.NotFound, definitionExitCode); + Assert.Equal(string.Empty, definitionStderr); + using (var definitionDocument = JsonDocument.Parse(definitionStdout)) + { + var definitionJson = definitionDocument.RootElement; + Assert.False(definitionJson.GetProperty("index_complete").GetBoolean()); + Assert.False(definitionJson.GetProperty("authoritative_count").GetBoolean()); + Assert.Equal(1, definitionJson.GetProperty("symbols_dropped_by_kind_filter").GetInt64()); + Assert.Contains( + "not authoritative", + definitionJson.GetProperty("index_generation_warning").GetString(), + StringComparison.Ordinal); + } + + var (humanDefinitionExitCode, _, humanDefinitionStderr) = + RunDefinitionAndCaptureIssue5224(["helper", "--db", dbPath]); + Assert.Equal(CommandExitCodes.Success, humanDefinitionExitCode); + Assert.Contains( + DbReader.SymbolKindFilterCoverageLimitedReason, + humanDefinitionStderr, + StringComparison.Ordinal); + Assert.Contains("not authoritative", humanDefinitionStderr, StringComparison.Ordinal); + + var (lspDefinitionExitCode, lspDefinitionStdout, lspDefinitionStderr) = + RunDefinitionAndCaptureIssue5224(["helper", "--db", dbPath, "--format", "lsp"]); + Assert.Equal(CommandExitCodes.Success, lspDefinitionExitCode); + Assert.Equal("[]", lspDefinitionStdout.Trim()); + Assert.Contains( + DbReader.SymbolKindFilterCoverageLimitedReason, + lspDefinitionStderr, + StringComparison.Ordinal); + Assert.Contains("not authoritative", lspDefinitionStderr, StringComparison.Ordinal); + + var (arraySymbolsExitCode, arraySymbolsStdout, arraySymbolsStderr) = + RunSymbolsAndCaptureIssue5224(["helper", "--db", dbPath, "--json=array"]); + Assert.Equal(CommandExitCodes.Success, arraySymbolsExitCode); + Assert.Equal("[]", arraySymbolsStdout.Trim()); + Assert.Contains( + DbReader.SymbolKindFilterCoverageLimitedReason, + arraySymbolsStderr, + StringComparison.Ordinal); + + var (ndjsonSymbolsExitCode, ndjsonSymbolsStdout, ndjsonSymbolsStderr) = + RunSymbolsAndCaptureIssue5224(["helper", "--db", dbPath, "--json=ndjson"]); + Assert.Equal(CommandExitCodes.Success, ndjsonSymbolsExitCode); + Assert.Equal(string.Empty, ndjsonSymbolsStdout); + Assert.Contains( + DbReader.SymbolKindFilterCoverageLimitedReason, + ndjsonSymbolsStderr, + StringComparison.Ordinal); + + var (filesExitCode, filesStdout, filesStderr) = + RunFilesAndCaptureIssue5224(["--db", dbPath, "--path", "missing/**", "--count", "--json"]); + Assert.Equal(CommandExitCodes.Success, filesExitCode); + Assert.Equal(string.Empty, filesStderr); + using var filesDocument = JsonDocument.Parse(filesStdout); + var filesJson = filesDocument.RootElement; + Assert.Equal(0, filesJson.GetProperty("count").GetInt32()); + Assert.True(filesJson.GetProperty("authoritative_count").GetBoolean()); + Assert.False(filesJson.TryGetProperty("index_generation_warning", out _)); + Assert.False(filesJson.TryGetProperty("symbol_kind_filter", out _)); + + var (searchExitCode, searchStdout, searchStderr) = + RunSearchAndCaptureIssue5224(["definitely_absent", "--db", dbPath, "--count", "--json"]); + Assert.Equal(CommandExitCodes.Success, searchExitCode); + Assert.Equal(string.Empty, searchStderr); + using var searchDocument = JsonDocument.Parse(searchStdout); + var searchJson = searchDocument.RootElement; + Assert.Equal(0, searchJson.GetProperty("count").GetInt32()); + Assert.True(searchJson.GetProperty("authoritative_count").GetBoolean()); + Assert.False(searchJson.TryGetProperty("index_generation_warning", out _)); + + var (validateExitCode, validateStdout, validateStderr) = + RunValidateAndCaptureIssue5224(["--db", dbPath, "--format", "count", "--json"]); + Assert.Equal(CommandExitCodes.Success, validateExitCode); + Assert.Equal(string.Empty, validateStderr); + using var validateDocument = JsonDocument.Parse(validateStdout); + var validateJson = validateDocument.RootElement; + Assert.Equal(0, validateJson.GetProperty("count").GetInt32()); + Assert.False(validateJson.GetProperty("index_complete").GetBoolean()); + Assert.False(validateJson.GetProperty("degraded").GetBoolean()); + Assert.True(validateJson.GetProperty("authoritative_count").GetBoolean()); + Assert.False(validateJson.TryGetProperty("index_generation_warning", out _)); + } + finally + { + DeleteDirectory(projectRoot); + } + } + + private (int ExitCode, string Stdout, string Stderr) RunDefinitionAndCaptureIssue5224( + string[] args) + => RunQueryAndCaptureIssue5224(() => QueryCommandRunner.RunDefinition(args, _jsonOptions)); + + private (int ExitCode, string Stdout, string Stderr) RunSymbolsAndCaptureIssue5224( + string[] args) + => RunQueryAndCaptureIssue5224(() => QueryCommandRunner.RunSymbols(args, _jsonOptions)); + + private (int ExitCode, string Stdout, string Stderr) RunFilesAndCaptureIssue5224( + string[] args) + => RunQueryAndCaptureIssue5224(() => QueryCommandRunner.RunFiles(args, _jsonOptions)); + + private (int ExitCode, string Stdout, string Stderr) RunSearchAndCaptureIssue5224( + string[] args) + => RunQueryAndCaptureIssue5224(() => QueryCommandRunner.RunSearch(args, _jsonOptions)); + + private (int ExitCode, string Stdout, string Stderr) RunValidateAndCaptureIssue5224( + string[] args) + => RunQueryAndCaptureIssue5224(() => QueryCommandRunner.RunValidate(args, _jsonOptions)); + + private static (int ExitCode, string Stdout, string Stderr) RunQueryAndCaptureIssue5224( + Func run) + { + lock (TestConsoleLock.Gate) + { + var originalOut = Console.Out; + var originalErr = Console.Error; + using var stdout = new StringWriter(); + using var stderr = new StringWriter(); + try + { + Console.SetOut(stdout); + Console.SetError(stderr); + var exitCode = run(); + return (exitCode, stdout.ToString(), stderr.ToString()); + } + finally + { + Console.SetOut(originalOut); + Console.SetError(originalErr); + } + } + } + + [Fact] + public void Run_SymbolKindPolicyPersistsAcrossIncrementalUpdateAndClearsOnUnfilteredRebuild_Issue5224() + { + var projectRoot = CreateTempProject(); + try + { + var sourcePath = Path.Combine(projectRoot, "app.py"); + File.WriteAllText(sourcePath, "class App:\n pass\n\ndef helper():\n return App()\n"); + + var (initialExitCode, initial) = RunAndCaptureJson( + [projectRoot, "--include-symbol-kind", "function,class", "--exclude-symbol-kind", "function", "--json", "--quiet"]); + Assert.Equal(CommandExitCodes.Success, initialExitCode); + Assert.Equal(1, initial.GetProperty("symbols_dropped_by_kind_filter").GetInt64()); + Assert.Equal( + ["class", "function"], + initial.GetProperty("symbol_kind_filter").GetProperty("include") + .EnumerateArray().Select(value => value.GetString()).ToArray()); + Assert.Contains( + DbReader.SymbolKindFilterCoverageLimitedReason, + ReadCompletenessReasons(initial, "index_incomplete_reasons")); + + // Simulate a pre-guard writer that preserved the policy and audit marker while + // resetting the per-file evidence. The permanent storage-contract bit must make + // the aggregate unavailable until a whole-workspace pass re-extracts unchanged files. + // policy と audit marker を残したまま per-file 証跡を reset する旧 writer を再現する。 + // 永続 storage-contract bit により、unchanged file の全体再抽出までは aggregate を省略する。 + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + SqliteConnection.ClearAllPools(); + using (var connection = OpenNonPoolingConnection(dbPath)) + { + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = $""" + PRAGMA user_version = {DbContext.CurrentSchemaVersion & ~DbContext.SymbolKindFilterAuditStorageContractFlag}; + UPDATE files SET symbols_dropped_by_kind_filter = 0; + """; + command.ExecuteNonQuery(); + } + SqliteConnection.ClearAllPools(); + + var (legacyStatusExitCode, legacyStatus) = RunStatusAndCaptureJson(["--db", dbPath, "--json"]); + Assert.Equal(CommandExitCodes.Success, legacyStatusExitCode); + Assert.True(legacyStatus.GetProperty("symbol_kind_filter_provenance_available").GetBoolean()); + Assert.False(legacyStatus.TryGetProperty("symbols_dropped_by_kind_filter", out _)); + + var (auditRepairExitCode, auditRepair) = RunAndCaptureJson( + [projectRoot, "--include-symbol-kind", "class,function", "--exclude-symbol-kind", "function", "--json", "--quiet"]); + Assert.Equal(CommandExitCodes.Success, auditRepairExitCode); + Assert.Equal(1, auditRepair.GetProperty("summary").GetProperty("files_extracted").GetInt64()); + Assert.Equal(1, auditRepair.GetProperty("symbols_dropped_by_kind_filter").GetInt64()); + using (var verificationDb = new DbContext(DbOpenIntent.QueryOnly, dbPath)) + using (var userVersionCommand = verificationDb.Connection.CreateCommand()) + { + userVersionCommand.CommandText = "PRAGMA user_version"; + var userVersion = (long)userVersionCommand.ExecuteScalar()!; + Assert.NotEqual(0, userVersion & DbContext.SymbolKindFilterAuditStorageContractFlag); + } + + File.WriteAllText(sourcePath, "class App:\n pass\n\ndef helper():\n return App()\n\ndef second():\n return App()\n"); + var (updateExitCode, update) = RunAndCaptureJson( + [projectRoot, "--include-symbol-kind", "class,function", "--exclude-symbol-kind", "function", "--json", "--quiet"]); + Assert.Equal(CommandExitCodes.Success, updateExitCode); + Assert.Equal("incremental", update.GetProperty("mode").GetString()); + Assert.Equal(2, update.GetProperty("symbols_dropped_by_kind_filter").GetInt64()); + Assert.False(update.GetProperty("index_complete").GetBoolean()); + + var (limitedStatusExitCode, limitedStatus) = RunStatusAndCaptureJson(["--db", dbPath, "--json"]); + Assert.Equal(CommandExitCodes.Success, limitedStatusExitCode); + Assert.Equal(2, limitedStatus.GetProperty("symbols_dropped_by_kind_filter").GetInt64()); + + var (rebuildExitCode, rebuilt) = RunAndCaptureJson( + [projectRoot, "--rebuild", "--yes", "--json", "--quiet"]); + Assert.Equal(CommandExitCodes.Success, rebuildExitCode); + Assert.Equal("rebuild", rebuilt.GetProperty("mode").GetString()); + Assert.True(rebuilt.GetProperty("index_complete").GetBoolean()); + Assert.True(rebuilt.GetProperty("reference_graph_complete").GetBoolean()); + Assert.Equal(0, rebuilt.GetProperty("symbols_dropped_by_kind_filter").GetInt64()); + Assert.Empty(rebuilt.GetProperty("symbol_kind_filter").GetProperty("include").EnumerateArray()); + Assert.Empty(rebuilt.GetProperty("symbol_kind_filter").GetProperty("exclude").EnumerateArray()); + } + finally + { + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_ScopedUpdateRejectsLegacyDatabaseWithoutSymbolKindProvenance_Issue5224() + { + var projectRoot = CreateTempProject(); + try + { + var sourceA = Path.Combine(projectRoot, "a.py"); + File.WriteAllText(sourceA, "class A:\n pass\n\ndef hidden_a():\n return 1\n"); + File.WriteAllText( + Path.Combine(projectRoot, "b.py"), + "class B:\n pass\n\ndef hidden_b():\n return 2\n"); + + var (initialExitCode, _) = RunAndCaptureJson( + [projectRoot, "--exclude-symbol-kind", "function", "--json", "--quiet"]); + Assert.Equal(CommandExitCodes.Success, initialExitCode); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + SqliteConnection.ClearAllPools(); + using (var connection = OpenNonPoolingConnection(dbPath)) + { + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = $""" + DELETE FROM codeindex_meta + WHERE key IN ('{DbContext.SymbolKindFilterMetaKey}', '{DbContext.SymbolKindFilterAuditVersionMetaKey}'); + PRAGMA user_version = {DbContext.CurrentSchemaVersion & ~DbContext.SymbolKindFilterAuditStorageContractFlag}; + """; + command.ExecuteNonQuery(); + } + SqliteConnection.ClearAllPools(); + File.AppendAllText(sourceA, "\n# changed\n"); + + var (scopedExitCode, scopedJson) = RunAndCaptureJson( + [projectRoot, "--files", "a.py", "--json", "--quiet"]); + + Assert.Equal(CommandExitCodes.UsageError, scopedExitCode); + Assert.Contains( + "per-file audit generation must be current", + scopedJson.GetProperty("message").GetString(), + StringComparison.Ordinal); + + var (statusExitCode, status) = RunStatusAndCaptureJson(["--db", dbPath, "--json"]); + Assert.Equal(CommandExitCodes.Success, statusExitCode); + Assert.False(status.GetProperty("index_complete").GetBoolean()); + Assert.False(status.GetProperty("symbol_kind_filter_provenance_available").GetBoolean()); + Assert.Contains( + DbReader.SymbolKindFilterProvenanceUnavailableReason, + ReadCompletenessReasons(status, "index_incomplete_reasons")); + } + finally + { + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_ScopedUpdateAllowsFreshDatabaseWithoutPriorSymbolKindProvenance_Issue5224() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "app.py"), + "class App:\n pass\n\ndef hidden():\n return 1\n"); + + var (exitCode, json) = RunAndCaptureJson( + [ + projectRoot, + "--files", "app.py", + "--exclude-symbol-kind", "function", + "--json", + "--quiet", + ]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.True(json.GetProperty("symbol_kind_filter_provenance_available").GetBoolean()); + Assert.Equal(1, json.GetProperty("symbols_dropped_by_kind_filter").GetInt64()); + Assert.Contains( + DbReader.SymbolKindFilterCoverageLimitedReason, + ReadCompletenessReasons(json, "index_incomplete_reasons")); + } + finally + { + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_ActiveSymbolKindPolicyWithZeroDropsStillLimitsPartialGeneration_Issue5224() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText(Path.Combine(projectRoot, "app.py"), "class App:\n pass\n"); + + var (exitCode, json) = RunAndCaptureJson( + [projectRoot, "--exclude-symbol-kind", "namespace", "--max-file-bytes", "1", "--json", "--quiet"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(0, json.GetProperty("symbols_dropped_by_kind_filter").GetInt64()); + var reasons = ReadCompletenessReasons(json, "index_incomplete_reasons"); + Assert.Contains(DbReader.SymbolKindFilterCoverageLimitedReason, reasons); + Assert.Contains("file_too_large", reasons); + Assert.False(json.GetProperty("graph_data_current").GetBoolean()); + Assert.False(json.GetProperty("reference_graph_complete").GetBoolean()); } finally { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 0c8e5e3c3..aca8e3b6a 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -3094,6 +3094,19 @@ public void ParseArgs_SymbolKindFilters_AcceptCommaSeparatedValues() Assert.Null(options.SymbolKindFilter.ParseError); } + [Theory] + [InlineData("class;custom", "reserved ',' or ';'")] + [InlineData("class\nspoof", "control characters")] + public void ParseArgs_SymbolKindFilters_RejectNonRoundTrippableValues_Issue5224( + string value, + string expectedError) + { + var options = IndexCommandRunner.ParseArgs([".", "--include-symbol-kind", value]); + + Assert.Contains(expectedError, options.SymbolKindFilter.ParseError, StringComparison.Ordinal); + Assert.Empty(options.SymbolKindFilter.Include); + } + [Theory] [InlineData("include=;exclude=", true)] [InlineData("include=class,FUNCTION,operator,property;exclude=", true)] diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerUpdateTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerUpdateTests.cs index 1526f316a..c5c1e3125 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerUpdateTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerUpdateTests.cs @@ -991,7 +991,7 @@ public void Run_UpdateMode_CancelledDuringMutualRecursionRefresh_LeavesReadiness Assert.Equal(CommandExitCodes.Interrupted, exitCode); Assert.Equal(CommandErrorCodes.Interrupted, json.GetProperty("error_code").GetString()); using var db = new DbContext(DbOpenIntent.WriteIndex, Path.Combine(projectRoot, ".cdidx", "codeindex.db")); - Assert.Equal(DbContext.HotspotReferenceAggregateFlags, db.GetUserVersion()); + Assert.Equal(DbContext.PreservedIndexStorageContractFlags, db.GetUserVersion()); Assert.Equal(initialHead, db.GetMetaString(DbContext.WorkspaceVerifiedHeadShaMetaKey)); } finally diff --git a/tests/CodeIndex.Tests/LegacySchemaMigrationTests.cs b/tests/CodeIndex.Tests/LegacySchemaMigrationTests.cs index 5f0ca1213..4f6eb7b65 100644 --- a/tests/CodeIndex.Tests/LegacySchemaMigrationTests.cs +++ b/tests/CodeIndex.Tests/LegacySchemaMigrationTests.cs @@ -1443,13 +1443,14 @@ public void InterruptedRefresh_OnAlreadyStampedDb_DemotesTrustUntilCompletion() writer.MarkGraphReady(); writer.MarkIssuesReady(); writer.MarkFoldReady(); + writer.MarkSymbolKindFilterAuditStorageContract(); Assert.Equal(DbContext.CurrentSchemaVersion, db.GetUserVersion()); // Simulate the start of a refresh: clear readiness. An interrupted refresh - // leaves only the aggregate contract bits — graph/issues trust is correctly demoted. + // leaves only the permanent storage-contract bits — graph/issues trust is correctly demoted. // refresh 開始を模擬。中断されればここで止まり、縮退のまま残る。 db.ClearReadyFlags(); - Assert.Equal(DbContext.HotspotReferenceAggregateFlags, db.GetUserVersion()); + Assert.Equal(DbContext.PreservedIndexStorageContractFlags, db.GetUserVersion()); var reader = new DbReader(db.Connection); var status = reader.GetStatus(); @@ -1803,14 +1804,15 @@ public void Rebuild_ClearsReadyFlagsBeforeDroppingTables() writer.MarkGraphReady(); writer.MarkIssuesReady(); writer.MarkFoldReady(); + writer.MarkSymbolKindFilterAuditStorageContract(); Assert.Equal(DbContext.CurrentSchemaVersion, db.GetUserVersion()); } SqliteConnection.ClearAllPools(); // Simulate the production rebuild sequence: open → ClearReadyFlags → DropAll → - // InitializeSchema. Interrupt before any writes or MarkReady. Only the aggregate - // contract bits may remain. - // 本番の rebuild 順序を模擬し、stamp 前に中断。aggregate の contract bit 以外の + // InitializeSchema. Interrupt before any writes or MarkReady. Only permanent + // storage-contract bits may remain. + // 本番の rebuild 順序を模擬し、stamp 前に中断。永続 storage-contract bit 以外の // readiness は 0 でなければならない。 using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) { @@ -1818,7 +1820,7 @@ public void Rebuild_ClearsReadyFlagsBeforeDroppingTables() db.DropAll(); db.InitializeSchema(); // intentionally no writes, no MarkGraphReady / MarkIssuesReady - Assert.Equal(DbContext.HotspotReferenceAggregateFlags, db.GetUserVersion()); + Assert.Equal(DbContext.PreservedIndexStorageContractFlags, db.GetUserVersion()); var reader = new DbReader(db.Connection); var status = reader.GetStatus(); @@ -1851,6 +1853,7 @@ public void CompletedIndex_MarkIndexCompleteStampsUserVersion_AndTablesAreTruste writer.MarkGraphReady(); writer.MarkIssuesReady(); writer.MarkFoldReady(); + writer.MarkSymbolKindFilterAuditStorageContract(); Assert.Equal(DbContext.CurrentSchemaVersion, db.GetUserVersion()); var reader = new DbReader(db.Connection); diff --git a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs index 8c027c88d..1422f5ea1 100644 --- a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs +++ b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs @@ -9999,6 +9999,144 @@ public void ToolsCall_Index_NullByteFilePersistsNullByteIssue_Issue3835() } } + [Fact] + public void ToolsCall_Index_SymbolKindPolicyMatchesPersistedStatusReadiness_Issue5224() + { + var fixtureDir = Path.Combine(Path.GetFullPath("."), $"mcp_index_symbol_policy_{Guid.NewGuid():N}"); + var dbPath = TestProjectHelper.CreateTempDbPath("cdidx_mcp_index_symbol_policy"); + try + { + Directory.CreateDirectory(fixtureDir); + File.WriteAllText( + Path.Combine(fixtureDir, "app.py"), + "class App:\n pass\n\ndef helper():\n return App()\n"); + + using var server = new McpServer(dbPath, ConsoleUi.LoadVersion()); + var response = CallIndex( + server, + fixtureDir, + args => args["includeSymbolKind"] = new JsonArray("class")); + + Assert.False(response["result"]?["isError"]?.GetValue() ?? false, response.ToJsonString()); + var structured = response["result"]!["structuredContent"]!; + Assert.False(structured["index_complete"]!.GetValue()); + Assert.False(structured["graph_data_current"]!.GetValue()); + Assert.False(structured["reference_graph_complete"]!.GetValue()); + Assert.True(structured["symbol_kind_filter_provenance_available"]!.GetValue()); + Assert.Equal( + "class", + structured["symbol_kind_filter"]!["include"]![0]!.GetValue()); + Assert.True(structured["symbols_dropped_by_kind_filter"]!.GetValue() > 0); + Assert.Contains( + DbReader.SymbolKindFilterCoverageLimitedReason, + structured["index_incomplete_reasons"]!.AsArray() + .Select(reason => reason!.GetValue())); + + using var db = new DbContext(DbOpenIntent.QueryOnly, dbPath); + var status = new DbReader(db.Connection, db.IsReadOnly).GetStatus(); + Assert.Equal(status.IndexComplete, structured["index_complete"]!.GetValue()); + Assert.Equal(status.GraphDataCurrent, structured["graph_data_current"]!.GetValue()); + Assert.Equal( + status.SymbolsDroppedByKindFilter, + structured["symbols_dropped_by_kind_filter"]!.GetValue()); + } + finally + { + TestProjectHelper.DeleteDirectory(fixtureDir); + TestProjectHelper.DeleteSqliteDatabaseFiles(dbPath); + } + } + + [Theory] + [InlineData("class;custom", "reserved ',' or ';'")] + [InlineData("class\nspoof", "control characters")] + public void ToolsCall_Index_RejectsNonRoundTrippableSymbolKinds_Issue5224( + string value, + string expectedError) + { + var fixtureDir = Path.Combine(Path.GetFullPath("."), $"mcp_index_symbol_policy_input_{Guid.NewGuid():N}"); + var dbPath = TestProjectHelper.CreateTempDbPath("cdidx_mcp_index_symbol_policy_input"); + try + { + Directory.CreateDirectory(fixtureDir); + using var server = new McpServer(dbPath, ConsoleUi.LoadVersion()); + + var response = CallIndex( + server, + fixtureDir, + args => args["includeSymbolKind"] = new JsonArray(value)); + + Assert.True(response["result"]!["isError"]!.GetValue(), response.ToJsonString()); + Assert.Equal( + McpErrorEnvelope.CategoryInvalidArgument, + response["result"]!["structuredContent"]!["category"]!.GetValue()); + var errorText = response["result"]!["content"]![0]!["text"]!.GetValue(); + Assert.Contains(expectedError, errorText, StringComparison.Ordinal); + } + finally + { + TestProjectHelper.DeleteDirectory(fixtureDir); + TestProjectHelper.DeleteSqliteDatabaseFiles(dbPath); + } + } + + [Fact] + public void ToolsCall_Index_InterruptedRebuildClearsSymbolKindProvenance_Issue5224() + { + var fixtureDir = Path.Combine(Path.GetFullPath("."), $"mcp_index_symbol_policy_rebuild_{Guid.NewGuid():N}"); + var dbPath = TestProjectHelper.CreateTempDbPath("cdidx_mcp_index_symbol_policy_rebuild"); + var previousContentLoadHook = McpServer.McpIndexFileContentLoadForTesting; + try + { + Directory.CreateDirectory(fixtureDir); + File.WriteAllText( + Path.Combine(fixtureDir, "app.py"), + "class App:\n pass\n\ndef helper():\n return App()\n"); + using var server = new McpServer(dbPath, ConsoleUi.LoadVersion()); + var initialResponse = CallIndex( + server, + fixtureDir, + args => args["includeSymbolKind"] = new JsonArray("class")); + Assert.False( + initialResponse["result"]?["isError"]?.GetValue() ?? false, + initialResponse.ToJsonString()); + + McpServer.McpIndexFileContentLoadForTesting = _ => + throw new IOException("simulated rebuild content load failure"); + var rebuildResponse = CallIndex( + server, + fixtureDir, + args => + { + args["rebuild"] = true; + args["includeSymbolKind"] = new JsonArray("function"); + }); + + Assert.False( + rebuildResponse["result"]?["isError"]?.GetValue() ?? false, + rebuildResponse.ToJsonString()); + Assert.Equal( + 1, + rebuildResponse["result"]!["structuredContent"]!["summary"]!["errors"]!.GetValue()); + using var db = new DbContext(DbOpenIntent.QueryOnly, dbPath); + Assert.Null(db.GetMetaString(DbContext.SymbolKindFilterMetaKey)); + Assert.Null(db.GetMetaString(DbContext.SymbolKindFilterAuditVersionMetaKey)); + var status = new DbReader(db.Connection, db.IsReadOnly).GetStatus(); + Assert.False(status.SymbolKindFilterProvenanceAvailable); + Assert.Null(status.SymbolsDroppedByKindFilter); + Assert.NotNull(status.IndexIncompleteReasons); + Assert.Contains( + DbReader.SymbolKindFilterProvenanceUnavailableReason, + status.IndexIncompleteReasons); + } + finally + { + McpServer.McpIndexFileContentLoadForTesting = previousContentLoadHook; + TestProjectHelper.DeleteDirectory(fixtureDir); + TestProjectHelper.DeleteSqliteDatabaseFiles(dbPath); + } + } + [Fact] public void ToolsCall_Index_MaxReferencesPerFilePersistsReferenceCountExceededIssue_Issue3719() { @@ -13315,7 +13453,9 @@ public void ToolsCall_BackfillFold_DryRunDoesNotWrite() Assert.False(structured["fold_ready"]!.GetValue()); Assert.Equal("DEADBEEFDEADBEEF", _db.GetMetaString("fold_key_fingerprint")); - Assert.Equal(DbContext.CurrentSchemaVersion, _db.GetUserVersion()); + Assert.Equal( + DbContext.CurrentSchemaVersion & ~DbContext.SymbolKindFilterAuditStorageContractFlag, + _db.GetUserVersion()); } [Fact] diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerStatusReadinessTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerStatusReadinessTests.cs index 6488f03c8..215f4c4a8 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerStatusReadinessTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerStatusReadinessTests.cs @@ -9,6 +9,62 @@ namespace CodeIndex.Tests; public sealed class QueryCommandRunnerStatusReadinessTests { + [Fact] + public void RunStatus_LegacyMissingSymbolKindPolicyUsesConservativeReadableFallback_Issue5224() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_status_symbol_policy_legacy_5224"); + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + try + { + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + var writer = new DbWriter(db.Connection); + writer.MarkIndexComplete(); + } + using (var connection = new SqliteConnection($"Data Source={dbPath};Pooling=False")) + { + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = $""" + DELETE FROM codeindex_meta WHERE key = '{DbContext.SymbolKindFilterMetaKey}'; + ALTER TABLE files DROP COLUMN {DbContext.SymbolsDroppedByKindFilterColumn}; + PRAGMA wal_checkpoint(TRUNCATE); + """; + command.ExecuteNonQuery(); + } + SqliteConnection.ClearAllPools(); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunStatus( + ["--db", dbPath, "--read-only", "--json"], + JsonOptions)); + + Assert.True( + exitCode == CommandExitCodes.Success, + $"exit={exitCode}\nstdout={stdout}\nstderr={stderr}"); + Assert.Equal(string.Empty, stderr); + using var document = ParseJsonOutput(stdout); + var json = document.RootElement; + Assert.False(json.GetProperty("symbol_kind_filter_provenance_available").GetBoolean()); + Assert.False(json.TryGetProperty("symbol_kind_filter", out _)); + Assert.False(json.TryGetProperty("symbols_dropped_by_kind_filter", out _)); + Assert.False(json.GetProperty("index_complete").GetBoolean()); + Assert.Contains( + json.GetProperty("index_incomplete_reasons").EnumerateArray(), + reason => reason.GetString() == DbReader.SymbolKindFilterProvenanceUnavailableReason); + + var (humanExitCode, humanStdout, _) = CaptureConsole(() => QueryCommandRunner.RunStatus( + ["--db", dbPath, "--read-only"], + JsonOptions)); + Assert.Equal(CommandExitCodes.Success, humanExitCode); + Assert.Contains("Kind policy", humanStdout, StringComparison.Ordinal); + Assert.Contains("unavailable (legacy generation)", humanStdout, StringComparison.Ordinal); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunStatus_Json_ReportsHotspotFamilyReadinessDegradationRebuild_2959() { diff --git a/tests/CodeIndex.Tests/WorkspaceCommandRunnerTests.cs b/tests/CodeIndex.Tests/WorkspaceCommandRunnerTests.cs index e204d8cc8..8429ffaed 100644 --- a/tests/CodeIndex.Tests/WorkspaceCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/WorkspaceCommandRunnerTests.cs @@ -49,7 +49,7 @@ public void WorkspaceList_ReadsManifestMembers() } [Fact] - public void WorkspaceStatusJson_ReportsBoundedMemberIndexHealth_Issue4726() + public void WorkspaceStatusJson_ReportsBoundedMemberIndexHealth_Issue4726AndIssue5224() { using var project = TestProjectHelper.CreateTempProjectScope("cdidx_workspace_status_health"); var root = project.Root; @@ -66,6 +66,15 @@ public void WorkspaceStatusJson_ReportsBoundedMemberIndexHealth_Issue4726() File.WriteAllText(Path.Combine(readyRoot, "App.cs"), IndexedContent); var readyDb = TestProjectHelper.CreateProjectDb(readyRoot); TestProjectHelper.InsertIndexedFile(readyDb, "App.cs", "csharp", IndexedContent); + using (var db = new DbContext(DbOpenIntent.WriteIndex, readyDb)) + { + var writer = new DbWriter(db.Connection); + writer.MarkSymbolKindFilterAuditStorageContract(); + writer.SetMeta(DbContext.SymbolKindFilterMetaKey, "include=class;exclude="); + using var command = db.Connection.CreateCommand(); + command.CommandText = $"UPDATE files SET {DbContext.SymbolsDroppedByKindFilterColumn} = 7"; + command.ExecuteNonQuery(); + } File.WriteAllText(Path.Combine(staleRoot, "App.cs"), IndexedContent); var staleDb = TestProjectHelper.CreateProjectDb(staleRoot); @@ -140,6 +149,16 @@ public void WorkspaceStatusJson_ReportsBoundedMemberIndexHealth_Issue4726() var graphDataCurrent = ready.GetProperty("graph_data_current").GetBoolean(); var referenceGraphComplete = ready.GetProperty("reference_graph_complete").GetBoolean(); var indexComplete = ready.GetProperty("index_complete").GetBoolean(); + Assert.False(indexComplete); + Assert.Equal("index_incomplete", ready.GetProperty("reason").GetString()); + Assert.Contains( + ready.GetProperty("index_incomplete_reasons").EnumerateArray(), + reason => reason.GetString() == DbReader.SymbolKindFilterCoverageLimitedReason); + Assert.True(ready.GetProperty("symbol_kind_filter_provenance_available").GetBoolean()); + Assert.Equal( + "class", + ready.GetProperty("symbol_kind_filter").GetProperty("include")[0].GetString()); + Assert.Equal(7, ready.GetProperty("symbols_dropped_by_kind_filter").GetInt64()); Assert.Equal( graphTableAvailable && graphDataCurrent && referenceGraphComplete && indexComplete, ready.GetProperty("graph_ready").GetBoolean()); diff --git a/tests/CodeIndex.Tests/golden/status.json b/tests/CodeIndex.Tests/golden/status.json index 9a0fc9db7..1491d0dee 100644 --- a/tests/CodeIndex.Tests/golden/status.json +++ b/tests/CodeIndex.Tests/golden/status.json @@ -202,6 +202,12 @@ "file_issues_data_current": false, "migration_in_progress": false, "index_complete": true, + "symbol_kind_filter_provenance_available": true, + "symbol_kind_filter": { + "include": [], + "exclude": [] + }, + "symbols_dropped_by_kind_filter": 0, "hotspot_family_ready": false, "hotspot_family_degraded_reason": "cross-file hotspot family grouping is degraded (hotspot_family_support_not_indexed=csharp); Run \u0060cdidx index \u003CprojectPath\u003E --rebuild\u0060 to rebuild and stamp authoritative hotspot families for every indexed row.", "language_readiness": {