diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 082bca1d6..90b71d8c2 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -20,6 +20,7 @@ #include "lsp/java_lsp.h" #include "lsp/kotlin_lsp.h" #include "lsp/rust_lsp.h" +#include "lsp/lsp_work.h" #include "preprocessor.h" #include "sql_values.h" // #1735: literal INSERT rows kept out of the SQL parse #include "foundation/compat.h" @@ -109,6 +110,190 @@ void cbm_reset_profile(void) { atomic_store(&total_files, 0); } +/* New facts use checked allocation and atomic publication; legacy extraction + * arrays retain their existing contracts. */ +static bool py_namespace_nonempty(const char *s) { + return s && s[0]; +} + +static bool py_namespace_fact_valid(const CBMPyNamespaceFact *f) { + if (!f || f->name_count < 0 || (size_t)f->name_count > SIZE_MAX / sizeof(*f->names) || + (f->name_count == 0 ? f->names != NULL : f->names == NULL)) + return false; + bool literal = f->kind == CBM_PY_NS_ALL_SET || f->kind == CBM_PY_NS_ALL_APPEND; + if (!literal && (f->names || f->name_count || f->sequence_kind != CBM_PY_NS_SEQUENCE_NONE)) + return false; + if (f->kind != CBM_PY_NS_OWN_DEF && f->def_kind != CBM_PY_NS_DEF_NONE) + return false; + if (f->kind != CBM_PY_NS_IMPORT_MODULE && f->flags) + return false; + if (f->kind != CBM_PY_NS_IMPORT_NAME && f->kind != CBM_PY_NS_IMPORT_STAR && f->relative_level) + return false; + bool unknown = f->kind == CBM_PY_NS_SHADOW || f->kind == CBM_PY_NS_UNKNOWN || + f->kind == CBM_PY_NS_ALL_UNKNOWN; + if (!unknown && f->reason != CBM_PY_NS_REASON_NONE) + return false; + switch (f->kind) { + case CBM_PY_NS_OWN_DEF: + return py_namespace_nonempty(f->local_name) && !f->module_name && !f->member_name && + (f->def_kind == CBM_PY_NS_DEF_FUNCTION || f->def_kind == CBM_PY_NS_DEF_CLASS); + case CBM_PY_NS_IMPORT_MODULE: + return py_namespace_nonempty(f->local_name) && py_namespace_nonempty(f->module_name) && + !f->member_name && !(f->flags & ~CBM_PY_NS_IMPORT_BINDS_ROOT); + case CBM_PY_NS_IMPORT_NAME: + case CBM_PY_NS_IMPORT_STAR: + if (!f->module_name || (!f->module_name[0] && !f->relative_level)) + return false; + return f->kind == CBM_PY_NS_IMPORT_NAME + ? py_namespace_nonempty(f->local_name) && py_namespace_nonempty(f->member_name) + : !f->local_name && !f->member_name; + case CBM_PY_NS_SHADOW: + return py_namespace_nonempty(f->local_name) && !f->module_name && !f->member_name && + (f->reason == CBM_PY_NS_REASON_VALUE || f->reason == CBM_PY_NS_REASON_DECORATED || + f->reason == CBM_PY_NS_REASON_UNPROVEN_CLASS || + f->reason == CBM_PY_NS_REASON_UNSUPPORTED); + case CBM_PY_NS_DELETE: + return py_namespace_nonempty(f->local_name) && strcmp(f->local_name, "__all__") != 0 && + !f->module_name && !f->member_name; + case CBM_PY_NS_UNKNOWN: + return !f->local_name && !f->module_name && !f->member_name && + (f->reason == CBM_PY_NS_REASON_COMPOUND || f->reason == CBM_PY_NS_REASON_EFFECT || + f->reason == CBM_PY_NS_REASON_UNSUPPORTED || f->reason == CBM_PY_NS_REASON_PARSE); + case CBM_PY_NS_ALL_UNKNOWN: + return !f->local_name && !f->module_name && !f->member_name && + (f->reason == CBM_PY_NS_REASON_VALUE || f->reason == CBM_PY_NS_REASON_EFFECT || + f->reason == CBM_PY_NS_REASON_UNSUPPORTED || f->reason == CBM_PY_NS_REASON_PARSE); + case CBM_PY_NS_ALL_SET: + case CBM_PY_NS_ALL_APPEND: + if (f->local_name || f->module_name || f->member_name || + (f->sequence_kind != CBM_PY_NS_SEQUENCE_LIST && + f->sequence_kind != CBM_PY_NS_SEQUENCE_TUPLE)) + return false; + for (int i = 0; i < f->name_count; i++) { + if (!f->names[i]) + return false; + } + return true; + case CBM_PY_NS_ALL_DELETE: + return !f->local_name && !f->module_name && !f->member_name; + default: + return false; + } +} + +bool cbm_py_namespace_facts_valid(const CBMPyNamespaceFacts *f) { + if (!f) + return false; + bool empty = !f->items && f->count == 0 && f->cap == 0; + if (f->version == 0) + return f->status == CBM_PY_NS_NOT_CAPTURED && f->failure == CBM_PY_NS_FAILURE_NONE && empty; + if (f->version != CBM_PY_NAMESPACE_FACTS_VERSION || f->language < 0 || + f->language >= CBM_LANG_COUNT) + return false; + switch (f->status) { + case CBM_PY_NS_NOT_CAPTURED: + return empty && f->failure == CBM_PY_NS_FAILURE_NONE; + case CBM_PY_NS_NOT_APPLICABLE: + return empty && f->language != CBM_LANG_PYTHON && f->failure == CBM_PY_NS_FAILURE_NONE; + case CBM_PY_NS_INCOMPLETE: + return empty && f->failure >= CBM_PY_NS_FAILURE_INVALID_INPUT && + f->failure <= CBM_PY_NS_FAILURE_LIMIT; + case CBM_PY_NS_COMPLETE: + if (f->language != CBM_LANG_PYTHON || f->failure != CBM_PY_NS_FAILURE_NONE || + f->count < 0 || f->cap < f->count || (size_t)f->cap > SIZE_MAX / sizeof(*f->items) || + (f->count == 0 ? !empty : f->items == NULL)) + return false; + for (int i = 0; i < f->count; i++) { + if (!py_namespace_fact_valid(&f->items[i])) + return false; + } + return true; + default: + return false; + } +} + +#ifdef CBM_ENABLE_TEST_SEAMS +static CBM_TLS int py_namespace_copy_left = -1; +void cbm_py_namespace_test_copy_fail_after(int successful_allocations) { + py_namespace_copy_left = successful_allocations; +} +#endif + +static void *py_namespace_copy_alloc(CBMArena *arena, size_t size) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (py_namespace_copy_left == 0) + return NULL; + if (py_namespace_copy_left > 0) + py_namespace_copy_left--; +#endif + return cbm_arena_alloc(arena, size); +} + +static bool py_namespace_copy_string(CBMArena *arena, const char *source, const char **out) { + *out = NULL; + if (!source) + return true; + size_t len = strlen(source); + if (len == SIZE_MAX) + return false; + char *copy = py_namespace_copy_alloc(arena, len + 1); + if (!copy) + return false; + memcpy(copy, source, len + 1); + *out = copy; + return true; +} + +bool cbm_py_namespace_facts_copy(CBMArena *arena, const CBMPyNamespaceFacts *source, + CBMPyNamespaceFacts *out) { + if (!out || source == out) + return false; + CBMPyNamespaceFacts failure = {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = CBM_LANG_PYTHON, + .status = CBM_PY_NS_INCOMPLETE, + .failure = CBM_PY_NS_FAILURE_INVALID_INPUT}; + if (!arena || !cbm_py_namespace_facts_valid(source)) { + *out = failure; + return false; + } + failure.language = source->language; + failure.failure = CBM_PY_NS_FAILURE_ALLOCATION; + CBMPyNamespaceFacts copy = *source; + copy.items = NULL; + copy.cap = copy.count; + if (copy.count) { + copy.items = py_namespace_copy_alloc(arena, (size_t)copy.count * sizeof(*copy.items)); + if (!copy.items) + goto failed; + memcpy(copy.items, source->items, (size_t)copy.count * sizeof(*copy.items)); + } + for (int i = 0; i < copy.count; i++) { + const CBMPyNamespaceFact *src = &source->items[i]; + CBMPyNamespaceFact *dst = ©.items[i]; + if (!py_namespace_copy_string(arena, src->local_name, &dst->local_name) || + !py_namespace_copy_string(arena, src->module_name, &dst->module_name) || + !py_namespace_copy_string(arena, src->member_name, &dst->member_name)) + goto failed; + dst->names = NULL; + if (src->name_count) { + dst->names = + py_namespace_copy_alloc(arena, (size_t)src->name_count * sizeof(*dst->names)); + if (!dst->names) + goto failed; + for (int j = 0; j < src->name_count; j++) { + if (!py_namespace_copy_string(arena, src->names[j], &dst->names[j])) + goto failed; + } + } + } + *out = copy; + return true; +failed: + *out = failure; + return false; +} + // --- Growable array push functions --- #define GROW_ARRAY(arr, arena) \ @@ -243,6 +428,11 @@ void cbm_typeassign_push(CBMTypeAssignArray *arr, CBMArena *a, CBMTypeAssign ta) arr->items[arr->count++] = ta; } +void cbm_fieldtype_push(CBMFieldTypeArray *arr, CBMArena *a, CBMFieldType ft) { + GROW_ARRAY(arr, a); + arr->items[arr->count++] = ft; +} + void cbm_stringref_push(CBMStringRefArray *arr, CBMArena *a, CBMStringRef sr) { GROW_ARRAY(arr, a); arr->items[arr->count++] = sr; @@ -2141,6 +2331,183 @@ static bool cbm_sql_values_exclusion_on(const char *rel_path) { return true; } +/* ── Innermost enclosing Function/Method per call (#1527) ────────────── + * The call-context metrics attribute each call to the Function/Method def with + * the SMALLEST line span containing the call's line, ties to the lowest def + * index. Scanning every def for every call made that O(calls x defs) per file + * -- quadratic in file size, the largest single cost on a module with + * thousands of methods. The sweep below gives the identical answer in + * O((calls + defs) log defs): calls in line order, candidates entering a + * min-heap keyed (span, index) once their start line is reached and leaving it + * lazily once their end line has passed -- lines only grow, so a def that has + * ended never contains a later call. The heap top is then the smallest + * (span, index) among the defs that contain the line. */ +static bool enclosing_is_callable(const CBMDefinition *d) { + return d->name && d->label && + (strcmp(d->label, "Function") == 0 || strcmp(d->label, "Method") == 0); +} + +/* The reference answer for one line (and the path when the sweep's arrays + * cannot be allocated). */ +static int cbm_enclosing_callable_scan(const CBMFileResult *result, int line) { + int best = -1; + int best_span = -1; + for (int di = 0; di < result->defs.count; di++) { + const CBMDefinition *d = &result->defs.items[di]; + CBM_LSP_WORK(1); + if (!enclosing_is_callable(d)) { + continue; + } + if ((int)d->start_line <= line && line <= (int)d->end_line) { + int span = (int)d->end_line - (int)d->start_line; + if (best < 0 || span < best_span) { + best_span = span; + best = di; + } + } + } + return best; +} + +static CBM_TLS const CBMFileResult *tl_enclosing_result; + +static int enclosing_def_by_start(const void *a, const void *b) { + int ia = *(const int *)a; + int ib = *(const int *)b; + uint32_t la = tl_enclosing_result->defs.items[ia].start_line; + uint32_t lb = tl_enclosing_result->defs.items[ib].start_line; + return la < lb ? -1 : la > lb ? 1 : ia - ib; +} + +static int enclosing_call_by_line(const void *a, const void *b) { + int ia = *(const int *)a; + int ib = *(const int *)b; + int la = tl_enclosing_result->calls.items[ia].start_line; + int lb = tl_enclosing_result->calls.items[ib].start_line; + return la < lb ? -1 : la > lb ? 1 : ia - ib; +} + +/* Heap order: smaller span first, then smaller def index. */ +static bool enclosing_heap_less(const CBMFileResult *r, int a, int b) { + const CBMDefinition *da = &r->defs.items[a]; + const CBMDefinition *db = &r->defs.items[b]; + int sa = (int)da->end_line - (int)da->start_line; + int sb = (int)db->end_line - (int)db->start_line; + return sa != sb ? sa < sb : a < b; +} + +static void enclosing_heap_push(const CBMFileResult *r, int *heap, int *n, int di) { + int i = (*n)++; + heap[i] = di; + while (i > 0) { + int parent = (i - 1) / 2; + CBM_LSP_WORK(1); + if (!enclosing_heap_less(r, heap[i], heap[parent])) { + break; + } + int t = heap[i]; + heap[i] = heap[parent]; + heap[parent] = t; + i = parent; + } +} + +static void enclosing_heap_pop(const CBMFileResult *r, int *heap, int *n) { + heap[0] = heap[--(*n)]; + int i = 0; + for (;;) { + int l = (2 * i) + 1; + int m = i; + CBM_LSP_WORK(1); + if (l < *n && enclosing_heap_less(r, heap[l], heap[m])) { + m = l; + } + if (l + 1 < *n && enclosing_heap_less(r, heap[l + 1], heap[m])) { + m = l + 1; + } + if (m == i) { + break; + } + int t = heap[i]; + heap[i] = heap[m]; + heap[m] = t; + i = m; + } +} + +/* enclosing[ci] = innermost Function/Method def index for call ci, or -1. + * Returns NULL when the working arrays cannot be allocated; the caller then + * asks cbm_enclosing_callable_scan per call (same answer, old cost). */ +static int *cbm_enclosing_callables(const CBMFileResult *result, int call_count) { + int def_count = result->defs.count; + if (call_count <= 0) { + return NULL; + } + size_t call_bytes = (size_t)call_count * sizeof(int); + size_t def_bytes = (size_t)(def_count > 0 ? def_count : 1) * sizeof(int); + int *enclosing = cbm_alloc(CBM_MEM_CLASS_OTHER, call_bytes); + int *call_order = cbm_alloc(CBM_MEM_CLASS_OTHER, call_bytes); + int *cands = cbm_alloc(CBM_MEM_CLASS_OTHER, def_bytes); + int *heap = cbm_alloc(CBM_MEM_CLASS_OTHER, def_bytes); + if (!enclosing || !call_order || !cands || !heap) { + cbm_free(CBM_MEM_CLASS_OTHER, enclosing); + cbm_free(CBM_MEM_CLASS_OTHER, call_order); + cbm_free(CBM_MEM_CLASS_OTHER, cands); + cbm_free(CBM_MEM_CLASS_OTHER, heap); + return NULL; + } + int ncand = 0; + for (int di = 0; di < def_count; di++) { + if (enclosing_is_callable(&result->defs.items[di])) { + cands[ncand++] = di; + } + } + for (int ci = 0; ci < call_count; ci++) { + call_order[ci] = ci; + enclosing[ci] = -1; + } + tl_enclosing_result = result; + qsort(cands, (size_t)ncand, sizeof(int), enclosing_def_by_start); + qsort(call_order, (size_t)call_count, sizeof(int), enclosing_call_by_line); + tl_enclosing_result = NULL; + + int heap_n = 0; + int next = 0; + for (int k = 0; k < call_count; k++) { + int ci = call_order[k]; + int line = result->calls.items[ci].start_line; + while (next < ncand && (int)result->defs.items[cands[next]].start_line <= line) { + enclosing_heap_push(result, heap, &heap_n, cands[next++]); + } + while (heap_n > 0 && (int)result->defs.items[heap[0]].end_line < line) { + enclosing_heap_pop(result, heap, &heap_n); + } + enclosing[ci] = heap_n > 0 ? heap[0] : -1; + } + cbm_free(CBM_MEM_CLASS_OTHER, call_order); + cbm_free(CBM_MEM_CLASS_OTHER, cands); + cbm_free(CBM_MEM_CLASS_OTHER, heap); + return enclosing; +} + +#ifdef CBM_ENABLE_TEST_SEAMS +int cbm_test_enclosing_sweep_mismatches(const CBMFileResult *result) { + int *enclosing = cbm_enclosing_callables(result, result->calls.count); + if (!enclosing) { + return result->calls.count > 0 ? -1 : 0; + } + int mismatches = 0; + for (int ci = 0; ci < result->calls.count; ci++) { + int line = result->calls.items[ci].start_line; + if (line > 0 && enclosing[ci] != cbm_enclosing_callable_scan(result, line)) { + mismatches++; + } + } + cbm_free(CBM_MEM_CLASS_OTHER, enclosing); + return mismatches; +} +#endif + static CBMFileResult *extract_file_ex_body(const char *source, int source_len, CBMLanguage language, const char *project, const char *rel_path, int64_t timeout_micros, const char **extra_defines, @@ -2154,6 +2521,10 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C return NULL; } + result->py_namespace.version = CBM_PY_NAMESPACE_FACTS_VERSION; + result->py_namespace.language = language; + result->py_namespace.status = + language == CBM_LANG_PYTHON ? CBM_PY_NS_NOT_CAPTURED : CBM_PY_NS_NOT_APPLICABLE; cbm_work_arena_take(&result->arena); CBMArena *a = &result->arena; @@ -2340,6 +2711,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C // then a single unified cursor walk handles the remaining 7 extractors. cbm_extract_definitions(&ctx); cbm_extract_imports(&ctx); + cbm_extract_python_namespace_facts(&ctx); cbm_extract_unified(&ctx); result->tree_nodes = ts_node_descendant_count(root); result->walk_nodes_visited = ctx.walk_nodes_visited; @@ -2504,8 +2876,10 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C // Also run LSP on expanded source for additional type-resolved // calls (language is already C/C++/CUDA — checked in enclosing // block). Runs in every mode. - cbm_run_c_lsp(a, result, expanded, expanded_len, pp_root, - language != CBM_LANG_C, CBM_SOURCE_ORIGIN_PREPROCESSED); + if (!result->lsp_skipped) { + cbm_run_c_lsp(a, result, expanded, expanded_len, pp_root, + language != CBM_LANG_C, CBM_SOURCE_ORIGIN_PREPROCESSED); + } /* All C-LSP emitters stamp origin directly so rewrite-time * comparisons are already safe. Keep this boundary sweep as @@ -2828,28 +3202,14 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C d->param_count = pc; } + int *enclosing = cbm_enclosing_callables(result, orig_calls_count); for (int ci = 0; ci < orig_calls_count; ci++) { const CBMCall *c = &result->calls.items[ci]; if (!c->callee_name || c->start_line <= 0) { continue; } // Innermost enclosing Function/Method def by line range (smallest span). - int best = -1; - int best_span = -1; - for (int di = 0; di < def_count; di++) { - const CBMDefinition *d = &result->defs.items[di]; - if (!d->name || !d->label || - (strcmp(d->label, "Function") != 0 && strcmp(d->label, "Method") != 0)) { - continue; - } - if ((int)d->start_line <= c->start_line && c->start_line <= (int)d->end_line) { - int span = (int)d->end_line - (int)d->start_line; - if (best < 0 || span < best_span) { - best_span = span; - best = di; - } - } - } + int best = enclosing ? enclosing[ci] : cbm_enclosing_callable_scan(result, c->start_line); if (best < 0) { continue; } @@ -2894,6 +3254,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C } free(has_self); free(has_guarded); + cbm_free(CBM_MEM_CLASS_OTHER, enclosing); uint64_t t2 = now_ns(); @@ -3057,6 +3418,35 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua CBMFileResult *result = extract_file_ex_body(source, source_len, language, project, rel_path, timeout_micros, extra_defines, include_paths, macro_table, return_type_table, &scratch); + /* Even early parse/quarantine exits publish conservative Python metadata. + * No tree is reparsed, and existing best-effort extraction flags stay intact. */ + if (result && language == CBM_LANG_PYTHON && + result->py_namespace.status == CBM_PY_NS_NOT_CAPTURED) { + CBMPyNamespaceFailure failure = CBM_PY_NS_FAILURE_NONE; + if (!result->module_qn || !result->module_qn[0]) { + if (!project || !project[0] || !rel_path || !rel_path[0]) { + failure = CBM_PY_NS_FAILURE_INVALID_INPUT; + } else { + result->module_qn = + cbm_fqn_module_source_lang(&result->arena, project, rel_path, language); + if (!result->module_qn || !result->module_qn[0]) + failure = CBM_PY_NS_FAILURE_ALLOCATION; + } + } + if (failure) { + result->py_namespace = (CBMPyNamespaceFacts){.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = language, + .status = CBM_PY_NS_INCOMPLETE, + .failure = failure}; + } else { + CBMExtractCtx facts_ctx = {.arena = &result->arena, + .result = result, + .source = source, + .source_len = source_len, + .language = language}; + cbm_extract_python_namespace_facts(&facts_ctx); + } + } /* !tl_scratch_live: a nested extraction (an embedded language inside this * file) may already have parked its own; never overwrite it. */ /* Kept up to CBM_EXTRACT_SCRATCH_KEEP_BYTES, grown blocks included: the diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index fb5e446a5..2a83d0bd7 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -310,6 +310,79 @@ typedef struct { bool is_default; // ES default import (`import X from "Y"`), JS/TS only (#1916) } CBMImport; +/* Content-only ordered Python namespace facts; no resolved target identities. */ +#define CBM_PY_NAMESPACE_FACTS_VERSION 1u + +typedef enum { + CBM_PY_NS_NOT_CAPTURED = 0, + CBM_PY_NS_NOT_APPLICABLE = 1, + CBM_PY_NS_COMPLETE = 2, + CBM_PY_NS_INCOMPLETE = 3 +} CBMPyNamespaceStatus; +typedef enum { + CBM_PY_NS_FAILURE_NONE = 0, + CBM_PY_NS_FAILURE_INVALID_INPUT = 1, + CBM_PY_NS_FAILURE_ALLOCATION = 2, + CBM_PY_NS_FAILURE_LIMIT = 3 +} CBMPyNamespaceFailure; +typedef enum { + CBM_PY_NS_OWN_DEF = 1, + CBM_PY_NS_IMPORT_MODULE = 2, + CBM_PY_NS_IMPORT_NAME = 3, + CBM_PY_NS_IMPORT_STAR = 4, + CBM_PY_NS_SHADOW = 5, + CBM_PY_NS_DELETE = 6, + CBM_PY_NS_UNKNOWN = 7, + CBM_PY_NS_ALL_SET = 8, + CBM_PY_NS_ALL_APPEND = 9, + CBM_PY_NS_ALL_UNKNOWN = 10, + CBM_PY_NS_ALL_DELETE = 11 +} CBMPyNamespaceFactKind; +typedef enum { + CBM_PY_NS_DEF_NONE = 0, + CBM_PY_NS_DEF_FUNCTION = 1, + CBM_PY_NS_DEF_CLASS = 2 +} CBMPyNamespaceDefKind; +typedef enum { + CBM_PY_NS_SEQUENCE_NONE = 0, + CBM_PY_NS_SEQUENCE_LIST = 1, + CBM_PY_NS_SEQUENCE_TUPLE = 2 +} CBMPyNamespaceSequenceKind; +typedef enum { + CBM_PY_NS_REASON_NONE = 0, + CBM_PY_NS_REASON_VALUE = 1, + CBM_PY_NS_REASON_DECORATED = 2, + CBM_PY_NS_REASON_UNPROVEN_CLASS = 3, + CBM_PY_NS_REASON_COMPOUND = 4, + CBM_PY_NS_REASON_EFFECT = 5, + CBM_PY_NS_REASON_UNSUPPORTED = 6, + CBM_PY_NS_REASON_PARSE = 7 +} CBMPyNamespaceUnknownReason; +enum { CBM_PY_NS_IMPORT_BINDS_ROOT = 1u << 0 }; + +typedef struct { + CBMPyNamespaceFactKind kind; + CBMPyNamespaceDefKind def_kind; + CBMPyNamespaceSequenceKind sequence_kind; + CBMPyNamespaceUnknownReason reason; + uint32_t flags; + uint32_t relative_level; + const char *local_name; + const char *module_name; + const char *member_name; + const char **names; + int name_count; +} CBMPyNamespaceFact; +typedef struct { + uint32_t version; + CBMLanguage language; + CBMPyNamespaceStatus status; + CBMPyNamespaceFailure failure; + CBMPyNamespaceFact *items; + int count; + int cap; +} CBMPyNamespaceFacts; + typedef enum { CBM_USAGE_VALUE = 0, CBM_USAGE_CALL_REFERENCE, @@ -405,6 +478,17 @@ typedef struct { CBMChannelDirection direction; } CBMChannel; +/* Python: one annotated instance field of a class -- `x: T` or `x: T = v` in + * the class body, `self.x: T = v` in __init__, or `self.x = p` where `p` is an + * annotated __init__ parameter. Not a graph node: it only carries the field's + * declared type to the cross-file LSP, so `obj.x.m()` on a class imported + * from another file can be typed (#1277). */ +typedef struct { + const char *class_qn; // QN of the owning class + const char *field_name; // attribute name + const char *type_text; // raw annotation text, resolved later per file +} CBMFieldType; + // Rust: impl Trait for Struct typedef struct { const char *trait_name; // trait name (raw text) @@ -439,6 +523,15 @@ typedef struct { int cap; } CBMResolvedCallArray; +/* Refinement status: existing parser/no-op exits are separate from these + * explicit evaluator failures. The caller retains prior completed output. */ +typedef enum { + CBM_LSP_COMPLETE = 0, + CBM_LSP_MEMO_FAILED, + CBM_LSP_DEPTH_EXCEEDED, + CBM_LSP_SCOPE_FAILED, +} CBMLSPStatus; + // Growable arrays used during extraction. typedef struct { CBMDefinition *items; @@ -518,6 +611,12 @@ typedef struct { int cap; } CBMChannelArray; +typedef struct { + CBMFieldType *items; + int count; + int cap; +} CBMFieldTypeArray; + // Full extraction result for one file. typedef struct CBMFileResult { CBMArena arena; // owns local memory; composites may also retain child arenas below @@ -525,6 +624,7 @@ typedef struct CBMFileResult { CBMDefArray defs; CBMCallArray calls; CBMImportArray imports; + CBMPyNamespaceFacts py_namespace; CBMUsageArray usages; CBMThrowArray throws; CBMRWArray rw; @@ -536,6 +636,7 @@ typedef struct CBMFileResult { CBMStringRefArray string_refs; // URL/config string literals from AST CBMInfraBindingArray infra_bindings; // topic→URL pairs from IaC configs CBMChannelArray channels; // Socket.IO / EventEmitter pub/sub participation + CBMFieldTypeArray field_types; // Python: annotated instance fields (#1277) const char *module_qn; // module qualified name const char *namespace_name; // declared namespace/package (Java/Kotlin/C#/PHP), NULL if none @@ -882,6 +983,15 @@ int cbm_macro_extraction_enabled(void); // --- Internal helpers used by extractors --- +CBMPyNamespaceStatus cbm_extract_python_namespace_facts(CBMExtractCtx *ctx); +bool cbm_py_namespace_facts_valid(const CBMPyNamespaceFacts *facts); +bool cbm_py_namespace_facts_copy(CBMArena *arena, const CBMPyNamespaceFacts *source, + CBMPyNamespaceFacts *out); +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_py_namespace_test_capture_fail_after(int successful_allocations); +void cbm_py_namespace_test_copy_fail_after(int successful_allocations); +#endif + // Growable array push functions (arena-allocated, no individual free needed). void cbm_defs_push(CBMDefArray *arr, CBMArena *a, CBMDefinition def); void cbm_calls_push(CBMCallArray *arr, CBMArena *a, CBMCall call); @@ -892,9 +1002,17 @@ void cbm_rw_push(CBMRWArray *arr, CBMArena *a, CBMReadWrite rw); void cbm_typerefs_push(CBMTypeRefArray *arr, CBMArena *a, CBMTypeRef tr); void cbm_envaccess_push(CBMEnvAccessArray *arr, CBMArena *a, CBMEnvAccess ea); void cbm_typeassign_push(CBMTypeAssignArray *arr, CBMArena *a, CBMTypeAssign ta); +void cbm_fieldtype_push(CBMFieldTypeArray *arr, CBMArena *a, CBMFieldType ft); void cbm_stringref_push(CBMStringRefArray *arr, CBMArena *a, CBMStringRef sr); void cbm_infrabinding_push(CBMInfraBindingArray *arr, CBMArena *a, CBMInfraBinding ib); void cbm_impltrait_push(CBMImplTraitArray *arr, CBMArena *a, CBMImplTrait it); + +#ifdef CBM_ENABLE_TEST_SEAMS +/* #1527 seam: how many of result's calls get a different innermost enclosing + * Function/Method from the call-context sweep than from the reference + * per-call scan (0 = identical), or -1 when the sweep could not allocate. */ +int cbm_test_enclosing_sweep_mismatches(const CBMFileResult *result); +#endif void cbm_resolvedcall_push(CBMResolvedCallArray *arr, CBMArena *a, CBMResolvedCall rc); void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch); @@ -948,4 +1066,9 @@ bool cbm_label_is_relation(const char *label); // `label` may be NULL (returns false). Defined in helpers.c. bool cbm_label_is_registry_symbol(const char *label); +/* Python symbol scope only: callers must establish the language explicitly. + * A trailing .__init__ is omitted when a nonempty prefix precedes it; raw + * Module/file identity is unchanged. NULL returns zero. */ +size_t cbm_fqn_symbol_scope_len(const char *module_qn); + #endif // CBM_H diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index bb56e2870..de5b2e90d 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -4584,6 +4584,169 @@ static bool extract_sql_ddl_class_def(CBMExtractCtx *ctx, TSNode node, const cha return true; } +/* ── Python annotated instance fields (#1277) ───────────────────── + * Exported as result->field_types (never graph nodes) so the cross-file LSP + * can type `obj.x` when obj's class lives in another file. Only DECLARED + * types count: a class-body annotation, a `self.x: T = v` annotation in + * __init__, or `self.x = p` where p is an annotated __init__ parameter. An + * unannotated right-hand side is never guessed at. */ + +static void py_push_field_type(CBMExtractCtx *ctx, const char *class_qn, TSNode name_node, + const char *type_text) { + char *name = cbm_node_text(ctx->arena, name_node, ctx->source); + if (!name || !name[0] || !type_text || !type_text[0]) { + return; + } + CBMFieldType ft = {.class_qn = class_qn, .field_name = name, .type_text = type_text}; + cbm_fieldtype_push(&ctx->result->field_types, ctx->arena, ft); +} + +/* The assignment inside an expression_statement, or a null node. */ +static TSNode py_statement_assignment(TSNode stmt) { + TSNode null_node = {0}; + if (strcmp(ts_node_type(stmt), "expression_statement") != 0 || + ts_node_named_child_count(stmt) == 0) { + return null_node; + } + TSNode inner = ts_node_named_child(stmt, 0); + return strcmp(ts_node_type(inner), "assignment") == 0 ? inner : null_node; +} + +/* Annotation text of the __init__ parameter called `name`, or NULL. */ +static const char *py_init_param_annotation(CBMExtractCtx *ctx, TSNode params, const char *name) { + uint32_t n = ts_node_named_child_count(params); + for (uint32_t i = 0; i < n; i++) { + TSNode p = ts_node_named_child(params, i); + const char *pk = ts_node_type(p); + TSNode pname = {0}; + if (strcmp(pk, "typed_default_parameter") == 0) { + pname = ts_node_child_by_field_name(p, TS_FIELD("name")); + } else if (strcmp(pk, "typed_parameter") == 0 && ts_node_named_child_count(p) > 0) { + pname = ts_node_named_child(p, 0); + } else { + continue; + } + TSNode ptype = ts_node_child_by_field_name(p, TS_FIELD("type")); + if (ts_node_is_null(pname) || ts_node_is_null(ptype) || + strcmp(ts_node_type(pname), "identifier") != 0) { + continue; + } + char *pn = cbm_node_text(ctx->arena, pname, ctx->source); + if (pn && strcmp(pn, name) == 0) { + return cbm_node_text(ctx->arena, ptype, ctx->source); + } + } + return NULL; +} + +/* `self.x: T = v` / `self.x = p` inside __init__ (self = its first parameter). */ +static void py_init_field_assignment(CBMExtractCtx *ctx, const char *class_qn, TSNode assign, + TSNode params, const char *self_name) { + TSNode left = ts_node_child_by_field_name(assign, TS_FIELD("left")); + if (ts_node_is_null(left) || strcmp(ts_node_type(left), "attribute") != 0) { + return; + } + TSNode obj = ts_node_child_by_field_name(left, TS_FIELD("object")); + TSNode attr = ts_node_child_by_field_name(left, TS_FIELD("attribute")); + if (ts_node_is_null(obj) || ts_node_is_null(attr) || + strcmp(ts_node_type(obj), "identifier") != 0) { + return; + } + char *obj_name = cbm_node_text(ctx->arena, obj, ctx->source); + if (!obj_name || strcmp(obj_name, self_name) != 0) { + return; + } + TSNode ann = ts_node_child_by_field_name(assign, TS_FIELD("type")); + if (!ts_node_is_null(ann)) { + py_push_field_type(ctx, class_qn, attr, cbm_node_text(ctx->arena, ann, ctx->source)); + return; + } + TSNode right = ts_node_child_by_field_name(assign, TS_FIELD("right")); + if (ts_node_is_null(right) || strcmp(ts_node_type(right), "identifier") != 0) { + return; + } + char *rhs = cbm_node_text(ctx->arena, right, ctx->source); + if (rhs && strcmp(rhs, self_name) != 0) { + py_push_field_type(ctx, class_qn, attr, py_init_param_annotation(ctx, params, rhs)); + } +} + +/* Every `self.x` field assignment in one __init__ body, nested blocks + * included; nested functions, lambdas and classes have their own `self`. */ +static void py_extract_init_fields(CBMExtractCtx *ctx, const char *class_qn, TSNode fn) { + TSNode params = ts_node_child_by_field_name(fn, TS_FIELD("parameters")); + TSNode body = ts_node_child_by_field_name(fn, TS_FIELD("body")); + if (ts_node_is_null(params) || ts_node_is_null(body) || + ts_node_named_child_count(params) == 0) { + return; + } + TSNode self_node = ts_node_named_child(params, 0); + if (strcmp(ts_node_type(self_node), "identifier") != 0) { + return; + } + char *self_name = cbm_node_text(ctx->arena, self_node, ctx->source); + if (!self_name || !self_name[0]) { + return; + } + TSNodeStack stack; + ts_nstack_init(&stack, ctx, CBM_SZ_32); + ts_nstack_push(&stack, body); + while (stack.count > 0) { + TSNode n = ts_nstack_pop(&stack); + const char *k = ts_node_type(n); + if (strcmp(k, "function_definition") == 0 || strcmp(k, "class_definition") == 0 || + strcmp(k, "lambda") == 0) { + continue; + } + TSNode assign = py_statement_assignment(n); + if (!ts_node_is_null(assign)) { + py_init_field_assignment(ctx, class_qn, assign, params, self_name); + continue; + } + /* Reverse push keeps source order on pop. */ + for (uint32_t i = ts_node_named_child_count(n); i > 0; i--) { + ts_nstack_push(&stack, ts_node_named_child(n, i - 1)); + } + } +} + +static void extract_py_field_types(CBMExtractCtx *ctx, TSNode class_node, const char *class_qn) { + TSNode body = ts_node_child_by_field_name(class_node, TS_FIELD("body")); + if (ts_node_is_null(body)) { + return; + } + uint32_t count = ts_node_named_child_count(body); + /* Class-body annotations first: `x: T` / `x: T = v`. */ + for (uint32_t i = 0; i < count; i++) { + TSNode assign = py_statement_assignment(ts_node_named_child(body, i)); + if (ts_node_is_null(assign)) { + continue; + } + TSNode left = ts_node_child_by_field_name(assign, TS_FIELD("left")); + TSNode ann = ts_node_child_by_field_name(assign, TS_FIELD("type")); + if (!ts_node_is_null(left) && !ts_node_is_null(ann) && + strcmp(ts_node_type(left), "identifier") == 0) { + py_push_field_type(ctx, class_qn, left, cbm_node_text(ctx->arena, ann, ctx->source)); + } + } + /* Then __init__. */ + for (uint32_t i = 0; i < count; i++) { + TSNode fn = ts_node_named_child(body, i); + if (strcmp(ts_node_type(fn), "decorated_definition") == 0) { + fn = ts_node_child_by_field_name(fn, TS_FIELD("definition")); + } + if (ts_node_is_null(fn) || strcmp(ts_node_type(fn), "function_definition") != 0) { + continue; + } + TSNode fname = ts_node_child_by_field_name(fn, TS_FIELD("name")); + char *fn_name = + ts_node_is_null(fname) ? NULL : cbm_node_text(ctx->arena, fname, ctx->source); + if (fn_name && strcmp(fn_name, "__init__") == 0) { + py_extract_init_fields(ctx, class_qn, fn); + } + } +} + static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec) { CBMArena *a = ctx->arena; const char *kind = ts_node_type(node); @@ -4970,6 +5133,10 @@ static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec // Extract class-level variables (field declarations) extract_class_variables(ctx, node, class_qn, spec); + if (ctx->language == CBM_LANG_PYTHON) { + extract_py_field_types(ctx, node, class_qn); + } + // C# 12 primary-constructor parameters: declared on the class line // (`class Foo(IBar bar, IBaz baz) : Base { ... }`) and bound to implicit // captured fields accessible from any instance member. Tree-sitter c-sharp diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index 7d416923c..71ec069c0 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -4,8 +4,10 @@ #include "lang_specs.h" // CBMLangSpec, CBMEmbeddedLangSpec, cbm_lang_spec, cbm_ts_language #include "tree_sitter/api.h" // TSNode, ts_node_* #include "foundation/constants.h" +#include "foundation/compat.h" /* CBM_TLS */ #include "extract_node_stack.h" #include // uint32_t +#include #include #include @@ -338,6 +340,490 @@ static void parse_python_imports(CBMExtractCtx *ctx) { ts_tree_cursor_delete(&cursor); } +/* Ordered Python namespace facts are a separate content-only pass. The older + * import records keep their existing graph/heuristic meaning. */ +typedef struct { + CBMExtractCtx *ctx; + CBMPyNamespaceFacts facts; + CBMPyNamespaceFailure failure; +} PyNamespaceBuilder; + +#ifdef CBM_ENABLE_TEST_SEAMS +static CBM_TLS int py_namespace_capture_left = -1; +void cbm_py_namespace_test_capture_fail_after(int successful_allocations) { + py_namespace_capture_left = successful_allocations; +} +#endif + +static void *py_ns_alloc(PyNamespaceBuilder *b, size_t size) { + if (b->failure) + return NULL; +#ifdef CBM_ENABLE_TEST_SEAMS + if (py_namespace_capture_left == 0) { + b->failure = CBM_PY_NS_FAILURE_ALLOCATION; + return NULL; + } + if (py_namespace_capture_left > 0) + py_namespace_capture_left--; +#endif + void *p = cbm_arena_alloc(b->ctx->arena, size); + if (!p) + b->failure = CBM_PY_NS_FAILURE_ALLOCATION; + return p; +} + +static const char *py_ns_copy(PyNamespaceBuilder *b, const char *text, size_t len) { + if (len == SIZE_MAX) { + b->failure = CBM_PY_NS_FAILURE_LIMIT; + return NULL; + } + char *copy = py_ns_alloc(b, len + 1); + if (!copy) + return NULL; + memcpy(copy, text, len); + copy[len] = '\0'; + return copy; +} + +static bool py_ns_slice(PyNamespaceBuilder *b, TSNode node, const char **text, size_t *len) { + if (ts_node_is_null(node)) + return false; + uint32_t start = ts_node_start_byte(node), end = ts_node_end_byte(node); + if (end < start || end > (uint32_t)b->ctx->source_len) + return false; + *text = b->ctx->source + start; + *len = (size_t)(end - start); + return memchr(*text, '\0', *len) == NULL; +} + +static bool py_ns_identifier_bytes(const char *s, size_t n, bool dotted) { + bool initial = true; + if (!n) + return false; + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)s[i]; + if (dotted && c == '.' && !initial) { + initial = true; + continue; + } + if (c == '_' || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || + (!initial && c >= '0' && c <= '9')) { + initial = false; + continue; + } + return false; + } + return !initial; +} + +static const char *py_ns_name(PyNamespaceBuilder *b, TSNode node, bool dotted) { + const char *text = NULL; + size_t len = 0; + if (!py_ns_slice(b, node, &text, &len) || !py_ns_identifier_bytes(text, len, dotted)) + return NULL; + return py_ns_copy(b, text, len); +} + +static bool py_ns_push(PyNamespaceBuilder *b, CBMPyNamespaceFact fact) { + if (b->failure) + return false; + if (b->facts.count == b->facts.cap) { + if (b->facts.cap > INT_MAX / 2) { + b->failure = CBM_PY_NS_FAILURE_LIMIT; + return false; + } + int cap = b->facts.cap ? b->facts.cap * 2 : 8; + if ((size_t)cap > SIZE_MAX / sizeof(*b->facts.items)) { + b->failure = CBM_PY_NS_FAILURE_LIMIT; + return false; + } + CBMPyNamespaceFact *items = py_ns_alloc(b, (size_t)cap * sizeof(*items)); + if (!items) + return false; + if (b->facts.count) + memcpy(items, b->facts.items, (size_t)b->facts.count * sizeof(*items)); + b->facts.items = items; + b->facts.cap = cap; + } + b->facts.items[b->facts.count++] = fact; + return true; +} + +static bool py_ns_unknown(PyNamespaceBuilder *b, CBMPyNamespaceUnknownReason reason) { + return py_ns_push(b, (CBMPyNamespaceFact){.kind = CBM_PY_NS_UNKNOWN, .reason = reason}); +} + +static bool py_ns_plain_string(PyNamespaceBuilder *b, TSNode node, const char **text, size_t *len) { + if (strcmp(ts_node_type(node), "string") != 0 || !py_ns_slice(b, node, text, len) || *len < 2) + return false; + char quote = (*text)[0]; + if ((quote != '\'' && quote != '"') || (*text)[*len - 1] != quote) + return false; + for (size_t i = 1; i + 1 < *len; i++) { + if ((*text)[i] == '\\' || (*text)[i] == quote || (*text)[i] == '\n' || (*text)[i] == '\r') + return false; + } + (*text)++; + *len -= 2; + return true; +} + +/* Bounded syntactic proof only. Unsupported or deep expressions become an + * unknown-effect barrier; no evaluator or source execution is involved. */ +static bool py_ns_safe_expr(PyNamespaceBuilder *b, TSNode node, int depth) { + if (ts_node_is_null(node)) + return true; + if (depth >= 64) + return false; + const char *kind = ts_node_type(node); + if (strcmp(kind, "identifier") == 0 || strcmp(kind, "integer") == 0 || + strcmp(kind, "float") == 0 || strcmp(kind, "true") == 0 || strcmp(kind, "false") == 0 || + strcmp(kind, "none") == 0 || strcmp(kind, "comment") == 0) + return true; + if (strcmp(kind, "string") == 0) { + const char *text = NULL; + size_t len = 0; + return py_ns_plain_string(b, node, &text, &len); + } + bool container = + strcmp(kind, "tuple") == 0 || strcmp(kind, "list") == 0 || + strcmp(kind, "parenthesized_expression") == 0 || strcmp(kind, "parameters") == 0 || + strcmp(kind, "default_parameter") == 0 || strcmp(kind, "typed_parameter") == 0 || + strcmp(kind, "typed_default_parameter") == 0 || strcmp(kind, "list_splat_pattern") == 0 || + strcmp(kind, "dictionary_splat_pattern") == 0 || strcmp(kind, "type") == 0; + if (!container) + return false; + TSTreeCursor cursor = ts_tree_cursor_new(node); + bool safe = true; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (ts_node_is_named(child) && !py_ns_safe_expr(b, child, depth + 1)) { + safe = false; + break; + } + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + return safe; +} + +static bool py_ns_signature_safe(PyNamespaceBuilder *b, TSNode node) { + return py_ns_safe_expr(b, ts_node_child_by_field_name(node, TS_FIELD("parameters")), 0) && + py_ns_safe_expr(b, ts_node_child_by_field_name(node, TS_FIELD("return_type")), 0); +} + +static bool py_ns_class_safe(PyNamespaceBuilder *b, TSNode node) { + if (!ts_node_is_null(ts_node_child_by_field_name(node, TS_FIELD("superclasses")))) + return false; + TSNode body = ts_node_child_by_field_name(node, TS_FIELD("body")); + if (ts_node_is_null(body)) + return false; + TSTreeCursor cursor = ts_tree_cursor_new(body); + bool safe = true; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (!ts_node_is_named(child)) + continue; + const char *kind = ts_node_type(child); + if (strcmp(kind, "pass_statement") == 0 || strcmp(kind, "comment") == 0) + continue; + if (strcmp(kind, "function_definition") == 0 && py_ns_signature_safe(b, child)) + continue; + if (strcmp(kind, "expression_statement") == 0 && + ts_node_named_child_count(child) == 1) { + TSNode expr = ts_node_named_child(child, 0); + if (strcmp(ts_node_type(expr), "string") == 0 && py_ns_safe_expr(b, expr, 0)) + continue; + /* Even an identifier assignment can install a descriptor whose + * __set_name__ hook executes during class creation. */ + } + safe = false; + break; + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + return safe; +} + +static bool py_ns_import(PyNamespaceBuilder *b, TSNode node, bool from) { + TSNode module = from ? ts_node_child_by_field_name(node, TS_FIELD("module_name")) : (TSNode){0}; + const char *module_name = NULL; + uint32_t level = 0; + if (from) { + const char *text = NULL; + size_t len = 0; + if (!py_ns_slice(b, module, &text, &len)) + return false; + while ((size_t)level < len && text[level] == '.') + level++; + if (level == len ? level == 0 : !py_ns_identifier_bytes(text + level, len - level, true)) + return false; + module_name = py_ns_copy(b, text + level, len - level); + if (!module_name) + return false; + } + TSTreeCursor cursor = ts_tree_cursor_new(node); + bool ok = true; + int emitted = 0; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (!ts_node_is_named(child) || (from && child.id == module.id) || + strcmp(ts_node_type(child), "comment") == 0) + continue; + const char *kind = ts_node_type(child); + CBMPyNamespaceFact f = {.kind = from ? CBM_PY_NS_IMPORT_NAME : CBM_PY_NS_IMPORT_MODULE, + .module_name = module_name, + .relative_level = level}; + if (from && strcmp(kind, "wildcard_import") == 0) { + f.kind = CBM_PY_NS_IMPORT_STAR; + } else { + bool alias = strcmp(kind, "aliased_import") == 0; + TSNode name = alias ? ts_node_child_by_field_name(child, TS_FIELD("name")) : child; + if (!alias && strcmp(kind, "dotted_name") != 0 && strcmp(kind, "identifier") != 0) { + ok = false; + break; + } + const char *imported = py_ns_name(b, name, !from); + if (!imported) { + ok = false; + break; + } + if (from) + f.member_name = imported; + else + f.module_name = imported; + if (alias) { + f.local_name = + py_ns_name(b, ts_node_child_by_field_name(child, TS_FIELD("alias")), false); + } else if (from) { + f.local_name = imported; + } else { + size_t n = strcspn(imported, "."); + f.local_name = py_ns_copy(b, imported, n); + f.flags = CBM_PY_NS_IMPORT_BINDS_ROOT; + } + if (!f.local_name) { + ok = false; + break; + } + } + if (!py_ns_push(b, f)) { + ok = false; + break; + } + emitted++; + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + return ok && emitted > 0; +} + +static bool py_ns_all_literal(PyNamespaceBuilder *b, TSNode node, CBMPyNamespaceFact *fact) { + if (ts_node_is_null(node)) + return false; + const char *kind = ts_node_type(node); + if (strcmp(kind, "list") != 0 && strcmp(kind, "tuple") != 0) + return false; + uint32_t count = ts_node_named_child_count(node); + if (count > INT_MAX || (size_t)count > SIZE_MAX / sizeof(*fact->names)) { + b->failure = CBM_PY_NS_FAILURE_LIMIT; + return false; + } + const char **names = count ? py_ns_alloc(b, (size_t)count * sizeof(*names)) : NULL; + if (count && !names) + return false; + int used = 0; + TSTreeCursor cursor = ts_tree_cursor_new(node); + bool ok = true; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (!ts_node_is_named(child) || strcmp(ts_node_type(child), "comment") == 0) + continue; + const char *text = NULL; + size_t len = 0; + if (!py_ns_plain_string(b, child, &text, &len) || + !(names[used] = py_ns_copy(b, text, len))) { + ok = false; + break; + } + used++; + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + if (!ok) + return false; + fact->sequence_kind = + strcmp(kind, "list") == 0 ? CBM_PY_NS_SEQUENCE_LIST : CBM_PY_NS_SEQUENCE_TUPLE; + fact->names = used ? names : NULL; + fact->name_count = used; + return true; +} + +static bool py_ns_assignment(PyNamespaceBuilder *b, TSNode node) { + TSNode left = ts_node_child_by_field_name(node, TS_FIELD("left")); + TSNode right = ts_node_child_by_field_name(node, TS_FIELD("right")); + TSNode annotation = ts_node_child_by_field_name(node, TS_FIELD("type")); + const char *name = py_ns_name(b, left, false); + if (!name) + return false; + if (!py_ns_safe_expr(b, right, 0) || !py_ns_safe_expr(b, annotation, 0)) { + if (!py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT)) + return false; + } + if (ts_node_is_null(right)) + return true; /* annotation alone does not bind the target */ + bool augmented = strcmp(ts_node_type(node), "augmented_assignment") == 0; + if (strcmp(name, "__all__") != 0) { + if (augmented) + return py_ns_unknown(b, CBM_PY_NS_REASON_UNSUPPORTED); + return py_ns_push(b, (CBMPyNamespaceFact){.kind = CBM_PY_NS_SHADOW, + .local_name = name, + .reason = CBM_PY_NS_REASON_VALUE}); + } + CBMPyNamespaceFact fact = {.kind = augmented ? CBM_PY_NS_ALL_APPEND : CBM_PY_NS_ALL_SET}; + bool plus = true; + if (augmented) { + const char *op = NULL; + size_t len = 0; + plus = py_ns_slice(b, ts_node_child_by_field_name(node, TS_FIELD("operator")), &op, &len) && + len == 2 && memcmp(op, "+=", 2) == 0; + } + if (plus && py_ns_all_literal(b, right, &fact)) + return py_ns_push(b, fact); + return py_ns_push( + b, (CBMPyNamespaceFact){.kind = CBM_PY_NS_ALL_UNKNOWN, .reason = CBM_PY_NS_REASON_VALUE}); +} + +static bool py_ns_delete(PyNamespaceBuilder *b, TSNode node, int depth) { + if (depth > 2) + return false; + if (strcmp(ts_node_type(node), "identifier") == 0) { + const char *name = py_ns_name(b, node, false); + if (!name) + return false; + return py_ns_push(b, + strcmp(name, "__all__") == 0 + ? (CBMPyNamespaceFact){.kind = CBM_PY_NS_ALL_DELETE} + : (CBMPyNamespaceFact){.kind = CBM_PY_NS_DELETE, .local_name = name}); + } + const char *kind = ts_node_type(node); + if (strcmp(kind, "delete_statement") != 0 && strcmp(kind, "expression_list") != 0) + return false; + TSTreeCursor cursor = ts_tree_cursor_new(node); + bool ok = true; + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode child = ts_tree_cursor_current_node(&cursor); + if (ts_node_is_named(child) && strcmp(ts_node_type(child), "comment") != 0 && + !py_ns_delete(b, child, depth + 1)) { + ok = false; + break; + } + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + return ok; +} + +static bool py_ns_statement(PyNamespaceBuilder *b, TSNode node) { + const char *kind = ts_node_type(node); + if (strcmp(kind, "comment") == 0 || strcmp(kind, "pass_statement") == 0) + return true; + if (strcmp(kind, "import_statement") == 0) + return py_ns_import(b, node, false); + if (strcmp(kind, "import_from_statement") == 0) + return py_ns_import(b, node, true); + bool decorated = strcmp(kind, "decorated_definition") == 0; + TSNode def = decorated ? ts_node_child_by_field_name(node, TS_FIELD("definition")) : node; + if (!ts_node_is_null(def)) { + const char *dk = ts_node_type(def); + bool function = strcmp(dk, "function_definition") == 0; + bool class_def = strcmp(dk, "class_definition") == 0; + if (function || class_def) { + const char *name = + py_ns_name(b, ts_node_child_by_field_name(def, TS_FIELD("name")), false); + if (!name) + return false; + bool proven = !decorated && (function || py_ns_class_safe(b, def)); + if (!proven || (function && !py_ns_signature_safe(b, def))) { + if (!py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT)) + return false; + } + return py_ns_push( + b, proven ? (CBMPyNamespaceFact){.kind = CBM_PY_NS_OWN_DEF, + .local_name = name, + .def_kind = function ? CBM_PY_NS_DEF_FUNCTION + : CBM_PY_NS_DEF_CLASS} + : (CBMPyNamespaceFact){.kind = CBM_PY_NS_SHADOW, + .local_name = name, + .reason = decorated + ? CBM_PY_NS_REASON_DECORATED + : CBM_PY_NS_REASON_UNPROVEN_CLASS}); + } + } + if (strcmp(kind, "delete_statement") == 0) + return py_ns_delete(b, node, 0); + if (strcmp(kind, "expression_statement") == 0 && ts_node_named_child_count(node) == 1) { + TSNode expr = ts_node_named_child(node, 0); + const char *ek = ts_node_type(expr); + if (strcmp(ek, "assignment") == 0 || strcmp(ek, "augmented_assignment") == 0) + return py_ns_assignment(b, expr); + return py_ns_safe_expr(b, expr, 0) || py_ns_unknown(b, CBM_PY_NS_REASON_EFFECT); + } + if (strcmp(kind, "if_statement") == 0 || strcmp(kind, "try_statement") == 0 || + strcmp(kind, "for_statement") == 0 || strcmp(kind, "while_statement") == 0 || + strcmp(kind, "with_statement") == 0 || strcmp(kind, "match_statement") == 0) + return py_ns_unknown(b, CBM_PY_NS_REASON_COMPOUND); + return false; +} + +CBMPyNamespaceStatus cbm_extract_python_namespace_facts(CBMExtractCtx *ctx) { + if (!ctx || !ctx->result) + return CBM_PY_NS_INCOMPLETE; + PyNamespaceBuilder b = {.ctx = ctx, + .facts = {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = ctx->language, + .status = CBM_PY_NS_COMPLETE}}; + if (ctx->language != CBM_LANG_PYTHON) { + b.facts.status = CBM_PY_NS_NOT_APPLICABLE; + } else if (!ctx->arena || !ctx->source || ctx->source_len < 0) { + b.failure = CBM_PY_NS_FAILURE_INVALID_INPUT; + } else if (ts_node_is_null(ctx->root) || ts_node_has_error(ctx->root)) { + py_ns_unknown(&b, CBM_PY_NS_REASON_PARSE); + } else { + TSTreeCursor cursor = ts_tree_cursor_new(ctx->root); + if (ts_tree_cursor_goto_first_child(&cursor)) { + do { + TSNode node = ts_tree_cursor_current_node(&cursor); + if (!ts_node_is_named(node)) + continue; + int start = b.facts.count; + if (!py_ns_statement(&b, node) && !b.failure) { + b.facts.count = start; + py_ns_unknown(&b, CBM_PY_NS_REASON_UNSUPPORTED); + } + if (b.failure) + break; + } while (ts_tree_cursor_goto_next_sibling(&cursor)); + } + ts_tree_cursor_delete(&cursor); + } + if (!b.failure && !cbm_py_namespace_facts_valid(&b.facts)) + b.failure = CBM_PY_NS_FAILURE_INVALID_INPUT; + if (b.failure) { + b.facts = (CBMPyNamespaceFacts){.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = ctx->language, + .status = CBM_PY_NS_INCOMPLETE, + .failure = b.failure}; + } + ctx->result->py_namespace = b.facts; + return b.facts.status; +} + // --- ES module imports (JS/TS/TSX) --- // import X from "Y"; import {A, B} from "Y"; import * as X from "Y" // const X = require("Y") diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c index 4c4d340f0..4ab8ba48e 100644 --- a/internal/cbm/helpers.c +++ b/internal/cbm/helpers.c @@ -1611,6 +1611,17 @@ char *cbm_fqn_compute(CBMArena *a, const char *project, const char *rel_path, co return buf; } +size_t cbm_fqn_symbol_scope_len(const char *module_qn) { + if (!module_qn) + return 0; + size_t len = strlen(module_qn); + static const char suffix[] = ".__init__"; + size_t suffix_len = sizeof(suffix) - 1; + if (len > suffix_len && memcmp(module_qn + len - suffix_len, suffix, suffix_len) == 0) + return len - suffix_len; + return len; +} + char *cbm_fqn_module(CBMArena *a, const char *project, const char *rel_path) { return cbm_fqn_compute(a, project, rel_path, NULL); } diff --git a/internal/cbm/lsp/c_lsp.c b/internal/cbm/lsp/c_lsp.c index 74b97d69d..350670761 100644 --- a/internal/cbm/lsp/c_lsp.c +++ b/internal/cbm/lsp/c_lsp.c @@ -1,10 +1,12 @@ #include "c_lsp.h" #include "lsp_node_iter.h" +#include "lsp_work.h" #include "../helpers.h" #include #include #include #include +#include /* Safe kind accessor — returns CBM_TYPE_UNKNOWN for NULL types. * Prevents SEGV in c_eval_expr_type_inner on unusual C++ AST shapes. */ @@ -12,6 +14,47 @@ static inline CBMTypeKind safe_kind(const CBMType *t) { return t ? t->kind : CBM_TYPE_UNKNOWN; } +#ifdef CBM_ENABLE_TEST_SEAMS +static _Thread_local int c_test_memo_allocations_left = -1; +static _Thread_local bool c_test_depth_failure; +void cbm_c_lsp_test_depth_fail(bool enabled) { + c_test_depth_failure = enabled; +} +static _Thread_local uint32_t c_test_walks; +uint32_t cbm_c_lsp_test_walks_take(void) { + uint32_t walks = c_test_walks; + c_test_walks = 0; + return walks; +} +void cbm_c_lsp_test_memo_fail_after(int successful_allocations) { + c_test_memo_allocations_left = successful_allocations; +} +#endif + +static void c_memo_test_stage(CLSPContext *ctx, const char *stage) { +#ifdef CBM_ENABLE_TEST_SEAMS + c_test_walks++; + const char *depth_stage = getenv("CBM_TEST_C_LSP_DEPTH_FAIL_STAGE"); + if (depth_stage && strcmp(depth_stage, stage) == 0) + ctx->eval_depth = 257; + const char *fail_stage = getenv("CBM_TEST_C_LSP_MEMO_FAIL_STAGE"); + if (fail_stage && strcmp(fail_stage, stage) == 0) + ctx->test_memo_allocations_left = 0; +#else + (void)ctx; + (void)stage; +#endif +} + +void cbm_c_lsp_record_failure(CBMFileResult *result, CBMLSPStatus status) { + result->has_error = true; + result->lsp_skipped = true; + /* Static lifetime keeps the reason available even when the arena is full. + * Result compaction/spilling copies strings, including non-arena strings. */ + result->error_msg = + status == CBM_LSP_DEPTH_EXCEEDED ? CBM_C_LSP_DEPTH_ERROR : CBM_C_LSP_MEMO_ERROR; +} + // Forward declarations static void c_resolve_calls_in_node(CLSPContext *ctx, TSNode node); static void c_emit_resolved_call(CLSPContext *ctx, const char *callee_qn, const char *strategy, @@ -59,6 +102,11 @@ void c_lsp_init(CLSPContext *ctx, CBMArena *arena, const char *source, int sourc const CBMTypeRegistry *registry, const char *module_qn, bool cpp_mode, CBMResolvedCallArray *out) { memset(ctx, 0, sizeof(CLSPContext)); +#ifdef CBM_ENABLE_TEST_SEAMS + ctx->test_memo_allocations_left = c_test_memo_allocations_left; + if (c_test_depth_failure) + ctx->eval_depth = 257; +#endif ctx->arena = arena; ctx->source = source; ctx->source_len = source_len; @@ -191,14 +239,7 @@ static bool c_scope_declares_name(const CBMScope *scope, const char *var_name) { if (!scope || !var_name) { return false; } - for (const CBMScopeChunk *chunk = scope->chunks; chunk; chunk = chunk->next) { - for (int i = 0; i < chunk->used; i++) { - if (chunk->bindings[i].name && strcmp(chunk->bindings[i].name, var_name) == 0) { - return true; - } - } - } - return false; + return cbm_scope_lookup_local(scope, var_name) != NULL; } /* Return the exact fp-state entry for the nearest lexical declaration. An @@ -1474,26 +1515,129 @@ static const char *type_to_qn(const CBMType *t) { static const CBMType *c_eval_expr_type_inner(CLSPContext *ctx, TSNode node); +/* Stack guard: the evaluator recurses once per expression nesting level. + * There is deliberately NO per-file work budget (#1527): the 10000-step cap + * that used to sit here (#323) left every later expression in a large file + * unresolved while the file still reported status `indexed`. What it was + * containing -- a fluent chain re-evaluating its receiver at every link, 2^N + * evaluations (the Bitcoin versionbits hang) -- is removed at the root by the + * per-evaluation memo below. */ #define C_EVAL_DEPTH_LIMIT 256 -#define C_EVAL_MAX_STEPS_PER_FILE 10000 +enum { C_EVAL_MEMO_INITIAL_CAP = 64 }; + +static uint32_t c_eval_memo_hash(const void *id) { + uintptr_t v = (uintptr_t)id >> 3; /* node structs are >=8-byte aligned */ + return (uint32_t)((v ^ (v >> 29)) * 2654435761u); +} + +/* Empty the memo for a new outermost evaluation: O(slots the last one used). */ +static void c_eval_memo_reset(CLSPContext *ctx) { + for (int i = 0; i < ctx->eval_memo_count; i++) { + ctx->eval_memo[ctx->eval_memo_used[i]].node_id = NULL; + } + ctx->eval_memo_count = 0; +} + +static const CBMType *c_eval_memo_lookup(const CLSPContext *ctx, const void *id) { + if (ctx->eval_memo_cap == 0) { + return NULL; + } + uint32_t mask = (uint32_t)ctx->eval_memo_cap - 1; + for (uint32_t i = c_eval_memo_hash(id) & mask;; i = (i + 1) & mask) { + const struct CLSPEvalMemoSlot *e = &ctx->eval_memo[i]; + if (!e->node_id) { + return NULL; + } + if (e->node_id == id) { + return e->result; + } + } +} + +static void c_eval_memo_place(CLSPContext *ctx, const void *id, const CBMType *result) { + uint32_t mask = (uint32_t)ctx->eval_memo_cap - 1; + uint32_t i = c_eval_memo_hash(id) & mask; + while (ctx->eval_memo[i].node_id) { + i = (i + 1) & mask; + } + ctx->eval_memo[i].node_id = id; + ctx->eval_memo[i].result = result; + ctx->eval_memo_used[ctx->eval_memo_count++] = (int)i; +} + +static void *c_eval_memo_alloc(CLSPContext *ctx, size_t size) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (ctx->test_memo_allocations_left == 0) + return NULL; + if (ctx->test_memo_allocations_left > 0) + ctx->test_memo_allocations_left--; +#endif + return cbm_arena_alloc(ctx->arena, size); +} + +/* Memoization is required to bound receiver re-evaluation. An allocation + * failure stops this file's refinement and is reported to the caller. */ +static bool c_eval_memo_insert(CLSPContext *ctx, const void *id, const CBMType *result) { + if (ctx->eval_memo_count >= ctx->eval_memo_cap / 2) { + if (ctx->eval_memo_cap > INT_MAX / 2) + return false; + int new_cap = ctx->eval_memo_cap ? ctx->eval_memo_cap * 2 : C_EVAL_MEMO_INITIAL_CAP; + if ((size_t)new_cap > SIZE_MAX / sizeof(struct CLSPEvalMemoSlot) || + (size_t)(new_cap / 2) > SIZE_MAX / sizeof(int)) + return false; + struct CLSPEvalMemoSlot *slots = c_eval_memo_alloc(ctx, (size_t)new_cap * sizeof(*slots)); + if (!slots) + return false; + int *used = c_eval_memo_alloc(ctx, (size_t)(new_cap / 2) * sizeof(int)); + if (!used) + return false; + memset(slots, 0, (size_t)new_cap * sizeof(*slots)); + struct CLSPEvalMemoSlot *old = ctx->eval_memo; + const int *old_used = ctx->eval_memo_used; + int old_count = ctx->eval_memo_count; + ctx->eval_memo = slots; + ctx->eval_memo_used = used; + ctx->eval_memo_cap = new_cap; + ctx->eval_memo_count = 0; + for (int k = 0; k < old_count; k++) { + const struct CLSPEvalMemoSlot *e = &old[old_used[k]]; + c_eval_memo_place(ctx, e->node_id, e->result); + } + } + c_eval_memo_place(ctx, id, result); + return true; +} const CBMType *c_eval_expr_type(CLSPContext *ctx, TSNode node) { - if (ts_node_is_null(node)) + if (ctx->eval_failure || ts_node_is_null(node)) return cbm_type_unknown(); - /* Expression type evaluation is best-effort. Some recovery-mode C++ ASTs - * can repeatedly drive member/type lookup without increasing recursion - * depth. Keep a generous per-file work budget so pathological expressions - * degrade to unknown instead of hanging repository indexing. */ - if (ctx->eval_depth > C_EVAL_DEPTH_LIMIT || ctx->eval_steps++ > C_EVAL_MAX_STEPS_PER_FILE) { - if (ctx->debug && ctx->eval_steps == C_EVAL_MAX_STEPS_PER_FILE + 2) { - fprintf(stderr, " [clsp] expression eval step budget exhausted; returning unknown\n"); - } + if (ctx->eval_depth > C_EVAL_DEPTH_LIMIT) { + ctx->eval_truncations++; + ctx->eval_failure = CBM_LSP_DEPTH_EXCEEDED; return cbm_type_unknown(); } + if (ctx->eval_depth == 0) { + c_eval_memo_reset(ctx); /* scope may have changed since the last evaluation */ + } + const CBMType *cached = c_eval_memo_lookup(ctx, node.id); + if (cached) { + return cached; + } + CBM_LSP_WORK(1); /* one real evaluation (complexity tests) */ + uint32_t trunc_before = ctx->eval_truncations; ctx->eval_depth++; const CBMType *result = c_eval_expr_type_inner(ctx, node); ctx->eval_depth--; - return result ? result : cbm_type_unknown(); + if (!result) { + result = cbm_type_unknown(); + } + /* Never memoize a result a depth cutoff may have truncated: the incomplete walk + * must stop before a second receiver branch repeats the truncated work. */ + if (!ctx->eval_failure && ctx->eval_truncations == trunc_before && + !c_eval_memo_insert(ctx, node.id, result)) { + ctx->eval_failure = CBM_LSP_MEMO_FAILED; + } + return ctx->eval_failure ? cbm_type_unknown() : result; } static const CBMType *c_eval_expr_type_inner(CLSPContext *ctx, TSNode node) { @@ -2963,6 +3107,8 @@ static const CBMType *c_parse_declaration_type(CLSPContext *ctx, TSNode decl_nod } void c_process_statement(CLSPContext *ctx, TSNode node) { + if (ctx->eval_failure) + return; if (ts_node_is_null(node)) return; const char *kind = ts_node_type(node); @@ -3930,6 +4076,8 @@ static void c_resolve_calls_in_node_inner(CLSPContext *ctx, TSNode node); * frames via c_adl_resolve). Past the cap the subtree is skipped — its calls * stay unresolved, which is graceful degradation, not a crash. */ static void c_resolve_calls_in_node(CLSPContext *ctx, TSNode node) { + if (ctx->eval_failure) + return; if (ctx->walk_depth >= C_LSP_MAX_WALK_DEPTH) return; ctx->walk_depth++; @@ -4783,6 +4931,8 @@ recurse:; // ============================================================================ static void c_process_function(CLSPContext *ctx, TSNode func_node) { + if (ctx->eval_failure) + return; TSNode decl = ts_node_child_by_field_name(func_node, "declarator", 10); if (ts_node_is_null(decl)) return; @@ -4973,6 +5123,8 @@ static void c_process_function(CLSPContext *ctx, TSNode func_node) { // Process a top-level or nested declaration within a namespace/class body, // handling template_declaration wrapping. static void c_process_body_child(CLSPContext *ctx, TSNode child) { + if (ctx->eval_failure) + return; if (ts_node_is_null(child)) return; const char *ck = ts_node_type(child); @@ -5429,6 +5581,8 @@ void c_lsp_process_file(CLSPContext *ctx, TSNode root) { if (ts_node_is_null(root)) return; + int resolved_before = ctx->resolved_calls ? ctx->resolved_calls->count : 0; + // Collect top-level children once (O(n)); both passes reuse the array. // Indexing ts_node_child(root,i) per iteration is O(i) → O(n²) on a wide root. uint32_t kn = 0; @@ -5437,7 +5591,7 @@ void c_lsp_process_file(CLSPContext *ctx, TSNode root) { TSNode inner; // Pass 1: process using declarations and global variables - for (uint32_t i = 0; i < kn; i++) { + for (uint32_t i = 0; i < kn && !ctx->eval_failure; i++) { child = kids[i]; const char *ck = ts_node_type(child); @@ -5461,7 +5615,7 @@ void c_lsp_process_file(CLSPContext *ctx, TSNode root) { } // Pass 2: process functions, namespaces, classes, templates - for (uint32_t i = 0; i < kn; i++) { + for (uint32_t i = 0; i < kn && !ctx->eval_failure; i++) { child = kids[i]; c_process_body_child(ctx, child); } @@ -5469,6 +5623,8 @@ void c_lsp_process_file(CLSPContext *ctx, TSNode root) { // Release the per-file negative-lookup memo (malloc-owned; only allocated in // shared-registry mode). No-op when it was never populated. c_neg_memo_free(ctx); + if (ctx->eval_failure && ctx->resolved_calls) + ctx->resolved_calls->count = resolved_before; } // ============================================================================ @@ -5682,8 +5838,116 @@ static void c_rewrite_proved_destructor_candidates(CBMFileResult *result) { } } +/* QN -> index of its FIRST registration in reg->types, for the single-file + * registration loop below (#1527). That loop asked "is this type registered + * yet, and where?" by scanning every registered type per Field and per + * Method -- O(defs x types) before the registry was indexed. Tracking the + * first index as types are added answers the same question in O(1); the + * types array may be reallocated as it grows, so indices, not pointers. */ +typedef struct { + const char **qn; /* NULL = empty slot */ + int *first; + int cap; /* power of two, or 0 when unavailable */ + int count; +} CTypeFirstIndex; + +static uint32_t c_type_first_hash(const char *s) { + uint32_t h = 2166136261u; + for (const unsigned char *p = (const unsigned char *)s; *p; p++) { + h ^= *p; + h *= 16777619u; + } + return h; +} + +/* Slot holding qn, or the empty slot where it belongs. */ +static int c_type_first_slot(const CTypeFirstIndex *ix, const char *qn) { + uint32_t mask = (uint32_t)ix->cap - 1; + uint32_t i = c_type_first_hash(qn) & mask; + while (ix->qn[i] && strcmp(ix->qn[i], qn) != 0) { + CBM_LSP_WORK(1); + i = (i + 1) & mask; + } + return (int)i; +} + +static bool c_type_first_alloc(CTypeFirstIndex *ix, CBMArena *arena, int cap) { + const char **qn = (const char **)cbm_arena_alloc(arena, (size_t)cap * sizeof(*qn)); + int *first = (int *)cbm_arena_alloc(arena, (size_t)cap * sizeof(*first)); + if (!qn || !first) { + return false; + } + memset(qn, 0, (size_t)cap * sizeof(*qn)); + ix->qn = qn; + ix->first = first; + ix->cap = cap; + ix->count = 0; + return true; +} + +/* Record reg->types[idx]; an existing QN keeps its first index. On OOM the + * index turns itself off (cap 0) and callers fall back to the scan. */ +static void c_type_first_note(CTypeFirstIndex *ix, CBMArena *arena, const CBMTypeRegistry *reg, + int idx) { + const char *qn = reg->types[idx].qualified_name; + if (ix->cap == 0 || !qn) { + return; + } + if ((ix->count + 1) * 2 > ix->cap) { + CTypeFirstIndex grown = {0}; + if (!c_type_first_alloc(&grown, arena, ix->cap * 2)) { + ix->cap = 0; + return; + } + for (int i = 0; i < ix->cap; i++) { + if (ix->qn[i]) { + int s = c_type_first_slot(&grown, ix->qn[i]); + grown.qn[s] = ix->qn[i]; + grown.first[s] = ix->first[i]; + grown.count++; + } + } + *ix = grown; + } + int s = c_type_first_slot(ix, qn); + if (!ix->qn[s]) { + ix->qn[s] = qn; + ix->first[s] = idx; + ix->count++; + } +} + +/* Index of the first registered type named qn, or -1. Same answer as the + * linear scan it replaces (which also serves as the OOM fallback). */ +static int c_type_first_find(const CTypeFirstIndex *ix, const CBMTypeRegistry *reg, + const char *qn) { + if (ix->cap > 0) { + int s = c_type_first_slot(ix, qn); + return ix->qn[s] ? ix->first[s] : -1; + } + for (int ri = 0; ri < reg->type_count; ri++) { + CBM_LSP_WORK(1); + if (strcmp(reg->types[ri].qualified_name, qn) == 0) { + return ri; + } + } + return -1; +} + +/* cbm_registry_add_type plus the first-index bookkeeping. */ +static void c_add_type_indexed(CBMTypeRegistry *reg, CTypeFirstIndex *ix, CBMArena *arena, + CBMRegisteredType rt) { + int before = reg->type_count; + cbm_registry_add_type(reg, rt); + if (reg->type_count > before) { + c_type_first_note(ix, arena, reg, before); + } +} + void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, int source_len, TSNode root, bool cpp_mode, CBMSourceOrigin source_origin) { + if (result->lsp_skipped) + return; CBMTypeRegistry reg; cbm_registry_init(®, arena); @@ -5695,6 +5959,18 @@ void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, i const char *module_qn = result->module_qn; + /* Scratch for the registration index and the registry's hash index; both + * die with this call (quadratic-pattern #6: never per-file index memory in + * a longer-lived arena). */ + CBMArena idx_arena; + cbm_arena_init(&idx_arena); + CTypeFirstIndex first_ix = {0}; + if (c_type_first_alloc(&first_ix, &idx_arena, 256)) { + for (int ti = 0; ti < reg.type_count; ti++) { + c_type_first_note(&first_ix, &idx_arena, ®, ti); + } + } + // Register file's own definitions for (int i = 0; i < result->defs.count; i++) { CBMDefinition *d = &result->defs.items[i]; @@ -5728,19 +6004,14 @@ void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, i rt.embedded_types = embedded; } } - cbm_registry_add_type(®, rt); + c_add_type_indexed(®, &first_ix, &idx_arena, rt); } // Field definitions → populate field_names/field_types on parent type if (d->label && strcmp(d->label, "Field") == 0 && d->parent_class && d->return_type) { // Find or add the parent type in registry - CBMRegisteredType *parent_rt = NULL; - for (int ri = 0; ri < reg.type_count; ri++) { - if (strcmp(reg.types[ri].qualified_name, d->parent_class) == 0) { - parent_rt = ®.types[ri]; - break; - } - } + int parent_idx = c_type_first_find(&first_ix, ®, d->parent_class); + CBMRegisteredType *parent_rt = parent_idx >= 0 ? ®.types[parent_idx] : NULL; if (parent_rt) { // Count existing fields int existing = 0; @@ -5807,14 +6078,14 @@ void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, i if (strcmp(d->label, "Method") == 0 && d->parent_class) { rf.receiver_type = d->parent_class; // Auto-create type if needed - if (!cbm_registry_lookup_type(®, d->parent_class)) { + if (c_type_first_find(&first_ix, ®, d->parent_class) < 0) { CBMRegisteredType auto_type; memset(&auto_type, 0, sizeof(auto_type)); auto_type.qualified_name = d->parent_class; // Extract short name const char *dot = strrchr(d->parent_class, '.'); auto_type.short_name = dot ? dot + 1 : d->parent_class; - cbm_registry_add_type(®, auto_type); + c_add_type_indexed(®, &first_ix, &idx_arena, auto_type); } } @@ -5822,13 +6093,27 @@ void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, i } } + /* Hash-index the registry before the walk (#1527): unindexed, every + * resolver lookup scanned the stdlib plus all of this file's definitions, + * O(calls x defs) per file. First-registered chains keep each duplicated + * name resolving to the entry the scan returned; resolver additions after + * this point stay visible through the registry's tail scan. */ + reg.index_first_registered = true; + cbm_registry_finalize_into(®, &idx_arena); + // Initialize context and run CLSPContext ctx; c_lsp_init(&ctx, arena, source, source_len, ®, module_qn, cpp_mode, &result->resolved_calls); ctx.source_origin = source_origin; + c_memo_test_stage(&ctx, + source_origin == CBM_SOURCE_ORIGIN_PREPROCESSED ? "preprocessed" : "raw"); c_lsp_process_file(&ctx, root); - c_rewrite_proved_destructor_candidates(result); + if (ctx.eval_failure) + cbm_c_lsp_record_failure(result, ctx.eval_failure); + else + c_rewrite_proved_destructor_candidates(result); + cbm_arena_destroy(&idx_arena); } // ============================================================================ @@ -6000,13 +6285,22 @@ CBMTypeRegistry *cbm_c_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, in return reg; } -void cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, +bool cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, CBMTypeRegistry *reg, const char **include_paths, const char **include_ns_qns, int include_count, TSTree *cached_tree, CBMResolvedCallArray *out) { + return cbm_run_c_lsp_cross_with_registry_status( + arena, source, source_len, module_qn, cpp_mode, reg, include_paths, include_ns_qns, + include_count, cached_tree, out) == CBM_LSP_COMPLETE; +} + +CBMLSPStatus cbm_run_c_lsp_cross_with_registry_status( + CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, + CBMTypeRegistry *reg, const char **include_paths, const char **include_ns_qns, + int include_count, TSTree *cached_tree, CBMResolvedCallArray *out) { if (!source || source_len == 0 || !out || !reg) - return; + return CBM_LSP_COMPLETE; TSParser *parser = NULL; TSTree *tree = cached_tree; @@ -6014,14 +6308,14 @@ void cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int if (!tree) { parser = ts_parser_new(); if (!parser) - return; + return CBM_LSP_COMPLETE; const TSLanguage *ts_lang = cpp_mode ? tree_sitter_cpp() : tree_sitter_c(); ts_parser_set_language(parser, ts_lang); tree = ts_parser_parse_string(parser, NULL, source, source_len); ts_parser_delete(parser); owns_tree = true; if (!tree) - return; + return CBM_LSP_COMPLETE; } TSNode root = ts_tree_root_node(tree); @@ -6031,20 +6325,32 @@ void cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int for (int i = 0; i < include_count; i++) { c_lsp_add_include(&ctx, include_paths[i], include_ns_qns[i]); } + c_memo_test_stage(&ctx, "cross"); c_lsp_process_file(&ctx, root); if (owns_tree) { ts_tree_delete(tree); } + return ctx.eval_failure; } -void cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, +bool cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, CBMLSPDef *defs, int def_count, const char **include_paths, const char **include_ns_qns, int include_count, TSTree *cached_tree, CBMResolvedCallArray *out) { + return cbm_run_c_lsp_cross_status(arena, source, source_len, module_qn, cpp_mode, defs, + def_count, include_paths, include_ns_qns, include_count, + cached_tree, out) == CBM_LSP_COMPLETE; +} + +CBMLSPStatus cbm_run_c_lsp_cross_status(CBMArena *arena, const char *source, int source_len, + const char *module_qn, bool cpp_mode, CBMLSPDef *defs, + int def_count, const char **include_paths, + const char **include_ns_qns, int include_count, + TSTree *cached_tree, CBMResolvedCallArray *out) { if (!source || source_len == 0 || !out) - return; + return CBM_LSP_COMPLETE; CBMTypeRegistry reg; cbm_registry_init(®, arena); @@ -6064,14 +6370,14 @@ void cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, co if (!tree) { parser = ts_parser_new(); if (!parser) - return; + return CBM_LSP_COMPLETE; const TSLanguage *ts_lang = cpp_mode ? tree_sitter_cpp() : tree_sitter_c(); ts_parser_set_language(parser, ts_lang); tree = ts_parser_parse_string(parser, NULL, source, source_len); ts_parser_delete(parser); owns_tree = true; if (!tree) - return; + return CBM_LSP_COMPLETE; } TSNode root = ts_tree_root_node(tree); @@ -6090,20 +6396,23 @@ void cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, co c_lsp_add_include(&ctx, include_paths[i], include_ns_qns[i]); } + c_memo_test_stage(&ctx, "cross"); c_lsp_process_file(&ctx, root); if (owns_tree) { ts_tree_delete(tree); } + return ctx.eval_failure; } // --- Batch cross-file LSP --- -void cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_count, +bool cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_count, CBMResolvedCallArray *out) { if (!files || file_count <= 0 || !out) - return; + return true; + bool complete = true; for (int f = 0; f < file_count; f++) { CBMBatchCLSPFile *file = &files[f]; memset(&out[f], 0, sizeof(CBMResolvedCallArray)); @@ -6119,10 +6428,12 @@ void cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_co memset(&file_out, 0, sizeof(file_out)); // Delegate to existing per-file function - cbm_run_c_lsp_cross(&file_arena, file->source, file->source_len, file->module_qn, - file->cpp_mode, file->defs, file->def_count, file->include_paths, - file->include_ns_qns, file->include_count, file->cached_tree, - &file_out); + bool file_complete = cbm_run_c_lsp_cross( + &file_arena, file->source, file->source_len, file->module_qn, file->cpp_mode, + file->defs, file->def_count, file->include_paths, file->include_ns_qns, + file->include_count, file->cached_tree, &file_out); + + complete = complete && file_complete; // Copy results to output arena (must outlive per-file arena) if (file_out.count > 0) { @@ -6147,4 +6458,5 @@ void cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_co cbm_arena_destroy(&file_arena); } + return complete; } diff --git a/internal/cbm/lsp/c_lsp.h b/internal/cbm/lsp/c_lsp.h index af81d5bbd..bbbf23655 100644 --- a/internal/cbm/lsp/c_lsp.h +++ b/internal/cbm/lsp/c_lsp.h @@ -99,12 +99,33 @@ typedef struct { // READ-ONLY across resolve workers — never mutate it // (and never store per-worker arena pointers into it) bool debug; - int eval_depth; // recursion depth for c_eval_expr_type (crash guard) - int eval_steps; // total expression eval calls for current file (hang guard) + int eval_depth; // recursion depth for c_eval_expr_type (crash guard) + uint32_t eval_truncations; // depth-cap cutoffs; a truncated result is never memoized + // Per-evaluation memo (#1527): results of the sub-expressions evaluated + // during ONE outermost c_eval_expr_type call, keyed by node identity. The + // evaluator re-evaluates a call's receiver (member lookup, then template + // substitution), which made an N-link fluent chain cost 2^N; the per-file + // step cap that used to hide that is gone. Evaluation never mutates scope + // or context, so within one outermost call a node's type cannot change; + // the memo is emptied when the next outermost call starts, because the + // resolver walk mutates scope between calls. + struct CLSPEvalMemoSlot *eval_memo; // open addressing, arena-owned + int *eval_memo_used; // slots filled by the current evaluation + int eval_memo_cap; // power of two, or 0 + int eval_memo_count; + CBMLSPStatus eval_failure; // sticky: stop refinement rather than repeat uncached work +#ifdef CBM_ENABLE_TEST_SEAMS + int test_memo_allocations_left; // -1 = normal; 0 = fail the next memo allocation +#endif int walk_depth; // c_resolve_calls_in_node self-recursion (AST nesting) int control_flow_depth; // if/loop/switch/catch nesting; assignments merge fail-closed } CLSPContext; +struct CLSPEvalMemoSlot { + const void *node_id; // TSNode.id; NULL = empty + const CBMType *result; +}; + // --- API --- void c_lsp_init(CLSPContext *ctx, CBMArena *arena, const char *source, int source_len, @@ -132,8 +153,28 @@ const CBMType *c_simplify_type(CLSPContext *ctx, const CBMType *t, bool unwrap_p void cbm_run_c_lsp(CBMArena *arena, CBMFileResult *result, const char *source, int source_len, TSNode root, bool cpp_mode, CBMSourceOrigin source_origin); +// A memo allocation/capacity failure is an explicit incomplete-refinement error. +#define CBM_C_LSP_DEPTH_ERROR "C/C++ expression recursion depth limit exceeded" +#define CBM_C_LSP_MEMO_ERROR "C/C++ expression memo allocation or capacity failure" +void cbm_c_lsp_record_failure(CBMFileResult *result, CBMLSPStatus status); +#ifdef CBM_ENABLE_TEST_SEAMS +// Thread-local, scoped to memo allocations only; restore with -1 after a test. +void cbm_c_lsp_test_memo_fail_after(int successful_allocations); +void cbm_c_lsp_test_depth_fail(bool enabled); +uint32_t cbm_c_lsp_test_walks_take(void); +#endif + // Cross-file LSP: build registry from defs + stdlib, re-parse and resolve. -void cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, +// Status variants distinguish memo/depth failure. Bool wrappers report success. +// Completed output before a failed walk is kept. +CBMLSPStatus cbm_run_c_lsp_cross_status(CBMArena *arena, const char *source, int source_len, + const char *module_qn, bool cpp_mode, CBMLSPDef *defs, + int def_count, const char **include_paths, + const char **include_ns_qns, int include_count, + TSTree *cached_tree, // NULL = parse internally + CBMResolvedCallArray *out); + +bool cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, CBMLSPDef *defs, int def_count, const char **include_paths, const char **include_ns_qns, int include_count, TSTree *cached_tree, // NULL = parse internally @@ -146,7 +187,14 @@ CBMTypeRegistry *cbm_c_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, in // Cross-file LSP using a pre-built shared registry (Tier 2). Skips the // per-file registry build; just parse + resolve. -void cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, +CBMLSPStatus cbm_run_c_lsp_cross_with_registry_status( + CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, + CBMTypeRegistry *reg, // pre-built, finalized, READ-ONLY + const char **include_paths, const char **include_ns_qns, int include_count, + TSTree *cached_tree, // NULL = parse internally + CBMResolvedCallArray *out); + +bool cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, const char *module_qn, bool cpp_mode, CBMTypeRegistry *reg, // pre-built, finalized, READ-ONLY const char **include_paths, const char **include_ns_qns, @@ -178,7 +226,7 @@ typedef struct { // Process multiple C/C++ files' cross-file LSP in one CGo call. // out must point to file_count pre-zeroed CBMResolvedCallArray structs. -void cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_count, +bool cbm_batch_c_lsp_cross(CBMArena *arena, CBMBatchCLSPFile *files, int file_count, CBMResolvedCallArray *out); #endif // CBM_LSP_C_LSP_H diff --git a/internal/cbm/lsp/kotlin_lsp.c b/internal/cbm/lsp/kotlin_lsp.c index 0ca49afd5..c0e2f3feb 100644 --- a/internal/cbm/lsp/kotlin_lsp.c +++ b/internal/cbm/lsp/kotlin_lsp.c @@ -489,13 +489,8 @@ static const CBMScope *kt_scope_declaring_name(const CBMScope *scope, const char if (!name) return NULL; for (const CBMScope *current = scope; current; current = current->parent) { - for (const CBMScopeChunk *chunk = current->chunks; chunk; chunk = chunk->next) { - for (int i = 0; i < chunk->used; i++) { - if (chunk->bindings[i].name && strcmp(chunk->bindings[i].name, name) == 0) { - return current; - } - } - } + if (cbm_scope_lookup_local(current, name)) + return current; } return NULL; } diff --git a/internal/cbm/lsp/lsp_node_iter.h b/internal/cbm/lsp/lsp_node_iter.h index 033962f14..65db5c21b 100644 --- a/internal/cbm/lsp/lsp_node_iter.h +++ b/internal/cbm/lsp/lsp_node_iter.h @@ -39,4 +39,33 @@ static inline TSNode* cbm_lsp_collect_children(CBMArena* arena, TSNode node, uin return kids; } +/* Named-children variant (#1527): the same single cursor pass, keeping only + * named nodes, so `for (i) ts_node_named_child(node, i)` over a wide node (a + * module holding thousands of classes, a body holding thousands of + * statements) becomes O(n). Returns NULL and sets *out_n=0 for a node without + * named children or on OOM; callers that must not lose children on OOM fall + * back to indexed access. */ +static inline TSNode *cbm_lsp_collect_named_children(CBMArena *arena, TSNode node, + uint32_t *out_n) { + uint32_t nc = ts_node_named_child_count(node); + *out_n = 0; + if (nc == 0) + return NULL; + TSNode *kids = (TSNode *)cbm_arena_alloc(arena, (size_t)nc * sizeof(TSNode)); + if (!kids) + return NULL; + uint32_t kn = 0; + TSTreeCursor cur = ts_tree_cursor_new(node); + if (ts_tree_cursor_goto_first_child(&cur)) { + do { + TSNode c = ts_tree_cursor_current_node(&cur); + if (ts_node_is_named(c)) + kids[kn++] = c; + } while (kn < nc && ts_tree_cursor_goto_next_sibling(&cur)); + } + ts_tree_cursor_delete(&cur); + *out_n = kn; + return kids; +} + #endif /* CBM_LSP_NODE_ITER_H */ diff --git a/internal/cbm/lsp/lsp_work.h b/internal/cbm/lsp/lsp_work.h new file mode 100644 index 000000000..fa286293d --- /dev/null +++ b/internal/cbm/lsp/lsp_work.h @@ -0,0 +1,24 @@ +/* + * lsp_work.h — deterministic work counter for LSP complexity tests (#1527). + * + * Seam builds only (CBM_ENABLE_TEST_SEAMS): every name comparison a resolver + * primitive performs while searching (scope frame probes, enclosing-definition + * candidates) adds to a thread-local counter, so a test can assert that the + * work grows linearly with input size without reading a clock (O9). Product + * builds compile CBM_LSP_WORK to nothing. + */ +#ifndef CBM_LSP_WORK_H +#define CBM_LSP_WORK_H + +#include + +#ifdef CBM_ENABLE_TEST_SEAMS +extern _Thread_local uint64_t cbm_lsp_work_steps; +#define CBM_LSP_WORK(n) (cbm_lsp_work_steps += (uint64_t)(n)) +/* Read and reset the calling thread's counter. */ +uint64_t cbm_lsp_work_take(void); +#else +#define CBM_LSP_WORK(n) ((void)0) +#endif + +#endif /* CBM_LSP_WORK_H */ diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index 2cf66ad01..bcea0c920 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -13,6 +13,7 @@ #include "py_lsp.h" #include "../cbm.h" #include "../helpers.h" +#include "lsp_node_iter.h" #include "tree_sitter/api.h" #include #include @@ -25,14 +26,106 @@ * only from py_lsp.c, never compiled standalone. */ #include "py_builtins.c" -/* Guards for py_eval_expr_type — mirrors c_eval_expr_type's guard design - * (C_EVAL_DEPTH_LIMIT / C_EVAL_MAX_STEPS_PER_FILE in c_lsp.c). */ +/* Stack guard for py_eval_expr_type (#720; same limit as C_EVAL_DEPTH_LIMIT). + * There is deliberately NO per-file work budget (#1527): the step cap that + * used to sit here stopped resolution after 10000 evaluations and left the + * rest of a large file silently unresolved while the file still reported + * status `indexed`. Memoized evaluation (#710), hashed scope frames, and + * cursor-based child access reduce the repeated work. Memo failure is reported + * explicitly; the existing depth bound now reports an incomplete walk too. */ #define PY_LSP_MAX_EVAL_DEPTH 256 -#define PY_EVAL_MAX_STEPS_PER_FILE 10000 +#include "lsp_work.h" + +#ifdef CBM_ENABLE_TEST_SEAMS +_Atomic uint64_t cbm_py_submodule_probes = 0; +_Atomic uint64_t cbm_py_submodule_probe_visits = 0; +_Atomic uint64_t cbm_py_module_class_binds = 0; +_Atomic uint64_t cbm_py_module_class_bind_visits = 0; +_Atomic uint64_t cbm_py_wildcard_invalidations = 0; +_Atomic uint64_t cbm_py_wildcard_invalidation_visits = 0; +static _Thread_local int py_module_scratch_allocations_left = -1; +static _Thread_local int py_module_prefix_allocations_left = -1; +void cbm_py_test_module_scratch_fail_after(int successful_allocations) { + py_module_scratch_allocations_left = successful_allocations; +} +void cbm_py_test_module_prefix_fail_after(int successful_allocations) { + py_module_prefix_allocations_left = successful_allocations; +} +#endif + +#ifdef CBM_ENABLE_TEST_SEAMS +static _Thread_local int py_test_memo_allocations_left = -1; +static _Thread_local bool py_test_depth_failure; +static _Thread_local int py_test_scope_allocations_left = -1; +static _Thread_local int py_test_class_frames_left = -1; +void cbm_py_lsp_test_class_frame_fail_after(int successful_allocations) { + py_test_class_frames_left = successful_allocations; +} +void cbm_py_lsp_test_scope_fail_after(int successful_allocations) { + py_test_scope_allocations_left = successful_allocations; +} +void cbm_py_lsp_test_depth_fail(bool enabled) { + py_test_depth_failure = enabled; +} +void cbm_py_lsp_test_memo_fail_after(int successful_allocations) { + py_test_memo_allocations_left = successful_allocations; +} +#endif + +static void py_memo_test_stage(PyLSPContext *ctx, const char *stage) { +#ifdef CBM_ENABLE_TEST_SEAMS + const char *depth_stage = getenv("CBM_TEST_PY_LSP_DEPTH_FAIL_STAGE"); + if (depth_stage && strcmp(depth_stage, stage) == 0) + ctx->eval_depth = 256; + const char *fail_stage = getenv("CBM_TEST_PY_LSP_MEMO_FAIL_STAGE"); + if (fail_stage && strcmp(fail_stage, stage) == 0) + ctx->test_memo_allocations_left = 0; +#else + (void)ctx; + (void)stage; +#endif +} + +void cbm_py_lsp_record_failure(CBMFileResult *result, CBMLSPStatus status) { + result->has_error = true; + result->lsp_skipped = true; + /* Static lifetime also works when the result arena cannot allocate. */ + result->error_msg = status == CBM_LSP_SCOPE_FAILED ? CBM_PY_LSP_SCOPE_ERROR + : status == CBM_LSP_DEPTH_EXCEEDED ? CBM_PY_LSP_DEPTH_ERROR + : CBM_PY_LSP_MEMO_ERROR; +} // Forward decls static void py_resolve_calls_in_inner(PyLSPContext *ctx, TSNode node); +/* Named children of one node for an index loop (#1527). ts_node_named_child(n, i) + * restarts tree-sitter's child iterator at the first child, so indexing every + * child of a wide node is O(n^2) -- a module of thousands of classes, or a body + * of thousands of statements, spent most of its resolve time re-walking its + * own children. Wide nodes are collected once (lsp_node_iter.h); narrow nodes + * keep plain indexed access, which is cheaper than an allocation. */ +enum { PY_KIDS_COLLECT_MIN = 64 }; +typedef struct { + TSNode node; + TSNode *kids; /* NULL = index the node directly */ + uint32_t n; +} PyKids; + +static PyKids py_kids(CBMArena *arena, TSNode node) { + PyKids k = {node, NULL, ts_node_named_child_count(node)}; + if (k.n >= PY_KIDS_COLLECT_MIN) { + uint32_t got = 0; + TSNode *kids = cbm_lsp_collect_named_children(arena, node, &got); + if (kids && got == k.n) + k.kids = kids; + } + return k; +} + +static TSNode py_kid(const PyKids *k, uint32_t i) { + return k->kids ? k->kids[i] : ts_node_named_child(k->node, i); +} + /* Decorators are extracted as raw syntax (`@property`, `@pkg.cache(...)`), * not resolved qualified names. Preserve the raw array for sound callable- * value admission, while parsing only enough of the spelling to retain the @@ -87,51 +180,32 @@ static bool py_func_is_exact_callable_value(const CBMRegisteredFunc *func) { (!func->decorator_qns || !func->decorator_qns[0]); } -static int py_func_qn_pointer_cmp(const void *left, const void *right) { - const CBMRegisteredFunc *a = *(CBMRegisteredFunc *const *)left; - const CBMRegisteredFunc *b = *(CBMRegisteredFunc *const *)right; - const char *aqn = a ? a->qualified_name : NULL; - const char *bqn = b ? b->qualified_name : NULL; - if (!aqn || !bqn) - return aqn ? 1 : bqn ? -1 : 0; - return strcmp(aqn, bqn); -} - -/* The graph identifies functions by QN. When Python source redefines the - * same QN, the runtime's last binding is knowable but the graph no longer has - * one occurrence-exact definition target. Mark the whole duplicate group so - * callable values fail closed without changing ordinary call overload/rebind - * behavior. */ -static void py_mark_ambiguous_callable_bindings(CBMTypeRegistry *registry) { - if (!registry || registry->func_count <= 1) +/* Reuse the QN snapshot for duplicate marks and later submodule probes. + * The graph cannot identify one exact callable value for a duplicate QN; + * ordinary exact lookups retain their original registration-order policy. */ +static void py_mark_ambiguous_callable_bindings(CBMTypeRegistry *registry, CBMArena *idx_arena) { + if (!registry) return; - size_t count = (size_t)registry->func_count; - CBMRegisteredFunc **sorted = count <= SIZE_MAX / sizeof(*sorted) - ? (CBMRegisteredFunc **)malloc(count * sizeof(*sorted)) - : NULL; - if (!sorted) { - /* Allocation failure must reduce precision, never fabricate it. */ + cbm_registry_build_func_qn_order(registry, idx_arena); + if (registry->func_count <= 1) + return; + const CBMRegistryQNEntry *sorted = registry->func_qn_sorted; + if (!sorted || registry->func_qn_sorted_upto != registry->func_count) { for (int i = 0; i < registry->func_count; i++) registry->funcs[i].flags |= CBM_FUNC_FLAG_AMBIGUOUS_BINDING; return; } - for (int i = 0; i < registry->func_count; i++) - sorted[i] = ®istry->funcs[i]; - qsort(sorted, count, sizeof(*sorted), py_func_qn_pointer_cmp); - for (size_t first = 0; first < count;) { - size_t end = first + 1; - const char *qn = sorted[first]->qualified_name; - while (qn && end < count && sorted[end]->qualified_name && - strcmp(qn, sorted[end]->qualified_name) == 0) { + int count = registry->func_qn_sorted_count; + for (int first = 0; first < count;) { + int end = first + 1; + while (end < count && strcmp(sorted[first].qualified_name, sorted[end].qualified_name) == 0) end++; - } - if (qn && end - first > 1) { - for (size_t i = first; i < end; i++) - sorted[i]->flags |= CBM_FUNC_FLAG_AMBIGUOUS_BINDING; + if (end - first > 1) { + for (int i = first; i < end; i++) + registry->funcs[sorted[i].func_index].flags |= CBM_FUNC_FLAG_AMBIGUOUS_BINDING; } first = end; } - free(sorted); } /* Depth-guarded entry for the AST call-resolution walk. The walk recurses once @@ -142,7 +216,7 @@ static void py_mark_ambiguous_callable_bindings(CBMTypeRegistry *registry) { * The walk_depth-- runs after the inner returns, so early returns in the body * never leak the counter. */ static void py_resolve_calls_in(PyLSPContext *ctx, TSNode node) { - if (ctx->walk_depth >= cbm_lsp_max_walk_depth()) + if (!ctx || ctx->eval_failure || ctx->walk_depth >= cbm_lsp_max_walk_depth()) return; ctx->walk_depth++; py_resolve_calls_in_inner(ctx, node); @@ -151,6 +225,7 @@ static void py_resolve_calls_in(PyLSPContext *ctx, TSNode node) { static const CBMType *py_eval_expr_type(PyLSPContext *ctx, TSNode node); static const CBMType *py_eval_expr_type_uncached(PyLSPContext *ctx, TSNode node); static void py_process_statement(PyLSPContext *ctx, TSNode node); +static void py_exec_class_definition(PyLSPContext *ctx, TSNode class_node); static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int depth); static const CBMRegisteredFunc *py_lookup_attribute(PyLSPContext *ctx, const char *type_qn, const char *member_name); @@ -212,6 +287,25 @@ static CBMScope *py_scope_push_checked(PyLSPContext *ctx) { return child; } +/* New class execution frames must never fall back to their parent: body + * bindings would escape and subsequent calls could use those stale bindings. */ +static CBMScope *py_class_frame_push(PyLSPContext *ctx, CBMScope *parent) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (py_test_class_frames_left == 0) { + ctx->eval_failure = CBM_LSP_SCOPE_FAILED; + return NULL; + } + if (py_test_class_frames_left > 0) + py_test_class_frames_left--; +#endif + CBMScope *frame = cbm_scope_push(ctx->arena, parent); + if (!frame || frame == parent) { + ctx->eval_failure = CBM_LSP_SCOPE_FAILED; + return NULL; + } + return frame; +} + static void py_scope_clear_callable(PyLSPContext *ctx, const char *name) { if (cbm_scope_update_callable(ctx->current_scope, name, NULL)) { ctx->type_cache_gen++; @@ -223,24 +317,64 @@ static void py_scope_restore(PyLSPContext *ctx, CBMScope *saved) { ctx->current_scope = saved; } -void py_lsp_init(PyLSPContext *ctx, CBMArena *arena, const char *source, int source_len, - const CBMTypeRegistry *registry, const char *module_qn, - CBMResolvedCallArray *out) { +/* Only a changed Python scope needs storage. Failure is explicit: never + * register or walk a package using the raw file scope as an OOM fallback. */ +static bool py_symbol_scope(CBMArena *arena, const char *module_qn, const char *stage, + const char **out) { + *out = module_qn; + size_t len = cbm_fqn_symbol_scope_len(module_qn); + if (!module_qn || module_qn[len] == '\0') + return true; +#ifdef CBM_ENABLE_TEST_SEAMS + const char *fail_stage = getenv("CBM_TEST_PY_LSP_SCOPE_FAIL_STAGE"); + if ((fail_stage && strcmp(fail_stage, stage) == 0) || py_test_scope_allocations_left == 0) + return false; + if (py_test_scope_allocations_left > 0) + py_test_scope_allocations_left--; +#else + (void)stage; +#endif + char *scope = cbm_arena_strndup(arena, module_qn, len); + if (!scope) + return false; + *out = scope; + return true; +} + +static void py_lsp_init_for_stage(PyLSPContext *ctx, CBMArena *arena, const char *source, + int source_len, const CBMTypeRegistry *registry, + const char *module_qn, CBMResolvedCallArray *out, + const char *stage) { if (!ctx) return; memset(ctx, 0, sizeof(PyLSPContext)); +#ifdef CBM_ENABLE_TEST_SEAMS + ctx->test_memo_allocations_left = py_test_memo_allocations_left; + if (py_test_depth_failure) + ctx->eval_depth = 256; +#endif ctx->arena = arena; ctx->source = source; ctx->source_len = source_len; ctx->registry = registry; ctx->registry_head = (CBMTypeRegistry *)registry; - ctx->module_qn = module_qn; + ctx->file_module_qn = module_qn; ctx->resolved_calls = out; + if (!py_symbol_scope(arena, module_qn, stage, &ctx->module_qn)) { + ctx->eval_failure = CBM_LSP_SCOPE_FAILED; + return; + } ctx->current_scope = py_scope_push_checked(ctx); const char *dbg = getenv("CBM_LSP_DEBUG"); ctx->debug = dbg && dbg[0] && dbg[0] != '0'; } +void py_lsp_init(PyLSPContext *ctx, CBMArena *arena, const char *source, int source_len, + const CBMTypeRegistry *registry, const char *module_qn, + CBMResolvedCallArray *out) { + py_lsp_init_for_stage(ctx, arena, source, source_len, registry, module_qn, out, "context"); +} + void py_lsp_add_import(PyLSPContext *ctx, const char *local_name, const char *module_qn) { if (!ctx || !local_name || !module_qn) return; @@ -505,16 +639,19 @@ static PyDirectImportKind py_import_kind_from_statement(PyLSPContext *ctx, TSNod return py_import_match_result(&match, qn_io); } -static PyDirectImportKind py_import_kind_from_ast(PyLSPContext *ctx, TSNode root, const char *local, - const char **qn_io) { +/* stmts: the root's import statements, collected once per root by the caller + * (#1527) -- only they can match, and rescanning every top-level statement + * once per import made classification O(imports x module size). */ +static PyDirectImportKind py_import_kind_from_ast(PyLSPContext *ctx, bool have_root, + const TSNode *stmts, uint32_t stmt_count, + const char *local, const char **qn_io) { const char *qn = qn_io ? *qn_io : NULL; - if (!ctx || !local || !qn || !qn_io || ts_node_is_null(root)) + if (!ctx || !local || !qn || !qn_io || !have_root) return PY_DIRECT_IMPORT_UNKNOWN; PyImportSyntaxMatch match = {0}; - uint32_t root_count = ts_node_named_child_count(root); - for (uint32_t i = 0; i < root_count; i++) { - py_import_match_statement(ctx, ts_node_named_child(root, i), local, qn, &match); + for (uint32_t i = 0; i < stmt_count; i++) { + py_import_match_statement(ctx, stmts[i], local, qn, &match); } return py_import_match_result(&match, qn_io); } @@ -561,12 +698,32 @@ static void py_bind_dotted_prefixes(PyLSPContext *ctx, const char *qn) { static void py_classify_imports_for_root(PyLSPContext *ctx, TSNode root) { if (!ctx) return; + /* A null root classifies every import UNKNOWN, as it always has; so does + * a root whose statement list cannot be allocated (no proof either way). */ + bool have_root = !ts_node_is_null(root); + TSNode *stmts = NULL; + uint32_t stmt_count = 0; + if (have_root && ctx->import_count > 0) { + PyKids rk = py_kids(ctx->arena, root); + if (rk.n > 0) { + stmts = (TSNode *)cbm_arena_alloc(ctx->arena, (size_t)rk.n * sizeof(TSNode)); + if (!stmts) + have_root = false; + } + for (uint32_t i = 0; stmts && i < rk.n; i++) { + TSNode c = py_kid(&rk, i); + const char *ck = ts_node_type(c); + if (strcmp(ck, "import_statement") == 0 || strcmp(ck, "import_from_statement") == 0) + stmts[stmt_count++] = c; + } + } for (int i = 0; i < ctx->import_count; i++) { const char *local = ctx->import_local_names[i]; const char *qn = ctx->import_module_qns[i]; if (!local || !qn) continue; - PyDirectImportKind direct_kind = py_import_kind_from_ast(ctx, root, local, &qn); + PyDirectImportKind direct_kind = + py_import_kind_from_ast(ctx, have_root, stmts, stmt_count, local, &qn); ctx->import_module_qns[i] = qn; if (ctx->import_kinds) ctx->import_kinds[i] = (unsigned char)direct_kind; @@ -783,6 +940,8 @@ static const char *py_lookup_dict_dispatch(PyLSPContext *ctx, const char *var, c return NULL; } +enum { PY_RESOLVED_DEDUP_WINDOW = 256 }; + static void py_emit_resolved_call_reason(PyLSPContext *ctx, const char *callee_qn, const char *strategy, float confidence, const char *reason, TSNode site) { @@ -796,9 +955,8 @@ static void py_emit_resolved_call_reason(PyLSPContext *ctx, const char *callee_q // entries catches the common case while keeping per-emission O(1). // Without this cap the dedup is O(N) per emission -> O(N^2) per file // and dominates above ~1k call sites. - enum { DEDUP_WINDOW = 256 }; int n = ctx->resolved_calls->count; - int start = n > DEDUP_WINDOW ? n - DEDUP_WINDOW : 0; + int start = n > PY_RESOLVED_DEDUP_WINDOW ? n - PY_RESOLVED_DEDUP_WINDOW : 0; for (int i = start; i < n; i++) { CBMResolvedCall *rc = &ctx->resolved_calls->items[i]; if (rc->kind == CBM_RESOLVED_INVOCATION && rc->site_start_byte == site_start && @@ -1422,8 +1580,24 @@ static const CBMType *py_iterable_element_type(PyLSPContext *ctx, const CBMType } /* The real recursive-descent evaluator. Never call directly — go through - * the memoizing, depth- and budget-guarded py_eval_expr_type wrapper below + * the memoizing, depth-guarded py_eval_expr_type wrapper below * (every recursive call inside this body already does). */ +/* Existing function-only submodule predicate, after exact symbol/type misses. */ +static bool py_names_submodule(PyLSPContext *ctx, const char *qn) { + const char *prefix = qn ? cbm_arena_sprintf(ctx->arena, "%s.", qn) : NULL; + if (!prefix) + return false; +#ifdef CBM_ENABLE_TEST_SEAMS + uint64_t visits = 0; + bool found = cbm_registry_has_func_qn_prefix(ctx->registry, prefix, &visits); + atomic_fetch_add_explicit(&cbm_py_submodule_probes, 1, memory_order_relaxed); + atomic_fetch_add_explicit(&cbm_py_submodule_probe_visits, visits, memory_order_relaxed); + return found; +#else + return cbm_registry_has_func_qn_prefix(ctx->registry, prefix, NULL); +#endif +} + static const CBMType *py_eval_expr_type_uncached(PyLSPContext *ctx, TSNode node) { if (!ctx || ts_node_is_null(node)) return cbm_type_unknown(); @@ -1544,25 +1718,7 @@ static const CBMType *py_eval_expr_type_uncached(PyLSPContext *ctx, TSNode node) const CBMRegisteredType *rt = cbm_registry_lookup_type(ctx->registry, qn); if (rt) return cbm_type_named(ctx->arena, qn); - // Submodule: if any registered function/type has qn starting - // with ".." then mod.attr is itself a module. - // Linear scan over registry funcs is O(R) per access; we - // skip it for the common case where mod.attr is already - // matched as a function/type above. With ~900 stdlib funcs - // and many module-attr accesses per file, this can dominate - // — keeping the loop tight and bailing early on first match. - const char *prefix = cbm_arena_sprintf(ctx->arena, "%s.", qn); - size_t prefix_len = strlen(prefix); - bool is_submodule = false; - CBMFreeFuncIter all_funcs; - cbm_registry_all_funcs_chain(ctx->registry, &all_funcs); - for (int i = -1; !is_submodule && (i = cbm_free_func_iter_next(&all_funcs)) >= 0;) { - const char *fqn = all_funcs.reg->funcs[i].qualified_name; - if (fqn && strncmp(fqn, prefix, prefix_len) == 0) { - is_submodule = true; - } - } - if (is_submodule) + if (py_names_submodule(ctx, qn)) return cbm_type_module(ctx->arena, qn); return cbm_type_unknown(); } @@ -2086,6 +2242,14 @@ static bool py_type_cache_grow(PyLSPContext *ctx) { if (ctx->type_cache_cap >= PY_TYPE_CACHE_MAX_CAP) return false; int new_cap = ctx->type_cache_cap ? ctx->type_cache_cap * 2 : PY_TYPE_CACHE_INITIAL_CAP; + if ((size_t)new_cap > SIZE_MAX / sizeof(CBMPyTypeCacheEntry)) + return false; +#ifdef CBM_ENABLE_TEST_SEAMS + if (ctx->test_memo_allocations_left == 0) + return false; + if (ctx->test_memo_allocations_left > 0) + ctx->test_memo_allocations_left--; +#endif CBMPyTypeCacheEntry *new_entries = (CBMPyTypeCacheEntry *)cbm_arena_alloc( ctx->arena, (size_t)new_cap * sizeof(CBMPyTypeCacheEntry)); if (!new_entries) @@ -2111,9 +2275,9 @@ static bool py_type_cache_grow(PyLSPContext *ctx) { return true; } -static void py_type_cache_insert(PyLSPContext *ctx, const void *id, const CBMType *result) { +static bool py_type_cache_insert(PyLSPContext *ctx, const void *id, const CBMType *result) { if (!id || !result) - return; + return true; /* Keep load strictly below 75% so probe chains stay short. If growth * fails (OOM / max cap), REJECT the insert rather than filling up: a * full table would turn every probe loop into a full-table scan, and @@ -2121,7 +2285,7 @@ static void py_type_cache_insert(PyLSPContext *ctx, const void *id, const CBMTyp if (!ctx->type_cache || (ctx->type_cache_count + 1) * 4 > ctx->type_cache_cap * 3) { if (!py_type_cache_grow(ctx) && (!ctx->type_cache || (ctx->type_cache_count + 1) * 4 > ctx->type_cache_cap * 3)) - return; + return false; } uint32_t mask = (uint32_t)ctx->type_cache_cap - 1; uint32_t idx = py_type_cache_hash(id) & mask; @@ -2132,31 +2296,33 @@ static void py_type_cache_insert(PyLSPContext *ctx, const void *id, const CBMTyp e->gen = ctx->type_cache_gen; e->result = result; ctx->type_cache_count++; - return; + return true; } if (e->node_id == id) { e->gen = ctx->type_cache_gen; /* refresh a stale-generation entry in place */ e->result = result; - return; + return true; } idx = (idx + 1) & mask; } /* Probe bound exhausted — only reachable with a corrupt count; drop the * insert instead of spinning. */ + return false; } -/* Memoizing, depth- and budget-guarded wrapper — the function every call - * site in this file goes through. */ +/* Memoizing, depth-guarded wrapper — the function every call site in this + * file goes through. */ static const CBMType *py_eval_expr_type(PyLSPContext *ctx, TSNode node) { - if (!ctx || ts_node_is_null(node)) + if (!ctx || ctx->eval_failure || ts_node_is_null(node)) return cbm_type_unknown(); /* Depth cap (issue #720): the evaluator recurses once per expression * nesting level, so a pathologically deep expression (tens of - * thousands of parens) overflowed the native stack. Same limit as - * C_EVAL_DEPTH_LIMIT. Past the cap: unknown, and NEVER cached. */ + * thousands of parens) overflowed the native stack. At the existing bound, + * stop this walk explicitly; never memoize a truncated result as complete. */ if (ctx->eval_depth >= PY_LSP_MAX_EVAL_DEPTH) { ctx->eval_truncations++; + ctx->eval_failure = CBM_LSP_DEPTH_EXCEEDED; return cbm_type_unknown(); } @@ -2164,17 +2330,9 @@ static const CBMType *py_eval_expr_type(PyLSPContext *ctx, TSNode node) { if (cached) return cached; - /* Per-file work budget (mirrors C_EVAL_MAX_STEPS_PER_FILE): expression - * type evaluation is best-effort, so pathological files degrade to - * unknown instead of stalling repository indexing. Only real - * evaluations consume budget — cache hits above are O(1). */ - if (ctx->eval_steps++ > PY_EVAL_MAX_STEPS_PER_FILE) { - ctx->eval_truncations++; - if (ctx->debug && ctx->eval_steps == PY_EVAL_MAX_STEPS_PER_FILE + 2) { - fprintf(stderr, " [pylsp] expression eval step budget exhausted; returning unknown\n"); - } - return cbm_type_unknown(); - } + /* One real (non-memoized) evaluation: the work the complexity tests count + * to prove evaluation stays linear without a budget. */ + CBM_LSP_WORK(1); uint32_t trunc_before = ctx->eval_truncations; ctx->eval_depth++; @@ -2184,19 +2342,25 @@ static const CBMType *py_eval_expr_type(PyLSPContext *ctx, TSNode node) { result = cbm_type_unknown(); /* Only cache full-fidelity results: if any descendant evaluation was - * cut off by the depth cap or the step budget, this result may be a - * truncated `unknown`. Caching it would poison later evaluations that - * reach the same node from a shallower frame (or with fresh budget) — - * exactly the reuse the guards are supposed to preserve. */ - if (ctx->eval_truncations == trunc_before) - py_type_cache_insert(ctx, node.id, result); - return result; + * cut off by the depth cap, this result may be a truncated `unknown`. + * The sticky failure stops this walk instead of repeating uncached branches; + * a fresh walk starts with a fresh cache. */ + if (!ctx->eval_failure && ctx->eval_truncations == trunc_before && + !py_type_cache_insert(ctx, node.id, result)) + ctx->eval_failure = CBM_LSP_MEMO_FAILED; + return ctx->eval_failure ? cbm_type_unknown() : result; } +#ifdef CBM_ENABLE_TEST_SEAMS +const CBMType *cbm_py_lsp_test_eval(PyLSPContext *ctx, TSNode node) { + return py_eval_expr_type(ctx, node); +} +#endif + /* ── statement processing: bind from assignments, for-loops, with-as ──── */ static void py_process_statement(PyLSPContext *ctx, TSNode node) { - if (!ctx || ts_node_is_null(node)) + if (!ctx || ctx->eval_failure || ts_node_is_null(node)) return; const char *k = ts_node_type(node); @@ -2305,13 +2469,16 @@ static void py_process_statement(PyLSPContext *ctx, TSNode node) { } else { py_scope_bind(ctx, name, rhs_type); } + // Class attributes must not replace file-wide dispatch entries. // Lambda registry: `fn = lambda x: ...`. - if (!ts_node_is_null(right) && strcmp(ts_node_type(right), "lambda") == 0) { + if (!ctx->class_body_outer_scope && !ts_node_is_null(right) && + strcmp(ts_node_type(right), "lambda") == 0) { py_register_lambda(ctx, name, right); } // Dict literal dispatch table: `funcs = {"a": foo, "b": bar}` // with all values being known function QNs. - if (!ts_node_is_null(right) && strcmp(ts_node_type(right), "dictionary") == 0) { + if (!ctx->class_body_outer_scope && !ts_node_is_null(right) && + strcmp(ts_node_type(right), "dictionary") == 0) { py_register_dict_literal(ctx, name, right); } } @@ -2502,6 +2669,8 @@ static void py_process_statement(PyLSPContext *ctx, TSNode node) { /* ── Recursive walker: process statements + emit resolved_calls ── */ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) { + if (!ctx || ctx->eval_failure) + return; if (!ctx || ctx->callable_value_proof_disabled) return; TSNode fn = ts_node_child_by_field_name(call_node, "function", 8); @@ -2517,7 +2686,11 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) { // Walk the lambda body for any nested call sites with the params // bound to the call's arg types. Resolved calls get caller_qn // set to a synthetic child of the enclosing function. - TSNode lambda_node = py_lookup_lambda(ctx, fname); + /* These legacy entries have no lexical frame identity. During class + * execution, leave lambda replay unresolved instead of bypassing a + * local shadow or running the lambda against class locals. */ + TSNode lambda_node = + ctx->class_body_outer_scope ? (TSNode){0} : py_lookup_lambda(ctx, fname); if (!ts_node_is_null(lambda_node)) { TSNode params = ts_node_child_by_field_name(lambda_node, "parameters", 10); TSNode body = ts_node_child_by_field_name(lambda_node, "body", 4); @@ -2628,7 +2801,7 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) { // Subscript-as-call: `funcs["a"]()` where funcs is a registered // dict-literal dispatch table. - if (strcmp(fk, "subscript") == 0) { + if (!ctx->class_body_outer_scope && strcmp(fk, "subscript") == 0) { TSNode container = ts_node_child_by_field_name(fn, "value", 5); TSNode key = ts_node_child_by_field_name(fn, "subscript", 9); if (!ts_node_is_null(container) && !ts_node_is_null(key) && @@ -2730,17 +2903,18 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) { // segment) and look up ".". A genuinely-external module // (requests, os) has no such project def, so it correctly stays // lsp_module_attr_unresolved. - if (mod && ctx->module_qn) { - const char *last_dot = strrchr(ctx->module_qn, '.'); - if (last_dot && last_dot > ctx->module_qn) { - size_t root_len = (size_t)(last_dot - ctx->module_qn); + if (mod && ctx->file_module_qn) { + const char *last_dot = strrchr(ctx->file_module_qn, '.'); + if (last_dot && last_dot > ctx->file_module_qn) { + size_t root_len = (size_t)(last_dot - ctx->file_module_qn); // Skip if mod is already rooted under the project to avoid // "..mod". - if (!(strncmp(mod, ctx->module_qn, root_len) == 0 && mod[root_len] == '.')) { + if (!(strncmp(mod, ctx->file_module_qn, root_len) == 0 && + mod[root_len] == '.')) { char *qual_mod = (char *)cbm_arena_alloc(ctx->arena, root_len + 1 + strlen(mod) + 1); if (qual_mod) { - memcpy(qual_mod, ctx->module_qn, root_len); + memcpy(qual_mod, ctx->file_module_qn, root_len); qual_mod[root_len] = '.'; strcpy(qual_mod + root_len + 1, mod); const CBMRegisteredFunc *qf = @@ -3202,6 +3376,65 @@ static void py_emit_dunder_call(PyLSPContext *ctx, const CBMType *recv, const ch } } +/* A class comprehension evaluates its leftmost iterable in the class frame, + * but the implicit body scope skips class locals. Nested comprehensions keep + * the surrounding comprehension frame. Keep class execution marked active so + * legacy bare-name dispatch cannot bypass either frame's lexical bindings. */ +static void py_resolve_class_comprehension(PyLSPContext *ctx, TSNode node) { + if (ctx->eval_failure) + return; + PyKids kids = py_kids(ctx->arena, node); + uint32_t first_for = kids.n; + TSNode first_right = {0}; + for (uint32_t i = 0; i < kids.n && !ctx->eval_failure; i++) { + TSNode child = py_kid(&kids, i); + if (strcmp(ts_node_type(child), "for_in_clause") == 0) { + first_for = i; + first_right = ts_node_child_by_field_name(child, "right", 5); + break; + } + } + if (first_for == kids.n) + return; + const CBMType *first_type = py_eval_expr_type(ctx, first_right); + py_resolve_calls_in(ctx, first_right); + if (ctx->eval_failure) + return; + const CBMType *first_elem = py_iterable_element_type(ctx, first_type); + CBMScope *saved_scope = ctx->current_scope; + bool saved_comprehension = ctx->class_body_in_comprehension; + CBMScope *parent = saved_comprehension ? saved_scope : ctx->class_body_outer_scope; + CBMScope *frame = py_class_frame_push(ctx, parent); + if (!frame) + return; + py_scope_restore(ctx, frame); + ctx->class_body_in_comprehension = true; + /* Clauses precede the body at evaluation time, although tree-sitter stores + * the body first. Walk each iterable/filter only once, before later binds. */ + for (uint32_t i = first_for; i < kids.n && !ctx->eval_failure; i++) { + TSNode child = py_kid(&kids, i); + if (strcmp(ts_node_type(child), "for_in_clause") == 0) { + TSNode left = ts_node_child_by_field_name(child, "left", 4); + const CBMType *elem = first_elem; + if (i != first_for) { + TSNode right = ts_node_child_by_field_name(child, "right", 5); + const CBMType *iter = py_eval_expr_type(ctx, right); + py_resolve_calls_in(ctx, right); + if (ctx->eval_failure) + break; + elem = py_iterable_element_type(ctx, iter); + } + py_bind_for_target(ctx, left, elem); + } else { + py_resolve_calls_in(ctx, child); + } + } + for (uint32_t i = 0; i < first_for && !ctx->eval_failure; i++) + py_resolve_calls_in(ctx, py_kid(&kids, i)); + ctx->class_body_in_comprehension = saved_comprehension; + py_scope_restore(ctx, saved_scope); +} + static void py_resolve_calls_in_inner(PyLSPContext *ctx, TSNode node) { if (!ctx || ts_node_is_null(node)) return; @@ -3404,6 +3637,10 @@ static void py_resolve_calls_in_inner(PyLSPContext *ctx, TSNode node) { // iterable's element type, then walk inner expressions. if (strcmp(k, "list_comprehension") == 0 || strcmp(k, "dictionary_comprehension") == 0 || strcmp(k, "set_comprehension") == 0 || strcmp(k, "generator_expression") == 0) { + if (ctx->class_body_outer_scope) { + py_resolve_class_comprehension(ctx, node); + return; + } CBMScope *saved = ctx->current_scope; ctx->current_scope = py_scope_push_checked(ctx); uint32_t cnc = ts_node_named_child_count(node); @@ -3436,16 +3673,20 @@ static void py_resolve_calls_in_inner(PyLSPContext *ctx, TSNode node) { // Recurse: children. We don't push scope for control-flow blocks // here (Python scoping is function-level apart from comprehension / // lambda / class), but we do for nested function / class / lambda. - if (strcmp(k, "function_definition") == 0 || strcmp(k, "class_definition") == 0 || - strcmp(k, "lambda") == 0) { + if (strcmp(k, "function_definition") == 0 || strcmp(k, "lambda") == 0) { // These are processed by the top-level pass; skip recursion to // avoid double-walking their bodies. return; } - uint32_t nc = ts_node_named_child_count(node); - for (uint32_t i = 0; i < nc; i++) { - py_resolve_calls_in(ctx, ts_node_named_child(node, i)); + if (strcmp(k, "class_definition") == 0) { + py_exec_class_definition(ctx, node); + return; + } + + PyKids nk = py_kids(ctx->arena, node); + for (uint32_t i = 0; i < nk.n && !ctx->eval_failure; i++) { + py_resolve_calls_in(ctx, py_kid(&nk, i)); } } @@ -4006,6 +4247,8 @@ static void py_bind_parameters(PyLSPContext *ctx, TSNode params) { } static void py_process_function(PyLSPContext *ctx, TSNode func_node, const char *container_qn) { + if (!ctx || ctx->eval_failure) + return; TSNode name_node = ts_node_child_by_field_name(func_node, "name", 4); if (ts_node_is_null(name_node)) return; @@ -4034,9 +4277,9 @@ static void py_process_function(PyLSPContext *ctx, TSNode func_node, const char if (!ts_node_is_null(body)) { py_resolve_calls_in(ctx, body); // Also descend into nested function/class definitions in the body. - uint32_t bnc = ts_node_named_child_count(body); - for (uint32_t i = 0; i < bnc; i++) { - TSNode c = ts_node_named_child(body, i); + PyKids bk = py_kids(ctx->arena, body); + for (uint32_t i = 0; i < bk.n; i++) { + TSNode c = py_kid(&bk, i); const char *ck = ts_node_type(c); if (strcmp(ck, "function_definition") == 0) { py_process_function(ctx, c, ctx->enclosing_func_qn); @@ -4063,7 +4306,123 @@ static bool py_is_init_method(PyLSPContext *ctx, TSNode func_node) { return nm && (strcmp(nm, "__init__") == 0 || strcmp(nm, "__post_init__") == 0); } +/* Decorator expressions of a decorated definition, in source order. They + * evaluate in the scope enclosing the definition, before it is bound. A bare + * `@name` is no call and emits nothing; `@name(args)` is one. */ +static void py_resolve_decorators(PyLSPContext *ctx, TSNode decorated) { + if (!ctx || ctx->eval_failure) + return; + PyKids kids = py_kids(ctx->arena, decorated); + for (uint32_t i = 0; i < kids.n && !ctx->eval_failure; i++) { + TSNode child = py_kid(&kids, i); + if (strcmp(ts_node_type(child), "decorator") == 0) + py_resolve_calls_in(ctx, child); + } +} + +/* Bind the name a class-body `def`/`class` statement defines in the class + * frame, so later class-body statements see it (and never the same-named + * module symbol). Its value is exact only for an undecorated function or a + * registered class whose QN is known; anything else is an UNKNOWN shadow. */ +static void py_bind_class_body_def(PyLSPContext *ctx, TSNode stmt) { + if (ctx->eval_failure) + return; + TSNode def = stmt; + bool decorated = strcmp(ts_node_type(stmt), "decorated_definition") == 0; + if (decorated) { + def = ts_node_child_by_field_name(stmt, "definition", 10); + if (ts_node_is_null(def)) + return; + } + const char *kind = ts_node_type(def); + bool is_class = strcmp(kind, "class_definition") == 0; + if (!is_class && strcmp(kind, "function_definition") != 0) + return; + char *name = py_node_text(ctx, ts_node_child_by_field_name(def, "name", 4)); + if (!name || !name[0]) { + py_disable_callable_value_proof(ctx); + return; + } + const char *qn = ctx->class_body_qn + ? cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->class_body_qn, name) + : NULL; + if (is_class) { + const CBMRegisteredType *type = qn ? cbm_registry_lookup_type(ctx->registry, qn) : NULL; + py_scope_bind(ctx, name, + type ? cbm_type_named(ctx->arena, type->qualified_name) : cbm_type_unknown()); + return; + } + const CBMRegisteredFunc *func = qn ? cbm_registry_lookup_func(ctx->registry, qn) : NULL; + if (!decorated && py_func_is_exact_callable_value(func)) { + py_scope_bind_callable(ctx, name, cbm_type_unknown(), func->qualified_name); + } else { + py_scope_bind(ctx, name, cbm_type_unknown()); + } +} + +/* Execute a class definition where it stands: its bases in the enclosing + * scope, then its body statements in source order in a class frame. Calls + * keep the caller the extraction gives them: the enclosing function, or the + * module QN at module level (enclosing_func_qn is left unchanged). + * + * Scoping follows Python: a name bound earlier in the body is visible to + * later body statements (and to decorators of methods), but the class frame + * is not an enclosing scope for anything nested in it. Method bodies are + * resolved later by py_process_class against the module scope, and a nested + * class body chains to the nearest enclosing non-class scope. Each body + * statement is walked exactly once. */ +static void py_exec_class_definition(PyLSPContext *ctx, TSNode class_node) { + if (!ctx || ctx->eval_failure || ts_node_is_null(class_node)) + return; + TSNode supers = ts_node_child_by_field_name(class_node, "superclasses", 12); + if (!ts_node_is_null(supers)) + py_resolve_calls_in(ctx, supers); + TSNode body = ts_node_child_by_field_name(class_node, "body", 4); + if (ctx->eval_failure || ts_node_is_null(body)) + return; + + char *cname = py_node_text(ctx, ts_node_child_by_field_name(class_node, "name", 4)); + const char *class_qn = NULL; + if (cname && cname[0]) { + if (ctx->class_body_outer_scope) { + class_qn = ctx->class_body_qn + ? cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->class_body_qn, cname) + : NULL; + } else if (ctx->module_qn && ctx->enclosing_func_qn && ctx->file_module_qn && + strcmp(ctx->enclosing_func_qn, ctx->file_module_qn) == 0) { + class_qn = cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->module_qn, cname); + } + } + + CBMScope *saved_scope = ctx->current_scope; + CBMScope *saved_outer = ctx->class_body_outer_scope; + const char *saved_qn = ctx->class_body_qn; + bool saved_comprehension = ctx->class_body_in_comprehension; + CBMScope *outer = saved_outer ? saved_outer : saved_scope; + CBMScope *frame = py_class_frame_push(ctx, outer); + if (!frame) + return; + py_scope_restore(ctx, frame); + ctx->class_body_outer_scope = outer; + ctx->class_body_qn = class_qn; + ctx->class_body_in_comprehension = false; + + PyKids bk = py_kids(ctx->arena, body); + for (uint32_t i = 0; i < bk.n && !ctx->eval_failure; i++) { + TSNode stmt = py_kid(&bk, i); + py_resolve_calls_in(ctx, stmt); + py_bind_class_body_def(ctx, stmt); + } + + ctx->class_body_outer_scope = saved_outer; + ctx->class_body_qn = saved_qn; + ctx->class_body_in_comprehension = saved_comprehension; + py_scope_restore(ctx, saved_scope); +} + static void py_process_class(PyLSPContext *ctx, TSNode class_node) { + if (!ctx || ctx->eval_failure) + return; TSNode name_node = ts_node_child_by_field_name(class_node, "name", 4); if (ts_node_is_null(name_node)) return; @@ -4076,13 +4435,14 @@ static void py_process_class(PyLSPContext *ctx, TSNode class_node) { TSNode body = ts_node_child_by_field_name(class_node, "body", 4); if (!ts_node_is_null(body)) { - uint32_t bnc = ts_node_named_child_count(body); + PyKids bk = py_kids(ctx->arena, body); + uint32_t bnc = bk.n; // First pass: process class-level annotated assignments (PEP 526 // class-body field annotations like `x: int`) and dunder __init__ // methods so fields are registered before sibling methods that // reference them. for (uint32_t i = 0; i < bnc; i++) { - TSNode c = ts_node_named_child(body, i); + TSNode c = py_kid(&bk, i); const char *ck = ts_node_type(c); if (strcmp(ck, "expression_statement") == 0 && ts_node_named_child_count(c) > 0) { TSNode inner = ts_node_named_child(c, 0); @@ -4105,7 +4465,7 @@ static void py_process_class(PyLSPContext *ctx, TSNode class_node) { } // Second pass: __init__ (so self.x assignments populate fields). for (uint32_t i = 0; i < bnc; i++) { - TSNode c = ts_node_named_child(body, i); + TSNode c = py_kid(&bk, i); const char *ck = ts_node_type(c); TSNode fn_node = c; bool is_decorated = strcmp(ck, "decorated_definition") == 0; @@ -4123,7 +4483,7 @@ static void py_process_class(PyLSPContext *ctx, TSNode class_node) { } // Third pass: every other method (and nested classes). for (uint32_t i = 0; i < bnc; i++) { - TSNode c = ts_node_named_child(body, i); + TSNode c = py_kid(&bk, i); const char *ck = ts_node_type(c); if (strcmp(ck, "function_definition") == 0) { if (!py_is_init_method(ctx, c)) { @@ -4148,10 +4508,17 @@ static void py_process_class(PyLSPContext *ctx, TSNode class_node) { /* Module bindings are replayed in source order before deferred function bodies * are analyzed. Class definitions always replace an earlier callable identity; * the registry decides whether the resulting class type is known precisely. */ -static bool py_root_defines_class_named(PyLSPContext *ctx, TSNode root, const char *name) { - uint32_t count = ts_node_named_child_count(root); - for (uint32_t i = 0; i < count; i++) { - TSNode node = ts_node_named_child(root, i); +/* The names of the classes this source defines at module level, as a name set + * (a detached, hash-indexed CBMScope frame). Built in one pass over the root: + * asking "does the root define a class named X?" by rescanning the root once + * per registry type made module assembly O(types x top-level statements). + * Returns NULL if a class name or set entry cannot be collected completely. */ +static CBMScope *py_root_class_names(PyLSPContext *ctx, const PyKids *rk) { + CBMScope *names = cbm_scope_push(ctx->arena, NULL); + if (!names) + return NULL; + for (uint32_t i = 0; i < rk->n; i++) { + TSNode node = py_kid(rk, i); const char *kind = ts_node_type(node); if (strcmp(kind, "decorated_definition") == 0) { node = ts_node_child_by_field_name(node, "definition", 10); @@ -4159,35 +4526,119 @@ static bool py_root_defines_class_named(PyLSPContext *ctx, TSNode root, const ch } if (strcmp(kind, "class_definition") != 0) continue; - TSNode name_node = ts_node_child_by_field_name(node, "name", 4); - if (py_import_node_text_equals(ctx, name_node, name)) - return true; + char *name = py_node_text(ctx, ts_node_child_by_field_name(node, "name", 4)); +#ifdef CBM_ENABLE_TEST_SEAMS + if (ctx->test_root_name_failure == 1) + name = NULL; + else if (ctx->test_root_name_failure == 2) + name = ""; +#endif + if (!name || !name[0] || !cbm_scope_bind_checked(names, name, cbm_type_unknown())) + return NULL; } - return false; + return names; +} + +/* The temporary query owner never supplies a retained binding/name/type. + * A disabled/failed optimization leaves the original module-aware scan usable. */ +static bool py_module_scratch_open(CBMArena *scratch) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (py_module_scratch_allocations_left == 0) + return false; + if (py_module_scratch_allocations_left > 0) + py_module_scratch_allocations_left--; +#endif + cbm_arena_init_lazy(scratch, 4096); + return true; +} + +static char *py_module_prefix(CBMArena *scratch, const char *module_qn, size_t length) { + /* The arena rounds up by seven bytes after this addition. */ + if (length > SIZE_MAX - 9) + return NULL; +#ifdef CBM_ENABLE_TEST_SEAMS + if (py_module_prefix_allocations_left == 0) + return NULL; + if (py_module_prefix_allocations_left > 0) + py_module_prefix_allocations_left--; +#endif + char *prefix = cbm_arena_alloc(scratch, length + 2); + if (!prefix) + return NULL; + memcpy(prefix, module_qn, length); + prefix[length] = '.'; + prefix[length + 1] = '\0'; + return prefix; } /* A project registry can contain another file's class in the same logical * Python module. Keep those cross-file globals available, but do not prebind * classes declared by this source: their binding epoch belongs in the ordered * replay below. */ -static void py_bind_external_module_classes(PyLSPContext *ctx, TSNode root) { +static void py_bind_external_module_classes(PyLSPContext *ctx, const PyKids *rk) { if (!ctx || !ctx->registry || !ctx->module_qn) return; - size_t prefix_len = strlen(ctx->module_qn); - CBMTypeShortIter all_types; - cbm_registry_all_types_chain(ctx->registry, &all_types); - for (int i = -1; (i = cbm_type_short_iter_next(&all_types)) >= 0;) { - const CBMRegisteredType *type = &all_types.reg->types[i]; - const char *qn = type->qualified_name; - const char *name = type->short_name; - if (!qn || !name || strncmp(qn, ctx->module_qn, prefix_len) != 0 || qn[prefix_len] != '.' || - py_root_defines_class_named(ctx, root, name)) { - continue; - } - py_scope_bind(ctx, name, cbm_type_named(ctx->arena, qn)); + CBMScope *defined = py_root_class_names(ctx, rk); + if (!defined) { + /* Without the set no external class can be proven undeclared here; + * binding none is the fail-closed direction. */ + py_disable_callable_value_proof(ctx); + return; } + size_t prefix_len = strlen(ctx->module_qn); + bool package_scope = ctx->file_module_qn && strcmp(ctx->file_module_qn, ctx->module_qn) != 0; + CBMArena scratch = {0}; + char *prefix = py_module_scratch_open(&scratch) + ? py_module_prefix(&scratch, ctx->module_qn, prefix_len) + : NULL; + uint64_t visits = 0; + for (const CBMTypeRegistry *link = ctx->registry; link; link = link->fallback) { + CBMQNPrefixIter it; + if (prefix) + cbm_registry_types_with_qn_prefix(link, prefix, package_scope, &scratch, &it); + int linear = 0; + for (;;) { + int i; + if (prefix) { + i = cbm_qn_prefix_iter_next(&it); + } else { + i = linear < link->type_count ? linear++ : -1; + if (i >= 0) + visits++; + } + if (i < 0) + break; + if (cbm_registry_type_shadowed(ctx->registry, link, i)) + continue; + const CBMRegisteredType *type = &link->types[i]; + const char *qn = type->qualified_name; + const char *name = type->short_name; + if (!qn || !name || strncmp(qn, ctx->module_qn, prefix_len) != 0 || + qn[prefix_len] != '.' || cbm_scope_lookup_local(defined, name)) + continue; + if (package_scope && strchr(qn + prefix_len + 1, '.')) + continue; /* submodule/nested classes are not package globals */ + py_scope_bind(ctx, name, cbm_type_named(ctx->arena, qn)); + } + if (prefix) + visits += it.visits; + } + cbm_arena_destroy(&scratch); +#ifdef CBM_ENABLE_TEST_SEAMS + atomic_fetch_add_explicit(&cbm_py_module_class_binds, 1, memory_order_relaxed); + atomic_fetch_add_explicit(&cbm_py_module_class_bind_visits, visits, memory_order_relaxed); +#else + (void)visits; +#endif } +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_py_lsp_test_bind_external_classes(PyLSPContext *ctx, TSNode root) { + PyKids rk = py_kids(ctx->arena, root); + py_bind_external_module_classes(ctx, &rk); +} +#endif + static void py_bind_module_class(PyLSPContext *ctx, TSNode class_node) { if (!ctx || ts_node_is_null(class_node) || !ctx->module_qn) return; @@ -4252,19 +4703,47 @@ static void py_invalidate_registry_module_functions(PyLSPContext *ctx, if (!ctx || !registry || !ctx->module_qn) return; size_t prefix_len = strlen(ctx->module_qn); - for (int i = 0; i < registry->func_count; i++) { - const CBMRegisteredFunc *func = ®istry->funcs[i]; - const char *qn = func->qualified_name; - if (!qn || func->receiver_type || strncmp(qn, ctx->module_qn, prefix_len) != 0 || - qn[prefix_len] != '.') { - continue; + CBMArena scratch = {0}; + char *prefix = py_module_scratch_open(&scratch) + ? py_module_prefix(&scratch, ctx->module_qn, prefix_len) + : NULL; + uint64_t visits = 0; + for (const CBMTypeRegistry *link = registry; link; link = link->fallback) { + CBMQNPrefixIter it; + if (prefix) + cbm_registry_funcs_with_qn_prefix(link, prefix, true, &scratch, &it); + int linear = 0; + for (;;) { + int i; + if (prefix) { + i = cbm_qn_prefix_iter_next(&it); + } else { + i = linear < link->func_count ? linear++ : -1; + if (i >= 0) + visits++; + } + if (i < 0) + break; + const CBMRegisteredFunc *func = &link->funcs[i]; + const char *qn = func->qualified_name; + if (!qn || func->receiver_type || strncmp(qn, ctx->module_qn, prefix_len) != 0 || + qn[prefix_len] != '.') + continue; + const char *suffix = qn + prefix_len + 1; + if (!suffix[0] || strchr(suffix, '.')) + continue; + py_scope_bind(ctx, func->short_name ? func->short_name : suffix, cbm_type_unknown()); } - const char *suffix = qn + prefix_len + 1; - if (!suffix[0] || strchr(suffix, '.')) - continue; - py_scope_bind(ctx, func->short_name ? func->short_name : suffix, cbm_type_unknown()); + if (prefix) + visits += it.visits; } - py_invalidate_registry_module_functions(ctx, registry->fallback); + cbm_arena_destroy(&scratch); +#ifdef CBM_ENABLE_TEST_SEAMS + atomic_fetch_add_explicit(&cbm_py_wildcard_invalidations, 1, memory_order_relaxed); + atomic_fetch_add_explicit(&cbm_py_wildcard_invalidation_visits, visits, memory_order_relaxed); +#else + (void)visits; +#endif } /* `from module import *` may overwrite any module global through `__all__`. @@ -4564,9 +5043,9 @@ static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int } return; } - uint32_t count = ts_node_named_child_count(node); - for (uint32_t i = 0; i < count; i++) - py_invalidate_possible_bindings(ctx, ts_node_named_child(node, i), depth + 1); + PyKids nk = py_kids(ctx->arena, node); + for (uint32_t i = 0; i < nk.n; i++) + py_invalidate_possible_bindings(ctx, py_kid(&nk, i), depth + 1); } static bool py_replayable_import_kind(PyDirectImportKind kind) { @@ -4654,10 +5133,22 @@ static bool py_expression_is_annotation_only_assignment(TSNode statement) { } void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { - if (!ctx || ts_node_is_null(root)) + if (!ctx || ctx->eval_failure || ts_node_is_null(root)) return; + int resolved_start = ctx->resolved_calls ? ctx->resolved_calls->count : 0; + int synthetic_start = ctx->syn_calls ? ctx->syn_calls->count : 0; + /* Only invocation dedup mutates prior rows, within this fixed tail. + * Snapshot once per walk: no full-output copy or fallible rollback allocation. */ + int tail_count = + resolved_start < PY_RESOLVED_DEDUP_WINDOW ? resolved_start : PY_RESOLVED_DEDUP_WINDOW; + int tail_start = resolved_start - tail_count; + CBMResolvedCall saved_tail[PY_RESOLVED_DEDUP_WINDOW]; + CBMResolvedCall *original_tail = tail_count ? ctx->resolved_calls->items + tail_start : NULL; + if (tail_count) + memcpy(saved_tail, original_tail, (size_t)tail_count * sizeof(*saved_tail)); + PyKids rk = py_kids(ctx->arena, root); py_classify_imports_for_root(ctx, root); - py_bind_external_module_classes(ctx, root); + py_bind_external_module_classes(ctx, &rk); unsigned char *consumed_imports = NULL; if (ctx->import_count > 0) { consumed_imports = (unsigned char *)cbm_arena_alloc(ctx->arena, (size_t)ctx->import_count); @@ -4665,28 +5156,36 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { memset(consumed_imports, 0, (size_t)ctx->import_count); } - uint32_t nc = ts_node_named_child_count(root); + uint32_t nc = rk.n; const char *prev_func = ctx->enclosing_func_qn; - ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s.__module__", ctx->module_qn); + /* Extraction uses the file's Module QN at top level. Keep the caller + * arena-owned for cross-file APIs whose module_qn input may be temporary. */ + ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s", ctx->file_module_qn); // Pass 1: execute top-level binding effects in source order. Function // bodies remain deferred until the final module scope has been assembled. - for (uint32_t i = 0; i < nc; i++) { - TSNode c = ts_node_named_child(root, i); + for (uint32_t i = 0; i < nc && !ctx->eval_failure; i++) { + TSNode c = py_kid(&rk, i); const char *ck = ts_node_type(c); if (strcmp(ck, "import_statement") == 0 || strcmp(ck, "import_from_statement") == 0) { py_replay_import_statement(ctx, c, consumed_imports); } else if (strcmp(ck, "function_definition") == 0) { py_bind_module_function(ctx, c); } else if (strcmp(ck, "class_definition") == 0) { - py_bind_module_class(ctx, c); + py_exec_class_definition(ctx, c); + if (!ctx->eval_failure) + py_bind_module_class(ctx, c); } else if (strcmp(ck, "decorated_definition") == 0) { + py_resolve_decorators(ctx, c); TSNode def = ts_node_child_by_field_name(c, "definition", 10); - if (!ts_node_is_null(def)) { + if (!ctx->eval_failure && !ts_node_is_null(def)) { const char *dk = ts_node_type(def); - if (strcmp(dk, "function_definition") == 0) + if (strcmp(dk, "function_definition") == 0) { py_bind_module_function(ctx, def); - else if (strcmp(dk, "class_definition") == 0) - py_bind_module_class(ctx, def); + } else if (strcmp(dk, "class_definition") == 0) { + py_exec_class_definition(ctx, def); + if (!ctx->eval_failure) + py_bind_module_class(ctx, def); + } } } else if (strcmp(ck, "expression_statement") == 0) { /* An expression statement is resolved for calls, not scanned as a @@ -4709,8 +5208,8 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { } } // Pass 2: top-level calls (rare) and nested definitions. - for (uint32_t i = 0; i < nc; i++) { - TSNode c = ts_node_named_child(root, i); + for (uint32_t i = 0; i < nc && !ctx->eval_failure; i++) { + TSNode c = py_kid(&rk, i); const char *ck = ts_node_type(c); if (strcmp(ck, "function_definition") == 0) { py_process_function(ctx, c, NULL); @@ -4729,6 +5228,17 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) { } } ctx->enclosing_func_qn = prev_func; + if (ctx->eval_failure) { + if (tail_count) { + memcpy(original_tail, saved_tail, (size_t)tail_count * sizeof(*saved_tail)); + memcpy(ctx->resolved_calls->items + tail_start, saved_tail, + (size_t)tail_count * sizeof(*saved_tail)); + } + if (ctx->resolved_calls) + ctx->resolved_calls->count = resolved_start; + if (ctx->syn_calls) + ctx->syn_calls->count = synthetic_start; + } } /* Register one definition into the registry. Returns true if recognized. */ @@ -4887,8 +5397,18 @@ static bool py_register_def(CBMArena *arena, CBMTypeRegistry *reg, CBMDefinition void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, int source_len, TSNode root) { - if (!arena || !result) + if (!arena || !result || result->lsp_skipped) + return; + + CBMTypeRegistry reg; + cbm_registry_init(®, arena); + PyLSPContext ctx; + py_lsp_init_for_stage(&ctx, arena, source, source_len, ®, result->module_qn, + &result->resolved_calls, "raw"); + if (ctx.eval_failure) { + cbm_py_lsp_record_failure(result, ctx.eval_failure); return; + } /* Inject minimal builtin definitions as real graph nodes (builtins.len, * builtins.str, builtins.str.upper, ...). The typeshed registry already @@ -4899,22 +5419,29 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, * builtin/constructor/method edges point at. Upsert dedups by QN. */ py_builtins_inject_defs(result, arena); - CBMTypeRegistry reg; - cbm_registry_init(®, arena); - cbm_python_stdlib_register(®, arena); - const char *module_qn = result->module_qn; + const char *module_qn = ctx.module_qn; // Register the file's own definitions so calls inside this file can // resolve via the registry. for (int i = 0; i < result->defs.count; i++) { py_register_def(arena, ®, &result->defs.items[i], module_qn); } - py_mark_ambiguous_callable_bindings(®); - PyLSPContext ctx; - py_lsp_init(&ctx, arena, source, source_len, ®, module_qn, &result->resolved_calls); + /* Hash-index the registry before the walk (#1527). Unfinalized, every + * lookup scanned the stdlib plus all of this file's definitions, which + * made resolving a large module O(calls x defs). The index answers each + * duplicated name with its first registration, as the scan did, so the + * resolution result is unchanged; post-walk additions (instance fields) + * stay visible through the registry's tail scan. The index lives in a + * scratch arena that dies with this call, not in the result arena. */ + CBMArena idx_arena; + cbm_arena_init(&idx_arena); + py_mark_ambiguous_callable_bindings(®, &idx_arena); + reg.index_first_registered = true; + cbm_registry_finalize_into(®, &idx_arena); + /* Let the resolver inject synthetic syntactic calls for operator/subscript * dunder desugaring so those recovered calls reach the CALLS-edge pipeline. */ ctx.syn_calls = &result->calls; @@ -4926,7 +5453,11 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, } } + py_memo_test_stage(&ctx, "raw"); py_lsp_process_file(&ctx, root); + if (ctx.eval_failure) + cbm_py_lsp_record_failure(result, ctx.eval_failure); + cbm_arena_destroy(&idx_arena); } /* ── Cross-file + batch ───────────────────────────────────────── */ @@ -4965,9 +5496,50 @@ static const char **py_split_pipe(CBMArena *arena, const char *text) { return out; } +/* Split a class def's field_defs ("name:QN|name:QN", built by the Python + * field fold in pass_lsp_cross.c) into the parallel NULL-terminated arrays a + * CBMRegisteredType carries. The types are already project QNs, so each one + * becomes a NAMED type verbatim. Without this, `obj.field.method()` on a class + * from another file lost the field's type (#1277); same-file classes get their + * fields from the per-file walk instead. Pure string work: no registry lookup + * happens here, so it is safe inside the type-registration pass. */ +static void py_split_field_defs(CBMArena *arena, const char *field_defs, const char ***names_out, + const CBMType ***types_out) { + *names_out = NULL; + *types_out = NULL; + const char **pairs = py_split_pipe(arena, field_defs); + if (!pairs) + return; + int n = 0; + while (pairs[n]) + n++; + const char **names = + (const char **)cbm_arena_alloc(arena, (size_t)(n + 1) * sizeof(const char *)); + const CBMType **types = + (const CBMType **)cbm_arena_alloc(arena, (size_t)(n + 1) * sizeof(const CBMType *)); + if (!names || !types) + return; + int kept = 0; + for (int i = 0; i < n; i++) { + char *colon = strchr(pairs[i], ':'); + if (!colon || colon == pairs[i] || !colon[1]) + continue; + *colon = '\0'; /* pairs[i] is this call's own arena copy */ + names[kept] = pairs[i]; + types[kept] = cbm_type_named(arena, colon + 1); + kept++; + } + if (kept == 0) + return; + names[kept] = NULL; + types[kept] = NULL; + *names_out = names; + *types_out = types; +} + /* Build a registry from CBMLSPDef[] supplied by the caller — covers both * the source file's own defs and cross-file referenced defs. */ -static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRegistry *reg, +static bool py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRegistry *reg, CBMLSPDef *defs, int def_count) { /* Pass 1: types only — the method pass probes the registry per Method def * (receiver auto-registration), which is a LINEAR scan pre-finalize: @@ -4990,6 +5562,7 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe if (d->method_names_str && d->method_names_str[0]) { rt.method_names = py_split_pipe(arena, d->method_names_str); } + py_split_field_defs(arena, d->field_defs, &rt.field_names, &rt.field_types); cbm_registry_add_type(reg, rt); } } @@ -5030,7 +5603,13 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe ret_types[n] = NULL; } } - PySignatureParamParserContext parser_ctx = {.module_qn = d->def_module_qn}; + const char *symbol_scope = d->def_module_qn; + /* Fallback definitions can be mixed-language. Only authoritative + * Python metadata opts into package scope normalization. */ + if (d->lang == CBM_LANG_PYTHON && + !py_symbol_scope(arena, d->def_module_qn, "register", &symbol_scope)) + return false; + PySignatureParamParserContext parser_ctx = {.module_qn = symbol_scope}; const CBMType **param_types = cbm_type_materialize_signature_params( arena, d->signature_param_types, d->signature_param_count, py_signature_param_type_adapter, &parser_ctx); @@ -5050,15 +5629,33 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe cbm_registry_add_func(reg, rf); } } + return true; } -void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, +#ifdef CBM_ENABLE_TEST_SEAMS +bool cbm_py_lsp_test_register_defs(CBMArena *arena, CBMTypeRegistry *reg, CBMLSPDef *defs, + int def_count) { + return py_register_lsp_defs(arena, NULL, reg, defs, def_count); +} +#endif + +bool cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, CBMLSPDef *defs, int def_count, const char **import_names, const char **import_qns, int import_count, TSTree *cached_tree, CBMResolvedCallArray *out, CBMCallArray *synthetic_calls) { + return cbm_run_py_lsp_cross_status(arena, source, source_len, module_qn, defs, def_count, + import_names, import_qns, import_count, cached_tree, out, + synthetic_calls) == CBM_LSP_COMPLETE; +} + +CBMLSPStatus cbm_run_py_lsp_cross_status(CBMArena *arena, const char *source, int source_len, + const char *module_qn, CBMLSPDef *defs, int def_count, + const char **import_names, const char **import_qns, + int import_count, TSTree *cached_tree, + CBMResolvedCallArray *out, CBMCallArray *synthetic_calls) { if (!arena || !source || source_len <= 0 || !out) - return; + return CBM_LSP_COMPLETE; TSParser *parser = NULL; TSTree *tree = cached_tree; @@ -5066,13 +5663,13 @@ void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, if (!tree) { parser = ts_parser_new(); if (!parser) - return; + return CBM_LSP_COMPLETE; ts_parser_set_language(parser, tree_sitter_python()); tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len); owns_tree = true; if (!tree) { ts_parser_delete(parser); - return; + return CBM_LSP_COMPLETE; } } TSNode root = ts_tree_root_node(tree); @@ -5084,28 +5681,36 @@ void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, /* Index allocations go to a per-call scratch arena (see php_lsp_cross). */ CBMArena idx_arena; cbm_arena_init(&idx_arena); - py_register_lsp_defs(arena, &idx_arena, ®, defs, def_count); - py_mark_ambiguous_callable_bindings(®); + PyLSPContext ctx; + py_lsp_init_for_stage(&ctx, arena, source, source_len, ®, module_qn, out, "cross"); + if (ctx.eval_failure) + goto cross_cleanup; + if (!py_register_lsp_defs(arena, &idx_arena, ®, defs, def_count)) { + ctx.eval_failure = CBM_LSP_SCOPE_FAILED; + goto cross_cleanup; + } + py_mark_ambiguous_callable_bindings(®, &idx_arena); /* Finalize registry — O(1) lookups. See go_lsp.c "3c. Finalize" * comment for the rationale. */ cbm_registry_finalize_into(®, &idx_arena); - PyLSPContext ctx; - py_lsp_init(&ctx, arena, source, source_len, ®, module_qn, out); ctx.syn_calls = synthetic_calls; for (int i = 0; i < import_count; i++) { if (import_names && import_qns && import_names[i] && import_qns[i]) { py_lsp_add_import(&ctx, import_names[i], import_qns[i]); } } + py_memo_test_stage(&ctx, "cross"); py_lsp_process_file(&ctx, root); +cross_cleanup: cbm_arena_destroy(&idx_arena); if (owns_tree && tree) ts_tree_delete(tree); if (parser) ts_parser_delete(parser); + return ctx.eval_failure; } /* ── Tier 2: pre-built per-language registry (mirrors Go pilot) ── */ @@ -5124,22 +5729,33 @@ CBMTypeRegistry *cbm_py_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, i if (d->lang != CBM_LANG_PYTHON) continue; /* Reuse the existing register fn on a single-def slice (n=1 inline). */ - py_register_lsp_defs(arena, NULL, reg, d, 1); + if (!py_register_lsp_defs(arena, NULL, reg, d, 1)) + return NULL; /* caller owns the shared arena and other registries */ } - py_mark_ambiguous_callable_bindings(reg); + py_mark_ambiguous_callable_bindings(reg, arena); + cbm_registry_build_type_qn_order(reg, arena); cbm_registry_finalize(reg); reg->read_only = true; /* seal: shared Tier-2 registry is read-only during resolve */ return reg; } -void cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, +bool cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, const char *module_qn, CBMTypeRegistry *reg, const char **import_names, const char **import_qns, int import_count, TSTree *cached_tree, CBMResolvedCallArray *out, CBMCallArray *synthetic_calls) { + return cbm_run_py_lsp_cross_with_registry_status( + arena, source, source_len, module_qn, reg, import_names, import_qns, import_count, + cached_tree, out, synthetic_calls) == CBM_LSP_COMPLETE; +} + +CBMLSPStatus cbm_run_py_lsp_cross_with_registry_status( + CBMArena *arena, const char *source, int source_len, const char *module_qn, + CBMTypeRegistry *reg, const char **import_names, const char **import_qns, int import_count, + TSTree *cached_tree, CBMResolvedCallArray *out, CBMCallArray *synthetic_calls) { if (!arena || !source || source_len <= 0 || !out || !reg) - return; + return CBM_LSP_COMPLETE; TSParser *parser = NULL; TSTree *tree = cached_tree; @@ -5147,38 +5763,43 @@ void cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int if (!tree) { parser = ts_parser_new(); if (!parser) - return; + return CBM_LSP_COMPLETE; ts_parser_set_language(parser, tree_sitter_python()); tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len); owns_tree = true; if (!tree) { ts_parser_delete(parser); - return; + return CBM_LSP_COMPLETE; } } TSNode root = ts_tree_root_node(tree); PyLSPContext ctx; - py_lsp_init(&ctx, arena, source, source_len, reg, module_qn, out); + py_lsp_init_for_stage(&ctx, arena, source, source_len, reg, module_qn, out, "cross"); + if (ctx.eval_failure) + goto prebuilt_cleanup; ctx.syn_calls = synthetic_calls; for (int i = 0; i < import_count; i++) { if (import_names && import_qns && import_names[i] && import_qns[i]) { py_lsp_add_import(&ctx, import_names[i], import_qns[i]); } } + py_memo_test_stage(&ctx, "cross"); py_lsp_process_file(&ctx, root); - +prebuilt_cleanup: if (owns_tree && tree) ts_tree_delete(tree); if (parser) ts_parser_delete(parser); + return ctx.eval_failure; } -void cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_count, +bool cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_count, CBMResolvedCallArray *out) { if (!arena || !files || file_count <= 0 || !out) - return; + return true; + bool complete = true; for (int f = 0; f < file_count; f++) { CBMBatchPyLSPFile *file = &files[f]; memset(&out[f], 0, sizeof(CBMResolvedCallArray)); @@ -5191,9 +5812,10 @@ void cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_ CBMResolvedCallArray file_out; memset(&file_out, 0, sizeof(file_out)); - cbm_run_py_lsp_cross(&file_arena, file->source, file->source_len, file->module_qn, - file->defs, file->def_count, file->import_names, file->import_qns, - file->import_count, file->cached_tree, &file_out, NULL); + if (!cbm_run_py_lsp_cross(&file_arena, file->source, file->source_len, file->module_qn, + file->defs, file->def_count, file->import_names, file->import_qns, + file->import_count, file->cached_tree, &file_out, NULL)) + complete = false; if (file_out.count > 0) { out[f].count = file_out.count; @@ -5222,6 +5844,7 @@ void cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_ cbm_arena_destroy(&file_arena); } + return complete; } /* ── Stdlib stub — Phase 10 replaces with auto-generated body ─── */ diff --git a/internal/cbm/lsp/py_lsp.h b/internal/cbm/lsp/py_lsp.h index 939025100..5f63c6ea9 100644 --- a/internal/cbm/lsp/py_lsp.h +++ b/internal/cbm/lsp/py_lsp.h @@ -61,7 +61,13 @@ typedef struct { // caller QNs. const char *enclosing_func_qn; const char *enclosing_class_qn; - const char *module_qn; + // Active class execution: nested class bodies skip this class frame. + // Local classes have no extracted graph QN, so class_body_qn stays NULL. + CBMScope *class_body_outer_scope; + const char *class_body_qn; + bool class_body_in_comprehension; // retain enclosing comprehension bindings + const char *module_qn; // Python symbol scope (package __init__ omitted) + const char *file_module_qn; // raw file identity; borrowed for this API call // Output: resolved calls accumulate here. CBMResolvedCallArray *resolved_calls; @@ -95,14 +101,18 @@ typedef struct { // py_eval_expr_type memoization + guards (issues #710/#720; mirrors // c_eval_expr_type's guard design in c_lsp.c). All zero via memset — - // each file starts with a cold cache and a full budget. + // each file starts with a cold cache. CBMPyTypeCacheEntry *type_cache; // open addressing, linear probe, arena-allocated int type_cache_count; // occupied slots (kept < 75% of cap) int type_cache_cap; // power-of-two capacity uint32_t type_cache_gen; // bumped on every scope mutation (O(1) flush) int eval_depth; // evaluator recursion depth (PY_LSP_MAX_EVAL_DEPTH) - int eval_steps; // per-file work budget used (PY_EVAL_MAX_STEPS_PER_FILE) - uint32_t eval_truncations; // depth/budget cutoff count — gates memo inserts + uint32_t eval_truncations; // depth-cap cutoff count — gates memo inserts + CBMLSPStatus eval_failure; // sticky: explicit failure makes this walk incomplete +#ifdef CBM_ENABLE_TEST_SEAMS + int test_memo_allocations_left; + int test_root_name_failure; // 1 = NULL, 2 = empty; only the root-name collection +#endif // Sticky fail-closed guard for exact callable-value proof. If a required // scope/name/overlay allocation fails, later semantic passes must leave @@ -161,9 +171,34 @@ void cbm_run_py_lsp(CBMArena *arena, CBMFileResult *result, const char *source, // Phase 10 fills the body; Phase 2 ships a no-op so linkage works. void cbm_python_stdlib_register(CBMTypeRegistry *reg, CBMArena *arena); +#define CBM_PY_LSP_DEPTH_ERROR "Python expression recursion depth limit exceeded" +#define CBM_PY_LSP_MEMO_ERROR "Python expression memo allocation or capacity failure" +#define CBM_PY_LSP_SCOPE_ERROR "Python symbol scope allocation failure" +void cbm_py_lsp_record_failure(CBMFileResult *result, CBMLSPStatus status); +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_py_lsp_test_memo_fail_after(int successful_allocations); +void cbm_py_lsp_test_depth_fail(bool enabled); +void cbm_py_lsp_test_scope_fail_after(int successful_allocations); +// Separate from the package symbol-scope seam; only new class execution frames. +void cbm_py_lsp_test_class_frame_fail_after(int successful_allocations); +bool cbm_py_lsp_test_register_defs(CBMArena *arena, CBMTypeRegistry *reg, CBMLSPDef *defs, + int def_count); +const CBMType *cbm_py_lsp_test_eval(PyLSPContext *ctx, TSNode node); +void cbm_py_lsp_test_bind_external_classes(PyLSPContext *ctx, TSNode root); +#endif + // --- Cross-file LSP resolution --- +// Status variants distinguish memo/depth failure; bool wrappers report success. +// Existing no-op/parser exits retain their previous behavior. + +CBMLSPStatus cbm_run_py_lsp_cross_status(CBMArena *arena, const char *source, int source_len, + const char *module_qn, CBMLSPDef *defs, int def_count, + const char **import_names, const char **import_qns, + int import_count, + TSTree *cached_tree, // NULL = parse internally + CBMResolvedCallArray *out, CBMCallArray *synthetic_calls); -void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, +bool cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, CBMLSPDef *defs, int def_count, const char **import_names, const char **import_qns, int import_count, TSTree *cached_tree, // NULL = parse internally @@ -173,7 +208,13 @@ void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, * Filters all_defs to Python entries, builds + finalizes once. */ CBMTypeRegistry *cbm_py_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, int def_count); -void cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, +CBMLSPStatus cbm_run_py_lsp_cross_with_registry_status( + CBMArena *arena, const char *source, int source_len, const char *module_qn, + CBMTypeRegistry *reg, // pre-built, finalized, READ-ONLY + const char **import_names, const char **import_qns, int import_count, TSTree *cached_tree, + CBMResolvedCallArray *out, CBMCallArray *synthetic_calls); + +bool cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int source_len, const char *module_qn, CBMTypeRegistry *reg, // pre-built, finalized, READ-ONLY const char **import_names, const char **import_qns, @@ -194,7 +235,24 @@ typedef struct { int import_count; } CBMBatchPyLSPFile; -void cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_count, +bool cbm_batch_py_lsp_cross(CBMArena *arena, CBMBatchPyLSPFile *files, int file_count, CBMResolvedCallArray *out); +#ifdef CBM_ENABLE_TEST_SEAMS +#include +/* Process-wide comparison counts for deterministic resolver operation tests. + * Only missing exact symbol/type attributes reach this submodule predicate. */ +extern _Atomic uint64_t cbm_py_submodule_probes; +extern _Atomic uint64_t cbm_py_submodule_probe_visits; +/* Complete module operations, including optimization fallbacks. Visits count + * inspected QNs/search comparisons, not sorting or string-length work. */ +extern _Atomic uint64_t cbm_py_module_class_binds; +extern _Atomic uint64_t cbm_py_module_class_bind_visits; +extern _Atomic uint64_t cbm_py_wildcard_invalidations; +extern _Atomic uint64_t cbm_py_wildcard_invalidation_visits; +/* -1 disables; zero fails until reset; positive counts down attempts. */ +void cbm_py_test_module_scratch_fail_after(int successful_allocations); +void cbm_py_test_module_prefix_fail_after(int successful_allocations); +#endif + #endif // CBM_LSP_PY_LSP_H diff --git a/internal/cbm/lsp/scope.c b/internal/cbm/lsp/scope.c index 2d9740db0..0b3c6505c 100644 --- a/internal/cbm/lsp/scope.c +++ b/internal/cbm/lsp/scope.c @@ -1,5 +1,21 @@ #include "scope.h" +#include "lsp_work.h" #include +#include + +#ifdef CBM_ENABLE_TEST_SEAMS +_Thread_local uint64_t cbm_lsp_work_steps = 0; +uint64_t cbm_lsp_work_take(void) { + uint64_t v = cbm_lsp_work_steps; + cbm_lsp_work_steps = 0; + return v; +} +#endif + +enum { + SCOPE_INDEX_MIN_CAP = 64, /* first table: > 2x one chunk, a power of two */ + SCOPE_INDEX_GROW = 2, +}; CBMScope* cbm_scope_push(CBMArena* a, CBMScope* current) { CBMScope* scope = (CBMScope*)cbm_arena_alloc(a, sizeof(CBMScope)); @@ -33,6 +49,114 @@ static CBMScopeChunk* alloc_chunk(CBMScope* scope) { return c; } +static uint32_t scope_name_hash(const char *name) { + uint32_t h = 2166136261u; /* FNV-1a */ + for (const unsigned char *p = (const unsigned char *)name; *p; p++) { + h ^= *p; + h *= 16777619u; + } + return h; +} + +/* Place one binding into a table that has room (load kept below 1/2). */ +static void scope_index_place(CBMVarBinding **slots, int cap, CBMVarBinding *b) { + uint32_t mask = (uint32_t)cap - 1; + uint32_t i = scope_name_hash(b->name) & mask; + while (slots[i]) { + i = (i + 1) & mask; + } + slots[i] = b; +} + +/* (Re)build the frame's table at cap slots from its chunks. On allocation + * failure the frame drops its index and stays on the always-correct scan. */ +static void scope_index_rebuild(CBMScope *scope, int cap) { + /* Recovery can start with many bindings and no index. Always size from + * the complete frame, never just from the previous table's capacity. */ + if (scope->binding_count < 0 || scope->binding_count > INT_MAX / 2) { + memset(&scope->index, 0, sizeof(scope->index)); + return; + } + if (cap < SCOPE_INDEX_MIN_CAP) + cap = SCOPE_INDEX_MIN_CAP; + while (cap < scope->binding_count * 2) { + if (cap > INT_MAX / SCOPE_INDEX_GROW) { + memset(&scope->index, 0, sizeof(scope->index)); + return; + } + cap *= SCOPE_INDEX_GROW; + } + if ((size_t)cap > SIZE_MAX / sizeof(CBMVarBinding *)) { + memset(&scope->index, 0, sizeof(scope->index)); + return; + } + CBMVarBinding **slots = + scope->arena + ? (CBMVarBinding **)cbm_arena_alloc(scope->arena, (size_t)cap * sizeof(CBMVarBinding *)) + : NULL; + if (!slots) { + memset(&scope->index, 0, sizeof(scope->index)); + return; + } + memset(slots, 0, (size_t)cap * sizeof(CBMVarBinding *)); + int count = 0; + for (CBMScopeChunk *c = scope->chunks; c != NULL; c = c->next) { + for (int i = 0; i < c->used; i++) { + if (c->bindings[i].name) { + scope_index_place(slots, cap, &c->bindings[i]); + count++; + } + } + } + scope->index.slots = slots; + scope->index.cap = cap; + scope->index.count = count; +} + +/* Record a freshly appended binding in the frame's index, building the index + * the first time the frame outgrows one chunk. */ +static void scope_index_note(CBMScope *scope, CBMVarBinding *b) { + if (scope->index.cap == 0) { + if (scope->binding_count > CBM_SCOPE_CHUNK_BINDINGS) { + scope_index_rebuild(scope, SCOPE_INDEX_MIN_CAP); + } + return; + } + if (scope->index.count >= scope->index.cap / 2) { + scope_index_rebuild(scope, scope->index.cap); /* includes b; sizes from binding_count */ + return; + } + scope_index_place(scope->index.slots, scope->index.cap, b); + scope->index.count++; +} + +/* The binding of name in one frame: hashed when the frame has an index, + * otherwise the linear chunk scan (small frames, or no index memory). */ +static CBMVarBinding *scope_frame_find(const CBMScope *scope, const char *name) { + if (scope->index.cap > 0) { + uint32_t mask = (uint32_t)scope->index.cap - 1; + for (uint32_t i = scope_name_hash(name) & mask;; i = (i + 1) & mask) { + CBMVarBinding *b = scope->index.slots[i]; + if (!b) { + return NULL; + } + CBM_LSP_WORK(1); + if (strcmp(b->name, name) == 0) { + return b; + } + } + } + for (CBMScopeChunk *c = scope->chunks; c != NULL; c = c->next) { + for (int i = 0; i < c->used; i++) { + CBM_LSP_WORK(1); + if (c->bindings[i].name && strcmp(c->bindings[i].name, name) == 0) { + return &c->bindings[i]; + } + } + } + return NULL; +} + /* Returns false when the binding could NOT be recorded in THIS frame. * * The failure that matters is arena exhaustion in alloc_chunk: the old void @@ -47,14 +171,11 @@ static bool cbm_scope_bind_value(CBMScope *scope, const char *name, const CBMTyp if (!scope || !name) { return false; } - for (CBMScopeChunk* c = scope->chunks; c != NULL; c = c->next) { - for (int i = 0; i < c->used; i++) { - if (c->bindings[i].name && strcmp(c->bindings[i].name, name) == 0) { - c->bindings[i].type = type; - c->bindings[i].callable_qn = callable_qn; - return true; - } - } + CBMVarBinding *existing = scope_frame_find(scope, name); + if (existing) { + existing->type = type; + existing->callable_qn = callable_qn; + return true; } CBMScopeChunk* head = scope->chunks; if (!head || head->used >= CBM_SCOPE_CHUNK_BINDINGS) { @@ -63,10 +184,13 @@ static bool cbm_scope_bind_value(CBMScope *scope, const char *name, const CBMTyp return false; /* arena exhausted: the shadow did NOT take effect */ } } - head->bindings[head->used].name = name; - head->bindings[head->used].type = type; - head->bindings[head->used].callable_qn = callable_qn; + CBMVarBinding *b = &head->bindings[head->used]; + b->name = name; + b->type = type; + b->callable_qn = callable_qn; head->used++; + scope->binding_count++; + scope_index_note(scope, b); return true; } @@ -88,6 +212,13 @@ bool cbm_scope_bind_callable_checked(CBMScope *scope, const char *name, const CB return cbm_scope_bind_value(scope, name, type, callable_qn); } +const CBMVarBinding *cbm_scope_lookup_local(const CBMScope *scope, const char *name) { + if (!scope || !name) { + return NULL; + } + return scope_frame_find(scope, name); +} + const CBMType* cbm_scope_lookup(const CBMScope* scope, const char* name) { const CBMVarBinding *binding = cbm_scope_lookup_binding(scope, name); if (binding) @@ -109,13 +240,10 @@ bool cbm_scope_update_callable(CBMScope *scope, const char *name, const char *ca return false; } for (CBMScope *s = scope; s != NULL; s = s->parent) { - for (CBMScopeChunk *c = s->chunks; c != NULL; c = c->next) { - for (int i = 0; i < c->used; i++) { - if (c->bindings[i].name && strcmp(c->bindings[i].name, name) == 0) { - c->bindings[i].callable_qn = callable_qn; - return true; - } - } + CBMVarBinding *b = scope_frame_find(s, name); + if (b) { + b->callable_qn = callable_qn; + return true; } } return false; diff --git a/internal/cbm/lsp/scope.h b/internal/cbm/lsp/scope.h index 5ba7265a3..af14cb7cd 100644 --- a/internal/cbm/lsp/scope.h +++ b/internal/cbm/lsp/scope.h @@ -25,27 +25,48 @@ typedef struct CBMScopeChunk { struct CBMScopeChunk* next; } CBMScopeChunk; +/* Name index for one frame (#1527). A frame that outgrows one chunk (a module + * scope binding thousands of classes and functions) is looked up through an + * open-addressing table of binding pointers instead of a linear chunk scan -- + * the scan made every identifier lookup O(frame size) and a whole file + * O(n^2). Bindings never move (chunks are arena-allocated and only + * prepended) and a name is bound at most once per frame (rebinding updates + * in place), so the table maps each name to exactly the binding the linear + * scan would have found. The table is an accelerator only: when it cannot be + * allocated or grown the frame drops it and every lookup falls back to the + * scan, which is always correct. */ +typedef struct { + CBMVarBinding **slots; // arena-owned; NULL = empty + int cap; // power of two, or 0 = no index (linear scan) + int count; +} CBMScopeIndex; + typedef struct CBMScope { struct CBMScope* parent; CBMScopeChunk* chunks; CBMArena* arena; // owning arena, propagated to children at push time + int binding_count; // bindings recorded in THIS frame + CBMScopeIndex index; // built once binding_count exceeds one chunk } CBMScope; +/* The binding of name in THIS frame only (no parent walk), or NULL. */ +const CBMVarBinding *cbm_scope_lookup_local(const CBMScope *scope, const char *name); + /* Return the complete nearest binding in one scope-chain walk, or NULL when * unbound. Keep this internal hot-path primitive inline: all language * resolvers use cbm_scope_lookup(), while Python also consumes the complete - * record to avoid repeating the same linear scan. */ + * record to avoid repeating the same walk. Each frame is probed through + * cbm_scope_lookup_local(), which uses the frame's hashed name index once the + * frame outgrows one chunk (#1527), so a lookup costs one probe per frame + * rather than a scan of every binding in scope. */ static inline const CBMVarBinding *cbm_scope_lookup_binding(const CBMScope *scope, const char *name) { if (!name) return NULL; for (const CBMScope *s = scope; s != NULL; s = s->parent) { - for (const CBMScopeChunk *c = s->chunks; c != NULL; c = c->next) { - for (int i = 0; i < c->used; i++) { - if (c->bindings[i].name && strcmp(c->bindings[i].name, name) == 0) - return &c->bindings[i]; - } - } + const CBMVarBinding *b = cbm_scope_lookup_local(s, name); + if (b) + return b; } return NULL; } diff --git a/internal/cbm/lsp/ts_lsp.c b/internal/cbm/lsp/ts_lsp.c index 7fe5055fd..a78c7433d 100644 --- a/internal/cbm/lsp/ts_lsp.c +++ b/internal/cbm/lsp/ts_lsp.c @@ -1391,12 +1391,8 @@ static const CBMScope *ts_scope_binding_owner(const CBMScope *scope, const char return NULL; } for (const CBMScope *s = scope; s; s = s->parent) { - for (const CBMScopeChunk *chunk = s->chunks; chunk; chunk = chunk->next) { - for (int i = 0; i < chunk->used; i++) { - if (chunk->bindings[i].name && strcmp(chunk->bindings[i].name, name) == 0) { - return s; - } - } + if (cbm_scope_lookup_local(s, name)) { + return s; } } return NULL; diff --git a/internal/cbm/lsp/type_registry.c b/internal/cbm/lsp/type_registry.c index 4e2f02012..e37277842 100644 --- a/internal/cbm/lsp/type_registry.c +++ b/internal/cbm/lsp/type_registry.c @@ -1,4 +1,5 @@ #include "type_registry.h" +#include "lsp_work.h" #include #include @@ -56,7 +57,12 @@ static void build_qn_index(CBMTypeRegistry *reg, CBMArena *idx_arena, bool for_f for (int i = 0; i < bucket_count; i++) buckets[i] = -1; - for (int i = 0; i < count; i++) { + /* Chains are built by prepending, so the entry inserted LAST is found + * first. Insert descending when the registry asked for first-registered + * answers (see index_first_registered). */ + bool first = reg->index_first_registered; + for (int k = 0; k < count; k++) { + int i = first ? count - 1 - k : k; const char *qn = for_funcs ? reg->funcs[i].qualified_name : reg->types[i].qualified_name; if (!qn) continue; @@ -96,7 +102,9 @@ static void build_method_index(CBMTypeRegistry *reg, CBMArena *idx_arena) { buckets[i] = -1; int idx = 0; - for (int i = 0; i < reg->func_count; i++) { + bool first = reg->index_first_registered; /* see build_qn_index */ + for (int k = 0; k < reg->func_count; k++) { + int i = first ? reg->func_count - 1 - k : k; const CBMRegisteredFunc *f = ®->funcs[i]; if (!f->receiver_type || !f->short_name) continue; @@ -677,8 +685,28 @@ CBMRegisteredType *cbm_registry_type_for_update(CBMTypeRegistry *head, const cha for (const CBMTypeRegistry *r = head->fallback; r; r = r->fallback) { const CBMRegisteredType *base = lookup_type_self(r, qualified_name); if (base) { + CBMRegisteredType copy = *base; + /* Field refinements overwrite entries in this array. The type + * objects remain immutable, but the writable overlay must own + * the pointer array rather than mutate its shared fallback. */ + if (base->field_types) { + size_t count = 0; + /* Types are parallel to names; a named field may have no + * type, so an interior NULL does not terminate that array. */ + while (base->field_names ? base->field_names[count] != NULL + : base->field_types[count] != NULL) { + count++; + } + const CBMType **field_types = (const CBMType **)cbm_arena_alloc( + head->arena, (count + 1) * sizeof(*field_types)); + if (!field_types) { + return NULL; + } + memcpy(field_types, base->field_types, (count + 1) * sizeof(*field_types)); + copy.field_types = field_types; + } int before = head->type_count; - cbm_registry_add_type(head, *base); + cbm_registry_add_type(head, copy); if (head->type_count == before + 1) { return &head->types[before]; } @@ -759,6 +787,7 @@ static const CBMRegisteredFunc *lookup_method_self(const CBMTypeRegistry *reg, int slot = (int)(h & (uint64_t)(reg->method_bucket_count - 1)); for (int idx = reg->method_buckets[slot]; idx >= 0; idx = reg->method_entries[idx].next_index) { + CBM_LSP_WORK(1); if (reg->method_entries[idx].hash != h) continue; const CBMRegisteredFunc *f = ®->funcs[reg->method_entries[idx].payload_index]; @@ -772,6 +801,7 @@ static const CBMRegisteredFunc *lookup_method_self(const CBMTypeRegistry *reg, * post-finalize additions stay visible instead of silently vanishing. */ for (int i = reg->func_qn_entry_count; i < reg->func_count; i++) { const CBMRegisteredFunc *f = ®->funcs[i]; + CBM_LSP_WORK(1); if (f->receiver_type && f->short_name && strcmp(f->receiver_type, receiver_qn) == 0 && strcmp(f->short_name, method_name) == 0) { return f; @@ -782,6 +812,7 @@ static const CBMRegisteredFunc *lookup_method_self(const CBMTypeRegistry *reg, for (int i = 0; i < reg->func_count; i++) { const CBMRegisteredFunc *f = ®->funcs[i]; + CBM_LSP_WORK(1); if (f->receiver_type && f->short_name && strcmp(f->receiver_type, receiver_qn) == 0 && strcmp(f->short_name, method_name) == 0) { return f; @@ -809,6 +840,7 @@ static const CBMRegisteredType *lookup_type_self(const CBMTypeRegistry *reg, int slot = (int)(h & (uint64_t)(reg->type_qn_bucket_count - 1)); for (int idx = reg->type_qn_buckets[slot]; idx >= 0; idx = reg->type_qn_entries[idx].next_index) { + CBM_LSP_WORK(1); if (reg->type_qn_entries[idx].hash != h) continue; int p = reg->type_qn_entries[idx].payload_index; @@ -819,6 +851,7 @@ static const CBMRegisteredType *lookup_type_self(const CBMTypeRegistry *reg, } /* Tail-scan types added after finalize (see lookup_method_self). */ for (int i = reg->type_qn_entry_count; i < reg->type_count; i++) { + CBM_LSP_WORK(1); if (reg->types[i].qualified_name && strcmp(reg->types[i].qualified_name, qualified_name) == 0) { return ®->types[i]; @@ -828,6 +861,7 @@ static const CBMRegisteredType *lookup_type_self(const CBMTypeRegistry *reg, } for (int i = 0; i < reg->type_count; i++) { + CBM_LSP_WORK(1); if (strcmp(reg->types[i].qualified_name, qualified_name) == 0) { return ®->types[i]; } @@ -853,6 +887,7 @@ static const CBMRegisteredFunc *lookup_func_self(const CBMTypeRegistry *reg, int slot = (int)(h & (uint64_t)(reg->func_qn_bucket_count - 1)); for (int idx = reg->func_qn_buckets[slot]; idx >= 0; idx = reg->func_qn_entries[idx].next_index) { + CBM_LSP_WORK(1); if (reg->func_qn_entries[idx].hash != h) continue; int p = reg->func_qn_entries[idx].payload_index; @@ -863,6 +898,7 @@ static const CBMRegisteredFunc *lookup_func_self(const CBMTypeRegistry *reg, } /* Tail-scan funcs added after finalize (see lookup_method_self). */ for (int i = reg->func_qn_entry_count; i < reg->func_count; i++) { + CBM_LSP_WORK(1); if (reg->funcs[i].qualified_name && strcmp(reg->funcs[i].qualified_name, qualified_name) == 0) { return ®->funcs[i]; @@ -872,6 +908,7 @@ static const CBMRegisteredFunc *lookup_func_self(const CBMTypeRegistry *reg, } for (int i = 0; i < reg->func_count; i++) { + CBM_LSP_WORK(1); if (strcmp(reg->funcs[i].qualified_name, qualified_name) == 0) { return ®->funcs[i]; } @@ -948,6 +985,299 @@ const CBMRegisteredFunc *cbm_registry_lookup_symbol(const CBMTypeRegistry *reg, return cbm_registry_lookup_func(reg, buf); } +/* A stable total order, independent of qsort's treatment of equal keys. */ +static int registry_qn_entry_cmp(const void *left, const void *right) { + const CBMRegistryQNEntry *a = left; + const CBMRegistryQNEntry *b = right; + int cmp = strcmp(a->qualified_name, b->qualified_name); + return cmp ? cmp : (a->func_index > b->func_index) - (a->func_index < b->func_index); +} + +#ifdef CBM_ENABLE_TEST_SEAMS +static _Thread_local int registry_qn_order_allocations_left = -1; +void cbm_registry_test_qn_order_fail_after(int successful_allocations) { + registry_qn_order_allocations_left = successful_allocations; +} +#endif + +void cbm_registry_build_func_qn_order(CBMTypeRegistry *reg, CBMArena *arena) { + if (!reg || !arena || reg->read_only) + return; + /* A failed rebuild must not leave an old snapshot authoritative. */ + reg->func_qn_sorted = NULL; + reg->func_qn_sorted_count = 0; + reg->func_qn_sorted_upto = 0; + if (reg->func_count <= 0 || + (size_t)reg->func_count > (SIZE_MAX - 7) / sizeof(CBMRegistryQNEntry)) + return; +#ifdef CBM_ENABLE_TEST_SEAMS + if (registry_qn_order_allocations_left == 0) + return; + if (registry_qn_order_allocations_left > 0) + registry_qn_order_allocations_left--; +#endif + CBMRegistryQNEntry *sorted = cbm_arena_alloc(arena, (size_t)reg->func_count * sizeof(*sorted)); + if (!sorted) + return; + int count = 0; + for (int i = 0; i < reg->func_count; i++) { + if (reg->funcs[i].qualified_name) { + sorted[count].qualified_name = reg->funcs[i].qualified_name; + sorted[count].func_index = i; + count++; + } + } + qsort(sorted, (size_t)count, sizeof(*sorted), registry_qn_entry_cmp); + reg->func_qn_sorted = sorted; + reg->func_qn_sorted_count = count; + reg->func_qn_sorted_upto = reg->func_count; +} + +static bool registry_link_has_func_qn_prefix(const CBMTypeRegistry *reg, const char *prefix, + size_t length, uint64_t *visits) { + int scan_from = 0; + if (reg->func_qn_sorted && reg->func_qn_sorted_upto <= reg->func_count) { + int lo = 0, hi = reg->func_qn_sorted_count; + while (lo < hi) { + int mid = lo + (hi - lo) / 2; + if (visits) + (*visits)++; + if (strcmp(reg->func_qn_sorted[mid].qualified_name, prefix) < 0) + lo = mid + 1; + else + hi = mid; + } + if (lo < reg->func_qn_sorted_count) { + if (visits) + (*visits)++; + if (strncmp(reg->func_qn_sorted[lo].qualified_name, prefix, length) == 0) + return true; + } + scan_from = reg->func_qn_sorted_upto; + } + for (int i = scan_from; i < reg->func_count; i++) { + if (visits) + (*visits)++; + const char *qn = reg->funcs[i].qualified_name; + if (qn && strncmp(qn, prefix, length) == 0) + return true; + } + return false; +} + +bool cbm_registry_has_func_qn_prefix(const CBMTypeRegistry *reg, const char *prefix, + uint64_t *visits) { + if (!prefix) + return false; + size_t length = strlen(prefix); + /* The old iterator omits shadowed fallback QNs, whose equal head QN has + * the same prefix answer. This boolean union needs no winner selection. */ + for (const CBMTypeRegistry *link = reg; link; link = link->fallback) { + if (registry_link_has_func_qn_prefix(link, prefix, length, visits)) + return true; + } + return false; +} + +static int registry_type_qn_entry_cmp(const void *left, const void *right) { + const CBMRegistryTypeQNEntry *a = left; + const CBMRegistryTypeQNEntry *b = right; + int cmp = strcmp(a->qualified_name, b->qualified_name); + return cmp ? cmp : (a->type_index > b->type_index) - (a->type_index < b->type_index); +} + +#ifdef CBM_ENABLE_TEST_SEAMS +static _Thread_local int registry_type_qn_order_allocations_left = -1; +static _Thread_local int registry_qn_prefix_block_allocations_left = -1; +void cbm_registry_test_type_qn_order_fail_after(int successful_allocations) { + registry_type_qn_order_allocations_left = successful_allocations; +} +void cbm_registry_test_qn_prefix_block_fail_after(int successful_allocations) { + registry_qn_prefix_block_allocations_left = successful_allocations; +} +#endif + +void cbm_registry_build_type_qn_order(CBMTypeRegistry *reg, CBMArena *arena) { + if (!reg || !arena || reg->read_only) + return; + reg->type_qn_sorted = NULL; + reg->type_qn_sorted_count = 0; + reg->type_qn_sorted_upto = 0; + if (!reg->types || reg->type_count <= 0 || + (size_t)reg->type_count > (SIZE_MAX - 7) / sizeof(CBMRegistryTypeQNEntry)) + return; +#ifdef CBM_ENABLE_TEST_SEAMS + if (registry_type_qn_order_allocations_left == 0) + return; + if (registry_type_qn_order_allocations_left > 0) + registry_type_qn_order_allocations_left--; +#endif + CBMRegistryTypeQNEntry *sorted = + cbm_arena_alloc(arena, (size_t)reg->type_count * sizeof(*sorted)); + if (!sorted) + return; + int count = 0; + for (int i = 0; i < reg->type_count; i++) { + if (reg->types[i].qualified_name) { + sorted[count].qualified_name = reg->types[i].qualified_name; + sorted[count].type_index = i; + count++; + } + } + qsort(sorted, (size_t)count, sizeof(*sorted), registry_type_qn_entry_cmp); + reg->type_qn_sorted = sorted; + reg->type_qn_sorted_count = count; + reg->type_qn_sorted_upto = reg->type_count; +} + +bool cbm_registry_type_shadowed(const CBMTypeRegistry *head, const CBMTypeRegistry *reg, + int index) { + return reg && reg->types && index >= 0 && index < reg->type_count && + type_shadowed(head, reg, index); +} + +static const char *registry_prefix_sorted_qn(const CBMQNPrefixIter *it, int index) { + return it->types ? it->reg->type_qn_sorted[index].qualified_name + : it->reg->func_qn_sorted[index].qualified_name; +} + +static int registry_prefix_sorted_index(const CBMQNPrefixIter *it, int index) { + return it->types ? it->reg->type_qn_sorted[index].type_index + : it->reg->func_qn_sorted[index].func_index; +} + +static int registry_prefix_index_cmp(const void *left, const void *right) { + int a = *(const int *)left, b = *(const int *)right; + return (a > b) - (a < b); +} + +/* Count/fill the exact range. For a direct query, jump over dotted subtrees + * with a binary upper bound on the prefix INCLUDING its separating dot. The + * undotted siblings A-b and AB must remain visible around A.child. */ +static int registry_prefix_collect(CBMQNPrefixIter *it, int start, int sorted_count, int covered, + int *indices, int capacity) { + int count = 0; + for (int pos = start; pos < sorted_count;) { + const char *qn = registry_prefix_sorted_qn(it, pos); + it->visits++; + if (!qn) + return -1; + if (strncmp(qn, it->prefix, it->prefix_len) != 0) + break; + const char *dot = it->direct_only ? strchr(qn + it->prefix_len, '.') : NULL; + if (dot) { + size_t subtree_len = (size_t)(dot - qn) + 1; + int lo = pos + 1, hi = sorted_count; + while (lo < hi) { + int mid = lo + (hi - lo) / 2; + const char *candidate = registry_prefix_sorted_qn(it, mid); + it->visits++; + if (!candidate) + return -1; + if (strncmp(candidate, qn, subtree_len) <= 0) + lo = mid + 1; + else + hi = mid; + } + pos = lo; + continue; + } + int index = registry_prefix_sorted_index(it, pos++); + if (index < 0 || index >= covered || (indices && count >= capacity)) + return -1; + if (indices) + indices[count] = index; + count++; + } + return count; +} + +static bool registry_prefix_open(const CBMTypeRegistry *reg, const char *prefix, bool direct_only, + bool types, CBMArena *scratch, CBMQNPrefixIter *out) { + if (!out) + return false; + memset(out, 0, sizeof(*out)); + out->reg = reg; + out->prefix = prefix; + out->types = types; + out->direct_only = direct_only; + if (!reg || !prefix || (types ? !reg->types : !reg->funcs)) + return true; + out->prefix_len = strlen(prefix); + int total = types ? reg->type_count : reg->func_count; + out->tail_end = total > 0 ? total : 0; + int covered = types ? reg->type_qn_sorted_upto : reg->func_qn_sorted_upto; + int sorted_count = types ? reg->type_qn_sorted_count : reg->func_qn_sorted_count; + bool has_order = types ? reg->type_qn_sorted != NULL : reg->func_qn_sorted != NULL; + if (!has_order || covered < 0 || covered > total || sorted_count < 0 || sorted_count > covered) + return true; /* initialized full-link fallback */ + + int lo = 0, hi = sorted_count; + while (lo < hi) { + int mid = lo + (hi - lo) / 2; + const char *qn = registry_prefix_sorted_qn(out, mid); + out->visits++; + if (!qn) + return true; + if (strcmp(qn, prefix) < 0) + lo = mid + 1; + else + hi = mid; + } + int count = registry_prefix_collect(out, lo, sorted_count, covered, NULL, 0); + if (count < 0) + return true; + if (count == 0) { + out->tail_i = covered; + return true; + } + if (!scratch || (size_t)count > (SIZE_MAX - 7) / sizeof(int)) + return true; +#ifdef CBM_ENABLE_TEST_SEAMS + if (registry_qn_prefix_block_allocations_left == 0) + return true; + if (registry_qn_prefix_block_allocations_left > 0) + registry_qn_prefix_block_allocations_left--; +#endif + int *indices = cbm_arena_alloc(scratch, (size_t)count * sizeof(*indices)); + if (!indices || + registry_prefix_collect(out, lo, sorted_count, covered, indices, count) != count) + return true; + qsort(indices, (size_t)count, sizeof(*indices), registry_prefix_index_cmp); + /* Publish only a complete block; every earlier return scans from zero. */ + out->indices = indices; + out->count = count; + out->tail_i = covered; + return true; +} + +bool cbm_registry_types_with_qn_prefix(const CBMTypeRegistry *reg, const char *prefix, + bool direct_only, CBMArena *scratch, CBMQNPrefixIter *out) { + return registry_prefix_open(reg, prefix, direct_only, true, scratch, out); +} + +bool cbm_registry_funcs_with_qn_prefix(const CBMTypeRegistry *reg, const char *prefix, + bool direct_only, CBMArena *scratch, CBMQNPrefixIter *out) { + return registry_prefix_open(reg, prefix, direct_only, false, scratch, out); +} + +int cbm_qn_prefix_iter_next(CBMQNPrefixIter *it) { + if (!it || !it->reg || !it->prefix) + return -1; + if (it->pos < it->count) + return it->indices[it->pos++]; + while (it->tail_i < it->tail_end) { + int index = it->tail_i++; + const char *qn = + it->types ? it->reg->types[index].qualified_name : it->reg->funcs[index].qualified_name; + it->visits++; + if (qn && strncmp(qn, it->prefix, it->prefix_len) == 0 && + (!it->direct_only || !strchr(qn + it->prefix_len, '.'))) + return index; + } + return -1; +} + // Count parameters in a FUNC signature. static int count_func_params(const CBMRegisteredFunc *f) { if (!f || !f->signature || f->signature->kind != CBM_TYPE_FUNC) diff --git a/internal/cbm/lsp/type_registry.h b/internal/cbm/lsp/type_registry.h index c752350b1..954e2f0df 100644 --- a/internal/cbm/lsp/type_registry.h +++ b/internal/cbm/lsp/type_registry.h @@ -74,6 +74,18 @@ typedef struct { int slot; // bucket slot this entry sits in (for resize) } CBMRegistryHashEntry; +/* A borrowed immutable QN and stable array index, never a movable func pointer. */ +typedef struct { + const char *qualified_name; + int func_index; +} CBMRegistryQNEntry; + +/* Type identities use their own representation; func_index remains unchanged. */ +typedef struct { + const char *qualified_name; + int type_index; +} CBMRegistryTypeQNEntry; + // Cross-file type/function registry. typedef struct CBMTypeRegistry { CBMRegisteredFunc *funcs; @@ -143,6 +155,23 @@ typedef struct CBMTypeRegistry { * every lookup linear-scans it -> O(files*defs) (the Linux-kernel full-index * hang) plus a heap data race across workers. */ bool read_only; + // Set before finalize by a caller whose lookups used to run unfinalized + // (#1527): the exact-QN and method indexes then chain entries in + // registration order, so a duplicated name resolves to its FIRST + // registration -- the answer the linear scan gave. Without it the + // hashed chains answer with the LAST registration (the Tier-2 behavior, + // left unchanged). + bool index_first_registered; + + /* Additional prefix/duplicate index; exact lookup order is unchanged. + * Covers non-NULL QNs in funcs[0..func_qn_sorted_upto), ordered by (QN,index). + * Appended functions remain visible through a tail scan. */ + CBMRegistryQNEntry *func_qn_sorted; + int func_qn_sorted_count; + int func_qn_sorted_upto; + CBMRegistryTypeQNEntry *type_qn_sorted; + int type_qn_sorted_count; + int type_qn_sorted_upto; } CBMTypeRegistry; // Initialize a registry. @@ -191,6 +220,65 @@ const CBMRegisteredFunc *cbm_registry_lookup_func(const CBMTypeRegistry *reg, const CBMRegisteredFunc *cbm_registry_lookup_symbol(const CBMTypeRegistry *reg, const char *package_qn, const char *name); +/* Build in an arena that outlives every use (owner arena for shared registries, + * walk scratch for per-file registries). QN strings and registered identities + * must remain immutable while a snapshot is used; end its use before changing + * identities, then rebuild before querying again. Appends do not invalidate it. + * Never frees caller arenas. Sealed registries are unchanged. A failed rebuild + * discards the old order and queries scan the complete registry instead. */ +void cbm_registry_build_func_qn_order(CBMTypeRegistry *reg, CBMArena *arena); + +/* Boolean starts-with over all function/method QNs in the fallback chain. + * Does not allocate or mutate. NULL prefix is false; empty prefix matches any + * non-NULL QN. Optional visits accumulates compared entries, including tail + * scans; fully indexed links use a lower bound plus at most one candidate. */ +bool cbm_registry_has_func_qn_prefix(const CBMTypeRegistry *reg, const char *prefix, + uint64_t *visits); +#ifdef CBM_ENABLE_TEST_SEAMS +/* -1 disables; zero fails this allocation until reset; positive counts down. */ +void cbm_registry_test_qn_order_fail_after(int successful_allocations); +#endif + +/* Optional type order, with the same ownership/immutability/rebuild contract as + * the function order. Build shared orders before sealing; queries never build. */ +void cbm_registry_build_type_qn_order(CBMTypeRegistry *reg, CBMArena *arena); + +/* Single-link literal-prefix query. Results retain original array order, + * including duplicate QNs, followed by the captured unindexed tail. Registry + * identities and the borrowed prefix must stay alive and unchanged while open; + * no concurrent registry mutation. No movable element pointers are retained. + * direct_only excludes a dot anywhere after the prefix (an empty suffix matches). + * With a non-NULL out, open always succeeds: missing inputs are empty; missing + * scratch, an invalid snapshot or optimization OOM selects the full linear scan. + * NULL out returns false; next(NULL) returns -1. Caller owns/destroys scratch. + * visits includes compared/inspected QNs in search, count, fill and tail passes, + * but excludes string-length work and sorting the original-index block. */ +typedef struct { + const CBMTypeRegistry *reg; + const char *prefix; + size_t prefix_len; + bool types; + bool direct_only; + int *indices; + int count; + int pos; + int tail_i; + int tail_end; + uint64_t visits; +} CBMQNPrefixIter; + +bool cbm_registry_types_with_qn_prefix(const CBMTypeRegistry *reg, const char *prefix, + bool direct_only, CBMArena *scratch, CBMQNPrefixIter *out); +bool cbm_registry_funcs_with_qn_prefix(const CBMTypeRegistry *reg, const char *prefix, + bool direct_only, CBMArena *scratch, CBMQNPrefixIter *out); +int cbm_qn_prefix_iter_next(CBMQNPrefixIter *it); +/* Preserve the existing chain iterator's original-head-only type shadow rule. */ +bool cbm_registry_type_shadowed(const CBMTypeRegistry *head, const CBMTypeRegistry *reg, int index); +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_registry_test_type_qn_order_fail_after(int successful_allocations); +void cbm_registry_test_qn_prefix_block_fail_after(int successful_allocations); +#endif + // Resolve type alias chain: follow alias_of until concrete type found (max 16 levels). const CBMRegisteredType *cbm_registry_resolve_alias(const CBMTypeRegistry *reg, const char *type_qn); diff --git a/internal/cbm/result_compact.c b/internal/cbm/result_compact.c index 90c8c6d82..9bb92fa72 100644 --- a/internal/cbm/result_compact.c +++ b/internal/cbm/result_compact.c @@ -299,6 +299,54 @@ static void cr_walk_call(cr_ctx_t *c, CBMCall *call) { } } +/* Facts have counted lists, not sentinel arrays. Bail before touching a failed + * array copy so no string rewrite can land in the original result's storage. */ +static bool cr_py_size(cr_ctx_t *c, size_t size) { + if (c->failed) + return false; + if (c->phase == CR_MEASURE && + (size > SIZE_MAX - CR_ALIGN || c->bytes > SIZE_MAX - cr_aligned(size))) { + c->failed = true; + return false; + } + return true; +} + +static bool cr_py_string(cr_ctx_t *c, const char **field) { + if (!*field) + return true; + if (c->phase == CR_COUNT && c->refs == SIZE_MAX) { + c->failed = true; + return false; + } + if (c->phase == CR_MEASURE && !cr_py_size(c, strlen(*field) + 1)) + return false; + cr_str(c, field); + return !c->failed; +} + +static void cr_walk_py_namespace(cr_ctx_t *c, CBMPyNamespaceFacts *facts) { + if (!cr_py_size(c, (size_t)facts->count * sizeof(*facts->items))) + return; + cr_array(c, (void **)&facts->items, facts->count, sizeof(*facts->items)); + if (c->failed) + return; + for (int i = 0; i < facts->count; i++) { + CBMPyNamespaceFact *f = &facts->items[i]; + if (!cr_py_string(c, &f->local_name) || !cr_py_string(c, &f->module_name) || + !cr_py_string(c, &f->member_name) || + !cr_py_size(c, (size_t)f->name_count * sizeof(*f->names))) + return; + cr_array(c, (void **)&f->names, f->name_count, sizeof(*f->names)); + if (c->failed) + return; + for (int j = 0; j < f->name_count; j++) { + if (!cr_py_string(c, &f->names[j])) + return; + } + } +} + static void cr_walk(cr_ctx_t *c, CBMFileResult *r) { cr_array(c, (void **)&r->defs.items, r->defs.count, sizeof(CBMDefinition)); for (int i = 0; i < r->defs.count && r->defs.items; i++) { @@ -377,6 +425,12 @@ static void cr_walk(cr_ctx_t *c, CBMFileResult *r) { cr_str(c, &r->channels.items[i].transport); cr_str(c, &r->channels.items[i].enclosing_func_qn); } + cr_array(c, (void **)&r->field_types.items, r->field_types.count, sizeof(CBMFieldType)); + for (int i = 0; i < r->field_types.count && r->field_types.items; i++) { + cr_str(c, &r->field_types.items[i].class_qn); + cr_str(c, &r->field_types.items[i].field_name); + cr_str(c, &r->field_types.items[i].type_text); + } cr_str(c, &r->module_qn); cr_str(c, &r->namespace_name); cr_list(c, &r->exports); @@ -386,6 +440,7 @@ static void cr_walk(cr_ctx_t *c, CBMFileResult *r) { cr_str(c, &r->error_msg); cr_str(c, &r->error_ranges); cr_blob(c, (const void **)&r->source, r->source ? (size_t)r->source_len + SKIP_ONE : 0); + cr_walk_py_namespace(c, &r->py_namespace); } void cbm_result_relocate(CBMFileResult *result, const char *old_base, size_t len, char *new_base) { @@ -468,7 +523,8 @@ static void cr_scratch_put(cr_ctx_t *c) { } void cbm_result_compact(CBMFileResult *result) { - if (!result || result->arena.nblocks == 0) { + if (!result || result->arena.nblocks == 0 || + !cbm_py_namespace_facts_valid(&result->py_namespace)) { return; } cr_ctx_t c; @@ -480,14 +536,23 @@ void cbm_result_compact(CBMFileResult *result) { c.phase = CR_COUNT; cr_walk(&c, &tmp); + /* Bound intern-table/replay sizing before the existing power-of-two step. */ + if (c.failed || c.refs > (SIZE_MAX / (sizeof(cr_slot_t) * 8) - CR_MIN_TABLE)) + return; c.cap = cr_pow2_at_least(c.refs * CR_TABLE_LOAD + CR_MIN_TABLE); + if (c.cap > UINT32_MAX) /* replay entries store slot indexes in uint32_t */ + return; if (!cr_scratch_get(&c)) { return; } c.phase = CR_MEASURE; cr_walk(&c, &tmp); + if (c.failed) { + cr_scratch_put(&c); + return; + } for (size_t i = 0; i < c.cap; i++) { c.slots[i].dst = NULL; /* MEASURE used dst as a booked marker */ } @@ -512,6 +577,7 @@ void cbm_result_compact(CBMFileResult *result) { tmp.defs.cap = tmp.defs.count; tmp.calls.cap = tmp.calls.count; tmp.imports.cap = tmp.imports.count; + tmp.py_namespace.cap = tmp.py_namespace.count; tmp.usages.cap = tmp.usages.count; tmp.throws.cap = tmp.throws.count; tmp.rw.cap = tmp.rw.count; @@ -523,6 +589,7 @@ void cbm_result_compact(CBMFileResult *result) { tmp.string_refs.cap = tmp.string_refs.count; tmp.infra_bindings.cap = tmp.infra_bindings.count; tmp.channels.cap = tmp.channels.count; + tmp.field_types.cap = tmp.field_types.count; /* A composite kept its per-unit results only so shallow-copied strings * stayed valid; every string is now a copy of its own. */ diff --git a/internal/cbm/result_spill.c b/internal/cbm/result_spill.c index 33a732451..503e1fb2a 100644 --- a/internal/cbm/result_spill.c +++ b/internal/cbm/result_spill.c @@ -258,7 +258,8 @@ bool cbm_result_spill_park(cbm_result_spill_t *sp, int writer, int slot, CBMFile * result that owns sub-results (embedded languages) has more arenas than * that and stays in memory. A retained parse tree is a re-parse cache, * not data: it is dropped with the in-memory result below. */ - if (result->arena.nblocks != 1 || result->owned_result_count != 0) { + if (result->arena.nblocks != 1 || result->owned_result_count != 0 || + !cbm_py_namespace_facts_valid(&result->py_namespace)) { return false; } /* The disk ceiling. Refusing a park leaves the result in memory, which is @@ -356,6 +357,10 @@ CBMFileResult *cbm_result_spill_load(const cbm_result_spill_t *sp, int slot) { } cbm_result_relocate(r, (const char *)(uintptr_t)hdr.old_base, (size_t)hdr.block_len, r->arena.blocks[0]); + if (!cbm_py_namespace_facts_valid(&r->py_namespace)) { + cbm_free_result(r); + return NULL; + } atomic_fetch_add_explicit((_Atomic int64_t *)&sp->loads, 1, memory_order_relaxed); return r; } diff --git a/internal/cbm/result_spill.h b/internal/cbm/result_spill.h index 88cfd3720..8777a5bca 100644 --- a/internal/cbm/result_spill.h +++ b/internal/cbm/result_spill.h @@ -43,7 +43,10 @@ CBMFileResult *cbm_result_spill_load(const cbm_result_spill_t *sp, int slot); /* Read only the parked header of slot `slot` into *out: every count in it * is valid, every pointer meaningless. False when the slot is empty or the - * read fails. Lets a consumer skip the load when there is nothing to read. */ + * read fails. Lets a consumer skip the load when there is nothing to read. + * Python namespace version/language/status/count are scalar evidence only: + * zero facts with NOT_CAPTURED/INCOMPLETE is not a known-empty namespace, and + * event pointers require load. Failed peek/load provides no namespace evidence. */ bool cbm_result_spill_peek_header(const cbm_result_spill_t *sp, int slot, CBMFileResult *out); /* Read only the parked header of slot `slot`: how many defs and impl diff --git a/scripts/memory-core-baseline.txt b/scripts/memory-core-baseline.txt index 138ef0f77..2321c430e 100644 --- a/scripts/memory-core-baseline.txt +++ b/scripts/memory-core-baseline.txt @@ -10,7 +10,6 @@ internal/cbm/extract_defs.c 4 internal/cbm/helpers.c 3 internal/cbm/lsp/c_lsp.c 4 internal/cbm/lsp/perl_lsp.c 21 -internal/cbm/lsp/py_lsp.c 2 internal/cbm/macro_table.c 1 internal/cbm/service_patterns.c 2 internal/cbm/sqlite_writer.c 74 diff --git a/src/pipeline/lsp_surface.c b/src/pipeline/lsp_surface.c index 4c3cde336..5bb52a55e 100644 --- a/src/pipeline/lsp_surface.c +++ b/src/pipeline/lsp_surface.c @@ -19,6 +19,7 @@ #include "pipeline/lsp_surface.h" #include "pipeline/pipeline_internal.h" +#include #include #include #include @@ -29,7 +30,7 @@ #include "pipeline/worker_pool.h" #include "yyjson/yyjson.h" -enum { SURFACE_CODEC_VERSION = 1 }; +enum { SURFACE_CODEC_VERSION = 2 }; /* Labels the incremental name registry serves that pxc_map_label does NOT * carry into the CBMLSPDef set. Their (name, qn, label) triple must still @@ -71,10 +72,288 @@ static void add_str_array_or_null(yyjson_mut_doc *doc, yyjson_mut_val *obj, cons yyjson_mut_obj_add_val(doc, obj, key, arr); } +/* The namespace payload is content-only. Its ordered records never depend on + * the registry used to build ordinary LSP definitions. */ +static bool surface_py_add_nullable(yyjson_mut_doc *doc, yyjson_mut_val *obj, const char *key, + const char *value) { + return value ? yyjson_mut_obj_add_str(doc, obj, key, value) + : yyjson_mut_obj_add_null(doc, obj, key); +} + +static bool surface_add_python_namespace(yyjson_mut_doc *doc, yyjson_mut_val *root, + const CBMFileResult *result, CBMLanguage language, + const char *rel_path) { + if (language != CBM_LANG_PYTHON) { + return yyjson_mut_obj_add_null(doc, root, "py"); + } + if (!result || !result->module_qn || !result->module_qn[0] || !rel_path || !rel_path[0] || + result->py_namespace.status != CBM_PY_NS_COMPLETE || + result->py_namespace.language != CBM_LANG_PYTHON || + !cbm_py_namespace_facts_valid(&result->py_namespace)) { + return false; + } + const CBMPyNamespaceFacts *facts = &result->py_namespace; + yyjson_mut_val *py = yyjson_mut_obj(doc); + yyjson_mut_val *events = yyjson_mut_arr(doc); + if (!py || !events || !yyjson_mut_obj_add_uint(doc, py, "v", facts->version) || + !yyjson_mut_obj_add_int(doc, py, "lang", (int)facts->language) || + !yyjson_mut_obj_add_str(doc, py, "module", result->module_qn) || + !yyjson_mut_obj_add_str(doc, py, "path", rel_path) || + !yyjson_mut_obj_add_int(doc, py, "status", (int)facts->status) || + !yyjson_mut_obj_add_int(doc, py, "failure", (int)facts->failure)) { + return false; + } + for (int i = 0; i < facts->count; i++) { + const CBMPyNamespaceFact *f = &facts->items[i]; + yyjson_mut_val *event = yyjson_mut_obj(doc); + yyjson_mut_val *names = yyjson_mut_arr(doc); + if (!event || !names || !yyjson_mut_obj_add_int(doc, event, "k", (int)f->kind) || + !yyjson_mut_obj_add_int(doc, event, "d", (int)f->def_kind) || + !yyjson_mut_obj_add_int(doc, event, "s", (int)f->sequence_kind) || + !yyjson_mut_obj_add_int(doc, event, "r", (int)f->reason) || + !yyjson_mut_obj_add_uint(doc, event, "f", f->flags) || + !yyjson_mut_obj_add_uint(doc, event, "l", f->relative_level) || + !surface_py_add_nullable(doc, event, "n", f->local_name) || + !surface_py_add_nullable(doc, event, "m", f->module_name) || + !surface_py_add_nullable(doc, event, "i", f->member_name)) { + return false; + } + for (int j = 0; j < f->name_count; j++) { + if (!yyjson_mut_arr_add_str(doc, names, f->names[j])) { + return false; + } + } + if (!yyjson_mut_obj_add_val(doc, event, "a", names) || + !yyjson_mut_arr_add_val(events, event)) { + return false; + } + } + return yyjson_mut_obj_add_val(doc, py, "events", events) && + yyjson_mut_obj_add_val(doc, root, "py", py); +} + +/* Reject duplicate fields rather than letting object lookup choose one. */ +static yyjson_val *surface_unique_field(yyjson_val *obj, const char *name) { + if (!yyjson_is_obj(obj)) { + return NULL; + } + yyjson_val *found = NULL; + yyjson_obj_iter it = yyjson_obj_iter_with(obj); + yyjson_val *key; + while ((key = yyjson_obj_iter_next(&it))) { + const char *text = yyjson_get_str(key); + if (text && yyjson_get_len(key) == strlen(name) && strcmp(text, name) == 0) { + if (found) { + return NULL; + } + found = yyjson_obj_iter_get_val(key); + } + } + return found; +} + +static bool surface_py_uint(yyjson_val *value, uint32_t maximum, uint32_t *out) { + if (!yyjson_is_int(value)) { + return false; + } + int64_t n = yyjson_get_sint(value); + if (n < 0 || (uint64_t)n > maximum) { + return false; + } + *out = (uint32_t)n; + return true; +} + +static bool surface_py_string(yyjson_val *value, bool nullable, const char **out) { + *out = NULL; + if (!value) { + return false; + } + if (yyjson_is_null(value)) { + return nullable; + } + const char *text = yyjson_get_str(value); + if (!text || strlen(text) != yyjson_get_len(value)) { + return false; + } + *out = text; + return true; +} + +/* Arrays belong to scratch; strings borrow the immutable JSON document. + * The caller keeps both alive until validation/copy/comparison is finished. */ +static int surface_python_namespace_parse(CBMArena *scratch, yyjson_val *root, + cbm_lsp_python_namespace_t *out) { + memset(out, 0, sizeof(*out)); + uint32_t version = 0, file_language = 0; + yyjson_val *py = surface_unique_field(root, "py"); + if (!surface_py_uint(surface_unique_field(root, "v"), UINT32_MAX, &version) || + version != SURFACE_CODEC_VERSION || !py || + !surface_py_uint(surface_unique_field(root, "lang"), CBM_LANG_COUNT - 1, &file_language) || + !yyjson_is_arr(surface_unique_field(root, "lsp"))) { + return -1; + } + if (yyjson_is_null(py)) { + return file_language == CBM_LANG_PYTHON ? -1 : 0; + } + if (file_language != CBM_LANG_PYTHON || !yyjson_is_obj(py) || yyjson_obj_size(py) != 7) { + return -1; + } + CBMPyNamespaceFacts *facts = &out->facts; + uint32_t language, status, failure; + if (!surface_py_uint(surface_unique_field(py, "v"), UINT32_MAX, &facts->version) || + !surface_py_uint(surface_unique_field(py, "lang"), INT_MAX, &language) || + !surface_py_uint(surface_unique_field(py, "status"), INT_MAX, &status) || + !surface_py_uint(surface_unique_field(py, "failure"), INT_MAX, &failure) || + !surface_py_string(surface_unique_field(py, "module"), false, &out->module_qn) || + !surface_py_string(surface_unique_field(py, "path"), false, &out->rel_path) || + !out->module_qn[0] || !out->rel_path[0] || language != CBM_LANG_PYTHON || + status != CBM_PY_NS_COMPLETE || failure != CBM_PY_NS_FAILURE_NONE) { + return -1; + } + facts->language = (CBMLanguage)language; + facts->status = (CBMPyNamespaceStatus)status; + facts->failure = (CBMPyNamespaceFailure)failure; + yyjson_val *events = surface_unique_field(py, "events"); + if (!yyjson_is_arr(events)) { + return -1; + } + size_t count = yyjson_arr_size(events); + if (count > INT_MAX || count > SIZE_MAX / sizeof(*facts->items)) { + return -1; + } + facts->count = (int)count; + facts->cap = (int)count; + if (count) { + facts->items = cbm_arena_alloc(scratch, count * sizeof(*facts->items)); + if (!facts->items) { + return -1; + } + memset(facts->items, 0, count * sizeof(*facts->items)); + } + yyjson_arr_iter it = yyjson_arr_iter_with(events); + yyjson_val *event; + int i = 0; + while ((event = yyjson_arr_iter_next(&it))) { + if (!yyjson_is_obj(event) || yyjson_obj_size(event) != 10) { + return -1; + } + CBMPyNamespaceFact *f = &facts->items[i++]; + uint32_t kind, def_kind, sequence_kind, reason; + if (!surface_py_uint(surface_unique_field(event, "k"), INT_MAX, &kind) || + !surface_py_uint(surface_unique_field(event, "d"), INT_MAX, &def_kind) || + !surface_py_uint(surface_unique_field(event, "s"), INT_MAX, &sequence_kind) || + !surface_py_uint(surface_unique_field(event, "r"), INT_MAX, &reason) || + !surface_py_uint(surface_unique_field(event, "f"), UINT32_MAX, &f->flags) || + !surface_py_uint(surface_unique_field(event, "l"), UINT32_MAX, &f->relative_level) || + !surface_py_string(surface_unique_field(event, "n"), true, &f->local_name) || + !surface_py_string(surface_unique_field(event, "m"), true, &f->module_name) || + !surface_py_string(surface_unique_field(event, "i"), true, &f->member_name)) { + return -1; + } + f->kind = (CBMPyNamespaceFactKind)kind; + f->def_kind = (CBMPyNamespaceDefKind)def_kind; + f->sequence_kind = (CBMPyNamespaceSequenceKind)sequence_kind; + f->reason = (CBMPyNamespaceUnknownReason)reason; + yyjson_val *names = surface_unique_field(event, "a"); + if (!yyjson_is_arr(names)) { + return -1; + } + size_t n = yyjson_arr_size(names); + if (n > INT_MAX || n > SIZE_MAX / sizeof(*f->names)) { + return -1; + } + f->name_count = (int)n; + if (n) { + f->names = cbm_arena_alloc(scratch, n * sizeof(*f->names)); + if (!f->names) { + return -1; + } + } + yyjson_arr_iter ni = yyjson_arr_iter_with(names); + yyjson_val *name; + int j = 0; + while ((name = yyjson_arr_iter_next(&ni))) { + if (!surface_py_string(name, false, &f->names[j++])) { + return -1; + } + } + } + return cbm_py_namespace_facts_valid(facts) ? 1 : -1; +} + +int cbm_lsp_surface_python_namespace_from_json(CBMArena *arena, const char *json, + cbm_lsp_python_namespace_t *out) { + if (!out) { + return -1; + } + memset(out, 0, sizeof(*out)); + if (!arena || !json) { + return -1; + } + yyjson_doc *doc = yyjson_read(json, strlen(json), 0); + if (!doc) { + return -1; + } + CBMArena scratch; + cbm_arena_init(&scratch); + cbm_lsp_python_namespace_t parsed = {0}, result = {0}; + int status = surface_python_namespace_parse(&scratch, yyjson_doc_get_root(doc), &parsed); + if (status == 1) { + if (!cbm_py_namespace_facts_copy(arena, &parsed.facts, &result.facts)) { + status = -1; + } else { + result.module_qn = cbm_arena_strdup(arena, parsed.module_qn); + result.rel_path = cbm_arena_strdup(arena, parsed.rel_path); + if (!result.module_qn || !result.rel_path) { + status = -1; + } + } + } + cbm_arena_destroy(&scratch); + yyjson_doc_free(doc); + if (status == 1) { + *out = result; + } + return status; +} + +int cbm_lsp_surface_python_change(const char *old_json, const char *new_json) { + if (!old_json) { + return -1; + } + yyjson_doc *old_doc = yyjson_read(old_json, strlen(old_json), 0); + yyjson_doc *new_doc = new_json ? yyjson_read(new_json, strlen(new_json), 0) : NULL; + if (!old_doc || (new_json && !new_doc)) { + yyjson_doc_free(old_doc); + yyjson_doc_free(new_doc); + return -1; + } + CBMArena scratch; + cbm_arena_init(&scratch); + cbm_lsp_python_namespace_t old_facts, new_facts; + int old_state = + surface_python_namespace_parse(&scratch, yyjson_doc_get_root(old_doc), &old_facts); + int new_state = + new_doc ? surface_python_namespace_parse(&scratch, yyjson_doc_get_root(new_doc), &new_facts) + : 0; + int result = -1; + if (old_state >= 0 && new_state >= 0) { + result = (old_state == 0 && new_state == 0) ? 0 + : (old_state != new_state || !new_json || strcmp(old_json, new_json) != 0) ? 1 + : 0; + } + cbm_arena_destroy(&scratch); + yyjson_doc_free(old_doc); + yyjson_doc_free(new_doc); + return result; +} + /* Serialize one file's surface: its slice of all_defs plus the registry-only * symbols from its raw extraction defs. Returns a malloc'd JSON string and its * length. */ -static char *surface_file_to_json(const CBMFileResult *result, const CBMLSPDef *defs, int def_count, +static char *surface_file_to_json(const CBMFileResult *result, CBMLanguage language, + const char *rel_path, const CBMLSPDef *defs, int def_count, size_t *out_len) { yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); if (!doc) { @@ -82,7 +361,12 @@ static char *surface_file_to_json(const CBMFileResult *result, const CBMLSPDef * } yyjson_mut_val *root = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root); - yyjson_mut_obj_add_int(doc, root, "v", SURFACE_CODEC_VERSION); + if ((unsigned)language >= CBM_LANG_COUNT || + !yyjson_mut_obj_add_int(doc, root, "v", SURFACE_CODEC_VERSION) || + !yyjson_mut_obj_add_int(doc, root, "lang", (int)language)) { + yyjson_mut_doc_free(doc); + return NULL; + } yyjson_mut_val *lsp = yyjson_mut_arr(doc); for (int i = 0; i < def_count; i++) { @@ -150,6 +434,10 @@ static char *surface_file_to_json(const CBMFileResult *result, const CBMLSPDef * yyjson_mut_obj_add_val(doc, root, "http", http); } + if (!surface_add_python_namespace(doc, root, result, language, rel_path)) { + yyjson_mut_doc_free(doc); + return NULL; + } char *json = yyjson_mut_write(doc, 0, out_len); yyjson_mut_doc_free(doc); return json; @@ -186,8 +474,9 @@ static void surface_row_one(int i, void *arg) { int start = job->def_starts ? job->def_starts[i] : 0; int end = job->def_starts ? job->def_starts[i + 1] : 0; size_t json_len = 0; - char *json = surface_file_to_json(fr, job->all_defs ? job->all_defs + start : NULL, end - start, - &json_len); + char *json = + surface_file_to_json(fr, job->files[i].language, job->files[i].rel_path, + job->all_defs ? job->all_defs + start : NULL, end - start, &json_len); cbm_pipeline_result_release(fr, loaded); if (!json) { atomic_store_explicit(&job->failed, true, memory_order_relaxed); @@ -302,9 +591,14 @@ int cbm_lsp_surface_defs_from_json(CBMArena *arena, const char *defs_json, CBMLS return -1; } yyjson_val *root = yyjson_doc_get_root(doc); - yyjson_val *ver = root ? yyjson_obj_get(root, "v") : NULL; - yyjson_val *lsp = root ? yyjson_obj_get(root, "lsp") : NULL; - if (!ver || yyjson_get_int(ver) != SURFACE_CODEC_VERSION || !lsp || !yyjson_is_arr(lsp)) { + CBMArena namespace_scratch; + cbm_arena_init(&namespace_scratch); + cbm_lsp_python_namespace_t namespace_payload = {0}; + int namespace_status = + surface_python_namespace_parse(&namespace_scratch, root, &namespace_payload); + cbm_arena_destroy(&namespace_scratch); + yyjson_val *lsp = surface_unique_field(root, "lsp"); + if (namespace_status < 0) { yyjson_doc_free(doc); return -1; } diff --git a/src/pipeline/lsp_surface.h b/src/pipeline/lsp_surface.h index de3390d85..dc198bb65 100644 --- a/src/pipeline/lsp_surface.h +++ b/src/pipeline/lsp_surface.h @@ -10,8 +10,10 @@ * unchanged — a body edit — and skip recomputing its dependents, and * (b) rehydrate the cross registries for files it does not re-parse. * - * The JSON is a versioned object {"v":1,"lsp":[...],"reg":[...]}. Field - * order and array order are fixed by the writer, so byte equality of the + * The JSON is a versioned object {"v":2,"lang":N,"lsp":[...],"reg":[...],"py":...}. + * Python rows carry content-only namespace facts, including ordered binding + * events and unknown effects; other languages carry an explicit null. + * Field order and array order are fixed by the writer, so byte equality of the * serialization is surface equality; the sha256 of these bytes is the * stored surface_sha. Rows written by a different codec version fail the * "v" check on load and route the run to a full rebuild. @@ -28,7 +30,8 @@ * all_defs[def_starts[i] .. def_starts[i+1]) are file i's defs. Rows carry * heap strings; release with cbm_store_free_lsp_surfaces. Files with an * empty surface still get a row (empty arrays hash too — "no defs" must be - * distinguishable from "no data"). Returns 0, or -1 on allocation failure. */ + * distinguishable from "no data"). Returns 0, or -1 on allocation failure + * or invalid/incomplete Python namespace metadata. */ int cbm_lsp_surface_build_rows(const cbm_pipeline_ctx_t *ctx, const char *project, CBMFileResult **cache, const cbm_file_info_t *files, int file_count, const CBMLSPDef *all_defs, const int *def_starts, @@ -40,4 +43,29 @@ int cbm_lsp_surface_build_rows(const cbm_pipeline_ctx_t *ctx, const char *projec * different codec version — callers route that to a full rebuild. */ int cbm_lsp_surface_defs_from_json(CBMArena *arena, const char *defs_json, CBMLSPDef **out_defs); +/* A restored Python namespace retains its raw module identity and source path. + * Facts and origin strings live in the caller arena; no JSON storage is borrowed. */ +typedef struct { + const char *module_qn; + const char *rel_path; + CBMPyNamespaceFacts facts; +} cbm_lsp_python_namespace_t; + +/* Returns 1 for a complete Python payload, 0 for a valid non-Python row, or + * -1 for invalid/incomplete metadata or allocation failure. Output is empty + * unless a complete Python payload was copied successfully. */ +int cbm_lsp_surface_python_namespace_from_json(CBMArena *arena, const char *json, + cbm_lsp_python_namespace_t *out); + +/* Conservative namespace admission for incremental planning. Both rows must + * have a current, valid fact envelope before any equality shortcut is used. + * Returns -1 for invalid/incomplete metadata, 0 when both rows are non-Python + * or their complete Python surfaces are identical, and 1 for a Python surface + * change or a language transition involving Python. A NULL new_json denotes + * deletion: valid Python metadata returns 1, valid non-Python metadata 0. + * NULL old_json is invalid. The full Python surface comparison deliberately + * includes definition properties until a narrower dependency closure is proven. + */ +int cbm_lsp_surface_python_change(const char *old_json, const char *new_json); + #endif /* CBM_PIPELINE_LSP_SURFACE_H */ diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index f068ad411..1c0dc1a62 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -377,7 +377,8 @@ bool cbm_pipeline_http_client_call_url(const cbm_gbuf_t *gbuf, const char *proje /* Handle a route registration call: create Route node + HANDLES edge. */ static void handle_route_registration(cbm_pipeline_ctx_t *ctx, const CBMCall *call, const cbm_gbuf_node_t *source_node, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { + const char **imp_keys, const char **imp_vals, int imp_count, + CBMLanguage lang) { const char *method = cbm_service_pattern_route_method(call->callee_name); char route_qn[CBM_ROUTE_QN_SIZE]; char cpath[CBM_SZ_256]; @@ -397,8 +398,8 @@ static void handle_route_registration(cbm_pipeline_ctx_t *ctx, const CBMCall *ca esc_fa); cbm_gbuf_insert_edge(ctx->gbuf, source_node->id, route_id, "CALLS", props); if (call->second_arg_name != NULL && call->second_arg_name[0] != '\0') { - cbm_resolution_t hres = cbm_registry_resolve(ctx->registry, call->second_arg_name, - module_qn, imp_keys, imp_vals, imp_count); + cbm_resolution_t hres = cbm_registry_resolve_lang( + ctx->registry, call->second_arg_name, module_qn, imp_keys, imp_vals, imp_count, lang); if (hres.qualified_name != NULL && hres.qualified_name[0] != '\0') { const cbm_gbuf_node_t *handler = cbm_gbuf_find_by_qn(ctx->gbuf, hres.qualified_name); if (handler != NULL) { @@ -588,10 +589,11 @@ static void emit_classified_edge(cbm_pipeline_ctx_t *ctx, const CBMCall *call, const cbm_gbuf_node_t *source, const cbm_gbuf_node_t *target, const cbm_resolution_t *res, const char *module_qn, const char **imp_keys, const char **imp_vals, int imp_count, - bool suppress_plain_calls) { + bool suppress_plain_calls, CBMLanguage lang) { cbm_svc_kind_t svc = cbm_service_pattern_match(res->qualified_name); if (svc == CBM_SVC_ROUTE_REG && call->first_string_arg && call->first_string_arg[0] == '/') { - handle_route_registration(ctx, call, source, module_qn, imp_keys, imp_vals, imp_count); + handle_route_registration(ctx, call, source, module_qn, imp_keys, imp_vals, imp_count, + lang); return; } if (svc == CBM_SVC_HTTP || svc == CBM_SVC_ASYNC) { @@ -675,7 +677,7 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM res.strategy = lsp->strategy; res.candidate_count = 1; emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, - imp_vals, imp_count, false); + imp_vals, imp_count, false, lang); return SKIP_ONE; } } @@ -741,8 +743,8 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM lang, lsp->strategy, lsp->callee_qn, ctx->project_name); cbm_resolution_t res = {0}; if (!rust_external) { - res = cbm_registry_resolve(ctx->registry, call->callee_name, module_qn, imp_keys, imp_vals, - imp_count); + res = cbm_registry_resolve_lang(ctx->registry, call->callee_name, module_qn, imp_keys, + imp_vals, imp_count, lang); } if (!res.qualified_name || res.qualified_name[0] == '\0') { /* Resolution is empty when the callee belongs to an EXTERNAL client @@ -767,7 +769,7 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM if (cbm_service_pattern_route_method(call->callee_name) != NULL && call->first_string_arg && call->first_string_arg[0] == '/') { handle_route_registration(ctx, call, source_node, module_qn, imp_keys, imp_vals, - imp_count); + imp_count, lang); return SKIP_ONE; } cbm_svc_kind_t esvc = cbm_service_pattern_match(call->callee_name); @@ -888,7 +890,7 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM return 0; } emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, imp_vals, - imp_count, drop_plain_call); + imp_count, drop_plain_call, lang); return SKIP_ONE; } @@ -1125,7 +1127,8 @@ static int scan_depends_in_sig(cbm_pipeline_ctx_t *ctx, const cbm_regex_t *re, c } memcpy(func_ref, scan + match[SKIP_ONE].rm_so, (size_t)ref_len); func_ref[ref_len] = '\0'; - cbm_resolution_t res = cbm_registry_resolve(ctx->registry, func_ref, module_qn, ik, iv, ic); + cbm_resolution_t res = cbm_registry_resolve_lang(ctx->registry, func_ref, module_qn, ik, iv, + ic, CBM_LANG_PYTHON); if (res.qualified_name && res.qualified_name[0] != '\0') { const cbm_gbuf_node_t *sn = cbm_gbuf_find_by_qn(ctx->gbuf, def->qualified_name); const cbm_gbuf_node_t *tn = cbm_gbuf_find_by_qn(ctx->gbuf, res.qualified_name); diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index 3b74cc1eb..3ae60db0e 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -227,11 +227,12 @@ static bool pxc_base_strategy_is_weak(const char *strategy) { * stdlib and third-party bases land here and keep their raw spelling. */ static const char *pxc_resolve_base_qn(const cbm_registry_t *reg, const char *raw, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { + const char **imp_vals, int imp_count, CBMLanguage lang) { if (!reg || !raw || !raw[0]) { return NULL; } - cbm_resolution_t res = cbm_registry_resolve(reg, raw, module_qn, imp_keys, imp_vals, imp_count); + cbm_resolution_t res = + cbm_registry_resolve_lang(reg, raw, module_qn, imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || !res.qualified_name[0]) { return NULL; } @@ -249,7 +250,8 @@ static const char *pxc_resolve_base_qn(const cbm_registry_t *reg, const char *ra * registry does not know keeps working exactly as before. */ static const char *pxc_join_base_qns(CBMArena *arena, const char *const *bases, const cbm_registry_t *reg, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { + const char **imp_keys, const char **imp_vals, int imp_count, + CBMLanguage lang) { if (!bases || !bases[0]) { return NULL; } @@ -264,7 +266,7 @@ static const char *pxc_join_base_qns(CBMArena *arena, const char *const *bases, } for (int i = 0; i < count; i++) { const char *qn = - pxc_resolve_base_qn(reg, bases[i], module_qn, imp_keys, imp_vals, imp_count); + pxc_resolve_base_qn(reg, bases[i], module_qn, imp_keys, imp_vals, imp_count, lang); resolved[i] = qn ? qn : bases[i]; } resolved[count] = NULL; @@ -419,7 +421,7 @@ static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src, const ch * raw source spelling their own registrar already knows how to handle. */ dst->embedded_types = (reg && pxc_lang_resolves_base_qns(lang)) ? pxc_join_base_qns(arena, src->base_classes, reg, module_qn, - imp_keys, imp_vals, imp_count) + imp_keys, imp_vals, imp_count, lang) : pxc_join_pipe(arena, src->base_classes); dst->signature_param_types = NULL; dst->signature_param_count = 0; @@ -529,6 +531,133 @@ static void pxc_fold_go_struct_fields(CBMArena *arena, const CBMFileResult *resu } } +/* Python annotation text -> the one type name it declares, or NULL. Strips a + * string-literal forward reference and unwraps `Optional[X]`, `X | None` and + * `None | X`; anything else with brackets, commas or a real union (generics, + * Callable, Union[A, B]) names no single receiver type and is dropped. */ +static const char *pxc_py_annotation_type_name(CBMArena *arena, const char *text) { + if (!text) { + return NULL; + } + while (*text == ' ') { + text++; + } + size_t n = strlen(text); + while (n > 0 && text[n - 1] == ' ') { + n--; + } + if (n >= 2 && (text[0] == '"' || text[0] == '\'') && text[n - 1] == text[0]) { + text++; + n -= 2; + } + static const char *const optional_prefixes[] = {"Optional[", "typing.Optional["}; + for (size_t i = 0; i < sizeof(optional_prefixes) / sizeof(optional_prefixes[0]); i++) { + size_t plen = strlen(optional_prefixes[i]); + if (n > plen + 1 && strncmp(text, optional_prefixes[i], plen) == 0 && text[n - 1] == ']') { + text += plen; + n -= plen + 1; + break; + } + } + static const char none_tail[] = " | None"; + static const char none_head[] = "None | "; + const size_t none_len = sizeof(none_tail) - 1; + if (n > none_len && strncmp(text + n - none_len, none_tail, none_len) == 0) { + n -= none_len; + } else if (n > none_len && strncmp(text, none_head, none_len) == 0) { + text += none_len; + n -= none_len; + } + if (n == 0) { + return NULL; + } + for (size_t i = 0; i < n; i++) { + char c = text[i]; + bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '_' || c == '.'; + if (!ok) { + return NULL; + } + } + return cbm_arena_strndup(arena, text, n); +} + +/* Python: fold the file's annotated instance fields (result->field_types, + * #1277) into their class defs' field_defs as "name:QN|name:QN", the format + * py_register_lsp_defs reads. Each annotation resolves through the same + * registry + import map, and the same weak-strategy and type-like vetoes, as + * the file's base classes (pxc_resolve_base_qn): a field whose type is not a + * confidently-known project type is left out rather than guessed. Linear in + * the file: one hash of the file's class defs, one pass over its fields. */ +typedef struct { + int owner; /* index into defs[], or -1 */ + const char *type_qn; +} pxc_py_field_t; + +static void pxc_fold_py_field_types(CBMArena *arena, const CBMFileResult *result, CBMLSPDef *defs, + int start, int end, const cbm_registry_t *reg, + const char *module_qn, const char **imp_keys, + const char **imp_vals, int imp_count) { + int nf = result ? result->field_types.count : 0; + if (!arena || nf == 0 || !defs || start >= end || !reg) { + return; + } + CBMHashTable *by_qn = cbm_ht_create((uint32_t)(end - start)); + pxc_py_field_t *fields = (pxc_py_field_t *)cbm_arena_alloc(arena, (size_t)nf * sizeof(*fields)); + /* Per class def: exact buffer size, then the write cursor into it. */ + size_t *lens = (size_t *)cbm_arena_alloc(arena, (size_t)(end - start) * sizeof(size_t)); + size_t *used = (size_t *)cbm_arena_alloc(arena, (size_t)(end - start) * sizeof(size_t)); + if (!by_qn || !fields || !lens || !used) { + cbm_ht_free(by_qn); /* NULL-safe */ + return; + } + memset(lens, 0, (size_t)(end - start) * sizeof(size_t)); + memset(used, 0, (size_t)(end - start) * sizeof(size_t)); + for (int si = start; si < end; si++) { + if (defs[si].label && strcmp(defs[si].label, "Class") == 0 && defs[si].qualified_name) { + cbm_ht_set(by_qn, defs[si].qualified_name, &defs[si]); + } + } + for (int f = 0; f < nf; f++) { + const CBMFieldType *ft = &result->field_types.items[f]; + fields[f].owner = -1; + fields[f].type_qn = NULL; + CBMLSPDef *owner = ft->class_qn ? (CBMLSPDef *)cbm_ht_get(by_qn, ft->class_qn) : NULL; + const char *name = owner ? pxc_py_annotation_type_name(arena, ft->type_text) : NULL; + const char *qn = name ? pxc_resolve_base_qn(reg, name, module_qn, imp_keys, imp_vals, + imp_count, CBM_LANG_PYTHON) + : NULL; + if (!qn || !ft->field_name || !ft->field_name[0]) { + continue; + } + fields[f].owner = (int)(owner - defs); + fields[f].type_qn = qn; + /* "name:QN" plus one byte for the '|' or the NUL after it. */ + lens[fields[f].owner - start] += strlen(ft->field_name) + 1 + strlen(qn) + 1; + } + cbm_ht_free(by_qn); + for (int si = start; si < end; si++) { + char *buf = lens[si - start] ? (char *)cbm_arena_alloc(arena, lens[si - start]) : NULL; + if (buf) { + buf[0] = '\0'; + } + defs[si].field_defs = buf; + } + for (int f = 0; f < nf; f++) { + char *buf = fields[f].owner >= 0 ? (char *)defs[fields[f].owner].field_defs : NULL; + if (!buf) { + continue; + } + int slot = fields[f].owner - start; + int wrote = + snprintf(buf + used[slot], lens[slot] - used[slot], "%s%s:%s", used[slot] ? "|" : "", + result->field_types.items[f].field_name, fields[f].type_qn); + if (wrote > 0) { + used[slot] += (size_t)wrote; + } + } +} + /* Carry one Rust type-level impl independently of any method definition. * `impl Trait for Type {}` is semantically meaningful even when the block is * empty (the trait may provide defaults), so attaching the relation only to @@ -634,6 +763,10 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *are idx++; } } + if (files[fi].language == CBM_LANG_PYTHON) { + pxc_fold_py_field_types(arena, fr, defs, file_start, idx, base_reg, def_modules[fi], + imp_keys, imp_vals, imp_count); + } cbm_pxc_free_import_map(imp_keys, imp_vals, imp_count); /* NULL-safe */ if (files[fi].language == CBM_LANG_GO) { pxc_fold_go_struct_fields(arena, fr, defs, file_start, idx); @@ -1301,14 +1434,22 @@ void cbm_pxc_run_one(CBMLanguage lang, CBMFileResult *r, const char *source, int * imports — the existing pipeline doesn't carry C-style include * resolution as a separate map, so pass NULL/0 and let the LSP * fall back to its own #include scan. */ - cbm_run_c_lsp_cross(&scratch, source, source_len, module_qn, cpp_mode, defs, def_count, - NULL, NULL, 0, tree, &out); + { + CBMLSPStatus status = + cbm_run_c_lsp_cross_status(&scratch, source, source_len, module_qn, cpp_mode, defs, + def_count, NULL, NULL, 0, tree, &out); + if (status != CBM_LSP_COMPLETE) + cbm_c_lsp_record_failure(r, status); + } break; } - case CBM_LANG_PYTHON: - cbm_run_py_lsp_cross(&scratch, source, source_len, module_qn, defs, def_count, imp_names, - imp_qns, imp_count, tree, &out, &synthetic_calls); - break; + case CBM_LANG_PYTHON: { + CBMLSPStatus status = cbm_run_py_lsp_cross_status(&scratch, source, source_len, module_qn, + defs, def_count, imp_names, imp_qns, + imp_count, tree, &out, &synthetic_calls); + if (status != CBM_LSP_COMPLETE) + cbm_py_lsp_record_failure(r, status); + } break; case CBM_LANG_PHP: cbm_run_php_lsp_cross(&scratch, source, source_len, module_qn, defs, def_count, imp_names, imp_qns, imp_count, tree, &out); @@ -1456,18 +1597,24 @@ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char * &out); used_prebuilt = true; break; - case CBM_LANG_PYTHON: - cbm_run_py_lsp_cross_with_registry(&scratch, source, source_len, def_module, &overlay, - imp_keys, imp_vals, imp_count, result->cached_tree, - &out, &synthetic_calls); + case CBM_LANG_PYTHON: { + CBMLSPStatus status = cbm_run_py_lsp_cross_with_registry_status( + &scratch, source, source_len, def_module, &overlay, imp_keys, imp_vals, imp_count, + result->cached_tree, &out, &synthetic_calls); + if (status != CBM_LSP_COMPLETE) + cbm_py_lsp_record_failure(result, status); + } used_prebuilt = true; break; case CBM_LANG_C: case CBM_LANG_CPP: - case CBM_LANG_CUDA: - cbm_run_c_lsp_cross_with_registry(&scratch, source, source_len, def_module, - (lang != CBM_LANG_C), &overlay, imp_keys, imp_vals, - imp_count, result->cached_tree, &out); + case CBM_LANG_CUDA: { + CBMLSPStatus status = cbm_run_c_lsp_cross_with_registry_status( + &scratch, source, source_len, def_module, (lang != CBM_LANG_C), &overlay, imp_keys, + imp_vals, imp_count, result->cached_tree, &out); + if (status != CBM_LSP_COMPLETE) + cbm_c_lsp_record_failure(result, status); + } used_prebuilt = true; break; case CBM_LANG_CSHARP: @@ -1730,10 +1877,14 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t * * file, not to a stale extraction marker (the innocent-quarantine * failure mode). */ cbm_index_mark_start(files[i].rel_path); + bool had_error = cache[i]->has_error; cbm_pxc_dispatch_file(lang, cache[i], source, source_len, files[i].rel_path, def_modules[i], &cross_registries, module_def_index, all_defs, def_count, imp_keys, imp_vals, imp_count, NULL, NULL); cbm_index_mark_done(files[i].rel_path); + if (!had_error && cache[i]->has_error && ctx->pipeline) + cbm_pipeline_add_file_error(ctx->pipeline, files[i].rel_path, cache[i]->error_msg, + "lsp_skipped"); per_lang_calls++; processed++; @@ -1889,16 +2040,33 @@ static void pxc_mark_module_defs(const CBMModuleDefIndex *idx, bool *selected, * (`project.module`). Select the nearest materialized module prefix; the * language registry still has to prove the full target QN before an edge is * emitted, so this broadens the candidate set without weakening precision. */ -static void pxc_mark_import_defs(const CBMModuleDefIndex *idx, bool *selected, +#ifdef CBM_ENABLE_TEST_SEAMS +static _Thread_local int pxc_import_probe_allocations_left = -1; +void cbm_pxc_test_import_probe_fail_after(int successful_allocations) { + pxc_import_probe_allocations_left = successful_allocations; +} +#endif + +static bool pxc_mark_import_defs(const CBMModuleDefIndex *idx, bool *selected, const CBMLSPDef *all_defs, CBMLanguage caller_lang, const char *import_qn, int *total) { if (!idx || !idx->ht || !import_qn || !import_qn[0]) { - return; - } - char *candidate = strdup(import_qn); - if (!candidate) { - return; + return true; } + static const char init_seg[] = ".__init__"; + size_t import_len = strlen(import_qn); + if (import_len > SIZE_MAX - sizeof(init_seg)) + return false; +#ifdef CBM_ENABLE_TEST_SEAMS + if (pxc_import_probe_allocations_left == 0) + return false; + if (pxc_import_probe_allocations_left > 0) + pxc_import_probe_allocations_left--; +#endif + char *candidate = (char *)malloc(import_len + sizeof(init_seg)); + if (!candidate) + return false; + memcpy(candidate, import_qn, import_len + 1); bool matched = false; for (;;) { pxc_module_entry_t *entry = (pxc_module_entry_t *)cbm_ht_get(idx->ht, candidate); @@ -1910,6 +2078,31 @@ static void pxc_mark_import_defs(const CBMModuleDefIndex *idx, bool *selected, matched = true; break; } + if (caller_lang == CBM_LANG_PYTHON) { + size_t cand_len = strlen(candidate); + memcpy(candidate + cand_len, init_seg, sizeof(init_seg)); + entry = (pxc_module_entry_t *)cbm_ht_get(idx->ht, candidate); + candidate[cand_len] = '\0'; + /* A suffixed module may be a non-Python directory named __init__. + * Only this new probe filters strictly; normal-entry precedence + * above remains unchanged. Eligibility is independent of dedup. */ + bool eligible = false; + for (int j = 0; entry && j < entry->count; j++) { + int di = entry->indices[j]; + if (all_defs[di].lang != CBM_LANG_PYTHON) + continue; + eligible = true; + if (!selected[di]) { + selected[di] = true; + if (total) + (*total)++; + } + } + if (eligible) { + matched = true; + break; + } + } char *dot = strrchr(candidate, '.'); if (!dot) { break; @@ -1940,6 +2133,7 @@ static void pxc_mark_import_defs(const CBMModuleDefIndex *idx, bool *selected, } } free(candidate); + return true; } CBMModuleDefIndex *cbm_pxc_build_module_def_index(CBMLSPDef *all_defs, int def_count) { @@ -2018,7 +2212,9 @@ CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx, CBMLSPDef int total = 0; pxc_mark_module_defs(idx, selected, all_defs, caller_lang, own_module, &total); for (int i = 0; i < imp_count; i++) { - pxc_mark_import_defs(idx, selected, all_defs, caller_lang, imp_qns[i], &total); + if (!pxc_mark_import_defs(idx, selected, all_defs, caller_lang, imp_qns[i], &total)) { + goto failed; /* out_success remains false: dispatcher uses all_defs */ + } } if (pxc_is_jvm_lang(caller_lang) && idx->namespace_ht) { const char *namespace_key = pxc_namespace_index_key(caller_namespace); @@ -2034,8 +2230,7 @@ CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx, CBMLSPDef CBMLSPDef *out = (CBMLSPDef *)malloc((size_t)total * sizeof(CBMLSPDef)); if (!out) { - free(selected); - return NULL; + goto failed; } int n = 0; @@ -2048,4 +2243,7 @@ CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx, CBMLSPDef *out_success = true; free(selected); return out; +failed: + free(selected); + return NULL; } diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index a91acaf83..265017c50 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -75,6 +75,8 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 }; #include "pipeline/pass_lsp_cross.h" /* cbm_pxc_* helpers for fused cross-file LSP */ #include "pipeline/lsp_resolve.h" #include "lsp/rust_cargo.h" +#include "lsp/c_lsp.h" +#include "lsp/py_lsp.h" #include "helpers.h" /* cbm_kind_in_set_free_cache — per-worker-thread cache teardown */ #include "pipeline/worker_pool.h" #include "foundation/compat.h" @@ -571,9 +573,9 @@ static bool is_checked_exception(const char *name) { static const char *resolve_as_class(const cbm_registry_t *reg, const char *name, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { + const char **imp_vals, int imp_count, CBMLanguage lang) { cbm_resolution_t res = - cbm_registry_resolve(reg, name, module_qn, imp_keys, imp_vals, imp_count); + cbm_registry_resolve_lang(reg, name, module_qn, imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { return NULL; } @@ -1292,12 +1294,12 @@ static void extract_worker(int worker_id, void *ctx_ptr) { snprintf(how_far, sizeof(how_far), "%u/%u nodes walked", result->walk_nodes_visited, result->tree_nodes); pp_err_add(errs, fi->rel_path, how_far, "walk_truncated"); - } else if (result->lsp_skipped) { - /* Indexed, but without the per-file and cross-file LSP refinement: - * the same kind of hole from the other direction. Nothing in - * production sets this any more except a truncated walk (handled - * above) and the test seam — it is reported anyway, so that if - * something sets it again the gap arrives named, not silent. */ + } else if (result->lsp_skipped && !result->has_error) { + /* has_error already recorded its precise cause above. Indexed, but without the per-file + * and cross-file LSP refinement: the same kind of hole from the other direction. + * Nothing in production sets this any more except a truncated walk (handled above) and + * the test seam — it is reported anyway, so that if something sets it again the gap + * arrives named, not silent. */ char size_text[CBM_SZ_64]; snprintf(size_text, sizeof(size_text), "%u nodes", result->tree_nodes); pp_err_add(errs, fi->rel_path, size_text, "lsp_skipped"); @@ -1916,6 +1918,9 @@ typedef struct { /* Counters for parallel.resolve.lsp_cross_done summary. */ _Atomic int lsp_cross_processed; _Atomic int lsp_cross_skipped_no_source; + /* Each file is owned by one worker. The coordinator reports failures + * after join, without mutating the pipeline's error list concurrently. */ + const char **lsp_failures; /* Per-sub-phase timing (ns aggregated across workers) — surfaces * exactly where parallel_resolve's wall time is spent so we stop @@ -2214,7 +2219,7 @@ static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sou const CBMCall *call, const char *route_path, const char *handler_ref, const char *module_qn, const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, - const char **ik, const char **iv, int ic) { + const char **ik, const char **iv, int ic, CBMLanguage lang) { const char *method = cbm_service_pattern_route_method(call->callee_name); char rqn[CBM_ROUTE_QN_SIZE]; char cpath[CBM_SZ_256]; @@ -2233,7 +2238,8 @@ static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sou esc_rp); cbm_gbuf_insert_edge(gbuf, source->id, rid, "CALLS", props); if (handler_ref && handler_ref[0] != '\0') { - cbm_resolution_t hres = cbm_registry_resolve(registry, handler_ref, module_qn, ik, iv, ic); + cbm_resolution_t hres = + cbm_registry_resolve_lang(registry, handler_ref, module_qn, ik, iv, ic, lang); if (hres.qualified_name && hres.qualified_name[0] != '\0') { const cbm_gbuf_node_t *h = cbm_gbuf_find_by_qn(main_gbuf, hres.qualified_name); if (h) { @@ -2526,7 +2532,7 @@ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const cbm_resolution_t *res, const char *module_qn, const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, const char **imp_keys, const char **imp_vals, int imp_count, - bool suppress_plain_calls) { + bool suppress_plain_calls, CBMLanguage lang) { cbm_svc_kind_t svc = cbm_service_pattern_match(res->qualified_name); const char *arg = call->first_string_arg; @@ -2553,7 +2559,7 @@ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const char *route_path = find_route_path_in_args(call, &handler_ref); if (route_path) { emit_route_registration(gbuf, source, call, route_path, handler_ref, module_qn, - registry, main_gbuf, imp_keys, imp_vals, imp_count); + registry, main_gbuf, imp_keys, imp_vals, imp_count, lang); return; } /* No path found — fall through to normal CALLS edge */ @@ -2991,8 +2997,8 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB lang, lsp->strategy, lsp->callee_qn, rc->project_name); if ((!res.qualified_name || !res.qualified_name[0]) && !call->requires_lsp_resolution && !rust_external) { - res = cbm_registry_resolve(rc->registry, call->callee_name, module_qn, imp_keys, - imp_vals, imp_count); + res = cbm_registry_resolve_lang(rc->registry, call->callee_name, module_qn, imp_keys, + imp_vals, imp_count, lang); } atomic_fetch_add_explicit(&rc->time_ns_rc_resolve, extract_now_ns() - _rc_t0, memory_order_relaxed); @@ -3096,7 +3102,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB .strategy = "service_pattern"}; emit_service_edge(ws->local_edge_buf, source_node, source_node, call, &svc_res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, - imp_count, false); + imp_count, false, lang); continue; } } @@ -3113,7 +3119,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB * self-call, so it keeps only the route/service edges. */ emit_service_edge(ws->local_edge_buf, source_node, source_node, call, &fake_res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, - imp_count, rust_external); + imp_count, rust_external, lang); } else if (cbm_service_pattern_is_global_fetch(call->callee_name)) { /* Native `fetch()` (#856): only the global API once resolution * has failed to find a local/imported `fetch`. Call the low-level @@ -3165,7 +3171,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB if (url_or_topic) { emit_service_edge(ws->local_edge_buf, source_node, NULL, call, &res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count, - false); + false, lang); ws->calls_resolved++; } } @@ -3174,7 +3180,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB _rc_t0 = extract_now_ns(); emit_service_edge(ws->local_edge_buf, source_node, target_node, call, &res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count, - drop_plain_call); + drop_plain_call, lang); atomic_fetch_add_explicit(&rc->time_ns_rc_emit, extract_now_ns() - _rc_t0, memory_order_relaxed); ws->calls_resolved++; @@ -3240,8 +3246,8 @@ static void resolve_file_usages(resolve_ctx_t *rc, resolve_worker_state_t *ws, (lang == CBM_LANG_SQL) ? cbm_registry_resolve_lineage(rc->registry, usage->ref_name, module_qn, imp_keys, imp_vals, imp_count) - : cbm_registry_resolve(rc->registry, usage->ref_name, module_qn, imp_keys, - imp_vals, imp_count); + : cbm_registry_resolve_lang(rc->registry, usage->ref_name, module_qn, imp_keys, + imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { continue; } @@ -3290,7 +3296,8 @@ static void resolve_file_usages(resolve_ctx_t *rc, resolve_worker_state_t *ws, /* Resolve throws/raises for one file. */ static void resolve_file_throws(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFileResult *result, const char *rel, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { + const char **imp_keys, const char **imp_vals, int imp_count, + CBMLanguage lang) { for (int t = 0; t < result->throws.count; t++) { CBMThrow *thr = &result->throws.items[t]; if (!thr->exception_name || !thr->enclosing_func_qn) { @@ -3305,8 +3312,8 @@ static void resolve_file_throws(resolve_ctx_t *rc, resolve_worker_state_t *ws, continue; } const char *edge_type = is_checked_exception(thr->exception_name) ? "THROWS" : "RAISES"; - cbm_resolution_t res = cbm_registry_resolve(rc->registry, thr->exception_name, module_qn, - imp_keys, imp_vals, imp_count); + cbm_resolution_t res = cbm_registry_resolve_lang( + rc->registry, thr->exception_name, module_qn, imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { continue; } @@ -3332,8 +3339,8 @@ static void resolve_file_rw(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFi if (!src) { continue; } - cbm_resolution_t res = cbm_registry_resolve(rc->registry, rw->var_name, module_qn, imp_keys, - imp_vals, imp_count); + cbm_resolution_t res = cbm_registry_resolve_lang(rc->registry, rw->var_name, module_qn, + imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { continue; } @@ -3360,12 +3367,14 @@ static void resolve_file_rw(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFi /* Resolve base_classes → INHERITS edges for one definition. */ static void resolve_def_inherits(resolve_ctx_t *rc, resolve_worker_state_t *ws, const CBMDefinition *def, const cbm_gbuf_node_t *node, - const char *mq, const char **ik, const char **iv, int ic) { + const char *mq, const char **ik, const char **iv, int ic, + CBMLanguage lang) { if (!def->base_classes) { return; } for (int b = 0; def->base_classes[b]; b++) { - const char *bqn = resolve_as_class(rc->registry, def->base_classes[b], mq, ik, iv, ic); + const char *bqn = + resolve_as_class(rc->registry, def->base_classes[b], mq, ik, iv, ic, lang); if (!bqn) { continue; } @@ -3384,7 +3393,8 @@ static void resolve_def_inherits(resolve_ctx_t *rc, resolve_worker_state_t *ws, /* Resolve decorators → DECORATES edges for one definition. */ static void resolve_def_decorators(resolve_ctx_t *rc, resolve_worker_state_t *ws, const CBMDefinition *def, const cbm_gbuf_node_t *node, - const char *mq, const char **ik, const char **iv, int ic) { + const char *mq, const char **ik, const char **iv, int ic, + CBMLanguage lang) { if (!def->decorators) { return; } @@ -3394,14 +3404,14 @@ static void resolve_def_decorators(resolve_ctx_t *rc, resolve_worker_state_t *ws if (fn[0] == '\0') { continue; } - cbm_resolution_t res = cbm_registry_resolve(rc->registry, fn, mq, ik, iv, ic); + cbm_resolution_t res = cbm_registry_resolve_lang(rc->registry, fn, mq, ik, iv, ic, lang); if ((!res.qualified_name || res.qualified_name[0] == '\0') && !strchr(fn, '.')) { /* C# attributes are referenced by their short name (`[Log]`) but * declared with an `Attribute` suffix (`class LogAttribute`). */ char with_suffix[CBM_SZ_256]; int wn = snprintf(with_suffix, sizeof(with_suffix), "%sAttribute", fn); if (wn > 0 && (size_t)wn < sizeof(with_suffix)) { - res = cbm_registry_resolve(rc->registry, with_suffix, mq, ik, iv, ic); + res = cbm_registry_resolve_lang(rc->registry, with_suffix, mq, ik, iv, ic, lang); } } const cbm_gbuf_node_t *dn = NULL; @@ -3454,7 +3464,8 @@ static void resolve_def_decorators(resolve_ctx_t *rc, resolve_worker_state_t *ws /* Resolve INHERITS + DECORATES + IMPLEMENTS for one file. */ static void resolve_file_semantic(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFileResult *result, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { + const char **imp_keys, const char **imp_vals, int imp_count, + CBMLanguage lang) { for (int d = 0; d < result->defs.count; d++) { CBMDefinition *def = &result->defs.items[d]; if (!def->qualified_name) { @@ -3464,8 +3475,8 @@ static void resolve_file_semantic(resolve_ctx_t *rc, resolve_worker_state_t *ws, if (!node) { continue; } - resolve_def_inherits(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count); - resolve_def_decorators(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count); + resolve_def_inherits(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count, lang); + resolve_def_decorators(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count, lang); } for (int t = 0; t < result->impl_traits.count; t++) { CBMImplTrait *it = &result->impl_traits.items[t]; @@ -3473,9 +3484,9 @@ static void resolve_file_semantic(resolve_ctx_t *rc, resolve_worker_state_t *ws, continue; } const char *tqn = resolve_as_class(rc->registry, it->trait_name, module_qn, imp_keys, - imp_vals, imp_count); + imp_vals, imp_count, lang); const char *sqn = resolve_as_class(rc->registry, it->struct_name, module_qn, imp_keys, - imp_vals, imp_count); + imp_vals, imp_count, lang); if (!tqn || !sqn) { continue; } @@ -3748,11 +3759,21 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { * file around the resolve so a hang HERE is attributed to * this file, not to a stale extraction marker. */ cbm_index_mark_start(rel); + bool had_error = result->has_error; cbm_pxc_dispatch_file(lang, result, lsp_source, lsp_source_len, rel, def_module, rc->cross_registries, rc->module_def_index, rc->all_defs, rc->def_count, imp_keys, imp_vals, imp_count, pp_rust_shared_registry_get, rc); cbm_index_mark_done(rel); + if (!had_error && result->has_error && result->error_msg && + (strcmp(result->error_msg, CBM_C_LSP_MEMO_ERROR) == 0 || + strcmp(result->error_msg, CBM_PY_LSP_MEMO_ERROR) == 0 || + strcmp(result->error_msg, CBM_C_LSP_DEPTH_ERROR) == 0 || + strcmp(result->error_msg, CBM_PY_LSP_DEPTH_ERROR) == 0 || + strcmp(result->error_msg, CBM_PY_LSP_SCOPE_ERROR) == 0)) { + rc->lsp_failures[file_idx] = result->error_msg; + ws->errors++; + } /* Free the on-demand re-read (no-op when source was retained). */ free_source(lsp_source_owned); /* Contract: cbm_slab_reclaim() requires the thread parser to be @@ -3802,7 +3823,7 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { /* ── THROWS / RAISES ───────────────────────────────────── */ _ph_t0 = extract_now_ns(); - resolve_file_throws(rc, ws, result, rel, module_qn, imp_keys, imp_vals, imp_count); + resolve_file_throws(rc, ws, result, rel, module_qn, imp_keys, imp_vals, imp_count, lang); atomic_fetch_add_explicit(&rc->time_ns_throws, extract_now_ns() - _ph_t0, memory_order_relaxed); @@ -3813,7 +3834,7 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { /* ── INHERITS + DECORATES + IMPLEMENTS ──────────────────── */ _ph_t0 = extract_now_ns(); - resolve_file_semantic(rc, ws, result, module_qn, imp_keys, imp_vals, imp_count); + resolve_file_semantic(rc, ws, result, module_qn, imp_keys, imp_vals, imp_count, lang); atomic_fetch_add_explicit(&rc->time_ns_semantic, extract_now_ns() - _ph_t0, memory_order_relaxed); @@ -3856,6 +3877,10 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, return 0; } + if (file_count < 0 || (size_t)file_count > SIZE_MAX / sizeof(const char *)) { + return CBM_NOT_FOUND; + } + cbm_log_info("parallel.resolve.start", "files", itoa_log(file_count), "workers", itoa_log(worker_count)); @@ -3865,6 +3890,12 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, return CBM_NOT_FOUND; } memset(workers, 0, (size_t)worker_count * sizeof(resolve_worker_state_t)); + const char **lsp_failures = + cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)file_count * sizeof(*lsp_failures)); + if (!lsp_failures) { + cbm_aligned_free(workers); + return CBM_NOT_FOUND; + } bool have_rust = false; for (int i = 0; i < file_count; i++) { @@ -3905,6 +3936,7 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, .module_def_index = module_def_index, .cross_registries = cross_registries, .rust_manifest = rust_manifest_ptr, + .lsp_failures = lsp_failures, }; atomic_init(&rc.next_file_idx, 0); atomic_init(&rc.lsp_cross_processed, 0); @@ -3922,6 +3954,12 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, cbm_scale_end(&rc.scale); CBM_PROF_END_N("parallel_resolve", "1_dispatch_workers_parallel", t_resolve_dispatch, file_count); + for (int i = 0; i < file_count; i++) { + if (lsp_failures[i] && ctx->pipeline) + cbm_pipeline_add_file_error(ctx->pipeline, files[i].rel_path, lsp_failures[i], + "lsp_skipped"); + } + cbm_free(CBM_MEM_CLASS_OTHER, lsp_failures); /* Workers joined: the shared Rust registry (if built) is no longer read. * Free its dedicated arena + the lock (registry was self-contained: it strdup'd * all QNs, so freeing all_defs afterward is safe). */ diff --git a/src/pipeline/pass_semantic.c b/src/pipeline/pass_semantic.c index 48dab0083..434ff769f 100644 --- a/src/pipeline/pass_semantic.c +++ b/src/pipeline/pass_semantic.c @@ -177,9 +177,9 @@ static void free_import_map(const char **keys, const char **vals, int count) { /* Resolve a class/type name through the registry. Returns borrowed QN or NULL. */ static const char *resolve_as_class(const cbm_registry_t *reg, const char *name, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { + const char **imp_vals, int imp_count, CBMLanguage lang) { cbm_resolution_t res = - cbm_registry_resolve(reg, name, module_qn, imp_keys, imp_vals, imp_count); + cbm_registry_resolve_lang(reg, name, module_qn, imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { return NULL; } @@ -546,14 +546,14 @@ static void synth_decorator_qn(const char *func_name, char *out, size_t outsz) { /* Resolve one decorator and create DECORATES edge. */ static void resolve_decorator(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *node, const char *decorator, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count, int *count) { + const char **imp_vals, int imp_count, int *count, CBMLanguage lang) { char func_name[CBM_SZ_256]; extract_decorator_func(decorator, func_name, sizeof(func_name)); if (func_name[0] == '\0') { return; } - cbm_resolution_t res = - cbm_registry_resolve(ctx->registry, func_name, module_qn, imp_keys, imp_vals, imp_count); + cbm_resolution_t res = cbm_registry_resolve_lang(ctx->registry, func_name, module_qn, imp_keys, + imp_vals, imp_count, lang); if ((!res.qualified_name || res.qualified_name[0] == '\0') && !strchr(func_name, '.')) { /* C# attributes are referenced by their short name (`[Log]`) but declared * with the conventional `Attribute` suffix (`class LogAttribute`). Retry @@ -561,8 +561,8 @@ static void resolve_decorator(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *no char with_suffix[CBM_SZ_256]; int wn = snprintf(with_suffix, sizeof(with_suffix), "%sAttribute", func_name); if (wn > 0 && (size_t)wn < sizeof(with_suffix)) { - res = cbm_registry_resolve(ctx->registry, with_suffix, module_qn, imp_keys, imp_vals, - imp_count); + res = cbm_registry_resolve_lang(ctx->registry, with_suffix, module_qn, imp_keys, + imp_vals, imp_count, lang); } } const cbm_gbuf_node_t *dec = NULL; @@ -613,7 +613,7 @@ static void resolve_decorator(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *no static void sem_process_def_edges(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const char *module_qn, const char **imp_keys, const char **imp_vals, int imp_count, int *inherits_count, - int *decorates_count) { + int *decorates_count, CBMLanguage lang) { if (!def->qualified_name) { return; } @@ -624,7 +624,7 @@ static void sem_process_def_edges(cbm_pipeline_ctx_t *ctx, const CBMDefinition * if (def->base_classes) { for (int b = 0; def->base_classes[b]; b++) { const char *base_qn = resolve_as_class(ctx->registry, def->base_classes[b], module_qn, - imp_keys, imp_vals, imp_count); + imp_keys, imp_vals, imp_count, lang); if (!base_qn) { continue; } @@ -646,7 +646,7 @@ static void sem_process_def_edges(cbm_pipeline_ctx_t *ctx, const CBMDefinition * if (def->decorators) { for (int dc = 0; def->decorators[dc]; dc++) { resolve_decorator(ctx, node, def->decorators[dc], module_qn, imp_keys, imp_vals, - imp_count, decorates_count); + imp_count, decorates_count, lang); } } } @@ -683,12 +683,12 @@ static int resolve_impl_traits(cbm_pipeline_ctx_t *ctx, const CBMFileResult *res continue; } const char *trait_qn = resolve_as_class(ctx->registry, it->trait_name, module_qn, imp_keys, - imp_vals, imp_count); + imp_vals, imp_count, CBM_LANG_RUST); if (!trait_qn) { continue; } const char *struct_qn = resolve_as_class(ctx->registry, it->struct_name, module_qn, - imp_keys, imp_vals, imp_count); + imp_keys, imp_vals, imp_count, CBM_LANG_RUST); if (!struct_qn) { continue; } @@ -737,7 +737,7 @@ int cbm_pipeline_pass_semantic(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *f /* ── INHERITS + DECORATES from definitions ──────────────── */ for (int d = 0; d < result->defs.count; d++) { sem_process_def_edges(ctx, &result->defs.items[d], module_qn, imp_keys, imp_vals, - imp_count, &inherits_count, &decorates_count); + imp_count, &inherits_count, &decorates_count, files[i].language); } /* ── IMPLEMENTS from impl_traits (Rust) ─────────────────── */ diff --git a/src/pipeline/pass_usages.c b/src/pipeline/pass_usages.c index 9738f7198..096130775 100644 --- a/src/pipeline/pass_usages.c +++ b/src/pipeline/pass_usages.c @@ -200,8 +200,8 @@ static int resolve_usage_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *res (lang == CBM_LANG_SQL) ? cbm_registry_resolve_lineage(ctx->registry, usage->ref_name, module_qn, imp_keys, imp_vals, imp_count) - : cbm_registry_resolve(ctx->registry, usage->ref_name, module_qn, imp_keys, - imp_vals, imp_count); + : cbm_registry_resolve_lang(ctx->registry, usage->ref_name, module_qn, imp_keys, + imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { continue; } @@ -248,7 +248,7 @@ static int resolve_usage_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *res /* Resolve THROWS/RAISES edges for one file's extracted throws. */ static int resolve_throw_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result, const char *rel, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { + const char **imp_vals, int imp_count, CBMLanguage lang) { int resolved = 0; for (int t = 0; t < result->throws.count; t++) { CBMThrow *thr = &result->throws.items[t]; @@ -262,8 +262,8 @@ static int resolve_throw_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *res } const char *edge_type = is_checked_exception(thr->exception_name) ? "THROWS" : "RAISES"; - cbm_resolution_t res = cbm_registry_resolve(ctx->registry, thr->exception_name, module_qn, - imp_keys, imp_vals, imp_count); + cbm_resolution_t res = cbm_registry_resolve_lang( + ctx->registry, thr->exception_name, module_qn, imp_keys, imp_vals, imp_count, lang); const cbm_gbuf_node_t *tgt = NULL; if (res.qualified_name && res.qualified_name[0]) { @@ -295,8 +295,8 @@ static int resolve_rw_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result continue; } - cbm_resolution_t res = cbm_registry_resolve(ctx->registry, rw->var_name, module_qn, - imp_keys, imp_vals, imp_count); + cbm_resolution_t res = cbm_registry_resolve_lang(ctx->registry, rw->var_name, module_qn, + imp_keys, imp_vals, imp_count, lang); if (!res.qualified_name || res.qualified_name[0] == '\0') { continue; } @@ -379,8 +379,8 @@ int cbm_pipeline_pass_usages(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fil usage_resolved += resolve_usage_edges(ctx, result, rel, module_qn, imp_keys, imp_vals, imp_count, files[i].language); - throw_resolved += - resolve_throw_edges(ctx, result, rel, module_qn, imp_keys, imp_vals, imp_count); + throw_resolved += resolve_throw_edges(ctx, result, rel, module_qn, imp_keys, imp_vals, + imp_count, files[i].language); rw_resolved += resolve_rw_edges(ctx, result, rel, module_qn, imp_keys, imp_vals, imp_count, files[i].language); diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index ba41b877c..32f9df6a5 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1156,7 +1156,8 @@ static void log_result_census(const char *tag, CBMFileResult **cache, int file_c (size_t)r->string_refs.count * sizeof(CBMStringRef) + (size_t)r->impl_traits.count * sizeof(CBMImplTrait) + (size_t)r->infra_bindings.count * sizeof(CBMInfraBinding) + - (size_t)r->channels.count * sizeof(CBMChannel); + (size_t)r->channels.count * sizeof(CBMChannel) + + (size_t)r->field_types.count * sizeof(CBMFieldType); cap_other += (size_t)r->imports.cap * sizeof(CBMImport) + (size_t)r->resolved_calls.cap * sizeof(CBMResolvedCall) + (size_t)r->throws.cap * sizeof(CBMThrow) + @@ -1165,7 +1166,8 @@ static void log_result_census(const char *tag, CBMFileResult **cache, int file_c (size_t)r->string_refs.cap * sizeof(CBMStringRef) + (size_t)r->impl_traits.cap * sizeof(CBMImplTrait) + (size_t)r->infra_bindings.cap * sizeof(CBMInfraBinding) + - (size_t)r->channels.cap * sizeof(CBMChannel); + (size_t)r->channels.cap * sizeof(CBMChannel) + + (size_t)r->field_types.cap * sizeof(CBMFieldType); n_defs += (size_t)r->defs.count; n_calls += (size_t)r->calls.count; n_usages += (size_t)r->usages.count; diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 154518658..0e06e8032 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1817,8 +1817,9 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p goto done; } - /* Early cutoff per changed file; surface-changed files seed the - * dependent query. Deleted files always seed it. */ + /* Validate Python namespace metadata before any hash cutoff or dependent + * query. Python changes require a full rebuild; other surface changes and + * deletions seed the dependent query. */ int n_surface_changed = 0; const char **dep_targets = (const char **)calloc((size_t)(n_changed + n_deleted), sizeof(char *)); @@ -1843,6 +1844,15 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p decline = "missing_surface_row"; goto done; } + int python_change = + cbm_lsp_surface_python_change(stored_row->defs_json, fresh_row->defs_json); + if (python_change != 0) { + free(dep_targets); + cbm_ht_free(rows_by_path); + decline = python_change < 0 ? "python_namespace_metadata_invalid" + : "python_namespace_changed"; + goto done; + } if (strcmp(stored_row->surface_sha, fresh_row->surface_sha) == 0) { continue; /* body edit: the file re-resolves, nobody else does */ } @@ -1858,6 +1868,17 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p dep_targets[dep_target_count++] = changed_paths[i]; } for (int i = 0; i < n_deleted; i++) { + const cbm_lsp_surface_row_t *stored_row = + cbm_ht_get(rows_by_path, changed_paths[n_changed + i]); + int python_change = + cbm_lsp_surface_python_change(stored_row ? stored_row->defs_json : NULL, NULL); + if (python_change != 0) { + free(dep_targets); + cbm_ht_free(rows_by_path); + decline = python_change < 0 ? "python_namespace_metadata_invalid" + : "python_namespace_changed"; + goto done; + } dep_targets[dep_target_count++] = changed_paths[n_changed + i]; } cbm_ht_free(rows_by_path); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 7f266eae9..397691ee3 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -21,6 +21,15 @@ #include #include +/* Internal explicit-language resolution; public raw/lineage APIs stay raw. */ +cbm_resolution_t cbm_registry_resolve_lang(const cbm_registry_t *r, const char *callee_name, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count, + CBMLanguage lang); +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_pxc_test_import_probe_fail_after(int successful_allocations); +#endif + /* ── Shared pipeline constants ─────────────────────────────────── */ /* Per-file tree-sitter parse budget, in MICROSECONDS of this thread's CPU time diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index 1f5335c10..bb8d19bdd 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -24,7 +24,7 @@ enum { REG_MAX_CANDIDATES = 256 }; #define REG_HALF_PENALTY 0.5 #define DEFAULT_CONFIDENCE 0.5 -#include "pipeline/pipeline.h" +#include "pipeline_internal.h" #include "cbm.h" /* cbm_label_is_relation — the resolve-time relation veto */ #include "foundation/compat.h" /* CBM_TLS */ #include "foundation/hash_table.h" @@ -387,6 +387,7 @@ void cbm_registry_import_map_cache_end(void) { * eliminated. */ typedef struct { cbm_resolution_t res; + size_t scope_len; /* raw and Python-scoped requests may share one file cache */ } resolve_cache_entry_t; static CBM_TLS CBMHashTable *_resolve_cache = NULL; @@ -1099,22 +1100,45 @@ static cbm_resolution_t resolve_import_map(const cbm_registry_t *r, const char * return empty_result(); } -/* Strategy 2: Same-module match */ -static cbm_resolution_t resolve_same_module(const cbm_registry_t *r, const char *callee_name, - const char *suffix, const char *module_qn) { +/* Exact same-module lookup without ever hashing a truncated candidate. The + * long path uses the canonical short-name bucket, not heuristic name limits. */ +static const char *same_module_candidate(const cbm_registry_t *r, const char *module_qn, + size_t scope_len, const char *name) { + if (!module_qn || !name) + return NULL; + size_t name_len = strlen(name); char candidate[CBM_SZ_512]; - snprintf(candidate, sizeof(candidate), "%s.%s", module_qn, callee_name); - const char *stored_key = cbm_ht_get_key(r->exact, candidate); - if (stored_key) { - return (cbm_resolution_t){stored_key, "same_module", CONF_SAME_MODULE, REG_RESOLVED}; - } - if (suffix && suffix[0]) { - snprintf(candidate, sizeof(candidate), "%s.%s", module_qn, suffix); - stored_key = cbm_ht_get_key(r->exact, candidate); - if (stored_key) { - return (cbm_resolution_t){stored_key, "same_module", CONF_SAME_MODULE, REG_RESOLVED}; - } + if (scope_len < sizeof(candidate) && name_len < sizeof(candidate) - scope_len - 1) { + memcpy(candidate, module_qn, scope_len); + candidate[scope_len] = '.'; + memcpy(candidate + scope_len + 1, name, name_len + 1); + return cbm_ht_get_key(r->exact, candidate); + } + qn_array_t *arr = (qn_array_t *)cbm_ht_get(r->by_name, simple_name(name)); + if (!arr) + return NULL; + for (int i = 0; i < arr->count; i++) { + const char *qn = arr->items[i]; + if (!qn) + continue; + size_t qn_len = strlen(qn); + if (qn_len > scope_len && qn_len - scope_len - 1 == name_len && + memcmp(qn, module_qn, scope_len) == 0 && qn[scope_len] == '.' && + memcmp(qn + scope_len + 1, name, name_len) == 0) + return qn; } + return NULL; +} + +/* Strategy 2: Same-module match. Scoring elsewhere retains the raw file QN. */ +static cbm_resolution_t resolve_same_module(const cbm_registry_t *r, const char *callee_name, + const char *suffix, const char *module_qn, + size_t scope_len) { + const char *stored_key = same_module_candidate(r, module_qn, scope_len, callee_name); + if (!stored_key && suffix && suffix[0]) + stored_key = same_module_candidate(r, module_qn, scope_len, suffix); + if (stored_key) + return (cbm_resolution_t){stored_key, "same_module", CONF_SAME_MODULE, REG_RESOLVED}; return empty_result(); } @@ -1360,7 +1384,8 @@ static cbm_resolution_t resolve_name_lookup(const cbm_registry_t *r, const char * cbm_registry_resolve owns the per-file cache). */ static cbm_resolution_t registry_resolve_chain(const cbm_registry_t *r, const char *callee_name, const char *module_qn, const char **import_map_keys, - const char **import_map_vals, int import_map_count) { + const char **import_map_vals, int import_map_count, + size_t scope_len) { /* Split callee at the first path separator: "pkg.Func" → prefix="pkg", * suffix="Func". Rust/C++ use "::" rather than ".", so honor whichever * separator appears first ("lib::square" → prefix="lib", suffix="square"). @@ -1393,7 +1418,7 @@ static cbm_resolution_t registry_resolve_chain(const cbm_registry_t *r, const ch resolve_import_map(r, prefix, suffix, import_map_keys, import_map_vals, import_map_count); if (!(res.qualified_name && res.qualified_name[0])) { /* Strategy 2: same module */ - res = resolve_same_module(r, callee_name, suffix, module_qn); + res = resolve_same_module(r, callee_name, suffix, module_qn, scope_len); } if (!(res.qualified_name && res.qualified_name[0])) { /* Strategy 3+4: name lookup */ @@ -1402,9 +1427,10 @@ static cbm_resolution_t registry_resolve_chain(const cbm_registry_t *r, const ch return res; } -cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *callee_name, - const char *module_qn, const char **import_map_keys, - const char **import_map_vals, int import_map_count) { +static cbm_resolution_t registry_resolve_scoped(const cbm_registry_t *r, const char *callee_name, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count, + size_t scope_len) { if (!r || !callee_name) { return empty_result(); } @@ -1412,16 +1438,13 @@ cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *calle /* Per-file cache: same callee_name in N call sites → 1 chain walk * + N-1 O(1) hash hits. module_qn is constant per file so the * cache key only needs callee_name. */ - if (_resolve_cache) { - resolve_cache_entry_t *cached = - (resolve_cache_entry_t *)cbm_ht_get(_resolve_cache, callee_name); - if (cached) { - return cached->res; - } - } + resolve_cache_entry_t *cached = + _resolve_cache ? (resolve_cache_entry_t *)cbm_ht_get(_resolve_cache, callee_name) : NULL; + if (cached && cached->scope_len == scope_len) + return cached->res; cbm_resolution_t res = registry_resolve_chain(r, callee_name, module_qn, import_map_keys, - import_map_vals, import_map_count); + import_map_vals, import_map_count, scope_len); /* Data relations (Table/View) are lineage-only registry members: common * table names (users, orders, config) collide with code identifiers across @@ -1437,10 +1460,15 @@ cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *calle /* Cache the result (including empty — caching the negative answer * is just as valuable; same name asks the same question). */ - if (_resolve_cache) { + if (cached) { + /* Same key, different effective scope: reuse its owned entry/key. */ + cached->res = res; + cached->scope_len = scope_len; + } else if (_resolve_cache) { resolve_cache_entry_t *e = (resolve_cache_entry_t *)malloc(sizeof(*e)); if (e) { e->res = res; + e->scope_len = scope_len; char *kdup = strdup(callee_name); if (kdup) { cbm_ht_set(_resolve_cache, kdup, e); @@ -1452,6 +1480,23 @@ cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *calle return res; } +cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *callee_name, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count) { + return registry_resolve_scoped(r, callee_name, module_qn, import_map_keys, import_map_vals, + import_map_count, module_qn ? strlen(module_qn) : 0); +} + +cbm_resolution_t cbm_registry_resolve_lang(const cbm_registry_t *r, const char *callee_name, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count, + CBMLanguage lang) { + size_t scope_len = lang == CBM_LANG_PYTHON ? cbm_fqn_symbol_scope_len(module_qn) + : (module_qn ? strlen(module_qn) : 0); + return registry_resolve_scoped(r, callee_name, module_qn, import_map_keys, import_map_vals, + import_map_count, scope_len); +} + cbm_resolution_t cbm_registry_resolve_lineage(const cbm_registry_t *r, const char *callee_name, const char *module_qn, const char **import_map_keys, const char **import_map_vals, int import_map_count) { @@ -1464,7 +1509,7 @@ cbm_resolution_t cbm_registry_resolve_lineage(const cbm_registry_t *r, const cha * it would poison one variant with the other's semantics. SQL files hold * few distinct relation refs, so the chain walk stays cheap. */ return registry_resolve_chain(r, callee_name, module_qn, import_map_keys, import_map_vals, - import_map_count); + import_map_count, module_qn ? strlen(module_qn) : 0); } /* ── Fuzzy Resolve ──────────────────────────────────────────────── */ diff --git a/src/store/store.h b/src/store/store.h index 06ea1fbb7..12e4e6483 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -23,7 +23,7 @@ typedef struct cbm_store cbm_store_t; #define CBM_STORE_OK 0 #define CBM_STORE_ERR (-1) #define CBM_STORE_NOT_FOUND (-2) -#define CBM_INDEX_FORMAT_VERSION 1 +#define CBM_INDEX_FORMAT_VERSION 2 #define CBM_STORE_CANCELLED (-3) #define CBM_STORE_SCAN_LIMIT (-4) #define CBM_STORE_CALLBACK_ERR (-5) @@ -112,10 +112,11 @@ void cbm_store_node_degree(cbm_store_t *s, int64_t node_id, int *in_deg, int *ou * stores that have no File nodes. Caller frees each out[i] and out itself. */ int cbm_store_list_files(cbm_store_t *s, const char *project, char ***out, int *count); -/* Persisted index-format identity. Bump when a change alters the QN scheme - * or node identity of an already-written graph, so an old DB is routed - * through the full-reindex path instead of producing a mixed graph. - * 1 = File QNs keep the file extension (#769). */ +/* Persisted index-format identity. Bump when a change alters the QN scheme, + * node identity, or required persisted semantic metadata, so an old DB is + * routed through the full-reindex path before incremental/no-op admission. + * 1 = File QNs keep the file extension (#769). + * 2 = Python namespace facts use the versioned LSP surface contract. */ int cbm_store_get_format_version(cbm_store_t *s, int *out); int cbm_store_set_format_version(cbm_store_t *s, int version); diff --git a/tests/test_c_lsp.c b/tests/test_c_lsp.c index 933cacc36..dbbef1d85 100644 --- a/tests/test_c_lsp.c +++ b/tests/test_c_lsp.c @@ -19,6 +19,7 @@ * - DLL patterns, SFINAE, placement new */ #include "test_framework.h" +#include "../src/foundation/compat.h" #include "cbm.h" #include "lsp/c_lsp.h" #include "lsp/py_lsp.h" @@ -26,7 +27,9 @@ #include "lsp/ts_lsp.h" #include "lsp/go_lsp.h" #include "lsp/type_registry.h" +#include "lsp/lsp_work.h" #include "../src/pipeline/lsp_resolve.h" +#include "../src/pipeline/pass_lsp_cross.h" #include "arena.h" #include "preprocessor.h" #include @@ -15350,6 +15353,10 @@ TEST(registry_overlay_chain_iterates_and_copies_on_write) { memset(&t, 0, sizeof(t)); t.qualified_name = "pkg.T"; t.short_name = "T"; + const char *field_names[] = {"untyped", "typed", NULL}; + const CBMType *field_types[] = {NULL, cbm_type_named(&arena, "pkg.Value"), NULL}; + t.field_names = field_names; + t.field_types = field_types; cbm_registry_add_type(&base, t); cbm_registry_finalize(&base); base.read_only = true; @@ -15414,6 +15421,10 @@ TEST(registry_overlay_chain_iterates_and_copies_on_write) { CBMRegisteredType *wt = cbm_registry_type_for_update(&overlay, "pkg.T"); ASSERT_NOT_NULL(wt); ASSERT_EQ(overlay.type_count, 1); + /* A missing type for an earlier named field must not truncate the copy. */ + bool field_slots_preserved = wt->field_types && wt->field_types != field_types && + wt->field_types[0] == NULL && + wt->field_types[1] == field_types[1] && wt->field_types[2] == NULL; cbm_registry_all_types_chain(&overlay, &ti); seen = 0; while (cbm_type_short_iter_next(&ti) >= 0) { @@ -15430,6 +15441,7 @@ TEST(registry_overlay_chain_iterates_and_copies_on_write) { cbm_arena_destroy(&scratch); cbm_arena_destroy(&arena); + ASSERT_TRUE(field_slots_preserved); PASS(); } @@ -15804,6 +15816,111 @@ TEST(seal_py_shared_registry_readonly_fields) { PASS(); } +/* #1277: the production resolver receives a writable overlay, so refining an + * existing field must detach its pointer array from the sealed fallback. */ +TEST(seal_py_overlay_existing_field_keeps_shared_type) { + CBMArena arena; + cbm_arena_init(&arena); + CBMLSPDef defs[4] = {0}; + defs[0].qualified_name = "test.mod.Foo"; + defs[0].short_name = "Foo"; + defs[0].label = "Class"; + defs[0].def_module_qn = "test.mod"; + defs[0].field_defs = "x:test.contracts.Contract"; + defs[1].qualified_name = "test.mod.Foo.m"; + defs[1].short_name = "m"; + defs[1].label = "Method"; + defs[1].receiver_type = "test.mod.Foo"; + defs[1].def_module_qn = "test.mod"; + defs[2].qualified_name = "test.contracts.Contract"; + defs[2].short_name = "Contract"; + defs[2].label = "Class"; + defs[2].def_module_qn = "test.contracts"; + defs[3].qualified_name = "test.contracts.Contract.process_batch"; + defs[3].short_name = "process_batch"; + defs[3].label = "Method"; + defs[3].receiver_type = "test.contracts.Contract"; + defs[3].def_module_qn = "test.contracts"; + for (int i = 0; i < 4; i++) { + defs[i].lang = CBM_LANG_PYTHON; + } + CBMTypeRegistry *reg = cbm_py_build_cross_registry(&arena, defs, 4); + ASSERT_NOT_NULL(reg); + ASSERT_TRUE(reg->read_only); + const CBMRegisteredType *base = cbm_registry_lookup_type(reg, "test.mod.Foo"); + ASSERT_NOT_NULL(base); + ASSERT_NOT_NULL(base->field_names); + ASSERT_NOT_NULL(base->field_types); + const char **names_before = base->field_names; + const CBMType **types_before = base->field_types; + const CBMType *type_before = base->field_types[0]; + ASSERT_NOT_NULL(type_before); + ASSERT_NULL(base->field_types[1]); + + CBMArena scratch; + cbm_arena_init(&scratch); + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &scratch); + overlay.fallback = reg; + const char *writer = "class Foo:\n" + " def m(self):\n" + " self.x = 1\n"; + CBMResolvedCallArray out = {0}; + cbm_run_py_lsp_cross_with_registry(&scratch, writer, (int)strlen(writer), "test.mod", &overlay, + NULL, NULL, 0, NULL, &out, NULL); + + /* Observe while scratch is live, then release both arenas before any + * failure assertion so the regression also fails cleanly under LSan. */ + bool shared_names_unchanged = base->field_names == names_before; + bool shared_types_unchanged = base->field_types == types_before; + bool shared_element_unchanged = base->field_types[0] == type_before; + bool shared_terminator_unchanged = base->field_types[1] == NULL; + const CBMRegisteredType *refined = cbm_registry_lookup_type(&overlay, "test.mod.Foo"); + bool local_refined = refined && refined != base && refined->field_types && + refined->field_types != types_before && refined->field_types[0] && + refined->field_types[0] != type_before && + refined->field_types[0]->kind == CBM_TYPE_BUILTIN && + refined->field_types[1] == NULL; + cbm_arena_destroy(&scratch); + + /* Never read a corrupted shared pointer after scratch destruction. When + * intact, a later file must still see the original Contract field. */ + int hits = 0; + if (shared_names_unchanged && shared_types_unchanged && shared_element_unchanged && + shared_terminator_unchanged && local_refined) { + cbm_arena_init(&scratch); + cbm_registry_init(&overlay, &scratch); + overlay.fallback = reg; + const char *reader = "from mod import Foo\n\n" + "def run(foo: Foo):\n" + " foo.x.process_batch()\n"; + const char *import_names[] = {"Foo"}; + const char *import_qns[] = {"test.mod.Foo"}; + memset(&out, 0, sizeof(out)); + cbm_run_py_lsp_cross_with_registry(&scratch, reader, (int)strlen(reader), "test.reader", + &overlay, import_names, import_qns, 1, NULL, &out, NULL); + for (int i = 0; i < out.count; i++) { + const CBMResolvedCall *call = &out.items[i]; + if (call->caller_qn && call->callee_qn && + strcmp(call->caller_qn, "test.reader.run") == 0 && + strcmp(call->callee_qn, "test.contracts.Contract.process_batch") == 0 && + call->kind == CBM_RESOLVED_INVOCATION && + call->confidence >= CBM_LSP_CONFIDENCE_FLOOR) { + hits++; + } + } + cbm_arena_destroy(&scratch); + } + cbm_arena_destroy(&arena); + ASSERT_TRUE(shared_names_unchanged); + ASSERT_TRUE(shared_types_unchanged); + ASSERT_TRUE(shared_element_unchanged); + ASSERT_TRUE(shared_terminator_unchanged); + ASSERT_TRUE(local_refined); + ASSERT_EQ(hits, 1); + PASS(); +} + TEST(seal_cs_shared_registry_readonly) { CBMArena arena; cbm_arena_init(&arena); @@ -16385,7 +16502,479 @@ TEST(clsp_preprocessed_destructor_rewrite_respects_origin_during_rewrite) { PASS(); } +/* ── #1527: no per-file eval budget; cost fixed at the root ────────────── */ + +/* Resolved rows that name a real target (not an lsp_unresolved marker) whose + * callee QN contains sub. */ +static int count_real_resolved(const CBMFileResult *r, const char *sub) { + int n = 0; + for (int i = 0; i < r->resolved_calls.count; i++) { + const CBMResolvedCall *rc = &r->resolved_calls.items[i]; + if (rc->callee_qn && strstr(rc->callee_qn, sub) && rc->confidence > 0.0f && + !(rc->strategy && strcmp(rc->strategy, "lsp_unresolved") == 0)) + n++; + } + return n; +} + +/* A versionbits-style fluent chain (the #323 hang): every link returns + * Tester&, and the evaluator re-evaluates a call's receiver for template + * substitution, so without a memo an N-link chain costs 2^N evaluations. The + * per-file step cap that used to contain it is gone; the per-evaluation memo + * must keep the work polynomial AND every link must resolve. */ +static char *clsp_chain_source(int links, int stmts) { + size_t sz = 512 + (size_t)stmts * (32 + (size_t)links * 40); + char *src = malloc(sz); + if (!src) + return NULL; + size_t pos = (size_t)snprintf(src, sz, + "struct Tester {\n" + " Tester& Mine(int h, int t, int v);\n" + " Tester& TestDefined();\n" + " Tester& TestStateSinceHeight(int h);\n" + " Tester& Reset();\n" + "};\n" + "void run() {\n" + " Tester t;\n"); + for (int s = 0; s < stmts; s++) { + pos += (size_t)snprintf(src + pos, sz - pos, " t.Reset()"); + for (int l = 0; l < links; l++) { + if (l % 3 == 0) + pos += (size_t)snprintf(src + pos, sz - pos, ".Mine(%d, %d, 0x100)", l, l); + else if (l % 3 == 1) + pos += (size_t)snprintf(src + pos, sz - pos, ".TestDefined()"); + else + pos += (size_t)snprintf(src + pos, sz - pos, ".TestStateSinceHeight(%d)", l); + } + pos += (size_t)snprintf(src + pos, sz - pos, ";\n"); + } + snprintf(src + pos, sz - pos, "}\n"); + return src; +} + +TEST(clsp_issue1527_fluent_chain_work_is_polynomial) { + enum { LINKS = 14, STMTS = 3 }; + char *src = clsp_chain_source(LINKS, STMTS); + ASSERT_NOT_NULL(src); + (void)cbm_lsp_work_take(); + CBMFileResult *r = extract_cpp(src); + uint64_t work = cbm_lsp_work_take(); + free(src); + ASSERT_NOT_NULL(r); + printf(" chain %dx%d: work=%llu\n", LINKS, STMTS, (unsigned long long)work); + /* Every link of every statement resolves to its Tester method. */ + ASSERT_EQ(count_real_resolved(r, ".Tester."), STMTS * (LINKS + 1)); + /* Memoized: ~LINKS^2 evaluations per statement (each call node starts a + * fresh outermost evaluation of its receiver chain) -- ~1.5k in total. + * Unmemoized: ~2^LINKS per call node, ~100k in total. */ + ASSERT(work < 10000); + cbm_free_result(r); + PASS(); +} + +/* Deterministic complexity gate for the C/C++ resolver (O9: no clock): 4x the + * definitions and calls must cost ~4x the counted work (scope-frame probes, + * registry lookup probes, enclosing-definition candidates, real + * evaluations). Methods are DEFINED (out of class), so they are registered + * before the registry is indexed. Declaration-only methods are registered by + * the walk itself, after indexing, and are still found by a linear tail scan + * -- a separate, known quadratic (see the #1527 follow-up), not measured here. */ +static uint64_t clsp_member_calls_work(int n, int *out_real) { + size_t sz = (size_t)n * 128 + 512; + char *src = malloc(sz); + if (!src) + return 0; + size_t pos = (size_t)snprintf(src, sz, "struct S {\n"); + for (int i = 0; i < n; i++) + pos += (size_t)snprintf(src + pos, sz - pos, " S& m%d();\n", i); + pos += (size_t)snprintf(src + pos, sz - pos, "};\n"); + for (int i = 0; i < n; i++) + pos += (size_t)snprintf(src + pos, sz - pos, "S& S::m%d() { return *this; }\n", i); + pos += (size_t)snprintf(src + pos, sz - pos, "void use() {\n S s;\n"); + for (int i = 0; i < n; i++) + pos += (size_t)snprintf(src + pos, sz - pos, " s.m%d().m%d();\n", i, (i + 1) % n); + snprintf(src + pos, sz - pos, "}\n"); + (void)cbm_lsp_work_take(); + CBMFileResult *r = extract_cpp(src); + uint64_t work = cbm_lsp_work_take(); + free(src); + *out_real = r ? count_real_resolved(r, ".S.m") : -1; + if (r) + cbm_free_result(r); + return work; +} + +TEST(clsp_issue1527_scale_work_is_linear) { + int real1 = 0, real4 = 0; + uint64_t w1 = clsp_member_calls_work(1000, &real1); + uint64_t w4 = clsp_member_calls_work(4000, &real4); + double ratio = w1 ? (double)w4 / (double)w1 : 0.0; + printf(" work: 1000=%llu (real=%d) 4000=%llu (real=%d) ratio=%.2fx\n", + (unsigned long long)w1, real1, (unsigned long long)w4, real4, ratio); + ASSERT_EQ(real1, 2 * 1000); + ASSERT_EQ(real4, 2 * 4000); + ASSERT_GT(w1, 0); + ASSERT(ratio < 5.0); + PASS(); +} + +/* A reachable post-allocation-failure state, pinned without memory pressure. + * The old recovery path returned a full table here. Check capacity BEFORE + * any missing-name lookup, and free the arena before the predicted RED. */ +TEST(scope_issue1527_recovery_sizes_from_all_bindings) { + CBMArena a; + cbm_arena_init(&a); + CBMScope *scope = cbm_scope_push(&a, NULL); + char names[64][24]; + bool setup_ok = scope != NULL; + for (int i = 0; i < 63 && setup_ok; i++) { + snprintf(names[i], sizeof(names[i]), "binding_%d", i); + setup_ok = cbm_scope_bind_callable_checked(scope, names[i], cbm_type_unknown(), "old"); + } + const CBMVarBinding *stable = setup_ok ? cbm_scope_lookup_local(scope, names[7]) : NULL; + if (setup_ok) + memset(&scope->index, 0, sizeof(scope->index)); + snprintf(names[63], sizeof(names[63]), "binding_63"); + bool bound = + setup_ok && cbm_scope_bind_callable_checked(scope, names[63], cbm_type_unknown(), "old"); + bool spare = bound && scope->binding_count == 64 && scope->index.cap >= 128; + bool preserved = false; + if (spare) { + preserved = cbm_scope_lookup_local(scope, "absent") == NULL; + for (int i = 0; i < 64; i++) + preserved = preserved && cbm_scope_lookup_local(scope, names[i]) != NULL; + preserved = preserved && + cbm_scope_bind_callable_checked(scope, names[7], cbm_type_unknown(), "new") && + cbm_scope_lookup_local(scope, names[7]) == stable && stable->callable_qn && + strcmp(stable->callable_qn, "new") == 0 && scope->binding_count == 64; + CBMScope *child = cbm_scope_push(&a, scope); + preserved = preserved && child && + cbm_scope_bind_callable_checked(child, names[7], cbm_type_unknown(), "child"); + const CBMVarBinding *shadow = child ? cbm_scope_lookup_binding(child, names[7]) : NULL; + preserved = preserved && shadow && strcmp(shadow->callable_qn, "child") == 0 && + strcmp(stable->callable_qn, "new") == 0; + } + cbm_arena_destroy(&a); + ASSERT_TRUE(setup_ok); + ASSERT_TRUE(bound); + ASSERT_TRUE(spare); + ASSERT_TRUE(preserved); + PASS(); +} + +/* Initial allocation, second allocation, and later growth fail independently + * of the arena. The failure must be explicit, not an uncached exponential walk. */ +TEST(clsp_issue1527_memo_allocation_failure_is_reported) { + char *src = clsp_chain_source(40, 1); + ASSERT_NOT_NULL(src); + bool failures_reported = true; + for (int successful_allocations = 0; successful_allocations <= 2; successful_allocations++) { + (void)cbm_c_lsp_test_walks_take(); + cbm_c_lsp_test_memo_fail_after(successful_allocations); + CBMFileResult *r = extract_cpp(src); + uint32_t walks = cbm_c_lsp_test_walks_take(); + cbm_c_lsp_test_memo_fail_after(-1); + failures_reported = failures_reported && r && r->has_error && r->lsp_skipped && + r->error_msg && strcmp(r->error_msg, CBM_C_LSP_MEMO_ERROR) == 0 && + r->resolved_calls.count == 0 && walks == 1; + if (r) { + /* The allocation-free diagnostic also survives result compaction. */ + cbm_result_compact(r); + failures_reported = failures_reported && r->error_msg && + strcmp(r->error_msg, CBM_C_LSP_MEMO_ERROR) == 0; + cbm_free_result(r); + } + } + CBMFileResult *fresh = extract_cpp(src); + bool recovered = fresh && !fresh->has_error && !fresh->lsp_skipped && + count_real_resolved(fresh, ".Tester.") == 41; + if (fresh) + cbm_free_result(fresh); + free(src); + ASSERT_TRUE(failures_reported); + ASSERT_TRUE(recovered); + PASS(); +} + +extern const TSLanguage *tree_sitter_cpp(void); + +TEST(clsp_issue1527_memo_and_depth_failures_are_sticky) { + const char *src = "void f() { x; }"; + TSParser *parser = ts_parser_new(); + ASSERT_NOT_NULL(parser); + ts_parser_set_language(parser, tree_sitter_cpp()); + TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)strlen(src)); + if (!tree) { + ts_parser_delete(parser); + FAIL("failed to parse memo fixture"); + } + TSNode fn = ts_node_named_child(ts_tree_root_node(tree), 0); + TSNode body = ts_node_child_by_field_name(fn, "body", 4); + TSNode expr = ts_node_named_child(ts_node_named_child(body, 0), 0); + bool identifier = !ts_node_is_null(expr) && strcmp(ts_node_type(expr), "identifier") == 0; + CBMArena a; + cbm_arena_init(&a); + CBMTypeRegistry reg; + cbm_registry_init(®, &a); + CBMResolvedCallArray out = {0}; + CLSPContext ctx; + c_lsp_init(&ctx, &a, src, (int)strlen(src), ®, "test", true, &out); + const CBMType *first = cbm_type_named(&a, "test.First"); + const CBMType *second = cbm_type_named(&a, "test.Second"); + cbm_scope_bind(ctx.current_scope, "x", first); + ctx.eval_depth = 257; + bool depth_unknown = cbm_type_is_unknown(c_eval_expr_type(&ctx, expr)) && + ctx.eval_failure == CBM_LSP_DEPTH_EXCEEDED; + ctx.eval_depth = 0; + (void)cbm_lsp_work_take(); + bool depth_stopped = cbm_type_is_unknown(c_eval_expr_type(&ctx, expr)); + depth_stopped = depth_stopped && cbm_lsp_work_take() == 0; + /* A depth-truncated walk is incomplete; only a fresh walk may resume. */ + c_lsp_init(&ctx, &a, src, (int)strlen(src), ®, "test", true, &out); + cbm_scope_bind(ctx.current_scope, "x", first); + bool shallow_ok = c_eval_expr_type(&ctx, expr) == first && !ctx.eval_failure; + cbm_scope_bind(ctx.current_scope, "x", second); + bool rebound = c_eval_expr_type(&ctx, expr) == second; + /* Force a new memo's allocation to fail; old slots remain arena-owned. */ + ctx.eval_memo_cap = ctx.eval_memo_count = 0; + ctx.eval_memo = NULL; + ctx.eval_memo_used = NULL; + ctx.test_memo_allocations_left = 0; + bool failed = cbm_type_is_unknown(c_eval_expr_type(&ctx, expr)) && ctx.eval_failure; + (void)cbm_lsp_work_take(); + bool stopped = true; + for (int i = 0; i < 20; i++) + stopped = stopped && cbm_type_is_unknown(c_eval_expr_type(&ctx, expr)); + uint64_t later_work = cbm_lsp_work_take(); + cbm_arena_destroy(&a); + ts_tree_delete(tree); + ts_parser_delete(parser); + ASSERT_TRUE(identifier); + ASSERT_TRUE(depth_unknown); + ASSERT_TRUE(depth_stopped); + ASSERT_TRUE(shallow_ok); + ASSERT_TRUE(rebound); + ASSERT_TRUE(failed); + ASSERT_TRUE(stopped); + ASSERT_EQ(later_work, 0); + PASS(); +} + +TEST(clsp_issue1527_preprocessed_memo_failure_is_reported) { + const char *key = "CBM_TEST_C_LSP_MEMO_FAIL_STAGE"; + const char *old = getenv(key); + char *saved = old ? strdup(old) : NULL; + if (old && !saved) + FAIL("failed to save memo seam"); + int set_rc = cbm_setenv(key, "preprocessed", 1); + (void)cbm_c_lsp_test_walks_take(); + CBMFileResult *r = set_rc == 0 ? extract_cpp("struct S { int m() { return 1; } };\n" + "#define INVOKE(s) s.m()\n" + "void run() { S s; INVOKE(s); }\n") + : NULL; + uint32_t walks = cbm_c_lsp_test_walks_take(); + bool reported = r && r->has_error && r->lsp_skipped && r->error_msg && + strcmp(r->error_msg, CBM_C_LSP_MEMO_ERROR) == 0 && walks == 2; + if (r) + cbm_free_result(r); + int restore_rc = saved ? cbm_setenv(key, saved, 1) : cbm_unsetenv(key); + free(saved); + ASSERT_EQ(set_rc, 0); + ASSERT_EQ(restore_rc, 0); + ASSERT_TRUE(reported); + PASS(); +} + +TEST(clsp_issue1527_preprocessed_depth_failure_is_reported) { + const char *key = "CBM_TEST_C_LSP_DEPTH_FAIL_STAGE"; + const char *old = getenv(key); + char *saved = old ? strdup(old) : NULL; + if (old && !saved) + FAIL("failed to save memo seam"); + int set_rc = cbm_setenv(key, "preprocessed", 1); + (void)cbm_c_lsp_test_walks_take(); + CBMFileResult *r = set_rc == 0 ? extract_cpp("struct S { int m() { return 1; } };\n" + "#define INVOKE(s) s.m()\n" + "void run() { S s; INVOKE(s); }\n") + : NULL; + uint32_t walks = cbm_c_lsp_test_walks_take(); + bool reported = r && r->has_error && r->lsp_skipped && r->error_msg && + strcmp(r->error_msg, CBM_C_LSP_DEPTH_ERROR) == 0 && walks == 2; + if (r) + cbm_free_result(r); + int restore_rc = saved ? cbm_setenv(key, saved, 1) : cbm_unsetenv(key); + free(saved); + ASSERT_EQ(set_rc, 0); + ASSERT_EQ(restore_rc, 0); + ASSERT_TRUE(reported); + PASS(); +} + +TEST(clsp_issue1527_cross_file_memo_failures_propagate) { + const char *src = "struct S { int m() { return 1; } }; void run() { S s; s.m(); }"; + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_CPP}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_CPP}, + }; + bool failed[3] = {false}, recovered[3] = {false}; + bool shared_ready = true; + for (int mode = 0; mode < 3; mode++) { + CBMArena a; + cbm_arena_init(&a); + CBMTypeRegistry reg; + cbm_registry_init(®, &a); + CBMTypeRegistry *shared = cbm_c_build_cross_registry(&a, defs, 2); + shared_ready = shared_ready && shared && shared->read_only; + reg.fallback = shared; /* the production prebuilt path uses a writable overlay */ + CBMResolvedCallArray out = {0}; + CBMBatchCLSPFile file = {.source = src, + .source_len = (int)strlen(src), + .module_qn = "test", + .cpp_mode = true, + .defs = defs, + .def_count = 2}; + cbm_c_lsp_test_memo_fail_after(0); + bool ok = mode == 0 ? cbm_run_c_lsp_cross(&a, src, (int)strlen(src), "test", true, defs, 2, + NULL, NULL, 0, NULL, &out) + : mode == 1 + ? cbm_run_c_lsp_cross_with_registry(&a, src, (int)strlen(src), "test", true, + ®, NULL, NULL, 0, NULL, &out) + : cbm_batch_c_lsp_cross(&a, &file, 1, &out); + cbm_c_lsp_test_memo_fail_after(-1); + failed[mode] = !ok && out.count == 0; + ok = mode == 0 ? cbm_run_c_lsp_cross(&a, src, (int)strlen(src), "test", true, defs, 2, NULL, + NULL, 0, NULL, &out) + : mode == 1 ? cbm_run_c_lsp_cross_with_registry(&a, src, (int)strlen(src), "test", + true, ®, NULL, NULL, 0, NULL, &out) + : cbm_batch_c_lsp_cross(&a, &file, 1, &out); + recovered[mode] = ok && out.count > 0; + cbm_arena_destroy(&a); + } + ASSERT_TRUE(shared_ready); + for (int i = 0; i < 3; i++) { + ASSERT_TRUE(failed[i]); + ASSERT_TRUE(recovered[i]); + } + PASS(); +} + +TEST(registry_issue1527_index_preserves_registration_order) { + CBMArena a; + cbm_arena_init(&a); + CBMTypeRegistry reg; + cbm_registry_init(®, &a); + CBMRegisteredType t = {.qualified_name = "test.S", .short_name = "S"}; + cbm_registry_add_type(®, t); + cbm_registry_add_type(®, t); + CBMRegisteredFunc f = { + .qualified_name = "test.S.m", .short_name = "m", .receiver_type = "test.S"}; + cbm_registry_add_func(®, f); + cbm_registry_add_func(®, f); + bool linear_first = cbm_registry_lookup_type(®, "test.S") == ®.types[0] && + cbm_registry_lookup_func(®, "test.S.m") == ®.funcs[0] && + cbm_registry_lookup_method(®, "test.S", "m") == ®.funcs[0]; + reg.index_first_registered = true; + cbm_registry_finalize(®); + bool indexed_first = cbm_registry_lookup_type(®, "test.S") == ®.types[0] && + cbm_registry_lookup_func(®, "test.S.m") == ®.funcs[0] && + cbm_registry_lookup_method(®, "test.S", "m") == ®.funcs[0]; + reg.index_first_registered = false; + cbm_registry_finalize(®); + bool default_last = cbm_registry_lookup_type(®, "test.S") == ®.types[1] && + cbm_registry_lookup_func(®, "test.S.m") == ®.funcs[1] && + cbm_registry_lookup_method(®, "test.S", "m") == ®.funcs[1]; + f.qualified_name = "test.S.tail"; + f.short_name = "tail"; + cbm_registry_add_func(®, f); + bool tail = cbm_registry_lookup_func(®, "test.S.tail") == ®.funcs[2] && + cbm_registry_lookup_method(®, "test.S", "tail") == ®.funcs[2]; + cbm_arena_destroy(&a); + ASSERT_TRUE(linear_first); + ASSERT_TRUE(indexed_first); + ASSERT_TRUE(default_last); + ASSERT_TRUE(tail); + PASS(); +} + +TEST(clsp_issue1527_depth_failure_reports_all_entry_points) { + const char *src = "struct S { int m() { return 1; } }; void run() { S s; s.m(); }"; + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_CPP}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_CPP}, + }; + cbm_c_lsp_test_depth_fail(true); + CBMFileResult *raw = extract_cpp(src); + bool correct = raw && raw->has_error && raw->lsp_skipped && raw->error_msg && + strcmp(raw->error_msg, CBM_C_LSP_DEPTH_ERROR) == 0 && + raw->resolved_calls.count == 0; + if (raw) + cbm_free_result(raw); + for (int mode = 0; mode < 3; mode++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *base = mode == 1 ? cbm_c_build_cross_registry(&shared, defs, 2) : NULL; + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + CBMResolvedCallArray out = {0}; + CBMBatchCLSPFile file = {.source = src, + .source_len = (int)strlen(src), + .module_qn = "test", + .defs = defs, + .def_count = 2, + .cpp_mode = true}; + if (mode == 2) { + correct = !cbm_batch_c_lsp_cross(&arena, &file, 1, &out) && correct; + } else { + CBMLSPStatus status = + mode == 0 ? cbm_run_c_lsp_cross_status(&arena, src, (int)strlen(src), "test", true, + defs, 2, NULL, NULL, 0, NULL, &out) + : cbm_run_c_lsp_cross_with_registry_status(&arena, src, (int)strlen(src), + "test", true, &overlay, NULL, + NULL, 0, NULL, &out); + correct = correct && status == CBM_LSP_DEPTH_EXCEEDED; + } + correct = correct && out.count == 0; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + CBMFileResult fallback = {0}; + cbm_arena_init(&fallback.arena); + cbm_pxc_run_one(CBM_LANG_CPP, &fallback, src, (int)strlen(src), "test", defs, 2, NULL, NULL, 0); + correct = correct && fallback.has_error && fallback.lsp_skipped && fallback.error_msg && + strcmp(fallback.error_msg, CBM_C_LSP_DEPTH_ERROR) == 0 && + fallback.resolved_calls.count == 0; + cbm_arena_destroy(&fallback.arena); + cbm_c_lsp_test_depth_fail(false); + CBMFileResult *fresh = extract_cpp(src); + bool recovered = fresh && !fresh->has_error && fresh->resolved_calls.count > 0; + if (fresh) + cbm_free_result(fresh); + ASSERT_TRUE(correct); + ASSERT_TRUE(recovered); + PASS(); +} + SUITE(c_lsp) { + RUN_TEST(clsp_issue1527_depth_failure_reports_all_entry_points); + RUN_TEST(scope_issue1527_recovery_sizes_from_all_bindings); + RUN_TEST(clsp_issue1527_memo_allocation_failure_is_reported); + RUN_TEST(clsp_issue1527_memo_and_depth_failures_are_sticky); + RUN_TEST(clsp_issue1527_preprocessed_memo_failure_is_reported); + RUN_TEST(clsp_issue1527_preprocessed_depth_failure_is_reported); + RUN_TEST(registry_issue1527_index_preserves_registration_order); + RUN_TEST(clsp_issue1527_cross_file_memo_failures_propagate); RUN_TEST(clsp_c_reassigned_function_pointer_calls_join_exact_occurrences); RUN_TEST(clsp_c_nested_function_pointer_shadow_restores_outer_target); RUN_TEST(clsp_cpp_repeated_same_leaf_calls_join_exact_occurrences); @@ -16402,6 +16991,7 @@ SUITE(c_lsp) { RUN_TEST(registry_overlay_chain_iterates_and_copies_on_write); RUN_TEST(clsp_method_return_refinement_is_copy_on_write); RUN_TEST(seal_py_shared_registry_readonly_fields); + RUN_TEST(seal_py_overlay_existing_field_keeps_shared_type); RUN_TEST(seal_cs_shared_registry_readonly); RUN_TEST(seal_ts_shared_registry_readonly); RUN_TEST(seal_go_shared_registry_readonly); @@ -17149,4 +17739,6 @@ SUITE(c_lsp) { RUN_TEST(clsp_dll_multiple_functions); RUN_TEST(clsp_dll_func_ptr_typedef); RUN_TEST(clsp_easy_win_sfinaeconditional_return); + RUN_TEST(clsp_issue1527_fluent_chain_work_is_polynomial); + RUN_TEST(clsp_issue1527_scale_work_is_linear); } diff --git a/tests/test_complexity.c b/tests/test_complexity.c index f20eac16e..0c8f3ab4d 100644 --- a/tests/test_complexity.c +++ b/tests/test_complexity.c @@ -50,11 +50,13 @@ #include "cbm.h" #include "discover/discover.h" #include "lang_specs.h" +#include "lsp/py_lsp.h" #include "tree_sitter/api.h" #include "pipeline/pass_lsp_cross.h" #include "pipeline/pipeline.h" #include "pipeline/pipeline_internal.h" #include "store/store.h" +#include "yyjson/yyjson.h" #include #include @@ -86,6 +88,9 @@ enum { * code path. The sequential path's own per-file registry work is bounded * by the 50-file ceiling and is deliberately out of scope here. */ CX_BIGPKG_FILES_PER_K = 20, + CX_PYEXT_FILES_PER_MOD = 20, + CX_PYEXT_DEFS_PER_FILE = 120, + CX_PYEXT_PROBES_PER_FILE = 3, }; /* Linear growth bounds for a 2x input doubling. Fixed structural overhead @@ -306,6 +311,44 @@ static int cx_build_bigpkg(const char *root, int k) { return 0; } +/* Every file probes absent external submodules and retains a real local-call + * control. Independent module copies grow the shared registry without adding + * cross-copy dependencies. Both sizes exceed the parallel-driver threshold. */ +static int cx_build_pyext(const char *root, int k) { + char dir[1024]; + char path[1200]; + for (int m = 0; m < k; m++) { + int n = snprintf(dir, sizeof(dir), "%s/pyext/mod%d", root, m); + if (n < 0 || (size_t)n >= sizeof(dir) || th_mkdir_p(dir) != 0) + return -1; + for (int j = 0; j < CX_PYEXT_FILES_PER_MOD; j++) { + n = snprintf(path, sizeof(path), "%s/p%d.py", dir, j); + if (n < 0 || (size_t)n >= sizeof(path)) + return -1; + FILE *f = fopen(path, "w"); + if (!f) + return -1; + bool written = fprintf(f, "import extlib\n\n") >= 0; + for (int d = 0; written && d < CX_PYEXT_DEFS_PER_FILE; d++) { + written = + fprintf(f, "def cx_pyext_definition_%d(x):\n return x + %d\n\n", d, d) >= 0; + } + if (written) + written = + fprintf(f, "def cx_pyext_use():\n return (cx_pyext_definition_0(0)") >= 0; + for (int probe = 0; written && probe < CX_PYEXT_PROBES_PER_FILE; probe++) { + written = fprintf(f, " + extlib.sub%d.helper(%d)", probe, probe) >= 0; + } + if (written) + written = fprintf(f, ")\n") >= 0; + int closed = fclose(f); + if (!written || closed != 0) + return -1; + } + } + return 0; +} + /* ── Metrics ─────────────────────────────────────────────────────────── */ /* Per-pass timing capture: a TEE log sink parses `pass.timing` lines during a @@ -357,16 +400,253 @@ typedef struct { uint64_t fallback_rows; uint64_t imp_nodes; /* symbols scored by the importance pass */ uint64_t imp_name_visits; /* same-name-group members visited by that pass */ + uint64_t py_probes; + uint64_t py_probe_visits; + int pyext_files; + int pyext_defs; + int pyext_controls; + uint64_t py_class_binds; + uint64_t py_class_visits; + uint64_t py_wildcards; + uint64_t py_wildcard_visits; + int pyrange_files; + int pyrange_defs; + int pyrange_classes; + int pyrange_peer_classes; + int pyrange_controls; + int errors; double wall_s; } CxMetrics; +/* Count the intended corpus, then prove each use() kept its exact local LSP + * call. Query errors are failures, never indistinguishable from zero rows. */ +static bool cx_pyext_control(cbm_store_t *store, const char *project, const cbm_node_t *caller) { + if (!caller->qualified_name || !caller->file_path) + return false; + const char *dot = strrchr(caller->qualified_name, '.'); + if (!dot || dot - caller->qualified_name > 1000) + return false; + char target_qn[1200]; + int n = snprintf(target_qn, sizeof(target_qn), "%.*s.cx_pyext_definition_0", + (int)(dot - caller->qualified_name), caller->qualified_name); + if (n < 0 || (size_t)n >= sizeof(target_qn)) + return false; + cbm_node_t target = {0}; + cbm_edge_t *edges = NULL; + int count = 0; + bool found = false; + if (cbm_store_find_node_by_qn(store, project, target_qn, &target) == CBM_STORE_OK && + target.label && strcmp(target.label, "Function") == 0 && target.file_path && + strcmp(target.file_path, caller->file_path) == 0 && + cbm_store_find_edges_by_source_type(store, caller->id, "CALLS", &edges, &count) == + CBM_STORE_OK) { + for (int i = 0; i < count; i++) { + if (edges[i].target_id != target.id || !edges[i].properties_json) + continue; + yyjson_doc *doc = + yyjson_read(edges[i].properties_json, strlen(edges[i].properties_json), 0); + const char *source = + doc ? yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(doc), "source")) : NULL; + found = found || (source && strcmp(source, "lsp") == 0); + yyjson_doc_free(doc); + } + } + cbm_store_free_edges(edges, count); + cbm_node_free_fields(&target); + return found; +} + +static bool cx_pyext_population(cbm_store_t *store, const char *project, CxMetrics *out) { + cbm_node_t *nodes = NULL; + int count = 0; + bool ok = cbm_store_find_nodes_by_label(store, project, "File", &nodes, &count) == CBM_STORE_OK; + for (int i = 0; ok && i < count; i++) { + if (nodes[i].file_path && strncmp(nodes[i].file_path, "pyext/", 6) == 0) + out->pyext_files++; + } + cbm_store_free_nodes(nodes, count); + nodes = NULL; + count = 0; + ok = ok && + cbm_store_find_nodes_by_label(store, project, "Function", &nodes, &count) == CBM_STORE_OK; + for (int i = 0; ok && i < count; i++) { + const cbm_node_t *node = &nodes[i]; + if (!node->file_path || strncmp(node->file_path, "pyext/", 6) != 0 || !node->name) + continue; + static const char definition_prefix[] = "cx_pyext_definition_"; + if (strncmp(node->name, definition_prefix, sizeof(definition_prefix) - 1) == 0) + out->pyext_defs++; + if (strcmp(node->name, "cx_pyext_use") == 0 && cx_pyext_control(store, project, node)) + out->pyext_controls++; + } + cbm_store_free_nodes(nodes, count); + return ok; +} + +/* Each package has many registered direct members; other packages are decoys. + * Wildcards precede definitions so the fixture retains explicit LSP positives. + * Exact invalidation-order negatives live in test_py_lsp.c. */ +enum { CX_PYRANGE_FILES_PER_K = 20, CX_PYRANGE_DEFS = 40, CX_PYRANGE_WILDCARDS = 4 }; + +static int cx_build_pyrange(const char *root, int k) { + for (int m = 0; m < k; m++) { + for (int j = 0; j < CX_PYRANGE_FILES_PER_K; j++) { + char dir[1024], path[1200]; + int dn = snprintf(dir, sizeof(dir), "%s/pyrange/mod%d/p%d", root, m, j); + if (dn < 0 || (size_t)dn >= sizeof(dir) || th_mkdir_p(dir) != 0) + return -1; + int pn = snprintf(path, sizeof(path), "%s/__init__.py", dir); + if (pn < 0 || (size_t)pn >= sizeof(path)) + return -1; + FILE *f = fopen(path, "w"); + if (!f) + return -1; + bool written = true; + for (int w = 0; written && w < CX_PYRANGE_WILDCARDS; w++) + written = fprintf(f, "from range_vendor_%d import *\n", w) >= 0; + int peer = (j + 1) % CX_PYRANGE_FILES_PER_K; + if (written) + written = fprintf(f, "from pyrange.mod%d.p%d import Peer_%d_%d\n\n", m, peer, m, + peer) >= 0; + for (int d = 0; written && d < CX_PYRANGE_DEFS; d++) { + written = fprintf(f, + "def cx_pyrange_definition_%d(x):\n return x + %d\n\n" + "class RangeClass_%d:\n def value(self):\n" + " return %d\n\n", + d, d, d, d) >= 0; + } + if (written) + written = fprintf(f, + "class Peer_%d_%d:\n def remote_%d_%d(self):\n" + " return %d\n\n" + "def cx_pyrange_use():\n obj = RangeClass_0()\n" + " peer = Peer_%d_%d()\n" + " return (cx_pyrange_definition_0(1) + obj.value()\n" + " + peer.remote_%d_%d())\n", + m, j, m, j, j, m, peer, m, peer) >= 0; + int closed = fclose(f); + if (!written || closed != 0) + return -1; + } + } + return 0; +} + +static bool cx_pyrange_exact_control(cbm_store_t *store, const char *project, + const cbm_node_t *caller, const char *qn, + const char *target_file, const char *label) { + if (!caller->qualified_name || !caller->file_path || !caller->label || + strcmp(caller->label, "Function") != 0) + return false; + cbm_node_t target = {0}; + cbm_edge_t *edges = NULL; + int count = 0, exact = 0; + bool ok = cbm_store_find_node_by_qn(store, project, qn, &target) == CBM_STORE_OK && + target.qualified_name && strcmp(target.qualified_name, qn) == 0 && target.label && + strcmp(target.label, label) == 0 && target.file_path && + strcmp(target.file_path, target_file) == 0 && + cbm_store_find_edges_by_source_type(store, caller->id, "CALLS", &edges, &count) == + CBM_STORE_OK; + for (int i = 0; ok && i < count; i++) { + if (edges[i].target_id != target.id) + continue; + yyjson_doc *doc = + edges[i].properties_json + ? yyjson_read(edges[i].properties_json, strlen(edges[i].properties_json), 0) + : NULL; + const char *source = + doc ? yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(doc), "source")) : NULL; + if (source && strcmp(source, "lsp") == 0) + exact++; + yyjson_doc_free(doc); + } + cbm_store_free_edges(edges, count); + cbm_node_free_fields(&target); + return ok && exact == 1; +} + +/* A remote method has no declaration in the importer/raw file registry. Its + * source=lsp edge proves that cross-file registry evidence reached resolution. */ +static bool cx_pyrange_controls(cbm_store_t *store, const char *project, const cbm_node_t *caller) { + if (!caller->qualified_name || !caller->file_path) + return false; + const char *dot = strrchr(caller->qualified_name, '.'); + if (!dot || dot - caller->qualified_name > 1000) + return false; + char module[1200], qn[1400]; + int n = snprintf(module, sizeof(module), "%.*s", (int)(dot - caller->qualified_name), + caller->qualified_name); + if (n < 0 || (size_t)n >= sizeof(module)) + return false; + n = snprintf(qn, sizeof(qn), "%s.cx_pyrange_definition_0", module); + bool function = + n >= 0 && (size_t)n < sizeof(qn) && + cx_pyrange_exact_control(store, project, caller, qn, caller->file_path, "Function"); + n = snprintf(qn, sizeof(qn), "%s.RangeClass_0.value", module); + bool method = n >= 0 && (size_t)n < sizeof(qn) && + cx_pyrange_exact_control(store, project, caller, qn, caller->file_path, "Method"); + int m = -1, j = -1; + if (sscanf(caller->file_path, "pyrange/mod%d/p%d/__init__.py", &m, &j) != 2 || m < 0 || j < 0 || + j >= CX_PYRANGE_FILES_PER_K) + return false; + char path[1200], slot_name[64]; + n = snprintf(path, sizeof(path), "pyrange/mod%d/p%d/__init__.py", m, j); + if (n < 0 || (size_t)n >= sizeof(path) || strcmp(path, caller->file_path) != 0) + return false; + n = snprintf(slot_name, sizeof(slot_name), "p%d", j); + char *slot = strrchr(module, '.'); + if (n < 0 || (size_t)n >= sizeof(slot_name) || !slot || strcmp(slot + 1, slot_name) != 0) + return false; + *slot = '\0'; + int peer = (j + 1) % CX_PYRANGE_FILES_PER_K; + n = snprintf(qn, sizeof(qn), "%s.p%d.Peer_%d_%d.remote_%d_%d", module, peer, m, peer, m, peer); + int pn = snprintf(path, sizeof(path), "pyrange/mod%d/p%d/__init__.py", m, peer); + bool remote = n >= 0 && (size_t)n < sizeof(qn) && pn >= 0 && (size_t)pn < sizeof(path) && + strcmp(path, caller->file_path) != 0 && + cx_pyrange_exact_control(store, project, caller, qn, path, "Method"); + return function && method && remote; +} + +static bool cx_pyrange_population(cbm_store_t *store, const char *project, CxMetrics *out) { + static const char *const labels[] = {"File", "Function", "Class"}; + bool ok = true; + for (int l = 0; l < 3; l++) { + cbm_node_t *nodes = NULL; + int count = 0; + bool queried = cbm_store_find_nodes_by_label(store, project, labels[l], &nodes, &count) == + CBM_STORE_OK; + ok = queried && ok; + for (int i = 0; queried && i < count; i++) { + const cbm_node_t *node = &nodes[i]; + if (!node->file_path || strncmp(node->file_path, "pyrange/", 8) != 0) + continue; + if (l == 0) + out->pyrange_files++; + if (l == 1 && node->name) { + static const char def_prefix[] = "cx_pyrange_definition_"; + if (strncmp(node->name, def_prefix, sizeof(def_prefix) - 1) == 0) + out->pyrange_defs++; + if (strcmp(node->name, "cx_pyrange_use") == 0 && + cx_pyrange_controls(store, project, node)) + out->pyrange_controls++; + } + if (l == 2 && node->name && strncmp(node->name, "RangeClass_", 11) == 0) + out->pyrange_classes++; + if (l == 2 && node->name && strncmp(node->name, "Peer_", 5) == 0) + out->pyrange_peer_classes++; + } + cbm_store_free_nodes(nodes, count); + } + return ok; +} + static double cx_now_s(void) { struct timespec ts; cbm_profile_now(&ts); return (double)ts.tv_sec + (double)ts.tv_nsec / 1e9; } -static int cx_run(const char *root, const char *db_path, CxMetrics *out) { +static int cx_run(const char *root, const char *db_path, CxMetrics *out, bool pyext, bool pyrange) { memset(out, 0, sizeof(*out)); uint64_t d0; @@ -379,6 +659,13 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) { uint64_t fb0 = cbm_pp_lsp_linear_fallback_rows(); uint64_t in0 = atomic_load_explicit(&g_importance_nodes, memory_order_relaxed); uint64_t iv0 = atomic_load_explicit(&g_importance_name_visits, memory_order_relaxed); + uint64_t pp0 = atomic_load_explicit(&cbm_py_submodule_probes, memory_order_relaxed); + uint64_t pv0 = atomic_load_explicit(&cbm_py_submodule_probe_visits, memory_order_relaxed); + uint64_t pc0 = atomic_load_explicit(&cbm_py_module_class_binds, memory_order_relaxed); + uint64_t pcv0 = atomic_load_explicit(&cbm_py_module_class_bind_visits, memory_order_relaxed); + uint64_t pw0 = atomic_load_explicit(&cbm_py_wildcard_invalidations, memory_order_relaxed); + uint64_t pwv0 = + atomic_load_explicit(&cbm_py_wildcard_invalidation_visits, memory_order_relaxed); atomic_store_explicit(&g_cx_pass_count, 0, memory_order_relaxed); cbm_log_set_sink_ex(cx_pass_sink, CBM_LOG_SINK_TEE); @@ -390,6 +677,7 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) { } int rc = cbm_pipeline_run(p); out->wall_s = cx_now_s() - t0; + cbm_pipeline_get_file_errors(p, NULL, &out->errors); cbm_log_set_sink(NULL); int captured = atomic_load_explicit(&g_cx_pass_count, memory_order_relaxed); out->pass_count = captured < CX_MAX_PASSES ? captured : CX_MAX_PASSES; @@ -417,6 +705,10 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) { out->imp_name_visits = atomic_load_explicit(&g_importance_name_visits, memory_order_relaxed) - iv0; + out->py_probes = atomic_load_explicit(&cbm_py_submodule_probes, memory_order_relaxed) - pp0; + out->py_probe_visits = + atomic_load_explicit(&cbm_py_submodule_probe_visits, memory_order_relaxed) - pv0; + cbm_store_t *s = cbm_store_open_path(db_path); if (!s) { return -1; @@ -427,8 +719,18 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) { out->lang_nodes[t] = cbm_store_count_nodes_scoped(s, project, CX_TEMPLATES[t].dirname); out->lang_edges[t] = cbm_store_count_edges_scoped(s, project, CX_TEMPLATES[t].dirname); } + out->py_class_binds = + atomic_load_explicit(&cbm_py_module_class_binds, memory_order_relaxed) - pc0; + out->py_class_visits = + atomic_load_explicit(&cbm_py_module_class_bind_visits, memory_order_relaxed) - pcv0; + out->py_wildcards = + atomic_load_explicit(&cbm_py_wildcard_invalidations, memory_order_relaxed) - pw0; + out->py_wildcard_visits = + atomic_load_explicit(&cbm_py_wildcard_invalidation_visits, memory_order_relaxed) - pwv0; + bool extra_ok = (!pyext || cx_pyext_population(s, project, out)) && + (!pyrange || cx_pyrange_population(s, project, out)); cbm_store_close(s); - return 0; + return extra_ok ? 0 : -1; } static double cx_ratio(double num, double den) { @@ -437,10 +739,12 @@ static double cx_ratio(double num, double den) { typedef int (*CxCorpusBuilder)(const char *root, int k); -static void cx_remove_corpus(const char *root) { - if (th_rmtree(root) != 0) { +static int cx_remove_corpus(const char *root) { + int rc = th_rmtree(root); + if (rc != 0) { fprintf(stderr, " [complexity] WARN: could not remove temp corpus %s\n", root); } + return rc; } /* Build one corpus shape at k and 2k in two fresh temp dirs, run the pipeline @@ -453,26 +757,30 @@ static int cx_measure(CxCorpusBuilder build, const char *prefix_a, const char *p const char *tmp = cbm_tmpdir(); char root_a[512]; char root_b[512]; - snprintf(root_a, sizeof(root_a), "%s/%s_XXXXXX", tmp, prefix_a); - snprintf(root_b, sizeof(root_b), "%s/%s_XXXXXX", tmp, prefix_b); + int na = snprintf(root_a, sizeof(root_a), "%s/%s_XXXXXX", tmp, prefix_a); + int nb = snprintf(root_b, sizeof(root_b), "%s/%s_XXXXXX", tmp, prefix_b); + if (na < 0 || (size_t)na >= sizeof(root_a) || nb < 0 || (size_t)nb >= sizeof(root_b)) + return -1; bool made_a = cbm_mkdtemp(root_a) != NULL; bool made_b = made_a && cbm_mkdtemp(root_b) != NULL; int rc = -1; if (made_b && build(root_a, CX_K_BASE) == 0 && build(root_b, CX_K_BASE * 2) == 0) { char db_a[600]; char db_b[600]; - snprintf(db_a, sizeof(db_a), "%s/cx.db", root_a); - snprintf(db_b, sizeof(db_b), "%s/cx.db", root_b); - if (cx_run(root_a, db_a, out_a) == 0 && cx_run(root_b, db_b, out_b) == 0) { + int da = snprintf(db_a, sizeof(db_a), "%s/cx.db", root_a); + int db = snprintf(db_b, sizeof(db_b), "%s/cx.db", root_b); + bool paths = da >= 0 && (size_t)da < sizeof(db_a) && db >= 0 && (size_t)db < sizeof(db_b); + bool pyext = build == cx_build_pyext; + bool pyrange = build == cx_build_pyrange; + if (paths && cx_run(root_a, db_a, out_a, pyext, pyrange) == 0 && + cx_run(root_b, db_b, out_b, pyext, pyrange) == 0) { rc = 0; } } - if (made_b) { - cx_remove_corpus(root_b); - } - if (made_a) { - cx_remove_corpus(root_a); - } + if (made_b && cx_remove_corpus(root_b) != 0 && build == cx_build_pyrange) + rc = -1; + if (made_a && cx_remove_corpus(root_a) != 0 && build == cx_build_pyrange) + rc = -1; return rc; } @@ -616,6 +924,58 @@ TEST(complexity_shared_package_growth_stays_linear) { PASS(); } +TEST(complexity_py_submodule_probe_avoids_registry_scan) { + static const char *const keys[] = {"CBM_WORKERS", "CBM_INDEX_SINGLE_THREAD"}; + char *saved[2] = {NULL, NULL}; + bool ready = true; + for (int i = 0; i < 2; i++) { + const char *old = getenv(keys[i]); + saved[i] = old ? strdup(old) : NULL; + if (old && !saved[i]) + ready = false; + } + CxMetrics a = {0}, b = {0}; + int rc = -1, restored = 0; + if (ready) { + int configured = cbm_setenv(keys[0], "2", 1); + configured |= cbm_unsetenv(keys[1]); + if (configured == 0) + rc = cx_measure(cx_build_pyext, "cbm_cxpy_a", "cbm_cxpy_b", &a, &b); + for (int i = 0; i < 2; i++) + restored |= saved[i] ? cbm_setenv(keys[i], saved[i], 1) : cbm_unsetenv(keys[i]); + } + for (int i = 0; i < 2; i++) + free(saved[i]); + /* cx_measure removes both owned corpora before any assertion below. */ + ASSERT_TRUE(ready); + ASSERT_EQ(restored, 0); + ASSERT_EQ(rc, 0); + int files = CX_K_BASE * CX_PYEXT_FILES_PER_MOD; + ASSERT_EQ(a.errors, 0); + ASSERT_EQ(b.errors, 0); + ASSERT_EQ(a.pyext_files, files); + ASSERT_EQ(b.pyext_files, files * 2); + ASSERT_EQ(a.pyext_defs, files * CX_PYEXT_DEFS_PER_FILE); + ASSERT_EQ(b.pyext_defs, files * 2 * CX_PYEXT_DEFS_PER_FILE); + ASSERT_EQ(a.pyext_controls, files); + ASSERT_EQ(b.pyext_controls, files * 2); + ASSERT_GT((long long)a.py_probes, (long long)CX_MIN_BASE_WORK); + ASSERT_GT((long long)b.py_probes, (long long)CX_MIN_BASE_WORK); + ASSERT_GT((long long)a.py_probe_visits, 0); + ASSERT_GT((long long)b.py_probe_visits, 0); + double probes_ratio = cx_ratio((double)b.py_probes, (double)a.py_probes); + double visits_ratio = cx_ratio((double)b.py_probe_visits, (double)a.py_probe_visits); + printf(" py_probes %llu -> %llu (%.2f), visits %llu -> %llu (%.2f)\n", + (unsigned long long)a.py_probes, (unsigned long long)b.py_probes, probes_ratio, + (unsigned long long)a.py_probe_visits, (unsigned long long)b.py_probe_visits, + visits_ratio); + ASSERT_TRUE(probes_ratio >= CX_RATIO_LO && probes_ratio <= CX_RATIO_HI); + ASSERT_TRUE(visits_ratio <= CX_RATIO_HI); + /* The direct registry test also enforces a logarithmic comparison bound, + * independently of fixed stdlib size and this corpus-growth ratio. */ + PASS(); +} + /* Throughput report — information only, never a gate (CI-determinism rule: * rates depend on the machine and scheduler, so a threshold would be a * lottery). Written locally under private/ (gitignored); CBM_COMPLEXITY_ @@ -841,11 +1201,73 @@ TEST(complexity_rescript_error_recovery_lexing_is_linear) { PASS(); } +TEST(complexity_py_module_ranges_preserve_calls_with_bounded_work) { + static const char *const keys[] = {"CBM_WORKERS", "CBM_INDEX_SINGLE_THREAD"}; + char *saved[2] = {NULL, NULL}; + bool ready = true; + for (int i = 0; i < 2; i++) { + const char *old = getenv(keys[i]); + saved[i] = old ? strdup(old) : NULL; + if (old && !saved[i]) + ready = false; + } + CxMetrics a = {0}, b = {0}; + int rc = -1, restored = 0; + if (ready) { + int configured = cbm_setenv(keys[0], "2", 1); + configured |= cbm_unsetenv(keys[1]); + if (configured == 0) + rc = cx_measure(cx_build_pyrange, "cbm_cxrange_a", "cbm_cxrange_b", &a, &b); + for (int i = 0; i < 2; i++) + restored |= saved[i] ? cbm_setenv(keys[i], saved[i], 1) : cbm_unsetenv(keys[i]); + } + for (int i = 0; i < 2; i++) + free(saved[i]); + /* Both owned corpora are removed and environment restored before assertions. */ + ASSERT_TRUE(ready); + ASSERT_EQ(restored, 0); + ASSERT_EQ(rc, 0); + const int files = CX_K_BASE * CX_PYRANGE_FILES_PER_K; + ASSERT_GT(files, 50); + ASSERT_EQ(a.errors, 0); + ASSERT_EQ(b.errors, 0); + ASSERT_EQ(a.pyrange_files, files); + ASSERT_EQ(b.pyrange_files, files * 2); + ASSERT_EQ(a.pyrange_defs, files * CX_PYRANGE_DEFS); + ASSERT_EQ(b.pyrange_defs, files * 2 * CX_PYRANGE_DEFS); + ASSERT_EQ(a.pyrange_classes, files * CX_PYRANGE_DEFS); + ASSERT_EQ(b.pyrange_classes, files * 2 * CX_PYRANGE_DEFS); + ASSERT_EQ(a.pyrange_peer_classes, files); + ASSERT_EQ(b.pyrange_peer_classes, files * 2); + ASSERT_EQ(a.pyrange_controls, files); + ASSERT_EQ(b.pyrange_controls, files * 2); + ASSERT_GTE(a.py_class_binds, (uint64_t)files); + ASSERT_GTE(b.py_class_binds, (uint64_t)(files * 2)); + ASSERT_GTE(a.py_wildcards, (uint64_t)(files * CX_PYRANGE_WILDCARDS)); + ASSERT_GTE(b.py_wildcards, (uint64_t)(files * 2 * CX_PYRANGE_WILDCARDS)); + ASSERT_GT(a.py_class_visits, (uint64_t)0); + ASSERT_GT(b.py_class_visits, (uint64_t)0); + ASSERT_GT(a.py_wildcard_visits, (uint64_t)0); + ASSERT_GT(b.py_wildcard_visits, (uint64_t)0); + double classes = cx_ratio((double)b.py_class_binds, (double)a.py_class_binds); + double wildcards = cx_ratio((double)b.py_wildcards, (double)a.py_wildcards); + ASSERT_TRUE(classes >= CX_RATIO_LO && classes <= CX_RATIO_HI); + ASSERT_TRUE(wildcards >= CX_RATIO_LO && wildcards <= CX_RATIO_HI); + ASSERT_TRUE(cx_ratio((double)b.py_class_visits, (double)a.py_class_visits) <= CX_RATIO_HI); + ASSERT_TRUE(cx_ratio((double)b.py_wildcard_visits, (double)a.py_wildcard_visits) <= + CX_RATIO_HI); + /* These counters exclude sorting/string-length work. The direct registry + * tests separately pin absolute bounds and exact ordered answers. */ + PASS(); +} + SUITE(complexity) { + RUN_TEST(complexity_py_module_ranges_preserve_calls_with_bounded_work); RUN_TEST(complexity_rescript_error_recovery_lexing_is_linear); RUN_TEST(complexity_replicated_modules_scale_linearly); RUN_TEST(complexity_perfile_registry_work_is_linear); RUN_TEST(complexity_importance_scoring_is_linear); RUN_TEST(complexity_shared_package_growth_stays_linear); + RUN_TEST(complexity_py_submodule_probe_avoids_registry_scan); RUN_TEST(complexity_throughput_report_written); } diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 0b6800cd1..efbeac315 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1182,6 +1182,62 @@ TEST(cpp_method_return_type_preserves_pointer_and_qualifier) { * ═══════════════════════════════════════════════════════════════════ */ /* --- Python --- */ +/* #1277: annotated Python instance fields reach result->field_types (the + * cross-file LSP's only source for them); nothing unannotated is guessed and + * no graph def is minted for a field. */ +static const char *py_field_type_of(const CBMFileResult *r, const char *cls_suffix, + const char *field) { + for (int i = 0; i < r->field_types.count; i++) { + const CBMFieldType *ft = &r->field_types.items[i]; + size_t cl = strlen(ft->class_qn); + size_t sl = strlen(cls_suffix); + if (cl >= sl && strcmp(ft->class_qn + cl - sl, cls_suffix) == 0 && + strcmp(ft->field_name, field) == 0) { + return ft->type_text; + } + } + return NULL; +} + +TEST(python_annotated_instance_fields_exported_issue1277) { + const char *src = "class Trainer(Base):\n" + " engine: Contract\n" + " limit: int = 3\n" + " plain = 5\n\n" + " def __init__(self, strategies: Contract, n, opt: 'Other' = None):\n" + " self.strategies = strategies\n" + " self.typed: Contract = strategies\n" + " self.count = n\n" + " self.made = make()\n" + " self.opt = opt\n" + " if n:\n" + " self.nested: Nested = n\n" + " def inner(other: Inner):\n" + " self.hidden = other\n\n" + " def later(self, x: Later):\n" + " self.late = x\n"; + CBMFileResult *r = extract(src, CBM_LANG_PYTHON, "t", "trainer.py"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "engine"), "Contract"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "limit"), "int"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "strategies"), "Contract"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "typed"), "Contract"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "opt"), "'Other'"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "nested"), "Nested"); + /* Controls: no declared type, a nested function's binding, a non-__init__ method. */ + ASSERT_NULL(py_field_type_of(r, ".Trainer", "plain")); + ASSERT_NULL(py_field_type_of(r, ".Trainer", "count")); + ASSERT_NULL(py_field_type_of(r, ".Trainer", "made")); + ASSERT_NULL(py_field_type_of(r, ".Trainer", "hidden")); + ASSERT_NULL(py_field_type_of(r, ".Trainer", "late")); + ASSERT_EQ(r->field_types.count, 6); + for (int i = 0; i < r->defs.count; i++) { + ASSERT_FALSE(r->defs.items[i].label && strcmp(r->defs.items[i].label, "Field") == 0); + } + cbm_free_result(r); + PASS(); +} + TEST(python_function) { CBMFileResult *r = extract( "def greet(name):\n return f\"Hello {name}\"\n\ndef main():\n greet(\"World\")\n", @@ -8863,6 +8919,530 @@ TEST(extract_walk_truncated_when_a_node_budget_is_set) { PASS(); } +/* #1527: the call-context metrics find each call's innermost enclosing + * Function/Method with a line sweep instead of scanning every def per call. + * The sweep must give the scan's exact answer -- smallest span, ties to the + * lowest def index -- on nested, overlapping, same-line, malformed + * (end < start) and unnamed/non-callable ranges. Fixed-seed generator: + * deterministic, many shapes. */ +static uint32_t enc_rand(uint32_t *state) { + *state = (*state * 1103515245u) + 12345u; + return (*state >> 16) & 0x7fffu; +} + +TEST(extract_enclosing_sweep_matches_scan_issue1527) { + static const char *const labels[] = {"Function", "Method", "Class", "Variable"}; + uint32_t seed = 1527; + for (int round = 0; round < 40; round++) { + CBMArena a; + cbm_arena_init(&a); + CBMFileResult r; + memset(&r, 0, sizeof(r)); + int ndefs = round == 0 ? 0 : (int)(enc_rand(&seed) % 300); + int ncalls = (int)(enc_rand(&seed) % 1500) + 1; + int max_line = (int)(enc_rand(&seed) % 400) + 5; + for (int i = 0; i < ndefs; i++) { + CBMDefinition d; + memset(&d, 0, sizeof(d)); + d.name = (enc_rand(&seed) % 10) == 0 ? NULL : "f"; + d.label = labels[enc_rand(&seed) % 4]; + d.start_line = 1 + (enc_rand(&seed) % (uint32_t)max_line); + int len = (int)(enc_rand(&seed) % 60) - 3; /* a few malformed end < start */ + d.end_line = (uint32_t)((int)d.start_line + len > 0 ? (int)d.start_line + len : 0); + cbm_defs_push(&r.defs, &a, d); + } + for (int i = 0; i < ncalls; i++) { + CBMCall c; + memset(&c, 0, sizeof(c)); + c.callee_name = "g"; + c.start_line = (int)(enc_rand(&seed) % (uint32_t)(max_line + 10)) - 2; + cbm_calls_push(&r.calls, &a, c); + } + ASSERT_EQ(cbm_test_enclosing_sweep_mismatches(&r), 0); + cbm_arena_destroy(&a); + } + PASS(); +} + +/* Python namespace facts: provenance is ordered, content-only and independently + * owned. These tests do not enable any reexport resolution. */ +static bool py_facts_equal(const CBMPyNamespaceFacts *a, const CBMPyNamespaceFacts *b) { + if (!cbm_py_namespace_facts_valid(a) || !cbm_py_namespace_facts_valid(b) || + a->version != b->version || a->language != b->language || a->status != b->status || + a->failure != b->failure || a->count != b->count) + return false; + for (int i = 0; i < a->count; i++) { + const CBMPyNamespaceFact *x = &a->items[i], *y = &b->items[i]; + if (x->kind != y->kind || x->def_kind != y->def_kind || + x->sequence_kind != y->sequence_kind || x->reason != y->reason || + x->flags != y->flags || x->relative_level != y->relative_level || + x->name_count != y->name_count || !cmp_str_eq(x->local_name, y->local_name) || + !cmp_str_eq(x->module_name, y->module_name) || + !cmp_str_eq(x->member_name, y->member_name)) + return false; + for (int j = 0; j < x->name_count; j++) { + if (!cmp_str_eq(x->names[j], y->names[j])) + return false; + } + } + return true; +} + +static bool py_facts_owned_range(const CBMArena *arena, const void *ptr, size_t bytes) { + if (!ptr) + return bytes == 0; + uintptr_t address = (uintptr_t)ptr; + for (int i = 0; i < arena->nblocks; i++) { + uintptr_t start = (uintptr_t)arena->blocks[i]; + size_t size = arena->block_sizes[i]; + if (address >= start && bytes <= size && address - start <= size - bytes) + return true; + } + return false; +} + +static bool py_facts_owned_string(const CBMArena *arena, const char *str) { + return !str || (py_facts_owned_range(arena, str, 1) && + py_facts_owned_range(arena, str, strlen(str) + 1)); +} + +static bool py_facts_owned(const CBMArena *arena, const CBMPyNamespaceFacts *facts) { + if (!py_facts_owned_range(arena, facts->items, (size_t)facts->count * sizeof(*facts->items))) + return false; + for (int i = 0; i < facts->count; i++) { + const CBMPyNamespaceFact *f = &facts->items[i]; + if (!py_facts_owned_string(arena, f->local_name) || + !py_facts_owned_string(arena, f->module_name) || + !py_facts_owned_string(arena, f->member_name) || + !py_facts_owned_range(arena, f->names, (size_t)f->name_count * sizeof(*f->names))) + return false; + for (int j = 0; j < f->name_count; j++) { + if (!py_facts_owned_string(arena, f->names[j])) + return false; + } + } + return true; +} + +static const char PY_FACTS_ORDERED[] = "def item():\n return 1\n" + "from first import item\n" + "from second import item as item\n" + "item = unknown\n" + "del item\n" + "import pkg.mod\n" + "import pkg.mod as alias\n" + "from . import child as local\n" + "from ..outer import member\n" + "from one.shared import *\n" + "from two.shared import *\n" + "class Local:\n pass\n"; + +TEST(python_namespace_facts_preserve_order_and_import_provenance) { + CBMFileResult *r = extract(PY_FACTS_ORDERED, CBM_LANG_PYTHON, "t", "pkg/__init__.py"); + bool ok = r && !r->has_error && !r->parse_incomplete && + r->py_namespace.status == CBM_PY_NS_COMPLETE && r->py_namespace.count == 12 && + cbm_py_namespace_facts_valid(&r->py_namespace) && has_def(r, "Function", "item") && + has_def(r, "Class", "Local") && r->module_qn && + strcmp(r->module_qn, "t.pkg.__init__") == 0; + if (ok) { + const CBMPyNamespaceFact *f = r->py_namespace.items; + ok = f[0].kind == CBM_PY_NS_OWN_DEF && f[0].def_kind == CBM_PY_NS_DEF_FUNCTION && + strcmp(f[0].local_name, "item") == 0 && f[1].kind == CBM_PY_NS_IMPORT_NAME && + strcmp(f[1].module_name, "first") == 0 && strcmp(f[1].member_name, "item") == 0 && + strcmp(f[1].local_name, "item") == 0 && f[2].kind == CBM_PY_NS_IMPORT_NAME && + strcmp(f[2].module_name, "second") == 0 && strcmp(f[2].local_name, "item") == 0 && + f[3].kind == CBM_PY_NS_SHADOW && strcmp(f[3].local_name, "item") == 0 && + f[4].kind == CBM_PY_NS_DELETE && strcmp(f[4].local_name, "item") == 0 && + f[5].kind == CBM_PY_NS_IMPORT_MODULE && strcmp(f[5].module_name, "pkg.mod") == 0 && + strcmp(f[5].local_name, "pkg") == 0 && f[5].flags == CBM_PY_NS_IMPORT_BINDS_ROOT && + f[6].kind == CBM_PY_NS_IMPORT_MODULE && strcmp(f[6].module_name, "pkg.mod") == 0 && + strcmp(f[6].local_name, "alias") == 0 && f[6].flags == 0 && f[7].relative_level == 1 && + strcmp(f[7].module_name, "") == 0 && strcmp(f[7].member_name, "child") == 0 && + strcmp(f[7].local_name, "local") == 0 && f[8].relative_level == 2 && + strcmp(f[8].module_name, "outer") == 0 && strcmp(f[8].member_name, "member") == 0 && + f[9].kind == CBM_PY_NS_IMPORT_STAR && strcmp(f[9].module_name, "one.shared") == 0 && + f[10].kind == CBM_PY_NS_IMPORT_STAR && strcmp(f[10].module_name, "two.shared") == 0 && + f[11].kind == CBM_PY_NS_OWN_DEF && f[11].def_kind == CBM_PY_NS_DEF_CLASS && + strcmp(f[11].local_name, "Local") == 0 && py_facts_owned(&r->arena, &r->py_namespace); + } + if (r) + cbm_free_result(r); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_distinguish_all_operations_and_unknowns) { + static const char source[] = "__all__ = ['old']; __all__ = []\n" + "__all__ = dynamic\n" + "__all__ = ('new', '_private', 'a,b', '')\n" + "__all__ += ('new',)\n" + "__all__ -= []\n" + "del __all__\n"; + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "main.py"); + CBMFileResult *empty = extract("# no bindings\n", CBM_LANG_PYTHON, "t", "empty.py"); + bool ok = r && empty && !r->has_error && !r->parse_incomplete && !empty->has_error && + r->py_namespace.status == CBM_PY_NS_COMPLETE && r->py_namespace.count == 7 && + empty->py_namespace.status == CBM_PY_NS_COMPLETE && empty->py_namespace.count == 0 && + cbm_py_namespace_facts_valid(&r->py_namespace); + if (ok) { + const CBMPyNamespaceFact *f = r->py_namespace.items; + ok = f[0].kind == CBM_PY_NS_ALL_SET && f[0].name_count == 1 && + strcmp(f[0].names[0], "old") == 0 && f[1].kind == CBM_PY_NS_ALL_SET && + f[1].name_count == 0 && f[1].names == NULL && + f[1].sequence_kind == CBM_PY_NS_SEQUENCE_LIST && f[2].kind == CBM_PY_NS_ALL_UNKNOWN && + f[3].kind == CBM_PY_NS_ALL_SET && f[3].sequence_kind == CBM_PY_NS_SEQUENCE_TUPLE && + f[3].name_count == 4 && strcmp(f[3].names[0], "new") == 0 && + strcmp(f[3].names[1], "_private") == 0 && strcmp(f[3].names[2], "a,b") == 0 && + strcmp(f[3].names[3], "") == 0 && f[4].kind == CBM_PY_NS_ALL_APPEND && + f[4].name_count == 1 && strcmp(f[4].names[0], "new") == 0 && + f[5].kind == CBM_PY_NS_ALL_UNKNOWN && f[6].kind == CBM_PY_NS_ALL_DELETE; + } + if (r) + cbm_free_result(r); + if (empty) + cbm_free_result(empty); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_unknown_effects_and_body_stability) { + static const char one[] = "from pkg import Item\ndef f():\n return 1\n"; + static const char two[] = + "from pkg import Item\ndef f():\n from elsewhere import Other\n return Other()\n"; + CBMFileResult *a = extract(one, CBM_LANG_PYTHON, "t", "main.py"); + CBMFileResult *b = extract(two, CBM_LANG_PYTHON, "t", "main.py"); + bool ok = a && b && !a->has_error && !b->has_error && !a->parse_incomplete && + !b->parse_incomplete && a->py_namespace.count == 2 && + py_facts_equal(&a->py_namespace, &b->py_namespace); + if (a) + cbm_free_result(a); + if (b) + cbm_free_result(b); + static const char unknown[] = "from pkg import Item\n" + "if enabled:\n from other import Item\n" + "@deco\ndef wrapped():\n pass\n" + "class Derived(Base):\n pass\n" + "Item = factory()\n"; + a = extract(unknown, CBM_LANG_PYTHON, "t", "main.py"); + ok = a && !a->has_error && !a->parse_incomplete && a->py_namespace.count == 8 && ok; + if (a && a->py_namespace.count == 8) { + const CBMPyNamespaceFact *f = a->py_namespace.items; + ok = f[0].kind == CBM_PY_NS_IMPORT_NAME && f[1].kind == CBM_PY_NS_UNKNOWN && + f[1].reason == CBM_PY_NS_REASON_COMPOUND && f[2].kind == CBM_PY_NS_UNKNOWN && + f[3].kind == CBM_PY_NS_SHADOW && f[3].reason == CBM_PY_NS_REASON_DECORATED && + strcmp(f[3].local_name, "wrapped") == 0 && f[4].kind == CBM_PY_NS_UNKNOWN && + f[5].kind == CBM_PY_NS_SHADOW && f[5].reason == CBM_PY_NS_REASON_UNPROVEN_CLASS && + f[6].kind == CBM_PY_NS_UNKNOWN && f[7].kind == CBM_PY_NS_SHADOW && ok; + } + if (a) + cbm_free_result(a); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_parse_errors_are_complete_unknown) { + const char *source = "from pkg import Before\ndef broken(:\n"; + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "main.py"); + bool ok = r && r->parse_incomplete && r->py_namespace.status == CBM_PY_NS_COMPLETE && + r->py_namespace.count == 1 && r->py_namespace.items[0].kind == CBM_PY_NS_UNKNOWN && + r->py_namespace.items[0].reason == CBM_PY_NS_REASON_PARSE && + cbm_py_namespace_facts_valid(&r->py_namespace); + if (r) + cbm_free_result(r); + CBMFileResult fallback = {0}; + cbm_arena_init(&fallback.arena); + CBMExtractCtx ctx = {.arena = &fallback.arena, + .result = &fallback, + .source = "", + .source_len = 0, + .language = CBM_LANG_PYTHON}; + ok = cbm_extract_python_namespace_facts(&ctx) == CBM_PY_NS_COMPLETE && + fallback.py_namespace.count == 1 && + fallback.py_namespace.items[0].reason == CBM_PY_NS_REASON_PARSE && ok; + cbm_arena_destroy(&fallback.arena); + r = extract("const x = 1;\n", CBM_LANG_JAVASCRIPT, "t", "__init__.js"); + ok = r && !r->has_error && r->py_namespace.status == CBM_PY_NS_NOT_APPLICABLE && + r->py_namespace.language == CBM_LANG_JAVASCRIPT && r->py_namespace.count == 0 && ok; + if (r) + cbm_free_result(r); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_capture_and_copy_fail_atomically) { + static const char source[] = "from .pkg import Item as Local\n__all__ = ['Local', '']\n"; + bool ok = true, capture_complete = false, copy_complete = false; + int capture_failures = 0, copy_failures = 0; + for (int limit = 0; limit < 64 && !capture_complete; limit++) { + cbm_py_namespace_test_capture_fail_after(limit); + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "pkg/__init__.py"); + cbm_py_namespace_test_capture_fail_after(-1); + if (!r) { + ok = false; + break; + } + const CBMPyNamespaceFacts *f = &r->py_namespace; + capture_complete = f->status == CBM_PY_NS_COMPLETE; + if (capture_complete) { + ok = f->count == 2 && cbm_py_namespace_facts_valid(f) && ok; + } else { + capture_failures++; + ok = f->status == CBM_PY_NS_INCOMPLETE && f->failure == CBM_PY_NS_FAILURE_ALLOCATION && + !f->items && f->count == 0 && f->cap == 0 && cbm_py_namespace_facts_valid(f) && ok; + } + cbm_free_result(r); + } + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "pkg/__init__.py"); + CBMArena reference_arena; + cbm_arena_init(&reference_arena); + CBMPyNamespaceFacts original = {0}; + ok = r && cbm_py_namespace_facts_copy(&reference_arena, &r->py_namespace, &original) && ok; + for (int limit = 0; r && limit < 64 && !copy_complete; limit++) { + CBMArena arena; + cbm_arena_init(&arena); + CBMPyNamespaceFacts copy = {0}; + cbm_py_namespace_test_copy_fail_after(limit); + copy_complete = cbm_py_namespace_facts_copy(&arena, &r->py_namespace, ©); + cbm_py_namespace_test_copy_fail_after(-1); + if (copy_complete) { + ok = py_facts_equal(©, &r->py_namespace) && py_facts_owned(&arena, ©) && ok; + } else { + copy_failures++; + ok = copy.status == CBM_PY_NS_INCOMPLETE && + copy.failure == CBM_PY_NS_FAILURE_ALLOCATION && !copy.items && copy.count == 0 && + copy.cap == 0 && ok; + } + ok = py_facts_equal(&r->py_namespace, &original) && ok; + cbm_arena_destroy(&arena); + } + /* Rebuilding into an existing result must replace old complete facts on + * failure; do not leave a stale successful manifest attached. */ + if (r) { + CBMExtractCtx ctx = {.arena = &r->arena, + .result = r, + .source = source, + .source_len = (int)strlen(source), + .language = CBM_LANG_PYTHON}; + cbm_py_namespace_test_capture_fail_after(0); + CBMPyNamespaceStatus status = cbm_extract_python_namespace_facts(&ctx); + cbm_py_namespace_test_capture_fail_after(-1); + ok = status == CBM_PY_NS_INCOMPLETE && !r->py_namespace.items && + r->py_namespace.count == 0 && ok; + cbm_free_result(r); + } + cbm_arena_destroy(&reference_arena); + ASSERT_TRUE(ok); + ASSERT_TRUE(capture_complete); + ASSERT_TRUE(copy_complete); + ASSERT_GT(capture_failures, 2); + ASSERT_GT(copy_failures, 2); + PASS(); +} + +TEST(python_namespace_facts_validation_rejects_partial_and_unknown_schema) { + CBMPyNamespaceFacts zero = {0}; + bool ok = cbm_py_namespace_facts_valid(&zero) && zero.status == CBM_PY_NS_NOT_CAPTURED; + CBMPyNamespaceFact event = {.kind = CBM_PY_NS_ALL_SET, + .sequence_kind = CBM_PY_NS_SEQUENCE_LIST}; + CBMPyNamespaceFacts facts = {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = CBM_LANG_PYTHON, + .status = CBM_PY_NS_COMPLETE, + .items = &event, + .count = 1, + .cap = 1}; + ok = cbm_py_namespace_facts_valid(&facts) && ok; + for (int variant = 0; variant < 8; variant++) { + CBMPyNamespaceFact bad_event = event; + CBMPyNamespaceFacts bad = facts; + bad.items = &bad_event; + if (variant == 0) + bad.version++; + if (variant == 1) + bad.cap = 0; + if (variant == 2) + bad.count = -1; + if (variant == 3) + bad_event.kind = (CBMPyNamespaceFactKind)999; + if (variant == 4) + bad_event.name_count = 1; + if (variant == 5) + bad_event.flags = UINT32_MAX; + if (variant == 6) + bad.language = CBM_LANG_JAVA; + if (variant == 7) + bad.status = CBM_PY_NS_INCOMPLETE; + CBMArena arena; + cbm_arena_init(&arena); + CBMPyNamespaceFacts copy = {0}; + ok = !cbm_py_namespace_facts_valid(&bad) && + !cbm_py_namespace_facts_copy(&arena, &bad, ©) && + copy.status == CBM_PY_NS_INCOMPLETE && copy.count == 0 && !copy.items && ok; + cbm_arena_destroy(&arena); + } + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_compact_spill_and_copy_keep_owned_payload) { + char source[] = "from . import Thing as Local\n" + "__all__ = ['Local', '', 'a,b', '_private']\n" + "if enabled:\n Local = dynamic\n" + "__all__ = []\n"; + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "pkg/__init__.py"); + CBMArena retained; + cbm_arena_init(&retained); + CBMPyNamespaceFacts expected = {0}; + bool ok = r && !r->has_error && !r->parse_incomplete && r->py_namespace.count == 4 && + cbm_py_namespace_facts_copy(&retained, &r->py_namespace, &expected); + memset(source, 'x', strlen(source)); /* facts cannot borrow the caller's input */ + char dir[512]; + int n = snprintf(dir, sizeof(dir), "%s/cbm_py_facts_XXXXXX", cbm_tmpdir()); + bool made = n > 0 && (size_t)n < sizeof(dir) && cbm_mkdtemp(dir); + size_t saved = cbm_result_spill_pin_free_bytes_for_tests(SIZE_MAX); + cbm_result_spill_t *spill = made ? cbm_result_spill_open(dir, 1, 1) : NULL; + cbm_result_spill_pin_free_bytes_for_tests(saved); + ok = spill && ok; + if (r && spill && ok) { + cbm_result_compact(r); + cbm_result_compact(r); + ok = r->arena.nblocks == 1 && r->py_namespace.cap == r->py_namespace.count && + py_facts_equal(&r->py_namespace, &expected) && + py_facts_owned(&r->arena, &r->py_namespace); + if (ok && cbm_result_spill_park(spill, 0, 0, r)) { + r = NULL; + CBMFileResult header = {0}; + ok = cbm_result_spill_peek_header(spill, 0, &header) && + header.py_namespace.status == CBM_PY_NS_COMPLETE && header.py_namespace.count == 4; + for (int round = 0; round < 2; round++) { + CBMFileResult *loaded = cbm_result_spill_load(spill, 0); + ok = loaded && loaded->cached_tree == NULL && + py_facts_owned(&loaded->arena, &loaded->py_namespace) && + py_facts_equal(&loaded->py_namespace, &expected) && ok; + if (loaded) + cbm_free_result(loaded); + } + } else { + ok = false; + } + } + if (r) + cbm_free_result(r); + cbm_result_spill_close(spill); + if (made) { + char subdir[1024]; + snprintf(subdir, sizeof(subdir), "%s/spill", dir); + cbm_rmdir(subdir); + cbm_rmdir(dir); + } + /* Original result/source tree and both reloads are gone; the independent + * copied payload still owns the relative empty module and literal strings. */ + ok = py_facts_owned(&retained, &expected) && cbm_py_namespace_facts_valid(&expected) && + expected.count == 4 && expected.items[0].relative_level == 1 && + strcmp(expected.items[0].module_name, "") == 0 && expected.items[1].name_count == 4 && + strcmp(expected.items[1].names[2], "a,b") == 0 && ok; + cbm_arena_destroy(&retained); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_unavailable_and_empty_survive_spill) { + CBMPyNamespaceFacts states[] = { + {0}, + {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = CBM_LANG_PYTHON, + .status = CBM_PY_NS_COMPLETE}, + {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = CBM_LANG_PYTHON, + .status = CBM_PY_NS_INCOMPLETE, + .failure = CBM_PY_NS_FAILURE_ALLOCATION}, + {.version = CBM_PY_NAMESPACE_FACTS_VERSION, + .language = CBM_LANG_JAVASCRIPT, + .status = CBM_PY_NS_NOT_APPLICABLE}, + }; + char dir[512]; + int n = snprintf(dir, sizeof(dir), "%s/cbm_py_fact_states_XXXXXX", cbm_tmpdir()); + bool made = n > 0 && (size_t)n < sizeof(dir) && cbm_mkdtemp(dir); + size_t saved = cbm_result_spill_pin_free_bytes_for_tests(SIZE_MAX); + cbm_result_spill_t *spill = made ? cbm_result_spill_open(dir, 1, 4) : NULL; + cbm_result_spill_pin_free_bytes_for_tests(saved); + bool ok = spill != NULL; + for (int i = 0; spill && i < 4; i++) { + CBMFileResult *r = cbm_result_alloc(); + if (!r) { + ok = false; + break; + } + cbm_arena_init(&r->arena); + r->module_qn = cbm_arena_strdup(&r->arena, "t.metadata"); + r->py_namespace = states[i]; + cbm_result_compact(r); + cbm_result_compact(r); + bool ready = + r->module_qn && r->arena.nblocks == 1 && py_facts_equal(&r->py_namespace, &states[i]); + if (!ready || !cbm_result_spill_park(spill, 0, i, r)) { + ok = false; + cbm_free_result(r); + continue; + } + CBMFileResult header = {0}; + ok = cbm_result_spill_peek_header(spill, i, &header) && + header.py_namespace.status == states[i].status && header.py_namespace.count == 0 && ok; + for (int round = 0; round < 2; round++) { + CBMFileResult *loaded = cbm_result_spill_load(spill, i); + ok = loaded && py_facts_equal(&loaded->py_namespace, &states[i]) && + py_facts_owned(&loaded->arena, &loaded->py_namespace) && ok; + if (loaded) + cbm_free_result(loaded); + } + } + cbm_result_spill_close(spill); + if (made) { + char subdir[1024]; + snprintf(subdir, sizeof(subdir), "%s/spill", dir); + cbm_rmdir(subdir); + cbm_rmdir(dir); + } + ASSERT_TRUE(ok); + PASS(); +} + +TEST(python_namespace_facts_multiple_bindings_and_unsupported_effects) { + static const char source[] = "import alpha, beta.mod as Beta\n" + "from pkg import (One, Two as Local)\n" + "from pkg import later\n" + "def later():\n pass\n" + "from pkg import __all__\n" + "def __all__():\n pass\n" + "values = {obj: 1}\n" + "class HasDescriptor:\n field = descriptor\n" + "__all__ = ['escaped\\n']\n"; + CBMFileResult *r = extract(source, CBM_LANG_PYTHON, "t", "main.py"); + bool ok = r && !r->has_error && !r->parse_incomplete && + r->py_namespace.status == CBM_PY_NS_COMPLETE && r->py_namespace.count == 14 && + cbm_py_namespace_facts_valid(&r->py_namespace) && has_def(r, "Function", "later") && + has_def(r, "Class", "HasDescriptor"); + if (ok) { + const CBMPyNamespaceFact *f = r->py_namespace.items; + ok = f[0].kind == CBM_PY_NS_IMPORT_MODULE && strcmp(f[0].local_name, "alpha") == 0 && + f[1].kind == CBM_PY_NS_IMPORT_MODULE && strcmp(f[1].local_name, "Beta") == 0 && + strcmp(f[1].module_name, "beta.mod") == 0 && f[2].kind == CBM_PY_NS_IMPORT_NAME && + strcmp(f[2].member_name, "One") == 0 && f[3].kind == CBM_PY_NS_IMPORT_NAME && + strcmp(f[3].member_name, "Two") == 0 && strcmp(f[3].local_name, "Local") == 0 && + f[4].kind == CBM_PY_NS_IMPORT_NAME && f[5].kind == CBM_PY_NS_OWN_DEF && + strcmp(f[5].local_name, "later") == 0 && f[6].kind == CBM_PY_NS_IMPORT_NAME && + strcmp(f[6].local_name, "__all__") == 0 && f[7].kind == CBM_PY_NS_OWN_DEF && + strcmp(f[7].local_name, "__all__") == 0 && f[8].kind == CBM_PY_NS_UNKNOWN && + f[9].kind == CBM_PY_NS_SHADOW && strcmp(f[9].local_name, "values") == 0 && + f[10].kind == CBM_PY_NS_UNKNOWN && f[11].kind == CBM_PY_NS_SHADOW && + f[11].reason == CBM_PY_NS_REASON_UNPROVEN_CLASS && f[12].kind == CBM_PY_NS_UNKNOWN && + f[13].kind == CBM_PY_NS_ALL_UNKNOWN; + } + if (r) + cbm_free_result(r); + ASSERT_TRUE(ok); + PASS(); +} + SUITE(extraction) { RUN_TEST(extract_compact_keeps_every_field_and_shrinks_the_arena); RUN_TEST(extract_compact_is_idempotent_and_survives_empty_results); @@ -8873,6 +9453,16 @@ SUITE(extraction) { /* Initialize extraction library */ cbm_init(); + RUN_TEST(python_namespace_facts_preserve_order_and_import_provenance); + RUN_TEST(python_namespace_facts_distinguish_all_operations_and_unknowns); + RUN_TEST(python_namespace_facts_unknown_effects_and_body_stability); + RUN_TEST(python_namespace_facts_parse_errors_are_complete_unknown); + RUN_TEST(python_namespace_facts_capture_and_copy_fail_atomically); + RUN_TEST(python_namespace_facts_validation_rejects_partial_and_unknown_schema); + RUN_TEST(python_namespace_facts_compact_spill_and_copy_keep_owned_payload); + RUN_TEST(python_namespace_facts_unavailable_and_empty_survive_spill); + RUN_TEST(python_namespace_facts_multiple_bindings_and_unsupported_effects); + /* Wide-flat-file linearity (ms-typescript hang) */ RUN_TEST(extract_wide_flat_file_is_linear); #if defined(CBM_CALL_REFERENCE_LOOKUP_TEST_API) && CBM_CALL_REFERENCE_LOOKUP_TEST_API @@ -8991,6 +9581,7 @@ SUITE(extraction) { /* Scripting */ RUN_TEST(python_function); RUN_TEST(python_class); + RUN_TEST(python_annotated_instance_fields_exported_issue1277); RUN_TEST(js_function); RUN_TEST(js_class); RUN_TEST(ts_function); @@ -9304,6 +9895,7 @@ SUITE(extraction) { RUN_TEST(json_toplevel_description_promoted_to_module_issue519); RUN_TEST(config_description_only_at_top_level_issue519); RUN_TEST(non_config_language_module_has_no_promoted_description_issue519); + RUN_TEST(extract_enclosing_sweep_matches_scan_issue1527); cbm_shutdown(); } diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 56be6b88e..a7d38e79a 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -10,7 +10,10 @@ #include "test_helpers.h" #include "foundation/mem.h" // cbm_mem_init/budget (back-pressure futile-nap test) #include "pipeline/pipeline.h" +#include "lsp/c_lsp.h" +#include "lsp/py_lsp.h" #include "pipeline/lsp_surface.h" +#include "pipeline/pass_lsp_cross.h" #include "pipeline/pipeline_internal.h" #include "pipeline/artifact.h" #include "store/store.h" @@ -8166,6 +8169,1330 @@ static void teardown_usages_repo(void) { g_usages_tmpdir[0] = '\0'; } +/* Real cross-file memo failure, after successful extraction. The failure + * must reach the public per-file error list on both production drivers. */ +static int pipeline_lsp_failure_case(bool parallel, const char *stage, bool python, bool depth) { + const char *src = "struct S { int m() { return 1; } };\n" + "void run() { S s; s.m(); }\n"; + if (python) + src = "class S:\n def m(self):\n return 1\ndef run():\n s = S()\n s.m()\n"; + const char *filename = python ? "memo.py" : "memo.cpp"; + if (setup_usages_repo(filename, src, NULL, NULL) != 0) + FAIL("failed to create memo fixture"); + char path[512]; + if (parallel) { + for (int i = 0; i < 51; i++) { + snprintf(path, sizeof(path), "%s/memo_pad_%02d.%s", g_usages_tmpdir, i, + python ? "py" : "cpp"); + if (th_write_file(path, python ? "# inert parallel routing fixture\n" + : "// inert parallel routing fixture\n") != 0) { + teardown_usages_repo(); + FAIL("failed to create parallel routing fixture"); + } + } + } + const char *keys[] = { + "CBM_WORKERS", "CBM_INDEX_SINGLE_THREAD", + depth ? (python ? "CBM_TEST_PY_LSP_DEPTH_FAIL_STAGE" : "CBM_TEST_C_LSP_DEPTH_FAIL_STAGE") + : (python ? "CBM_TEST_PY_LSP_MEMO_FAIL_STAGE" : "CBM_TEST_C_LSP_MEMO_FAIL_STAGE")}; + char *saved[3] = {0}; + bool env_saved = true; + for (int i = 0; i < 3; i++) { + const char *old = getenv(keys[i]); + saved[i] = old ? strdup(old) : NULL; + env_saved = env_saved && (!old || saved[i]); + } + if (!env_saved) { + for (int i = 0; i < 3; i++) + free(saved[i]); + teardown_usages_repo(); + FAIL("failed to save memo fixture environment"); + } + int set_rc = cbm_setenv(keys[0], parallel ? "4" : "1", 1); + set_rc |= parallel ? cbm_unsetenv(keys[1]) : cbm_setenv(keys[1], "1", 1); + set_rc |= cbm_setenv(keys[2], stage, 1); + snprintf(path, sizeof(path), "%s/memo.db", g_usages_tmpdir); + cbm_pipeline_t *p = set_rc == 0 ? cbm_pipeline_new(g_usages_tmpdir, path, CBM_MODE_FULL) : NULL; + int run_rc = p ? cbm_pipeline_run(p) : -1; + cbm_file_error_t *errors = NULL; + int error_count = 0; + cbm_pipeline_get_file_errors(p, &errors, &error_count); + int memo_errors = 0; + const char *expected_phase = strcmp(stage, "cross") == 0 ? "lsp_skipped" : "extract"; + bool precise = true; + for (int i = 0; i < error_count; i++) { + if (errors[i].path && strcmp(errors[i].path, filename) == 0) { + memo_errors++; + precise = precise && errors[i].phase && strcmp(errors[i].phase, expected_phase) == 0 && + errors[i].reason && + strcmp(errors[i].reason, + depth ? (python ? CBM_PY_LSP_DEPTH_ERROR : CBM_C_LSP_DEPTH_ERROR) + : (python ? CBM_PY_LSP_MEMO_ERROR : CBM_C_LSP_MEMO_ERROR)) == 0; + } + } + cbm_pipeline_free(p); + int restore_rc = 0; + for (int i = 0; i < 3; i++) { + restore_rc |= saved[i] ? cbm_setenv(keys[i], saved[i], 1) : cbm_unsetenv(keys[i]); + free(saved[i]); + } + teardown_usages_repo(); + ASSERT_EQ(set_rc, 0); + ASSERT_EQ(restore_rc, 0); + ASSERT_EQ(run_rc, 0); + ASSERT_EQ(memo_errors, 1); + ASSERT_TRUE(precise); + PASS(); +} + +TEST(pipeline_issue1527_cross_memo_failure_sequential) { + return pipeline_lsp_failure_case(false, "cross", false, false); +} +TEST(pipeline_issue1527_cross_memo_failure_parallel) { + return pipeline_lsp_failure_case(true, "cross", false, false); +} + +TEST(pipeline_issue1527_raw_memo_failure_sequential) { + return pipeline_lsp_failure_case(false, "raw", false, false); +} +TEST(pipeline_issue1527_raw_memo_failure_parallel) { + return pipeline_lsp_failure_case(true, "raw", false, false); +} + +TEST(pipeline_issue1527_python_raw_memo_failure_sequential) { + return pipeline_lsp_failure_case(false, "raw", true, false); +} + +TEST(pipeline_issue1527_python_raw_memo_failure_parallel) { + return pipeline_lsp_failure_case(true, "raw", true, false); +} + +TEST(pipeline_issue1527_python_cross_memo_failure_sequential) { + return pipeline_lsp_failure_case(false, "cross", true, false); +} + +TEST(pipeline_issue1527_python_cross_memo_failure_parallel) { + return pipeline_lsp_failure_case(true, "cross", true, false); +} + +TEST(pipeline_issue1527_c_raw_depth_failure_sequential) { + return pipeline_lsp_failure_case(false, "raw", false, true); +} + +TEST(pipeline_issue1527_c_cross_depth_failure_sequential) { + return pipeline_lsp_failure_case(false, "cross", false, true); +} + +TEST(pipeline_issue1527_c_raw_depth_failure_parallel) { + return pipeline_lsp_failure_case(true, "raw", false, true); +} + +TEST(pipeline_issue1527_c_cross_depth_failure_parallel) { + return pipeline_lsp_failure_case(true, "cross", false, true); +} + +TEST(pipeline_issue1527_python_raw_depth_failure_sequential) { + return pipeline_lsp_failure_case(false, "raw", true, true); +} + +TEST(pipeline_issue1527_python_cross_depth_failure_sequential) { + return pipeline_lsp_failure_case(false, "cross", true, true); +} + +TEST(pipeline_issue1527_python_raw_depth_failure_parallel) { + return pipeline_lsp_failure_case(true, "raw", true, true); +} + +TEST(pipeline_issue1527_python_cross_depth_failure_parallel) { + return pipeline_lsp_failure_case(true, "cross", true, true); +} + +TEST(pipeline_issue1527_cross_memo_failure_fallback) { + const char *src = "struct S { int m() { return 1; } }; void run() { S s; s.m(); }"; + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_CPP}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_CPP}, + }; + CBMFileResult result = {0}; + cbm_arena_init(&result.arena); + cbm_c_lsp_test_memo_fail_after(0); + cbm_pxc_run_one(CBM_LANG_CPP, &result, src, (int)strlen(src), "test", defs, 2, NULL, NULL, 0); + cbm_c_lsp_test_memo_fail_after(-1); + bool reported = result.has_error && result.lsp_skipped && result.error_msg && + strcmp(result.error_msg, CBM_C_LSP_MEMO_ERROR) == 0 && + result.resolved_calls.count == 0; + cbm_arena_destroy(&result.arena); + ASSERT_TRUE(reported); + PASS(); +} + +static bool python_module_write_fixture_file(const char *root, const char *name, const char *text) { + char path[768]; + int n = snprintf(path, sizeof(path), "%s/%s", root, name); + return n > 0 && (size_t)n < sizeof(path) && th_write_file(path, text) == 0; +} + +/* An absent node, failed edge query, or heuristic edge must not satisfy this + * integration check. Exact source/target identities precede strategy checks. */ +static bool python_module_exact_lsp_edge(cbm_store_t *store, const char *project, + const char *caller, const char *caller_label, + const char *target, const char *target_label) { + char source_qn[768], target_qn[768]; + int sn = snprintf(source_qn, sizeof(source_qn), "%s.%s", project, caller); + int tn = snprintf(target_qn, sizeof(target_qn), "%s.%s", project, target); + if (sn <= 0 || (size_t)sn >= sizeof(source_qn) || tn <= 0 || (size_t)tn >= sizeof(target_qn)) + return false; + cbm_node_t source = {0}, destination = {0}; + bool correct = + cbm_store_find_node_by_qn(store, project, source_qn, &source) == CBM_STORE_OK && + cbm_store_find_node_by_qn(store, project, target_qn, &destination) == CBM_STORE_OK && + source.id > 0 && destination.id > 0 && source.label && destination.label && + strcmp(source.label, caller_label) == 0 && strcmp(destination.label, target_label) == 0; + cbm_edge_t *edges = NULL; + int edge_count = 0, hits = 0; + if (correct) { + correct = cbm_store_find_edges_by_source_type(store, source.id, "CALLS", &edges, + &edge_count) == CBM_STORE_OK; + for (int i = 0; correct && i < edge_count; i++) { + if (edges[i].source_id != source.id || edges[i].target_id != destination.id) + continue; + hits++; + const char *properties = edges[i].properties_json; + yyjson_doc *doc = properties ? yyjson_read(properties, strlen(properties), 0) : NULL; + const char *strategy = + doc ? yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(doc), "strategy")) : NULL; + correct = strategy && strncmp(strategy, "lsp_", 4) == 0; + if (doc) + yyjson_doc_free(doc); + } + } + cbm_store_free_edges(edges, edge_count); + cbm_node_free_fields(&source); + cbm_node_free_fields(&destination); + return correct && hits == 1; +} + +static int python_module_callers_pipeline_case(bool parallel) { + static const char helpers[] = "def helper():\n" + " return 1\n" + "class Service:\n" + " def run(self):\n" + " return 2\n"; + static const char app[] = "from helpers import helper, Service\n" + "def local_helper():\n" + " return 3\n" + "class Local:\n" + " def run(self):\n" + " return 4\n" + "class Controller:\n" + " def run(self):\n" + " helper()\n" + " method_service = Service()\n" + " method_service.run()\n" + "def control():\n" + " helper()\n" + " function_service = Service()\n" + " function_service.run()\n" + " local_helper()\n" + " function_local = Local()\n" + " function_local.run()\n" + "helper()\n" + "module_service = Service()\n" + "module_service.run()\n" + "local_helper()\n" + "module_local = Local()\n" + "module_local.run()\n"; + static const struct { + const char *caller; + const char *caller_label; + const char *target; + const char *target_label; + } expected[] = { + {"app", "Module", "helpers.helper", "Function"}, + {"app", "Module", "helpers.Service.run", "Method"}, + {"app", "Module", "app.local_helper", "Function"}, + {"app", "Module", "app.Local.run", "Method"}, + {"app.control", "Function", "helpers.helper", "Function"}, + {"app.control", "Function", "helpers.Service.run", "Method"}, + {"app.control", "Function", "app.local_helper", "Function"}, + {"app.control", "Function", "app.Local.run", "Method"}, + {"app.Controller.run", "Method", "helpers.helper", "Function"}, + {"app.Controller.run", "Method", "helpers.Service.run", "Method"}, + {"pkg.__init__", "Module", "helpers.helper", "Function"}, + }; + char root[512]; + int n = snprintf(root, sizeof(root), "%s/cbm-python-module-XXXXXX", cbm_tmpdir()); + if (n <= 0 || (size_t)n >= sizeof(root) || !cbm_mkdtemp(root)) + FAIL("failed to create Python module fixture"); + char path[768]; + n = snprintf(path, sizeof(path), "%s/pkg", root); + bool written = n > 0 && (size_t)n < sizeof(path); + if (written) { + cbm_mkdir_p(path, 0755); + written = python_module_write_fixture_file(root, "helpers.py", helpers) && + python_module_write_fixture_file(root, "app.py", app) && + python_module_write_fixture_file(root, "pkg/__init__.py", + "from helpers import helper\n" + "PACKAGE_VALUE = helper()\n"); + } + for (int i = 0; written && parallel && i < 52; i++) { + char name[64]; + snprintf(name, sizeof(name), "module_pad_%02d.py", i); + written = python_module_write_fixture_file(root, name, "padding = 0\n"); + } + if (!written) { + th_rmtree(root); + FAIL("failed to write Python module fixture"); + } + + const char *keys[] = {"CBM_WORKERS", "CBM_INDEX_SINGLE_THREAD"}; + char *saved[2] = {0}; + bool saved_ok = true; + for (int i = 0; i < 2; i++) { + const char *old = getenv(keys[i]); + saved[i] = old ? strdup(old) : NULL; + saved_ok = saved_ok && (!old || saved[i]); + } + if (!saved_ok) { + free(saved[0]); + free(saved[1]); + th_rmtree(root); + FAIL("failed to save Python module fixture environment"); + } + int set_rc = cbm_setenv(keys[0], parallel ? "4" : "1", 1); + set_rc |= parallel ? cbm_unsetenv(keys[1]) : cbm_setenv(keys[1], "1", 1); + n = snprintf(path, sizeof(path), "%s/module.db", root); + cbm_pipeline_t *pipeline = set_rc == 0 && n > 0 && (size_t)n < sizeof(path) + ? cbm_pipeline_new(root, path, CBM_MODE_FULL) + : NULL; + int run_rc = pipeline ? cbm_pipeline_run(pipeline) : -1; + cbm_file_error_t *errors = NULL; + int error_count = 0; + cbm_pipeline_get_file_errors(pipeline, &errors, &error_count); + cbm_store_t *store = run_rc == 0 ? cbm_store_open_path(path) : NULL; + bool correct = store != NULL; + int file_count = 0; + if (store) { + const char *project = cbm_pipeline_project_name(pipeline); + cbm_node_t *files = NULL; + correct = project && + cbm_store_find_nodes_by_label(store, project, "File", &files, &file_count) == + CBM_STORE_OK && + file_count == (parallel ? 55 : 3); + cbm_store_free_nodes(files, file_count); + for (size_t i = 0; i < sizeof(expected) / sizeof(expected[0]); i++) { + bool edge = project && python_module_exact_lsp_edge( + store, project, expected[i].caller, expected[i].caller_label, + expected[i].target, expected[i].target_label); + correct = correct && edge; + } + /* Neither the ordinary nor package Module should acquire a fake + * __module__ owner node. A failed lookup query is not absence proof. */ + const char *phantoms[] = {"app.__module__", "pkg.__init__.__module__"}; + for (int i = 0; project && i < 2; i++) { + char qn[768]; + n = snprintf(qn, sizeof(qn), "%s.%s", project, phantoms[i]); + cbm_node_t node = {0}; + bool absent = + n > 0 && (size_t)n < sizeof(qn) && + cbm_store_find_node_by_qn(store, project, qn, &node) == CBM_STORE_NOT_FOUND; + correct = correct && absent; + cbm_node_free_fields(&node); + } + cbm_store_close(store); + } + cbm_pipeline_free(pipeline); + int restore_rc = 0; + for (int i = 0; i < 2; i++) { + restore_rc |= saved[i] ? cbm_setenv(keys[i], saved[i], 1) : cbm_unsetenv(keys[i]); + free(saved[i]); + } + int cleanup_rc = th_rmtree(root); + ASSERT_EQ(set_rc, 0); + ASSERT_EQ(restore_rc, 0); + ASSERT_EQ(cleanup_rc, 0); + ASSERT_EQ(run_rc, 0); + ASSERT_EQ(error_count, 0); + ASSERT_TRUE(correct); + PASS(); +} + +/* Insert after python_module_exact_lsp_edge. Parent registers the four TESTs. */ +static const char namespace_admission_control[] = "def target():\n" + " return 1\n" + "def keep():\n" + " return target()\n"; + +static const char *const namespace_admission_env_keys[] = {"CBM_WORKERS", + "CBM_INDEX_SINGLE_THREAD"}; + +typedef struct { + char root[512]; + char db[768]; + char *saved[2]; + bool created; + bool saved_env; +} namespace_admission_fixture; + +typedef struct { + int rc; + int errors; + int format; + cbm_incremental_route_t route; + bool migrated; + bool healthy; + char project[512]; + char *json; +} namespace_admission_result; + +static bool namespace_admission_begin(namespace_admission_fixture *f) { + int n = snprintf(f->root, sizeof(f->root), "%s/cbm-namespace-admission-XXXXXX", cbm_tmpdir()); + if (n <= 0 || (size_t)n >= sizeof(f->root) || !cbm_mkdtemp(f->root)) + return false; + f->created = true; + n = snprintf(f->db, sizeof(f->db), "%s/namespace.db", f->root); + if (n <= 0 || (size_t)n >= sizeof(f->db)) + return false; + for (int i = 0; i < 2; i++) { + const char *old = getenv(namespace_admission_env_keys[i]); + f->saved[i] = old ? strdup(old) : NULL; + if (old && !f->saved[i]) + return false; + } + f->saved_env = true; + int env_rc = cbm_setenv(namespace_admission_env_keys[0], "1", 1); + env_rc |= cbm_setenv(namespace_admission_env_keys[1], "1", 1); + return env_rc == 0 && + python_module_write_fixture_file(f->root, "control.py", namespace_admission_control); +} + +static bool namespace_admission_finish(namespace_admission_fixture *f) { + int restore_rc = 0; + for (int i = 0; i < 2; i++) { + if (f->saved_env) + restore_rc |= f->saved[i] ? cbm_setenv(namespace_admission_env_keys[i], f->saved[i], 1) + : cbm_unsetenv(namespace_admission_env_keys[i]); + free(f->saved[i]); + } + cbm_pipeline_incremental_test_reset_faults(); + int cleanup_rc = f->created ? th_rmtree(f->root) : 0; + return restore_rc == 0 && cleanup_rc == 0; +} + +static bool namespace_admission_exact_hash(cbm_store_t *store, const char *project, + const char *path, const char *source) { + cbm_file_hash_t row = {0}; + char expected[CBM_SHA256_HEX_LEN + 1]; + cbm_sha256_hex(source, strlen(source), expected); + bool correct = cbm_store_get_file_hash(store, project, path, &row) == CBM_STORE_OK && + row.sha256 && strcmp(row.sha256, expected) == 0; + cbm_store_clear_file_hash(&row); + return correct; +} + +static namespace_admission_result namespace_admission_run(namespace_admission_fixture *f, + const char *watch, const char *source, + int expected_files) { + namespace_admission_result result = { + .rc = -1, .errors = -1, .format = -1, .route = CBM_INCREMENTAL_ROUTE_NONE}; + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *pipeline = cbm_pipeline_new(f->root, f->db, CBM_MODE_FULL); + if (!pipeline) + return result; + result.rc = cbm_pipeline_run(pipeline); + result.route = cbm_pipeline_incremental_test_last_route(); + result.migrated = cbm_pipeline_had_format_migration(pipeline); + cbm_file_error_t *errors = NULL; + cbm_pipeline_get_file_errors(pipeline, &errors, &result.errors); + const char *project = cbm_pipeline_project_name(pipeline); + int n = project ? snprintf(result.project, sizeof(result.project), "%s", project) : -1; + cbm_store_t *store = result.rc == 0 && n > 0 && (size_t)n < sizeof(result.project) + ? cbm_store_open_path(f->db) + : NULL; + if (store) { + cbm_node_t *files = NULL; + int file_count = 0; + bool file_query = cbm_store_find_nodes_by_label(store, project, "File", &files, + &file_count) == CBM_STORE_OK; + int watched_files = 0; + for (int i = 0; file_query && i < file_count; i++) + if (files[i].file_path && strcmp(files[i].file_path, watch) == 0) + watched_files++; + cbm_store_free_nodes(files, file_count); + bool positive = python_module_exact_lsp_edge(store, project, "control.keep", "Function", + "control.target", "Function"); + bool hashes = namespace_admission_exact_hash(store, project, "control.py", + namespace_admission_control) && + namespace_admission_exact_hash(store, project, watch, source); + bool format_ok = cbm_store_get_format_version(store, &result.format) == CBM_STORE_OK; + cbm_lsp_surface_row_t *rows = NULL; + int row_count = 0; + bool row_query = + cbm_store_get_lsp_surfaces(store, project, &rows, &row_count) == CBM_STORE_OK; + int matches = 0; + for (int i = 0; row_query && i < row_count; i++) { + if (!rows[i].rel_path || strcmp(rows[i].rel_path, watch) != 0) + continue; + matches++; + if (rows[i].defs_json && !result.json) + result.json = strdup(rows[i].defs_json); + } + cbm_store_free_lsp_surfaces(rows, row_count); + bool python_row = matches == 1 && result.json && + cbm_lsp_surface_python_change(result.json, result.json) == 0 && + cbm_lsp_surface_python_change(result.json, NULL) == 1; + result.healthy = result.errors == 0 && file_query && file_count == expected_files && + watched_files == 1 && positive && hashes && format_ok && + result.format == CBM_INDEX_FORMAT_VERSION && row_query && python_row; + cbm_store_close(store); + } + cbm_pipeline_free(pipeline); + return result; +} + +TEST(pipeline_python_namespace_admission_upgrade_then_noop) { + namespace_admission_fixture fixture = {0}; + bool setup = namespace_admission_begin(&fixture); + namespace_admission_result first = {0}, upgraded = {0}, unchanged = {0}; + bool stamped = false; + if (setup) { + first = namespace_admission_run(&fixture, "control.py", namespace_admission_control, 1); + cbm_store_t *store = first.healthy ? cbm_store_open_path(fixture.db) : NULL; + if (store) { + int old_format = -1; + stamped = cbm_store_set_format_version(store, 1) == CBM_STORE_OK && + cbm_store_get_format_version(store, &old_format) == CBM_STORE_OK && + old_format == 1; + cbm_store_close(store); + } + if (stamped) + upgraded = + namespace_admission_run(&fixture, "control.py", namespace_admission_control, 1); + if (upgraded.healthy) + unchanged = + namespace_admission_run(&fixture, "control.py", namespace_admission_control, 1); + } + bool stable = first.json && upgraded.json && unchanged.json && + cbm_lsp_surface_python_change(first.json, upgraded.json) == 0 && + cbm_lsp_surface_python_change(upgraded.json, unchanged.json) == 0; + free(first.json); + free(upgraded.json); + free(unchanged.json); + bool cleaned = namespace_admission_finish(&fixture); + ASSERT_TRUE(setup && stamped && cleaned); + ASSERT_TRUE(first.healthy && upgraded.healthy && unchanged.healthy && stable); + ASSERT_TRUE(upgraded.migrated); + ASSERT_FALSE(unchanged.migrated); + ASSERT_EQ(unchanged.route, CBM_INCREMENTAL_ROUTE_NOOP); + PASS(); +} + +TEST(pipeline_python_namespace_admission_body_alias_and_all) { + static const char initial[] = "from math import floor as Alias\n" + "__all__ = ['Alias']\n" + "def body():\n return 1\n"; + static const char body_edit[] = "from math import floor as Alias\n" + "__all__ = ['Alias']\n" + "def body():\n return 2\n"; + static const char alias_edit[] = "from math import ceil as Alias\n" + "__all__ = ['Alias']\n" + "def body():\n return 2\n"; + static const char all_edit[] = "from math import ceil as Alias\n" + "__all__ = []\n" + "def body():\n return 2\n"; + namespace_admission_fixture fixture = {0}; + bool setup = namespace_admission_begin(&fixture); + namespace_admission_result phases[4] = {0}; + const char *sources[] = {initial, body_edit, alias_edit, all_edit}; + bool wrote_all = setup; + for (int i = 0; i < 4 && wrote_all; i++) { + wrote_all = python_module_write_fixture_file(fixture.root, "subject.py", sources[i]); + if (wrote_all) + phases[i] = namespace_admission_run(&fixture, "subject.py", sources[i], 2); + if (!phases[i].healthy) + break; + } + bool all_healthy = + phases[0].healthy && phases[1].healthy && phases[2].healthy && phases[3].healthy; + bool comparisons = all_healthy && + cbm_lsp_surface_python_change(phases[0].json, phases[1].json) == 0 && + cbm_lsp_surface_python_change(phases[1].json, phases[2].json) == 1 && + cbm_lsp_surface_python_change(phases[2].json, phases[3].json) == 1; + for (int i = 0; i < 4; i++) + free(phases[i].json); + bool cleaned = namespace_admission_finish(&fixture); + ASSERT_TRUE(setup && wrote_all && cleaned && all_healthy && comparisons); + ASSERT_EQ(phases[1].route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); + ASSERT_EQ(phases[2].route, CBM_INCREMENTAL_ROUTE_FORCED_FULL); + ASSERT_EQ(phases[3].route, CBM_INCREMENTAL_ROUTE_FORCED_FULL); + PASS(); +} + +TEST(pipeline_python_namespace_admission_deleted_and_empty) { + const char *sources[] = {"def removed():\n return 3\n", "# empty namespace\n"}; + bool setups[2] = {false, false}, removed[2] = {false, false}; + bool healthy[2] = {false, false}, absent[2] = {false, false}, cleaned[2] = {false, false}; + cbm_incremental_route_t routes[2] = {CBM_INCREMENTAL_ROUTE_NONE, CBM_INCREMENTAL_ROUTE_NONE}; + for (int i = 0; i < 2; i++) { + namespace_admission_fixture fixture = {0}; + setups[i] = namespace_admission_begin(&fixture); + if (setups[i]) + setups[i] = python_module_write_fixture_file(fixture.root, "gone.py", sources[i]); + namespace_admission_result before = {0}, after = {0}; + if (setups[i]) + before = namespace_admission_run(&fixture, "gone.py", sources[i], 2); + if (before.healthy) { + char path[768]; + int n = snprintf(path, sizeof(path), "%s/gone.py", fixture.root); + removed[i] = n > 0 && (size_t)n < sizeof(path) && cbm_unlink(path) == 0; + } + if (removed[i]) + after = namespace_admission_run(&fixture, "control.py", namespace_admission_control, 1); + healthy[i] = before.healthy && after.healthy; + routes[i] = after.route; + cbm_store_t *store = after.healthy ? cbm_store_open_path(fixture.db) : NULL; + if (store) { + cbm_node_t *nodes = NULL; + int count = 0; + absent[i] = cbm_store_find_nodes_by_file(store, after.project, "gone.py", &nodes, + &count) == CBM_STORE_OK && + count == 0; + cbm_store_free_nodes(nodes, count); + cbm_store_close(store); + } + free(before.json); + free(after.json); + cleaned[i] = namespace_admission_finish(&fixture); + } + for (int i = 0; i < 2; i++) { + ASSERT_TRUE(setups[i] && removed[i] && healthy[i] && absent[i] && cleaned[i]); + ASSERT_EQ(routes[i], CBM_INCREMENTAL_ROUTE_FORCED_FULL); + } + PASS(); +} + +TEST(pipeline_python_namespace_admission_invalid_equal_hash) { + static const char initial[] = "def body():\n return 1\n"; + static const char edited[] = "def body():\n return 2\n"; + namespace_admission_fixture fixture = {0}; + bool setup = namespace_admission_begin(&fixture); + if (setup) + setup = python_module_write_fixture_file(fixture.root, "subject.py", initial); + namespace_admission_result before = {0}, after = {0}; + bool corrupted = false, wrote = false; + char *old_sha = NULL, *new_sha = NULL; + if (setup) + before = namespace_admission_run(&fixture, "subject.py", initial, 2); + cbm_store_t *store = before.healthy ? cbm_store_open_path(fixture.db) : NULL; + if (store) { + cbm_lsp_surface_row_t *rows = NULL; + int count = 0; + int rc = cbm_store_get_lsp_surfaces(store, before.project, &rows, &count); + for (int i = 0; rc == CBM_STORE_OK && i < count; i++) { + if (!rows[i].rel_path || strcmp(rows[i].rel_path, "subject.py") != 0) + continue; + old_sha = rows[i].surface_sha ? strdup(rows[i].surface_sha) : NULL; + cbm_lsp_surface_row_t invalid = rows[i]; + invalid.defs_json = "{}"; /* Leave the old hash intact: validate before equality. */ + corrupted = old_sha && cbm_lsp_surface_python_change("{}", "{}") == -1 && + cbm_store_upsert_lsp_surface_batch(store, &invalid, 1) == CBM_STORE_OK; + break; + } + cbm_store_free_lsp_surfaces(rows, count); + rows = NULL; + count = 0; + bool confirmed = false; + rc = corrupted ? cbm_store_get_lsp_surfaces(store, before.project, &rows, &count) + : CBM_STORE_ERR; + for (int i = 0; rc == CBM_STORE_OK && i < count; i++) { + if (!rows[i].rel_path || strcmp(rows[i].rel_path, "subject.py") != 0) + continue; + confirmed = rows[i].defs_json && strcmp(rows[i].defs_json, "{}") == 0 && + rows[i].surface_sha && strcmp(rows[i].surface_sha, old_sha) == 0; + break; + } + corrupted = corrupted && confirmed; + cbm_store_free_lsp_surfaces(rows, count); + cbm_store_close(store); + } + if (corrupted) + wrote = python_module_write_fixture_file(fixture.root, "subject.py", edited); + if (wrote) + after = namespace_admission_run(&fixture, "subject.py", edited, 2); + store = after.healthy ? cbm_store_open_path(fixture.db) : NULL; + if (store) { + cbm_lsp_surface_row_t *rows = NULL; + int count = 0; + int rc = cbm_store_get_lsp_surfaces(store, after.project, &rows, &count); + for (int i = 0; rc == CBM_STORE_OK && i < count; i++) { + if (rows[i].rel_path && strcmp(rows[i].rel_path, "subject.py") == 0 && + rows[i].surface_sha) { + new_sha = strdup(rows[i].surface_sha); + break; + } + } + cbm_store_free_lsp_surfaces(rows, count); + cbm_store_close(store); + } + bool equal_hash = old_sha && new_sha && strcmp(old_sha, new_sha) == 0; + bool equal_valid_surface = before.healthy && after.healthy && + cbm_lsp_surface_python_change(before.json, after.json) == 0; + free(old_sha); + free(new_sha); + free(before.json); + free(after.json); + bool cleaned = namespace_admission_finish(&fixture); + ASSERT_TRUE(setup && corrupted && wrote && cleaned); + ASSERT_TRUE(before.healthy && after.healthy && equal_hash && equal_valid_surface); + ASSERT_EQ(after.route, CBM_INCREMENTAL_ROUTE_FORCED_FULL); + PASS(); +} + +TEST(pipeline_python_module_callers_sequential) { + return python_module_callers_pipeline_case(false); +} + +TEST(pipeline_python_module_callers_parallel) { + return python_module_callers_pipeline_case(true); +} + +/* Distinct targets keep each newly walked class/decorator site observable + * even though the stored graph aggregates calls by caller/target pair. */ +static int python_class_body_pipeline_case(bool parallel) { + static const char helpers[] = + "def top_field():\n return 1\n" + "def nested_field():\n return 2\n" + "def class_decorator(value):\n return lambda target: target\n" + "def class_argument():\n return 3\n" + "def method_decorator(value):\n return lambda target: target\n" + "def method_argument():\n return 4\n" + "def method_control():\n return 5\n" + "def local_field():\n return 6\n" + "def method_local_field():\n return 7\n" + "def base_factory():\n return object\n" + "def package_field():\n return 8\n" + "def package_decorator(value):\n return lambda target: target\n" + "def package_argument():\n return 9\n"; + static const char app[] = + "from helpers import top_field, nested_field, class_decorator, class_argument\n" + "from helpers import method_decorator, method_argument, method_control\n" + "from helpers import local_field, method_local_field, base_factory\n" + "class Base(base_factory()):\n" + " value = top_field()\n" + " @method_decorator(method_argument())\n" + " def run(self):\n" + " method_control()\n" + " class MethodLocal:\n" + " value = method_local_field()\n" + "@class_decorator(class_argument())\n" + "class Admin:\n" + " class Meta:\n" + " value = nested_field()\n" + "def outer():\n" + " class Local:\n" + " value = local_field()\n" + " return Local\n"; + static const char package[] = + "from helpers import package_field, package_decorator, package_argument\n" + "@package_decorator(package_argument())\n" + "class Package:\n" + " value = package_field()\n"; + static const struct { + const char *caller; + const char *caller_label; + const char *target; + } expected[] = { + {"app", "Module", "helpers.base_factory"}, + {"app", "Module", "helpers.top_field"}, + {"app", "Module", "helpers.nested_field"}, + {"app", "Module", "helpers.class_decorator"}, + {"app", "Module", "helpers.class_argument"}, + {"app", "Module", "helpers.method_decorator"}, + {"app", "Module", "helpers.method_argument"}, + {"app.Base.run", "Method", "helpers.method_control"}, + {"app.Base.run", "Method", "helpers.method_local_field"}, + {"app.outer", "Function", "helpers.local_field"}, + {"pkg.__init__", "Module", "helpers.package_decorator"}, + {"pkg.__init__", "Module", "helpers.package_argument"}, + {"pkg.__init__", "Module", "helpers.package_field"}, + }; + char root[512], path[768]; + int n = snprintf(root, sizeof(root), "%s/cbm-python-class-body-XXXXXX", cbm_tmpdir()); + if (n <= 0 || (size_t)n >= sizeof(root) || !cbm_mkdtemp(root)) + FAIL("failed to create Python class-body fixture"); + n = snprintf(path, sizeof(path), "%s/pkg", root); + bool written = n > 0 && (size_t)n < sizeof(path); + if (written) { + cbm_mkdir_p(path, 0755); + written = python_module_write_fixture_file(root, "helpers.py", helpers) && + python_module_write_fixture_file(root, "app.py", app) && + python_module_write_fixture_file(root, "pkg/__init__.py", package); + } + for (int i = 0; written && parallel && i < 52; i++) { + char name[64]; + snprintf(name, sizeof(name), "class_body_pad_%02d.py", i); + written = python_module_write_fixture_file(root, name, "# inert parallel routing file\n"); + } + if (!written) { + th_rmtree(root); + FAIL("failed to write Python class-body fixture"); + } + const char *keys[] = {"CBM_WORKERS", "CBM_INDEX_SINGLE_THREAD"}; + char *saved[2] = {0}; + bool saved_ok = true; + for (int i = 0; i < 2; i++) { + const char *old = getenv(keys[i]); + saved[i] = old ? strdup(old) : NULL; + saved_ok = saved_ok && (!old || saved[i]); + } + if (!saved_ok) { + free(saved[0]); + free(saved[1]); + th_rmtree(root); + FAIL("failed to save Python class-body fixture environment"); + } + int set_rc = cbm_setenv(keys[0], parallel ? "4" : "1", 1); + set_rc |= parallel ? cbm_unsetenv(keys[1]) : cbm_setenv(keys[1], "1", 1); + n = snprintf(path, sizeof(path), "%s/class-body.db", root); + cbm_pipeline_t *pipeline = set_rc == 0 && n > 0 && (size_t)n < sizeof(path) + ? cbm_pipeline_new(root, path, CBM_MODE_FULL) + : NULL; + int run_rc = pipeline ? cbm_pipeline_run(pipeline) : -1; + cbm_file_error_t *errors = NULL; + int error_count = 0; + cbm_pipeline_get_file_errors(pipeline, &errors, &error_count); + cbm_store_t *store = run_rc == 0 ? cbm_store_open_path(path) : NULL; + bool correct = store != NULL; + if (store) { + const char *project = cbm_pipeline_project_name(pipeline); + cbm_node_t *files = NULL; + int file_count = 0; + correct = project && + cbm_store_find_nodes_by_label(store, project, "File", &files, &file_count) == + CBM_STORE_OK && + file_count == (parallel ? 55 : 3); + cbm_store_free_nodes(files, file_count); + for (size_t i = 0; i < sizeof(expected) / sizeof(expected[0]); i++) { + bool edge = project && python_module_exact_lsp_edge(store, project, expected[i].caller, + expected[i].caller_label, + expected[i].target, "Function"); + correct = correct && edge; + } + cbm_store_close(store); + } + cbm_pipeline_free(pipeline); + int restore_rc = 0; + for (int i = 0; i < 2; i++) { + restore_rc |= saved[i] ? cbm_setenv(keys[i], saved[i], 1) : cbm_unsetenv(keys[i]); + free(saved[i]); + } + int cleanup_rc = th_rmtree(root); + ASSERT_EQ(set_rc, 0); + ASSERT_EQ(restore_rc, 0); + ASSERT_EQ(cleanup_rc, 0); + ASSERT_EQ(run_rc, 0); + ASSERT_EQ(error_count, 0); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pipeline_python_class_body_calls_sequential) { + return python_class_body_pipeline_case(false); +} +TEST(pipeline_python_class_body_calls_parallel) { + return python_class_body_pipeline_case(true); +} + +/* Python file identity keeps .__init__; its exported symbols do not. */ +static bool python_init_local_inheritance(cbm_store_t *store, const char *project, + const char *module) { + char child_qn[768], parent_qn[768], wrong_qn[768]; + int cn = snprintf(child_qn, sizeof(child_qn), "%s.%s.BooleanField", project, module); + int pn = snprintf(parent_qn, sizeof(parent_qn), "%s.%s.Field", project, module); + int wn = snprintf(wrong_qn, sizeof(wrong_qn), "%s.other.Field", project); + if (cn <= 0 || (size_t)cn >= sizeof(child_qn) || pn <= 0 || (size_t)pn >= sizeof(parent_qn) || + wn <= 0 || (size_t)wn >= sizeof(wrong_qn)) + return false; + cbm_node_t child = {0}, parent = {0}, wrong = {0}; + bool ok = cbm_store_find_node_by_qn(store, project, child_qn, &child) == CBM_STORE_OK && + cbm_store_find_node_by_qn(store, project, parent_qn, &parent) == CBM_STORE_OK && + cbm_store_find_node_by_qn(store, project, wrong_qn, &wrong) == CBM_STORE_OK && + child.id > 0 && parent.id > 0 && wrong.id > 0 && parent.id != wrong.id && + child.label && parent.label && wrong.label && strcmp(child.label, "Class") == 0 && + strcmp(parent.label, "Class") == 0 && strcmp(wrong.label, "Class") == 0; + cbm_edge_t *edges = NULL; + int count = 0, local = 0, foreign = 0; + if (ok) { + ok = cbm_store_find_edges_by_source_type(store, child.id, "INHERITS", &edges, &count) == + CBM_STORE_OK; + for (int i = 0; ok && i < count; i++) { + local += edges[i].target_id == parent.id; + foreign += edges[i].target_id == wrong.id; + } + } + cbm_store_free_edges(edges, count); + cbm_node_free_fields(&child); + cbm_node_free_fields(&parent); + cbm_node_free_fields(&wrong); + return ok && local == 1 && foreign == 0; +} + +static int python_init_scope_pipeline_case(bool parallel, const char *failure_stage) { + static const char body[] = "from util import normalize\n" + "from other import Field as OtherField\n" + "def helper(value):\n return normalize(value)\n" + "class Field:\n" + " def clean(self, value):\n return helper(value)\n" + " def validate(self, value):\n return self.clean(value)\n" + "class BooleanField(Field):\n" + " def to_python(self, value):\n" + " self.validate(value)\n return helper(value)\n" + "def build(value):\n" + " field = BooleanField()\n" + " field.to_python(value)\n return helper(value)\n"; + static const char consumer[] = "from pkg import BooleanField, helper\n" + "def consume(value):\n" + " field = BooleanField()\n" + " field.to_python(value)\n return helper(value)\n"; + static const struct { + const char *caller; + const char *caller_label; + const char *target; + const char *target_label; + } local_edges[] = { + {"Field.clean", "Method", "helper", "Function"}, + {"Field.validate", "Method", "Field.clean", "Method"}, + {"BooleanField.to_python", "Method", "Field.validate", "Method"}, + {"BooleanField.to_python", "Method", "helper", "Function"}, + {"build", "Function", "BooleanField.to_python", "Method"}, + {"build", "Function", "helper", "Function"}, + }; + char root[512], path[768]; + int n = snprintf(root, sizeof(root), "%s/cbm-python-init-XXXXXX", cbm_tmpdir()); + if (n <= 0 || (size_t)n >= sizeof(root) || !cbm_mkdtemp(root)) + FAIL("failed to create Python package fixture"); + n = snprintf(path, sizeof(path), "%s/pkg", root); + bool written = n > 0 && (size_t)n < sizeof(path); + if (written) { + cbm_mkdir_p(path, 0755); + written = python_module_write_fixture_file( + root, "pkg/__init__.py", + failure_stage ? "class S:\n def m(self):\n return 1\n" + "def run():\n s = S()\n return s.m()\n" + : body); + } + if (written && !failure_stage) { + written = python_module_write_fixture_file(root, "plain.py", body) && + python_module_write_fixture_file(root, "consumer.py", consumer) && + python_module_write_fixture_file(root, "util.py", + "def normalize(value):\n return value\n") && + python_module_write_fixture_file(root, "other.py", "class Field:\n pass\n"); + } + for (int i = 0; written && parallel && i < 52; i++) { + char name[64]; + snprintf(name, sizeof(name), "init_pad_%02d.py", i); + written = python_module_write_fixture_file(root, name, "# inert parallel routing file\n"); + } + if (!written) { + th_rmtree(root); + FAIL("failed to write Python package fixture"); + } + const char *keys[] = {"CBM_WORKERS", "CBM_INDEX_SINGLE_THREAD", + "CBM_TEST_PY_LSP_SCOPE_FAIL_STAGE"}; + char *saved[3] = {0}; + bool saved_ok = true; + for (int i = 0; i < 3; i++) { + const char *old = getenv(keys[i]); + saved[i] = old ? strdup(old) : NULL; + saved_ok = saved_ok && (!old || saved[i]); + } + if (!saved_ok) { + for (int i = 0; i < 3; i++) + free(saved[i]); + th_rmtree(root); + FAIL("failed to save Python package fixture environment"); + } + int set_rc = cbm_setenv(keys[0], parallel ? "4" : "1", 1); + set_rc |= parallel ? cbm_unsetenv(keys[1]) : cbm_setenv(keys[1], "1", 1); + set_rc |= failure_stage ? cbm_setenv(keys[2], failure_stage, 1) : cbm_unsetenv(keys[2]); + n = snprintf(path, sizeof(path), "%s/package.db", root); + cbm_pipeline_t *pipeline = set_rc == 0 && n > 0 && (size_t)n < sizeof(path) + ? cbm_pipeline_new(root, path, CBM_MODE_FULL) + : NULL; + int run_rc = pipeline ? cbm_pipeline_run(pipeline) : -1; + cbm_file_error_t *errors = NULL; + int error_count = 0, scope_errors = 0; + bool precise = true; + cbm_pipeline_get_file_errors(pipeline, &errors, &error_count); + for (int i = 0; i < error_count; i++) { + bool is_package = errors[i].path && strcmp(errors[i].path, "pkg/__init__.py") == 0; + if (failure_stage && is_package) { + scope_errors++; + const char *phase = strcmp(failure_stage, "raw") == 0 ? "extract" : "lsp_skipped"; + precise = precise && errors[i].phase && strcmp(errors[i].phase, phase) == 0 && + errors[i].reason && strcmp(errors[i].reason, CBM_PY_LSP_SCOPE_ERROR) == 0; + } else { + precise = false; + } + } + cbm_store_t *store = run_rc == 0 ? cbm_store_open_path(path) : NULL; + bool correct = store != NULL; + if (store) { + const char *project = cbm_pipeline_project_name(pipeline); + cbm_node_t *files = NULL; + int file_count = 0; + correct = project && + cbm_store_find_nodes_by_label(store, project, "File", &files, &file_count) == + CBM_STORE_OK && + file_count == (failure_stage ? 1 : 5) + (parallel ? 52 : 0); + cbm_store_free_nodes(files, file_count); + if (project && !failure_stage) { + const char *modules[] = {"plain", "pkg"}; + for (int m = 0; m < 2; m++) { + for (size_t i = 0; i < sizeof(local_edges) / sizeof(local_edges[0]); i++) { + char caller[128], target[128]; + snprintf(caller, sizeof(caller), "%s.%s", modules[m], local_edges[i].caller); + snprintf(target, sizeof(target), "%s.%s", modules[m], local_edges[i].target); + bool edge = python_module_exact_lsp_edge(store, project, caller, + local_edges[i].caller_label, target, + local_edges[i].target_label); + correct = correct && edge; + } + char caller[128]; + snprintf(caller, sizeof(caller), "%s.helper", modules[m]); + bool imported = python_module_exact_lsp_edge(store, project, caller, "Function", + "util.normalize", "Function"); + bool inherits = python_init_local_inheritance(store, project, modules[m]); + correct = correct && imported && inherits; + } + bool method = + python_module_exact_lsp_edge(store, project, "consumer.consume", "Function", + "pkg.BooleanField.to_python", "Method"); + bool function = python_module_exact_lsp_edge(store, project, "consumer.consume", + "Function", "pkg.helper", "Function"); + correct = correct && method && function; + } + cbm_store_close(store); + } + cbm_pipeline_free(pipeline); + int restore_rc = 0; + for (int i = 0; i < 3; i++) { + restore_rc |= saved[i] ? cbm_setenv(keys[i], saved[i], 1) : cbm_unsetenv(keys[i]); + free(saved[i]); + } + int cleanup_rc = th_rmtree(root); + ASSERT_EQ(set_rc, 0); + ASSERT_EQ(restore_rc, 0); + ASSERT_EQ(cleanup_rc, 0); + ASSERT_EQ(run_rc, 0); + ASSERT_EQ(error_count, failure_stage ? 1 : 0); + ASSERT_EQ(scope_errors, failure_stage ? 1 : 0); + ASSERT_TRUE(precise && correct); + PASS(); +} + +TEST(pipeline_python_init_scope_sequential) { + return python_init_scope_pipeline_case(false, NULL); +} +TEST(pipeline_python_init_scope_parallel) { + return python_init_scope_pipeline_case(true, NULL); +} +TEST(pipeline_python_init_raw_scope_failure_sequential) { + return python_init_scope_pipeline_case(false, "raw"); +} +TEST(pipeline_python_init_raw_scope_failure_parallel) { + return python_init_scope_pipeline_case(true, "raw"); +} +TEST(pipeline_python_init_cross_scope_failure_sequential) { + return python_init_scope_pipeline_case(false, "cross"); +} +TEST(pipeline_python_init_cross_scope_failure_parallel) { + return python_init_scope_pipeline_case(true, "cross"); +} +TEST(pipeline_python_init_register_scope_failure_sequential) { + return python_init_scope_pipeline_case(false, "register"); +} +TEST(pipeline_python_init_register_scope_failure_parallel) { + return python_init_scope_pipeline_case(true, "register"); +} + +/* Compare identities, not just counts; a duplicate import must stop at the + * same nearest eligible entry even when it contributes no new selections. */ +static bool python_init_filter_is(CBMLSPDef *defs, int def_count, const char *const *imports, + int import_count, const char *const *expected, int count) { + CBMModuleDefIndex *index = cbm_pxc_build_module_def_index(defs, def_count); + int got_count = -1; + bool succeeded = false; + CBMLSPDef *got = + cbm_pxc_filter_defs_for_file(index, defs, CBM_LANG_PYTHON, NULL, "test.consumer", imports, + import_count, &got_count, &succeeded); + bool correct = index && succeeded && got_count == count && (count == 0 || got); + for (int i = 0; correct && i < count; i++) { + int hits = 0; + for (int j = 0; j < got_count; j++) + hits += got[j].qualified_name && strcmp(got[j].qualified_name, expected[i]) == 0; + correct = hits == 1; + } + free(got); + cbm_pxc_free_module_def_index(index); + return correct; +} + +TEST(pipeline_python_init_import_filter_precedence) { + CBMLSPDef defs[] = { + {.qualified_name = "test.consumer.use", + .def_module_qn = "test.consumer", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Store", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Foreign", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_GO}, + {.qualified_name = "test.Parent", + .def_module_qn = "test.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.foreign.Only", + .def_module_qn = "test.foreign.__init__", + .lang = CBM_LANG_GO}, + {.qualified_name = "test.unknown.Only", + .def_module_qn = "test.unknown.__init__", + .lang = (CBMLanguage)-1}, + {.qualified_name = "test.plain.Exact", + .def_module_qn = "test.plain", + .lang = CBM_LANG_JAVA}, + {.qualified_name = "test.plain.Package", + .def_module_qn = "test.plain.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.nested.Item", + .def_module_qn = "test.pkg.nested.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.sub.Item", + .def_module_qn = "test.pkg.sub", + .lang = CBM_LANG_PYTHON}, + }; + const char *pkg[] = {"test.pkg", "test.pkg.Store", "test.pkg.Store"}; + const char *want_pkg[] = {"test.consumer.use", "test.pkg.Store"}; + const char *foreign[] = {"test.foreign.Only", "test.unknown.Only"}; + const char *want_parent[] = {"test.consumer.use", "test.Parent"}; + const char *normal[] = {"test.plain.Exact", "test.plain.Exact"}; + const char *want_normal[] = {"test.consumer.use", "test.plain.Exact"}; + const char *nested[] = {"test.pkg.nested.Item", "test.pkg.nested.Item"}; + const char *want_nested[] = {"test.consumer.use", "test.pkg.nested.Item"}; + const char *submodule[] = {"test.pkg.sub.Item"}; + const char *want_submodule[] = {"test.consumer.use", "test.pkg.sub.Item"}; + int n = (int)(sizeof(defs) / sizeof(defs[0])); + bool pkg_ok = python_init_filter_is(defs, n, pkg, 3, want_pkg, 2); + bool foreign_ok = python_init_filter_is(defs, n, foreign, 2, want_parent, 2); + bool normal_ok = python_init_filter_is(defs, n, normal, 2, want_normal, 2); + bool nested_ok = python_init_filter_is(defs, n, nested, 2, want_nested, 2); + bool submodule_ok = python_init_filter_is(defs, n, submodule, 1, want_submodule, 2); + ASSERT_TRUE(pkg_ok && foreign_ok && normal_ok && nested_ok && submodule_ok); + PASS(); +} + +TEST(pipeline_python_init_import_probe_failure_falls_back) { + const char *source = "from pkg import Store\n" + "def use():\n store = Store()\n return store.get()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.consumer.use", + .short_name = "use", + .label = "Function", + .def_module_qn = "test.consumer", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Store", + .short_name = "Store", + .label = "Class", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Store.get", + .short_name = "get", + .label = "Method", + .def_module_qn = "test.pkg.__init__", + .receiver_type = "test.pkg.Store", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + }; + const char *keys[] = {"Store"}; + const char *values[] = {"test.pkg.Store"}; + CBMModuleDefIndex *index = cbm_pxc_build_module_def_index(defs, 3); + int count = -1; + bool succeeded = true; + cbm_pxc_test_import_probe_fail_after(0); + CBMLSPDef *filtered = cbm_pxc_filter_defs_for_file( + index, defs, CBM_LANG_PYTHON, NULL, "test.consumer", values, 1, &count, &succeeded); + cbm_pxc_test_import_probe_fail_after(-1); + bool failed_precisely = index && !filtered && count == 0 && !succeeded; + free(filtered); + CBMFileResult result = {0}; + cbm_arena_init(&result.arena); + cbm_pxc_test_import_probe_fail_after(0); + cbm_pxc_dispatch_file(CBM_LANG_PYTHON, &result, source, (int)strlen(source), "consumer.py", + "test.consumer", NULL, index, defs, 3, keys, values, 1, NULL, NULL); + cbm_pxc_test_import_probe_fail_after(-1); + const char *site = strstr(source, "store.get()"); + int exact = 0; + for (int i = 0; i < result.resolved_calls.count; i++) { + const CBMResolvedCall *call = &result.resolved_calls.items[i]; + if (call->caller_qn && strcmp(call->caller_qn, "test.consumer.use") == 0 && + call->callee_qn && strcmp(call->callee_qn, "test.pkg.Store.get") == 0 && + call->kind == CBM_RESOLVED_INVOCATION && call->strategy && + strncmp(call->strategy, "lsp_", 4) == 0 && site && + call->site_start_byte == (uint32_t)(site - source) && + call->site_end_byte == (uint32_t)(site - source + strlen("store.get()"))) + exact++; + } + bool resolved = !result.has_error && !result.lsp_skipped && exact == 1; + cbm_arena_destroy(&result.arena); + cbm_pxc_free_module_def_index(index); + ASSERT_TRUE(failed_precisely && resolved); + PASS(); +} + +/* #1277: typed instance fields of a Python class imported from another file. + * Counts CALLS edges caller -> callee (QN suffixes) whose properties name an + * LSP strategy. */ +static int count_lsp_calls(cbm_store_t *s, const char *project, const char *caller_suffix, + const char *callee_suffix) { + cbm_edge_t *edges = NULL; + int edge_count = 0; + cbm_store_find_edges_by_type(s, project, "CALLS", &edges, &edge_count); + int hits = 0; + for (int i = 0; i < edge_count; i++) { + cbm_node_t src = {0}; + cbm_node_t tgt = {0}; + if (cbm_store_find_node_by_id(s, edges[i].source_id, &src) == CBM_STORE_OK && + cbm_store_find_node_by_id(s, edges[i].target_id, &tgt) == CBM_STORE_OK) { + size_t sl = strlen(src.qualified_name); + size_t tl = strlen(tgt.qualified_name); + size_t cs = strlen(caller_suffix); + size_t ts = strlen(callee_suffix); + if (sl >= cs && tl >= ts && strcmp(src.qualified_name + sl - cs, caller_suffix) == 0 && + strcmp(tgt.qualified_name + tl - ts, callee_suffix) == 0 && + edges[i].properties_json && strstr(edges[i].properties_json, "lsp")) { + hits++; + } + } + cbm_node_free_fields(&src); + cbm_node_free_fields(&tgt); + } + if (edges) + cbm_store_free_edges(edges, edge_count); + return hits; +} + +static int check_python_crossfile_typed_field_calls_issue1277(bool parallel) { + const char *contracts = "class Contract:\n" + " def process_batch(self) -> None:\n" + " ...\n"; + const char *trainer = "from contracts import Contract\n\n" + "def make():\n" + " return None\n\n" + "class Trainer:\n" + " engine: Contract\n\n" + " def __init__(self, strategies: Contract) -> None:\n" + " self.strategies = strategies\n" + " self.typed: Contract = strategies\n" + " self.plain = make()\n"; + const char *loop = "from trainer import Trainer\n\n" + "def run(trainer: Trainer) -> None:\n" + " strategies = trainer.strategies\n" + " strategies.process_batch()\n\n" + "def run_typed(trainer: Trainer) -> None:\n" + " trainer.typed.process_batch()\n\n" + "def run_classlevel(trainer: Trainer) -> None:\n" + " trainer.engine.process_batch()\n\n" + "def run_untyped(trainer: Trainer) -> None:\n" + " trainer.plain.process_batch()\n"; + if (setup_usages_repo("contracts.py", contracts, "trainer.py", trainer) != 0) { + FAIL("failed to create temp dir"); + } + char path[512]; + snprintf(path, sizeof(path), "%s/loop.py", g_usages_tmpdir); + if (th_write_file(path, loop) != 0) { + teardown_usages_repo(); + FAIL("failed to write loop.py"); + } + if (parallel) { + /* Three semantic files plus 50 inert files exceed the production + * threshold for the fused parallel pipeline. */ + for (int i = 0; i < 50; i++) { + snprintf(path, sizeof(path), "%s/field_pad_%02d.py", g_usages_tmpdir, i); + if (th_write_file(path, "field_padding = 0\n") != 0) { + teardown_usages_repo(); + FAIL("failed to write parallel-selection fixture"); + } + } + } + + const char *old_workers = getenv("CBM_WORKERS"); + char *saved_workers = old_workers ? strdup(old_workers) : NULL; + const char *old_single = getenv("CBM_INDEX_SINGLE_THREAD"); + char *saved_single = old_single ? strdup(old_single) : NULL; + if ((old_workers && !saved_workers) || (old_single && !saved_single)) { + free(saved_workers); + free(saved_single); + teardown_usages_repo(); + FAIL("failed to save pipeline environment"); + } + int pin_workers_rc = parallel ? cbm_setenv("CBM_WORKERS", "4", 1) : 0; + int pin_single_rc = parallel ? cbm_unsetenv("CBM_INDEX_SINGLE_THREAD") : 0; + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/test_1277.db", g_usages_tmpdir); + cbm_pipeline_t *p = pin_workers_rc == 0 && pin_single_rc == 0 + ? cbm_pipeline_new(g_usages_tmpdir, db_path, CBM_MODE_FULL) + : NULL; + int run_rc = p ? cbm_pipeline_run(p) : -1; + cbm_store_t *s = run_rc == 0 ? cbm_store_open_path(db_path) : NULL; + bool store_opened = s != NULL; + int via_alias = -1, via_typed = -1, via_class = -1, via_untyped = -1; + if (s) { + const char *project = cbm_pipeline_project_name(p); + via_alias = count_lsp_calls(s, project, "loop.run", "contracts.Contract.process_batch"); + via_typed = + count_lsp_calls(s, project, "loop.run_typed", "contracts.Contract.process_batch"); + via_class = + count_lsp_calls(s, project, "loop.run_classlevel", "contracts.Contract.process_batch"); + via_untyped = count_lsp_calls(s, project, "loop.run_untyped", "process_batch"); + cbm_store_close(s); + } + + cbm_pipeline_free(p); + int restore_workers_rc = + saved_workers ? cbm_setenv("CBM_WORKERS", saved_workers, 1) : cbm_unsetenv("CBM_WORKERS"); + int restore_single_rc = saved_single ? cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1) + : cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + free(saved_workers); + free(saved_single); + teardown_usages_repo(); + + ASSERT_EQ(pin_workers_rc, 0); + ASSERT_EQ(pin_single_rc, 0); + ASSERT_EQ(restore_workers_rc, 0); + ASSERT_EQ(restore_single_rc, 0); + ASSERT_EQ(run_rc, 0); + ASSERT_TRUE(store_opened); + ASSERT_EQ(via_alias, 1); /* self.x = */ + ASSERT_EQ(via_typed, 1); /* self.x: T = ... */ + ASSERT_EQ(via_class, 1); /* class-level x: T */ + ASSERT_EQ(via_untyped, 0); /* control: no annotation, no guess */ + PASS(); +} + +TEST(python_crossfile_typed_field_calls_issue1277) { + return check_python_crossfile_typed_field_calls_issue1277(false); +} + +TEST(python_crossfile_typed_field_calls_issue1277_parallel) { + return check_python_crossfile_typed_field_calls_issue1277(true); +} + TEST(usages_creates_edges) { /* Port of TestPassUsagesCreatesEdges. * Go source with callback reference → USAGE edge. */ @@ -15247,6 +16574,247 @@ TEST(pipeline_seq_ts_cross_uses_shared_registry) { PASS(); } +/* Build a real extraction surface without publishing a database. The returned + * JSON outlives both the extraction and collector arenas. */ +static char *pipeline_namespace_surface(const char *source, bool include_defs) { + CBMFileResult *fr = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, + "namespace_probe", "pkg/__init__.py", 0, NULL, NULL); + if (!fr || fr->has_error || fr->parse_incomplete) { + cbm_free_result(fr); + return NULL; + } + CBMArena arena; + cbm_arena_init(&arena); + CBMFileResult *cache[] = {fr}; + cbm_file_info_t files[] = {{.rel_path = "pkg/__init__.py", .language = CBM_LANG_PYTHON}}; + char *modules[] = {NULL}; + int starts[2] = {0, 0}; + int def_count = 0; + CBMLSPDef *defs = include_defs + ? cbm_pxc_collect_all_defs(NULL, &arena, cache, files, 1, + "namespace_probe", modules, &def_count, starts) + : NULL; + cbm_lsp_surface_row_t *rows = NULL; + int row_count = 0; + char *json = NULL; + if (cbm_lsp_surface_build_rows(NULL, "namespace_probe", cache, files, 1, defs, starts, &rows, + &row_count) == 0 && + row_count == 1 && rows[0].defs_json) { + json = strdup(rows[0].defs_json); + } + cbm_store_free_lsp_surfaces(rows, row_count); + free(defs); + free(modules[0]); + cbm_arena_destroy(&arena); + cbm_free_result(fr); + return json; +} + +TEST(pipeline_python_namespace_change_admission) { + const char *sources[] = { + "def target(value):\n return 1\n", + "def target(value):\n return 2\n", + "def target(value, extra):\n return 1\n", + "from first import Target as Alias\n", + "from second import Target as Alias\n", + "from first import Target as Alias\nfrom second import Target as Alias\n", + "from second import Target as Alias\nfrom first import Target as Alias\n", + "__all__ = []\n", + "__all__ = ['Alias']\n", + "__all__ = dynamic_names()\n", + "# empty namespace\n", + }; + enum { count = (int)(sizeof(sources) / sizeof(sources[0])) }; + char *json[count]; + bool complete = true; + for (int i = 0; i < count; i++) { + json[i] = pipeline_namespace_surface(sources[i], true); + complete = complete && json[i] != NULL; + } + int body = -9, signature = -9, alias = -9, order = -9; + int empty = -9, literal = -9, unknown = -9, deletion = -9, transition = -9; + const char *non_python = "{\"v\":2,\"lang\":0,\"lsp\":[],\"reg\":[],\"py\":null}"; + if (complete) { + body = cbm_lsp_surface_python_change(json[0], json[1]); + signature = cbm_lsp_surface_python_change(json[0], json[2]); + alias = cbm_lsp_surface_python_change(json[3], json[4]); + order = cbm_lsp_surface_python_change(json[5], json[6]); + empty = cbm_lsp_surface_python_change(json[10], json[7]); + literal = cbm_lsp_surface_python_change(json[7], json[8]); + unknown = cbm_lsp_surface_python_change(json[7], json[9]); + deletion = cbm_lsp_surface_python_change(json[10], NULL); + transition = cbm_lsp_surface_python_change(non_python, json[10]); + } + for (int i = 0; i < count; i++) { + free(json[i]); + } + ASSERT_TRUE(complete); + ASSERT_EQ(body, 0); + ASSERT_EQ(signature, 1); + ASSERT_EQ(alias, 1); + ASSERT_EQ(order, 1); + ASSERT_EQ(empty, 1); + ASSERT_EQ(literal, 1); + ASSERT_EQ(unknown, 1); + ASSERT_EQ(deletion, 1); + ASSERT_EQ(transition, 1); + ASSERT_EQ(cbm_lsp_surface_python_change(non_python, NULL), 0); + ASSERT_EQ(cbm_lsp_surface_python_change(non_python, non_python), 0); + PASS(); +} + +static char *pipeline_namespace_payload_json(const char *json) { + yyjson_doc *doc = json ? yyjson_read(json, strlen(json), 0) : NULL; + if (!doc) { + return NULL; + } + yyjson_val *payload = yyjson_obj_get(yyjson_doc_get_root(doc), "py"); + char *copy = payload ? yyjson_val_write(payload, 0, NULL) : NULL; + yyjson_doc_free(doc); + return copy; +} + +TEST(pipeline_python_namespace_codec_owns_ordered_facts) { + const char *source = "def plain(value):\n return value\n" + "from .one import First as Alias\n" + "from .two import Second as Alias\n" + "__all__ = ['Alias', '_private', '', 'odd|name']\n" + "__all__ += ('tail',)\n" + "__all__ = []\n" + "__all__ = dynamic_names()\n"; + char *full = pipeline_namespace_surface(source, true); + char *probe = pipeline_namespace_surface(source, false); + char *full_facts = pipeline_namespace_payload_json(full); + char *probe_facts = pipeline_namespace_payload_json(probe); + bool parity = full_facts && probe_facts && strcmp(full_facts, probe_facts) == 0; + bool distinct_defs = full && probe && strcmp(full, probe) != 0; + CBMArena arena; + cbm_arena_init(&arena); + cbm_lsp_python_namespace_t copied = {0}, failed = {0}; + cbm_py_namespace_test_copy_fail_after(0); + int failed_rc = cbm_lsp_surface_python_namespace_from_json(&arena, full, &failed); + cbm_py_namespace_test_copy_fail_after(-1); + int rc = cbm_lsp_surface_python_namespace_from_json(&arena, full, &copied); + free(full); + free(probe); + free(full_facts); + free(probe_facts); + + /* The original result, collector, JSON documents and serialized bytes are + * all gone before the copied strings and nested arrays are inspected. */ + bool origin = rc == 1 && copied.module_qn && copied.rel_path && + strcmp(copied.module_qn, "namespace_probe.pkg.__init__") == 0 && + strcmp(copied.rel_path, "pkg/__init__.py") == 0; + bool valid = rc == 1 && cbm_py_namespace_facts_valid(&copied.facts) && + copied.facts.status == CBM_PY_NS_COMPLETE && + copied.facts.language == CBM_LANG_PYTHON; + int imports = 0; + bool import_order = true, names = false, append = false, empty = false, unknown = false; + if (valid) { + for (int i = 0; i < copied.facts.count; i++) { + const CBMPyNamespaceFact *f = &copied.facts.items[i]; + if (f->kind == CBM_PY_NS_IMPORT_NAME) { + const char *module = imports == 0 ? "one" : "two"; + const char *member = imports == 0 ? "First" : "Second"; + import_order = import_order && imports < 2 && f->relative_level == 1 && + f->module_name && strcmp(f->module_name, module) == 0 && + f->member_name && strcmp(f->member_name, member) == 0 && + f->local_name && strcmp(f->local_name, "Alias") == 0; + imports++; + } + if (f->kind == CBM_PY_NS_ALL_SET && f->name_count == 4) { + names = f->sequence_kind == CBM_PY_NS_SEQUENCE_LIST && + strcmp(f->names[0], "Alias") == 0 && strcmp(f->names[1], "_private") == 0 && + strcmp(f->names[2], "") == 0 && strcmp(f->names[3], "odd|name") == 0; + } + if (f->kind == CBM_PY_NS_ALL_APPEND) { + append = f->sequence_kind == CBM_PY_NS_SEQUENCE_TUPLE && f->name_count == 1 && + strcmp(f->names[0], "tail") == 0; + } + empty = + empty || (f->kind == CBM_PY_NS_ALL_SET && f->name_count == 0 && f->names == NULL); + unknown = unknown || f->kind == CBM_PY_NS_ALL_UNKNOWN; + } + } + bool failure_empty = failed.facts.items == NULL && failed.facts.count == 0 && + failed.module_qn == NULL && failed.rel_path == NULL; + cbm_arena_destroy(&arena); + ASSERT_TRUE(parity); + ASSERT_TRUE(distinct_defs); + ASSERT_EQ(failed_rc, -1); + ASSERT_TRUE(failure_empty); + ASSERT_EQ(rc, 1); + ASSERT_TRUE(origin); + ASSERT_TRUE(valid); + ASSERT_EQ(imports, 2); + ASSERT_TRUE(import_order); + ASSERT_TRUE(names); + ASSERT_TRUE(append); + ASSERT_TRUE(empty); + ASSERT_TRUE(unknown); + PASS(); +} + +TEST(pipeline_python_namespace_codec_rejects_non_authoritative_rows) { + /* Numeric values below are the frozen version-1 namespace wire values. + * Even an empty LSP array must validate its namespace envelope. */ + const char *bad[] = { + "{\"v\":1,\"lsp\":[]}", + "{\"v\":2,\"lang\":1,\"lsp\":[]}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":null}", + "{\"v\":2,\"lang\":1.0,\"lsp\":[],\"py\":null}", + "{\"v\":2,\"v\":2,\"lang\":0,\"lsp\":[],\"py\":null}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":1,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\",\"status\":3,\"failure\":2,\"events\":[]}}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":99,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\",\"status\":2,\"failure\":0,\"events\":[]}}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":1,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\\u0000suffix\",\"status\":2," + "\"failure\":0,\"events\":[]}}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":1,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\",\"status\":2,\"failure\":0,\"events\":[" + "{\"k\":999,\"d\":0,\"s\":0,\"r\":0,\"f\":0,\"l\":0," + "\"n\":null,\"m\":null,\"i\":null,\"a\":[]}]}}", + "{\"v\":2,\"lang\":1,\"lsp\":[],\"py\":{\"v\":1,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\",\"status\":2,\"failure\":0,\"events\":[" + "{\"k\":8,\"d\":0,\"s\":1,\"r\":0,\"f\":0,\"l\":4294967296," + "\"n\":null,\"m\":null,\"i\":null,\"a\":[]}]}}", + }; + bool rejected = true; + for (size_t i = 0; i < sizeof(bad) / sizeof(bad[0]); i++) { + CBMArena arena; + cbm_arena_init(&arena); + cbm_lsp_python_namespace_t facts = {0}; + CBMLSPDef *defs = NULL; + int ns_rc = cbm_lsp_surface_python_namespace_from_json(&arena, bad[i], &facts); + int defs_rc = cbm_lsp_surface_defs_from_json(&arena, bad[i], &defs); + int compare_rc = cbm_lsp_surface_python_change(bad[i], bad[i]); + int delete_rc = cbm_lsp_surface_python_change(bad[i], NULL); + rejected = rejected && ns_rc == -1 && defs_rc == -1 && compare_rc == -1 && + delete_rc == -1 && facts.facts.items == NULL && defs == NULL; + cbm_arena_destroy(&arena); + } + const char *unknown = + "{\"v\":2,\"lang\":1,\"lsp\":[],\"reg\":[],\"py\":{\"v\":1,\"lang\":1," + "\"module\":\"p.m\",\"path\":\"m.py\",\"status\":2,\"failure\":0,\"events\":[" + "{\"k\":7,\"d\":0,\"s\":0,\"r\":7,\"f\":0,\"l\":0," + "\"n\":null,\"m\":null,\"i\":null,\"a\":[]}]}}"; + CBMArena arena; + cbm_arena_init(&arena); + cbm_lsp_python_namespace_t facts = {0}; + int unknown_rc = cbm_lsp_surface_python_namespace_from_json(&arena, unknown, &facts); + int equal_rc = cbm_lsp_surface_python_change(unknown, unknown); + bool retained = unknown_rc == 1 && facts.facts.count == 1 && + facts.facts.items[0].kind == CBM_PY_NS_UNKNOWN && + facts.facts.items[0].reason == CBM_PY_NS_REASON_PARSE; + cbm_arena_destroy(&arena); + ASSERT_TRUE(rejected); + ASSERT_EQ(unknown_rc, 1); + ASSERT_EQ(equal_rc, 0); + ASSERT_TRUE(retained); + PASS(); +} + /* Object arrays are non-flat in yyjson. Exercise ordered decoding and arena * ownership across a large surface, including nested string arrays. */ TEST(pipeline_lsp_surface_large_object_array_decode) { @@ -15254,7 +16822,7 @@ TEST(pipeline_lsp_surface_large_object_array_decode) { const size_t capacity = (size_t)count * 192 + 64; char *json = malloc(capacity); ASSERT_NOT_NULL(json); - size_t used = (size_t)snprintf(json, capacity, "{\"v\":1,\"lsp\":["); + size_t used = (size_t)snprintf(json, capacity, "{\"v\":2,\"lang\":0,\"py\":null,\"lsp\":["); for (int i = 0; i < count; i++) { int n = snprintf(json + used, capacity - used, "%s{\"qn\":\"pkg.f%d\",\"sn\":\"f%d\",\"lb\":\"Function\"," @@ -15346,7 +16914,7 @@ TEST(pipeline_lsp_surface_persisted_and_body_edit_invariant) { snprintf(sha_baseline, sizeof(sha_baseline), "%s", row_a->surface_sha); /* The row is the versioned codec envelope with the def present. */ snprintf(json_probe, sizeof(json_probe), "%.20s", row_a->defs_json); - ASSERT_TRUE(strstr(row_a->defs_json, "\"v\":1") != NULL); + ASSERT_TRUE(strstr(row_a->defs_json, "\"v\":2") != NULL); ASSERT_TRUE(strstr(row_a->defs_json, "surface_probe") != NULL); } else if (round == 1) { ASSERT_STR_EQ(row_a->surface_sha, sha_baseline); @@ -16271,10 +17839,48 @@ TEST(pipeline_objectscript_export_range_join_keeps_one_trailing_marker) { #endif SUITE(pipeline) { + RUN_TEST(pipeline_python_module_callers_sequential); + RUN_TEST(pipeline_python_module_callers_parallel); + RUN_TEST(pipeline_python_class_body_calls_sequential); + RUN_TEST(pipeline_python_class_body_calls_parallel); + RUN_TEST(pipeline_python_init_scope_sequential); + RUN_TEST(pipeline_python_init_scope_parallel); + RUN_TEST(pipeline_python_init_raw_scope_failure_sequential); + RUN_TEST(pipeline_python_init_raw_scope_failure_parallel); + RUN_TEST(pipeline_python_init_cross_scope_failure_sequential); + RUN_TEST(pipeline_python_init_cross_scope_failure_parallel); + RUN_TEST(pipeline_python_init_register_scope_failure_sequential); + RUN_TEST(pipeline_python_init_register_scope_failure_parallel); + RUN_TEST(pipeline_python_init_import_filter_precedence); + RUN_TEST(pipeline_python_init_import_probe_failure_falls_back); + RUN_TEST(pipeline_issue1527_python_raw_memo_failure_sequential); + RUN_TEST(pipeline_issue1527_python_raw_memo_failure_parallel); + RUN_TEST(pipeline_issue1527_python_cross_memo_failure_sequential); + RUN_TEST(pipeline_issue1527_python_cross_memo_failure_parallel); + RUN_TEST(pipeline_issue1527_c_raw_depth_failure_sequential); + RUN_TEST(pipeline_issue1527_c_cross_depth_failure_sequential); + RUN_TEST(pipeline_issue1527_c_raw_depth_failure_parallel); + RUN_TEST(pipeline_issue1527_c_cross_depth_failure_parallel); + RUN_TEST(pipeline_issue1527_python_raw_depth_failure_sequential); + RUN_TEST(pipeline_issue1527_python_cross_depth_failure_sequential); + RUN_TEST(pipeline_issue1527_python_raw_depth_failure_parallel); + RUN_TEST(pipeline_issue1527_python_cross_depth_failure_parallel); + RUN_TEST(pipeline_issue1527_cross_memo_failure_sequential); + RUN_TEST(pipeline_issue1527_cross_memo_failure_parallel); + RUN_TEST(pipeline_issue1527_raw_memo_failure_sequential); + RUN_TEST(pipeline_issue1527_raw_memo_failure_parallel); + RUN_TEST(pipeline_issue1527_cross_memo_failure_fallback); RUN_TEST(pipeline_nested_fixture_files_are_written); RUN_TEST(pipeline_fixture_file_parent_is_preserved); RUN_TEST(pipeline_fixture_node_count_requires_exact_source_nodes); RUN_TEST(pipeline_nested_fixture_graph_has_endpoints); + RUN_TEST(pipeline_python_namespace_admission_upgrade_then_noop); + RUN_TEST(pipeline_python_namespace_admission_body_alias_and_all); + RUN_TEST(pipeline_python_namespace_admission_deleted_and_empty); + RUN_TEST(pipeline_python_namespace_admission_invalid_equal_hash); + RUN_TEST(pipeline_python_namespace_change_admission); + RUN_TEST(pipeline_python_namespace_codec_owns_ordered_facts); + RUN_TEST(pipeline_python_namespace_codec_rejects_non_authoritative_rows); RUN_TEST(pipeline_lsp_surface_large_object_array_decode); RUN_TEST(pipeline_lsp_surface_persisted_and_body_edit_invariant); /* Index lock */ @@ -16378,6 +17984,8 @@ SUITE(pipeline) { RUN_TEST(implements_creates_override); RUN_TEST(implements_no_match); /* Usages pass (full pipeline integration) */ + RUN_TEST(python_crossfile_typed_field_calls_issue1277); + RUN_TEST(python_crossfile_typed_field_calls_issue1277_parallel); RUN_TEST(usages_creates_edges); RUN_TEST(usages_no_duplicate_calls); RUN_TEST(calls_edge_carries_call_site_line); diff --git a/tests/test_py_lsp.c b/tests/test_py_lsp.c index 3aaadee51..91d3db951 100644 --- a/tests/test_py_lsp.c +++ b/tests/test_py_lsp.c @@ -10,6 +10,7 @@ #include "test_framework.h" #include "cbm.h" #include "lsp/py_lsp.h" +#include "lsp/lsp_work.h" #include "pipeline/lsp_resolve.h" #include "pipeline/pass_lsp_cross.h" @@ -580,6 +581,119 @@ TEST(pylsp_crossfile_method_dispatch) { PASS(); } +/* Issue #1277 (reproducer contributed by @Enferlain): a method call through a + * typed instance field of a class imported from ANOTHER file. The field type + * reaches the resolver only through CBMLSPDef.field_defs on the class def. */ +TEST(pylsp_crossfile_receiver_through_typed_field) { + const char *source = "from trainer import Trainer\n" + "def run(trainer: Trainer):\n" + " strategies = trainer.strategies\n" + " return strategies.process_batch()\n"; + + CBMLSPDef defs[3]; + memset(defs, 0, sizeof(defs)); + + defs[0].qualified_name = "contracts.Contract"; + defs[0].short_name = "Contract"; + defs[0].label = "Class"; + defs[0].def_module_qn = "contracts"; + + defs[1].qualified_name = "contracts.Contract.process_batch"; + defs[1].short_name = "process_batch"; + defs[1].label = "Method"; + defs[1].receiver_type = "contracts.Contract"; + defs[1].def_module_qn = "contracts"; + + defs[2].qualified_name = "trainer.Trainer"; + defs[2].short_name = "Trainer"; + defs[2].label = "Class"; + defs[2].def_module_qn = "trainer"; + defs[2].field_defs = "strategies:contracts.Contract"; + + const char *imp_names[] = {"Trainer"}; + const char *imp_qns[] = {"trainer.Trainer"}; + + CBMArena arena; + cbm_arena_init(&arena); + CBMResolvedCallArray out = {0}; + + cbm_run_py_lsp_cross(&arena, source, (int)strlen(source), "test.loop", defs, 3, imp_names, + imp_qns, 1, NULL, &out, NULL); + + ASSERT_GTE(find_resolved_arr(&out, "run", "process_batch"), 0); + cbm_arena_destroy(&arena); + PASS(); +} + +/* #1277 on the production path: the sealed shared Tier-2 registry. Covers the + * direct chain, the local alias, a field inherited from a base class, and two + * controls: an undeclared field and a field whose type has no such method + * must stay unresolved (never guessed). */ +TEST(pylsp_crossfile_typed_field_shared_registry) { + CBMArena arena; + cbm_arena_init(&arena); + + CBMLSPDef defs[5]; + memset(defs, 0, sizeof(defs)); + defs[0].qualified_name = "contracts.Contract"; + defs[0].short_name = "Contract"; + defs[0].label = "Class"; + defs[0].def_module_qn = "contracts"; + defs[1].qualified_name = "contracts.Contract.process_batch"; + defs[1].short_name = "process_batch"; + defs[1].label = "Method"; + defs[1].receiver_type = "contracts.Contract"; + defs[1].def_module_qn = "contracts"; + defs[2].qualified_name = "trainer.Trainer"; + defs[2].short_name = "Trainer"; + defs[2].label = "Class"; + defs[2].def_module_qn = "trainer"; + defs[2].field_defs = "strategies:contracts.Contract|name:builtins.str"; + defs[3].qualified_name = "trainer.SubTrainer"; + defs[3].short_name = "SubTrainer"; + defs[3].label = "Class"; + defs[3].def_module_qn = "trainer"; + defs[3].embedded_types = "trainer.Trainer"; + defs[4].qualified_name = "trainer.Other"; + defs[4].short_name = "Other"; + defs[4].label = "Class"; + defs[4].def_module_qn = "trainer"; + for (int i = 0; i < 5; i++) { + defs[i].lang = CBM_LANG_PYTHON; + } + + CBMTypeRegistry *reg = cbm_py_build_cross_registry(&arena, defs, 5); + ASSERT_NOT_NULL(reg); + ASSERT_TRUE(reg->read_only); + + const char *src = "from trainer import Trainer, SubTrainer, Other\n" + "def direct(t: Trainer):\n" + " return t.strategies.process_batch()\n" + "def alias(t: Trainer):\n" + " s = t.strategies\n" + " return s.process_batch()\n" + "def inherited(t: SubTrainer):\n" + " return t.strategies.process_batch()\n" + "def undeclared(t: Other):\n" + " return t.strategies.process_batch()\n" + "def wrong_type(t: Trainer):\n" + " return t.name.process_batch()\n"; + const char *imp_names[] = {"Trainer", "SubTrainer", "Other"}; + const char *imp_qns[] = {"trainer.Trainer", "trainer.SubTrainer", "trainer.Other"}; + CBMResolvedCallArray out = {0}; + cbm_run_py_lsp_cross_with_registry(&arena, src, (int)strlen(src), "test.loop", reg, imp_names, + imp_qns, 3, NULL, &out, NULL); + + ASSERT_GTE(find_resolved_arr(&out, "direct", "Contract.process_batch"), 0); + ASSERT_GTE(find_resolved_arr(&out, "alias", "Contract.process_batch"), 0); + ASSERT_GTE(find_resolved_arr(&out, "inherited", "Contract.process_batch"), 0); + ASSERT_EQ(find_resolved_arr(&out, "undeclared", "process_batch"), -1); + ASSERT_EQ(find_resolved_arr(&out, "wrong_type", "process_batch"), -1); + + cbm_arena_destroy(&arena); + PASS(); +} + /* Graph-quality guard for the Python seal fix (per-file field overlay). In the * FUSED path the shared Tier-2 registry is read_only, so `self.b = Bar()` can no * longer be recorded by mutating the shared registry; it is recorded in the per-file @@ -2110,16 +2224,30 @@ TEST(pylsp_issue710_heterogeneous_receiver_chain) { PASS(); } -TEST(pylsp_eval_steps_budget_degrades_gracefully) { - /* PY_EVAL_MAX_STEPS_PER_FILE guard: a file whose expressions demand - * more evaluator work than the per-file budget must still complete - * quickly and keep everything resolved BEFORE exhaustion — graceful - * partial degradation, not a hang, crash, or corrupted result. 90 - * statements x 30-link chains re-evaluated across the bind + emit - * phases (the assignment bind flushes the memo generation in between) - * comfortably exceed the 10000-step budget. */ - enum { BUDGET_STMTS = 90, BUDGET_LINKS = 30 }; - size_t sz = 4096 + (size_t)BUDGET_STMTS * (32 + (size_t)BUDGET_LINKS * 8); +/* Resolved INVOCATION rows whose callee QN ends with `suffix` (e.g. ".compile"). */ +static int count_resolved_suffix(const CBMFileResult *r, const char *suffix) { + size_t sl = strlen(suffix); + int n = 0; + for (int i = 0; i < r->resolved_calls.count; i++) { + const CBMResolvedCall *rc = &r->resolved_calls.items[i]; + size_t ql = rc->callee_qn ? strlen(rc->callee_qn) : 0; + if (rc->kind == CBM_RESOLVED_INVOCATION && ql >= sl && + strcmp(rc->callee_qn + ql - sl, suffix) == 0) { + n++; + } + } + return n; +} + +TEST(pylsp_issue1527_chain_statements_resolve_completely) { + /* #1527 replaces the old budget-degradation guard: with no per-file + * evaluation budget, the #710 chain shape -- 90 statements x 30-link + * chains, re-evaluated across the bind + emit phases (the assignment bind + * flushes the memo generation in between) -- must still finish quickly + * AND resolve EVERY link and EVERY final call (each is a distinct call + * site), not merely the first statements. */ + enum { CHAIN_STMTS = 90, CHAIN_LINKS = 30 }; + size_t sz = 4096 + (size_t)CHAIN_STMTS * (32 + (size_t)CHAIN_LINKS * 8); char *src = malloc(sz); ASSERT_NOT_NULL(src); char *p = src; @@ -2134,11 +2262,11 @@ TEST(pylsp_eval_steps_budget_degrades_gracefully) { "def run():\n"); p += n; left -= (size_t)n; - for (int s = 0; s < BUDGET_STMTS; s++) { + for (int s = 0; s < CHAIN_STMTS; s++) { n = snprintf(p, left, " v%d = G()", s); p += n; left -= (size_t)n; - for (int l = 0; l < BUDGET_LINKS; l++) { + for (int l = 0; l < CHAIN_LINKS; l++) { n = snprintf(p, left, ".add(%d)", l); p += n; left -= (size_t)n; @@ -2151,15 +2279,2387 @@ TEST(pylsp_eval_steps_budget_degrades_gracefully) { CBMFileResult *r = extract_py(src); free(src); ASSERT_NOT_NULL(r); - /* The first statements run with budget to spare — they must resolve. */ - ASSERT_GTE(require_resolved(r, "run", "G.compile"), 0); + ASSERT_EQ(count_resolved_suffix(r, ".G.compile"), CHAIN_STMTS); + ASSERT_EQ(count_resolved_suffix(r, ".G.add"), CHAIN_STMTS * CHAIN_LINKS); + cbm_free_result(r); + PASS(); +} + +TEST(pylsp_issue1527_large_module_resolves_every_call) { + /* #1527 reproduction: a module of 8000 classes plus one function calling + * `ClsN().chain().chain().method()` for each. Before the fix the per-file + * step cap stopped resolution after ~3333 statements (CALLS 14,666 of + * 24,000 in the graph) while the file still reported status `indexed`. + * Every one of the 8000 x 3 method call sites must resolve. */ + enum { N_CLASSES = 8000 }; + size_t sz = (size_t)N_CLASSES * 192 + 1024; + char *src = malloc(sz); + ASSERT_NOT_NULL(src); + size_t pos = (size_t)snprintf(src, sz, "from typing import Self\n"); + for (int i = 0; i < N_CLASSES; i++) { + pos += (size_t)snprintf(src + pos, sz - pos, + "class Cls%d:\n" + " def method(self) -> int:\n" + " return %d\n" + " def chain(self) -> Self:\n" + " return self\n", + i, i); + } + pos += (size_t)snprintf(src + pos, sz - pos, "def use():\n"); + for (int i = 0; i < N_CLASSES; i++) { + pos += (size_t)snprintf(src + pos, sz - pos, " Cls%d().chain().chain().method()\n", i); + } + ASSERT(pos < sz); + CBMFileResult *r = + cbm_extract_file(src, (int)pos, CBM_LANG_PYTHON, "test", "scale.py", 0, NULL, NULL); + free(src); + ASSERT_NOT_NULL(r); + ASSERT_EQ(count_resolved_suffix(r, ".method"), N_CLASSES); + ASSERT_EQ(count_resolved_suffix(r, ".chain"), 2 * N_CLASSES); + /* The last class is resolved exactly, not just some class. */ + ASSERT_GTE(require_resolved(r, "use", "Cls7999.method"), 0); cbm_free_result(r); PASS(); } /* ── Suite ─────────────────────────────────────────────────────── */ +static char *pylsp_work_fixture(int n_classes, int *out_len) { + /* Per class: ~140 chars (5-line def). Per call: ~50 chars. Overhead + * for the class number digits scales with log10(n) but the constant + * 256 covers up to 9-digit indices comfortably. */ + int approx = n_classes * 256 + 1024; + char *buf = (char *)malloc((size_t)approx); + if (!buf) + return NULL; + int pos = 0; + pos += snprintf(buf + pos, (size_t)(approx - pos), "from typing import Self\n"); + for (int i = 0; i < n_classes; i++) { + int n = snprintf(buf + pos, (size_t)(approx - pos), + "class Cls%d:\n" + " def method(self) -> int:\n" + " return %d\n" + " def chain(self) -> Self:\n" + " return self\n", + i, i); + if (n < 0 || pos + n >= approx) + break; + pos += n; + } + int n = snprintf(buf + pos, (size_t)(approx - pos), "def use():\n"); + pos += n; + for (int i = 0; i < n_classes; i++) { + int m = snprintf(buf + pos, (size_t)(approx - pos), + " Cls%d().chain().chain().method()\n", i); + if (m < 0 || pos + m >= approx) + break; + pos += m; + } + *out_len = pos; + return buf; +} + +/* Deterministic complexity gate (#1527, O9: no clock). The resolver + * primitives count every candidate they examine -- scope-frame probes, + * registry lookup probes, enclosing-definition candidates -- plus every real + * (non-memoized) expression evaluation, into a seam-build work counter. 4x + * the input must cost about 4x the work: a linear-scan lookup or a lost memo + * lands at ~16x (quadratic) or worse. Also asserts every call site resolved, + * so the work is not "linear" because resolution stopped early. */ +static uint64_t work_for(int n_classes, int *out_resolved) { + int slen = 0; + char *src = pylsp_work_fixture(n_classes, &slen); + if (!src) + return 0; + (void)cbm_lsp_work_take(); + CBMFileResult *r = + cbm_extract_file(src, slen, CBM_LANG_PYTHON, "test", "scale.py", 0, NULL, NULL); + uint64_t work = cbm_lsp_work_take(); + *out_resolved = r ? r->resolved_calls.count : -1; + if (r) + cbm_free_result(r); + free(src); + return work; +} + +TEST(pylsp_scale_work_is_linear) { + int r1 = 0, r4 = 0; + uint64_t w1 = work_for(1000, &r1); + uint64_t w4 = work_for(4000, &r4); + double ratio = w1 ? (double)w4 / (double)w1 : 0.0; + printf(" work: 1000=%llu (resolved=%d) 4000=%llu (resolved=%d) ratio=%.2fx " + "(linear ~4x, quadratic ~16x)\n", + (unsigned long long)w1, r1, (unsigned long long)w4, r4, ratio); + /* Each class line: ClsN() + chain() + chain() + method() = 4 resolved sites. */ + ASSERT_EQ(r1, 4 * 1000); + ASSERT_EQ(r4, 4 * 4000); + ASSERT_GT(w1, 0); + ASSERT(ratio < 5.0); + PASS(); +} + +extern const TSLanguage *tree_sitter_python(void); + +TEST(pylsp_issue1527_memo_allocation_failure_is_reported) { + int len = 0; + char *src = pylsp_work_fixture(200, &len); + ASSERT_NOT_NULL(src); + bool failures_reported = true; + for (int fail_after = 0; fail_after <= 1; fail_after++) { + cbm_py_lsp_test_memo_fail_after(fail_after); + CBMFileResult *r = extract_py(src); + cbm_py_lsp_test_memo_fail_after(-1); + failures_reported = failures_reported && r && r->has_error && r->lsp_skipped && + r->error_msg && strcmp(r->error_msg, CBM_PY_LSP_MEMO_ERROR) == 0 && + r->resolved_calls.count == 0; + if (r) { + cbm_result_compact(r); + failures_reported = failures_reported && r->error_msg && + strcmp(r->error_msg, CBM_PY_LSP_MEMO_ERROR) == 0; + cbm_free_result(r); + } + } + CBMFileResult *fresh = extract_py(src); + bool recovered = fresh && !fresh->has_error && fresh->resolved_calls.count > 0; + if (fresh) + cbm_free_result(fresh); + free(src); + ASSERT_TRUE(failures_reported); + ASSERT_TRUE(recovered); + PASS(); +} + +TEST(pylsp_issue1527_memo_and_depth_failures_stop_work) { + const char *src = "x\n"; + TSParser *parser = ts_parser_new(); + ASSERT_NOT_NULL(parser); + ts_parser_set_language(parser, tree_sitter_python()); + TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)strlen(src)); + if (!tree) { + ts_parser_delete(parser); + FAIL("failed to parse memo fixture"); + } + TSNode node = ts_node_named_child(ts_node_named_child(ts_tree_root_node(tree), 0), 0); + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + CBMResolvedCallArray out = {0}; + PyLSPContext ctx; + py_lsp_init(&ctx, &arena, src, (int)strlen(src), ®, "test", &out); + const CBMType *known = cbm_type_named(&arena, "test.Known"); + cbm_scope_bind(ctx.current_scope, "x", known); + ctx.eval_depth = 256; + bool depth_unknown = cbm_type_is_unknown(cbm_py_lsp_test_eval(&ctx, node)) && + ctx.eval_failure == CBM_LSP_DEPTH_EXCEEDED; + ctx.eval_depth = 0; + (void)cbm_lsp_work_take(); + bool depth_stopped = cbm_type_is_unknown(cbm_py_lsp_test_eval(&ctx, node)); + depth_stopped = depth_stopped && cbm_lsp_work_take() == 0; + /* A fresh walk can resolve this shallow expression; the failed one cannot. */ + py_lsp_init(&ctx, &arena, src, (int)strlen(src), ®, "test", &out); + cbm_scope_bind(ctx.current_scope, "x", known); + bool depth_recovered = cbm_py_lsp_test_eval(&ctx, node) == known && !ctx.eval_failure; + bool stopped = true; + for (int capacity_failure = 0; capacity_failure < 2; capacity_failure++) { + py_lsp_init(&ctx, &arena, src, (int)strlen(src), ®, "test", &out); + cbm_scope_bind(ctx.current_scope, "x", known); + ctx.test_memo_allocations_left = 0; + /* Pin the maximum-capacity refusal without allocating a huge table. + * A NULL table bypasses lookup; insertion must report failed growth. */ + if (capacity_failure) + ctx.type_cache_cap = 1 << 26; + stopped = + stopped && cbm_type_is_unknown(cbm_py_lsp_test_eval(&ctx, node)) && ctx.eval_failure; + (void)cbm_lsp_work_take(); + for (int i = 0; i < 20; i++) + stopped = cbm_type_is_unknown(cbm_py_lsp_test_eval(&ctx, node)) && stopped; + stopped = stopped && cbm_lsp_work_take() == 0; + } + cbm_arena_destroy(&arena); + ts_tree_delete(tree); + ts_parser_delete(parser); + ASSERT_TRUE(depth_unknown); + ASSERT_TRUE(depth_stopped); + ASSERT_TRUE(depth_recovered); + ASSERT_TRUE(stopped); + PASS(); +} + +TEST(pylsp_issue1527_root_class_name_failure_binds_no_external_class) { + const char *src = "class Local:\n pass\n"; + TSParser *parser = ts_parser_new(); + ASSERT_NOT_NULL(parser); + ts_parser_set_language(parser, tree_sitter_python()); + TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)strlen(src)); + if (!tree) { + ts_parser_delete(parser); + FAIL("failed to parse root-name fixture"); + } + bool closed = true, control = false; + for (int failure = 0; failure <= 2; failure++) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + cbm_registry_add_type( + ®, (CBMRegisteredType){.qualified_name = "test.Local", .short_name = "Local"}); + cbm_registry_add_type( + ®, (CBMRegisteredType){.qualified_name = "test.External", .short_name = "External"}); + CBMResolvedCallArray out = {0}; + PyLSPContext ctx; + py_lsp_init(&ctx, &arena, src, (int)strlen(src), ®, "test", &out); + ctx.test_root_name_failure = failure; + cbm_py_lsp_test_bind_external_classes(&ctx, ts_tree_root_node(tree)); + bool local_absent = !cbm_scope_lookup_local(ctx.current_scope, "Local"); + bool external_present = cbm_scope_lookup_local(ctx.current_scope, "External") != NULL; + if (failure) + closed = + closed && ctx.callable_value_proof_disabled && local_absent && !external_present; + else + control = !ctx.callable_value_proof_disabled && local_absent && external_present; + cbm_arena_destroy(&arena); + } + ts_tree_delete(tree); + ts_parser_delete(parser); + ASSERT_TRUE(control); + ASSERT_TRUE(closed); + PASS(); +} + +TEST(pylsp_issue1527_cross_memo_failure_preserves_completed_rows) { + char src[4096]; + size_t pos = (size_t)snprintf(src, sizeof(src), "def run(s: S):\n s + s\n"); + for (int i = 0; i < 200; i++) + pos += (size_t)snprintf(src + pos, sizeof(src) - pos, " s.m()\n"); + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.S.__add__", + .short_name = "__add__", + .label = "Method", + .receiver_type = "test.S", + .return_types = "S", + .lang = CBM_LANG_PYTHON}, + }; + bool correct = true; + for (int mode = 0; mode < 3; mode++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMResolvedCall prior = {.caller_qn = "test.run", + .callee_qn = "test.S.__add__", + .kind = CBM_RESOLVED_INVOCATION, + .confidence = 0.1f, + .strategy = "prior.strategy", + .reason = "prior.reason", + .site_start_byte = (uint32_t)(strstr(src, "s + s") - src), + .site_end_byte = (uint32_t)(strstr(src, "s + s") - src) + 5}; + CBMCall prior_synthetic = {.callee_name = "completed.synthetic"}; + CBMResolvedCallArray out = {.items = &prior, .count = 1, .cap = 1}; + CBMCallArray synthetic = {.items = &prior_synthetic, .count = 1, .cap = 1}; + CBMTypeRegistry *base = mode == 1 ? cbm_py_build_cross_registry(&shared, defs, 3) : NULL; + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + CBMBatchPyLSPFile file = {.source = src, + .source_len = (int)strlen(src), + .module_qn = "test", + .defs = defs, + .def_count = 3}; + cbm_py_lsp_test_memo_fail_after(1); + bool complete = + mode == 0 ? cbm_run_py_lsp_cross(&arena, src, (int)strlen(src), "test", defs, 3, NULL, + NULL, 0, NULL, &out, &synthetic) + : mode == 1 ? cbm_run_py_lsp_cross_with_registry(&arena, src, (int)strlen(src), "test", + &overlay, NULL, NULL, 0, NULL, &out, + &synthetic) + : cbm_batch_py_lsp_cross(&arena, &file, 1, &out); + cbm_py_lsp_test_memo_fail_after(-1); + correct = correct && !complete && out.count == (mode == 2 ? 0 : 1) && + synthetic.count == 1 && prior.confidence == 0.1f && + strcmp(prior.strategy, "prior.strategy") == 0 && + strcmp(prior.reason, "prior.reason") == 0 && + strcmp(prior_synthetic.callee_name, "completed.synthetic") == 0; + if (mode != 2) + correct = correct && out.items[0].confidence == 0.1f && + strcmp(out.items[0].strategy, "prior.strategy") == 0 && + strcmp(out.items[0].reason, "prior.reason") == 0; + if (mode == 2) + out = (CBMResolvedCallArray){0}; + complete = mode == 0 ? cbm_run_py_lsp_cross(&arena, src, (int)strlen(src), "test", defs, 3, + NULL, NULL, 0, NULL, &out, &synthetic) + : mode == 1 ? cbm_run_py_lsp_cross_with_registry(&arena, src, (int)strlen(src), + "test", &overlay, NULL, NULL, 0, + NULL, &out, &synthetic) + : cbm_batch_py_lsp_cross(&arena, &file, 1, &out); + correct = correct && complete && out.count > 200 && + (mode == 2 || (synthetic.count > 1 && out.items[0].confidence > 0.1f)); + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_issue1527_cross_memo_failure_fallback_reports_error) { + const char *src = "def run(s: S):\n s.m()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + }; + CBMFileResult result = {0}; + cbm_arena_init(&result.arena); + cbm_py_lsp_test_memo_fail_after(0); + cbm_pxc_run_one(CBM_LANG_PYTHON, &result, src, (int)strlen(src), "test", defs, 2, NULL, NULL, + 0); + cbm_py_lsp_test_memo_fail_after(-1); + bool reported = result.has_error && result.lsp_skipped && result.error_msg && + strcmp(result.error_msg, CBM_PY_LSP_MEMO_ERROR) == 0 && + result.resolved_calls.count == 0; + cbm_arena_destroy(&result.arena); + ASSERT_TRUE(reported); + PASS(); +} + +TEST(pylsp_issue1527_depth_failure_reports_all_entry_points) { + const char *src = "class S:\n def m(self):\n return 1\ndef run(s: S):\n s.m()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.S", .short_name = "S", .label = "Class", .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.S", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + }; + cbm_py_lsp_test_depth_fail(true); + CBMFileResult *raw = extract_py(src); + bool correct = raw && raw->has_error && raw->lsp_skipped && raw->error_msg && + strcmp(raw->error_msg, CBM_PY_LSP_DEPTH_ERROR) == 0 && + raw->resolved_calls.count == 0; + if (raw) + cbm_free_result(raw); + for (int mode = 0; mode < 3; mode++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *base = mode == 1 ? cbm_py_build_cross_registry(&shared, defs, 2) : NULL; + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + CBMResolvedCallArray out = {0}; + CBMBatchPyLSPFile file = {.source = src, + .source_len = (int)strlen(src), + .module_qn = "test", + .defs = defs, + .def_count = 2}; + if (mode == 2) { + correct = !cbm_batch_py_lsp_cross(&arena, &file, 1, &out) && correct; + } else { + CBMLSPStatus status = + mode == 0 ? cbm_run_py_lsp_cross_status(&arena, src, (int)strlen(src), "test", defs, + 2, NULL, NULL, 0, NULL, &out, NULL) + : cbm_run_py_lsp_cross_with_registry_status(&arena, src, (int)strlen(src), + "test", &overlay, NULL, NULL, + 0, NULL, &out, NULL); + correct = correct && status == CBM_LSP_DEPTH_EXCEEDED; + } + correct = correct && out.count == 0; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + CBMFileResult fallback = {0}; + cbm_arena_init(&fallback.arena); + cbm_pxc_run_one(CBM_LANG_PYTHON, &fallback, src, (int)strlen(src), "test", defs, 2, NULL, NULL, + 0); + correct = correct && fallback.has_error && fallback.lsp_skipped && fallback.error_msg && + strcmp(fallback.error_msg, CBM_PY_LSP_DEPTH_ERROR) == 0 && + fallback.resolved_calls.count == 0; + cbm_arena_destroy(&fallback.arena); + cbm_py_lsp_test_depth_fail(false); + CBMFileResult *fresh = extract_py(src); + bool recovered = fresh && !fresh->has_error && fresh->resolved_calls.count > 0; + if (fresh) + cbm_free_result(fresh); + ASSERT_TRUE(correct); + ASSERT_TRUE(recovered); + PASS(); +} + +/* Join one known source occurrence, not merely two equal aggregate counts. + * Changing the owner or range must not borrow a neighbouring resolution. */ +static bool pylsp_module_site_joins(const CBMCallArray *calls, const CBMResolvedCallArray *resolved, + const char *source, const char *site_text, const char *caller, + const char *target) { + const char *site = strstr(source, site_text); + if (!site || strstr(site + 1, site_text)) + return false; + uint32_t start = (uint32_t)(site - source); + uint32_t end = start + (uint32_t)strlen(site_text); + const CBMCall *call = NULL; + int call_count = 0; + for (int i = 0; i < calls->count; i++) { + const CBMCall *candidate = &calls->items[i]; + if (candidate->enclosing_func_qn && strcmp(candidate->enclosing_func_qn, caller) == 0 && + candidate->site_start_byte == start && candidate->site_end_byte == end) { + call = candidate; + call_count++; + } + } + if (call_count != 1) + return false; + const CBMResolvedCall *hit = cbm_pipeline_find_lsp_resolution(resolved, call, false); + if (!hit || !hit->caller_qn || strcmp(hit->caller_qn, caller) != 0 || !hit->callee_qn || + strcmp(hit->callee_qn, target) != 0 || hit->kind != CBM_RESOLVED_INVOCATION || + hit->site_start_byte != start || hit->site_end_byte != end || + hit->source_origin != call->source_origin) + return false; + int semantic_count = 0; + for (int i = 0; i < resolved->count; i++) { + const CBMResolvedCall *candidate = &resolved->items[i]; + if (candidate->kind == CBM_RESOLVED_INVOCATION && candidate->caller_qn && + strcmp(candidate->caller_qn, caller) == 0 && candidate->site_start_byte == start && + candidate->site_end_byte == end) + semantic_count++; + } + CBMCall wrong = *call; + wrong.enclosing_func_qn = "unrelated.module.caller"; + bool rejects_owner = cbm_pipeline_find_lsp_resolution(resolved, &wrong, false) == NULL; + wrong = *call; + wrong.site_start_byte++; + return semantic_count == 1 && rejects_owner && + cbm_pipeline_find_lsp_resolution(resolved, &wrong, false) == NULL; +} + +static bool pylsp_module_has_def(const CBMFileResult *r, const char *qn) { + for (int i = 0; i < r->defs.count; i++) { + if (r->defs.items[i].qualified_name && strcmp(r->defs.items[i].qualified_name, qn) == 0) + return true; + } + return false; +} + +TEST(pylsp_module_callers_join_exact_raw_occurrences) { + static const char source[] = "class Alpha:\n" + " def render(self):\n" + " return 1\n" + "class Beta:\n" + " def render(self):\n" + " return 2\n" + "def helper():\n" + " return 3\n" + "def control():\n" + " function_a = Alpha()\n" + " function_a.render()\n" + " helper()\n" + "class Controller:\n" + " def control(self):\n" + " method_b = Beta()\n" + " method_b.render()\n" + "module_a = Alpha()\n" + "module_b = Beta()\n" + "module_a.render()\n" + "module_b.render()\n"; + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + "app.py", 0, NULL, NULL); + bool correct = r && !r->has_error && !r->parse_incomplete && !r->lsp_skipped && r->module_qn && + strcmp(r->module_qn, "test.app") == 0; + if (correct) { + correct = + pylsp_module_has_def(r, "test.app.Alpha.render") && + pylsp_module_has_def(r, "test.app.Beta.render") && + pylsp_module_has_def(r, "test.app.control") && + pylsp_module_has_def(r, "test.app.Controller.control") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, "module_a.render()", + "test.app", "test.app.Alpha.render") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, "module_b.render()", + "test.app", "test.app.Beta.render") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, "function_a.render()", + "test.app.control", "test.app.Alpha.render") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, "method_b.render()", + "test.app.Controller.control", "test.app.Beta.render"); + } + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + + /* Package symbol-scope normalization is separate. A builtin isolates + * this package Module's raw caller identity from that remaining issue. */ + static const char init_source[] = "PACKAGE_SIZE = len([1])\n"; + r = cbm_extract_file(init_source, (int)strlen(init_source), CBM_LANG_PYTHON, "test", + "pkg/__init__.py", 0, NULL, NULL); + correct = r && !r->has_error && !r->parse_incomplete && !r->lsp_skipped && r->module_qn && + strcmp(r->module_qn, "test.pkg.__init__") == 0 && + pylsp_module_has_def(r, "builtins.len") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, init_source, "len([1])", + "test.pkg.__init__", "builtins.len"); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_module_named_lambda_keeps_exact_dunder_pair) { + static const char source[] = "class Number:\n" + " def __add__(self, other):\n" + " return self\n" + "module_number = Number()\n" + "add = lambda left, right: left + right\n" + "first = add(module_number, module_number)\n" + "second = add(module_number, module_number)\n"; + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + "app.py", 0, NULL, NULL); + bool correct = r && !r->has_error && !r->parse_incomplete && !r->lsp_skipped && + pylsp_module_has_def(r, "test.app.Number.__add__") && + pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, "left + right", + "test.app.", "test.app.Number.__add__"); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_module_cross_callers_own_input_and_join_exact_sites) { + static const char source[] = "from helpers import Alpha, Beta\n" + "module_a = Alpha()\n" + "module_b = Beta()\n" + "module_a.render()\n" + "module_b.render()\n"; + static const char *const paths[] = {"app.py", "pkg/__init__.py"}; + static const char *const modules[] = {"test.app", "test.pkg.__init__"}; + CBMLSPDef defs[] = { + {.qualified_name = "test.helpers.Alpha", + .short_name = "Alpha", + .label = "Class", + .def_module_qn = "test.helpers", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.helpers.Alpha.render", + .short_name = "render", + .label = "Method", + .receiver_type = "test.helpers.Alpha", + .def_module_qn = "test.helpers", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.helpers.Beta", + .short_name = "Beta", + .label = "Class", + .def_module_qn = "test.helpers", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.helpers.Beta.render", + .short_name = "render", + .label = "Method", + .receiver_type = "test.helpers.Beta", + .def_module_qn = "test.helpers", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + }; + const char *import_names[] = {"Alpha", "Beta"}; + const char *import_qns[] = {"test.helpers.Alpha", "test.helpers.Beta"}; + bool correct = true; + for (int path = 0; path < 2; path++) { + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + paths[path], 0, NULL, NULL); + if (!r || r->has_error || r->parse_incomplete || r->lsp_skipped || !r->module_qn || + strcmp(r->module_qn, modules[path]) != 0) { + correct = false; + if (r) + cbm_free_result(r); + continue; + } + for (int prebuilt = 0; prebuilt < 2; prebuilt++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *base = prebuilt ? cbm_py_build_cross_registry(&shared, defs, 4) : NULL; + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + char module_input[64]; + snprintf(module_input, sizeof(module_input), "%s", modules[path]); + CBMResolvedCallArray out = {0}; + CBMLSPStatus status = + prebuilt ? cbm_run_py_lsp_cross_with_registry_status( + &arena, source, (int)strlen(source), module_input, &overlay, + import_names, import_qns, 2, NULL, &out, NULL) + : cbm_run_py_lsp_cross_status(&arena, source, (int)strlen(source), + module_input, defs, 4, import_names, + import_qns, 2, NULL, &out, NULL); + /* Returned caller strings must outlive the caller's input buffer. + * Overwrite live storage, avoiding a dangling-pointer test. */ + memset(module_input, 'x', strlen(module_input)); + correct = correct && (!prebuilt || (base && base->read_only)) && + status == CBM_LSP_COMPLETE && + pylsp_module_site_joins(&r->calls, &out, source, "module_a.render()", + modules[path], "test.helpers.Alpha.render") && + pylsp_module_site_joins(&r->calls, &out, source, "module_b.render()", + modules[path], "test.helpers.Beta.render"); + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + cbm_free_result(r); + } + ASSERT_TRUE(correct); + PASS(); +} + +/* Package symbols use their parent scope while Module callers retain the file. */ +TEST(pylsp_init_scope_context_and_helper) { + static const struct { + const char *raw; + const char *scope; + } cases[] = { + {"test.__init__", "test"}, + {"test.pkg.__init__", "test.pkg"}, + {"test.pkg.nested.__init__", "test.pkg.nested"}, + {"test.plain", "test.plain"}, + {"test.x__init__", "test.x__init__"}, + {"__init__", "__init__"}, + {"test.index", "test.index"}, + {NULL, NULL}, + }; + bool correct = true; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry registry; + cbm_registry_init(®istry, &arena); + CBMResolvedCallArray out = {0}; + PyLSPContext ctx; + py_lsp_init(&ctx, &arena, "pass\n", 5, ®istry, cases[i].raw, &out); + correct = + correct && ctx.eval_failure == CBM_LSP_COMPLETE && ctx.current_scope && + ctx.file_module_qn == cases[i].raw && + cbm_fqn_symbol_scope_len(cases[i].raw) == (cases[i].scope ? strlen(cases[i].scope) : 0); + correct = + correct && (cases[i].scope ? ctx.module_qn && strcmp(ctx.module_qn, cases[i].scope) == 0 + : ctx.module_qn == NULL); + cbm_arena_destroy(&arena); + } + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_init_scope_raw_exact_occurrences_and_annotations) { + static const char source[] = + "class Store:\n def get(self):\n return 1\n" + "def identity(value: Store) -> Store:\n return value\n" + "def quoted(value: 'Store') -> 'Store':\n return value\n" + "def wrapped(values: list[Store]) -> Store:\n return values[0]\n" + "def run(first: Store, second: 'Store', values: list[Store]):\n" + " first.get()\n second.get()\n values[0].get()\n" + " identity(first).get()\n quoted(second).get()\n" + " wrapped(values).get()\n" + "class User:\n def use(self, item: Store):\n return item.get()\n" + "top = Store()\ntop.get()\n"; + static const char *paths[] = {"plain.py", "__init__.py", "pkg/nested/__init__.py"}; + static const char *scopes[] = {"test.plain", "test", "test.pkg.nested"}; + static const char *modules[] = {"test.plain", "test.__init__", "test.pkg.nested.__init__"}; + static const char *sites[] = {"first.get()", "second.get()", + "values[0].get()", "identity(first).get()", + "quoted(second).get()", "wrapped(values).get()"}; + bool correct = true; + for (int mode = 0; mode < 3; mode++) { + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + paths[mode], 0, NULL, NULL); + bool extracted = r && !r->has_error && !r->parse_incomplete && !r->lsp_skipped && + r->module_qn && strcmp(r->module_qn, modules[mode]) == 0; + correct = correct && extracted; + if (extracted) { + char caller[128], target[128], method[128]; + snprintf(caller, sizeof(caller), "%s.run", scopes[mode]); + snprintf(target, sizeof(target), "%s.Store.get", scopes[mode]); + snprintf(method, sizeof(method), "%s.User.use", scopes[mode]); + correct = correct && pylsp_module_has_def(r, caller) && + pylsp_module_has_def(r, target) && pylsp_module_has_def(r, method); + for (size_t j = 0; j < sizeof(sites) / sizeof(sites[0]); j++) { + bool joined = pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, + sites[j], caller, target); + correct = correct && joined; + } + bool method_join = pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, + "item.get()", method, target); + bool module_join = pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, + "top.get()", modules[mode], target); + correct = correct && method_join && module_join; + } + if (r) + cbm_free_result(r); + } + ASSERT_TRUE(correct); + PASS(); +} + +static bool pylsp_init_named_type(const CBMType *type, const char *qn) { + return type && type->kind == CBM_TYPE_NAMED && type->data.named.qualified_name && + strcmp(type->data.named.qualified_name, qn) == 0; +} + +static bool pylsp_init_signature(const CBMTypeRegistry *registry, const char *function, + const char *expected_type) { + const CBMRegisteredFunc *f = cbm_registry_lookup_func(registry, function); + const CBMType *sig = f ? f->signature : NULL; + if (!sig || sig->kind != CBM_TYPE_FUNC || !sig->data.func.param_types) + return false; + const CBMType **params = sig->data.func.param_types; + if (!pylsp_init_named_type(params[0], expected_type) || + !pylsp_init_named_type(params[1], expected_type) || !params[2] || params[3]) + return false; + const CBMType *container = params[2]; + return container->kind == CBM_TYPE_TEMPLATE && container->data.template_type.template_name && + strcmp(container->data.template_type.template_name, "list") == 0 && + container->data.template_type.arg_count == 1 && + container->data.template_type.template_args && + pylsp_init_named_type(container->data.template_type.template_args[0], expected_type); +} + +TEST(pylsp_init_scope_cross_signatures_keep_definition_language) { + const char *params[] = {"Token", "'Token'", "list[Token]"}; + CBMLSPDef defs[] = { + {.qualified_name = "test.pkg.call", + .short_name = "call", + .label = "Function", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON, + .signature_param_types = params, + .signature_param_count = 3}, + {.qualified_name = "test.foreign.__init__.call", + .short_name = "call", + .label = "Function", + .def_module_qn = "test.foreign.__init__", + .lang = CBM_LANG_GO, + .signature_param_types = params, + .signature_param_count = 3}, + {.qualified_name = "test.unknown.__init__.call", + .short_name = "call", + .label = "Function", + .def_module_qn = "test.unknown.__init__", + .lang = (CBMLanguage)-1, + .signature_param_types = params, + .signature_param_count = 3}, + }; + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry fallback; + cbm_registry_init(&fallback, &arena); + bool registered = cbm_py_lsp_test_register_defs(&arena, &fallback, defs, 3); + bool correct = registered && + pylsp_init_signature(&fallback, "test.pkg.call", "test.pkg.Token") && + pylsp_init_signature(&fallback, "test.foreign.__init__.call", + "test.foreign.__init__.Token") && + pylsp_init_signature(&fallback, "test.unknown.__init__.call", + "test.unknown.__init__.Token"); + CBMTypeRegistry *base = cbm_py_build_cross_registry(&shared, defs, 3); + correct = correct && base && base->read_only && + pylsp_init_signature(base, "test.pkg.call", "test.pkg.Token") && + !cbm_registry_lookup_func(base, "test.foreign.__init__.call") && + !cbm_registry_lookup_func(base, "test.unknown.__init__.call"); + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_init_scope_external_classes_and_cross_exact_joins) { + const char *source = "from pkg.sub import Widget as Imported\n" + "def use():\n store = Store()\n store.get()\n" + " imported = Imported()\n imported.get()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.pkg.Store", + .short_name = "Store", + .label = "Class", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Store.get", + .short_name = "get", + .label = "Method", + .def_module_qn = "test.pkg.__init__", + .receiver_type = "test.pkg.Store", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.sub.Widget", + .short_name = "Widget", + .label = "Class", + .def_module_qn = "test.pkg.sub", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.sub.Widget.get", + .short_name = "get", + .label = "Method", + .def_module_qn = "test.pkg.sub", + .receiver_type = "test.pkg.sub.Widget", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Outer.Nested", + .short_name = "Nested", + .label = "Class", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON}, + }; + const char *names[] = {"Imported"}; + const char *qns[] = {"test.pkg.sub.Widget"}; + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + "pkg/__init__.py", 0, NULL, NULL); + TSParser *parser = ts_parser_new(); + if (parser) + ts_parser_set_language(parser, tree_sitter_python()); + TSTree *tree = + parser ? ts_parser_parse_string(parser, NULL, source, (uint32_t)strlen(source)) : NULL; + bool correct = r && !r->has_error && !r->parse_incomplete && !r->lsp_skipped && tree && + pylsp_module_has_def(r, "test.pkg.use"); + for (int prebuilt = 0; correct && prebuilt < 2; prebuilt++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *base = cbm_py_build_cross_registry(&shared, defs, 5); + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + CBMResolvedCallArray out = {0}; + PyLSPContext ctx; + py_lsp_init(&ctx, &arena, source, (int)strlen(source), &overlay, "test.pkg.__init__", &out); + cbm_py_lsp_test_bind_external_classes(&ctx, ts_tree_root_node(tree)); + bool globals = base && ctx.eval_failure == CBM_LSP_COMPLETE && + pylsp_init_named_type(cbm_scope_lookup_local(ctx.current_scope, "Store"), + "test.pkg.Store") && + !cbm_scope_lookup_local(ctx.current_scope, "Widget") && + !cbm_scope_lookup_local(ctx.current_scope, "Nested"); + char module_input[] = "test.pkg.__init__"; + CBMLSPStatus status = + prebuilt + ? cbm_run_py_lsp_cross_with_registry_status(&arena, source, (int)strlen(source), + module_input, &overlay, names, qns, 1, + tree, &out, NULL) + : cbm_run_py_lsp_cross_status(&arena, source, (int)strlen(source), module_input, + defs, 5, names, qns, 1, tree, &out, NULL); + memset(module_input, 'x', sizeof(module_input) - 1); + bool store = pylsp_module_site_joins(&r->calls, &out, source, "store.get()", "test.pkg.use", + "test.pkg.Store.get"); + bool imported = pylsp_module_site_joins(&r->calls, &out, source, "imported.get()", + "test.pkg.use", "test.pkg.sub.Widget.get"); + correct = correct && globals && status == CBM_LSP_COMPLETE && store && imported; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + if (tree) + ts_tree_delete(tree); + if (parser) + ts_parser_delete(parser); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_init_scope_failure_preserves_outputs_and_wrappers) { + const char *source = "def run():\n value = Store()\n return value.get()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.pkg.Store", + .short_name = "Store", + .label = "Class", + .def_module_qn = "test.pkg.__init__", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.pkg.Store.get", + .short_name = "get", + .label = "Method", + .def_module_qn = "test.pkg.__init__", + .receiver_type = "test.pkg.Store", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + }; + bool correct = true; + cbm_py_lsp_test_scope_fail_after(0); + CBMFileResult *raw = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + "pkg/__init__.py", 0, NULL, NULL); + cbm_py_lsp_test_scope_fail_after(-1); + correct = raw && raw->has_error && raw->lsp_skipped && raw->error_msg && + strcmp(raw->error_msg, CBM_PY_LSP_SCOPE_ERROR) == 0 && + raw->resolved_calls.count == 0 && pylsp_module_has_def(raw, "test.pkg.run"); + if (raw) + cbm_free_result(raw); + for (int mode = 0; mode < 3; mode++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *base = cbm_py_build_cross_registry(&shared, defs, 2); + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + CBMResolvedCall prior = {.caller_qn = "completed.caller", + .callee_qn = "completed.target", + .strategy = "prior.strategy", + .reason = "prior.reason", + .confidence = 0.125f, + .kind = CBM_RESOLVED_INVOCATION, + .site_start_byte = 7, + .site_end_byte = 14}; + unsigned char saved[sizeof(prior)]; + memcpy(saved, &prior, sizeof(prior)); + CBMCall previous_synthetic = {.callee_name = "completed.synthetic"}; + CBMCallArray synthetic = {.items = &previous_synthetic, .count = 1, .cap = 1}; + CBMResolvedCallArray out = {.items = &prior, .count = 1, .cap = 1}; + cbm_py_lsp_test_scope_fail_after(mode == 2 ? 1 : 0); + /* mode 2 gets past context normalization, then fails signature scope. */ + CBMLSPStatus status = mode == 1 + ? cbm_run_py_lsp_cross_with_registry_status( + &arena, source, (int)strlen(source), "test.pkg.__init__", + &overlay, NULL, NULL, 0, NULL, &out, &synthetic) + : cbm_run_py_lsp_cross_status(&arena, source, (int)strlen(source), + "test.pkg.__init__", defs, 2, NULL, + NULL, 0, NULL, &out, &synthetic); + cbm_py_lsp_test_scope_fail_after(-1); + correct = correct && base && status == CBM_LSP_SCOPE_FAILED && out.count == 1 && + out.items == &prior && memcmp(saved, &prior, sizeof(prior)) == 0 && + synthetic.count == 1 && synthetic.items == &previous_synthetic && + strcmp(previous_synthetic.callee_name, "completed.synthetic") == 0; + cbm_py_lsp_test_scope_fail_after(0); + bool complete = + mode == 1 + ? cbm_run_py_lsp_cross_with_registry(&arena, source, (int)strlen(source), + "test.pkg.__init__", &overlay, NULL, NULL, 0, + NULL, &out, &synthetic) + : cbm_run_py_lsp_cross(&arena, source, (int)strlen(source), "test.pkg.__init__", + defs, 2, NULL, NULL, 0, NULL, &out, &synthetic); + cbm_py_lsp_test_scope_fail_after(-1); + correct = + correct && !complete && out.count == 1 && memcmp(saved, &prior, sizeof(prior)) == 0; + CBMBatchPyLSPFile file = {.source = source, + .source_len = (int)strlen(source), + .module_qn = "test.pkg.__init__", + .defs = defs, + .def_count = 2}; + CBMResolvedCallArray batch = {0}; + cbm_py_lsp_test_scope_fail_after(0); + bool batch_complete = cbm_batch_py_lsp_cross(&arena, &file, 1, &batch); + cbm_py_lsp_test_scope_fail_after(-1); + correct = correct && !batch_complete && batch.count == 0; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_init_scope_shared_failure_keeps_arena_and_needed_fallback) { + const char *source = "def use():\n obj = Item()\n return obj.get()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.plain.Item", + .short_name = "Item", + .label = "Class", + .def_module_qn = "test.plain", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.plain.Item.get", + .short_name = "get", + .label = "Method", + .def_module_qn = "test.plain", + .receiver_type = "test.plain.Item", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.unused.work", + .short_name = "work", + .label = "Function", + .def_module_qn = "test.unused.__init__", + .lang = CBM_LANG_PYTHON}, + }; + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + char *sentinel = cbm_arena_strdup(&shared, "caller-owned shared data"); + cbm_py_lsp_test_scope_fail_after(0); + CBMTypeRegistry *failed = cbm_py_build_cross_registry(&shared, defs, 3); + CBMResolvedCallArray out = {0}; + CBMLSPStatus fallback = + cbm_run_py_lsp_cross_status(&arena, source, (int)strlen(source), "test.plain", defs, 2, + NULL, NULL, 0, NULL, &out, NULL); + cbm_py_lsp_test_scope_fail_after(-1); + bool correct = !failed && sentinel && strcmp(sentinel, "caller-owned shared data") == 0 && + fallback == CBM_LSP_COMPLETE; + CBMFileResult *extracted = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, + "test", "plain.py", 0, NULL, NULL); + correct = correct && extracted && !extracted->has_error && !extracted->parse_incomplete && + pylsp_module_has_def(extracted, "test.plain.use") && + pylsp_module_site_joins(&extracted->calls, &out, source, "obj.get()", + "test.plain.use", "test.plain.Item.get"); + CBMTypeRegistry *fresh = cbm_py_build_cross_registry(&shared, defs, 3); + correct = correct && fresh && fresh->read_only && + cbm_registry_lookup_func(fresh, "test.unused.work") && + strcmp(sentinel, "caller-owned shared data") == 0; + if (extracted) + cbm_free_result(extracted); + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + ASSERT_TRUE(correct); + PASS(); +} + +/* Class execution keeps the extracted caller and exact source occurrence. */ +static bool pylsp_class_site_unjoined(const CBMFileResult *r, const char *source, + const char *site_text, const char *caller) { + const char *site = strstr(source, site_text); + if (!site || strstr(site + 1, site_text)) + return false; + uint32_t start = (uint32_t)(site - source); + uint32_t end = start + (uint32_t)strlen(site_text); + const CBMCall *call = NULL; + int count = 0; + for (int i = 0; i < r->calls.count; i++) { + const CBMCall *candidate = &r->calls.items[i]; + if (candidate->enclosing_func_qn && strcmp(candidate->enclosing_func_qn, caller) == 0 && + candidate->site_start_byte == start && candidate->site_end_byte == end) { + call = candidate; + count++; + } + } + if (count != 1 || cbm_pipeline_find_lsp_resolution(&r->resolved_calls, call, false)) + return false; + for (int i = 0; i < r->resolved_calls.count; i++) { + const CBMResolvedCall *hit = &r->resolved_calls.items[i]; + if (hit->kind == CBM_RESOLVED_INVOCATION && hit->site_start_byte == start && + hit->site_end_byte == end) + return false; + } + return true; +} + +typedef struct { + const char *site; + const char *caller; + const char *target; /* NULL requires an extracted, unresolved occurrence. */ +} PylspClassSite; + +static bool pylsp_class_sites(const CBMFileResult *r, const char *source, + const PylspClassSite *sites, size_t count) { + if (!r || r->has_error || r->parse_incomplete || r->lsp_skipped || !r->module_qn) + return false; + for (size_t i = 0; i < count; i++) { + const PylspClassSite *s = &sites[i]; + if (strcmp(s->caller, r->module_qn) != 0 && !pylsp_module_has_def(r, s->caller)) + return false; + if (s->target) { + if (!pylsp_module_has_def(r, s->target) || + !pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, s->site, s->caller, + s->target)) + return false; + } else if (!pylsp_class_site_unjoined(r, source, s->site, s->caller)) { + return false; + } + } + return true; +} + +TEST(pylsp_classbody_calls_join_extracted_caller) { + static const char source[] = + "def field(n):\n return n\n" + "class Svc:\n" + " @staticmethod\n def make(n):\n return Svc()\n" + "class Model:\n" + " first = field(11)\n second = Svc(12)\n third = Svc.make(13)\n" + "def control():\n field(21)\n Svc(22)\n Svc.make(23)\n"; + static const char *const paths[] = {"main.py", "pkg/__init__.py"}; + static const char *const modules[] = {"test.main", "test.pkg.__init__"}; + static const char *const scopes[] = {"test.main", "test.pkg"}; + bool correct = true; + for (int path = 0; path < 2; path++) { + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + paths[path], 0, NULL, NULL); + char field[64], svc[64], make[64], control[64]; + snprintf(field, sizeof(field), "%s.field", scopes[path]); + snprintf(svc, sizeof(svc), "%s.Svc", scopes[path]); + snprintf(make, sizeof(make), "%s.Svc.make", scopes[path]); + snprintf(control, sizeof(control), "%s.control", scopes[path]); + const PylspClassSite sites[] = {{"field(11)", modules[path], field}, + {"Svc(12)", modules[path], svc}, + {"Svc.make(13)", modules[path], make}, + {"field(21)", control, field}, + {"Svc(22)", control, svc}, + {"Svc.make(23)", control, make}}; + correct = pylsp_class_sites(r, source, sites, 6) && correct; + if (r) + cbm_free_result(r); + } + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_classbody_names_visible_to_later_statements_only) { + static const char source[] = + "def helper(n):\n return n\n" + "class Svc:\n def run(self, n):\n return n\n" + "class Model:\n" + " obj = Svc()\n result = obj.run(101)\n" + " def helper(self):\n return 2\n" + " later = helper(None)\n" + " def m(self):\n return helper(102)\n" + " def n(self):\n return obj.run(103)\n" + "class Shadow:\n helper = None\n value = helper(104)\n" + "def control():\n obj = Svc()\n obj.run(105)\n helper(106)\n"; + const PylspClassSite sites[] = { + {"obj.run(101)", "test.main", "test.main.Svc.run"}, + {"helper(None)", "test.main", "test.main.Model.helper"}, + {"helper(102)", "test.main.Model.m", "test.main.helper"}, + {"obj.run(103)", "test.main.Model.n", NULL}, + {"helper(104)", "test.main", NULL}, + {"obj.run(105)", "test.main.control", "test.main.Svc.run"}, + {"helper(106)", "test.main.control", "test.main.helper"}, + }; + CBMFileResult *r = extract_py(source); + bool correct = pylsp_class_sites(r, source, sites, 7); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_decorator_and_base_calls_join_extracted_caller) { + static const char source[] = "def deco(fn):\n return fn\n" + "def deco_arg(n):\n return deco\n" + "def base_factory(n):\n return object\n" + "class Svc:\n def run(self):\n return 1\n" + "@deco_arg(11)\ndef top():\n pass\n" + "@deco\ndef bare():\n pass\n" + "@deco_arg(12)\nclass Model(base_factory(13)):\n" + " marker = Svc()\n" + " @deco_arg(marker.run())\n def m(self):\n pass\n" + " @property\n def p(self):\n return 1\n" + "def outer():\n @deco_arg(14)\n def inner():\n pass\n" + " return inner\n"; + bool correct = true; + for (int package = 0; package < 2; package++) { + const char *scope = package ? "test.pkg" : "test.main"; + const char *module = package ? "test.pkg.__init__" : "test.main"; + char deco[64], base[64], run[64], outer[64]; + snprintf(deco, sizeof(deco), "%s.deco_arg", scope); + snprintf(base, sizeof(base), "%s.base_factory", scope); + snprintf(run, sizeof(run), "%s.Svc.run", scope); + snprintf(outer, sizeof(outer), "%s.outer", scope); + const PylspClassSite sites[] = { + {"deco_arg(11)", module, deco}, {"deco_arg(12)", module, deco}, + {"base_factory(13)", module, base}, {"deco_arg(marker.run())", module, deco}, + {"marker.run()", module, run}, {"deco_arg(14)", outer, deco}}; + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", + package ? "pkg/__init__.py" : "main.py", 0, NULL, NULL); + correct = pylsp_class_sites(r, source, sites, 6) && correct; + if (r) { + const char *bare_sites[] = {"@deco\n", "@property\n"}; + for (int b = 0; b < 2; b++) { + const char *hit = strstr(source, bare_sites[b]); + uint32_t lo = (uint32_t)(hit - source); + uint32_t hi = lo + (uint32_t)strlen(bare_sites[b]); + for (int i = 0; i < r->calls.count; i++) + correct = !(r->calls.items[i].site_start_byte >= lo && + r->calls.items[i].site_start_byte < hi) && + correct; + for (int i = 0; i < r->resolved_calls.count; i++) + correct = !(r->resolved_calls.items[i].kind == CBM_RESOLVED_INVOCATION && + r->resolved_calls.items[i].site_start_byte >= lo && + r->resolved_calls.items[i].site_start_byte < hi) && + correct; + } + cbm_free_result(r); + } + } + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_nested_and_local_classbody_calls_join_extracted_caller) { + static const char source[] = + "def field(n):\n return n\n" + "class Svc:\n def run(self):\n return 1\n" + "class Outer:\n tag = Svc()\n" + " class Meta:\n ordering = field(31)\n probe = tag.run()\n" + " inner = Meta()\n" + "def build():\n before = field(32)\n" + " class Local:\n value = field(33)\n" + " class Inner:\n value = field(34)\n" + " return Local\n" + "class Controller:\n def build(self):\n" + " class Local:\n value = field(35)\n" + " return Local\n"; + const PylspClassSite sites[] = { + {"field(31)", "test.main", "test.main.field"}, + {"tag.run()", "test.main", NULL}, + {"Meta()", "test.main", "test.main.Outer.Meta"}, + {"field(32)", "test.main.build", "test.main.field"}, + {"field(33)", "test.main.build", "test.main.field"}, + {"field(34)", "test.main.build", "test.main.field"}, + {"field(35)", "test.main.Controller.build", "test.main.field"}, + }; + CBMFileResult *r = extract_py(source); + bool correct = pylsp_class_sites(r, source, sites, 7) && + !pylsp_module_has_def(r, "test.main.build.Local") && + !pylsp_module_has_def(r, "test.main.Controller.build.Local"); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_class_comprehensions_separate_iterable_and_body_scopes) { + static const char source[] = + "class ModuleSvc:\n def run(self, n):\n return n\n" + "class ClassSvc:\n" + " def run(self, n):\n return n\n" + " def items(self) -> list[ModuleSvc]:\n return []\n" + "shared = ModuleSvc()\n" + "class Model:\n" + " shared = ClassSvc()\n local = ClassSvc()\n" + " a = [item.run(41) for item in local.items()]\n" + " b = {shared.run(42) for item in [1]}\n" + " c = {item: shared.run(43) for item in [1]}\n" + " d = (shared.run(44) for item in [1])\n" + " e = [local.run(45) for item in [1]]\n" + " f = [item for item in [1] if local.run(46)]\n" + " g = [item for item in [1] for other in local.items(47)]\n" + " h = [[inner.run(48) for inner in [outer]] for outer in [ModuleSvc()]]\n" + " class Nested:\n" + " shared = ClassSvc()\n" + " a = [shared.run(49) for item in [1]]\n" + " after = shared.run(50)\n" + "def control():\n return [item.run(51) for item in [ModuleSvc()]]\n"; + const PylspClassSite sites[] = { + {"local.items()", "test.main", "test.main.ClassSvc.items"}, + {"item.run(41)", "test.main", "test.main.ModuleSvc.run"}, + {"shared.run(42)", "test.main", "test.main.ModuleSvc.run"}, + {"shared.run(43)", "test.main", "test.main.ModuleSvc.run"}, + {"shared.run(44)", "test.main", "test.main.ModuleSvc.run"}, + {"local.run(45)", "test.main", NULL}, + {"local.run(46)", "test.main", NULL}, + {"local.items(47)", "test.main", NULL}, + {"inner.run(48)", "test.main", "test.main.ModuleSvc.run"}, + {"shared.run(49)", "test.main", "test.main.ModuleSvc.run"}, + {"shared.run(50)", "test.main", "test.main.ClassSvc.run"}, + {"item.run(51)", "test.main.control", "test.main.ModuleSvc.run"}, + }; + CBMFileResult *r = extract_py(source); + bool correct = pylsp_class_sites(r, source, sites, 12) && + pylsp_module_has_def(r, "test.main.ClassSvc.run") && + pylsp_module_has_def(r, "test.main.ClassSvc.items"); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_class_legacy_dispatch_cannot_bypass_or_publish_shadows) { + static const char source[] = "def wanted(n):\n return n\n" + "def other(n):\n return n\n" + "class Svc:\n def run(self, n):\n return n\n" + "table = {'x': wanted}\n" + "fn = lambda obj: obj.run(61)\n" + "stable = lambda obj: obj.run(62)\n" + "class Shadow:\n" + " table = None\n a = table['x'](63)\n" + " fn = None\n b = fn(Svc(64))\n" + "class Replacements:\n" + " table = {'x': other}\n c = table['x'](65)\n" + " stable = lambda obj: obj.run(66)\n" + "outside = table['x'](67)\n" + "outside_lambda = stable(Svc(68))\n"; + const PylspClassSite sites[] = { + {"table['x'](63)", "test.main", NULL}, + {"Svc(64)", "test.main", "test.main.Svc"}, + {"table['x'](65)", "test.main", NULL}, + {"table['x'](67)", "test.main", "test.main.wanted"}, + {"Svc(68)", "test.main", "test.main.Svc"}, + }; + CBMFileResult *r = extract_py(source); + bool correct = pylsp_class_sites(r, source, sites, 5) && + pylsp_module_has_def(r, "test.main.other") && + pylsp_module_has_def(r, "test.main.Svc.run"); + /* A replay has the lambda body's site, not the invocation's site. Verify + * those exact sites independently so output dedup cannot hide a replay. */ + if (r) { + const char *body[] = {"obj.run(61)", "obj.run(62)", "obj.run(66)"}; + for (int b = 0; b < 3; b++) { + uint32_t lo = (uint32_t)(strstr(source, body[b]) - source); + uint32_t hi = lo + (uint32_t)strlen(body[b]); + int extracted = 0, resolved = 0; + for (int i = 0; i < r->calls.count; i++) + extracted += r->calls.items[i].site_start_byte == lo && + r->calls.items[i].site_end_byte == hi; + for (int i = 0; i < r->resolved_calls.count; i++) { + const CBMResolvedCall *hit = &r->resolved_calls.items[i]; + if (hit->kind == CBM_RESOLVED_INVOCATION && hit->site_start_byte == lo && + hit->site_end_byte == hi) { + resolved++; + correct = hit->caller_qn && strcmp(hit->caller_qn, "test.main.") == 0 && + hit->callee_qn && strcmp(hit->callee_qn, "test.main.Svc.run") == 0 && + correct; + } + } + correct = extracted == 1 && resolved == (b == 1 ? 1 : 0) && correct; + } + cbm_free_result(r); + } + ASSERT_TRUE(correct); + PASS(); +} + +TEST(pylsp_class_frame_failure_restores_context_and_stops) { + static const char source[] = "class Outer:\n leaked = 1\n" + " first = hit(71)\n" + " values = [hit(72) for item in [1]]\n" + "later = hit(73)\n"; + TSParser *parser = ts_parser_new(); + bool setup = parser && ts_parser_set_language(parser, tree_sitter_python()); + TSTree *tree = + setup ? ts_parser_parse_string(parser, NULL, source, (uint32_t)strlen(source)) : NULL; + bool correct = tree && !ts_node_has_error(ts_tree_root_node(tree)); + CBMLSPDef defs[] = {{.qualified_name = "test.main.hit", + .short_name = "hit", + .label = "Function", + .def_module_qn = "test.main", + .lang = CBM_LANG_PYTHON}}; + for (int failure = 0; tree && failure < 2; failure++) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + bool registered = cbm_py_lsp_test_register_defs(&arena, ®, defs, 1); + CBMResolvedCallArray out = {0}; + PyLSPContext ctx; + py_lsp_init(&ctx, &arena, source, (int)strlen(source), ®, "test.main", &out); + CBMScope *root_scope = ctx.current_scope; + ctx.enclosing_func_qn = "saved.caller"; + ctx.enclosing_class_qn = "saved.class"; + cbm_py_lsp_test_class_frame_fail_after(failure); + py_lsp_process_file(&ctx, ts_tree_root_node(tree)); + cbm_py_lsp_test_class_frame_fail_after(-1); + correct = registered && root_scope && ctx.eval_failure == CBM_LSP_SCOPE_FAILED && + ctx.current_scope == root_scope && !ctx.class_body_outer_scope && + !ctx.class_body_qn && !ctx.class_body_in_comprehension && + strcmp(ctx.enclosing_func_qn, "saved.caller") == 0 && + strcmp(ctx.enclosing_class_qn, "saved.class") == 0 && out.count == 0 && + !cbm_scope_lookup_local(root_scope, "leaked") && + !cbm_scope_lookup_local(root_scope, "Outer") && + !cbm_scope_lookup_local(root_scope, "later") && correct; + (void)cbm_lsp_work_take(); + py_lsp_process_file(&ctx, ts_tree_root_node(tree)); + correct = cbm_lsp_work_take() == 0 && out.count == 0 && correct; + cbm_arena_destroy(&arena); + } + if (tree) + ts_tree_delete(tree); + if (parser) + ts_parser_delete(parser); + /* The pre-existing package seam must still ignore ordinary module names, + * including the new class/comprehension frames. */ + static const char healthy[] = "def hit(n):\n return n\n" + "class Outer:\n first = hit(74)\n" + " values = [hit(75) for item in [1]]\n"; + cbm_py_lsp_test_scope_fail_after(0); + CBMFileResult *r = extract_py(healthy); + cbm_py_lsp_test_scope_fail_after(-1); + const PylspClassSite sites[] = {{"hit(74)", "test.main", "test.main.hit"}, + {"hit(75)", "test.main", "test.main.hit"}}; + bool recovered = pylsp_class_sites(r, healthy, sites, 2); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + ASSERT_TRUE(recovered); + PASS(); +} + +TEST(pylsp_class_frame_failure_rolls_back_raw_and_cross_outputs) { + static const char source[] = "class Outer:\n item = S()\n item + item\n item.m()\n" + " class Inner:\n later = S(81)\n" + "after = S(82)\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.main.S", + .short_name = "S", + .label = "Class", + .def_module_qn = "test.main", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.main.S.m", + .short_name = "m", + .label = "Method", + .receiver_type = "test.main.S", + .def_module_qn = "test.main", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "test.main.S.__add__", + .short_name = "__add__", + .label = "Method", + .receiver_type = "test.main.S", + .def_module_qn = "test.main", + .return_types = "S", + .lang = CBM_LANG_PYTHON}, + }; + bool correct = true; + for (int prebuilt = 0; prebuilt < 2; prebuilt++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *base = prebuilt ? cbm_py_build_cross_registry(&shared, defs, 3) : NULL; + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &arena); + overlay.fallback = base; + CBMResolvedCall prior = { + .caller_qn = "test.main", + .callee_qn = "test.main.S.__add__", + .kind = CBM_RESOLVED_INVOCATION, + .confidence = 0.1f, + .strategy = "prior.strategy", + .reason = "prior.reason", + .site_start_byte = (uint32_t)(strstr(source, "item + item") - source), + .site_end_byte = (uint32_t)(strstr(source, "item + item") - source) + 11}; + CBMResolvedCall saved_prior; + memcpy(&saved_prior, &prior, sizeof(prior)); + CBMCall prior_synthetic = {.callee_name = "completed.synthetic"}; + CBMResolvedCallArray out = {.items = &prior, .count = 1, .cap = 1}; + CBMCallArray synthetic = {.items = &prior_synthetic, .count = 1, .cap = 1}; + cbm_py_lsp_test_class_frame_fail_after(1); + CBMLSPStatus status = + prebuilt + ? cbm_run_py_lsp_cross_with_registry_status(&arena, source, (int)strlen(source), + "test.main", &overlay, NULL, NULL, 0, + NULL, &out, &synthetic) + : cbm_run_py_lsp_cross_status(&arena, source, (int)strlen(source), "test.main", + defs, 3, NULL, NULL, 0, NULL, &out, &synthetic); + cbm_py_lsp_test_class_frame_fail_after(-1); + correct = (!prebuilt || base) && status == CBM_LSP_SCOPE_FAILED && out.count == 1 && + synthetic.count == 1 && memcmp(&prior, &saved_prior, sizeof(prior)) == 0 && + memcmp(&out.items[0], &saved_prior, sizeof(prior)) == 0 && + strcmp(prior_synthetic.callee_name, "completed.synthetic") == 0 && correct; + bool complete = + prebuilt ? cbm_run_py_lsp_cross_with_registry(&arena, source, (int)strlen(source), + "test.main", &overlay, NULL, NULL, 0, + NULL, &out, &synthetic) + : cbm_run_py_lsp_cross(&arena, source, (int)strlen(source), "test.main", defs, + 3, NULL, NULL, 0, NULL, &out, &synthetic); + correct = complete && out.count > 1 && synthetic.count > 1 && + out.items[0].confidence > saved_prior.confidence && correct; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + static const char raw_source[] = "class S:\n def m(self):\n return 1\n" + "class Outer:\n item = S()\n item.m()\n" + " class Inner:\n later = S(83)\n"; + cbm_py_lsp_test_class_frame_fail_after(2); + CBMFileResult *r = extract_py(raw_source); + cbm_py_lsp_test_class_frame_fail_after(-1); + correct = r && r->has_error && r->lsp_skipped && !r->parse_incomplete && r->error_msg && + strcmp(r->error_msg, CBM_PY_LSP_SCOPE_ERROR) == 0 && r->resolved_calls.count == 0 && + r->calls.count > 0 && correct; + if (r) { + cbm_result_compact(r); + correct = r->error_msg && strcmp(r->error_msg, CBM_PY_LSP_SCOPE_ERROR) == 0 && correct; + cbm_free_result(r); + } + r = extract_py(raw_source); + const PylspClassSite sites[] = {{"item.m()", "test.main", "test.main.S.m"}, + {"S(83)", "test.main", "test.main.S"}}; + bool recovered = pylsp_class_sites(r, raw_source, sites, 2); + if (r) + cbm_free_result(r); + ASSERT_TRUE(correct); + ASSERT_TRUE(recovered); + PASS(); +} + +/* The prefix index is an existence oracle, never an exact-symbol selector. */ +static bool pylsp_prefix_scan(const CBMTypeRegistry *reg, const char *prefix) { + if (!reg || !prefix) + return false; + CBMFreeFuncIter iter; + cbm_registry_all_funcs_chain(reg, &iter); + size_t length = strlen(prefix); + for (int i; (i = cbm_free_func_iter_next(&iter)) >= 0;) { + const char *qn = iter.reg->funcs[i].qualified_name; + if (qn && strncmp(qn, prefix, length) == 0) + return true; + } + return false; +} + +static bool pylsp_prefix_add(CBMTypeRegistry *reg, const char *qn, const char *receiver) { + int before = reg->func_count; + cbm_registry_add_func( + reg, + (CBMRegisteredFunc){.qualified_name = qn, .short_name = "call", .receiver_type = receiver}); + return reg->func_count == before + 1; +} + +static bool pylsp_prefix_oracles(const CBMTypeRegistry *reg) { + const char *prefixes[] = {NULL, "", "pkg.", "pkg.sub.", + "pkg.sub.call", "pkg.su.", "pkg.submarine.", "pkg.Type.", + "only.tail.", "missing.", "zzzz."}; + for (size_t i = 0; i < sizeof(prefixes) / sizeof(prefixes[0]); i++) { + uint64_t visits = 17; + bool indexed = cbm_registry_has_func_qn_prefix(reg, prefixes[i], &visits); + if (indexed != pylsp_prefix_scan(reg, prefixes[i]) || visits < 17 || + cbm_registry_has_func_qn_prefix(reg, prefixes[i], NULL) != indexed || + (!prefixes[i] && visits != 17)) + return false; + } + return true; +} + +TEST(pylsp_prefix_order_matches_iterator_and_chain_shadowing) { + CBMArena arena, scratch; + cbm_arena_init(&arena); + cbm_arena_init(&scratch); + CBMTypeRegistry head, middle, tail, empty; + cbm_registry_init(&head, &arena); + cbm_registry_init(&middle, &arena); + cbm_registry_init(&tail, &arena); + cbm_registry_init(&empty, &arena); + bool ok = pylsp_prefix_add(&head, NULL, NULL) && pylsp_prefix_add(&head, "", NULL) && + pylsp_prefix_add(&head, "pkg.sub.call", NULL) && + pylsp_prefix_add(&head, "pkg.sub.call", NULL) && + pylsp_prefix_add(&head, "pkg.Type.call", "pkg.Type") && + pylsp_prefix_add(&middle, "pkg.sub.call", "different.receiver") && + pylsp_prefix_add(&middle, "pkg.submarine.call", NULL) && + pylsp_prefix_add(&tail, "only.tail.call", NULL); + head.fallback = &middle; + middle.fallback = &tail; + ok = pylsp_prefix_oracles(NULL) && pylsp_prefix_oracles(&empty) && + pylsp_prefix_oracles(&head) && ok; + cbm_registry_build_func_qn_order(&head, &scratch); + ok = head.func_qn_sorted && head.func_qn_sorted_count == 4 && head.func_qn_sorted_upto == 5 && + pylsp_prefix_oracles(&head) && ok; + cbm_registry_build_func_qn_order(&tail, &arena); + ok = tail.func_qn_sorted && pylsp_prefix_oracles(&head) && ok; + cbm_registry_build_func_qn_order(&middle, &scratch); + ok = middle.func_qn_sorted && pylsp_prefix_oracles(&head) && ok; + if (head.func_qn_sorted) { + int duplicate_count = 0, previous = -1; + for (int i = 0; i < head.func_qn_sorted_count; i++) { + const CBMRegistryQNEntry *entry = &head.func_qn_sorted[i]; + ok = entry->func_index >= 0 && entry->func_index < head.func_count && + entry->qualified_name == head.funcs[entry->func_index].qualified_name && ok; + if (strcmp(entry->qualified_name, "pkg.sub.call") == 0) { + ok = entry->func_index > previous && ok; + previous = entry->func_index; + duplicate_count++; + } + } + ok = duplicate_count == 2 && ok; + } + size_t owned_before = cbm_arena_total(&arena), scratch_before = cbm_arena_total(&scratch); + ok = pylsp_prefix_oracles(&head) && cbm_arena_total(&arena) == owned_before && + cbm_arena_total(&scratch) == scratch_before && ok; + /* An indexed fallback also works behind an unindexed empty head. */ + empty.fallback = &head; + ok = pylsp_prefix_oracles(&empty) && ok; + cbm_arena_destroy(&scratch); + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_prefix_order_survives_array_growth_rebuild_and_seal) { + CBMArena arena, scratch; + cbm_arena_init(&arena); + cbm_arena_init(&scratch); + CBMTypeRegistry reg, unindexed; + cbm_registry_init(®, &arena); + cbm_registry_init(&unindexed, &arena); + bool ok = pylsp_prefix_add(®, "pkg.old.call", NULL); + cbm_registry_build_func_qn_order(®, &scratch); + CBMRegisteredFunc *original_array = reg.funcs; + const CBMRegistryQNEntry *order = reg.func_qn_sorted; + for (int i = 0; i < 140; i++) { + const char *qn = cbm_arena_sprintf(&arena, "tail.mod%03d.call", i); + ok = qn && pylsp_prefix_add(®, qn, NULL) && ok; + } + ok = reg.func_count == 141 && reg.funcs != original_array && reg.func_qn_sorted == order && + reg.func_qn_sorted_upto == 1 && + cbm_registry_has_func_qn_prefix(®, "tail.mod139.", NULL) && + !cbm_registry_has_func_qn_prefix(®, "tail.mod140.", NULL) && ok; + cbm_registry_build_func_qn_order(®, &scratch); + ok = reg.func_qn_sorted && reg.func_qn_sorted_upto == reg.func_count && + cbm_registry_has_func_qn_prefix(®, "tail.mod139.", NULL) && ok; + /* End use of the old snapshot before replacing an identity. No lookup is + * permitted until the complete order has been rebuilt. */ + if (reg.func_count == 141) { + reg.funcs[0].qualified_name = "pkg.renamed.call"; + cbm_registry_build_func_qn_order(®, &scratch); + ok = !cbm_registry_has_func_qn_prefix(®, "pkg.old.", NULL) && + cbm_registry_has_func_qn_prefix(®, "pkg.renamed.", NULL) && ok; + } + order = reg.func_qn_sorted; + reg.read_only = true; + cbm_registry_test_qn_order_fail_after(0); + cbm_registry_build_func_qn_order(®, &scratch); + cbm_registry_test_qn_order_fail_after(-1); + cbm_registry_add_func(®, (CBMRegisteredFunc){.qualified_name = "forbidden.call"}); + ok = reg.func_qn_sorted == order && reg.func_count == 141 && + cbm_registry_has_func_qn_prefix(®, "pkg.renamed.", NULL) && + !cbm_registry_has_func_qn_prefix(®, "forbidden.", NULL) && ok; + ok = pylsp_prefix_add(&unindexed, "sealed.call", NULL) && ok; + unindexed.read_only = true; + cbm_registry_build_func_qn_order(&unindexed, &scratch); + uint64_t visits = 0; + ok = !unindexed.func_qn_sorted && + cbm_registry_has_func_qn_prefix(&unindexed, "sealed.", &visits) && visits == 1 && ok; + cbm_arena_destroy(&scratch); + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_prefix_order_allocation_failure_preserves_oracle) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + bool ok = pylsp_prefix_add(®, "pkg.sub.call", NULL) && pylsp_prefix_add(®, NULL, NULL) && + pylsp_prefix_add(®, "only.tail.call", NULL); + cbm_registry_test_qn_order_fail_after(0); + cbm_registry_build_func_qn_order(®, &arena); + cbm_registry_test_qn_order_fail_after(-1); + uint64_t visits = 0; + ok = !reg.func_qn_sorted && reg.func_qn_sorted_count == 0 && reg.func_qn_sorted_upto == 0 && + pylsp_prefix_oracles(®) && !cbm_registry_has_func_qn_prefix(®, "zzzz.", &visits) && + visits == 3 && ok; + cbm_registry_test_qn_order_fail_after(1); + cbm_registry_build_func_qn_order(®, &arena); + bool first_built = reg.func_qn_sorted && reg.func_qn_sorted_upto == reg.func_count; + cbm_registry_build_func_qn_order(®, &arena); + cbm_registry_test_qn_order_fail_after(-1); + ok = first_built && !reg.func_qn_sorted && reg.func_qn_sorted_count == 0 && + reg.func_qn_sorted_upto == 0 && pylsp_prefix_oracles(®) && ok; + cbm_registry_build_func_qn_order(®, &arena); + ok = reg.func_qn_sorted && reg.func_qn_sorted_count == 2 && reg.func_qn_sorted_upto == 3 && + pylsp_prefix_oracles(®) && ok; + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_prefix_order_has_absolute_comparison_bound) { + enum { FUNCTIONS = 1024, PROBES = 32, BOUND = 12 }; + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + bool ok = true; + for (int i = FUNCTIONS - 1; i >= 0; i--) { + const char *qn = cbm_arena_sprintf(&arena, "project.mod%04d.call", i); + ok = qn && pylsp_prefix_add(®, qn, NULL) && ok; + } + cbm_registry_build_func_qn_order(®, &arena); + ok = reg.func_count == FUNCTIONS && reg.func_qn_sorted_count == FUNCTIONS && + reg.func_qn_sorted_upto == FUNCTIONS && reg.func_qn_sorted && ok; + for (int i = 0; i < PROBES; i++) { + char hit[64], miss[64]; + int a = snprintf(hit, sizeof(hit), "project.mod%04d.", i * 31); + int b = snprintf(miss, sizeof(miss), "project.mod%04dx.", i * 31); + uint64_t positive = 0, negative = 0; + bool ready = a > 0 && (size_t)a < sizeof(hit) && b > 0 && (size_t)b < sizeof(miss); + ok = ready && cbm_registry_has_func_qn_prefix(®, hit, &positive) && + !cbm_registry_has_func_qn_prefix(®, miss, &negative) && positive > 0 && + positive <= BOUND && negative > 0 && negative <= BOUND && + pylsp_prefix_scan(®, hit) && !pylsp_prefix_scan(®, miss) && ok; + } + /* Failure restores the reference scan, which must inspect every entry on + * this absent suffix. This is a deterministic contrast, not a timing ratio. */ + cbm_registry_test_qn_order_fail_after(0); + cbm_registry_build_func_qn_order(®, &arena); + cbm_registry_test_qn_order_fail_after(-1); + uint64_t visits = 0; + ok = !reg.func_qn_sorted && !cbm_registry_has_func_qn_prefix(®, "zzzz.", &visits) && + visits == FUNCTIONS && visits > BOUND && ok; + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_prefix_order_preserves_exact_selection_and_ambiguity) { + CBMArena arena; + cbm_arena_init(&arena); + bool ok = true; + for (int first = 0; first < 2; first++) { + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + ok = pylsp_prefix_add(®, "pkg.duplicate", NULL) && + pylsp_prefix_add(®, "pkg.duplicate", NULL) && ok; + const CBMRegisteredFunc *linear = cbm_registry_lookup_func(®, "pkg.duplicate"); + ok = reg.func_count == 2 && linear == ®.funcs[0] && ok; + cbm_registry_build_func_qn_order(®, &arena); + ok = cbm_registry_lookup_func(®, "pkg.duplicate") == linear && ok; + reg.index_first_registered = first != 0; + cbm_registry_finalize(®); + const CBMRegisteredFunc *selected = cbm_registry_lookup_func(®, "pkg.duplicate"); + ok = reg.func_count == 2 && selected == ®.funcs[first ? 0 : 1] && ok; + cbm_registry_build_func_qn_order(®, &arena); + ok = cbm_registry_lookup_func(®, "pkg.duplicate") == selected && ok; + } + CBMLSPDef defs[] = { + {.qualified_name = "pkg.duplicate", + .short_name = "duplicate", + .label = "Function", + .def_module_qn = "pkg", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "pkg.duplicate", + .short_name = "duplicate", + .label = "Function", + .def_module_qn = "pkg", + .return_types = "str", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "pkg.unique", + .short_name = "unique", + .label = "Function", + .def_module_qn = "pkg", + .lang = CBM_LANG_PYTHON}, + }; + for (int fail = 0; fail < 2; fail++) { + cbm_registry_test_qn_order_fail_after(fail ? 0 : -1); + CBMTypeRegistry *reg = cbm_py_build_cross_registry(&arena, defs, 3); + cbm_registry_test_qn_order_fail_after(-1); + ok = reg && reg->read_only && reg->func_count > 3 && ok; + if (reg) { + int duplicate_count = 0, unique_count = 0; + for (int i = 0; i < reg->func_count; i++) { + const CBMRegisteredFunc *f = ®->funcs[i]; + bool ambiguous = (f->flags & CBM_FUNC_FLAG_AMBIGUOUS_BINDING) != 0; + if (fail) + ok = ambiguous && ok; + if (f->qualified_name && strcmp(f->qualified_name, "pkg.duplicate") == 0) { + duplicate_count++; + ok = ambiguous && ok; + } + if (f->qualified_name && strcmp(f->qualified_name, "pkg.unique") == 0) { + unique_count++; + ok = ambiguous == (fail != 0) && ok; + } + } + ok = duplicate_count == 2 && unique_count == 1 && + (fail ? reg->func_qn_sorted == NULL : reg->func_qn_sorted != NULL) && + cbm_registry_has_func_qn_prefix(reg, "pkg.", NULL) && + !cbm_registry_has_func_qn_prefix(reg, "pkg.missing.", NULL) && ok; + } + } + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +/* A negative must have one real extracted source occurrence and no accepted + * resolution; a missing carrier cannot make it pass. */ +static bool pylsp_prefix_site_unresolved(const CBMCallArray *calls, + const CBMResolvedCallArray *resolved, const char *source, + const char *site_text, const char *caller) { + const char *site = strstr(source, site_text); + if (!site || strstr(site + 1, site_text)) + return false; + uint32_t start = (uint32_t)(site - source), end = start + (uint32_t)strlen(site_text); + const CBMCall *found = NULL; + int count = 0; + for (int i = 0; i < calls->count; i++) { + const CBMCall *call = &calls->items[i]; + if (call->enclosing_func_qn && strcmp(call->enclosing_func_qn, caller) == 0 && + call->site_start_byte == start && call->site_end_byte == end) { + found = call; + count++; + } + } + return count == 1 && cbm_pipeline_find_lsp_resolution(resolved, found, false) == NULL; +} + +TEST(pylsp_prefix_raw_per_file_and_shared_keep_exact_calls) { + static const char source[] = "import pkg\nimport vendor\n" + "def ping():\n return 1\n" + "def run():\n" + " pkg.sub.ping()\n" + " pkg.su.ping()\n" + " vendor.external.ping()\n"; + CBMLSPDef defs[] = { + {.qualified_name = "pkg.sub.ping", + .short_name = "ping", + .label = "Function", + .def_module_qn = "pkg.sub", + .return_types = "int", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "pkg.submarine.ping", + .short_name = "ping", + .label = "Function", + .def_module_qn = "pkg.submarine", + .return_types = "str", + .lang = CBM_LANG_PYTHON}, + }; + const char *names[] = {"pkg", "vendor"}; + const char *qns[] = {"pkg", "vendor"}; + bool ok = true; + for (int fail = 0; fail < 2; fail++) { + uint64_t probes_before = + atomic_load_explicit(&cbm_py_submodule_probes, memory_order_relaxed); + uint64_t visits_before = + atomic_load_explicit(&cbm_py_submodule_probe_visits, memory_order_relaxed); + cbm_registry_test_qn_order_fail_after(fail ? 0 : -1); + CBMFileResult *r = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "pkg", + "sub.py", 0, NULL, NULL); + cbm_registry_test_qn_order_fail_after(-1); + bool healthy = r && !r->has_error && !r->parse_incomplete && !r->lsp_skipped && + pylsp_module_has_def(r, "pkg.sub.ping") && + pylsp_module_has_def(r, "pkg.sub.run"); + ok = healthy && + atomic_load_explicit(&cbm_py_submodule_probes, memory_order_relaxed) > probes_before && + atomic_load_explicit(&cbm_py_submodule_probe_visits, memory_order_relaxed) > + visits_before && + ok; + if (healthy) { + ok = pylsp_module_site_joins(&r->calls, &r->resolved_calls, source, "pkg.sub.ping()", + "pkg.sub.run", "pkg.sub.ping") && + pylsp_prefix_site_unresolved(&r->calls, &r->resolved_calls, source, + "pkg.su.ping()", "pkg.sub.run") && + pylsp_prefix_site_unresolved(&r->calls, &r->resolved_calls, source, + "vendor.external.ping()", "pkg.sub.run") && + ok; + for (int shared_mode = 0; shared_mode < 2; shared_mode++) { + CBMArena arena, shared; + cbm_arena_init(&arena); + cbm_arena_init(&shared); + CBMTypeRegistry *reg = NULL; + cbm_registry_test_qn_order_fail_after(fail ? 0 : -1); + if (shared_mode) + reg = cbm_py_build_cross_registry(&shared, defs, 2); + CBMResolvedCallArray out = {0}; + CBMLSPStatus status = CBM_LSP_SCOPE_FAILED; + probes_before = + atomic_load_explicit(&cbm_py_submodule_probes, memory_order_relaxed); + visits_before = + atomic_load_explicit(&cbm_py_submodule_probe_visits, memory_order_relaxed); + if (!shared_mode || reg) { + status = shared_mode + ? cbm_run_py_lsp_cross_with_registry_status( + &arena, source, (int)strlen(source), "pkg.sub", reg, names, + qns, 2, r->cached_tree, &out, NULL) + : cbm_run_py_lsp_cross_status(&arena, source, (int)strlen(source), + "pkg.sub", defs, 2, names, qns, 2, + r->cached_tree, &out, NULL); + } + cbm_registry_test_qn_order_fail_after(-1); + ok = status == CBM_LSP_COMPLETE && + atomic_load_explicit(&cbm_py_submodule_probes, memory_order_relaxed) > + probes_before && + atomic_load_explicit(&cbm_py_submodule_probe_visits, memory_order_relaxed) > + visits_before && + pylsp_module_site_joins(&r->calls, &out, source, "pkg.sub.ping()", + "pkg.sub.run", "pkg.sub.ping") && + pylsp_prefix_site_unresolved(&r->calls, &out, source, "pkg.su.ping()", + "pkg.sub.run") && + pylsp_prefix_site_unresolved(&r->calls, &out, source, "vendor.external.ping()", + "pkg.sub.run") && + ok; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&shared); + } + } + if (r) + cbm_free_result(r); + } + ASSERT_TRUE(ok); + PASS(); +} + +/* Module-range queries must return the old array sequence, including duplicates. */ +static bool pylsp_range_add_type(CBMTypeRegistry *reg, const char *qn, const char *name) { + int before = reg->type_count; + cbm_registry_add_type(reg, (CBMRegisteredType){.qualified_name = qn, .short_name = name}); + return reg->type_count == before + 1; +} + +static bool pylsp_range_member(const char *qn, const char *prefix, bool direct) { + if (!qn || !prefix) + return false; + size_t n = strlen(prefix); + return strncmp(qn, prefix, n) == 0 && (!direct || !strchr(qn + n, '.')); +} + +/* No fixed result buffer and no call to the implementation's member predicate. */ +static bool pylsp_range_oracle(const CBMTypeRegistry *reg, bool types, const char *prefix, + bool direct, CBMArena *scratch, uint64_t *visits) { + CBMQNPrefixIter it; + bool opened = types ? cbm_registry_types_with_qn_prefix(reg, prefix, direct, scratch, &it) + : cbm_registry_funcs_with_qn_prefix(reg, prefix, direct, scratch, &it); + if (!opened) + return false; + int total = reg ? (types ? reg->type_count : reg->func_count) : 0; + bool ok = true; + for (int i = 0; i < total; i++) { + const char *qn = types ? reg->types[i].qualified_name : reg->funcs[i].qualified_name; + if (pylsp_range_member(qn, prefix, direct)) + ok = cbm_qn_prefix_iter_next(&it) == i && ok; + } + ok = cbm_qn_prefix_iter_next(&it) == -1 && cbm_qn_prefix_iter_next(&it) == -1 && ok; + if (visits) + *visits = it.visits; + return ok; +} + +static void pylsp_range_reset_failures(void) { + cbm_registry_test_type_qn_order_fail_after(-1); + cbm_registry_test_qn_prefix_block_fail_after(-1); + cbm_py_test_module_scratch_fail_after(-1); + cbm_py_test_module_prefix_fail_after(-1); +} + +TEST(pylsp_range_order_boundaries_and_null_contract) { + static const char *const qns[] = { + "p.foo.B", "p.foo.A.child", NULL, "p.foo.A-b", "p.foobar.C", "p.foo.AB", + "p.foo.A.z", "p.foo.", "", "p.foo.B", "p.foo.C.child", "p.foo.C-plain"}; + static const char *const prefixes[] = {NULL, "", "p.foo.", "p.foo", "p.foo.A", + "p.foo.A.", "p.foo.B", "p.foo..", "zzzz."}; + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg, empty; + cbm_registry_init(®, &arena); + cbm_registry_init(&empty, &arena); + bool ok = true; + for (size_t i = 0; i < sizeof(qns) / sizeof(qns[0]); i++) + ok = pylsp_range_add_type(®, qns[i], "Shared") && + pylsp_prefix_add(®, qns[i], i == 1 ? "Receiver" : NULL) && ok; + for (int indexed = 0; indexed < 2; indexed++) { + if (indexed) { + cbm_registry_build_type_qn_order(®, &arena); + cbm_registry_build_func_qn_order(®, &arena); + ok = reg.type_qn_sorted && reg.func_qn_sorted && + reg.type_qn_sorted_count == reg.type_count - 1 && + reg.type_qn_sorted_upto == reg.type_count && ok; + if (reg.type_qn_sorted) { + for (int i = 0; i < reg.type_qn_sorted_count; i++) { + const CBMRegistryTypeQNEntry *e = ®.type_qn_sorted[i]; + ok = e->type_index >= 0 && e->type_index < reg.type_count && + e->qualified_name == reg.types[e->type_index].qualified_name && ok; + } + } + } + for (int types = 0; types < 2; types++) { + for (int direct = 0; direct < 2; direct++) { + for (size_t p = 0; p < sizeof(prefixes) / sizeof(prefixes[0]); p++) { + CBMArena scratch; + cbm_arena_init(&scratch); + ok = pylsp_range_oracle(®, types != 0, prefixes[p], direct != 0, &scratch, + NULL) && + pylsp_range_oracle(®, types != 0, prefixes[p], direct != 0, NULL, + NULL) && + pylsp_range_oracle(&empty, types != 0, prefixes[p], direct != 0, &scratch, + NULL) && + pylsp_range_oracle(NULL, types != 0, prefixes[p], direct != 0, &scratch, + NULL) && + ok; + cbm_arena_destroy(&scratch); + } + } + } + } + ok = !cbm_registry_types_with_qn_prefix(®, "p.", true, &arena, NULL) && + !cbm_registry_funcs_with_qn_prefix(®, "p.", false, &arena, NULL) && + cbm_qn_prefix_iter_next(NULL) == -1 && + cbm_registry_has_func_qn_prefix(®, "p.foo.", NULL) && ok; + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_range_growth_rebuild_seal_and_independent_failure) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg, sealed; + cbm_registry_init(®, &arena); + cbm_registry_init(&sealed, &arena); + bool ok = pylsp_range_add_type(®, "p.old", "Old") && pylsp_prefix_add(®, "p.old", NULL); + cbm_registry_build_type_qn_order(®, &arena); + cbm_registry_build_func_qn_order(®, &arena); + const CBMRegisteredType *old_types = reg.types; + const CBMRegisteredFunc *old_funcs = reg.funcs; + for (int i = 0; i < 140; i++) { + const char *qn = cbm_arena_sprintf(&arena, "p.tail%03d", i); + ok = qn && pylsp_range_add_type(®, qn, "Tail") && pylsp_prefix_add(®, qn, NULL) && ok; + } + ok = pylsp_range_add_type(®, NULL, "NoQN") && pylsp_prefix_add(®, NULL, NULL) && + reg.type_count == 142 && reg.func_count == 142 && reg.types != old_types && + reg.funcs != old_funcs && reg.type_qn_sorted_upto == 1 && reg.func_qn_sorted_upto == 1 && + pylsp_range_oracle(®, true, "p.", true, &arena, NULL) && + pylsp_range_oracle(®, false, "p.", true, &arena, NULL) && ok; + if (reg.type_count == 142 && reg.func_count == 142) { + /* The previous queries ended before either registered identity changes. */ + reg.types[0].qualified_name = "p.renamed"; + reg.funcs[0].qualified_name = "p.renamed"; + cbm_registry_build_func_qn_order(®, &arena); + const CBMRegistryQNEntry *func_order = reg.func_qn_sorted; + cbm_registry_test_type_qn_order_fail_after(0); + cbm_registry_build_type_qn_order(®, &arena); + cbm_registry_test_type_qn_order_fail_after(-1); + ok = func_order && reg.func_qn_sorted == func_order && !reg.type_qn_sorted && + reg.type_qn_sorted_count == 0 && reg.type_qn_sorted_upto == 0 && + pylsp_range_oracle(®, true, "p.", true, &arena, NULL) && ok; + cbm_registry_test_type_qn_order_fail_after(1); + cbm_registry_build_type_qn_order(®, &arena); + bool first_built = reg.type_qn_sorted != NULL; + cbm_registry_build_type_qn_order(®, &arena); + cbm_registry_test_type_qn_order_fail_after(-1); + ok = first_built && !reg.type_qn_sorted && reg.func_qn_sorted == func_order && ok; + cbm_registry_build_type_qn_order(®, &arena); + const CBMRegistryTypeQNEntry *type_order = reg.type_qn_sorted; + cbm_registry_test_qn_order_fail_after(0); + cbm_registry_build_func_qn_order(®, &arena); + cbm_registry_test_qn_order_fail_after(-1); + ok = type_order && reg.type_qn_sorted == type_order && !reg.func_qn_sorted && + pylsp_range_oracle(®, false, "p.", true, &arena, NULL) && ok; + cbm_registry_build_func_qn_order(®, &arena); + func_order = reg.func_qn_sorted; + reg.read_only = true; + cbm_registry_test_type_qn_order_fail_after(0); + cbm_registry_test_qn_order_fail_after(0); + cbm_registry_build_type_qn_order(®, &arena); + cbm_registry_build_func_qn_order(®, &arena); + pylsp_range_reset_failures(); + cbm_registry_test_qn_order_fail_after(-1); + cbm_registry_add_type(®, (CBMRegisteredType){.qualified_name = "forbidden.Type"}); + cbm_registry_add_func(®, (CBMRegisteredFunc){.qualified_name = "forbidden.call"}); + ok = reg.type_qn_sorted == type_order && reg.func_qn_sorted == func_order && + reg.type_count == 142 && reg.func_count == 142 && + pylsp_range_oracle(®, true, "p.", true, &arena, NULL) && + pylsp_range_oracle(®, false, "p.", true, &arena, NULL) && ok; + } + ok = pylsp_range_add_type(&sealed, "p.sealed", "Sealed") && + pylsp_prefix_add(&sealed, "p.sealed", NULL) && ok; + sealed.read_only = true; + cbm_registry_build_type_qn_order(&sealed, &arena); + cbm_registry_build_func_qn_order(&sealed, &arena); + ok = !sealed.type_qn_sorted && !sealed.func_qn_sorted && + pylsp_range_oracle(&sealed, true, "p.", true, &arena, NULL) && + pylsp_range_oracle(&sealed, false, "p.", true, &arena, NULL) && ok; + pylsp_range_reset_failures(); + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_range_invalid_snapshot_uses_complete_scan) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + bool ok = pylsp_range_add_type(®, "p.Z", "Z") && pylsp_range_add_type(®, "p.A", "A") && + pylsp_prefix_add(®, "p.Z", NULL) && pylsp_prefix_add(®, "p.A", NULL); + cbm_registry_build_type_qn_order(®, &arena); + cbm_registry_build_func_qn_order(®, &arena); + ok = reg.type_count == 2 && reg.func_count == 2 && reg.type_qn_sorted && reg.func_qn_sorted && + ok; + size_t allocated = cbm_arena_total(&arena); + for (int types = 0; types < 2; types++) { + int *covered = types ? ®.type_qn_sorted_upto : ®.func_qn_sorted_upto; + int *count = types ? ®.type_qn_sorted_count : ®.func_qn_sorted_count; + int saved_covered = *covered, saved_count = *count; + for (int invalid = 0; invalid < 3; invalid++) { + *covered = invalid == 0 ? 3 : saved_covered; + *count = invalid == 1 ? -1 : (invalid == 2 ? 3 : saved_count); + int expected_covered = *covered, expected_count = *count; + uint64_t visits = 0; + ok = pylsp_range_oracle(®, types != 0, "p.", true, &arena, &visits) && visits >= 2 && + *covered == expected_covered && *count == expected_count && + cbm_arena_total(&arena) == allocated && ok; + *covered = saved_covered; + *count = saved_count; + } + ok = pylsp_range_oracle(®, types != 0, "p.", true, &arena, NULL) && ok; + allocated = cbm_arena_total(&arena); + } + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_range_block_failure_preserves_each_link_and_tail) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry links[3]; + bool ok = true; + for (int j = 0; j < 3; j++) { + cbm_registry_init(&links[j], &arena); + ok = pylsp_range_add_type(&links[j], "p.Z", "Z") && + pylsp_range_add_type(&links[j], "p.A", "A") && + pylsp_prefix_add(&links[j], "p.Z", NULL) && pylsp_prefix_add(&links[j], "p.A", NULL) && + ok; + cbm_registry_build_type_qn_order(&links[j], &arena); + cbm_registry_build_func_qn_order(&links[j], &arena); + ok = links[j].type_qn_sorted && links[j].func_qn_sorted && + pylsp_range_add_type(&links[j], "p.Tail", "Tail") && + pylsp_prefix_add(&links[j], "p.Tail", NULL) && ok; + links[j].fallback = j < 2 ? &links[j + 1] : NULL; + } + for (int types = 0; types < 2; types++) { + for (int fail = 0; fail < 3; fail++) { + CBMArena scratch; + cbm_arena_init(&scratch); + cbm_registry_test_qn_prefix_block_fail_after(fail); + for (int j = 0; j < 3; j++) { + uint64_t visits = 0; + ok = pylsp_range_oracle(&links[j], types != 0, "p.", true, &scratch, &visits) && + visits > 0 && ok; + } + cbm_registry_test_qn_prefix_block_fail_after(-1); + cbm_arena_destroy(&scratch); + } + } + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_range_direct_subtree_pruning_has_absolute_bound) { + enum { NESTED = 1024, BOUND = 100 }; + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry reg; + cbm_registry_init(®, &arena); + bool ok = true; + for (int i = NESTED - 1; i >= 0; i--) { + const char *qn = cbm_arena_sprintf(&arena, "p.foo.A.n%04d", i); + ok = qn && pylsp_range_add_type(®, qn, "Nested") && + pylsp_prefix_add(®, qn, "p.foo.A") && ok; + } + static const char *const direct[] = {"p.foo.B", "p.foo.A-b", "p.foo.AB"}; + for (size_t i = 0; i < sizeof(direct) / sizeof(direct[0]); i++) + ok = pylsp_range_add_type(®, direct[i], "Direct") && + pylsp_prefix_add(®, direct[i], NULL) && ok; + cbm_registry_build_type_qn_order(®, &arena); + cbm_registry_build_func_qn_order(®, &arena); + ok = reg.type_count == NESTED + 3 && reg.func_count == NESTED + 3 && reg.type_qn_sorted && + reg.func_qn_sorted && ok; + for (int types = 0; types < 2; types++) { + uint64_t indexed = 0, linear = 0, miss = 0; + ok = pylsp_range_oracle(®, types != 0, "p.foo.", true, &arena, &indexed) && + indexed > 0 && indexed <= BOUND && + pylsp_range_oracle(®, types != 0, "zzzz.", true, &arena, &miss) && miss > 0 && + miss <= BOUND && ok; + cbm_registry_test_qn_prefix_block_fail_after(0); + ok = pylsp_range_oracle(®, types != 0, "p.foo.", true, &arena, &linear) && + linear >= NESTED + 3 && linear > BOUND && ok; + cbm_registry_test_qn_prefix_block_fail_after(-1); + } + cbm_arena_destroy(&arena); + ASSERT_TRUE(ok); + PASS(); +} + +/* Test scratch storage must be gone before these retained bindings are read. */ +TEST(pylsp_range_class_binding_keeps_order_shadow_and_package_filters) { + static const char source[] = "class Local:\n pass\n"; + TSParser *parser = ts_parser_new(); + bool language = parser && ts_parser_set_language(parser, tree_sitter_python()); + TSTree *tree = + language ? ts_parser_parse_string(parser, NULL, source, (uint32_t)strlen(source)) : NULL; + bool ok = tree && !ts_node_has_error(ts_tree_root_node(tree)); + for (int package = 0; tree && package < 2; package++) { + for (int failure = 0; failure < 6; failure++) { + CBMArena owner, arena; + cbm_arena_init(&owner); + cbm_arena_init(&arena); + CBMTypeRegistry head, middle, tail; + cbm_registry_init(&head, &owner); + cbm_registry_init(&middle, &owner); + cbm_registry_init(&tail, &owner); + head.fallback = &middle; + middle.fallback = &tail; + bool built = pylsp_range_add_type(&head, "test.pkg.Head", "Shared") && + pylsp_range_add_type(&head, "test.pkg.Same", "HeadSame") && + pylsp_range_add_type(&head, "test.pkg.Local", "Local") && + pylsp_range_add_type(&middle, "test.pkg.Middle", "Shared") && + pylsp_range_add_type(&middle, "test.pkg.Same", "Masked") && + pylsp_range_add_type(&middle, "test.pkg.Deep", "MiddleOnly") && + pylsp_range_add_type(&tail, "test.pkg.Tail", "Shared") && + pylsp_range_add_type(&tail, "test.pkg.Deep", "TailOnly") && + pylsp_range_add_type(&tail, "test.pkg.sub.Hidden", "Hidden"); + if (failure == 5) + cbm_registry_test_type_qn_order_fail_after(0); + CBMTypeRegistry *links[] = {&head, &middle, &tail}; + for (int i = 0; i < 3; i++) + cbm_registry_build_type_qn_order(links[i], &owner); + cbm_registry_test_type_qn_order_fail_after(-1); + if (failure != 5) + built = + head.type_qn_sorted && middle.type_qn_sorted && tail.type_qn_sorted && built; + else + built = + !head.type_qn_sorted && !middle.type_qn_sorted && !tail.type_qn_sorted && built; + /* The two fallback Deep entries remain visible: only the head shadows. */ + bool shadowed = built && cbm_registry_type_shadowed(&head, &middle, 1) && + !cbm_registry_type_shadowed(&head, &tail, 1) && + !cbm_registry_type_shadowed(&head, &head, 1); + PyLSPContext ctx; + CBMResolvedCallArray out = {0}; + py_lsp_init(&ctx, &arena, source, (int)strlen(source), &head, + package ? "test.pkg.__init__" : "test.pkg", &out); + if (failure == 1 || failure == 2) + cbm_registry_test_qn_prefix_block_fail_after(failure - 1); + if (failure == 3) + cbm_py_test_module_prefix_fail_after(0); + if (failure == 4) + cbm_py_test_module_scratch_fail_after(0); + cbm_py_lsp_test_bind_external_classes(&ctx, ts_tree_root_node(tree)); + pylsp_range_reset_failures(); + ok = shadowed && !ctx.callable_value_proof_disabled && + ctx.eval_failure == CBM_LSP_COMPLETE && + pylsp_init_named_type(cbm_scope_lookup_local(ctx.current_scope, "Shared"), + "test.pkg.Tail") && + pylsp_init_named_type(cbm_scope_lookup_local(ctx.current_scope, "HeadSame"), + "test.pkg.Same") && + pylsp_init_named_type(cbm_scope_lookup_local(ctx.current_scope, "MiddleOnly"), + "test.pkg.Deep") && + pylsp_init_named_type(cbm_scope_lookup_local(ctx.current_scope, "TailOnly"), + "test.pkg.Deep") && + !cbm_scope_lookup_local(ctx.current_scope, "Masked") && + !cbm_scope_lookup_local(ctx.current_scope, "Local") && ok; + const CBMType *hidden = cbm_scope_lookup_local(ctx.current_scope, "Hidden"); + ok = + (package ? hidden == NULL : pylsp_init_named_type(hidden, "test.pkg.sub.Hidden")) && + ok; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&owner); + } + } + pylsp_range_reset_failures(); + if (tree) + ts_tree_delete(tree); + if (parser) + ts_parser_delete(parser); + ASSERT_TRUE(ok); + PASS(); +} + +TEST(pylsp_range_wildcard_and_scratch_failures_keep_exact_later_calls) { + static const char source[] = "external = External()\nexternal.ping()\n" + "from vendor import *\nshadow_only(72)\nExternal(74)\n" + "def later(v):\n return v\nlater(73)\n"; + CBMFileResult *raw = extract_py(source); + TSParser *parser = ts_parser_new(); + bool language = parser && ts_parser_set_language(parser, tree_sitter_python()); + TSTree *tree = + language ? ts_parser_parse_string(parser, NULL, source, (uint32_t)strlen(source)) : NULL; + bool healthy = raw && !raw->has_error && !raw->parse_incomplete && !raw->lsp_skipped && tree && + !ts_node_has_error(ts_tree_root_node(tree)) && + pylsp_module_has_def(raw, "test.main.later"); + bool ok = healthy; + for (int failure = 0; healthy && failure < 8; failure++) { + CBMArena owner, arena; + cbm_arena_init(&owner); + cbm_arena_init(&arena); + CBMTypeRegistry head, tail; + cbm_registry_init(&head, &owner); + cbm_registry_init(&tail, &owner); + head.fallback = &tail; + bool built = pylsp_range_add_type(&head, "test.main.External", "External") && + pylsp_range_add_type(&tail, "test.main.Other", "Other"); + cbm_registry_add_func(&head, + (CBMRegisteredFunc){.qualified_name = "test.main.External.ping", + .short_name = "ping", + .receiver_type = "test.main.External"}); + cbm_registry_add_func( + &head, (CBMRegisteredFunc){.qualified_name = "test.main.later", .short_name = "later"}); + /* A head method must not suppress invalidation of a fallback free function. */ + cbm_registry_add_func(&head, (CBMRegisteredFunc){.qualified_name = "test.main.hidden", + .short_name = "hidden", + .receiver_type = "test.main.External"}); + cbm_registry_add_func(&tail, (CBMRegisteredFunc){.qualified_name = "test.main.hidden", + .short_name = "shadow_only"}); + cbm_registry_build_type_qn_order(&head, &owner); + cbm_registry_build_type_qn_order(&tail, &owner); + cbm_registry_build_func_qn_order(&head, &owner); + cbm_registry_build_func_qn_order(&tail, &owner); + built = built && head.func_count == 3 && tail.func_count == 1 && head.type_qn_sorted && + tail.type_qn_sorted && head.func_qn_sorted && tail.func_qn_sorted; + head.read_only = true; + tail.read_only = true; + PyLSPContext ctx; + CBMResolvedCallArray out = {0}; + py_lsp_init(&ctx, &arena, source, (int)strlen(source), &head, "test.main", &out); + if (failure >= 1 && failure <= 4) + cbm_registry_test_qn_prefix_block_fail_after(failure - 1); + if (failure == 5) + cbm_py_test_module_prefix_fail_after(0); + if (failure == 6) + cbm_py_test_module_scratch_fail_after(0); + if (failure == 7) + cbm_py_test_module_prefix_fail_after(1); + uint64_t before = + atomic_load_explicit(&cbm_py_wildcard_invalidations, memory_order_relaxed); + py_lsp_process_file(&ctx, ts_tree_root_node(tree)); + pylsp_range_reset_failures(); + const CBMType *shadow = cbm_scope_lookup_local(ctx.current_scope, "shadow_only"); + ok = built && ctx.eval_failure == CBM_LSP_COMPLETE && !ctx.callable_value_proof_disabled && + shadow && cbm_type_is_unknown(shadow) && + atomic_load_explicit(&cbm_py_wildcard_invalidations, memory_order_relaxed) > before && + pylsp_module_site_joins(&raw->calls, &out, source, "external.ping()", "test.main", + "test.main.External.ping") && + pylsp_module_site_joins(&raw->calls, &out, source, "later(73)", "test.main", + "test.main.later") && + pylsp_prefix_site_unresolved(&raw->calls, &out, source, "shadow_only(72)", + "test.main") && + pylsp_prefix_site_unresolved(&raw->calls, &out, source, "External(74)", "test.main") && + ok; + cbm_arena_destroy(&arena); + cbm_arena_destroy(&owner); + } + pylsp_range_reset_failures(); + if (tree) + ts_tree_delete(tree); + if (parser) + ts_parser_delete(parser); + if (raw) + cbm_free_result(raw); + ASSERT_TRUE(ok); + PASS(); +} + SUITE(py_lsp) { + RUN_TEST(pylsp_range_order_boundaries_and_null_contract); + RUN_TEST(pylsp_range_growth_rebuild_seal_and_independent_failure); + RUN_TEST(pylsp_range_invalid_snapshot_uses_complete_scan); + RUN_TEST(pylsp_range_block_failure_preserves_each_link_and_tail); + RUN_TEST(pylsp_range_direct_subtree_pruning_has_absolute_bound); + RUN_TEST(pylsp_range_class_binding_keeps_order_shadow_and_package_filters); + RUN_TEST(pylsp_range_wildcard_and_scratch_failures_keep_exact_later_calls); + + RUN_TEST(pylsp_prefix_order_matches_iterator_and_chain_shadowing); + RUN_TEST(pylsp_prefix_order_survives_array_growth_rebuild_and_seal); + RUN_TEST(pylsp_prefix_order_allocation_failure_preserves_oracle); + RUN_TEST(pylsp_prefix_order_has_absolute_comparison_bound); + RUN_TEST(pylsp_prefix_order_preserves_exact_selection_and_ambiguity); + RUN_TEST(pylsp_prefix_raw_per_file_and_shared_keep_exact_calls); + + RUN_TEST(pylsp_classbody_calls_join_extracted_caller); + RUN_TEST(pylsp_classbody_names_visible_to_later_statements_only); + RUN_TEST(pylsp_decorator_and_base_calls_join_extracted_caller); + RUN_TEST(pylsp_nested_and_local_classbody_calls_join_extracted_caller); + RUN_TEST(pylsp_class_comprehensions_separate_iterable_and_body_scopes); + RUN_TEST(pylsp_class_legacy_dispatch_cannot_bypass_or_publish_shadows); + RUN_TEST(pylsp_class_frame_failure_restores_context_and_stops); + RUN_TEST(pylsp_class_frame_failure_rolls_back_raw_and_cross_outputs); + + RUN_TEST(pylsp_init_scope_context_and_helper); + RUN_TEST(pylsp_init_scope_raw_exact_occurrences_and_annotations); + RUN_TEST(pylsp_init_scope_cross_signatures_keep_definition_language); + RUN_TEST(pylsp_init_scope_external_classes_and_cross_exact_joins); + RUN_TEST(pylsp_init_scope_failure_preserves_outputs_and_wrappers); + RUN_TEST(pylsp_init_scope_shared_failure_keeps_arena_and_needed_fallback); + + RUN_TEST(pylsp_module_callers_join_exact_raw_occurrences); + RUN_TEST(pylsp_module_named_lambda_keeps_exact_dunder_pair); + RUN_TEST(pylsp_module_cross_callers_own_input_and_join_exact_sites); + RUN_TEST(pylsp_issue1527_depth_failure_reports_all_entry_points); + RUN_TEST(pylsp_issue1527_cross_memo_failure_fallback_reports_error); + RUN_TEST(pylsp_issue1527_cross_memo_failure_preserves_completed_rows); + RUN_TEST(pylsp_issue1527_root_class_name_failure_binds_no_external_class); + RUN_TEST(pylsp_issue1527_memo_and_depth_failures_stop_work); + RUN_TEST(pylsp_issue1527_memo_allocation_failure_is_reported); + RUN_TEST(pylsp_scale_work_is_linear); /* Phase 2 — smoke */ RUN_TEST(pylsp_smoke_empty); RUN_TEST(pylsp_smoke_one_function); @@ -2193,6 +4693,8 @@ SUITE(py_lsp) { RUN_TEST(pylsp_pep695_generic_class); /* Phase 9 — cross-file + batch */ RUN_TEST(pylsp_crossfile_method_dispatch); + RUN_TEST(pylsp_crossfile_receiver_through_typed_field); + RUN_TEST(pylsp_crossfile_typed_field_shared_registry); RUN_TEST(pylsp_fused_self_attr_chain_via_overlay); RUN_TEST(pylsp_crossfile_classmethod_on_class_issue228); RUN_TEST(pylsp_crossfile_inheritance); @@ -2251,5 +4753,6 @@ SUITE(py_lsp) { /* Evaluator guards — #710/#720 (distilled PRs #732 + #758) */ RUN_TEST(pylsp_issue710_deep_call_chain_resolves); RUN_TEST(pylsp_issue710_heterogeneous_receiver_chain); - RUN_TEST(pylsp_eval_steps_budget_degrades_gracefully); + RUN_TEST(pylsp_issue1527_chain_statements_resolve_completely); + RUN_TEST(pylsp_issue1527_large_module_resolves_every_call); } diff --git a/tests/test_py_lsp_scale.c b/tests/test_py_lsp_scale.c index f20e76275..1982e0623 100644 --- a/tests/test_py_lsp_scale.c +++ b/tests/test_py_lsp_scale.c @@ -1,7 +1,7 @@ /* * test_py_lsp_scale.c — measure scaling behavior at 100 / 500 / 2000 - * classes-and-calls. Asserts that doubling the input doesn't more than - * 4x the runtime (catches accidental O(n^2) in the resolver). + * classes-and-calls. Timing is diagnostic; deterministic work is gated in + * the normal py_lsp suite. */ #include "test_framework.h" #include "cbm.h" @@ -88,26 +88,10 @@ TEST(pylsp_scale_linear_growth) { ASSERT(r_pct_100 > 0.5); ASSERT(r_pct_2000 > 0.5); - /* Quadratic-growth detector. 20x input: linear ~20-30x time, clear - * quadratic ~400x. The bound sits at 200x — mid-way in log space — so a - * real quadratic regression still fails by 2x while the CURRENT, KNOWN - * superlinear resolve curve does not produce false release blocks. - * - * The old bound of 100x was calibrated as "linear plus generous overhead", - * but the resolve path has never been linear here: measured 2026-08-11 - * (sanitized builds, deterministic across runs) — quiet arm64 host - * 57-68x, macos-15-intel CI runner 101.1x, per-function cost growing - * 0.5ms -> 1.7ms from 100 to 2000 functions. That ~O(n^1.4-1.5) curve is - * the audited short-name-lookup/negative-memo gap, tracked as #1527; this - * detector was flagging host CONSTANTS, not a complexity change. When - * #1527 lands, tighten this back down (~40x holds linear honestly). */ - if (t100 > 0.5) { // skip when t100 too small to compare reliably - double ratio = t2000 / t100; - printf(" scale ratio 2000/100: %.1fx (linear ~20x, known-superlinear ~60-100x, " - "quadratic ~400x)\n", - ratio); - ASSERT(ratio < 200.0); // flags clear quadratic; #1527 tracks the curve itself - } + /* Timing is diagnostic only. The normal py_lsp suite gates deterministic + * resolver work and complete resolution independently of host scheduling. */ + if (t100 > 0.0) + printf(" diagnostic scale ratio 2000/100: %.1fx\n", t2000 / t100); PASS(); } diff --git a/tests/test_registry.c b/tests/test_registry.c index 278e97d9b..4d305356c 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -1288,7 +1288,277 @@ TEST(resolve_import_map_alias_with_suffix_hits_method) { PASS(); } +/* Python package scope is an explicit resolver input, not a property inferred + * from a QN shared by other languages. Keep these fixtures separate from the + * raw public-API tests above. */ +static bool registry_scope_match(cbm_resolution_t result, const char *qn, const char *strategy) { + return qn && strategy && result.qualified_name && result.strategy && + strcmp(result.qualified_name, qn) == 0 && strcmp(result.strategy, strategy) == 0 && + result.candidate_count == 1; +} + +TEST(python_symbol_scope_len_preserves_file_identity) { + const char *inputs[] = {"proj.pkg.__init__", + "proj.__init__", + "proj.pkg.__init__.__init__", + "proj.pkg.mod", + "proj.pkg.x__init__", + "__init__", + ".__init__", + "proj.pkg.index", + "", + NULL}; + const char *scopes[] = {"proj.pkg", + "proj", + "proj.pkg.__init__", + "proj.pkg.mod", + "proj.pkg.x__init__", + "__init__", + ".__init__", + "proj.pkg.index", + "", + ""}; + bool lengths = true; + for (size_t i = 0; i < sizeof(inputs) / sizeof(inputs[0]); i++) { + lengths = cbm_fqn_symbol_scope_len(inputs[i]) == strlen(scopes[i]) && lengths; + } + char *module = cbm_pipeline_fqn_module("proj", "pkg/__init__.py"); + char *symbol = cbm_pipeline_fqn_compute("proj", "pkg/__init__.py", "Field"); + bool identities = module && symbol && strcmp(module, "proj.pkg.__init__") == 0 && + strcmp(symbol, "proj.pkg.Field") == 0 && + cbm_fqn_symbol_scope_len(module) == strlen("proj.pkg"); + free(module); + free(symbol); + ASSERT_TRUE(lengths && identities); + PASS(); +} + +TEST(registry_python_package_scope_is_explicit) { + cbm_registry_t *r = cbm_registry_new(); + ASSERT_NOT_NULL(r); + cbm_registry_add(r, "Field", "proj.forms.fields.Field", "Class"); + cbm_registry_add(r, "Field", "proj.db.fields.Field", "Class"); + cbm_registry_add(r, "Field", "proj.db.fields.__init__.Field", "Class"); + cbm_registry_add(r, "Field", "proj.db.plain.Field", "Class"); + cbm_registry_add(r, "Field", "proj.db.index.Field", "Class"); + const char *keys[] = {"forms"}; + const char *vals[] = {"proj.forms"}; + const char *module = "proj.db.fields.__init__"; + bool seeded = cbm_registry_size(r) == 5; + cbm_resolution_t package = + cbm_registry_resolve_lang(r, "Field", module, keys, vals, 1, CBM_LANG_PYTHON); + bool python = registry_scope_match(package, "proj.db.fields.Field", "same_module"); + cbm_resolution_t plain = + cbm_registry_resolve_lang(r, "Field", "proj.db.plain", keys, vals, 1, CBM_LANG_PYTHON); + bool normal = registry_scope_match(plain, "proj.db.plain.Field", "same_module"); + cbm_resolution_t raw = cbm_registry_resolve(r, "Field", module, keys, vals, 1); + bool preserved = registry_scope_match(raw, "proj.db.fields.__init__.Field", "same_module"); + const CBMLanguage raw_languages[] = {CBM_LANG_JAVA, CBM_LANG_GO, CBM_LANG_JAVASCRIPT, + (CBMLanguage)-1, CBM_LANG_COUNT}; + for (size_t i = 0; i < sizeof(raw_languages) / sizeof(raw_languages[0]); i++) { + cbm_resolution_t result = + cbm_registry_resolve_lang(r, "Field", module, keys, vals, 1, raw_languages[i]); + preserved = registry_scope_match(result, raw.qualified_name, "same_module") && + result.confidence == raw.confidence && preserved; + } + cbm_resolution_t index = + cbm_registry_resolve_lang(r, "Field", "proj.db.index", keys, vals, 1, CBM_LANG_JAVASCRIPT); + bool index_raw = registry_scope_match(index, "proj.db.index.Field", "same_module"); + cbm_registry_free(r); + ASSERT_TRUE(seeded && python && normal && preserved && index_raw); + PASS(); +} + +TEST(registry_python_scope_keeps_import_precedence_and_raw_scoring) { + cbm_registry_t *r = cbm_registry_new(); + ASSERT_NOT_NULL(r); + cbm_registry_add(r, "Field", "proj.vendor.Field", "Class"); + cbm_registry_add(r, "Field", "proj.pkg.Field", "Class"); + cbm_registry_add(r, "Field", "proj.pkg.__init__.Field", "Class"); + cbm_registry_add(r, "choose", "proj.pkg.__init__.child.choose", "Function"); + cbm_registry_add(r, "choose", "proj.pkg.peer.choose", "Function"); + const char *keys[] = {"Field"}; + const char *vals[] = {"proj.vendor.Field"}; + bool seeded = cbm_registry_size(r) == 5; + cbm_resolution_t imported = + cbm_registry_resolve_lang(r, "Field", "proj.pkg.__init__", keys, vals, 1, CBM_LANG_PYTHON); + bool precedence = registry_scope_match(imported, "proj.vendor.Field", "import_map"); + cbm_resolution_t raw = cbm_registry_resolve(r, "choose", "proj.pkg.__init__", NULL, NULL, 0); + cbm_resolution_t python = + cbm_registry_resolve_lang(r, "choose", "proj.pkg.__init__", NULL, NULL, 0, CBM_LANG_PYTHON); + bool scoring = + raw.qualified_name && python.qualified_name && raw.strategy && python.strategy && + strcmp(raw.qualified_name, "proj.pkg.__init__.child.choose") == 0 && + strcmp(python.qualified_name, raw.qualified_name) == 0 && + strcmp(raw.strategy, "suffix_match") == 0 && strcmp(python.strategy, raw.strategy) == 0 && + python.confidence == raw.confidence && python.candidate_count == raw.candidate_count; + cbm_registry_free(r); + ASSERT_TRUE(seeded && precedence && scoring); + PASS(); +} + +TEST(registry_python_scope_cache_modes_and_raw_lineage) { + cbm_registry_t *r = cbm_registry_new(); + ASSERT_NOT_NULL(r); + cbm_registry_add(r, "Entry", "proj.pkg.Entry", "Function"); + cbm_registry_add(r, "Entry", "proj.pkg.__init__.Entry", "Function"); + cbm_registry_add(r, "Reject", "proj.pkg.Reject", "Table"); + cbm_registry_add(r, "Reject", "proj.pkg.__init__.Reject", "Function"); + cbm_registry_add(r, "Accept", "proj.pkg.Accept", "Function"); + cbm_registry_add(r, "Accept", "proj.pkg.__init__.Accept", "Table"); + cbm_registry_add(r, "Relation", "proj.pkg.Relation", "View"); + cbm_registry_add(r, "Relation", "proj.pkg.__init__.Relation", "Table"); + bool seeded = cbm_registry_size(r) == 8; + bool correct = true; + /* Each cache lifetime has one fixed module/import map. Alternate modes in + * both starting orders; repeated same-mode requests also exercise hits. */ + for (int first_python = 0; first_python < 2; first_python++) { + cbm_registry_resolve_cache_begin(8); + for (int turn = 0; turn < 16; turn++) { + bool python = ((turn / 2 + first_python) % 2) != 0; + const char *names[] = {"Entry", "Reject", "Accept", "Relation", "Missing"}; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + cbm_resolution_t result = + python ? cbm_registry_resolve_lang(r, names[i], "proj.pkg.__init__", NULL, NULL, + 0, CBM_LANG_PYTHON) + : cbm_registry_resolve(r, names[i], "proj.pkg.__init__", NULL, NULL, 0); + const char *expected = NULL; + if (i == 0) { + expected = python ? "proj.pkg.Entry" : "proj.pkg.__init__.Entry"; + } else if (i == 1 && !python) { + expected = "proj.pkg.__init__.Reject"; + } else if (i == 2 && python) { + expected = "proj.pkg.Accept"; + } + correct = + (expected ? registry_scope_match(result, expected, "same_module") + : result.qualified_name == NULL && result.candidate_count == 0) && + correct; + } + /* The cached relation veto must never poison the uncached SQL API, + * nor may the SQL result bypass the next default cached veto. */ + cbm_resolution_t lineage = + cbm_registry_resolve_lineage(r, "Relation", "proj.pkg.__init__", NULL, NULL, 0); + correct = registry_scope_match(lineage, "proj.pkg.__init__.Relation", "same_module") && + correct; + cbm_resolution_t veto = cbm_registry_resolve_lang(r, "Relation", "proj.pkg.__init__", + NULL, NULL, 0, CBM_LANG_PYTHON); + correct = veto.qualified_name == NULL && veto.candidate_count == 0 && correct; + } + cbm_registry_resolve_cache_end(); + } + cbm_registry_free(r); + ASSERT_TRUE(seeded && correct); + PASS(); +} + +static const char *registry_scope_canonical(cbm_registry_t *r, const char *name, const char *qn) { + const char **candidates = NULL; + int count = 0; + cbm_registry_find_by_name(r, name, &candidates, &count); + for (int i = 0; i < count; i++) { + if (strcmp(candidates[i], qn) == 0) { + return candidates[i]; + } + } + return NULL; +} + +TEST(registry_python_scope_complete_candidate_boundaries) { + const size_t lengths[] = {511, 512, 513, 1536}; + bool complete = true; + for (size_t i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) { + cbm_registry_t *r = cbm_registry_new(); + if (!r) { + complete = false; + break; + } + char scope[2048]; + char module[2048]; + char target[2048]; + const char *name = "LongTarget"; + size_t scope_len = lengths[i] - strlen(name) - 1; + memcpy(scope, "proj.", 5); + memset(scope + 5, 'a', scope_len - 5); + scope[scope_len] = '\0'; + int mn = snprintf(module, sizeof(module), "%s.__init__", scope); + int tn = snprintf(target, sizeof(target), "%s.%s", scope, name); + bool shape = mn > 0 && (size_t)mn < sizeof(module) && tn > 0 && (size_t)tn == lengths[i] && + (size_t)tn < sizeof(target); + cbm_registry_add(r, name, target, "Function"); + if (lengths[i] > 511) { + char truncated[512]; + memcpy(truncated, target, sizeof(truncated) - 1); + truncated[sizeof(truncated) - 1] = '\0'; + cbm_registry_add(r, "truncated_decoy", truncated, "Function"); + } + const char *canonical = registry_scope_canonical(r, name, target); + cbm_resolution_t python = + cbm_registry_resolve_lang(r, name, module, NULL, NULL, 0, CBM_LANG_PYTHON); + cbm_resolution_t normal = cbm_registry_resolve(r, name, scope, NULL, NULL, 0); + complete = shape && canonical && python.qualified_name == canonical && + normal.qualified_name == canonical && + registry_scope_match(python, target, "same_module") && + registry_scope_match(normal, target, "same_module") && complete; + cbm_registry_free(r); + } + ASSERT_TRUE(complete); + PASS(); +} + +TEST(registry_python_scope_qualified_terminal_bucket_uncapped) { + cbm_registry_t *r = cbm_registry_new(); + ASSERT_NOT_NULL(r); + /* The exact targets are inserted after the heuristic limit. A long exact + * comparison must use the canonical terminal bucket, not "Owner.perform" + * as the bucket key, and must not stop at 256 candidates. */ + bool shape = true; + for (int i = 0; i < 300; i++) { + char qn[80]; + int n = snprintf(qn, sizeof(qn), "proj.noise%d.perform", i); + shape = n > 0 && (size_t)n < sizeof(qn) && shape; + cbm_registry_add(r, "perform", qn, "Method"); + } + char scope[1024]; + memcpy(scope, "proj.", 5); + memset(scope + 5, 'b', 900); + scope[905] = '\0'; + char module[1024]; + int mn = snprintf(module, sizeof(module), "%s.__init__", scope); + shape = mn > 0 && (size_t)mn < sizeof(module) && shape; + const char *names[] = {"Owner.perform", "Owner::perform"}; + bool resolved = true; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + char target[1024]; + char prefixed[80]; + int tn = snprintf(target, sizeof(target), "%s.%s", scope, names[i]); + int pn = snprintf(prefixed, sizeof(prefixed), "receiver.%s", names[i]); + shape = tn > 0 && (size_t)tn < sizeof(target) && pn > 0 && (size_t)pn < sizeof(prefixed) && + shape; + cbm_registry_add(r, "perform", target, "Method"); + const char *canonical = registry_scope_canonical(r, "perform", target); + cbm_resolution_t direct = + cbm_registry_resolve_lang(r, names[i], module, NULL, NULL, 0, CBM_LANG_PYTHON); + cbm_resolution_t suffix = + cbm_registry_resolve_lang(r, prefixed, module, NULL, NULL, 0, CBM_LANG_PYTHON); + resolved = canonical && direct.qualified_name == canonical && + suffix.qualified_name == canonical && + registry_scope_match(direct, target, "same_module") && + registry_scope_match(suffix, target, "same_module") && resolved; + } + bool seeded = cbm_registry_size(r) == 302; + cbm_registry_free(r); + ASSERT_TRUE(shape && seeded && resolved); + PASS(); +} + SUITE(registry) { + RUN_TEST(python_symbol_scope_len_preserves_file_identity); + RUN_TEST(registry_python_package_scope_is_explicit); + RUN_TEST(registry_python_scope_keeps_import_precedence_and_raw_scoring); + RUN_TEST(registry_python_scope_cache_modes_and_raw_lineage); + RUN_TEST(registry_python_scope_complete_candidate_boundaries); + RUN_TEST(registry_python_scope_qualified_terminal_bucket_uncapped); /* FQN */ RUN_TEST(fqn_simple); RUN_TEST(fqn_no_name);