From 5d9abdfd7ce1390ff34b33ce49f60a6a803e6f47 Mon Sep 17 00:00:00 2001 From: Eric Minish Date: Wed, 23 Sep 2026 12:07:50 -0400 Subject: [PATCH 1/5] fix(python): resolve imports from the package roots the project has (#1704) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python absolute specifiers never went through resolveImportPath. Every caller bolted on a suffix match against any `…/a/b/c.py`, which bound `import json` to a project's `app/utils/json.py` (and `yaml`, `requests`, `openai` to any same-named nested file) and picked an arbitrary twin when two services both carry `core/models.py`. Python named imports couldn't resolve through the import at all: Python nodes are never isExported, and re-exports were JS-only. One resolver now answers every Python specifier, relative and absolute, from two kinds of evidence-derived root: - project roots: the repo root, and each service with its own pyproject.toml/setup.cfg/setup.py, plus its `src/` and the package dirs its config declares (read from the right tables, nested configs included). These may answer any importer, but only unambiguously. A service inside a package, or under a directory named as non-project code (tests, fixtures, examples, vendor, ...), only answers within that directory. - local roots: the parent of each top-level regular package, a script's own directory, and a namespace package seen from inside it. These answer only importers inside them, so a fixture's `tests/fixtures/proj/requests/` never becomes the application's `requests`. A containing root answers first. Among other roots, a regular package or module shadows a namespace portion. In one directory `pkg/__init__.py` beats `pkg.py`, and a dotted path is walked level by level, so a plain module ends it. A relative import can't climb above its top-level package. `from P import N` looks N up on the package first, as CPython's `_handle_fromlist` does: a def or re-export of N in `P/__init__.py` beats a submodule `P/N`, which answers only when the package doesn't bind N. Module-level defs are importable without an export flag, and a module's `from X import …` lines are its re-exports, chased with the existing cycle guard and depth cap. `from m import *` honours a literal `__all__`, and otherwise binds only public names. Import parsing reads parenthesised lists, continuations, `import a, b` and indented imports over comment- and docstring-blanked source. An attribute of an imported function (`task.delay()`) keeps the unresolved answer it had before, rather than landing on the function. Sync: the Python root set (each root with its tier and reach) is fingerprinted in project_metadata. When it changes, every Python resolution edge is re-opened (reusing the CG-33 resurrect helper) and failed Python refs are requeued. When only modules or packages below the roots appear or vanish, sync re-opens just the edges into those module names, the edges out of a changed package, and the parked failures naming them. When a package's `__init__.py` is edited, it re-opens its importers' references to the names whose binding moved (a def or re-export added, removed or repointed). The narrow sets are resolved directly; either way sync converges to what a fresh index produces. Trade-off: a package nested under a directory named as non-project code (third_party/, vendor/, examples/, tests/, ...) doesn't answer imports from outside that directory, so a `third_party/foo` used through an editable install is unresolved: a missed edge rather than a fixture capturing the application's imports. --- CHANGELOG.md | 1 + __tests__/python-module-resolution.test.ts | 951 +++++++++++++++++++++ src/db/queries.ts | 205 +++++ src/extraction/index.ts | 91 ++ src/index.ts | 71 ++ src/resolution/import-resolver.ts | 883 ++++++++++++++----- src/resolution/index.ts | 51 +- 7 files changed, 2050 insertions(+), 203 deletions(-) create mode 100644 __tests__/python-module-resolution.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 4722c5fb64..064a3e8afa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -154,6 +154,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - In React with TypeScript, a component that renders a generic component and passes it type arguments, like outline's ` items={…} />` or excalidraw's ` …>`, now links to the component it renders. Before, a tag followed by type arguments was skipped, so the generic component looked unused by the screens that render it, and `codegraph_explore`'s call paths, callers and impact stopped short of it. Re-index React projects after upgrading. - In React, a component that a file declares itself as a value is now what that file's JSX renders, whatever else in the repository shares its name. A lazily loaded page, like `const RegisterPage = Loadable(lazy(() => import('./pages/Register')))`, a `dynamic(() => import(…))` component or one loaded only in development (`IS_DEV ? lazy(…) : null`), links to the component its module exports, including the one its loader picks and one an `export default observer(Login)` wraps; a wrapped component like `const Settings = observer(function Settings() {…})`, `React.memo(Rows, areEqual)` or `forwardRef(ComboboxInner) as {…}` links to the function it wraps; and an alias like `const DialogPortal = DialogPrimitive.Portal` links to its own declaration. Before, `` was linked to any lone component of that name elsewhere, often in another app of the same repository, so `codegraph_explore`, callers and impact followed it into the wrong app while the page it really renders had no caller. React Router layouts declared this way follow the same rule, and a TSX type named like a value the file declares now stays in that file. Re-index React projects after upgrading. - In React, a JSX tag imported from one of your own modules now links to the component that module exports under that name, read the way a lazily loaded page's module already is. That includes going through an `index` file or barrel that passes the component on, like outline's `Sidebar/index.ts` with `import Sidebar from './App'; export default Sidebar` or its `Avatar/index.ts` with `const Avatar = AvatarWithHoverCard`, and past a wrapper such as `export default observer(AppSidebar)`, `React.memo(StaticCanvas, areEqual)` or mastodon's `const MemoizedBlurhash = memo(Blurhash)`. Before, a tag was matched by its name and the import only broke ties, so outline's `` linked to an unrelated styled `Sidebar` in a sibling file, a default import under another name linked to whatever other component its module exported, and an import through a barrel could land on another app's component of the same name. A tag whose module exports an unnamed component, like `export default () => …`, now links to nothing instead of to a namesake elsewhere. A lazily loaded React Router page exported as such a value or wrapper links to the component it holds too, the one with the markup. `codegraph sync` and the file watcher redraw these links when only the file that passes the component on changes. Re-index React projects after upgrading. +- Python imports now resolve the way Python finds modules: from your project's package roots — the repository root, `src/` layouts, each service with its own `pyproject.toml`, `setup.cfg` or `setup.py`, and the package directories those declare — so `from shop.cart import total` reaches `src/shop/cart.py`, `from pkg import name` reaches what the package's `__init__.py` defines or re-exports (respecting `__all__`), and `import json` or `import requests` no longer links to a same-named file nested in a test fixture, an example, a vendored copy or another package. After a module, package or `__init__.py` is added or removed, a package's `__init__.py` is edited, or a build config is edited, the next sync brings Python imports back in line with a fresh index; the file watcher doesn't react to a build-config edit by itself, so run `codegraph sync` after one. Re-index Python projects after upgrading. (#1704) ## [1.6.2] - 2026-10-03 diff --git a/__tests__/python-module-resolution.test.ts b/__tests__/python-module-resolution.test.ts new file mode 100644 index 0000000000..10f89f64ca --- /dev/null +++ b/__tests__/python-module-resolution.test.ts @@ -0,0 +1,951 @@ +/** + * Python module-specifier resolution: one resolver for every Python import. + * + * An absolute dotted specifier (`pkg.sub.mod`) used to reach a file only + * through a suffix match against ANY `…/pkg/sub/mod.py` in the repo. That + * bound `import json` to a project's `app/utils/json.py`, and picked an + * arbitrary twin when two services both carry `core/models.py`. Named imports + * that an `__init__.py` re-exports got no edge at all. Each case below plants + * a same-named decoy, so only the import can pick the right target. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { CodeGraph } from '../src'; +import { extractImportMappings } from '../src/resolution/import-resolver'; + +describe('Python module resolution', () => { + let tempDir: string; + let cg: CodeGraph | undefined; + + beforeEach(() => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-pymod-')); + cg = undefined; + }); + + afterEach(() => { + if (cg) cg.destroy(); + else if (fs.existsSync(tempDir)) fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + function write(files: Record): void { + for (const [rel, content] of Object.entries(files)) { + const abs = path.join(tempDir, rel); + fs.mkdirSync(path.dirname(abs), { recursive: true }); + fs.writeFileSync(abs, content); + } + } + + /** `kind` edges leaving `fnName` in `file`, as `name@file` of each target. */ + function targets(fnName: string, file: string, kind: 'calls' | 'imports'): string[] { + const src = cg! + .getNodesInFile(file) + .find((n) => n.name === fnName && (n.kind === 'function' || n.kind === 'file')); + expect(src, `${fnName} in ${file}`).toBeDefined(); + return cg! + .getOutgoingEdges(src!.id) + .filter((e) => e.kind === kind) + .map((e) => cg!.getNode(e.target)) + .map((n) => `${n!.name}@${n!.filePath.replace(/\\/g, '/')}`); + } + + it('resolves an absolute dotted specifier to the module at a package root, not a same-named nested module', async () => { + write({ + 'pkg/__init__.py': '', + 'pkg/sub/__init__.py': '', + 'pkg/sub/mod.py': 'def run():\n return 1\n', + // `aaa.pkg.sub.mod` — ends in `pkg/sub/mod.py` but is not `pkg.sub.mod`. + 'aaa/__init__.py': '', + 'aaa/pkg/__init__.py': '', + 'aaa/pkg/sub/__init__.py': '', + 'aaa/pkg/sub/mod.py': 'def run():\n return 2\n', + 'app/__init__.py': '', + 'app/main.py': `import pkg.sub.mod +from pkg.sub.mod import run + + +def qualified(): + pkg.sub.mod.run() + + +def named(): + run() +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('qualified', 'app/main.py', 'calls')).toEqual(['run@pkg/sub/mod.py']); + expect(targets('named', 'app/main.py', 'calls')).toEqual(['run@pkg/sub/mod.py']); + expect(targets('main.py', 'app/main.py', 'imports')).toContain('mod.py@pkg/sub/mod.py'); + expect(targets('main.py', 'app/main.py', 'imports')).not.toContain('mod.py@aaa/pkg/sub/mod.py'); + }); + + it('finds packages under a src/ layout from the __init__.py chain', async () => { + write({ + 'src/shop/__init__.py': '', + 'src/shop/cart.py': 'def total():\n return 1\n', + 'legacy/__init__.py': '', + 'legacy/shop/__init__.py': '', + 'legacy/shop/cart.py': 'def total():\n return 2\n', + 'tests/test_cart.py': `from shop.cart import total + + +def test_total(): + total() +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('test_total', 'tests/test_cart.py', 'calls')).toEqual(['total@src/shop/cart.py']); + }); + + it('prefers the monorepo service that contains the importer', async () => { + write({ + 'svc_a/core/__init__.py': '', + 'svc_a/core/models.py': 'def save():\n return 1\n', + 'svc_b/core/__init__.py': '', + 'svc_b/core/models.py': 'def save():\n return 2\n', + 'svc_b/core/views.py': `import core.models + + +def view(): + core.models.save() +`, + 'svc_a/core/views.py': `import core.models + + +def view(): + core.models.save() +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('views.py', 'svc_b/core/views.py', 'imports')).toContain('models.py@svc_b/core/models.py'); + expect(targets('views.py', 'svc_b/core/views.py', 'imports')).not.toContain('models.py@svc_a/core/models.py'); + expect(targets('views.py', 'svc_a/core/views.py', 'imports')).toContain('models.py@svc_a/core/models.py'); + expect(targets('views.py', 'svc_a/core/views.py', 'imports')).not.toContain('models.py@svc_b/core/models.py'); + expect(targets('view', 'svc_b/core/views.py', 'calls')).toEqual(['save@svc_b/core/models.py']); + expect(targets('view', 'svc_a/core/views.py', 'calls')).toEqual(['save@svc_a/core/models.py']); + }); + + it('draws no edge when two package roots both provide the module and neither contains the importer', async () => { + write({ + 'svc_a/core/__init__.py': '', + 'svc_a/core/models.py': 'def save():\n return 1\n', + 'svc_b/core/__init__.py': '', + 'svc_b/core/models.py': 'def save():\n return 2\n', + 'tools/check.py': `import core.models + + +def check(): + core.models.save() +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + // The import names no module this file can prove. (The `save()` call is + // emitted as a bare name, which the name matcher still answers on its own.) + const intoServices = (t: string) => t.includes('@svc_'); + expect(targets('check.py', 'tools/check.py', 'imports').filter(intoServices)).toEqual([]); + }); + + it('reads package roots declared in pyproject.toml, for namespace packages without __init__.py', async () => { + write({ + 'pyproject.toml': '[tool.setuptools.packages.find]\nwhere = ["lib"]\n', + 'lib/nsp/tools.py': 'def go():\n return 1\n', + 'vendor/__init__.py': '', + 'vendor/nsp/tools.py': 'def go():\n return 2\n', + 'tests/test_tools.py': `import nsp.tools + + +def test_go(): + nsp.tools.go() +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('test_go', 'tests/test_tools.py', 'calls')).toEqual(['go@lib/nsp/tools.py']); + }); + + it('resolves a namespace package from inside itself', async () => { + write({ + 'backend/myapp/models.py': 'def fetch():\n return 1\n', + 'backend/myapp/views.py': `from myapp.models import fetch + + +def index(): + fetch() +`, + 'other/__init__.py': '', + 'other/myapp/__init__.py': '', + 'other/myapp/models.py': 'def fetch():\n return 2\n', + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('index', 'backend/myapp/views.py', 'calls')).toEqual(['fetch@backend/myapp/models.py']); + }); + + it('resolves a package import to its __init__.py', async () => { + write({ + 'shop/__init__.py': 'def checkout():\n return 1\n', + 'misc/__init__.py': '', + 'misc/tools.py': 'def checkout():\n return 2\n', + 'main.py': `import shop + + +def buy(): + shop.checkout() +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('buy', 'main.py', 'calls')).toEqual(['checkout@shop/__init__.py']); + expect(targets('main.py', 'main.py', 'imports')).toContain('__init__.py@shop/__init__.py'); + }); + + it('follows __init__.py re-exports: relative, absolute, aliased, parenthesised and star', async () => { + write({ + 'shop/__init__.py': `from .billing.invoice import make_invoice +from shop.billing.tax import compute_tax as tax +from .cart import ( + add_item, # the common multi-line form + remove_item, +) +from .money import * +`, + 'shop/billing/__init__.py': '', + 'shop/billing/invoice.py': 'def make_invoice():\n return 1\n', + 'shop/billing/tax.py': 'def compute_tax():\n return 1\n', + 'shop/cart.py': 'def add_item():\n return 1\n\n\ndef remove_item():\n return 1\n', + 'shop/money.py': 'def to_cents():\n return 1\n', + // A same-named decoy for every re-exported name. + 'decoys/__init__.py': '', + 'decoys/all.py': `def make_invoice(): + return 2 + + +def compute_tax(): + return 2 + + +def tax(): + return 2 + + +def add_item(): + return 2 + + +def remove_item(): + return 2 + + +def to_cents(): + return 2 +`, + 'main.py': `from shop import make_invoice, tax, add_item, remove_item, to_cents + + +def order(): + make_invoice() + tax() + add_item() + remove_item() + to_cents() +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('order', 'main.py', 'calls').sort()).toEqual([ + 'add_item@shop/cart.py', + 'compute_tax@shop/billing/tax.py', + 'make_invoice@shop/billing/invoice.py', + 'remove_item@shop/cart.py', + 'to_cents@shop/money.py', + ]); + }); + + it('terminates on a re-export cycle and draws no edge', async () => { + write({ + 'a/__init__.py': 'from b import thing\n', + 'b/__init__.py': 'from a import thing\n', + 'main.py': `from a import thing + + +def use(): + thing() +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('use', 'main.py', 'calls')).toEqual([]); + }); + + it('never binds a stdlib or third-party import to a same-named module nested in the project', async () => { + write({ + 'app/__init__.py': '', + 'app/utils/__init__.py': '', + 'app/utils/json.py': 'def loads(s):\n return s\n', + 'app/utils/yaml.py': 'def safe_load(s):\n return s\n', + 'app/views.py': `import json +import yaml + + +def view(): + json.loads("1") + yaml.safe_load("x") +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('view', 'app/views.py', 'calls')).toEqual([]); + expect(targets('views.py', 'app/views.py', 'imports')).toEqual([]); + }); + + it('binds a top-level module the project really provides at a package root', async () => { + write({ + 'yaml.py': 'def safe_load(s):\n return s\n', + 'app/__init__.py': '', + 'app/views.py': `import yaml + + +def view(): + yaml.safe_load("x") +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('view', 'app/views.py', 'calls')).toEqual(['safe_load@yaml.py']); + }); + + it('keeps script-directory imports: a sibling module of a file outside any package', async () => { + write({ + 'scripts/helpers.py': 'def helper():\n return 1\n', + 'lib/__init__.py': '', + 'lib/helpers.py': 'def helper():\n return 2\n', + 'scripts/run.py': `import helpers + + +def main(): + helpers.helper() +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('main', 'scripts/run.py', 'calls')).toEqual(['helper@scripts/helpers.py']); + }); + + it('does not resolve a relative import that climbs above the top-level package', async () => { + write({ + 'helpers.py': 'def helper():\n return 1\n', + 'pkg/__init__.py': '', + 'pkg/mod.py': `from .. import helpers + + +def use(): + helpers.helper() +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('mod.py', 'pkg/mod.py', 'imports')).not.toContain('helpers.py@helpers.py'); + expect(targets('use', 'pkg/mod.py', 'calls')).toEqual([]); + }); + // Before this change a Python named import never reached the import + // resolver's member branch, so `send_welcome.delay()` stayed unresolved. + // Now that it does, it must keep that answer rather than land on the task. + it('keeps an attribute of an imported function unresolved, as before', async () => { + write({ + 'api/tasks.py': 'def send_welcome(item_id):\n return item_id\n', + 'api/items.py': `from .tasks import send_welcome + + +def create(): + send_welcome.delay(1) + send_welcome(2) +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + // `send_welcome.delay()` enqueues a Celery job; only the direct call is a call. + const edges = cg + .getOutgoingEdges(cg.getNodesInFile('api/items.py').find((n) => n.name === 'create')!.id) + .filter((e) => e.kind === 'calls'); + expect(edges.map((e) => e.line)).toEqual([6]); + }); + + it('never lets a package nested in a test fixture capture an import from outside it', async () => { + write({ + 'app/__init__.py': '', + 'app/client.py': `import requests + + +def fetch(): + requests.get("x") +`, + // A fixture project's own copy of a PyPI name — importable only by the + // fixture's own code, never by the application. + 'tests/fixtures/proj/requests/__init__.py': 'def get(u):\n return u\n', + 'tests/fixtures/proj/pyproject.toml': '[project]\nname = "proj"\n', + 'tests/fixtures/proj/use.py': `import requests + + +def use(): + requests.get("y") +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + const intoFixture = (t: string) => t.includes('@tests/fixtures/'); + expect(targets('client.py', 'app/client.py', 'imports').filter(intoFixture)).toEqual([]); + expect(targets('fetch', 'app/client.py', 'calls').filter(intoFixture)).toEqual([]); + // Inside its own subtree the fixture still resolves its own package. + expect(targets('use', 'tests/fixtures/proj/use.py', 'calls')).toEqual(['get@tests/fixtures/proj/requests/__init__.py']); + }); + + it('never lets a package nested inside another package capture an import from outside it', async () => { + write({ + 'mylib/__init__.py': '', + 'mylib/_vendor/__init__.py': '', + 'mylib/_vendor/six.py': 'def ensure_str(s):\n return s\n', + 'app/__init__.py': '', + 'app/main.py': `import six + + +def run(): + six.ensure_str("x") +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('run', 'app/main.py', 'calls')).toEqual([]); + }); + + it('treats src/ as a package root for a namespace package with no __init__.py and no build config', async () => { + write({ + 'src/shop/cart.py': 'def total():\n return 0\n', + 'legacy/cart.py': 'def total():\n return 9\n', + 'tests/test_cart.py': `from shop.cart import total + + +def test_it(): + total() +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('test_cart.py', 'tests/test_cart.py', 'imports')).toContain('cart.py@src/shop/cart.py'); + expect(targets('test_it', 'tests/test_cart.py', 'calls')).toEqual(['total@src/shop/cart.py']); + }); + + it("reads a service's own src/ layout and nested build configs, scoped to their tables", async () => { + write({ + // A monorepo service with a src/ layout and nothing at the repo root. + 'services/billing/pyproject.toml': '[project]\nname = "billing"\n', + 'services/billing/src/billing/__init__.py': '', + 'services/billing/src/billing/invoice.py': 'def make():\n return 1\n', + // A nested setup.cfg declaring its package directory. + 'services/auth/setup.cfg': '[metadata]\nname = auth\n\n[options]\npackage_dir =\n =lib\n', + 'services/auth/lib/authn/tokens.py': 'def issue():\n return 1\n', + // `where` under an unrelated table must not declare a root. + 'pyproject.toml': '[tool.other]\nwhere = ["decoy"]\n', + 'decoy/authn/tokens.py': 'def issue():\n return 2\n', + 'tools/run.py': `from billing.invoice import make +from authn.tokens import issue + + +def go(): + make() + issue() +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('go', 'tools/run.py', 'calls').sort()).toEqual([ + 'issue@services/auth/lib/authn/tokens.py', + 'make@services/billing/src/billing/invoice.py', + ]); + }); + + it('prefers a regular package over a same-named module, and a module over a namespace directory', async () => { + write({ + 'pkg.py': 'def f():\n return "module"\n', + 'pkg/__init__.py': 'def f():\n return "package"\n', + 'mod.py': 'def g():\n return "module"\n', + 'mod/g.py': 'def g():\n return "namespace"\n', + 'main.py': `from pkg import f +from mod import g + + +def go(): + f() + g() +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('go', 'main.py', 'calls').sort()).toEqual(['f@pkg/__init__.py', 'g@mod.py']); + }); + + it('honours __all__ on a star re-export, and skips _private names without one', async () => { + write({ + 'pkg/__init__.py': `from .impl import * +from .noall import * +from .priv import _priv +`, + 'pkg/impl.py': `__all__ = ["pub"] + + +def pub(): + return 1 + + +def hidden(): + return 2 + + +def _priv(): + return 3 +`, + 'pkg/noall.py': `def hidden(): + return 4 + + +def _priv(): + return 5 + + +def util(): + return 6 +`, + 'pkg/priv.py': 'def _priv():\n return 7\n', + 'main.py': `from pkg import pub, hidden, _priv, util + + +def go(): + pub() + hidden() + _priv() + util() +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('go', 'main.py', 'calls').sort()).toEqual([ + '_priv@pkg/priv.py', + 'hidden@pkg/noall.py', + 'pub@pkg/impl.py', + 'util@pkg/noall.py', + ]); + }); + + it('does not trust an __all__ assembled from other names, as django.db.models builds its own', async () => { + write({ + 'shop/__init__.py': 'from shop.models import *\n', + 'shop/models/__init__.py': `from shop.models.fields import * +from shop.models.fields import __all__ as fields_all + +__all__ = fields_all + ["Model"] +__all__ += ["Manager"] +`, + 'shop/models/fields.py': '__all__ = ["CharField"]\n\n\nclass CharField:\n pass\n', + 'decoy.py': 'class CharField:\n pass\n', + 'main.py': `from shop import CharField + + +def build(): + CharField() +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(cg.getOutgoingEdges(cg.getNodesInFile('main.py').find((n) => n.name === 'build')!.id) + .map((e) => cg!.getNode(e.target)!.filePath)).toEqual(['shop/models/fields.py']); + }); + + it("lets the importer's own tree answer before a same-named module next to it", async () => { + // A script run from `demo/` imports `utils.helpers`; `demo/utils/` also + // holds a `utils.py`, which must not shadow the package it sits in. + write({ + 'demo/run.py': 'from utils.helpers import go\n', + 'demo/utils/helpers.py': 'def go():\n return 1\n', + 'demo/utils/utils.py': 'def x():\n return 1\n', + 'demo/utils/runner.py': `from utils.helpers import go + + +def main(): + go() +`, + 'pkg/__init__.py': '', + 'pkg/utils.py': 'def go():\n return 2\n', + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('runner.py', 'demo/utils/runner.py', 'imports')).toContain('helpers.py@demo/utils/helpers.py'); + expect(targets('main', 'demo/utils/runner.py', 'calls')).toEqual(['go@demo/utils/helpers.py']); + }); + + describe('from P import N: the package attribute before the submodule', () => { + // CPython's `_handle_fromlist` looks N up on the package first and imports + // the submodule P.N only when the package has no such attribute. + it('binds a name the package __init__.py defines, not a same-named submodule', async () => { + write({ + 'pkg/__init__.py': 'def f():\n return 0\n', + 'pkg/f.py': 'X = 1\n', + 'main.py': 'from pkg import f\n\n\ndef main():\n f()\n', + }); + cg = await CodeGraph.init(tempDir, { index: true }); + expect(targets('main.py', 'main.py', 'imports')).toEqual(['f@pkg/__init__.py']); + }); + + it('binds it through a relative import too', async () => { + write({ + 'pkg/__init__.py': '', + 'pkg/sub/__init__.py': 'def f():\n return 0\n', + 'pkg/sub/f.py': 'X = 1\n', + 'pkg/a/__init__.py': '', + 'pkg/a/m.py': 'from ..sub import f\n\n\ndef main():\n f()\n', + }); + cg = await CodeGraph.init(tempDir, { index: true }); + expect(targets('m.py', 'pkg/a/m.py', 'imports')).toContain('f@pkg/sub/__init__.py'); + expect(targets('m.py', 'pkg/a/m.py', 'imports')).not.toContain('f.py@pkg/sub/f.py'); + }); + + it('binds a name the package re-exports, not a same-named subpackage', async () => { + write({ + 'pkg/__init__.py': 'from .impl import thing\n', + 'pkg/impl.py': 'def thing():\n return 1\n', + 'pkg/thing/__init__.py': '', + 'main.py': 'from pkg import thing\n\n\ndef main():\n thing()\n', + }); + cg = await CodeGraph.init(tempDir, { index: true }); + expect(targets('main.py', 'main.py', 'imports')).toEqual(['thing@pkg/impl.py']); + }); + + it('still binds the submodule when the package does not bind the name', async () => { + write({ + 'pkg/__init__.py': 'X = 1\n', + 'pkg/f.py': 'def g():\n return 1\n', + 'main.py': 'from pkg import f\n\n\ndef main():\n f.g()\n', + }); + cg = await CodeGraph.init(tempDir, { index: true }); + expect(targets('main.py', 'main.py', 'imports')).toEqual(['f.py@pkg/f.py']); + expect(targets('main', 'main.py', 'calls')).toEqual(['g@pkg/f.py']); + }); + }); + + describe('service roots under a shared test config', () => { + const services = (extra: Record) => ({ + 'services/billing/pyproject.toml': '[project]\nname = "billing"\n', + 'services/billing/src/billing/__init__.py': '', + 'services/billing/src/billing/models.py': 'def charge():\n return 1\n', + 'services/api/pyproject.toml': '[project]\nname = "api"\n', + 'services/api/src/api/__init__.py': '', + 'services/api/src/api/views.py': 'from billing.models import charge\n\n\ndef view():\n charge()\n', + ...extra, + }); + const linked = () => + expect(targets('views.py', 'services/api/src/api/views.py', 'imports')).toContain( + 'models.py@services/billing/src/billing/models.py' + ); + + it('keeps a service a project root when a shared conftest.py sits above it', async () => { + write(services({ 'services/conftest.py': 'import pytest\n' })); + cg = await CodeGraph.init(tempDir, { index: true }); + linked(); + }); + + it('keeps a service a project root when a smoke_test.py script sits above it', async () => { + write(services({ 'services/smoke_test.py': 'print(1)\n' })); + cg = await CodeGraph.init(tempDir, { index: true }); + linked(); + }); + + it('lets sibling projects under examples/ import each other, but not the code outside', async () => { + const moved = Object.fromEntries( + Object.entries(services({})).map(([k, v]) => [k.replace('services/', 'examples/'), v]) + ); + write({ + ...moved, + 'app/__init__.py': '', + 'app/main.py': 'from billing.models import charge\n\n\ndef run():\n charge()\n', + }); + cg = await CodeGraph.init(tempDir, { index: true }); + expect(targets('views.py', 'examples/api/src/api/views.py', 'imports')).toContain( + 'models.py@examples/billing/src/billing/models.py' + ); + expect(targets('main.py', 'app/main.py', 'imports')).not.toContain('models.py@examples/billing/src/billing/models.py'); + }); + }); + + describe('sync', () => { + /** Every Python calls/imports edge, as a sorted list of readable triples. */ + function pythonEdges(graph: CodeGraph): string[] { + const out: string[] = []; + for (const n of graph.getNodesByKind('function').concat(graph.getNodesByKind('file'))) { + if (!n.filePath.endsWith('.py')) continue; + for (const e of graph.getOutgoingEdges(n.id)) { + if (e.kind !== 'calls' && e.kind !== 'imports') continue; + const t = graph.getNode(e.target)!; + out.push(`${e.kind} ${n.name}@${n.filePath} -> ${t.kind}:${t.name}@${t.filePath}`); + } + } + return out.sort(); + } + + async function freshEdges(): Promise { + const copy = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-pymod-fresh-')); + try { + fs.cpSync(tempDir, copy, { recursive: true, filter: (src) => !src.includes('.codegraph') }); + const fresh = await CodeGraph.init(copy, { index: true }); + const edges = pythonEdges(fresh); + fresh.destroy(); + return edges; + } finally { + fs.rmSync(copy, { recursive: true, force: true }); + } + } + + it('re-resolves unchanged importers when an __init__.py changes where a package starts', async () => { + write({ + 'a/b/__init__.py': '', + 'a/helpers.py': 'def h():\n return 1\n', + 'a/b/mod.py': `from ..helpers import h + + +def use(): + h() +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + // `a/b` is the top-level package, so `..helpers` climbs out of it. + const before = pythonEdges(cg); + expect(before).not.toContain('imports mod.py@a/b/mod.py -> file:helpers.py@a/helpers.py'); + + write({ 'a/__init__.py': '' }); + await cg.sync(); + + const fresh = await freshEdges(); + expect(fresh).toContain('imports mod.py@a/b/mod.py -> file:helpers.py@a/helpers.py'); + expect(pythonEdges(cg)).toEqual(fresh); + }); + + it('re-resolves when pyproject.toml declares a new package root', async () => { + write({ + 'lib/nsp/tools.py': 'def go():\n return 1\n', + 'tests/test_tools.py': `import nsp.tools + + +def test_go(): + nsp.tools.go() +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + expect(pythonEdges(cg)).not.toContain('imports test_tools.py@tests/test_tools.py -> file:tools.py@lib/nsp/tools.py'); + + write({ 'pyproject.toml': '[tool.setuptools.packages.find]\nwhere = ["lib"]\n' }); + await cg.sync(); + + const fresh = await freshEdges(); + expect(fresh).toContain('imports test_tools.py@tests/test_tools.py -> file:tools.py@lib/nsp/tools.py'); + expect(pythonEdges(cg)).toEqual(fresh); + }); + + /** Spies on both re-open paths of the orchestrator. */ + function spyReopen(graph: CodeGraph) { + const orchestrator = (graph as unknown as { orchestrator: object }).orchestrator; + return { + all: vi.spyOn(orchestrator as { resurrectResolutionEdgesForLanguage: () => number }, 'resurrectResolutionEdgesForLanguage'), + narrow: vi.spyOn(orchestrator as { resurrectResolutionEdgesTouching: () => number }, 'resurrectResolutionEdgesTouching'), + }; + } + + const subpackageRepo = { + 'pkg/__init__.py': '', + 'pkg/core.py': 'def run():\n return 1\n', + 'app/__init__.py': '', + 'app/main.py': `import pkg.core +from pkg.fresh.mod import go + + +def main(): + pkg.core.run() + go() +`, + }; + + it('re-resolves only what a new subpackage touches, and matches a fresh index', async () => { + write(subpackageRepo); + cg = await CodeGraph.init(tempDir, { index: true }); + const spy = spyReopen(cg); + + write({ 'pkg/fresh/__init__.py': '', 'pkg/fresh/mod.py': 'def go():\n return 1\n' }); + await cg.sync(); + + const fresh = await freshEdges(); + expect(fresh).toContain('imports main.py@app/main.py -> file:mod.py@pkg/fresh/mod.py'); + expect(pythonEdges(cg)).toEqual(fresh); + expect(spy.all).not.toHaveBeenCalled(); + expect(spy.narrow).toHaveBeenCalledTimes(1); + }); + + it('converges when a subpackage loses its __init__.py and a same-named module takes over', async () => { + write({ + 'pkg/__init__.py': '', + 'pkg/sub.py': 'def f():\n return 1\n', + 'pkg/sub/__init__.py': '', + 'pkg/sub/x.py': 'def g():\n return 2\n', + 'main.py': 'import pkg.sub.x\n\n\ndef main():\n pkg.sub.x.g()\n', + }); + cg = await CodeGraph.init(tempDir, { index: true }); + expect(pythonEdges(cg)).toContain('imports main.py@main.py -> file:x.py@pkg/sub/x.py'); + + fs.rmSync(path.join(tempDir, 'pkg/sub/__init__.py')); + await cg.sync(); + + // `pkg.sub` is now the module `pkg/sub.py`, which has no `x`. + const fresh = await freshEdges(); + expect(fresh).not.toContain('imports main.py@main.py -> file:x.py@pkg/sub/x.py'); + expect(pythonEdges(cg)).toEqual(fresh); + }); + + it('converges when an __init__.py makes a package shadow the same-named module', async () => { + write({ + 'pkg/__init__.py': '', + 'pkg/sub.py': 'def f():\n return 1\n', + 'pkg/sub/g.py': 'def g():\n return 2\n', + 'main.py': 'import pkg.sub\n\n\ndef main():\n pkg.sub.f()\n', + }); + cg = await CodeGraph.init(tempDir, { index: true }); + expect(pythonEdges(cg)).toContain('imports main.py@main.py -> file:sub.py@pkg/sub.py'); + + write({ 'pkg/sub/__init__.py': '' }); + await cg.sync(); + + const fresh = await freshEdges(); + expect(fresh).toContain('imports main.py@main.py -> file:__init__.py@pkg/sub/__init__.py'); + expect(pythonEdges(cg)).toEqual(fresh); + }); + + it('keeps the package binding when a same-named submodule is added, and matches a fresh index', async () => { + write({ + 'pkg/__init__.py': 'def f():\n return 0\n', + 'main.py': 'from pkg import f\n\n\ndef main():\n f()\n', + }); + cg = await CodeGraph.init(tempDir, { index: true }); + const before = pythonEdges(cg); + + write({ 'pkg/f.py': 'X = 1\n' }); + await cg.sync(); + + const fresh = await freshEdges(); + expect(fresh).toEqual(before); + expect(pythonEdges(cg)).toEqual(fresh); + }); + + describe('an edited __init__.py moves what its importers bind', () => { + const SUB = 'def run():\n return "sub"\n'; + const MAIN = 'from pkg import N\n\n\ndef main():\n N()\n N.run()\n'; + const IMPL = 'class N:\n def run(self):\n return 1\n'; + + async function editAndCompare(initial: Record, edit: Record, expected: string) { + write(initial); + cg = await CodeGraph.init(tempDir, { index: true }); + write(edit); + await cg.sync(); + const fresh = await freshEdges(); + expect(fresh).toContain(expected); + expect(pythonEdges(cg)).toEqual(fresh); + } + + it('adding a def of N moves the import off the submodule (E1)', () => + editAndCompare( + { 'pkg/__init__.py': 'X = 1\n', 'pkg/N.py': SUB, 'main.py': MAIN }, + { 'pkg/__init__.py': 'X = 1\n\n\ndef N():\n return 0\n' }, + 'imports main.py@main.py -> function:N@pkg/__init__.py' + )); + + it('removing a def of N hands the import back to the submodule (E2)', () => + editAndCompare( + { 'pkg/__init__.py': 'X = 1\n\n\ndef N():\n return 0\n', 'pkg/N.py': SUB, 'main.py': MAIN }, + { 'pkg/__init__.py': 'X = 1\n' }, + 'calls main@main.py -> function:run@pkg/N.py' + )); + + it('removing a re-export of N hands the import back to the submodule (E4)', () => + editAndCompare( + { 'pkg/__init__.py': 'X = 1\nfrom .impl import N\n', 'pkg/N.py': SUB, 'pkg/impl.py': IMPL, 'main.py': MAIN }, + { 'pkg/__init__.py': 'X = 1\n' }, + 'imports main.py@main.py -> file:N.py@pkg/N.py' + )); + + it('repointing a re-export moves the importer with it (B1)', () => + editAndCompare( + { + 'pkg/__init__.py': 'from .impl import thing\n', + 'pkg/impl.py': 'def thing():\n return 1\n', + 'pkg/impl2.py': 'def thing():\n return 2\n', + 'main.py': 'from pkg import thing\n\n\ndef main():\n thing()\n', + }, + { 'pkg/__init__.py': 'from .impl2 import thing\n' }, + 'calls main@main.py -> function:thing@pkg/impl2.py' + )); + }); + + it('re-opens nothing on a sync that changes nothing', async () => { + write(subpackageRepo); + cg = await CodeGraph.init(tempDir, { index: true }); + const spy = spyReopen(cg); + + await cg.sync(); + + expect(spy.all).not.toHaveBeenCalled(); + expect(spy.narrow).not.toHaveBeenCalled(); + }); + + it('records a fingerprint for an index built before one existed, without re-resolving', async () => { + write(subpackageRepo); + cg = await CodeGraph.init(tempDir, { index: true }); + const queries = (cg as unknown as { + queries: { getMetadata(k: string): string | null; db: { prepare(sql: string): { run(...a: unknown[]): unknown } } }; + }).queries; + queries.db.prepare('DELETE FROM project_metadata WHERE key = ?').run('python_root_fingerprint'); + const spy = spyReopen(cg); + + write({ 'pyproject.toml': '[tool.setuptools.packages.find]\nwhere = ["lib"]\n' }); + await cg.sync(); + + expect(spy.all).not.toHaveBeenCalled(); + expect(queries.getMetadata('python_root_fingerprint')).toMatch(/^[0-9a-f]{40}$/); + }); + }); + + describe('import parsing', () => { + const names = (code: string) => + extractImportMappings('m.py', code, 'python').map((m) => `${m.localName}<-${m.source}:${m.exportedName}`); + + it('reads a parenthesised multi-line list, with comments and a trailing comma', () => { + expect(names('from .cart import (\n add_item, # note\n remove_item as rm,\n)\n')).toEqual([ + 'add_item<-.cart:add_item', + 'rm<-.cart:remove_item', + ]); + }); + + it('reads backslash continuations and comma-separated plain imports, at any indentation', () => { + expect(names('from a import b, \\\n c\nimport os, pkg.sub as s\ndef f():\n import lazy\n')).toEqual([ + 'b<-a:b', + 'c<-a:c', + 'os<-os:*', + 's<-pkg.sub:*', + 'lazy<-lazy:*', + ]); + }); + + it('ignores imports written inside a docstring or a comment', () => { + expect(names('"""\nfrom fake import thing\nimport other\n"""\n# from nope import x\nimport real\n')).toEqual([ + 'real<-real:*', + ]); + }); + }); +}); diff --git a/src/db/queries.ts b/src/db/queries.ts index 6421eec360..fbfa602faa 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -4397,6 +4397,211 @@ export class QueryBuilder { })); } + /** + * Every resolution edge leaving a node of `language` — the re-resolution set + * when a project-wide input to that language's import resolution changes + * (Python package roots). Same exclusion as + * {@link getResolutionEdgesByTargetName}: synthesized edges carry no refName + * stamp and are left alone. + */ + getResolutionEdgesBySourceLanguage( + language: Language + ): Array { + const rows = this.db + .prepare( + `SELECT e.*, src.file_path AS source_file_path, src.language AS source_language + FROM edges e + JOIN nodes src ON src.id = e.source + WHERE src.language = ? + AND e.kind != 'contains' + AND (e.provenance IS NULL OR e.provenance != 'heuristic')` + ) + .all(language) as Array; + return rows.map((row) => ({ + ...rowToEdge(row), + edgeId: row.id, + sourceFilePath: row.source_file_path, + sourceLanguage: row.source_language, + })); + } + + /** + * Resolution edges leaving a `language` node whose TARGET is in one of + * `targetFiles`, or whose SOURCE is under one of `sourceDirs` — the narrow + * re-resolution set when modules or packages appear or vanish below an + * unchanged set of Python roots. + */ + getResolutionEdgesTouchingFiles( + language: Language, + targetFiles: string[], + sourceDirs: string[], + importNodeSegments: string[] = [] + ): Array { + // Collect candidate edge ids through narrow indexed lookups, then load + // the full rows edge-first (CROSS JOIN pins the join order; left to + // itself the planner drove the query from the language index and one + // rename took 18 s). + const candidates = new Set(); + const idsWhere = (column: 'source' | 'target', nodeIds: string[]): void => { + for (let i = 0; i < nodeIds.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = nodeIds.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const rows = this.db + .prepare(`SELECT id FROM edges WHERE ${column} IN (${chunk.map(() => '?').join(',')})`) + .all(...chunk) as Array<{ id: number }>; + for (const r of rows) candidates.add(r.id); + } + }; + // Edges into the affected module files. + for (let i = 0; i < targetFiles.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = targetFiles.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const nodes = this.db + .prepare(`SELECT id FROM nodes WHERE file_path IN (${chunk.map(() => '?').join(',')})`) + .all(...chunk) as Array<{ id: string }>; + idsWhere('target', nodes.map((n) => n.id)); + } + // Edges out of a changed package: its nodes by an index range on the + // path (`dir/` up to `dir0`, the character after '/'). + for (const dir of sourceDirs) { + const nodes = (dir === '' + ? this.db.prepare('SELECT id FROM nodes WHERE language = ?').all(language) + : this.db + .prepare('SELECT id FROM nodes WHERE file_path >= ? AND file_path < ?') + .all(`${dir}/`, `${dir}0`)) as Array<{ id: string }>; + idsWhere('source', nodes.map((n) => n.id)); + } + // Imports the resolver couldn't place and bound to their own `import` + // node rather than parking as a failure, matched by the module name they + // spell: ONE read of the language's import nodes, filtered here. (A LIKE + // scan of the edges per name measured ~225 ms each on django's 213k.) + if (importNodeSegments.length > 0) { + const wanted = new Set(importNodeSegments); + const nodes = this.db + .prepare("SELECT id, name FROM nodes WHERE kind = 'import' AND language = ?") + .all(language) as Array<{ id: string; name: string }>; + idsWhere('target', nodes.filter((n) => n.name.split('.').some((seg) => wanted.has(seg))).map((n) => n.id)); + } + return this.loadResolutionEdges(language, [...candidates]); + } + + /** + * Resolution edges leaving any node of `files` — the candidates when an + * importer's bindings may have moved (a Python package's `__init__.py` + * edited). The caller narrows them by the names each file binds. + */ + getResolutionEdgesFromFiles( + language: Language, + files: string[] + ): Array { + const ids: number[] = []; + for (let i = 0; i < files.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = files.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const rows = this.db + .prepare( + `SELECT e.id AS id FROM nodes n CROSS JOIN edges e ON e.source = n.id + WHERE n.file_path IN (${chunk.map(() => '?').join(',')})` + ) + .all(...chunk) as Array<{ id: number }>; + for (const r of rows) ids.push(r.id); + } + return this.loadResolutionEdges(language, ids); + } + + /** + * Full rows for resolution edges by id, loaded edge-first (CROSS JOIN pins + * the join order; left to itself the planner drove the query from the + * language index and one rename took 18 s). + */ + private loadResolutionEdges( + language: Language, + ids: number[] + ): Array { + type Row = EdgeRow & { source_file_path: string; source_language: Language }; + const out: Array = []; + for (let i = 0; i < ids.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = ids.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const rows = this.db + .prepare( + `SELECT e.*, src.file_path AS source_file_path, src.language AS source_language + FROM edges e + CROSS JOIN nodes src ON src.id = e.source + CROSS JOIN nodes tgt ON tgt.id = e.target + WHERE e.id IN (${chunk.map(() => '?').join(',')}) + AND src.language = ? + AND e.kind != 'contains' + AND (e.provenance IS NULL OR e.provenance != 'heuristic')` + ) + .all(...chunk, language) as Row[]; + for (const row of rows) { + out.push({ ...rowToEdge(row), edgeId: row.id, sourceFilePath: row.source_file_path, sourceLanguage: row.source_language }); + } + } + return out; + } + + /** + * Re-queue a language's parked failures in `names`' files whose reference + * starts with one of the names bound there (`N`, `N.run`). + */ + requeueFailedReferencesBinding(language: Language, names: Map>): number { + const files = [...names.keys()]; + const ids: number[] = []; + for (let i = 0; i < files.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = files.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const rows = this.db + .prepare( + `SELECT id, file_path, reference_name FROM unresolved_refs + WHERE status = 'failed' AND language = ? AND file_path IN (${chunk.map(() => '?').join(',')})` + ) + .all(language, ...chunk) as Array<{ id: number; file_path: string; reference_name: string }>; + for (const r of rows) { + if (names.get(r.file_path)?.has(r.reference_name.split('.')[0]!)) ids.push(r.id); + } + } + let changed = 0; + this.db.transaction(() => { + for (let i = 0; i < ids.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = ids.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + changed += this.db + .prepare(`UPDATE unresolved_refs SET status = 'pending' WHERE id IN (${chunk.map(() => '?').join(',')})`) + .run(...chunk).changes; + } + })(); + return changed; + } + + /** + * Return a language's parked (status='failed') refs to the pending set — + * all of them, or only those naming one of `segments` as a dotted part + * (`x`, `a.x`, `x.b`, `.x`). + */ + requeueFailedReferencesByLanguage(language: Language, segments?: string[], files?: Set): number { + const sql = "UPDATE unresolved_refs SET status = 'pending' WHERE status = 'failed' AND language = ?"; + if (!segments) return this.db.prepare(sql).run(language).changes; + if (segments.length === 0) return 0; + // One read of the parked rows, matched here, then an update by id. + const wanted = new Set(segments); + const ids = ( + this.db + .prepare("SELECT id, reference_name, file_path FROM unresolved_refs WHERE status = 'failed' AND language = ?") + .all(language) as Array<{ id: number; reference_name: string; file_path: string }> + ) + .filter((r) => r.reference_name.split('.').some((seg) => wanted.has(seg))) + .map((r) => { + files?.add(r.file_path); + return r.id; + }); + let changed = 0; + this.db.transaction(() => { + for (let i = 0; i < ids.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = ids.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + changed += this.db + .prepare(`UPDATE unresolved_refs SET status = 'pending' WHERE id IN (${chunk.map(() => '?').join(',')})`) + .run(...chunk).changes; + } + })(); + return changed; + } + /** Delete edges by primary key — the rebind pass's half of a re-resolution. */ deleteEdgesByIds(edgeIds: number[]): number { if (edgeIds.length === 0) return 0; diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 8623d72034..debee6373b 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -3239,6 +3239,97 @@ export class ExtractionOrchestrator { return refs.length; } + /** + * Re-open every resolution a language made, for the sync's sweep to redo + * against the current graph — the answer when an input to that language's + * resolution changed project-wide rather than per definition. Python + * package roots are that kind of input: an added `__init__.py` or an edited + * `pyproject.toml` moves where modules start for importers in files the + * sync never touched, and CG-33's per-name rebind can't see it. Parked + * failures are re-queued too, since some of them now resolve. Same + * conservatism as {@link resurrectStaleResolutionEdges}: an edge without a + * refName stamp is never deleted. + */ + resurrectResolutionEdgesForLanguage(language: Language): number { + return ( + this.reopenEdges(this.queries.getResolutionEdgesBySourceLanguage(language)) + + this.queries.requeueFailedReferencesByLanguage(language) + ); + } + + /** + * The narrow form, for modules or packages that appeared or vanished below + * an unchanged set of roots: re-open the edges whose target lies in + * `targetFiles` or whose source lies under `sourceDirs`, and re-queue the + * parked failures naming one of `moduleLeaves`. Returns the files whose + * references were re-opened. + */ + resurrectResolutionEdgesTouching( + language: Language, + scope: { targetFiles: string[]; sourceDirs: string[]; moduleLeaves: string[] } + ): string[] { + const files = new Set(); + const edges = this.queries.getResolutionEdgesTouchingFiles(language, scope.targetFiles, scope.sourceDirs, scope.moduleLeaves); + for (const e of edges) files.add(e.sourceFilePath); + this.reopenEdges(edges); + this.queries.requeueFailedReferencesByLanguage(language, scope.moduleLeaves, files); + return [...files]; + } + + /** + * Re-open what importers bound from a Python package whose `__init__.py` + * was edited: in each importing file, the edges and parked failures whose + * reference starts with a name bound from that package (`N`, `N.run`, + * `pkg.N`). A def or re-export added, removed or repointed there moves + * exactly those answers — `from pkg import N` prefers what `__init__.py` + * binds over a submodule `pkg/N` — and nothing in the importer's other + * references. A name whose import edge already lands where the package + * binds it now is left alone, so an edit that moves nothing re-resolves + * nothing. Returns the files whose references were re-opened. + */ + resurrectPythonPackageImporters(bindings: Map>): string[] { + if (bindings.size === 0) return []; + const edges = this.queries.getResolutionEdgesFromFiles('python', [...bindings.keys()]); + const refHead = (e: { metadata?: Record }): string | null => { + const refName = e.metadata?.refName; + return typeof refName === 'string' ? refName.split('.')[0]! : null; + }; + // Where each bound name's own import edge lands today. + const bindsNow = new Map(); + for (const e of edges) { + const refName = e.metadata?.refName; + if (e.kind !== 'imports' || typeof refName !== 'string' || refName.includes('.')) continue; + const target = this.queries.getNodeById(e.target); + if (target) bindsNow.set(`${e.sourceFilePath}\0${refName}`, `${target.filePath}\0${target.kind === 'file' ? '' : target.name}`); + } + const moved = new Map>(); + for (const [file, names] of bindings) { + for (const [name, binding] of names) { + if (binding !== null && bindsNow.get(`${file}\0${name}`) === binding) continue; + let set = moved.get(file); + if (!set) moved.set(file, (set = new Set())); + set.add(name); + } + } + if (moved.size === 0) return []; + this.reopenEdges(edges.filter((e) => moved.get(e.sourceFilePath)?.has(refHead(e) ?? '') === true)); + this.queries.requeueFailedReferencesBinding('python', moved); + return [...moved.keys()]; + } + + private reopenEdges(edges: Array): number { + const edgeIds: number[] = []; + const refs: UnresolvedReference[] = []; + for (const e of edges) { + const ref = resurrectRefFromDroppedEdge(e); + if (!ref) continue; + edgeIds.push(e.edgeId); + refs.push(ref); + } + if (refs.length > 0) this.queries.replaceResolutionEdgesWithUnresolvedRefs(edgeIds, refs); + return refs.length; + } + /** * Sync the index with the current file state. * diff --git a/src/index.ts b/src/index.ts index 1f50d46ce7..479c4adbb8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -148,6 +148,9 @@ export interface IndexOptions { * * Provides the primary interface for interacting with the code knowledge graph. */ +/** project_metadata key for the Python package-root digest (see sync). */ +const PYTHON_ROOTS_METADATA_KEY = 'python_root_fingerprint'; + export class CodeGraph { private db: DatabaseConnection; private queries: QueryBuilder; @@ -737,6 +740,8 @@ export class CodeGraph { // engine produces richer extraction than the one on disk. Only on a // real full index — a sync touches a subset, so it must NOT advance the // extraction stamp (the bulk would still be stale). See extraction-version.ts. + if (result.success) this.stampPythonRootFingerprint(); + if (result.success && result.filesIndexed > 0) { try { this.queries.setMetadata('indexed_with_version', CodeGraphPackageVersion); @@ -834,6 +839,18 @@ export class CodeGraph { * * Uses a mutex to prevent concurrent indexing operations. */ + /** Indexed Python file paths — the before/after sets a sync diffs. */ + private pythonFilePaths(): Set { + return new Set(this.queries.getAllFilePaths().filter((f) => f.endsWith('.py'))); + } + + /** Record the Python package-root digest a later sync compares against. */ + private stampPythonRootFingerprint(fingerprint = this.resolver.getPythonRootFingerprint()): void { + try { + this.queries.setMetadata(PYTHON_ROOTS_METADATA_KEY, fingerprint); + } catch { /* metadata is advisory */ } + } + async sync(options: IndexOptions = {}): Promise { return this.indexMutex.withLock(async () => { const startedAt = Date.now(); @@ -920,6 +937,7 @@ export class CodeGraph { const fullReconcile = !options.paths || options.paths.length === 0; const gitState = this.orchestrator.beginGitIndexState(fullReconcile); + const pythonFilesBefore = this.pythonFilePaths(); // An interrupted index may have absorbed its changed files before // resolution/synthesis. Detect those orphans BEFORE this sync adds refs. let refreshSynthesis = this.queries.getMetadata('synthesis_pending') === '1' || @@ -1214,6 +1232,59 @@ export class CodeGraph { if (process.env.CODEGRAPH_SYNTH_TIMINGS) { console.error(`[phase-timing] sync-route-modules: ${Date.now() - tModules}ms (${reopened} refs re-opened)`); } + } + + // Python module answers this sync may have moved in files it never + // touched. If the root set changed (a top-level package appeared, a + // build config was edited) any import anywhere may bind differently: + // re-open them all. Otherwise only modules or packages appeared or + // vanished below the roots, and only the edges into those module + // names, out of a changed package, and the failures naming them can + // move. The orphan sweep below re-resolves whatever is re-opened + // exactly as a full index would. An index stamped by an older engine + // has no fingerprint — record one rather than re-resolving on upgrade. + { + const before = this.queries.getMetadata(PYTHON_ROOTS_METADATA_KEY); + const now = this.resolver.getPythonRootFingerprint(); + if (before !== null && before !== now) { + this.resolver.clearCaches(); + this.orchestrator.resurrectResolutionEdgesForLanguage('python'); + // Any Python edge may move, so any synthesis built on one may too. + refreshSynthesis = true; + this.queries.setMetadata('synthesis_pending', '1'); + } else if (before !== null) { + const after = this.pythonFilePaths(); + const moved = [...after].filter((f) => !pythonFilesBefore.has(f)); + for (const f of pythonFilesBefore) if (!after.has(f)) moved.push(f); + const reopened = new Set(); + if (moved.length > 0) { + this.resolver.clearCaches(); + for (const f of this.orchestrator.resurrectResolutionEdgesTouching('python', this.resolver.getPythonReopenScope(moved))) reopened.add(f); + } + // An edited `__init__.py` can add, drop or repoint what its + // package binds, which moves what its importers' names resolve to. + const editedInits = (result.changedFilePaths ?? []).filter( + (f) => pythonFilesBefore.has(f) && after.has(f) && /(?:^|[\\/])__init__\.py$/.test(f) + ); + if (editedInits.length > 0) { + this.resolver.clearCaches(); + for (const f of this.orchestrator.resurrectPythonPackageImporters(this.resolver.getPythonPackageImporters(editedInits))) reopened.add(f); + } + // Resolve the narrow set now, the way the changed files' own + // references were resolved above, rather than leaving a handful of + // refs to the orphan sweep and its whole-graph synthesis pass. + if (reopened.size > 0) { + this.resolver.resolveAndPersist(this.queries.getUnresolvedReferencesByFiles([...reopened])); + // Same rule as the changed files above: a moved edge in a file + // synthesis read or wrote means synthesis must be rebuilt. + if (!refreshSynthesis && [...reopened].some((f) => + this.queries.hasSynthesizedEdgesTouchingFile(f) || this.queries.wasSynthesisInput(f))) { + refreshSynthesis = true; + this.queries.setMetadata('synthesis_pending', '1'); + } + } + } + if (before !== now) this.stampPythonRootFingerprint(now); } // Orphan sweep (#1187). A resolution pass that dies mid-run — the #850 diff --git a/src/resolution/import-resolver.ts b/src/resolution/import-resolver.ts index 9423eaebff..7ccea8430d 100644 --- a/src/resolution/import-resolver.ts +++ b/src/resolution/import-resolver.ts @@ -4,6 +4,7 @@ * Resolves import paths to actual files and symbols. */ +import * as crypto from 'crypto'; import * as fs from 'fs'; import * as path from 'path'; import { Language, Node } from '../types'; @@ -248,7 +249,12 @@ function getFileExportIndex(filePath: string, context: ResolutionContext): FileE let idx = perFile.get(filePath); if (!idx) { idx = { byName: new Map(), defaultComponent: undefined, defaultFnClass: undefined }; - const exported = context.getExportedNodesInFile?.(filePath) ?? context.getNodesInFile(filePath).filter((n) => n.isExported); + // Python has no export keyword: every module-level def is importable, and + // the extractor never flags one isExported. Without this a Python named + // import could only ever resolve through the bare-name matcher. + const exported = PYTHON_MODULE_FILE.test(filePath) + ? context.getNodesInFile(filePath).filter((n) => n.isExported || isPythonModuleLevelDef(n)) + : context.getExportedNodesInFile?.(filePath) ?? context.getNodesInFile(filePath).filter((n) => n.isExported); for (const n of exported) { if (!idx.byName.has(n.name)) idx.byName.set(n.name, n); if (idx.defaultComponent === undefined && n.kind === 'component') idx.defaultComponent = n; @@ -321,9 +327,9 @@ export function clearImportResolverMemos(context: ResolutionContext): void { fileExportIndexes.delete(context); luaFileBasenameIndexes.delete(context); cobolCopybookIndexes.delete(context); - pythonModuleFileMemos.delete(context); - PY_MODULE_SYMBOLS.delete(context); PROJECT_FOLDER_SUFFIXES.delete(context); + pythonModuleIndexes.delete(context); + pythonStarExportMemos.delete(context); } export function resolveImportPath( @@ -367,6 +373,12 @@ function resolveImportPathUncached( return null; } + // Python names modules, not paths: one resolver answers relative and + // absolute specifiers alike, so every caller gets the same answer. + if (language === 'python') { + return resolvePythonModule(importPath, fromFile, context); + } + const projectRoot = context.getProjectRoot(); const fromDir = path.dirname(path.join(projectRoot, fromFile)); @@ -466,6 +478,549 @@ function resolveCobolCopybook( return best; } +/** + * Python module resolution. Python imports name MODULES (`pkg.sub.mod`, + * `..sibling`), which the interpreter finds by searching `sys.path` roots, so + * the answer depends on where packages start — not on any file that merely + * ends in `pkg/sub/mod.py`. The old suffix match bound `import json` to a + * project's `app/utils/json.py` and picked an arbitrary twin when two services + * both carry `core/models.py`. + * + * Two kinds of root, both derived from evidence: + * - PROJECT roots — what `sys.path` holds for any code in the repo: the repo + * root; each service root (a directory with its own `pyproject.toml`, + * `setup.cfg` or `setup.py`); the `src/` of any of those; and the package + * directories their build config declares. These may answer an importer + * anywhere, but only unambiguously. + * - LOCAL roots — the parent of every top-level regular package, a script's + * own directory, a namespace package's parent seen from inside it. These + * answer only importers within their REACH: a fixture's `tests/fixtures/ + * proj/requests/` must never become the application's `requests`. + * + * A service root counts as a project root unless it is nested inside a + * package (its reach is then itself) or under a directory whose NAME marks + * code that isn't the project's API ({@link PYTHON_NON_PROJECT_DIRS} — its + * reach is then that directory, so sibling projects under `examples/` still + * import each other). The nearest root containing the importer wins; + * otherwise exactly one root within reach must provide the module. Only + * indexed files count — a module the index doesn't hold has no node to link to. + */ +interface PythonModuleIndex { + files: Set; + /** Directories holding an `__init__.py` — regular packages. */ + packageDirs: Set; + /** Every directory that holds a `.py` file at any depth — namespace candidates. */ + dirs: Set; + projectRoots: Set; + /** Local roots, each with the directory whose importers it may answer. */ + localRoots: Map; + /** + * Per top-level name, the roots (project and local) that provide it, split by kind + * (a regular package or module shadows a namespace portion). Memoised so a + * large monorepo pays for the scan once per name, not once per reference. + */ + headsByName: Map; +} + +const pythonModuleIndexes = new WeakMap(); +const PYTHON_MODULE_FILE = /\.py$/; +const PYTHON_MODULE_LEVEL_KINDS = new Set(['function', 'class', 'variable', 'constant']); +/** + * Directory names that, by near-universal convention, hold code that is not + * the project's importable API: test inputs, sample projects, vendored copies. + * Used ONLY to keep a root nested under one from answering importers outside + * that directory — never to drop an edge inside it. A name, not content: a + * shared `conftest.py` or a `smoke_test.py` script proves a directory holds + * tests, not that the services beneath it aren't the project. + * + * The trade-off: a library kept under one of these names but installed into + * the environment (`third_party/foo` with its own `pyproject.toml`, `pip + * install -e`'d, imported as `import foo`) is not resolved from outside that + * directory — an editable install is invisible to a static index, and + * treating every such tree as importable would let fixtures capture the + * application's imports. A missed edge, never a wrong one. + */ +const PYTHON_NON_PROJECT_DIRS = new Set([ + 'tests', 'test', 'testing', 'fixtures', 'testdata', 'examples', 'example', 'samples', 'vendor', 'third_party', +]); + +/** A def at module level — what `from mod import name` can bind. */ +function isPythonModuleLevelDef(n: Node): boolean { + return PYTHON_MODULE_LEVEL_KINDS.has(n.kind) && !n.qualifiedName.includes('::'); +} + +function parentDir(dir: string): string { + const slash = dir.lastIndexOf('/'); + return slash < 0 ? '' : dir.slice(0, slash); +} + +function joinRel(dir: string, rel: string): string { + return dir ? `${dir}/${rel}` : rel; +} + +/** Whether `dir` is `scope` or inside it. */ +function isWithin(dir: string, scope: string): boolean { + return scope === '' || dir === scope || dir.startsWith(`${scope}/`); +} + +function pythonModuleIndex(context: ResolutionContext): PythonModuleIndex { + let index = pythonModuleIndexes.get(context); + if (!index) { + index = buildPythonModuleIndex(context); + pythonModuleIndexes.set(context, index); + } + return index; +} + +function buildPythonModuleIndex(context: ResolutionContext): PythonModuleIndex { + const files = new Set(); + const packageDirs = new Set(); + const dirs = new Set(['']); + for (const f of context.getAllFiles()) { + const norm = f.replace(/\\/g, '/'); + if (!PYTHON_MODULE_FILE.test(norm)) continue; + files.add(norm); + const dir = parentDir(norm); + if (norm === '__init__.py' || norm.endsWith('/__init__.py')) packageDirs.add(dir); + for (let d = dir; d && !dirs.has(d); d = parentDir(d)) dirs.add(d); + } + + // Where a service's roots may answer from: everywhere (null), only itself + // when a package encloses it, or the nearest enclosing directory named as + // non-project code. + const serviceReach = (service: string): string | null => { + if (packageDirs.has(service)) return service; + for (let a = parentDir(service); a; a = parentDir(a)) { + if (packageDirs.has(a)) return service; + if (PYTHON_NON_PROJECT_DIRS.has(a.slice(a.lastIndexOf('/') + 1))) return a; + } + return null; + }; + + const projectRoots = new Set(['']); + const localRoots = new Map(); + const addLocal = (root: string, reach: string): void => { + const prior = localRoots.get(root); + // Two claims on one root: the wider reach wins. + if (prior === undefined || isWithin(prior, reach)) localRoots.set(root, reach); + }; + for (const dir of packageDirs) { + let top = dir; + while (top && packageDirs.has(parentDir(top))) top = parentDir(top); + if (top) addLocal(parentDir(top), parentDir(top)); + } + for (const service of dirs) { + const hasConfig = + files.has(joinRel(service, 'setup.py')) || + context.fileExists(joinRel(service, 'pyproject.toml')) || + context.fileExists(joinRel(service, 'setup.cfg')); + if (!hasConfig && service !== '') continue; + const reach = service === '' ? null : serviceReach(service); + const roots = [service]; + const src = joinRel(service, 'src'); + if (dirs.has(src) && !packageDirs.has(src)) roots.push(src); + for (const declared of declaredPythonRoots(context, service)) { + if (dirs.has(declared)) roots.push(declared); + } + for (const root of roots) { + if (reach === null) projectRoots.add(root); + else addLocal(root, reach); + } + } + for (const root of projectRoots) localRoots.delete(root); + return { files, packageDirs, dirs, projectRoots, localRoots, headsByName: new Map() }; +} + +/** + * Package roots a build config declares, relative to the repo root. Only the + * keys that mean "packages live here" are read, each in its own table: + * setuptools `[tool.setuptools.packages.find] where` and + * `[tool.setuptools.package-dir] "" =` (or the inline `package-dir` of + * `[tool.setuptools]`), poetry `packages = [{ from = … }]`, hatch + * `packages = ["src/pkg"]`, and `setup.cfg`'s `[options] package_dir` and + * `[options.packages.find] where`. A miss only costs the namespace-package + * case, which the `__init__.py` chain can't see either. + */ +function declaredPythonRoots(context: ResolutionContext, service: string): string[] { + const out: string[] = []; + const add = (raw: string): void => { + const dir = raw.trim().replace(/^["']|["']$/g, '').replace(/^\.(?:\/|$)/, '').replace(/\/+$/, ''); + if (!dir || dir.startsWith('..') || path.isAbsolute(dir)) return; + out.push(joinRel(service, dir)); + }; + const toml = context.readFile(joinRel(service, 'pyproject.toml')); + if (toml) { + for (const [table, body] of iniTables(toml)) { + if (table === 'tool.setuptools.packages.find') { + const m = body.match(/^\s*where\s*=\s*\[([^\]]*)\]/m); + if (m) for (const item of m[1]!.split(',')) add(item); + } else if (table === 'tool.setuptools.package-dir') { + const m = body.match(/^\s*["']{2}\s*=\s*["']([^"']+)["']/m); + if (m) add(m[1]!); + } else if (table === 'tool.setuptools') { + const m = body.match(/package-dir\s*=\s*\{[^}]*?["']{2}\s*=\s*["']([^"']+)["']/); + if (m) add(m[1]!); + } else if (table === 'tool.poetry') { + const pkgs = body.match(/^\s*packages\s*=\s*\[([\s\S]*?)\]\s*$/m); + if (pkgs) for (const m of pkgs[1]!.matchAll(/\bfrom\s*=\s*["']([^"']+)["']/g)) add(m[1]!); + } else if (table === 'tool.hatch.build.targets.wheel') { + const pkgs = body.match(/^\s*packages\s*=\s*\[([^\]]*)\]/m); + if (pkgs) { + for (const item of pkgs[1]!.split(',')) { + const p = item.trim().replace(/^["']|["']$/g, ''); + if (p.includes('/')) add(p.slice(0, p.lastIndexOf('/'))); + } + } + } + } + } + const cfg = context.readFile(joinRel(service, 'setup.cfg')); + if (cfg) { + for (const [table, body] of iniTables(cfg)) { + if (table === 'options') { + const m = body.match(/^\s*package_dir\s*=\s*(?:\n\s+)?=\s*(\S+)\s*$/m); + if (m) add(m[1]!); + } else if (table === 'options.packages.find') { + const m = body.match(/^\s*where\s*=\s*(\S+)\s*$/m); + if (m) add(m[1]!); + } + } + } + return out; +} + +/** `[table]` sections of a TOML / INI file, with the body under each. */ +function iniTables(content: string): Array<[string, string]> { + const out: Array<[string, string]> = []; + const headers = [...content.matchAll(/^[ \t]*\[([^\[\]\n]+)\][ \t]*(?:#.*)?$/gm)]; + headers.forEach((h, i) => { + const end = i + 1 < headers.length ? headers[i + 1]!.index! : content.length; + out.push([h[1]!.trim().replace(/["']/g, ''), content.slice(h.index! + h[0].length, end)]); + }); + return out; +} + +/** + * The module file at `base` (dir-relative path), in Python's own order: a + * regular package `base/__init__.py` beats a module `base.py`; a bare + * directory (a namespace portion) has no file of its own. + */ +function pythonModuleAt(base: string, index: PythonModuleIndex): string | null { + const init = joinRel(base, '__init__.py'); + if (index.files.has(init)) return init; + return index.files.has(`${base}.py`) ? `${base}.py` : null; +} + +/** + * The module file for the dotted path `relPath` (slash-joined) under `root`, + * walking it the way the import system does: every prefix must be a package + * — regular, or a namespace directory — so a prefix that is a plain module + * (`pkg/sub.py` with no `pkg/sub/__init__.py`) ends the walk, and + * `pkg.sub.x` does not exist even if `pkg/sub/x.py` does. + */ +function pythonModuleUnder(root: string, relPath: string, index: PythonModuleIndex): string | null { + const parts = relPath.split('/'); + let base = root; + for (let i = 0; i < parts.length - 1; i++) { + base = joinRel(base, parts[i]!); + if (index.files.has(joinRel(base, '__init__.py'))) continue; + if (index.files.has(`${base}.py`) || !index.dirs.has(base)) return null; + } + return pythonModuleAt(joinRel(root, relPath), index); +} + +/** How `root` provides the top-level name `head`: concretely, as a namespace portion, or not at all. */ +function pythonHeadKind(root: string, head: string, index: PythonModuleIndex): 'concrete' | 'namespace' | null { + const base = joinRel(root, head); + if (pythonModuleAt(base, index)) return 'concrete'; + return index.dirs.has(base) ? 'namespace' : null; +} + +/** Every root (project or local) providing the top-level name `head`, by kind. */ +function pythonRootHeads(head: string, index: PythonModuleIndex): { concrete: string[]; namespace: string[] } { + let heads = index.headsByName.get(head); + if (!heads) { + heads = { concrete: [], namespace: [] }; + for (const root of [...index.projectRoots, ...index.localRoots.keys()]) { + const kind = pythonHeadKind(root, head, index); + if (kind) heads[kind].push(root); + } + index.headsByName.set(head, heads); + } + return heads; +} + +/** + * Resolve a Python import specifier to the file of the module it names — + * `pkg.sub.mod` → `pkg/sub/mod.py`, `pkg` → `pkg/__init__.py`, `..sibling` + * relative to `fromFile`'s package. Null when the module isn't in the index, + * when a relative import climbs above its top-level package, or when two + * project roots that don't contain the importer both provide it. + */ +function resolvePythonModule( + specifier: string, + fromFile: string, + context: ResolutionContext +): string | null { + if (!/^\.*[\w.]*$/.test(specifier) || specifier === '') return null; + const index = pythonModuleIndex(context); + const importerDir = parentDir(fromFile.replace(/\\/g, '/')); + + // Relative: leading dots are package levels (one dot = the importer's own + // package), the rest a dotted path below it. + const dots = specifier.length - specifier.replace(/^\.+/, '').length; + if (dots > 0) { + let base = importerDir; + for (let level = 1; level < dots; level++) { + // Python refuses to climb out of the top-level package ("attempted + // relative import beyond top-level package"). Only a regular package + // proves where the top is; a namespace package's boundary is unknown. + if (index.packageDirs.has(base) && !index.packageDirs.has(parentDir(base))) return null; + if (!base) return null; + base = parentDir(base); + } + const rest = specifier.slice(dots); + if (rest) return pythonModuleUnder(base, rest.replace(/\./g, '/'), index); + // `from . import x` — the package itself, i.e. its `__init__.py`. + const init = joinRel(base, '__init__.py'); + return index.files.has(init) ? init : null; + } + + const relPath = specifier.replace(/\./g, '/'); + const head = specifier.split('.')[0]!; + + // Roots that contain the importer, nearest first — the ones Python would + // search for this file before any other. + const containing: string[] = []; + for (let dir: string | null = importerDir; dir !== null; dir = dir ? parentDir(dir) : null) { + const isRoot = + index.projectRoots.has(dir) || + index.localRoots.has(dir) || + (!index.packageDirs.has(dir) && + (dir === importerDir || importerDir === joinRel(dir, head) || importerDir.startsWith(`${joinRel(dir, head)}/`))); + if (isRoot) containing.push(dir); + } + + // A root containing the importer is on its path: the nearest one that holds + // the module answers. If one holds `head` itself as a regular package or + // module but not the rest of the path, Python stops there too. + let headIsLocal = false; + for (const root of containing) { + const hit = pythonModuleUnder(root, relPath, index); + if (hit) return hit; + if (pythonHeadKind(root, head, index) === 'concrete') headIsLocal = true; + } + if (headIsLocal) return null; + + // Otherwise a root elsewhere whose reach includes the importer, as + // `sys.path` would rank them: a root holding `head` concretely shadows any + // namespace portion, and either way the answer must be unambiguous. + const heads = pythonRootHeads(head, index); + const reaches = (root: string): boolean => + !containing.includes(root) && + (index.projectRoots.has(root) || isWithin(importerDir, index.localRoots.get(root)!)); + const concrete = heads.concrete.filter(reaches); + if (concrete.length > 0) { + return concrete.length === 1 ? pythonModuleUnder(concrete[0]!, relPath, index) : null; + } + let found: string | null = null; + for (const root of heads.namespace) { + if (!reaches(root)) continue; + const hit = pythonModuleUnder(root, relPath, index); + if (!hit) continue; + if (found !== null && found !== hit) return null; + found = hit; + } + return found; +} + +/** + * A digest of the Python root set — each root with its tier and reach. A + * change here can move the answer for any import anywhere, so sync re-opens + * every Python resolution when it changes. Changes below the roots (a module + * or package added or removed) only move answers for the module names they + * touch; see {@link pythonReopenScope}. Built fresh, not from the memo. + */ +export function pythonRootFingerprint(context: ResolutionContext): string { + const index = buildPythonModuleIndex(context); + if (index.files.size === 0) return ''; + const hash = crypto.createHash('sha1'); + hash.update([...index.projectRoots].sort().join('\n')); + hash.update('\0'); + hash.update([...index.localRoots].map(([root, reach]) => `${root}\t${reach}`).sort().join('\n')); + return hash.digest('hex'); +} + +/** + * What a sync must re-open when Python files were added or removed but the + * root set held. Each file names a module under every root containing it + * (`pkg/sub/__init__.py` is `pkg.sub`); an already-resolved edge can move only + * if its target is that module or inside it — shadowed (`pkg/sub/` now beats + * `pkg/sub.py`), un-shadowed, or newly ambiguous. A changed `__init__.py` also + * moves where a package starts for code inside it (relative imports, a + * script's own directory), so edges FROM that directory re-open too. The + * module names' last segments pick out the parked failures that may now + * resolve. + */ +export function pythonReopenScope( + context: ResolutionContext, + changedFiles: string[] +): { targetFiles: string[]; sourceDirs: string[]; moduleLeaves: string[] } { + const index = buildPythonModuleIndex(context); + const roots = [...index.projectRoots, ...index.localRoots.keys()]; + const moduleNames = (file: string): string[] => { + const out: string[] = []; + const mod = file.replace(/(?:^|\/)__init__\.py$/, '').replace(/\.py$/, ''); + for (const root of roots) { + if (root && !mod.startsWith(`${root}/`)) continue; + const rel = root ? mod.slice(root.length + 1) : mod; + if (rel) out.push(rel.replace(/\//g, '.')); + } + return out; + }; + const affected = new Set(); + const sourceDirs = new Set(); + for (const raw of changedFiles) { + const file = raw.replace(/\\/g, '/'); + if (!PYTHON_MODULE_FILE.test(file)) continue; + for (const m of moduleNames(file)) affected.add(m); + if (file === '__init__.py' || file.endsWith('/__init__.py')) sourceDirs.add(parentDir(file)); + } + const targetFiles: string[] = []; + for (const file of index.files) { + const hit = moduleNames(file).some((m) => { + for (let i = m.length; i > 0; i = m.lastIndexOf('.', i - 1)) { + if (affected.has(m.slice(0, i))) return true; + } + return false; + }); + if (hit) targetFiles.push(file); + } + const moduleLeaves = [...new Set([...affected].map((m) => m.slice(m.lastIndexOf('.') + 1)))]; + return { targetFiles, sourceDirs: [...sourceDirs], moduleLeaves }; +} + +/** + * The Python files that import one of the packages whose `__init__.py` is in + * `initFiles`, each with the local names bound from it and what each name + * binds to NOW (`file\0name` of the definition, `file\0` for a submodule, + * null when it can't be said — a namespace import used as `pkg.N`). A caller + * compares that with what the importer's edge says it bound before and + * leaves unmoved names alone. An importer that is itself a package + * `__init__.py` passes the binding on (`from .sub import N` re-exported), so + * its own importers are included too, to a fixed point. + */ +export function pythonPackageImporters( + context: ResolutionContext, + initFiles: string[] +): Map> { + const index = pythonModuleIndex(context); + const packages = new Set(initFiles.map((f) => f.replace(/\\/g, '/'))); + const out = new Map>(); + const bindingNow = (imp: ImportMapping, file: string): string | null => { + if (imp.isNamespace || imp.exportedName === '*') return null; + const packageFile = resolveImportPath(imp.source, file, 'python', context); + if (!packageFile) return null; + const symbol = findExportedSymbol( + packageFile, + { isDefault: false, isNamespace: false, exportedName: imp.exportedName, memberName: null }, + 'python', + context, + new Set() + ); + if (symbol) return `${symbol.filePath}\0${symbol.name}`; + const sub = resolveImportPath( + imp.source.endsWith('.') ? imp.source + imp.exportedName : `${imp.source}.${imp.exportedName}`, + file, + 'python', + context + ); + return sub ? `${sub}\0` : '\0'; + }; + // Candidate importers from the import nodes (named by their specifier), + // so only files that import an edited package pay for a mapping parse. + const importNodes = context.getNodesByKind('import').filter((n) => PYTHON_MODULE_FILE.test(n.filePath)); + for (let grew = true; grew; ) { + grew = false; + const candidates = new Set(); + for (const n of importNodes) { + const target = resolveImportPath(n.name, n.filePath, 'python', context); + if (target && packages.has(target) && index.files.has(n.filePath)) candidates.add(n.filePath); + } + for (const file of candidates) { + for (const imp of context.getImportMappings(file, 'python')) { + const target = resolveImportPath(imp.source, file, 'python', context); + if (!target || !packages.has(target) || target === file) continue; + let names = out.get(file); + if (!names) out.set(file, (names = new Map())); + names.set(imp.localName, bindingNow(imp, file)); + // `import pkg.sub` is used as `pkg.sub.N`. + if (imp.isNamespace) names.set(imp.source.split('.')[0]!, null); + if (/(?:^|\/)__init__\.py$/.test(file) && !packages.has(file)) { + packages.add(file); + grew = true; + } + } + } + } + return out; +} + +/** The file node of the module `specifier` names, as seen from `fromFile`. */ +function pythonModuleFileNode(specifier: string, fromFile: string, context: ResolutionContext): Node | null { + const file = resolveImportPath(specifier, fromFile, 'python', context); + if (!file || file === fromFile) return null; + return context.getNodesInFile(file).find((n) => n.kind === 'file') ?? null; +} + +/** + * The names `from import *` binds: the string literals of the module's + * `__all__` when it assigns one, otherwise every public (non-`_`) name. + * Returns null for "every public name". Memoised per context. + */ +const pythonStarExportMemos = new WeakMap | null>>(); + +function pythonStarExports(filePath: string, context: ResolutionContext): Set | null { + let memo = pythonStarExportMemos.get(context); + if (!memo) { + memo = new Map(); + pythonStarExportMemos.set(context, memo); + } + if (memo.has(filePath)) return memo.get(filePath)!; + let names: Set | null = null; + const content = context.readFile(filePath); + if (content && content.includes('__all__')) { + // Locate every statement that builds `__all__` on blanked source (so a + // docstring mention is ignored), then read the literals from the original + // at the same offsets. Only a list written out in full is trusted: one + // assembled from other names (`__all__ = fields_all + [...]`, + // `__all__.extend(...)`) is unknown, and falls back to the public names. + const code = stripCommentsForRegex(content, 'python'); + let literal = true; + const collected = new Set(); + for (const m of code.matchAll(/^[ \t]*__all__\b[ \t]*(\.|\+=|=|:[^=\n]*=)?[ \t]*(.?)/gm)) { + const open = m.index! + m[0].length - 1; + const bracket = m[2]; + const close = bracket === '[' ? code.indexOf(']', open) : bracket === '(' ? code.indexOf(')', open) : -1; + const assigns = m[1] !== undefined && m[1] !== '.'; + const tail = close < 0 ? '' : code.slice(close + 1, code.indexOf('\n', close) < 0 ? undefined : code.indexOf('\n', close)); + if (!assigns || close < 0 || tail.trim() !== '') { + literal = false; + break; + } + for (const lit of content.slice(open, close).matchAll(/["']([A-Za-z_]\w*)["']/g)) collected.add(lit[1]!); + } + if (literal && collected.size > 0) names = collected; + } + memo.set(filePath, names); + return names; +} + +function pythonStarAllows(filePath: string, name: string, context: ResolutionContext): boolean { + const names = pythonStarExports(filePath, context); + return names ? names.has(name) : !name.startsWith('_'); +} + /** * C and C++ standard library header names (without delimiters). * Used by isExternalImport to filter system includes from resolution. @@ -626,24 +1181,6 @@ function resolveRelativeImport( const projectRoot = context.getProjectRoot(); const extensions = EXTENSION_RESOLUTION[language] || []; - // Python dotted-relative imports (`from .certs import x`, `from ..pkg.mod - // import y`): leading dots are PACKAGE levels (1 = current package), and the - // remainder is a dotted submodule path. `path.resolve(dir, '.certs')` would - // treat `.certs` as a literal hidden filename, so translate the Python form - // to a real filesystem-relative path before resolving. - if (language === 'python' && importPath.startsWith('.')) { - const dots = importPath.length - importPath.replace(/^\.+/, '').length; - const up = '../'.repeat(Math.max(0, dots - 1)); // 1 dot = current dir - const rest = importPath.slice(dots).replace(/\./g, '/'); // 'sub.mod' -> 'sub/mod' - const pyBase = path.resolve(fromDir, up + rest); - const pyRel = path.relative(projectRoot, pyBase).replace(/\\/g, '/'); - for (const ext of extensions) { - if (context.fileExists(pyRel + ext)) return pyRel + ext; - } - if (pyRel && context.fileExists(pyRel)) return pyRel; - return null; - } - // Try the path as-is first const basePath = path.resolve(fromDir, importPath); const relativePath = path.relative(projectRoot, basePath).replace(/\\/g, '/'); @@ -1212,55 +1749,79 @@ function extractJSImports(content: string): ImportMapping[] { return mappings; } +interface PythonFromImport { + source: string; + names: Array<{ name: string; alias: string }>; + star: boolean; +} + +/** + * The `from X import …` statements of a Python file, read over comment- and + * docstring-blanked source so a usage example in a docstring binds nothing. + * Handles the parenthesised multi-line list (`from .x import (\n a,\n b,\n)`, + * the usual shape of a package `__init__.py`) and backslash continuations, + * which a single-line match silently dropped. + */ +function pythonFromImports(code: string): PythonFromImport[] { + const out: PythonFromImport[] = []; + const fromRe = /^[ \t]*from[ \t]+(\.*[\w.]*)[ \t]+import[ \t]*(\([^)]*\)|(?:[^\n\\;]|\\\r?\n)+)/gm; + let m: RegExpExecArray | null; + while ((m = fromRe.exec(code)) !== null) { + const source = m[1]!; + if (!source) continue; + const list = m[2]!.replace(/^\(|\)$/g, '').replace(/\\\r?\n/g, ' '); + const names: PythonFromImport['names'] = []; + let star = false; + for (const raw of list.split(',')) { + const item = raw.trim(); + if (item === '*') { + star = true; + continue; + } + const im = item.match(/^(\w+)(?:\s+as\s+(\w+))?$/); + if (im) names.push({ name: im[1]!, alias: im[2] ?? im[1]! }); + } + out.push({ source, names, star }); + } + return out; +} + /** * Extract Python import mappings */ function extractPythonImports(content: string): ImportMapping[] { const mappings: ImportMapping[] = []; + const code = stripCommentsForRegex(content, 'python'); - // from X import Y, and the parenthesized form `from X import (\n Y,\n Z,\n)` - const fromImportRegex = /from\s+([\w.]+)\s+import\s+(\([^)]*\)|[^#\n]+)/g; - let match; - - while ((match = fromImportRegex.exec(content)) !== null) { - const [, source, imports] = match; - const names = imports!.trim().replace(/^\(|\)$/g, '') - .split(',').map((s) => s.trim()); - - for (const name of names) { - const aliasMatch = name.match(/(\w+)\s+as\s+(\w+)/); - if (aliasMatch) { - mappings.push({ - localName: aliasMatch[2]!, - exportedName: aliasMatch[1]!, - source: source!, - isDefault: false, - isNamespace: false, - }); - } else if (name && name !== '*') { - mappings.push({ - localName: name, - exportedName: name, - source: source!, - isDefault: false, - isNamespace: false, - }); - } + // from X import Y [as Z], … + for (const { source, names } of pythonFromImports(code)) { + for (const { name, alias } of names) { + mappings.push({ + localName: alias, + exportedName: name, + source, + isDefault: false, + isNamespace: false, + }); } } - // import X - const importRegex = /^import\s+([\w.]+)(?:\s+as\s+(\w+))?/gm; - while ((match = importRegex.exec(content)) !== null) { - const [, source, alias] = match; - const localName = alias || source!.split('.').pop()!; - mappings.push({ - localName, - exportedName: '*', - source: source!, - isDefault: false, - isNamespace: true, - }); + // import X [as Y], … — at any indentation: a function-local import binds + // the name for the calls in that function. + const importRe = /^[ \t]*import[ \t]+((?:[^\n\\;]|\\\r?\n)+)/gm; + let match: RegExpExecArray | null; + while ((match = importRe.exec(code)) !== null) { + for (const raw of match[1]!.replace(/\\\r?\n/g, ' ').split(',')) { + const im = raw.trim().match(/^([\w.]+)(?:\s+as\s+(\w+))?$/); + if (!im) continue; + mappings.push({ + localName: im[2] || im[1]!.split('.').pop()!, + exportedName: '*', + source: im[1]!, + isDefault: false, + isNamespace: true, + }); + } } return mappings; @@ -1524,6 +2085,7 @@ function stripJsComments(content: string): string { * fall through silently; resolution simply skips the broken file. */ export function extractReExports(content: string, language: Language): ReExport[] { + if (language === 'python') return extractPythonReExports(content); if ( language !== 'typescript' && language !== 'javascript' && @@ -1579,6 +2141,23 @@ export function extractReExports(content: string, language: Language): ReExport[ return out; } +/** + * A Python module's imports ARE its re-exports: `from .impl import foo` makes + * `foo` an attribute of the module, which is exactly how a package + * `__init__.py` publishes its API. Chased only after a direct lookup misses, + * with the shared cycle guard and depth cap. + */ +function extractPythonReExports(content: string): ReExport[] { + const out: ReExport[] = []; + for (const { source, names, star } of pythonFromImports(stripCommentsForRegex(content, 'python'))) { + if (star) out.push({ kind: 'wildcard', source }); + for (const { name, alias } of names) { + out.push({ kind: 'named', exportedName: alias, originalName: name, source }); + } + } + return out; +} + /** * Resolve a reference using import mappings */ @@ -2033,19 +2612,15 @@ export function resolveViaImport( for (const imp of imports) { if (imp.localName === ref.referenceName || ref.referenceName.startsWith(imp.localName + '.')) { // Resolve the import path - let resolvedPath = resolveImportPath( + // Python absolute and relative specifiers alike resolve here, from the + // project's package roots — including aliases used as receiver types (#1820). + const resolvedPath = resolveImportPath( imp.source, ref.filePath, ref.language, context ); - // Named Python imports need the same absolute-module lookup as namespace - // imports, including aliases used as receiver types (#1820). - if (!resolvedPath && ref.language === 'python') { - resolvedPath = findPythonModuleFile(imp.source, context, ref.filePath)?.filePath ?? null; - } - if (resolvedPath) { const exportedName = imp.isDefault ? 'default' : imp.exportedName; const memberName = imp.isNamespace @@ -2058,9 +2633,7 @@ export function resolveViaImport( ref.language, context, new Set() - ) ?? (ref.language === 'python' - ? pythonModuleSymbol(resolvedPath, memberName ?? exportedName, context, 0) - : undefined); + ); if (targetNode) { // `Foo.bar()` / `Foo.CONST` — a NAMED (non-namespace) class import @@ -2179,23 +2752,21 @@ function resolvePythonModuleMember( ? imp.source + moduleName : imp.source + '.' + moduleName; - // resolveImportPath only maps RELATIVE dotted paths (`.mod`, `..pkg.mod`); an - // ABSOLUTE package path (`pkg.module` from `from pkg import module`, or a bare - // `import pkg.mod`) resolves to null there, so fall back to the dotted-module - // file lookup — the same asymmetry resolveModuleImportToFile already handles - // for the file→file import edge. Without this, a `module.func()` call after - // `from pkg import module` dropped its `calls` edge even though the import - // edge resolved (#578). - let resolvedPath = resolveImportPath(modulePath, ref.filePath, ref.language, context); - if (!resolvedPath) { - resolvedPath = findPythonModuleFile(modulePath, context, ref.filePath)?.filePath ?? null; - } + if (!imp.isNamespace && pythonPackageBinds(imp, ref.filePath, context)) continue; + const resolvedPath = resolveImportPath(modulePath, ref.filePath, ref.language, context); if (!resolvedPath || resolvedPath === ref.filePath) continue; - // Find the member as a top-level definition in the module file, or one it - // re-exports (a package's `__init__.py`). Exclude `method` so `mod.foo` - // never lands on a same-named class method. - const target = pythonModuleSymbol(resolvedPath, member, context, 0); + // The member as the module binds it: a module-level def (never a method — + // `mod.foo` must not land on a same-named class method, and never a nested + // def), or a name the module re-exports from elsewhere (`pkg/__init__.py` + // doing `from .impl import foo`). + const target = findExportedSymbol( + resolvedPath, + { isDefault: false, isNamespace: false, exportedName: member, memberName: null }, + 'python', + context, + new Set() + ); if (target) { return { original: ref, targetNodeId: target.id, confidence: 0.85, resolvedBy: 'import' }; } @@ -2203,6 +2774,30 @@ function resolvePythonModuleMember( return null; } +/** + * Whether `from P import N` binds something `P/__init__.py` defines or + * re-exports, rather than the submodule `P.N`. CPython's `_handle_fromlist` + * looks `N` up as an attribute of the package first and imports the + * submodule only when the package has no such attribute — so a def, a + * constant or a re-export of `N` in `__init__.py` wins over a `P/N.py` or + * `P/N/` beside it. A package that doesn't bind `N` leaves the submodule to + * answer, as before. + */ +function pythonPackageBinds(imp: ImportMapping, fromFile: string, context: ResolutionContext): boolean { + if (imp.isNamespace || imp.exportedName === '*') return false; + const packageFile = resolveImportPath(imp.source, fromFile, 'python', context); + if (!packageFile || !/(?:^|\/)__init__\.py$/.test(packageFile)) return false; + return ( + findExportedSymbol( + packageFile, + { isDefault: false, isNamespace: false, exportedName: imp.exportedName, memberName: null }, + 'python', + context, + new Set() + ) !== undefined + ); +} + /** * Resolve a whole-MODULE import to that module's file (a file→file dependency). * The imported name is a module, not a symbol, so there's nothing to resolve to @@ -2312,9 +2907,11 @@ function resolveModuleImportToFile( // (no file), so `import React from 'react'` creates no edge. modulePath = imp.source; } else if (ref.language === 'python') { - // `from . import certs` — the imported NAME is a submodule of the source. + // `from . import certs` — the imported NAME is a submodule of the source, + // unless the package itself binds that name (see pythonPackageBinds). // As in resolvePythonModuleMember, use the exported name so an alias // still links to the real module file (#1626). + if (pythonPackageBinds(imp, ref.filePath, context)) continue; const moduleName = imp.exportedName === '*' ? imp.localName : imp.exportedName; modulePath = imp.source.endsWith('.') ? imp.source + moduleName @@ -2331,118 +2928,12 @@ function resolveModuleImportToFile( return { original: ref, targetNodeId: fileNode.id, confidence: 0.9, resolvedBy: 'import' }; } } - - // Python absolute `from a.b import submodule` (a FastAPI router aggregator's - // `from app.api.routes import authentication`): resolveImportPath only maps - // RELATIVE dotted paths to a file, so resolve the absolute dotted module - // directly to its file node. - if (ref.language === 'python') { - const modFile = findPythonModuleFile(modulePath, context, ref.filePath); - if (modFile) { - return { original: ref, targetNodeId: modFile.id, confidence: 0.9, resolvedBy: 'import' }; - } - } } return null; } /** - * Find the file node for a Python dotted module path `a.b.c` — a module file - * ending in `a/b/c.py`, or a package `a/b/c/__init__.py` (suffix-matched, so a - * package rooted under `src/` etc. still resolves). Returns null for - * stdlib/external modules (no matching repo file node), so `import os` creates - * no edge. Shared by absolute `import a.b.c` and absolute `from a.b import c` - * (where `c` is a submodule) resolution. - */ -/** - * Per-context memo for findPythonModuleFile: module path → the `.py` and - * `/__init__.py` file nodes whose path ends with it, in name-lookup - * order. Only the importing file's own path is excluded per call, so taking - * the first survivor returns the node the unmemoized scan found. Without it, - * every ref naming a module outside the project (`from unittest import mock`) - * rescanned every `__init__.py` in the tree. Same stable window as the name - * caches; dropped by clearImportResolverMemos. - */ -const pythonModuleFileMemos = new WeakMap>(); - -/** - * A top-level class / function / value named `name` in a Python module, or - * one the module re-exports — `from .users import *`, `from .users import - * User` — a few packages deep. netbox's `from users.models import User` names - * `users/models/__init__.py`, which star-imports `.users`, where `User` is. - */ -const PY_MODULE_SYMBOLS = new WeakMap>(); - -function pythonModuleSymbol(file: string, name: string, context: ResolutionContext, depth: number): Node | undefined { - let memo = PY_MODULE_SYMBOLS.get(context); - if (!memo) PY_MODULE_SYMBOLS.set(context, (memo = new Map())); - const key = `${file}\0${name}`; - const hit = memo.get(key); - if (hit !== undefined) return hit ?? undefined; - // (A cycle of star imports reads as "not here" while it is being walked.) - memo.set(key, null); - const found = pythonModuleSymbolUncached(file, name, context, depth); - memo.set(key, found ?? null); - return found; -} - -function pythonModuleSymbolUncached(file: string, name: string, context: ResolutionContext, depth: number): Node | undefined { - const own = context.getNodesInFile(file).find((n) => - n.name === name && !n.qualifiedName.includes('::') && - (n.kind === 'class' || n.kind === 'function' || n.kind === 'variable' || n.kind === 'constant')); - if (own || depth >= 3) return own; - // Re-exported by name (`from .users import User`), else through a star import - // (`from .users import *` — not among the import mappings, so read here). - const sources: Array<{ source: string; exported: string }> = context.getImportMappings(file, 'python') - .filter((imp) => !imp.isNamespace && imp.localName === name) - .map((imp) => ({ source: imp.source, exported: imp.exportedName })); - for (const m of (context.readFile(file) ?? '').matchAll(/^\s*from\s+([\w.]+)\s+import\s+\*/gm)) { - sources.push({ source: m[1]!, exported: name }); - } - for (const { source, exported } of sources) { - const target = resolveImportPath(source, file, 'python', context) ?? findPythonModuleFile(source, context, file)?.filePath ?? null; - if (!target || target === file) continue; - const found = pythonModuleSymbol(target, exported, context, depth + 1); - if (found) return found; - } - return undefined; -} - -function findPythonModuleFile( - mod: string, - context: ResolutionContext, - excludeFilePath: string -): Node | null { - if (!mod || mod.startsWith('.')) return null; // relative imports handled elsewhere - let memo = pythonModuleFileMemos.get(context); - if (!memo) { - memo = new Map(); - pythonModuleFileMemos.set(context, memo); - } - let files = memo.get(mod); - if (!files) { - const rel = mod.replace(/\./g, '/'); - const lastSeg = mod.split('.').pop()!; - const endsWith = (p: string, want: string): boolean => p === want || p.endsWith('/' + want); - files = { - module: context - .getNodesByName(`${lastSeg}.py`) - .filter((n) => n.kind === 'file' && endsWith(n.filePath, `${rel}.py`)), - pkg: context - .getNodesByName('__init__.py') - .filter((n) => n.kind === 'file' && endsWith(n.filePath, `${rel}/__init__.py`)), - }; - memo.set(mod, files); - } - return ( - files.module.find((n) => n.filePath !== excludeFilePath) ?? - files.pkg.find((n) => n.filePath !== excludeFilePath) ?? - null - ); -} - -/** - * Resolve a Python ABSOLUTE dotted module import (`import a.b.c`) to its file — + * Resolve a Python dotted module import (`import a.b.c`, `from .a.b import x`) to its file — * the Django `AppConfig.ready(): import myapp.signals` pattern and any * side-effect module import. */ @@ -2456,7 +2947,7 @@ function resolvePythonAbsoluteModule( // `authentication.py` files may exist — so leave it to resolveModuleImportToFile, // which uses the import's source (`app.api.routes`) to build the full path. if (!ref.referenceName.includes('.')) return null; - const hit = findPythonModuleFile(ref.referenceName, context, ref.filePath); + const hit = pythonModuleFileNode(ref.referenceName, ref.filePath, context); return hit ? { original: ref, targetNodeId: hit.id, confidence: 0.9, resolvedBy: 'import' } : null; } @@ -2894,6 +3385,8 @@ function findExportedSymbolWalk( if (rex.kind === 'wildcard') { const next = resolveImportPath(rex.source, filePath, language, context); if (!next) continue; + // Python's `from m import *` binds only `m.__all__`, else m's public names. + if (PYTHON_MODULE_FILE.test(next) && !want.isDefault && !pythonStarAllows(next, want.exportedName, context)) continue; const chained = findExportedSymbol(next, want, language, context, visited, depth + 1); if (chained) return chained; } diff --git a/src/resolution/index.ts b/src/resolution/index.ts index cac05d6ce9..9362c39c2f 100644 --- a/src/resolution/index.ts +++ b/src/resolution/index.ts @@ -33,7 +33,7 @@ import { gateDartLocal, clearDartLocalScopeMemos } from './dart-local-scope'; import { clearCppTypeAliasMemos } from './cpp-type-aliases'; import { clearCppIncluderMemos } from './cpp-includers'; import { matchShopifyThemeFile } from './shopify-themes'; -import { resolveViaImport, resolvePhpImportedStaticCall, resolvePhpQualifiedClassRef, resolveJvmImport, extractImportMappings, extractReExports, loadCppIncludeDirs, isPhpIncludePathRef, isCobolCopybookRef, isNixPathImportRef, isDartImportRef, isLuaRequireRef, isJsPathImportRef, isBoundToOutOfRepoImport, importBindingTail, clearImportResolverMemos, resolveImportPath, isExternalImport } from './import-resolver'; +import { resolveViaImport, resolvePhpImportedStaticCall, resolvePhpQualifiedClassRef, resolveJvmImport, extractImportMappings, extractReExports, loadCppIncludeDirs, isPhpIncludePathRef, isCobolCopybookRef, isNixPathImportRef, isDartImportRef, isLuaRequireRef, isJsPathImportRef, isBoundToOutOfRepoImport, importBindingTail, clearImportResolverMemos, resolveImportPath, isExternalImport, pythonRootFingerprint, pythonReopenScope, pythonPackageImporters } from './import-resolver'; import { ResolverPool, minRefsForPool, shouldEngageAdaptively } from './resolver-pool'; import { resolveAliasBinding } from './alias-binding'; import { detectFrameworks, getResolvingFrameworks } from './frameworks'; @@ -1010,15 +1010,16 @@ export class ReferenceResolver { this.reExportCache.set(filePath, []); return []; } - // Re-exports are a JS/TS-only construct, and what matters is the - // BARREL file's own language — not the consuming reference's. A - // `.svelte`/`.vue` consumer threads its own language down the - // re-export chase, which would make extractReExports() bail on a - // `.ts` index barrel and silently break the chain (#629). Re-key + // What matters is the BARREL file's own language — not the consuming + // reference's. A `.svelte`/`.vue` consumer threads its own language + // down the re-export chase, which would make extractReExports() bail + // on a `.ts` index barrel and silently break the chain (#629). Re-key // the parse on the barrel's extension so the chase works no matter - // what kind of file imports through it. + // what kind of file imports through it. Python re-exports through its + // imports (a package `__init__.py`), so a `.py` barrel parses as Python. const isJsFamily = /\.(?:d\.ts|[cm]?tsx?|[cm]?jsx?|ets)$/i.test(filePath); - const reExports = extractReExports(content, isJsFamily ? 'typescript' : language); + const barrelLanguage = isJsFamily ? 'typescript' : /\.py$/i.test(filePath) ? 'python' : language; + const reExports = extractReExports(content, barrelLanguage); this.reExportCache.set(filePath, reExports); return reExports; }, @@ -2634,6 +2635,40 @@ export class ReferenceResolver { } } + /** + * Digest of the package roots Python import resolution rests on (see + * `pythonRootFingerprint`). Build configs are read from disk rather than + * the file cache, so a sync after an edit sees the edit. Empty for a + * project with no Python. + */ + getPythonRootFingerprint(): string { + return pythonRootFingerprint(this.diskContext()); + } + + /** What a sync must re-open after these Python files were added or removed (see `pythonReopenScope`). */ + getPythonReopenScope(changedFiles: string[]): ReturnType { + return pythonReopenScope(this.diskContext(), changedFiles); + } + + /** Importers of Python packages whose `__init__.py` was edited, with the names they bind (see `pythonPackageImporters`). */ + getPythonPackageImporters(initFiles: string[]): Map> { + return pythonPackageImporters(this.context, initFiles); + } + + /** The resolution context, reading build configs from disk rather than the file cache. */ + private diskContext(): ResolutionContext { + return { + ...this.context, + readFile: (filePath: string) => { + try { + return fs.readFileSync(path.join(this.projectRoot, filePath), 'utf-8'); + } catch { + return null; + } + }, + }; + } + /** * Get detected frameworks */ From 8a63b2aa5cfeee9b2d2a525c4a8f8c59163988a0 Mon Sep 17 00:00:00 2001 From: Eric Minish Date: Wed, 30 Sep 2026 22:48:54 -0400 Subject: [PATCH 2/5] fix(python): a name imported from a package the project lacks never binds to a nested copy of it `from requests import get; get("x")` still linked the call and the import to `get` in a copy of `requests` nested in a test fixture. The bare-call gate decided whether the project provides `requests` by path suffix, so any file ending in `requests/__init__.py` counted. It now asks the import resolver, which finds modules from the package roots, through a new optional `resolveModuleFile` on the resolution context. A python `imports` reference for a name from a module the project does not provide is not name-matched either. --- __tests__/python-module-resolution.test.ts | 34 ++++++++++++++++++++++ src/resolution/index.ts | 2 ++ src/resolution/name-matcher.ts | 27 +++++++++++++---- src/resolution/types.ts | 6 ++++ 4 files changed, 64 insertions(+), 5 deletions(-) diff --git a/__tests__/python-module-resolution.test.ts b/__tests__/python-module-resolution.test.ts index 10f89f64ca..5b217e8c47 100644 --- a/__tests__/python-module-resolution.test.ts +++ b/__tests__/python-module-resolution.test.ts @@ -303,6 +303,40 @@ def view(): expect(targets('views.py', 'app/views.py', 'imports')).toEqual([]); }); + it('never binds a name imported from a third-party package to a copy of it nested in a test fixture', async () => { + write({ + 'tests/fixtures/proj/pyproject.toml': '[project]\nname = "proj"\n', + 'tests/fixtures/proj/requests/__init__.py': 'def get(u):\n return u\n', + 'app/__init__.py': '', + 'app/client.py': `from requests import get + + +def fetch(): + return get("x") +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('fetch', 'app/client.py', 'calls')).toEqual([]); + expect(targets('client.py', 'app/client.py', 'imports')).toEqual([]); + }); + + it('still binds a name imported from a package the project provides at a root', async () => { + write({ + 'requests/__init__.py': 'def get(u):\n return u\n', + 'app/__init__.py': '', + 'app/client.py': `from requests import get + + +def fetch(): + return get("x") +`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('fetch', 'app/client.py', 'calls')).toEqual(['get@requests/__init__.py']); + }); + it('binds a top-level module the project really provides at a package root', async () => { write({ 'yaml.py': 'def safe_load(s):\n return s\n', diff --git a/src/resolution/index.ts b/src/resolution/index.ts index 9362c39c2f..efc70e6461 100644 --- a/src/resolution/index.ts +++ b/src/resolution/index.ts @@ -682,6 +682,8 @@ export class ReferenceResolver { if (resolved !== null && this.isIndexedFile(resolved)) return false; return this.isDeclaredOutsidePackage(source, fromFile); }, + resolveModuleFile: (source, fromFile, language) => + resolveImportPath(source, fromFile, language, this.context), getNodesInFile: (filePath: string) => { if (!this.nodeCache.has(filePath)) { this.nodeCache.set(filePath, this.queries.getNodesByFile(filePath)); diff --git a/src/resolution/name-matcher.ts b/src/resolution/name-matcher.ts index 02f3266170..3c57fcd561 100644 --- a/src/resolution/name-matcher.ts +++ b/src/resolution/name-matcher.ts @@ -3583,12 +3583,21 @@ function isPythonNameImportedFromOutside(name: string, ref: UnresolvedRef, conte memo = new Map(); PY_MODULE_LOCAL.set(context, memo); } - let local = memo.get(module); + // The import resolver decides where a module lives (package roots), and the + // answer can depend on the importer (a monorepo service's own root), so key + // by both. A file that merely ENDS in `requests/__init__.py` — a copy nested + // in a test fixture — is not the `requests` an import names. + const key = `${ref.filePath}\0${module}`; + let local = memo.get(key); if (local === undefined) { - const rel = module.replace(/\./g, '/'); - local = context.getAllFiles().some((f) => - f === `${rel}.py` || f.endsWith(`/${rel}.py`) || f === `${rel}/__init__.py` || f.endsWith(`/${rel}/__init__.py`)); - memo.set(module, local); + if (context.resolveModuleFile) { + local = context.resolveModuleFile(module, ref.filePath, 'python') !== null; + } else { + const rel = module.replace(/\./g, '/'); + local = context.getAllFiles().some((f) => + f === `${rel}.py` || f.endsWith(`/${rel}.py`) || f === `${rel}/__init__.py` || f.endsWith(`/${rel}/__init__.py`)); + } + memo.set(key, local); } return !local; } @@ -16077,6 +16086,14 @@ function matchReferenceInner( return matchFunctionRef(ref, context); } + // A python name imported from a module the project does not provide — `from + // requests import get` with no `requests` at a package root — names nothing + // in the project. Name matching would bind the import to any same-named + // symbol, such as a copy of the package nested in a test fixture. + if (ref.language === 'python' && ref.referenceKind === 'imports' && isPythonNameImportedFromOutside(ref.referenceName, ref, context)) { + return null; + } + // ArkTS chained UI attributes — emitted with a leading dot (`.titleStyle`, // `.width`) by the extractor — resolve ONLY to decorator-marked attribute // helpers: `@Extend`/`@Styles`/`@AnimatableExtend` functions (and global diff --git a/src/resolution/types.ts b/src/resolution/types.ts index d61c9a40b3..176b9c433f 100644 --- a/src/resolution/types.ts +++ b/src/resolution/types.ts @@ -184,6 +184,12 @@ export interface ResolutionContext { * to no project file. Supplied by the coordinator, like `resolveImport`. */ isOutOfRepoImport?(source: string, fromFile: string, language: Language): boolean; + /** + * The project file a module specifier, written in `fromFile`, names — the + * import resolver's own answer (for Python: from the package roots), or null + * when no project file provides it. Supplied by the coordinator. + */ + resolveModuleFile?(source: string, fromFile: string, language: Language): string | null; /** * Project import-path aliases (tsconfig/jsconfig `paths`). Returns * `null` when the project doesn't define any. Cached per resolver From c23651fa880d2f9e0311977d194d320950c5e4d2 Mon Sep 17 00:00:00 2001 From: Eric Minish Date: Thu, 15 Oct 2026 11:16:30 -0400 Subject: [PATCH 3/5] =?UTF-8?q?test(python):=20pin=20#2566=20=E2=80=94=20`?= =?UTF-8?q?from=20=20import=20name`=20never=20binds=20a=20same-n?= =?UTF-8?q?amed=20project=20symbol?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `from scipy import stats` bound `stats` by exact name to a top-level `stats` in another project file, and reported an import cycle when that file imports this one. The stdlib form and the aliased form (`from scipy import stats as st`) did the same. Package-root resolution already removes the edge; these cases fail on main and pass here. --- __tests__/python-module-resolution.test.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/__tests__/python-module-resolution.test.ts b/__tests__/python-module-resolution.test.ts index 5b217e8c47..36fc0dc514 100644 --- a/__tests__/python-module-resolution.test.ts +++ b/__tests__/python-module-resolution.test.ts @@ -321,6 +321,27 @@ def fetch(): expect(targets('client.py', 'app/client.py', 'imports')).toEqual([]); }); + // #2566: `from scipy import stats` bound `stats` by exact name to a + // top-level `stats` in another project file, and when that file imports + // this one, a cycle that does not exist. The stdlib form (`from os import + // path`) did the same. A module import (`import scipy.stats as stats`) never did. + it.each([ + ['a third-party module, against a variable', 'from scipy import stats', 'stats', 'stats = summary([1, 2, 3])\n'], + ['a third-party module, against a function', 'from scipy import stats', 'stats', 'def stats():\n return 1\n'], + ['a stdlib module, against a variable', 'from os import path', 'path', 'path = summary([1])\n'], + ['a third-party module under an alias, against a variable', 'from scipy import stats as st', 'st', 'st = summary([1])\n'], + ])('never binds a name imported from %s to a same-named project symbol (#2566)', async (_label, importLine, name, decl) => { + write({ + 'analysis.py': `${importLine}\n\n\ndef summary(values):\n return ${name}.describe(values)\n`, + 'report.py': `from analysis import summary\n\n${decl}`, + }); + cg = await CodeGraph.init(tempDir, { index: true }); + + expect(targets('analysis.py', 'analysis.py', 'imports')).toEqual([]); + expect(targets('report.py', 'report.py', 'imports')).toEqual(['summary@analysis.py']); + expect(cg.findCircularDependencies()).toEqual([]); + }); + it('still binds a name imported from a package the project provides at a root', async () => { write({ 'requests/__init__.py': 'def get(u):\n return u\n', From 59faa2a390107c4eb10e8540650e9a98b4eb156a Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 20 Oct 2026 12:20:33 -0500 Subject: [PATCH 4/5] fix(python): carry the package-root lookup into code main added since Two pieces of main reached Python modules the old way, which this branch replaces with package roots: - A function-local `import a.b as X` (PR 2607) found its module with the suffix match and its member with pythonModuleSymbol, both gone here. It now resolves the module like any other import and reads the member with findExportedSymbol, as the module-level path below it. - The outside-module receiver check (PR 2606) split out isPythonProjectModule without the importer, which the root-aware lookup needs to key its memo and to ask resolveModuleFile; it now takes the importing file. Co-Authored-By: Claude Opus 5.5 --- src/resolution/import-resolver.ts | 9 ++++++--- src/resolution/name-matcher.ts | 16 ++++++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/resolution/import-resolver.ts b/src/resolution/import-resolver.ts index 58b7ef648a..f9622cae61 100644 --- a/src/resolution/import-resolver.ts +++ b/src/resolution/import-resolver.ts @@ -2991,9 +2991,12 @@ function resolvePythonModuleMember( // module-level binding of X, and the file's mappings don't hold it (#2600). const local = pythonFunctionModuleAlias(receiver, ref, context); if (local) { - const localPath = resolveImportPath(local, ref.filePath, ref.language, context) ?? - findPythonModuleFile(local, context, ref.filePath)?.filePath ?? null; - const target = localPath && localPath !== ref.filePath ? pythonModuleSymbol(localPath, member, context, 0) : null; + // The module from the package roots, and its member as the module binds it, + // the same lookup as for a module-level binding below. + const localPath = resolveImportPath(local, ref.filePath, ref.language, context); + const target = localPath && localPath !== ref.filePath + ? findExportedSymbol(localPath, { isDefault: false, isNamespace: false, exportedName: member, memberName: null }, 'python', context, new Set()) + : null; return target ? { original: ref, targetNodeId: target.id, confidence: 0.85, resolvedBy: 'import' } : null; } diff --git a/src/resolution/name-matcher.ts b/src/resolution/name-matcher.ts index 0c1b35c1fd..986d51b15b 100644 --- a/src/resolution/name-matcher.ts +++ b/src/resolution/name-matcher.ts @@ -3889,11 +3889,11 @@ function pythonFromImports(filePath: string, context: ResolutionContext): Map @@ -3937,7 +3937,7 @@ export function isPythonOutsideModuleReceiver(name: string, ref: UnresolvedRef, // the module `a` tells that `x` comes from outside. return [...keys].every((key) => { const module = key.slice(0, key.indexOf(':')); - return !module.startsWith('.') && !isPythonProjectModuleAnyRoot(module, context); + return !module.startsWith('.') && !isPythonProjectModuleAnyRoot(module, ref.filePath, context); }); }); return outside && !pythonBindsLocally(name, ref, context, false); @@ -3948,9 +3948,9 @@ export function isPythonOutsideModuleReceiver(name: string, ref: UnresolvedRef, * with the package itself indexed as the root — `b/c.py` or `c.py`. Erring * toward "project" only leaves a call to the name strategies, as before. */ -function isPythonProjectModuleAnyRoot(module: string, context: ResolutionContext): boolean { +function isPythonProjectModuleAnyRoot(module: string, fromFile: string, context: ResolutionContext): boolean { const parts = module.split('.'); - return parts.some((_, i) => isPythonProjectModule(parts.slice(i).join('.'), context)); + return parts.some((_, i) => isPythonProjectModule(parts.slice(i).join('.'), fromFile, context)); } const JAVA_TYPE_KINDS: ReadonlySet = new Set(['class', 'interface', 'enum', 'struct', 'record', 'trait']); From 7c843e2425576f082d43d02e0f41782774701f78 Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Tue, 20 Oct 2026 12:30:28 -0500 Subject: [PATCH 5/5] fix(python): keep function-local imports out of the file's mappings The import mappings apply to every function of the file, so reading `import X as Y` at any indentation bound a function's own alias in the others: a parameter named like it, or a module-level import of the same name in another function, took that function's module (the two scoping tests from PR 2607). Plain imports are read at module level again, keeping this branch's comma lists and backslash continuations; a function-local `import a.b as X` binds X in its own function only, through pythonFunctionModuleAlias. Co-Authored-By: Claude Opus 5.5 --- __tests__/python-module-resolution.test.ts | 6 ++++-- src/resolution/import-resolver.ts | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/__tests__/python-module-resolution.test.ts b/__tests__/python-module-resolution.test.ts index 36fc0dc514..dca9ddcd2a 100644 --- a/__tests__/python-module-resolution.test.ts +++ b/__tests__/python-module-resolution.test.ts @@ -987,13 +987,15 @@ def main(): ]); }); - it('reads backslash continuations and comma-separated plain imports, at any indentation', () => { + it('reads backslash continuations and comma-separated plain imports, at module level', () => { + // A function's own `import lazy` is not the file's: the mappings apply to + // every function, and a function-local `import a.b as X` binds X in its + // function only (python-local-import-alias.test.ts). expect(names('from a import b, \\\n c\nimport os, pkg.sub as s\ndef f():\n import lazy\n')).toEqual([ 'b<-a:b', 'c<-a:c', 'os<-os:*', 's<-pkg.sub:*', - 'lazy<-lazy:*', ]); }); diff --git a/src/resolution/import-resolver.ts b/src/resolution/import-resolver.ts index f9622cae61..a401af1dd1 100644 --- a/src/resolution/import-resolver.ts +++ b/src/resolution/import-resolver.ts @@ -2007,9 +2007,10 @@ function extractPythonImports(content: string): ImportMapping[] { } } - // import X [as Y], … — at any indentation: a function-local import binds - // the name for the calls in that function. - const importRe = /^[ \t]*import[ \t]+((?:[^\n\\;]|\\\r?\n)+)/gm; + // import X [as Y], … at module level. The mappings are the file's, so a + // function-local import is left to its function: `import a.b as X` there + // binds X in that function only (pythonFunctionModuleAlias). + const importRe = /^import[ \t]+((?:[^\n\\;]|\\\r?\n)+)/gm; let match: RegExpExecArray | null; while ((match = importRe.exec(code)) !== null) { for (const raw of match[1]!.replace(/\\\r?\n/g, ' ').split(',')) {