Skip to content

Commit 78b6356

Browse files
committed
perf(core): discover manifests in a single filesystem walk
find_files() started a separate recursive rglob traversal for every expanded manifest pattern, so a scan re-walked each root once per pattern and only filtered excluded directories after descending into them. Replace that with one os.walk() per scan root: - Expand and case-fold all active patterns once, then match in memory. - Prune excluded directories, including .git, before descending. - Reject non-manifests on the basename alone (one set lookup plus one compiled glob alternation) before building a relative path or running a path match. - Cache supported manifest patterns per Core instance, but only when the API lookup succeeds, so a transient failure does not pin the run to the smaller local fallback pattern set. - Emit INFO durations for organization setup, pattern retrieval and discovery, with files/directories visited, directories pruned and manifests found. Matching behaviour is unchanged apart from intentionally excluding .git metadata. Adds parity tests against the previous rglob implementation for every built-in ecosystem and pattern, covering case-insensitivity, brace expansion, nested patterns, dot-directories, exclusions, inclusions, symlinks, excluded ecosystems, multiple roots, sorting and deduplication, plus an opt-in benchmark that asserts old/new result equality on a synthetic large-monorepo fixture. Ref: CE-379
1 parent 0d3937d commit 78b6356

3 files changed

Lines changed: 665 additions & 67 deletions

File tree

benchmarks/manifest_discovery.py

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
#!/usr/bin/env python3
2+
"""Compare legacy per-pattern rglob discovery with the single-pass walker.
3+
4+
This is an opt-in developer benchmark, not a timing assertion in the test
5+
suite. It creates a synthetic monorepo so filesystem or CI-agent changes do not
6+
make regular tests flaky.
7+
"""
8+
9+
import argparse
10+
import tempfile
11+
import time
12+
from pathlib import Path
13+
from types import SimpleNamespace
14+
from unittest.mock import MagicMock
15+
16+
from socketsecurity.core import Core
17+
from socketsecurity.core.socket_config import SocketConfig
18+
from socketsecurity.core.utils import socket_globs
19+
20+
21+
def seed_tree(root: Path, directories: int, files_per_directory: int) -> None:
22+
for directory_index in range(directories):
23+
directory = root / "packages" / f"package-{directory_index:05d}"
24+
directory.mkdir(parents=True)
25+
(directory / "package.json").write_text("{}\n", encoding="utf-8")
26+
for file_index in range(files_per_directory):
27+
(directory / f"source-{file_index:03d}.txt").write_text(
28+
"not a manifest\n",
29+
encoding="utf-8",
30+
)
31+
32+
# These trees model the expensive directories that the new walker prunes
33+
# before descent rather than visiting once for every manifest pattern.
34+
for excluded in (".git/objects", "node_modules/example", ".venv/site-packages"):
35+
directory = root / excluded
36+
directory.mkdir(parents=True)
37+
for index in range(files_per_directory * 10):
38+
(directory / f"object-{index:05d}").write_text("x", encoding="utf-8")
39+
40+
41+
def legacy_discover(root: Path) -> set[str]:
42+
results = set()
43+
excluded_dirs = SocketConfig(api_key="benchmark").excluded_dirs
44+
for ecosystem_patterns in socket_globs.values():
45+
for details in ecosystem_patterns.values():
46+
for pattern in Core.expand_brace_pattern(details["pattern"]):
47+
insensitive = Core.to_case_insensitive_regex(pattern)
48+
for candidate in root.rglob(insensitive):
49+
if candidate.is_file() and not Core.is_excluded(
50+
str(candidate),
51+
excluded_dirs,
52+
):
53+
results.add(candidate.as_posix())
54+
return results
55+
56+
57+
def new_core() -> Core:
58+
core = Core.__new__(Core)
59+
core.config = SocketConfig(api_key="benchmark")
60+
core.cli_config = SimpleNamespace(exclude_paths=None)
61+
core.sdk = MagicMock()
62+
core._supported_patterns = socket_globs
63+
return core
64+
65+
66+
def timed(function, root: Path) -> tuple[set[str], float]:
67+
start = time.perf_counter()
68+
results = set(function(root))
69+
return results, time.perf_counter() - start
70+
71+
72+
def main() -> None:
73+
parser = argparse.ArgumentParser()
74+
parser.add_argument("--directories", type=int, default=500)
75+
parser.add_argument("--files-per-directory", type=int, default=20)
76+
args = parser.parse_args()
77+
78+
with tempfile.TemporaryDirectory(prefix="socket-manifest-benchmark-") as temp:
79+
root = Path(temp)
80+
seed_tree(root, args.directories, args.files_per_directory)
81+
legacy_results, legacy_seconds = timed(legacy_discover, root)
82+
new_results, new_seconds = timed(
83+
lambda path: new_core().find_files(str(path)),
84+
root,
85+
)
86+
87+
if legacy_results != new_results:
88+
raise SystemExit(
89+
"Manifest result mismatch: "
90+
f"legacy={len(legacy_results)}, single_pass={len(new_results)}"
91+
)
92+
93+
speedup = legacy_seconds / new_seconds if new_seconds else float("inf")
94+
print(f"Manifests: {len(new_results)}")
95+
print(f"Legacy per-pattern rglob: {legacy_seconds:.3f}s")
96+
print(f"Single-pass walk: {new_seconds:.3f}s")
97+
print(f"Speedup: {speedup:.1f}x")
98+
99+
100+
if __name__ == "__main__":
101+
main()

0 commit comments

Comments
 (0)