Skip to content

Optimize: skip logging and stats collection calls if they are no-ops #20839

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JukkaL merged 4 commits into from
Feb 17, 2026
+93 −50
Merged

Optimize: skip logging and stats collection calls if they are no-ops #20839

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
efd5c2b
Optimize: skip logging and stats collection calls if they are no-ops
JukkaL Feb 16, 2026
9d1bf7c
We need add_stats even when not collecting build stats
JukkaL Feb 17, 2026
b2b52c9
Update comment
JukkaL Feb 17, 2026
12e79e8
Simplify
JukkaL Feb 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 93 additions & 50 deletions mypy/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -799,6 +799,12 @@ def __init__(
if not isinstance(plugin, ChainedPlugin):
plugin = ChainedPlugin(options, [plugin])
self.plugin = plugin
# These allow quickly skipping logging and stats collection calls. Note
# that some stats impact mypy behavior, so be careful when skipping stats
# collection calls.
self.stats_enabled = self.options.dump_build_stats
self.logging_enabled = self.options.verbosity >= 1
self.tracing_enabled = self.options.verbosity >= 2
# Set of namespaces (module or class) that are being populated during semantic
# analysis and may have missing definitions.
self.incomplete_namespaces: set[str] = set()
Expand Down Expand Up @@ -913,7 +919,7 @@ def __init__(
self.known_partial_packages: dict[str, bool] = {}

def dump_stats(self) -> None:
if self.options.dump_build_stats:
if self.stats_enabled:
print("Stats:")
for key, value in sorted(self.stats_summary().items()):
print(f"{key + ':':24}{value}")
Expand Down Expand Up @@ -1463,14 +1469,23 @@ def read_deps_cache(manager: BuildManager, graph: Graph) -> dict[str, FgDepMeta]
return module_deps_metas


def _load_ff_file(file: str, manager: BuildManager, log_error: str) -> bytes | None:
t0 = time.time()
def _load_ff_file(
file: str, manager: BuildManager, log_error_fmt: str, id: str | None
) -> bytes | None:
if manager.stats_enabled:
t0 = time.time()
try:
data = manager.metastore.read(file)
except OSError:
manager.log(log_error + file)
if manager.logging_enabled:
if id:
message = log_error_fmt.format(id) + file
else:
message = log_error_fmt + file
manager.log(message)
return None
manager.add_stats(metastore_read_time=time.time() - t0)
if manager.stats_enabled:
manager.add_stats(metastore_read_time=time.time() - t0)
return data


Expand Down Expand Up @@ -1638,11 +1653,15 @@ def find_cache_meta(
"""
# TODO: May need to take more build options into account
meta_file, data_file, _ = get_cache_names(id, path, manager.options)
manager.trace(f"Looking for {id} at {meta_file}")
if manager.tracing_enabled:
manager.trace(f"Looking for {id} at {meta_file}")
meta: bytes | dict[str, Any] | None
t0 = time.time()
if manager.stats_enabled:
t0 = time.time()
if manager.options.fixed_format_cache:
meta = _load_ff_file(meta_file, manager, log_error=f"Could not load cache for {id}: ")
meta = _load_ff_file(
meta_file, manager, log_error_fmt="Could not load cache for {}: ", id=id
)
if meta is None:
return None
else:
Expand All @@ -1659,7 +1678,8 @@ def find_cache_meta(
f"Could not load cache for {id}: meta cache is not a dict: {repr(meta)}"
)
return None
t1 = time.time()
if manager.stats_enabled:
t1 = time.time()
if isinstance(meta, bytes):
# If either low-level buffer format or high-level cache layout changed, we
# cannot use the cache files, even with --skip-version-check.
Expand All @@ -1674,10 +1694,11 @@ def find_cache_meta(
if m is None:
manager.log(f"Metadata abandoned for {id}: cannot deserialize data")
return None
t2 = time.time()
manager.add_stats(
load_meta_time=t2 - t0, load_meta_load_time=t1 - t0, load_meta_from_dict_time=t2 - t1
)
if manager.stats_enabled:
t2 = time.time()
manager.add_stats(
load_meta_time=t2 - t0, load_meta_load_time=t1 - t0, load_meta_from_dict_time=t2 - t1
)
if skip_validation:
return m

Expand Down Expand Up @@ -1754,7 +1775,8 @@ def validate_meta(
manager.log(f"Metadata abandoned for {id}: errors were previously ignored")
return None

t0 = time.time()
if manager.stats_enabled:
t0 = time.time()
bazel = manager.options.bazel
assert path is not None, "Internal error: meta was provided without a path"
if not manager.options.skip_cache_mtime_checks:
Expand All @@ -1779,7 +1801,8 @@ def validate_meta(
manager.log(f"Metadata abandoned for {id}: file or directory {path} does not exist")
return None

manager.add_stats(validate_stat_time=time.time() - t0)
if manager.stats_enabled:
manager.add_stats(validate_stat_time=time.time() - t0)

# When we are using a fine-grained cache, we want our initial
# build() to load all of the cache information and then do a
Expand Down Expand Up @@ -1835,25 +1858,31 @@ def validate_meta(
manager.log(f"Metadata abandoned for {id}: file {path} has different hash")
return None
else:
t0 = time.time()
if manager.stats_enabled:
t0 = time.time()
# Optimization: update mtime and path (otherwise, this mismatch will reappear).
meta.mtime = mtime
meta.path = path
meta.size = size
meta.options = options_snapshot(id, manager)
meta_file, _, _ = get_cache_names(id, path, manager.options)
manager.log(
"Updating mtime for {}: file {}, meta {}, mtime {}".format(
id, path, meta_file, meta.mtime
if manager.logging_enabled:
manager.log(
"Updating mtime for {}: file {}, meta {}, mtime {}".format(
id, path, meta_file, meta.mtime
)
)
)
write_cache_meta(meta, manager, meta_file)
t1 = time.time()
manager.add_stats(validate_update_time=time.time() - t1, validate_munging_time=t1 - t0)
if manager.stats_enabled:
t1 = time.time()
manager.add_stats(
validate_update_time=time.time() - t1, validate_munging_time=t1 - t0
)
return meta

# It's a match on (id, path, size, hash, mtime).
manager.log(f"Metadata fresh for {id}: file {path}")
if manager.logging_enabled:
manager.log(f"Metadata fresh for {id}: file {path}")
return meta


Expand Down Expand Up @@ -2322,9 +2351,11 @@ def new_state(
if path and source is None and manager.fscache.isdir(path):
source = ""

t0 = time.time()
if manager.stats_enabled:
t0 = time.time()
meta = validate_meta(meta, id, path, ignore_all, manager)
manager.add_stats(validate_meta_time=time.time() - t0)
if manager.stats_enabled:
manager.add_stats(validate_meta_time=time.time() - t0)

if meta:
# Make copies, since we may modify these and want to
Expand Down Expand Up @@ -2672,7 +2703,7 @@ def load_tree(self, temporary: bool = False) -> None:

data: bytes | dict[str, Any] | None
if self.options.fixed_format_cache:
data = _load_ff_file(data_file, self.manager, "Could not load tree: ")
data = _load_ff_file(data_file, self.manager, "Could not load tree: ", None)
else:
data = _load_json_file(data_file, self.manager, "Load tree ", "Could not load tree: ")
if data is None:
Expand Down Expand Up @@ -3349,19 +3380,23 @@ def exist_removed_submodules(dependencies: list[str], manager: BuildManager) ->

def find_module_simple(id: str, manager: BuildManager) -> str | None:
"""Find a filesystem path for module `id` or `None` if not found."""
t0 = time.time()
if manager.stats_enabled:
t0 = time.time()
x = manager.find_module_cache.find_module(id, fast_path=True)
manager.add_stats(find_module_time=time.time() - t0, find_module_calls=1)
if manager.stats_enabled:
manager.add_stats(find_module_time=time.time() - t0, find_module_calls=1)
if isinstance(x, ModuleNotFoundReason):
return None
return x


def find_module_with_reason(id: str, manager: BuildManager) -> ModuleSearchResult:
"""Find a filesystem path for module `id` or the reason it can't be found."""
t0 = time.time()
if manager.stats_enabled:
t0 = time.time()
x = manager.find_module_cache.find_module(id, fast_path=False)
manager.add_stats(find_module_time=time.time() - t0, find_module_calls=1)
if manager.stats_enabled:
manager.add_stats(find_module_time=time.time() - t0, find_module_calls=1)
return x


Expand Down Expand Up @@ -3476,6 +3511,9 @@ def skipping_ancestor(manager: BuildManager, id: str, path: str, ancestor_for: S
def log_configuration(manager: BuildManager, sources: list[BuildSource]) -> None:
"""Output useful configuration information to LOG and TRACE"""

if not manager.logging_enabled:
return

config_file = manager.options.config_file
if config_file:
config_file = os.path.abspath(config_file)
Expand Down Expand Up @@ -3938,23 +3976,25 @@ def find_stale_sccs(
if stale_indirect is not None:
fresh = False

if fresh:
fresh_msg = "fresh"
elif stale_scc:
fresh_msg = "inherently stale"
if stale_scc != ascc.mod_ids:
fresh_msg += f" ({' '.join(sorted(stale_scc))})"
if stale_deps:
fresh_msg += f" with stale deps ({' '.join(sorted(stale_deps))})"
elif stale_deps:
fresh_msg = f"stale due to deps ({' '.join(sorted(stale_deps))})"
else:
assert stale_indirect is not None
fresh_msg = f"stale due to stale indirect dep(s): first {stale_indirect}"
if manager.logging_enabled:
if fresh:
fresh_msg = "fresh"
elif stale_scc:
fresh_msg = "inherently stale"
if stale_scc != ascc.mod_ids:
fresh_msg += f" ({' '.join(sorted(stale_scc))})"
if stale_deps:
fresh_msg += f" with stale deps ({' '.join(sorted(stale_deps))})"
elif stale_deps:
fresh_msg = f"stale due to deps ({' '.join(sorted(stale_deps))})"
else:
assert stale_indirect is not None
fresh_msg = f"stale due to stale indirect dep(s): first {stale_indirect}"
scc_str = " ".join(ascc.mod_ids)

scc_str = " ".join(ascc.mod_ids)
if fresh:
manager.trace(f"Found {fresh_msg} SCC ({scc_str})")
if manager.tracing_enabled:
manager.trace(f"Found {fresh_msg} SCC ({scc_str})")
# If there is at most one file with errors we can skip the ordering to save time.
mods_with_errors = [id for id in ascc.mod_ids if graph[id].error_lines]
if len(mods_with_errors) <= 1:
Expand All @@ -3971,11 +4011,14 @@ def find_stale_sccs(
manager.flush_errors(path, formatted, False)
fresh_sccs.append(ascc)
else:
size = len(ascc.mod_ids)
if size == 1:
manager.log(f"Scheduling SCC singleton ({scc_str}) as {fresh_msg}")
else:
manager.log("Scheduling SCC of size %d (%s) as %s" % (size, scc_str, fresh_msg))
if manager.logging_enabled:
size = len(ascc.mod_ids)
if size == 1:
manager.log(f"Scheduling SCC singleton ({scc_str}) as {fresh_msg}")
else:
manager.log(
"Scheduling SCC of size %d (%s) as %s" % (size, scc_str, fresh_msg)
)
stale_sccs.append(ascc)
return stale_sccs, fresh_sccs

Expand Down