diff --git a/CHANGELOG.md b/CHANGELOG.md index f15c72dc..e23fea4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ ### Fixed - (`core`) Fix a RuntimeException error when automatically correcting deprecated data paths in task args +### Changed +- (`core`) The `max_cores` parameter limits also the number of CPU cores pre-allocated for the training + ## 11.0.1.0 - 2026-07-02 ### Added diff --git a/khiops/core/api.py b/khiops/core/api.py index 3774c4c0..172eeed3 100644 --- a/khiops/core/api.py +++ b/khiops/core/api.py @@ -199,7 +199,12 @@ def _preprocess_arguments(args): if arg == "max_cores": max_cores = args[arg] if max_cores is not None: + # This `max_cores` system setting will be used in the khiops scenario + # to limit the CPU cores to use for the training system_settings.max_cores = int(max_cores) + # An additional environment variable (local to this specific run) + # MUST also be set to avoid allocating all the available CPU cores. + # Thus, allocated CPU cores = max number of CPU cores used elif arg == "memory_limit_mb": memory_limit_mb = args[arg] if memory_limit_mb is not None: diff --git a/khiops/core/internals/runner.py b/khiops/core/internals/runner.py index e210a3c5..ce9b889f 100644 --- a/khiops/core/internals/runner.py +++ b/khiops/core/internals/runner.py @@ -335,11 +335,18 @@ def _get_current_library_installer(): return "unknown" -def _build_khiops_process_environment(): +def _build_khiops_process_environment(system_settings=None): """Build a specific environment used for the execution of khiops in a process This environment can be modified freely without interfering with the global one. + + Parameters + ---------- + + system_settings: SystemSettings + Set of settings that must be taken into account + for this specific run """ khiops_env = os.environ.copy() @@ -347,6 +354,11 @@ def _build_khiops_process_environment(): # (using KHIOPS_MPI_HOME if it exists) if "HOME" not in khiops_env: khiops_env["HOME"] = khiops_env.get("KHIOPS_MPI_HOME", "") + if system_settings is not None and system_settings.max_cores is not None: + # An additional environment variable (local to this specific run) + # must also be set to avoid allocating all the available CPU cores. + # Thus, allocated CPU cores = max number of CPU cores used + khiops_env["KHIOPS_PROC_NUMBER"] = system_settings.max_cores return khiops_env @@ -697,6 +709,7 @@ class for more information. scenario_path, command_line_options, trace, + system_settings, ) # pylint: enable=assignment-from-no-return # Catch an OS level error if any @@ -902,6 +915,7 @@ def _run( scenario_path, command_line_options, trace, + system_settings, ): """Abstract run method to be implemented in child classes @@ -1466,7 +1480,14 @@ def _get_samples_dir(self): self._samples_dir_checked = True return self._samples_dir - def raw_run(self, tool_name, command_line_args=None, use_mpi=True, trace=False): + def raw_run( + self, + tool_name, + command_line_args=None, + use_mpi=True, + trace=False, + system_settings=None, + ): """Execute a Khiops tool with given command line arguments Parameters @@ -1479,6 +1500,11 @@ def raw_run(self, tool_name, command_line_args=None, use_mpi=True, trace=False): Whether to execute the application with MPI trace : bool, default False If `True` print the trace of the process. + system_settings: SystemSettings + Set of settings that must be taken into account + for this specific run + (Set the `KHIOPS_PROC_NUMBER` env var for each + training run using the value of `max_cores`) Examples -------- @@ -1521,9 +1547,9 @@ def raw_run(self, tool_name, command_line_args=None, use_mpi=True, trace=False): print(f"Khiops execution call: {khiops_call}") # Build custom Khiops process environment - # which makes sure HOME is defined and set + # which makes sure for example HOME is defined and set # according to khiops_env's KHIOPS_MPI_HOME - khiops_env = _build_khiops_process_environment() + khiops_env = _build_khiops_process_environment(system_settings) # Execute the process with subprocess.Popen( @@ -1546,11 +1572,15 @@ def _run( scenario_path, command_line_options, trace, + system_settings, ): # Execute the tool khiops_args = command_line_options.build_command_line_options(scenario_path) stdout, stderr, return_code = self.raw_run( - tool_name, command_line_args=khiops_args, trace=trace + tool_name, + command_line_args=khiops_args, + trace=trace, + system_settings=system_settings, ) return return_code, stdout, stderr diff --git a/khiops/extras/docker.py b/khiops/extras/docker.py index cd8d4df6..605f504e 100644 --- a/khiops/extras/docker.py +++ b/khiops/extras/docker.py @@ -103,6 +103,7 @@ def _run( scenario_path, command_line_options, trace, + system_settings, ): # Check arguments if command_line_options.output_scenario_path: diff --git a/tests/test_core.py b/tests/test_core.py index facdc667..345d05f5 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -2673,9 +2673,11 @@ def run( self, task, task_args, - command_line_options, + command_line_options=None, trace=False, system_settings=None, + stdout_file_path="", + stderr_file_path="", force_ansi_scenario=False, **kwargs, ): @@ -2711,6 +2713,7 @@ def _run( scenario_path, command_line_options, trace, + system_settings, ): return 0, "", ""