diff --git a/pathwaysutils/elastic/manager.py b/pathwaysutils/elastic/manager.py index 1f48d5c..7b4dd4a 100644 --- a/pathwaysutils/elastic/manager.py +++ b/pathwaysutils/elastic/manager.py @@ -52,7 +52,7 @@ class ScaleUpSignalError(Exception): def _elastic_event_cleanup() -> None: - """Cleans up JAX profiles, caches, and live arrays.""" + """Stops any ongoing JAX profiler trace.""" try: _logger.info("Cleaning up any ongoing traces") jax.profiler.stop_trace() @@ -62,10 +62,6 @@ def _elastic_event_cleanup() -> None: _logger.exception("Error cleaning up ongoing traces") raise - jax.clear_caches() - for array in jax.live_arrays(): - array.delete() - class ElasticRetryLimit: """A retry callback that limits the number of attempts.""" @@ -299,8 +295,7 @@ def elastic_retry( This decorator wraps a function to automatically retry execution in case of `jax.errors.JaxRuntimeError` caused by slice down events. It waits for - `minimum_slice_count` active slices before each attempt and cleans up JAX - caches on failure. + `minimum_slice_count` active slices before each attempt. If `minimum_slice_count` is not met, the function will wait until at least `minimum_slice_count` slices are active before execution. If @@ -313,6 +308,12 @@ def elastic_retry( boundaries) and raise a `ScaleUpSignalError` to gracefully interrupt the current execution and trigger a retry with the expanded hardware. + On an elastic event the arrays of the failed attempt are not deleted + explicitly; JAX frees them once the attempt has unwound and nothing + references them anymore. The wrapped function and the callbacks must not + keep references to them, or the memory stays allocated during the next + attempt. + Often, the function will dispatch JAX operations and wait for them to complete while creating a log message. If using Python logging, it is recommended to set `logging.raiseExceptions=True` to ensure that the