From 616e601846abb657a43051cd7492d28a30404ad6 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 20 Aug 2026 13:10:52 +0500 Subject: [PATCH] Document that idle_duration requires a nodes range --- mkdocs/docs/concepts/fleets.md | 24 ++++++++++++++----- .../docs/concepts/snippets/manage-fleets.ext | 3 ++- mkdocs/docs/guides/protips.md | 3 ++- src/dstack/_internal/core/models/fleets.py | 3 ++- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/mkdocs/docs/concepts/fleets.md b/mkdocs/docs/concepts/fleets.md index b74bd60c0..9e85b157e 100644 --- a/mkdocs/docs/concepts/fleets.md +++ b/mkdocs/docs/concepts/fleets.md @@ -318,12 +318,12 @@ Backend fleets allow you to set `backends` to specify which backends are allowed ### Idle duration -By default, instances of a backend fleet stay `idle` for 3 days and can be reused within that time. -If an instance is not reused within this period, it is automatically terminated. +Instances of a backend fleet that aren't running any jobs are `idle` and can be reused by new runs. +`dstack` terminates idle instances above [`nodes.min`](#nodes) after +[`idle_duration`](../reference/dstack.yml/fleet.md#idle_duration), which defaults to 3 days. -To change the default idle duration, set -[`idle_duration`](../reference/dstack.yml/fleet.md#idle_duration) in the fleet configuration (e.g., `0s`, `1m`, or `off` for -unlimited). +For `idle_duration` to have an effect, `nodes` must be a range. Set `min` to `0` and `target` to +the number of instances to pre-provision:
@@ -331,7 +331,9 @@ unlimited). type: fleet name: my-fleet -nodes: 2 +nodes: + min: 0 + target: 2 # Terminate instances idle for more than 1 hour idle_duration: 1h @@ -342,6 +344,16 @@ resources:
+`dstack apply` provisions `target` instances, and every instance that stays idle for +`idle_duration` is terminated until the fleet is down to `min`. Set `idle_duration` to `0s` to +terminate instances as soon as they become idle, or to `off` to never terminate them. + +!!! info "Fixed number of nodes" + Instances are never terminated on idle duration if the fleet is already at `nodes.min`, since + `dstack` would immediately provision new ones to maintain `min`. A fleet with a fixed size, + e.g. `nodes: 2`, has `min` equal to `2`, so it keeps its instances regardless of + `idle_duration`. + ### Blocks By default, a job uses the entire instance—e.g., all 8 GPUs. To allow multiple jobs on the same instance, set the `blocks` property to divide the instance. Each job can then use one or more blocks, up to the full instance. diff --git a/mkdocs/docs/concepts/snippets/manage-fleets.ext b/mkdocs/docs/concepts/snippets/manage-fleets.ext index 5680e497d..2b21b9f5a 100644 --- a/mkdocs/docs/concepts/snippets/manage-fleets.ext +++ b/mkdocs/docs/concepts/snippets/manage-fleets.ext @@ -36,6 +36,7 @@ the shorter of the fleet's and run's `idle_duration` is used. If the run reuses an existing fleet instance, only the fleet's [`idle_duration`](../reference/dstack.yml/fleet.md#idle_duration) applies. -If an instance remains `idle`, it is automatically terminated after `idle_duration`. +If an instance remains `idle`, it is automatically terminated after `idle_duration`, +unless the fleet is already at `nodes.min`. > Not applied for container-based backends (Kubernetes, Vast.ai, Runpod). diff --git a/mkdocs/docs/guides/protips.md b/mkdocs/docs/guides/protips.md index 02995f0c7..4a80ff6d2 100644 --- a/mkdocs/docs/guides/protips.md +++ b/mkdocs/docs/guides/protips.md @@ -270,7 +270,8 @@ If the run is submitted to a fleet with `nodes` set to a range and a new instanc If the run reuses an existing fleet instance, only the fleet's [`idle_duration`](../reference/dstack.yml/fleet.md#idle_duration) applies. -If an instance remains `idle`, it is automatically terminated after `idle_duration`. +If an instance remains `idle`, it is automatically terminated after `idle_duration`, +unless the fleet is already at `nodes.min`. > Not applied for container-based backends (Kubernetes, Vast.ai, Runpod). diff --git a/src/dstack/_internal/core/models/fleets.py b/src/dstack/_internal/core/models/fleets.py index 07911666a..94c44037c 100644 --- a/src/dstack/_internal/core/models/fleets.py +++ b/src/dstack/_internal/core/models/fleets.py @@ -296,7 +296,8 @@ class BackendFleetConfiguraionProps(CoreModel): Field( description=( "Time to wait before terminating idle instances." - " Instances are not terminated if the fleet is already at `nodes.min`." + " Only instances above `nodes.min` are terminated," + " so a fleet with a fixed `nodes: N` keeps its instances." " Defaults to `5m` for runs and `3d` for fleets." " Use `off` for unlimited duration" )