diff --git a/common/arg.cpp b/common/arg.cpp index c3b540f5a5fb..4353464d2847 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1692,13 +1692,18 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } ).set_env("LLAMA_ARG_CTX_CHECKPOINTS").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); add_opt(common_arg( - {"-cms", "--checkpoint-min-step"}, "N", - string_format("minimum spacing between context checkpoints in tokens (default: %d, 0 = no minimum)", params.checkpoint_min_step), - [](common_params & params, int value) { - if (value < 0) { - throw std::invalid_argument("checkpoint-min-step must be non-negative"); + {"-cms", "--checkpoint-min-step"}, "N|auto", + string_format("minimum spacing between context checkpoints in tokens (default: %d, 0 = no minimum, auto = each sequence's prompt over --ctx-checkpoints, so its checkpoints cover its whole prompt)", params.checkpoint_min_step), + [](common_params & params, const std::string & value) { + if (value == "auto") { + params.checkpoint_min_step = -1; // per sequence, from its own prompt (server_context::checkpoint_step) + return; + } + const int n = std::stoi(value); + if (n < 0) { + throw std::invalid_argument("checkpoint-min-step must be non-negative or auto"); } - params.checkpoint_min_step = value; + params.checkpoint_min_step = n; } ).set_env("LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT").set_examples({LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( @@ -2898,6 +2903,28 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } } ).set_env("LLAMA_ARG_FIT_TARGET")); + add_opt(common_arg( + { "--fit-budget" }, "MiB0,MiB1,MiB2,...", + "the memory the engine may use per device for --fit (Continuum fork): the fit keeps at least (free - budget) of each device " + "free, so the engine stays within the budget its node's memory authority grants; a single value is broadcast across all devices", + [](common_params & params, const std::string & value) { + const std::vector split_arg = string_split(value, ','); + if (split_arg.size() > llama_max_devices()) { + throw std::invalid_argument(string_format("got %zu budgets but only %zu devices are supported", split_arg.size(), llama_max_devices())); + } + // stoull wraps a leading '-' into a huge value, which would silently mean no budget + const auto mib = [](const std::string & v) -> size_t { + if (v.empty() || v[0] == '-') { + throw std::invalid_argument("--fit-budget takes non-negative MiB values, got '" + v + "'"); + } + return std::stoull(v) * 1024*1024; + }; + params.fit_params_budget.assign(llama_max_devices(), mib(split_arg[0])); + for (size_t i = 1; i < split_arg.size(); i++) { + params.fit_params_budget[i] = mib(split_arg[i]); + } + } + ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_FIT_PARAMS})); add_opt(common_arg( { "-fitc", "--fit-ctx" }, "N", string_format("minimum ctx size that can be set by --fit option, default: %" PRIu32, params.fit_params_min_ctx), diff --git a/common/common.cpp b/common/common.cpp index bda497246329..e95a1862b8d7 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1322,6 +1322,7 @@ common_init_result::common_init_result(common_params & params, bool model_only) params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), + params.fit_params_budget.empty() ? nullptr : params.fit_params_budget.data(), params.fit_params_min_ctx, has_draft || spec_mtp ? &extra : nullptr, params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR); diff --git a/common/common.h b/common/common.h index 42a259bf5502..1fb343b93eca 100644 --- a/common/common.h +++ b/common/common.h @@ -472,6 +472,7 @@ struct common_params { // margin per device in bytes for fitting parameters to free memory: std::vector fit_params_target = std::vector(llama_max_devices(), 1024 * 1024*1024); + std::vector fit_params_budget; // Continuum fork: bytes the engine may use per device (--fit-budget); empty = no budget enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs enum llama_load_mode load_mode = LLAMA_LOAD_MODE_AUTO; // how to load the model diff --git a/common/fit.cpp b/common/fit.cpp index c601fe405ea5..27d4b9d1d1b4 100644 --- a/common/fit.cpp +++ b/common/fit.cpp @@ -178,7 +178,7 @@ common_device_memory_data_vec common_get_device_memory_data( static void common_params_fit_impl( const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams, float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides, - size_t * margins_s, uint32_t n_ctx_min, const common_fit_extra_model * extra, enum ggml_log_level log_level) { + size_t * margins_s, const size_t * budgets, uint32_t n_ctx_min, const common_fit_extra_model * extra, enum ggml_log_level log_level) { if (mparams->split_mode == LLAMA_SPLIT_MODE_TENSOR) { throw common_params_fit_exception("llama_params_fit is not implemented for SPLIT_MODE_TENSOR, abort"); } @@ -289,6 +289,19 @@ static void common_params_fit_impl( } } + // Continuum fork: a budget is the memory the engine may use on a device, so the fit leaves at + // least (free - budget) of it free; the node's memory authority decides the budget, the fit + // decides what fits inside it. + if (budgets != nullptr) { + if (nd == 0) { + margins[0] = std::max(margins[0], int64_t(dmds_full.back().total) - int64_t(budgets[0])); + } else { + for (size_t id = 0; id < nd; id++) { + margins[id] = std::max(margins[id], int64_t(dmds_full[id].free) - int64_t(budgets[id])); + } + } + } + std::vector dev_names; { dev_names.reserve(nd); @@ -882,13 +895,14 @@ enum common_params_fit_status common_fit_params( float * tensor_split, llama_model_tensor_buft_override * tensor_buft_overrides, size_t * margins, + const size_t * budgets, uint32_t n_ctx_min, const common_fit_extra_model * extra, ggml_log_level log_level) { const int64_t t0_us = llama_time_us(); common_params_fit_status status = COMMON_PARAMS_FIT_STATUS_SUCCESS; try { - common_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level); + common_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, budgets, n_ctx_min, extra, log_level); LOG_TRC("%s: successfully fit params to free device memory\n", __func__); } catch (const common_params_fit_exception & e) { LOG_WRN("%s: failed to fit params to free device memory: %s\n", __func__, e.what()); diff --git a/common/fit.h b/common/fit.h index 824d386b07a1..f912ae57e720 100644 --- a/common/fit.h +++ b/common/fit.h @@ -33,6 +33,7 @@ common_params_fit_status common_fit_params( float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements size_t * margins, // margins of memory to leave per device in bytes + const size_t * budgets, // Continuum fork: bytes the engine may use per device, nullptr = no budget uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log diff --git a/tools/fit-params/fit-params.cpp b/tools/fit-params/fit-params.cpp index 20ca0be716a1..db8239616e90 100644 --- a/tools/fit-params/fit-params.cpp +++ b/tools/fit-params/fit-params.cpp @@ -34,7 +34,8 @@ int llama_fit_params(int argc, char ** argv) { if (!params.fit_params_print) { const common_params_fit_status status = common_fit_params(params.model.path.c_str(), &mparams, &cparams, - params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), params.fit_params_min_ctx, + params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), + params.fit_params_budget.empty() ? nullptr : params.fit_params_budget.data(), params.fit_params_min_ctx, nullptr, params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR); if (params.fit_params_json) { diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cpp index a2da93b9a282..8843d5ae1761 100644 --- a/tools/llama-bench/llama-bench.cpp +++ b/tools/llama-bench/llama-bench.cpp @@ -2293,6 +2293,7 @@ int llama_bench(int argc, char ** argv) { fit_tensor_split.data(), fit_overrides.data(), margins.data(), + nullptr, // no budget: the bench fits to free memory inst.fit_min_ctx, nullptr, params.verbose ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR); diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 9dd7a34d11b6..e80169b34519 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1363,8 +1363,13 @@ struct server_context_impl { SRV_TRC("%s", "for more info see https://github.com/ggml-org/llama.cpp/pull/16391\n"); if (params_base.n_ctx_checkpoints > 0) { - SRV_TRC("context checkpoints enabled, max = %d, min spacing = %d\n", - params_base.n_ctx_checkpoints, params_base.checkpoint_min_step); + if (params_base.checkpoint_min_step < 0) { + SRV_TRC("context checkpoints enabled, max = %d, min spacing = each prompt over %d\n", + params_base.n_ctx_checkpoints, params_base.n_ctx_checkpoints); + } else { + SRV_TRC("context checkpoints enabled, max = %d, min spacing = %d\n", + params_base.n_ctx_checkpoints, params_base.checkpoint_min_step); + } } else { SRV_TRC("%s", "context checkpoints disabled\n"); } @@ -2462,15 +2467,31 @@ struct server_context_impl { return true; } + // Continuum fork: the minimum spacing between this sequence's checkpoints. A fixed + // `--checkpoint-min-step` is used as given; `auto` spaces them so `--ctx-checkpoints` of them + // cover this sequence's own prompt, never a slot window: under `--kv-unified` a slot's window + // is the whole pool, and pool / 32 (8,192 on a 262,144 pool) skipped every divergence point + // of a ~5k-token turn (continuum, measured cache_n 0 on every turn, 2026-09-13). + int64_t checkpoint_step(const server_slot & slot) const { + if (params_base.checkpoint_min_step >= 0) { + return params_base.checkpoint_min_step; + } + if (params_base.n_ctx_checkpoints <= 0) { + return 0; + } + return (slot.task->n_tokens() + params_base.n_ctx_checkpoints - 1) / params_base.n_ctx_checkpoints; + } + // n_tokens_cur: the number of tokens added to the batch for the current slot void create_checkpoint(server_slot & slot, const int64_t n_tokens_cur, llama_pos pos_min, llama_pos pos_max) { const int id_task = slot.task->id; // evict checkpoints within min-step of a previous checkpoint, unless they were // created by the current task + const int64_t step = checkpoint_step(slot); int64_t last = -1; for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end(); ) { - if (it->id_task != id_task && last >= 0 && it->n_tokens <= last + params_base.checkpoint_min_step) { + if (it->id_task != id_task && last >= 0 && it->n_tokens <= last + step) { SLT_TRC(slot, "erasing context checkpoint too close to an earlier one (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", it->pos_min, it->pos_max, it->n_tokens, (float) it->size() / 1024 / 1024); @@ -4185,7 +4206,7 @@ struct server_context_impl { const auto pos = slot.prompt.n_tokens(); const auto & checkpoints = slot.prompt.checkpoints; - if (pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back().n_tokens + params_base.checkpoint_min_step) { + if (pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back().n_tokens + checkpoint_step(slot)) { break; } } @@ -4269,7 +4290,7 @@ struct server_context_impl { do_checkpoint = do_checkpoint && ( slot.prompt.checkpoints.empty() || is_last_user_message || near_prompt_end || is_view_end || - n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step); + n_tokens_start > slot.prompt.checkpoints.back().n_tokens + checkpoint_step(slot)); SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max); // note: we create the checkpoint before calling llama_decode(), so the current batch is not