From 993cfc0fa542bd94ca5ae8c3a3aaf481bd55ecda Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 3 Nov 2026 19:31:31 -0600 Subject: [PATCH 1/3] llama-fit-params --fit-json: the fit as one typed JSON report (status, n_ctx, n_gpu_layers, tensor_split, overrides) Continuum sizes its one launch from the engine's own fit instead of its own formula (continuum card 4a20255d, ONE-BUDGET 3b): it runs llama-fit-params before launch with -c unset, the authority's margin (--fit-target) and the largest measured need (--fit-ctx), and passes the fitted arguments to llama-server. A caller reads a typed report, never the CLI text, so --fit-json prints one object built with nlohmann::json (override patterns are regexes; their backslashes are escaped by construction). A failed fit reports its status and exits non-zero. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- common/arg.cpp | 7 +++++++ common/common.h | 1 + tools/fit-params/fit-params.cpp | 24 ++++++++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/common/arg.cpp b/common/arg.cpp index 7ce7eaad16ab..6eb9481fff58 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2866,6 +2866,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } } ).set_examples({LLAMA_EXAMPLE_FIT_PARAMS}).set_env("LLAMA_ARG_FIT_ESTIMATE")); + add_opt(common_arg( + { "--fit-json" }, + "report the fit as one JSON object on stdout: status, n_ctx, n_gpu_layers, tensor_split, overrides (Continuum fork; a caller reads a typed report, never the CLI text)", + [](common_params & params) { + params.fit_params_json = true; + } + ).set_examples({LLAMA_EXAMPLE_FIT_PARAMS})); add_opt(common_arg( { "-fitt", "--fit-target" }, "MiB0,MiB1,MiB2,...", string_format("target margin per device for --fit, comma-separated list of values, " diff --git a/common/common.h b/common/common.h index 29b171b7dda0..42a259bf5502 100644 --- a/common/common.h +++ b/common/common.h @@ -467,6 +467,7 @@ struct common_params { float tensor_split[128] = {0}; // how split tensors should be distributed across GPUs bool fit_params = true; // whether to fit unset model/context parameters to free device memory bool fit_params_print = false; // print the estimated required memory to run the model + bool fit_params_json = false; // Continuum fork: llama-fit-params reports the fit as one JSON object int32_t fit_params_min_ctx = 4096; // minimum context size to set when trying to reduce memory use // margin per device in bytes for fitting parameters to free memory: diff --git a/tools/fit-params/fit-params.cpp b/tools/fit-params/fit-params.cpp index 3e78c89290a7..b07cb25df020 100644 --- a/tools/fit-params/fit-params.cpp +++ b/tools/fit-params/fit-params.cpp @@ -8,6 +8,8 @@ #include +#include + #if defined(_MSC_VER) #pragma warning(disable: 4244 4267) // possible loss of data #endif @@ -35,6 +37,28 @@ int llama_fit_params(int argc, char ** argv) { params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), params.fit_params_min_ctx, nullptr, params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR); + if (params.fit_params_json) { + // One typed report (Continuum fork): the caller deserializes it, never the CLI text below. + const char * names[] = { "success", "failure", "error" }; + nlohmann::ordered_json report = { {"status", names[status]} }; + if (status == COMMON_PARAMS_FIT_STATUS_SUCCESS) { + report["n_ctx"] = cparams.n_ctx; + report["n_gpu_layers"] = mparams.n_gpu_layers; + auto split = nlohmann::ordered_json::array(); + for (size_t id = 0; id < llama_max_devices(); id++) { + split.push_back(mparams.tensor_split[id]); + } + report["tensor_split"] = split; + auto overrides = nlohmann::ordered_json::array(); + for (size_t itbo = 0; itbo < llama_max_tensor_buft_overrides() && mparams.tensor_buft_overrides[itbo].pattern != nullptr; itbo++) { + overrides.push_back({ {"pattern", mparams.tensor_buft_overrides[itbo].pattern}, + {"buft", ggml_backend_buft_name(mparams.tensor_buft_overrides[itbo].buft)} }); + } + report["overrides"] = overrides; + } + printf("%s\n", report.dump().c_str()); + return status == COMMON_PARAMS_FIT_STATUS_SUCCESS ? 0 : 1; + } if (status != COMMON_PARAMS_FIT_STATUS_SUCCESS) { LOG_ERR("%s: failed to fit CLI arguments to free memory, exiting...\n", __func__); exit(1); From 7233724bf9e49e4a747d320bf541c27a79744b6c Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 3 Nov 2026 19:35:43 -0600 Subject: [PATCH 2/3] llama-fit-params takes --kv-unified: the fit sizes the engine Continuum launches (one shared pool), not a per-sequence one Without it the fit of 5 sequences read 1,310,720 tokens (5 x the trained window, one cache per sequence) for an engine launched with one unified pool. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- common/arg.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/arg.cpp b/common/arg.cpp index 6eb9481fff58..c3b540f5a5fb 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1716,7 +1716,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, bool value) { params.kv_unified = value; } - ).set_env("LLAMA_ARG_KV_UNIFIED").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_BATCHED, LLAMA_EXAMPLE_BENCH, LLAMA_EXAMPLE_PARALLEL})); + ).set_env("LLAMA_ARG_KV_UNIFIED").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_BATCHED, LLAMA_EXAMPLE_BENCH, LLAMA_EXAMPLE_PARALLEL, LLAMA_EXAMPLE_FIT_PARAMS})); add_opt(common_arg( {"--cache-idle-slots"}, {"--no-cache-idle-slots"}, From 18ecc4991dbe0adc95df27b4c0f6b39667ae066b Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 3 Nov 2026 19:40:32 -0600 Subject: [PATCH 3/3] llama-fit-params --fit-json: name the status with a switch, so a new status is a compile warning (Cormac on #93) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- tools/fit-params/fit-params.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/fit-params/fit-params.cpp b/tools/fit-params/fit-params.cpp index b07cb25df020..20ca0be716a1 100644 --- a/tools/fit-params/fit-params.cpp +++ b/tools/fit-params/fit-params.cpp @@ -39,8 +39,13 @@ int llama_fit_params(int argc, char ** argv) { params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR); if (params.fit_params_json) { // One typed report (Continuum fork): the caller deserializes it, never the CLI text below. - const char * names[] = { "success", "failure", "error" }; - nlohmann::ordered_json report = { {"status", names[status]} }; + const char * name = "error"; + switch (status) { // every status named; a new one is a compile warning here (Cormac on #93) + case COMMON_PARAMS_FIT_STATUS_SUCCESS: name = "success"; break; + case COMMON_PARAMS_FIT_STATUS_FAILURE: name = "failure"; break; + case COMMON_PARAMS_FIT_STATUS_ERROR: name = "error"; break; + } + nlohmann::ordered_json report = { {"status", name} }; if (status == COMMON_PARAMS_FIT_STATUS_SUCCESS) { report["n_ctx"] = cparams.n_ctx; report["n_gpu_layers"] = mparams.n_gpu_layers;