diff --git a/common/arg.cpp b/common/arg.cpp index 7ce7eaad16ab..c3b540f5a5fb 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1716,7 +1716,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, bool value) { params.kv_unified = value; } - ).set_env("LLAMA_ARG_KV_UNIFIED").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_BATCHED, LLAMA_EXAMPLE_BENCH, LLAMA_EXAMPLE_PARALLEL})); + ).set_env("LLAMA_ARG_KV_UNIFIED").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_BATCHED, LLAMA_EXAMPLE_BENCH, LLAMA_EXAMPLE_PARALLEL, LLAMA_EXAMPLE_FIT_PARAMS})); add_opt(common_arg( {"--cache-idle-slots"}, {"--no-cache-idle-slots"}, @@ -2866,6 +2866,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } } ).set_examples({LLAMA_EXAMPLE_FIT_PARAMS}).set_env("LLAMA_ARG_FIT_ESTIMATE")); + add_opt(common_arg( + { "--fit-json" }, + "report the fit as one JSON object on stdout: status, n_ctx, n_gpu_layers, tensor_split, overrides (Continuum fork; a caller reads a typed report, never the CLI text)", + [](common_params & params) { + params.fit_params_json = true; + } + ).set_examples({LLAMA_EXAMPLE_FIT_PARAMS})); add_opt(common_arg( { "-fitt", "--fit-target" }, "MiB0,MiB1,MiB2,...", string_format("target margin per device for --fit, comma-separated list of values, " diff --git a/common/common.h b/common/common.h index 29b171b7dda0..42a259bf5502 100644 --- a/common/common.h +++ b/common/common.h @@ -467,6 +467,7 @@ struct common_params { float tensor_split[128] = {0}; // how split tensors should be distributed across GPUs bool fit_params = true; // whether to fit unset model/context parameters to free device memory bool fit_params_print = false; // print the estimated required memory to run the model + bool fit_params_json = false; // Continuum fork: llama-fit-params reports the fit as one JSON object int32_t fit_params_min_ctx = 4096; // minimum context size to set when trying to reduce memory use // margin per device in bytes for fitting parameters to free memory: diff --git a/tools/fit-params/fit-params.cpp b/tools/fit-params/fit-params.cpp index 3e78c89290a7..20ca0be716a1 100644 --- a/tools/fit-params/fit-params.cpp +++ b/tools/fit-params/fit-params.cpp @@ -8,6 +8,8 @@ #include +#include + #if defined(_MSC_VER) #pragma warning(disable: 4244 4267) // possible loss of data #endif @@ -35,6 +37,33 @@ int llama_fit_params(int argc, char ** argv) { params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), params.fit_params_min_ctx, nullptr, params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR); + if (params.fit_params_json) { + // One typed report (Continuum fork): the caller deserializes it, never the CLI text below. + const char * name = "error"; + switch (status) { // every status named; a new one is a compile warning here (Cormac on #93) + case COMMON_PARAMS_FIT_STATUS_SUCCESS: name = "success"; break; + case COMMON_PARAMS_FIT_STATUS_FAILURE: name = "failure"; break; + case COMMON_PARAMS_FIT_STATUS_ERROR: name = "error"; break; + } + nlohmann::ordered_json report = { {"status", name} }; + if (status == COMMON_PARAMS_FIT_STATUS_SUCCESS) { + report["n_ctx"] = cparams.n_ctx; + report["n_gpu_layers"] = mparams.n_gpu_layers; + auto split = nlohmann::ordered_json::array(); + for (size_t id = 0; id < llama_max_devices(); id++) { + split.push_back(mparams.tensor_split[id]); + } + report["tensor_split"] = split; + auto overrides = nlohmann::ordered_json::array(); + for (size_t itbo = 0; itbo < llama_max_tensor_buft_overrides() && mparams.tensor_buft_overrides[itbo].pattern != nullptr; itbo++) { + overrides.push_back({ {"pattern", mparams.tensor_buft_overrides[itbo].pattern}, + {"buft", ggml_backend_buft_name(mparams.tensor_buft_overrides[itbo].buft)} }); + } + report["overrides"] = overrides; + } + printf("%s\n", report.dump().c_str()); + return status == COMMON_PARAMS_FIT_STATUS_SUCCESS ? 0 : 1; + } if (status != COMMON_PARAMS_FIT_STATUS_SUCCESS) { LOG_ERR("%s: failed to fit CLI arguments to free memory, exiting...\n", __func__); exit(1);