llama-fit-params: --fit-json (one typed report) and --kv-unified (fit the engine we launch)

common/arg.cpp

@@ -1716,7 +1716,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         [](common_params & params, bool value) {
             params.kv_unified = value;
         }
-    ).set_env("LLAMA_ARG_KV_UNIFIED").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_BATCHED, LLAMA_EXAMPLE_BENCH, LLAMA_EXAMPLE_PARALLEL}));
+    ).set_env("LLAMA_ARG_KV_UNIFIED").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_BATCHED, LLAMA_EXAMPLE_BENCH, LLAMA_EXAMPLE_PARALLEL, LLAMA_EXAMPLE_FIT_PARAMS}));
     add_opt(common_arg(
         {"--cache-idle-slots"},
         {"--no-cache-idle-slots"},
@@ -2866,6 +2866,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             }
         }
     ).set_examples({LLAMA_EXAMPLE_FIT_PARAMS}).set_env("LLAMA_ARG_FIT_ESTIMATE"));
+    add_opt(common_arg(
+        { "--fit-json" },
+        "report the fit as one JSON object on stdout: status, n_ctx, n_gpu_layers, tensor_split, overrides (Continuum fork; a caller reads a typed report, never the CLI text)",
+        [](common_params & params) {
+            params.fit_params_json = true;
+        }
+    ).set_examples({LLAMA_EXAMPLE_FIT_PARAMS}));
     add_opt(common_arg(
         { "-fitt", "--fit-target" }, "MiB0,MiB1,MiB2,...",
         string_format("target margin per device for --fit, comma-separated list of values, "

common/common.h

@@ -467,6 +467,7 @@ struct common_params {
     float   tensor_split[128]  = {0};   // how split tensors should be distributed across GPUs
     bool    fit_params         = true;  // whether to fit unset model/context parameters to free device memory
     bool    fit_params_print   = false; // print the estimated required memory to run the model
+    bool fit_params_json  = false; // Continuum fork: llama-fit-params reports the fit as one JSON object
     int32_t fit_params_min_ctx = 4096;  // minimum context size to set when trying to reduce memory use
 
     // margin per device in bytes for fitting parameters to free memory:

tools/fit-params/fit-params.cpp

@@ -8,6 +8,8 @@
 
 #include <cinttypes>
 
+#include <nlohmann/json.hpp>
+
 #if defined(_MSC_VER)
 #pragma warning(disable: 4244 4267) // possible loss of data
 #endif
@@ -35,6 +37,33 @@ int llama_fit_params(int argc, char ** argv) {
                 params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), params.fit_params_min_ctx,
                 nullptr,
                 params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
+        if (params.fit_params_json) {
+            // One typed report (Continuum fork): the caller deserializes it, never the CLI text below.
+            const char * name = "error";
+            switch (status) { // every status named; a new one is a compile warning here (Cormac on #93)
+                case COMMON_PARAMS_FIT_STATUS_SUCCESS: name = "success"; break;
+                case COMMON_PARAMS_FIT_STATUS_FAILURE: name = "failure"; break;
+                case COMMON_PARAMS_FIT_STATUS_ERROR:   name = "error";   break;
+            }
+            nlohmann::ordered_json report = { {"status", name} };
+            if (status == COMMON_PARAMS_FIT_STATUS_SUCCESS) {
+                report["n_ctx"]        = cparams.n_ctx;
+                report["n_gpu_layers"] = mparams.n_gpu_layers;
+                auto split = nlohmann::ordered_json::array();
+                for (size_t id = 0; id < llama_max_devices(); id++) {
+                    split.push_back(mparams.tensor_split[id]);
+                }
+                report["tensor_split"] = split;
+                auto overrides = nlohmann::ordered_json::array();
+                for (size_t itbo = 0; itbo < llama_max_tensor_buft_overrides() && mparams.tensor_buft_overrides[itbo].pattern != nullptr; itbo++) {
+                    overrides.push_back({ {"pattern", mparams.tensor_buft_overrides[itbo].pattern},
+                                          {"buft",    ggml_backend_buft_name(mparams.tensor_buft_overrides[itbo].buft)} });
+                }
+                report["overrides"] = overrides;
+            }
+            printf("%s\n", report.dump().c_str());
+            return status == COMMON_PARAMS_FIT_STATUS_SUCCESS ? 0 : 1;
+        }
         if (status != COMMON_PARAMS_FIT_STATUS_SUCCESS) {
             LOG_ERR("%s: failed to fit CLI arguments to free memory, exiting...\n", __func__);
             exit(1);