diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index e80169b34519..5d0dcb9c99cf 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -858,6 +858,13 @@ struct server_context_impl { // measures her decode rate with it (see server-train.h, yield_for_slowdown) std::atomic n_tokens_generated{0}; + // the adapter set the main loop last published (publish_lora_report): GET /lora-adapters reads + // it here, never through the task queue + json lora_report_snapshot() const { + std::lock_guard lock(lora_report_mutex); + return lora_report; + } + mtmd_context * mctx = nullptr; const llama_vocab * vocab = nullptr; @@ -967,6 +974,46 @@ struct server_context_impl { bool sleeping = false; + // Continuum fork: the adapter set as GET /lora-adapters reports it, republished by the main loop + // every time the set changes (load, set, retire), so the route reads it under a lock and never + // waits behind decoding slots (on a loaded CPU engine the queued read took over 60 s, card d765a668). + mutable std::mutex lora_report_mutex; + json lora_report = json::array(); + + // the one place the report is built: every adapter still loaded, by the id requests address it by + void publish_lora_report() { + json report = json::array(); + const auto & loras = params_base.lora_adapters; + for (size_t i = 0; i < loras.size(); ++i) { + const auto & lora = loras[i]; + if (lora.ptr == nullptr) { + continue; // retired + } + json entry = { + {"id", i}, + {"path", lora.path}, + {"scale", lora.scale}, + {"task_name", lora.task_name}, + {"prompt_prefix", lora.prompt_prefix}, + }; + const uint64_t n_alora_tokens = llama_adapter_get_alora_n_invocation_tokens(lora.ptr); + if (n_alora_tokens) { + const llama_token * alora_tokens = llama_adapter_get_alora_invocation_tokens(lora.ptr); + std::string invocation_string; + llama_tokens invocation_tokens; + for (uint64_t j = 0; j < n_alora_tokens; ++j) { + invocation_string += common_token_to_piece(vocab, alora_tokens[j]); + invocation_tokens.push_back(alora_tokens[j]); + } + entry["alora_invocation_string"] = invocation_string; + entry["alora_invocation_tokens"] = invocation_tokens; + } + report.push_back(std::move(entry)); + } + std::lock_guard lock(lora_report_mutex); + lora_report = std::move(report); + } + int64_t t_last_load_progress_ms = 0; void destroy() { @@ -1391,6 +1438,8 @@ struct server_context_impl { // propagate new defaults back to caller params = params_base; + publish_lora_report(); + if (!is_resume) { return init(); } @@ -3135,36 +3184,6 @@ struct server_context_impl { res->n_erased = n_erased; queue_results.send(std::move(res)); } break; - case SERVER_TASK_TYPE_GET_LORA: - { - // TODO @ngxson : make lora_adapters a dedicated member of server_context - auto & loras = params_base.lora_adapters; - auto res = std::make_unique(); - res->id = task.id; - for (size_t i = 0; i < loras.size(); ++i) { - auto & lora = loras[i]; - if (lora.ptr == nullptr) { - continue; // retired - } - std::string alora_invocation_string = ""; - const uint64_t n_alora_tokens = llama_adapter_get_alora_n_invocation_tokens(lora.ptr); - llama_tokens alora_invocation_tokens; - if (n_alora_tokens) { - const llama_token * alora_tokens = llama_adapter_get_alora_invocation_tokens(lora.ptr); - for (uint64_t j = 0; j < n_alora_tokens; ++j) { - alora_invocation_string += common_token_to_piece(vocab, alora_tokens[j]); - alora_invocation_tokens.push_back(alora_tokens[j]); - } - } - res->loras.push_back(server_task_result_get_lora::lora{ - i, - lora, - alora_invocation_string, - alora_invocation_tokens, - }); - } - queue_results.send(std::move(res)); - } break; case SERVER_TASK_TYPE_SET_LORA: { auto new_loras = construct_lora_list(task.set_lora); @@ -3174,6 +3193,7 @@ struct server_context_impl { } // TODO @ngxson : make lora_adapters a dedicated member of server_context params_base.lora_adapters = new_loras; + publish_lora_report(); auto res = std::make_unique(); res->id = task.id; queue_results.send(std::move(res)); @@ -3213,6 +3233,7 @@ struct server_context_impl { llama_adapter_meta_val_str(la.ptr, "adapter.lora.prompt_prefix", buf, sizeof(buf)); la.prompt_prefix = buf; params_base.lora_adapters.push_back(la); + publish_lora_report(); SRV_INF("loaded lora adapter id=%zu path=%s (dormant)\n", params_base.lora_adapters.size() - 1, la.path.c_str()); auto res = std::make_unique(); res->id = task.id; @@ -3308,6 +3329,7 @@ struct server_context_impl { res->loaded = false; loras[id].ptr = nullptr; loras[id].path.clear(); + publish_lora_report(); SRV_INF("retired lora adapter id=%d path=%s\n", id, res->path.c_str()); queue_results.send(std::move(res)); } break; @@ -6040,31 +6062,10 @@ void server_routes::init_routes() { return res; }; - this->get_lora_adapters = [this](const server_http_req & req) { + // the adapter set the main loop last published: answered at once, never queued behind slots + this->get_lora_adapters = [this](const server_http_req &) { auto res = create_response(); - - auto & rd = res->rd; - { - server_task task(SERVER_TASK_TYPE_GET_LORA); - task.id = rd.get_new_id(); - rd.post_task(std::move(task)); - } - - // get the result - auto result = rd.next(req.should_stop); - if (!result) { - // connection was closed - GGML_ASSERT(req.should_stop()); - return res; - } - - if (result->is_error()) { - res->error(result->to_json()); - return res; - } - - GGML_ASSERT(dynamic_cast(result.get()) != nullptr); - res->ok(result->to_json()); + res->ok(ctx_server.lora_report_snapshot()); return res; }; diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 7b4e5692ccce..824a1aade71b 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -1694,30 +1694,6 @@ json server_task_result_slot_erase::to_json() { }; } -// -// server_task_result_get_lora -// - -json server_task_result_get_lora::to_json() { - json result = json::array(); - for (size_t i = 0; i < loras.size(); ++i) { - auto & lora = loras[i]; - json entry = { - {"id", lora.id}, - {"path", lora.info.path}, - {"scale", lora.info.scale}, - {"task_name", lora.info.task_name}, - {"prompt_prefix", lora.info.prompt_prefix}, - }; - if (!lora.alora_invocation_tokens.empty()) { - entry["alora_invocation_string"] = lora.alora_invocation_string; - entry["alora_invocation_tokens"] = lora.alora_invocation_tokens; - } - result.push_back(std::move(entry)); - } - return result; -} - // // server_task_result_apply_lora // diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 2d866581c0cd..0e9e7469e8f4 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -57,7 +57,6 @@ enum server_task_type { SERVER_TASK_TYPE_SLOT_SAVE, SERVER_TASK_TYPE_SLOT_RESTORE, SERVER_TASK_TYPE_SLOT_ERASE, - SERVER_TASK_TYPE_GET_LORA, SERVER_TASK_TYPE_SET_LORA, SERVER_TASK_TYPE_LOAD_LORA, SERVER_TASK_TYPE_UNLOAD_LORA, @@ -658,18 +657,6 @@ struct server_task_result_control : server_task_result { } }; -struct server_task_result_get_lora : server_task_result { - struct lora { - size_t id; // the index requests address it by; retired ids are not listed - common_adapter_lora_info info; - std::string alora_invocation_string; - llama_tokens alora_invocation_tokens; - }; - std::vector loras; - - virtual json to_json() override; -}; - struct server_task_result_apply_lora : server_task_result { virtual json to_json() override; }; diff --git a/tools/server/tests/unit/test_lora.py b/tools/server/tests/unit/test_lora.py index 00b2f245f60f..e84377745269 100644 --- a/tools/server/tests/unit/test_lora.py +++ b/tools/server/tests/unit/test_lora.py @@ -1,3 +1,4 @@ +import shutil import pytest from utils import * @@ -66,6 +67,40 @@ def test_lora_per_request(): assert match_regex(re_test, res.body["content"]) +# what this catches: GET /lora-adapters queued as a task behind decoding slots (over 60 s on a loaded +# CPU engine, card d765a668), so a core adopting a live engine could not learn its adapter set and +# relaunched it cold. The route reads the set the main loop publishes on every change: at launch, +# on a scale change, on a runtime load and on a retire, the report is what the engine holds. +def test_the_adapter_report_follows_every_change_to_the_set(tmp_path): + global server + lora = server.lora_files[0] + shutil.copy(lora, tmp_path / "second.gguf") + server.train_dir = str(tmp_path) + server.start() + + def report(): + res = server.make_request("GET", "/lora-adapters") + assert res.status_code == 200, res.body + return {entry["id"]: entry for entry in res.body} + + launched = report() + assert list(launched) == [0] and launched[0]["path"] == lora + + assert server.make_request("POST", "/lora-adapters", data=[{"id": 0, "scale": 0.25}]).status_code == 200 + assert report()[0]["scale"] == 0.25 + + loaded = server.make_request("POST", "/lora-adapters/load", data={"name": "second.gguf"}) + assert loaded.status_code == 200, loaded.body + assert sorted(report()) == [0, 1] + + # an adapter active by default is retired only once its scale is 0 (the engine refuses otherwise) + assert server.make_request("POST", "/lora-adapters", data=[{"id": 0, "scale": 0.0}, {"id": 1, "scale": 0.0}]).status_code == 200 + assert report()[0]["scale"] == 0.0 + unloaded = server.make_request("POST", "/lora-adapters/unload", data={"id": 0}) + assert unloaded.status_code == 200, unloaded.body + assert list(report()) == [1], "a retired adapter leaves the report; the ids it left stay taken" + + @pytest.mark.skipif(not is_slow_test_allowed(), reason="skipping slow test") def test_with_big_model(): server = ServerProcess() diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py index a0d2dfa3c591..cb66d76874f4 100644 --- a/tools/server/tests/utils.py +++ b/tools/server/tests/utils.py @@ -95,6 +95,7 @@ class ServerProcess: models_preset: str | None = None no_models_autoload: bool | None = None lora_files: List[str] | None = None + train_dir: str | None = None # --train-dir: where POST /lora-adapters/load finds adapters enable_ctx_shift: int | None = False spec_type: str | None = None spec_draft_n_min: int | None = None @@ -235,6 +236,8 @@ def start(self, timeout_seconds: int = DEFAULT_HTTP_TIMEOUT) -> None: if self.lora_files: for lora_file in self.lora_files: server_args.extend(["--lora", lora_file]) + if self.train_dir: + server_args.extend(["--train-dir", self.train_dir]) if self.enable_ctx_shift: server_args.append("--context-shift") if self.spec_type: