From a4901044831da921d37be8fbe7a2b790d5ee35a3 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Mon, 24 Aug 2026 13:02:26 +0200 Subject: [PATCH] fix(context): honour an explicit context_size of 512 llm_context_create*() decided whether the caller had asked for a context size by comparing the parsed value against llama_context_default_params().n_ctx: struct llama_context_params defaults = llama_context_default_params(); if (ai->model && ctx_params.n_ctx == defaults.n_ctx) { ctx_params.n_ctx = 0; // 0 = use the model's training window } That default is 512 (llama-context.cpp:2772), a value a caller can perfectly well pass, so an explicit context_size=512 was indistinguishable from unset and was silently replaced by the model's full training window: context_size=64 -> n_ctx=256 context_size=512 -> n_ctx=32768 <-- asked for 512, got 64x that context_size=513 -> n_ctx=768 n_ctx=512 hit it too, since the check looked at the resolved value rather than at which key was written. The intent was right, only the detection was wrong. llama.cpp already defines n_ctx = 0 as "use the training window", so start from that sentinel instead of inferring it after the fact: the caller's value now always survives, and 0 keeps its documented meaning. Also stops context_size=0 driving n_batch to 0, which llama will not accept. Behaviour change, and a quiet one: anyone passing context_size=512 or n_ctx=512 was getting the model's whole window and now gets 512. Nothing errors - the context simply becomes what was asked for, so a conversation that used to fit may now reach the limit. That is the point: silently ignoring the configuration is the bug. It does make #28 easier to hit, where a full context errors and leaves the chat unusable. API.md said llm_context_create_chat() and llm_context_create_textgen() were equivalent to context_size=4096. Their presets are empty, so both inherit the model's training window; documented as such, along with 0 on context_size/n_ctx. test_context_size_is_honoured covers 256/512/1024 exactly (llama pads n_ctx to a multiple of 256), both spellings, and that omitting the key - or passing 0 - still auto-sizes. Verified to fail against the pre-fix build with "context_size=512 produced n_ctx=32768". Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012KRojTrn4Q4SaZQpqcwpAt --- API.md | 14 ++++++--- src/sqlite-ai.c | 19 ++++++------ src/sqlite-ai.h | 2 +- tests/c/unittest.c | 76 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 14 deletions(-) diff --git a/API.md b/API.md index 2cc189b..73a5d40 100644 --- a/API.md +++ b/API.md @@ -108,8 +108,8 @@ The following keys are available in context_settings: | Key | Type | Meaning | | ------------------------ | -------- | ---------------------------------------------------------------- | -| `context_size` | `number` | Equivalent to n_ctx = N and n_batch = N. | -| `n_ctx` | `number` | Text context length (tokens). `0` = from model. | +| `context_size` | `number` | Equivalent to `n_ctx = N` and `n_batch = N`. `0` = use the model's training window. | +| `n_ctx` | `number` | Text context length (tokens). `0` = use the model's training window. | | `n_batch` | `number` | **Logical** max batch size submitted to `llama_decode`. | | `n_ubatch` | `number` | **Physical** max micro-batch size. | | `n_seq_max` | `number` | Max concurrent sequences (parallel states for recurrent models). | @@ -199,7 +199,10 @@ SELECT llm_context_create_embedding(); **Description:** Creates a new inference context specifically set for chat conversation. -It is equivalent to `SELECT llm_context_create('context_size=4096');` +It applies no context settings of its own, so anything you do not pass is +llama.cpp's default — in particular the context length defaults to the model's +training window (`llm_model_n_ctx_train()`), not to a fixed size. Pass +`context_size` to bound it, and use `llm_context_size()` to confirm what you got. Context must explicitly created before performing any AI operation! @@ -220,7 +223,10 @@ SELECT llm_context_create_chat(); **Description:** Creates a new inference context specifically set for text generation. -It is equivalent to `SELECT llm_context_create('context_size=4096');` +It applies no context settings of its own, so anything you do not pass is +llama.cpp's default — in particular the context length defaults to the model's +training window (`llm_model_n_ctx_train()`), not to a fixed size. Pass +`context_size` to bound it, and use `llm_context_size()` to confirm what you got. Context must explicitly created before performing any AI operation! diff --git a/src/sqlite-ai.c b/src/sqlite-ai.c index a20e9b8..263f742 100644 --- a/src/sqlite-ai.c +++ b/src/sqlite-ai.c @@ -511,8 +511,8 @@ static bool llm_context_options_callback (void *ctx, void *xdata, const char *ke if (KEY_MATCHES(key, key_len, OPTION_KEY_CONTEXT_SIZE)) { int value = (int)strtol(buffer, NULL, 0); if (value >= 0) { - options->n_ctx = value; - options->n_batch = value; + options->n_ctx = value; // 0 = use the model's training window + if (value > 0) options->n_batch = value; // never drive n_batch to 0 } return true; } @@ -2724,6 +2724,14 @@ static void llm_context_free (sqlite3_context *context, int argc, sqlite3_value static bool llm_context_create_with_options (sqlite3_context *context, ai_context *ai, const char *options1, const char *options2) { struct llama_context_params ctx_params = llama_context_default_params(); + + // n_ctx = 0 tells llama.cpp to use the model's training window. Start from it so + // that "the caller did not ask for a context size" is an explicit sentinel rather + // than something inferred after the fact from the value: llama's own default is + // 512, so an explicit context_size=512 used to be indistinguishable from unset and + // was silently replaced by n_ctx_train. + ctx_params.n_ctx = 0; + if (parse_keyvalue_string(ai, options1, llm_context_options_callback, &ctx_params) == false) { sqlite_context_result_error(context, SQLITE_ERROR, "An error occurred while parsing options (%s)", options1); return false; @@ -2742,13 +2750,6 @@ static bool llm_context_create_with_options (sqlite3_context *context, ai_contex return false; } - // auto-size n_ctx to the model's training window when no explicit context_size was set - // llama_context_default_params() sets n_ctx=512; setting n_ctx=0 tells llama.cpp to use n_ctx_train - struct llama_context_params defaults = llama_context_default_params(); - if (ai->model && ctx_params.n_ctx == defaults.n_ctx) { - ctx_params.n_ctx = 0; - } - // for embedding contexts, clamp n_ctx to n_ctx_train to avoid position overflow if (ctx_params.embeddings && ai->model) { int n_ctx_train = llama_model_n_ctx_train(ai->model); diff --git a/src/sqlite-ai.h b/src/sqlite-ai.h index a710ed5..713d066 100644 --- a/src/sqlite-ai.h +++ b/src/sqlite-ai.h @@ -24,7 +24,7 @@ extern "C" { #endif -#define SQLITE_AI_VERSION "1.0.5" +#define SQLITE_AI_VERSION "1.0.6" SQLITE_AI_API int sqlite3_ai_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); diff --git a/tests/c/unittest.c b/tests/c/unittest.c index 8868a35..963959b 100644 --- a/tests/c/unittest.c +++ b/tests/c/unittest.c @@ -607,6 +607,81 @@ static int test_llm_context_size_errors(const test_env *env) { return 1; } +// Regression: an explicit context_size=512 collided with the value +// llama_context_default_params() uses for n_ctx, so it was read as "the caller did +// not ask" and silently replaced by the model's training window - 32768 for the test +// model, 64x what was requested. +static int test_context_size_is_honoured(const test_env *env) { + sqlite3 *db = NULL; + if (open_db_and_load(env, &db) != SQLITE_OK) return 1; + + const char *model = env->model_path ? env->model_path : DEFAULT_MODEL_PATH; + char sqlbuf[512]; + snprintf(sqlbuf, sizeof(sqlbuf), "SELECT llm_model_load('%s');", model); + if (exec_expect_ok(env, db, sqlbuf) != 0) goto fail; + + int n_ctx_train = 0; + if (select_single_int(env, db, "SELECT llm_model_n_ctx_train();", &n_ctx_train) != 0) goto fail; + if (n_ctx_train <= 1024) { + fprintf(stderr, "[context_size_is_honoured] model trains at %d, too small to tell a " + "honoured context_size from an auto-sized one\n", n_ctx_train); + goto fail; + } + + // llama pads n_ctx up to a multiple of 256, so only exact multiples compare equal. + // 512 is the regression; the others guard against over-correcting. + const int sizes[] = {256, 512, 1024}; + for (size_t i = 0; i < sizeof(sizes) / sizeof(sizes[0]); ++i) { + snprintf(sqlbuf, sizeof(sqlbuf), "SELECT llm_context_create_chat('context_size=%d');", sizes[i]); + if (exec_expect_ok(env, db, sqlbuf) != 0) goto fail; + int n_ctx = 0; + if (select_single_int(env, db, "SELECT llm_context_size();", &n_ctx) != 0) goto fail; + if (env->verbose) printf("[context_size_is_honoured] asked %d, got %d\n", sizes[i], n_ctx); + if (n_ctx != sizes[i]) { + fprintf(stderr, "[context_size_is_honoured] context_size=%d produced n_ctx=%d\n", sizes[i], n_ctx); + goto fail; + } + if (exec_expect_ok(env, db, "SELECT llm_context_free();") != 0) goto fail; + } + + // n_ctx= is the other spelling of the same thing and hit the same collision + if (exec_expect_ok(env, db, "SELECT llm_context_create_chat('n_ctx=512');") != 0) goto fail; + int n_ctx = 0; + if (select_single_int(env, db, "SELECT llm_context_size();", &n_ctx) != 0) goto fail; + if (n_ctx != 512) { + fprintf(stderr, "[context_size_is_honoured] n_ctx=512 produced n_ctx=%d\n", n_ctx); + goto fail; + } + if (exec_expect_ok(env, db, "SELECT llm_context_free();") != 0) goto fail; + + // omitting it still auto-sizes to the model's training window + if (exec_expect_ok(env, db, "SELECT llm_context_create_chat();") != 0) goto fail; + if (select_single_int(env, db, "SELECT llm_context_size();", &n_ctx) != 0) goto fail; + if (n_ctx != n_ctx_train) { + fprintf(stderr, "[context_size_is_honoured] no context_size produced n_ctx=%d, expected %d\n", n_ctx, n_ctx_train); + goto fail; + } + if (exec_expect_ok(env, db, "SELECT llm_context_free();") != 0) goto fail; + + // and context_size=0 asks for that explicitly + if (exec_expect_ok(env, db, "SELECT llm_context_create_chat('context_size=0');") != 0) goto fail; + if (select_single_int(env, db, "SELECT llm_context_size();", &n_ctx) != 0) goto fail; + if (n_ctx != n_ctx_train) { + fprintf(stderr, "[context_size_is_honoured] context_size=0 produced n_ctx=%d, expected %d\n", n_ctx, n_ctx_train); + goto fail; + } + if (exec_expect_ok(env, db, "SELECT llm_context_free();") != 0) goto fail; + + if (exec_expect_ok(env, db, "SELECT llm_model_free();") != 0) goto fail; + + sqlite3_close_v2(db); + return assert_sqlite_memory_clean("context_size_is_honoured", env); + +fail: + if (db) sqlite3_close_v2(db); + return 1; +} + static int test_document_ingestion_flow(const test_env *env) { sqlite3 *db = NULL; if (open_db_and_load(env, &db) != SQLITE_OK) { @@ -2093,6 +2168,7 @@ static const test_case TESTS[] = { {"llm_embed_generate_basic", test_llm_embed_generate_basic}, {"llm_embedding_then_chat", test_llm_embedding_then_chat}, {"llm_context_size_errors", test_llm_context_size_errors}, + {"context_size_is_honoured", test_context_size_is_honoured}, {"document_ingestion_flow", test_document_ingestion_flow}, {"llm_sampler_roundtrip", test_llm_sampler_roundtrip}, {"chat_default_sampler_autocreate", test_chat_default_sampler_autocreate},