diff --git a/API.md b/API.md index 2cc189b..73a5d40 100644 --- a/API.md +++ b/API.md @@ -108,8 +108,8 @@ The following keys are available in context_settings: | Key | Type | Meaning | | ------------------------ | -------- | ---------------------------------------------------------------- | -| `context_size` | `number` | Equivalent to n_ctx = N and n_batch = N. | -| `n_ctx` | `number` | Text context length (tokens). `0` = from model. | +| `context_size` | `number` | Equivalent to `n_ctx = N` and `n_batch = N`. `0` = use the model's training window. | +| `n_ctx` | `number` | Text context length (tokens). `0` = use the model's training window. | | `n_batch` | `number` | **Logical** max batch size submitted to `llama_decode`. | | `n_ubatch` | `number` | **Physical** max micro-batch size. | | `n_seq_max` | `number` | Max concurrent sequences (parallel states for recurrent models). | @@ -199,7 +199,10 @@ SELECT llm_context_create_embedding(); **Description:** Creates a new inference context specifically set for chat conversation. -It is equivalent to `SELECT llm_context_create('context_size=4096');` +It applies no context settings of its own, so anything you do not pass is +llama.cpp's default — in particular the context length defaults to the model's +training window (`llm_model_n_ctx_train()`), not to a fixed size. Pass +`context_size` to bound it, and use `llm_context_size()` to confirm what you got. Context must explicitly created before performing any AI operation! @@ -220,7 +223,10 @@ SELECT llm_context_create_chat(); **Description:** Creates a new inference context specifically set for text generation. -It is equivalent to `SELECT llm_context_create('context_size=4096');` +It applies no context settings of its own, so anything you do not pass is +llama.cpp's default — in particular the context length defaults to the model's +training window (`llm_model_n_ctx_train()`), not to a fixed size. Pass +`context_size` to bound it, and use `llm_context_size()` to confirm what you got. Context must explicitly created before performing any AI operation! diff --git a/src/sqlite-ai.c b/src/sqlite-ai.c index a20e9b8..263f742 100644 --- a/src/sqlite-ai.c +++ b/src/sqlite-ai.c @@ -511,8 +511,8 @@ static bool llm_context_options_callback (void *ctx, void *xdata, const char *ke if (KEY_MATCHES(key, key_len, OPTION_KEY_CONTEXT_SIZE)) { int value = (int)strtol(buffer, NULL, 0); if (value >= 0) { - options->n_ctx = value; - options->n_batch = value; + options->n_ctx = value; // 0 = use the model's training window + if (value > 0) options->n_batch = value; // never drive n_batch to 0 } return true; } @@ -2724,6 +2724,14 @@ static void llm_context_free (sqlite3_context *context, int argc, sqlite3_value static bool llm_context_create_with_options (sqlite3_context *context, ai_context *ai, const char *options1, const char *options2) { struct llama_context_params ctx_params = llama_context_default_params(); + + // n_ctx = 0 tells llama.cpp to use the model's training window. Start from it so + // that "the caller did not ask for a context size" is an explicit sentinel rather + // than something inferred after the fact from the value: llama's own default is + // 512, so an explicit context_size=512 used to be indistinguishable from unset and + // was silently replaced by n_ctx_train. + ctx_params.n_ctx = 0; + if (parse_keyvalue_string(ai, options1, llm_context_options_callback, &ctx_params) == false) { sqlite_context_result_error(context, SQLITE_ERROR, "An error occurred while parsing options (%s)", options1); return false; @@ -2742,13 +2750,6 @@ static bool llm_context_create_with_options (sqlite3_context *context, ai_contex return false; } - // auto-size n_ctx to the model's training window when no explicit context_size was set - // llama_context_default_params() sets n_ctx=512; setting n_ctx=0 tells llama.cpp to use n_ctx_train - struct llama_context_params defaults = llama_context_default_params(); - if (ai->model && ctx_params.n_ctx == defaults.n_ctx) { - ctx_params.n_ctx = 0; - } - // for embedding contexts, clamp n_ctx to n_ctx_train to avoid position overflow if (ctx_params.embeddings && ai->model) { int n_ctx_train = llama_model_n_ctx_train(ai->model); diff --git a/src/sqlite-ai.h b/src/sqlite-ai.h index a710ed5..713d066 100644 --- a/src/sqlite-ai.h +++ b/src/sqlite-ai.h @@ -24,7 +24,7 @@ extern "C" { #endif -#define SQLITE_AI_VERSION "1.0.5" +#define SQLITE_AI_VERSION "1.0.6" SQLITE_AI_API int sqlite3_ai_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); diff --git a/tests/c/unittest.c b/tests/c/unittest.c index 8868a35..963959b 100644 --- a/tests/c/unittest.c +++ b/tests/c/unittest.c @@ -607,6 +607,81 @@ static int test_llm_context_size_errors(const test_env *env) { return 1; } +// Regression: an explicit context_size=512 collided with the value +// llama_context_default_params() uses for n_ctx, so it was read as "the caller did +// not ask" and silently replaced by the model's training window - 32768 for the test +// model, 64x what was requested. +static int test_context_size_is_honoured(const test_env *env) { + sqlite3 *db = NULL; + if (open_db_and_load(env, &db) != SQLITE_OK) return 1; + + const char *model = env->model_path ? env->model_path : DEFAULT_MODEL_PATH; + char sqlbuf[512]; + snprintf(sqlbuf, sizeof(sqlbuf), "SELECT llm_model_load('%s');", model); + if (exec_expect_ok(env, db, sqlbuf) != 0) goto fail; + + int n_ctx_train = 0; + if (select_single_int(env, db, "SELECT llm_model_n_ctx_train();", &n_ctx_train) != 0) goto fail; + if (n_ctx_train <= 1024) { + fprintf(stderr, "[context_size_is_honoured] model trains at %d, too small to tell a " + "honoured context_size from an auto-sized one\n", n_ctx_train); + goto fail; + } + + // llama pads n_ctx up to a multiple of 256, so only exact multiples compare equal. + // 512 is the regression; the others guard against over-correcting. + const int sizes[] = {256, 512, 1024}; + for (size_t i = 0; i < sizeof(sizes) / sizeof(sizes[0]); ++i) { + snprintf(sqlbuf, sizeof(sqlbuf), "SELECT llm_context_create_chat('context_size=%d');", sizes[i]); + if (exec_expect_ok(env, db, sqlbuf) != 0) goto fail; + int n_ctx = 0; + if (select_single_int(env, db, "SELECT llm_context_size();", &n_ctx) != 0) goto fail; + if (env->verbose) printf("[context_size_is_honoured] asked %d, got %d\n", sizes[i], n_ctx); + if (n_ctx != sizes[i]) { + fprintf(stderr, "[context_size_is_honoured] context_size=%d produced n_ctx=%d\n", sizes[i], n_ctx); + goto fail; + } + if (exec_expect_ok(env, db, "SELECT llm_context_free();") != 0) goto fail; + } + + // n_ctx= is the other spelling of the same thing and hit the same collision + if (exec_expect_ok(env, db, "SELECT llm_context_create_chat('n_ctx=512');") != 0) goto fail; + int n_ctx = 0; + if (select_single_int(env, db, "SELECT llm_context_size();", &n_ctx) != 0) goto fail; + if (n_ctx != 512) { + fprintf(stderr, "[context_size_is_honoured] n_ctx=512 produced n_ctx=%d\n", n_ctx); + goto fail; + } + if (exec_expect_ok(env, db, "SELECT llm_context_free();") != 0) goto fail; + + // omitting it still auto-sizes to the model's training window + if (exec_expect_ok(env, db, "SELECT llm_context_create_chat();") != 0) goto fail; + if (select_single_int(env, db, "SELECT llm_context_size();", &n_ctx) != 0) goto fail; + if (n_ctx != n_ctx_train) { + fprintf(stderr, "[context_size_is_honoured] no context_size produced n_ctx=%d, expected %d\n", n_ctx, n_ctx_train); + goto fail; + } + if (exec_expect_ok(env, db, "SELECT llm_context_free();") != 0) goto fail; + + // and context_size=0 asks for that explicitly + if (exec_expect_ok(env, db, "SELECT llm_context_create_chat('context_size=0');") != 0) goto fail; + if (select_single_int(env, db, "SELECT llm_context_size();", &n_ctx) != 0) goto fail; + if (n_ctx != n_ctx_train) { + fprintf(stderr, "[context_size_is_honoured] context_size=0 produced n_ctx=%d, expected %d\n", n_ctx, n_ctx_train); + goto fail; + } + if (exec_expect_ok(env, db, "SELECT llm_context_free();") != 0) goto fail; + + if (exec_expect_ok(env, db, "SELECT llm_model_free();") != 0) goto fail; + + sqlite3_close_v2(db); + return assert_sqlite_memory_clean("context_size_is_honoured", env); + +fail: + if (db) sqlite3_close_v2(db); + return 1; +} + static int test_document_ingestion_flow(const test_env *env) { sqlite3 *db = NULL; if (open_db_and_load(env, &db) != SQLITE_OK) { @@ -2093,6 +2168,7 @@ static const test_case TESTS[] = { {"llm_embed_generate_basic", test_llm_embed_generate_basic}, {"llm_embedding_then_chat", test_llm_embedding_then_chat}, {"llm_context_size_errors", test_llm_context_size_errors}, + {"context_size_is_honoured", test_context_size_is_honoured}, {"document_ingestion_flow", test_document_ingestion_flow}, {"llm_sampler_roundtrip", test_llm_sampler_roundtrip}, {"chat_default_sampler_autocreate", test_chat_default_sampler_autocreate},