From cde54f19b3976e66a1958837cc528de3fb2469b0 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Mon, 8 Jun 2026 10:55:37 +0800 Subject: [PATCH 1/2] {"schema":"decodex/commit/1","summary":"add generated OpenAPI and Scalar API docs","authority":"XY-790"} --- Cargo.lock | 39 + Cargo.toml | 2 + Makefile.toml | 4 + apps/elf-api/Cargo.toml | 2 + apps/elf-api/src/routes.rs | 699 +++++++++++++++++- apps/elf-api/tests/http.rs | 114 ++- apps/elf-eval/src/app.rs | 4 +- .../elf-eval/src/bin/trace_regression_gate.rs | 4 +- apps/elf-worker/src/lib.rs | 3 +- docs/guide/getting_started.md | 19 +- .../elf-config/tests/config_validation.rs | 54 +- packages/elf-service/src/add_event.rs | 16 +- packages/elf-service/src/add_note.rs | 13 +- packages/elf-service/src/delete.rs | 4 +- packages/elf-service/src/docs.rs | 17 +- packages/elf-service/src/graph_query.rs | 8 +- packages/elf-service/src/list.rs | 7 +- packages/elf-service/src/notes.rs | 5 +- .../elf-service/src/progressive_search.rs | 4 +- packages/elf-service/src/search.rs | 77 +- packages/elf-service/src/sharing.rs | 23 +- packages/elf-service/src/update.rs | 11 +- .../tests/acceptance/docs_extension_v1.rs | 8 +- .../tests/acceptance/graph_ingestion.rs | 33 +- .../acceptance/outbox_eventual_consistency.rs | 8 +- .../acceptance/structured_field_retrieval.rs | 6 +- 26 files changed, 1022 insertions(+), 162 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 647e5c0f..cbaf3dba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -911,6 +911,8 @@ dependencies = [ "tower 0.5.3", "tracing", "tracing-subscriber", + "utoipa", + "utoipa-scalar", "uuid", "vergen-gitcl", ] @@ -4124,6 +4126,43 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "utoipa" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bde15df68e80b16c7d16b9616e80770ad158988daa56a27dccd1e55558b0160" +dependencies = [ + "indexmap 2.13.0", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba0b99ee52df3028635d93840c797102da61f8a7bb3cf751032455895b52ef8" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "syn", + "uuid", +] + +[[package]] +name = "utoipa-scalar" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59559e1509172f6b26c1cdbc7247c4ddd1ac6560fe94b584f81ee489b141f719" +dependencies = [ + "axum 0.8.8", + "serde", + "serde_json", + "utoipa", +] + [[package]] name = "uuid" version = "1.22.0" diff --git a/Cargo.toml b/Cargo.toml index 26c81861..9a9f815a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,8 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } unicode-normalization = { version = "0.1" } unicode-script = { version = "0.5" } unicode-segmentation = { version = "1.12" } +utoipa = { version = "5.5", features = ["axum_extras", "time", "uuid"] } +utoipa-scalar = { version = "0.3", features = ["axum"] } uuid = { version = "1.22", features = ["serde", "v4", "v5"] } vergen-gitcl = { version = "9.1", features = ["cargo"] } whatlang = { version = "0.18" } diff --git a/Makefile.toml b/Makefile.toml index a1881736..637bf120 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -85,6 +85,8 @@ command = "cargo" args = [ "vstyle", "curate", + "--language", + "rust", "--workspace", "--all-features" ] @@ -95,6 +97,8 @@ command = "cargo" args = [ "vstyle", "tune", + "--language", + "rust", "--workspace", "--all-features", "--strict", diff --git a/apps/elf-api/Cargo.toml b/apps/elf-api/Cargo.toml index c4d02159..1d393479 100644 --- a/apps/elf-api/Cargo.toml +++ b/apps/elf-api/Cargo.toml @@ -14,6 +14,8 @@ time = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } +utoipa = { workspace = true } +utoipa-scalar = { workspace = true } uuid = { workspace = true } elf-cli = { workspace = true } diff --git a/apps/elf-api/src/routes.rs b/apps/elf-api/src/routes.rs index 148f3356..145b4e9f 100644 --- a/apps/elf-api/src/routes.rs +++ b/apps/elf-api/src/routes.rs @@ -7,7 +7,10 @@ use axum::{ DefaultBodyLimit, Extension, Path, Query, State, rejection::{JsonRejection, QueryRejection}, }, - http::{HeaderMap, Request, StatusCode}, + http::{ + HeaderMap, HeaderValue, Request, StatusCode, + header::{CONTENT_LENGTH, CONTENT_TYPE}, + }, middleware::{self, Next}, response::{IntoResponse, Response}, routing, @@ -15,6 +18,8 @@ use axum::{ use serde::{Deserialize, Serialize}; use serde_json::Value; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; +use utoipa::{OpenApi, ToSchema}; +use utoipa_scalar::{Scalar, Servable}; use uuid::Uuid; use crate::state::AppState; @@ -46,6 +51,11 @@ use elf_service::{ UpdateResponse, search::TraceBundleMode, }; +/// JSON OpenAPI contract route. +pub const OPENAPI_JSON_PATH: &str = "/openapi.json"; +/// Scalar API reference route. +pub const SCALAR_DOCS_PATH: &str = "/docs"; + const HEADER_TENANT_ID: &str = "X-ELF-Tenant-Id"; const HEADER_PROJECT_ID: &str = "X-ELF-Project-Id"; const HEADER_AGENT_ID: &str = "X-ELF-Agent-Id"; @@ -66,6 +76,72 @@ const MAX_TOP_K: u32 = 100; const MAX_CANDIDATE_K: u32 = 1_000; const MAX_ERROR_LOG_CHARS: usize = 1_024; +/// Generated OpenAPI document for the ELF HTTP API. +#[derive(OpenApi)] +#[openapi( + info( + title = "ELF API", + version = env!("CARGO_PKG_VERSION"), + description = "Evidence-linked fact memory HTTP and admin API." + ), + paths( + health, + notes_ingest, + events_ingest, + docs_put, + docs_get, + docs_search_l0, + docs_excerpts_get, + graph_query, + searches_create, + searches_get, + searches_timeline, + searches_notes, + notes_list, + notes_get, + notes_patch, + notes_delete, + notes_publish, + notes_unpublish, + space_grants_list, + space_grant_upsert, + space_grant_revoke, + admin_ingestion_profiles_list, + admin_ingestion_profile_create, + admin_ingestion_profile_get, + admin_ingestion_profile_versions_list, + admin_ingestion_profile_default_get, + admin_ingestion_profile_default_set, + rebuild_qdrant, + searches_raw, + trace_recent_list, + trace_get, + trace_bundle_get, + trace_trajectory_get, + trace_item_get, + admin_graph_predicates_list, + admin_graph_predicate_patch, + admin_graph_predicate_alias_add, + admin_graph_predicate_aliases_list, + admin_note_provenance_get, + ), + components(schemas( + AdminIngestionProfileDefaultResponseV2, + AdminIngestionProfileDefaultSetBody, + ErrorBody, + )), + tags( + (name = "health", description = "Health and process liveness."), + (name = "notes", description = "Memory note ingestion, listing, mutation, and sharing."), + (name = "events", description = "Event ingestion through the extractor pipeline."), + (name = "docs", description = "Document extension ingestion, search, and excerpt retrieval."), + (name = "search", description = "Progressive search sessions and raw search diagnostics."), + (name = "graph", description = "Graph query and predicate administration."), + (name = "admin", description = "Local admin and operator inspection routes."), + ) +)] +pub struct ApiDoc; + #[derive(Clone, Debug)] struct RequestContext { tenant_id: String, @@ -160,13 +236,6 @@ struct SearchCreateRequest { ranking: Option, } -#[derive(Clone, Copy, Debug, Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] -enum SearchMode { - QuickFind, - PlannedSearch, -} - #[derive(Clone, Debug, Serialize)] struct SearchIndexResponseV2 { mode: SearchMode, @@ -236,12 +305,19 @@ struct AdminIngestionProfileGetQuery { version: Option, } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, ToSchema)] struct AdminIngestionProfileDefaultSetBody { profile_id: String, version: Option, } +#[derive(Clone, Debug, Serialize, ToSchema)] +struct AdminIngestionProfileDefaultResponseV2 { + profile_id: String, + version: Option, + updated_at: String, +} + #[derive(Clone, Debug, Serialize)] struct SearchDetailsResponseV2 { search_id: Uuid, @@ -338,7 +414,7 @@ struct SpaceGrantsListResponseV2 { grants: Vec, } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, ToSchema)] struct ErrorBody { error_code: String, message: String, @@ -429,6 +505,13 @@ impl IntoResponse for ApiError { } } +#[derive(Clone, Copy, Debug, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +enum SearchMode { + QuickFind, + PlannedSearch, +} + /// Builds the authenticated public API router. pub fn router(state: AppState) -> Router { let auth_state = state.clone(); @@ -461,6 +544,7 @@ pub fn router(state: AppState) -> Router { .layer(DefaultBodyLimit::max(MAX_DOC_REQUEST_BYTES)); Router::new() + .merge(contract_router()) .merge(api_router) .merge(docs_router) .layer(middleware::from_fn_with_state(auth_state, api_auth_middleware)) @@ -510,6 +594,16 @@ pub fn admin_router(state: AppState) -> Router { .layer(middleware::from_fn_with_state(auth_state, admin_auth_middleware)) } +/// Builds the API contract router. +pub fn contract_router() -> Router +where + S: Clone + Send + Sync + 'static, +{ + Router::new() + .route(OPENAPI_JSON_PATH, routing::get(openapi_json)) + .merge(Scalar::with_url(SCALAR_DOCS_PATH, ::openapi())) +} + fn json_error( status: StatusCode, code: &str, @@ -808,6 +902,16 @@ fn parse_optional_rfc3339( }) } +async fn openapi_json() -> Response { + let mut response = Json(::openapi()).into_response(); + + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("application/vnd.oai.openapi+json")); + + response +} + async fn with_request_id(response: Response, request_id: Uuid) -> Response { let (mut parts, body) = response.into_parts(); @@ -818,7 +922,7 @@ async fn with_request_id(response: Response, request_id: Uuid) -> Response { let is_json_response = parts .headers - .get(axum::http::header::CONTENT_TYPE) + .get(CONTENT_TYPE) .and_then(|value| value.to_str().ok()) .map(|content_type| content_type.starts_with("application/json")) .unwrap_or(false); @@ -833,7 +937,7 @@ async fn with_request_id(response: Response, request_id: Uuid) -> Response { }; if let Some(response_body) = inject_request_id_into_json_body(&body_bytes, &request_id) { - parts.headers.remove(axum::http::header::CONTENT_LENGTH); + parts.headers.remove(CONTENT_LENGTH); Response::from_parts(parts, Body::from(response_body)) } else { @@ -934,10 +1038,30 @@ async fn admin_auth_middleware( with_request_id(response, request_id).await } +#[utoipa::path( + get, + path = "/health", + tag = "health", + responses((status = 200, description = "API process is healthy.")) +)] async fn health() -> StatusCode { StatusCode::OK } +#[utoipa::path( + post, + path = "/v2/notes/ingest", + tag = "notes", + request_body = Value, + responses( + (status = 200, description = "Notes were processed.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 422, description = "Non-English input rejected.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn notes_ingest( State(state): State, headers: HeaderMap, @@ -978,6 +1102,20 @@ async fn notes_ingest( Ok(Json(response)) } +#[utoipa::path( + post, + path = "/v2/events/ingest", + tag = "events", + request_body = Value, + responses( + (status = 200, description = "Event messages were processed.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 422, description = "Non-English input rejected.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn events_ingest( State(state): State, headers: HeaderMap, @@ -1031,6 +1169,20 @@ async fn events_ingest( Ok(Json(response)) } +#[utoipa::path( + post, + path = "/v2/docs", + tag = "docs", + request_body = Value, + responses( + (status = 200, description = "Document was stored.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 422, description = "Non-English input rejected.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn docs_put( State(state): State, headers: HeaderMap, @@ -1067,6 +1219,20 @@ async fn docs_put( Ok(Json(response)) } +#[utoipa::path( + get, + path = "/v2/docs/{doc_id}", + tag = "docs", + params(("doc_id" = Uuid, Path, description = "Document ID.")), + responses( + (status = 200, description = "Document was fetched.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 404, description = "Document was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn docs_get( State(state): State, headers: HeaderMap, @@ -1088,6 +1254,20 @@ async fn docs_get( Ok(Json(response)) } +#[utoipa::path( + post, + path = "/v2/docs/search/l0", + tag = "docs", + request_body = Value, + responses( + (status = 200, description = "L0 document search results.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 422, description = "Non-English input rejected.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn docs_search_l0( State(state): State, headers: HeaderMap, @@ -1182,6 +1362,20 @@ async fn docs_search_l0( Ok(Json(response)) } +#[utoipa::path( + post, + path = "/v2/docs/excerpts", + tag = "docs", + request_body = Value, + responses( + (status = 200, description = "Document excerpt result.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 404, description = "Document or excerpt was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn docs_excerpts_get( State(state): State, headers: HeaderMap, @@ -1213,6 +1407,20 @@ async fn docs_excerpts_get( Ok(Json(response)) } +#[utoipa::path( + post, + path = "/v2/graph/query", + tag = "graph", + request_body = Value, + responses( + (status = 200, description = "Graph facts matching the query.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 422, description = "Non-English input rejected.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn graph_query( State(state): State, headers: HeaderMap, @@ -1245,6 +1453,20 @@ async fn graph_query( Ok(Json(response)) } +#[utoipa::path( + post, + path = "/v2/searches", + tag = "search", + request_body = Value, + responses( + (status = 200, description = "Search session was created.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 422, description = "Non-English input rejected.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn searches_create( State(state): State, headers: HeaderMap, @@ -1339,6 +1561,25 @@ async fn searches_create( Ok(Json(response)) } +#[utoipa::path( + get, + path = "/v2/searches/{search_id}", + tag = "search", + params( + ("search_id" = Uuid, Path, description = "Search session ID."), + ("payload_level" = Option, Query, description = "Optional payload level."), + ("top_k" = Option, Query, description = "Optional result limit override."), + ("touch" = Option, Query, description = "Whether to extend the session TTL."), + ), + responses( + (status = 200, description = "Search session index view.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 404, description = "Search session was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn searches_get( State(state): State, headers: HeaderMap, @@ -1385,6 +1626,24 @@ async fn searches_get( })) } +#[utoipa::path( + get, + path = "/v2/searches/{search_id}/timeline", + tag = "search", + params( + ("search_id" = Uuid, Path, description = "Search session ID."), + ("payload_level" = Option, Query, description = "Optional payload level."), + ("group_by" = Option, Query, description = "Timeline grouping mode."), + ), + responses( + (status = 200, description = "Search session timeline.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 404, description = "Search session was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn searches_timeline( State(state): State, headers: HeaderMap, @@ -1421,6 +1680,21 @@ async fn searches_timeline( })) } +#[utoipa::path( + post, + path = "/v2/searches/{search_id}/notes", + tag = "search", + params(("search_id" = Uuid, Path, description = "Search session ID.")), + request_body = Value, + responses( + (status = 200, description = "Hydrated search note details.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 404, description = "Search session was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn searches_notes( State(state): State, headers: HeaderMap, @@ -1463,6 +1737,23 @@ async fn searches_notes( })) } +#[utoipa::path( + get, + path = "/v2/notes", + tag = "notes", + params( + ("scope" = Option, Query, description = "Optional note scope filter."), + ("status" = Option, Query, description = "Optional note status filter."), + ("type" = Option, Query, description = "Optional note type filter."), + ), + responses( + (status = 200, description = "Notes visible to the caller.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn notes_list( State(state): State, headers: HeaderMap, @@ -1494,6 +1785,20 @@ async fn notes_list( Ok(Json(response)) } +#[utoipa::path( + get, + path = "/v2/notes/{note_id}", + tag = "notes", + params(("note_id" = Uuid, Path, description = "Note ID.")), + responses( + (status = 200, description = "Note details.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 404, description = "Note was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn notes_get( State(state): State, headers: HeaderMap, @@ -1513,6 +1818,22 @@ async fn notes_get( Ok(Json(response)) } +#[utoipa::path( + patch, + path = "/v2/notes/{note_id}", + tag = "notes", + params(("note_id" = Uuid, Path, description = "Note ID.")), + request_body = Value, + responses( + (status = 200, description = "Note was updated.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 404, description = "Note was not found.", body = ErrorBody), + (status = 422, description = "Non-English input rejected.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn notes_patch( State(state): State, headers: HeaderMap, @@ -1542,6 +1863,20 @@ async fn notes_patch( Ok(Json(response)) } +#[utoipa::path( + delete, + path = "/v2/notes/{note_id}", + tag = "notes", + params(("note_id" = Uuid, Path, description = "Note ID.")), + responses( + (status = 200, description = "Note was deleted.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 404, description = "Note was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn notes_delete( State(state): State, headers: HeaderMap, @@ -1561,6 +1896,21 @@ async fn notes_delete( Ok(Json(response)) } +#[utoipa::path( + post, + path = "/v2/notes/{note_id}/publish", + tag = "notes", + params(("note_id" = Uuid, Path, description = "Note ID.")), + request_body = Value, + responses( + (status = 200, description = "Note was published to a shared space.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 404, description = "Note was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn notes_publish( State(state): State, headers: HeaderMap, @@ -1598,6 +1948,21 @@ async fn notes_publish( })) } +#[utoipa::path( + post, + path = "/v2/notes/{note_id}/unpublish", + tag = "notes", + params(("note_id" = Uuid, Path, description = "Note ID.")), + request_body = Value, + responses( + (status = 200, description = "Note was returned to private scope.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 404, description = "Note was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn notes_unpublish( State(state): State, headers: HeaderMap, @@ -1634,6 +1999,19 @@ async fn notes_unpublish( })) } +#[utoipa::path( + get, + path = "/v2/spaces/{space}/grants", + tag = "notes", + params(("space" = String, Path, description = "Shared space name.")), + responses( + (status = 200, description = "Space grants.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn space_grants_list( State(state): State, headers: HeaderMap, @@ -1666,6 +2044,20 @@ async fn space_grants_list( })) } +#[utoipa::path( + post, + path = "/v2/spaces/{space}/grants", + tag = "notes", + params(("space" = String, Path, description = "Shared space name.")), + request_body = Value, + responses( + (status = 200, description = "Space grant was upserted.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn space_grant_upsert( State(state): State, headers: HeaderMap, @@ -1706,6 +2098,20 @@ async fn space_grant_upsert( })) } +#[utoipa::path( + post, + path = "/v2/spaces/{space}/grants/revoke", + tag = "notes", + params(("space" = String, Path, description = "Shared space name.")), + request_body = Value, + responses( + (status = 200, description = "Space grant was revoked.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Scope denied.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn space_grant_revoke( State(state): State, headers: HeaderMap, @@ -1741,6 +2147,19 @@ async fn space_grant_revoke( Ok(Json(response)) } +#[utoipa::path( + get, + path = "/v2/admin/graph/predicates", + tag = "graph", + params(("scope" = Option, Query, description = "Predicate scope filter.")), + responses( + (status = 200, description = "Graph predicates.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn admin_graph_predicates_list( State(state): State, headers: HeaderMap, @@ -1770,6 +2189,22 @@ async fn admin_graph_predicates_list( Ok(Json(response)) } +#[utoipa::path( + patch, + path = "/v2/admin/graph/predicates/{predicate_id}", + tag = "graph", + params(("predicate_id" = Uuid, Path, description = "Predicate ID.")), + request_body = Value, + responses( + (status = 200, description = "Graph predicate was updated.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 404, description = "Predicate was not found.", body = ErrorBody), + (status = 409, description = "Predicate update conflicted.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn admin_graph_predicate_patch( State(state): State, headers: HeaderMap, @@ -1799,6 +2234,22 @@ async fn admin_graph_predicate_patch( Ok(Json(response)) } +#[utoipa::path( + post, + path = "/v2/admin/graph/predicates/{predicate_id}/aliases", + tag = "graph", + params(("predicate_id" = Uuid, Path, description = "Predicate ID.")), + request_body = Value, + responses( + (status = 200, description = "Graph predicate alias was added.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 404, description = "Predicate was not found.", body = ErrorBody), + (status = 409, description = "Predicate update conflicted.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn admin_graph_predicate_alias_add( State(state): State, headers: HeaderMap, @@ -1827,6 +2278,20 @@ async fn admin_graph_predicate_alias_add( Ok(Json(response)) } +#[utoipa::path( + get, + path = "/v2/admin/graph/predicates/{predicate_id}/aliases", + tag = "graph", + params(("predicate_id" = Uuid, Path, description = "Predicate ID.")), + responses( + (status = 200, description = "Graph predicate aliases.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 404, description = "Predicate was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn admin_graph_predicate_aliases_list( State(state): State, headers: HeaderMap, @@ -1846,6 +2311,20 @@ async fn admin_graph_predicate_aliases_list( Ok(Json(response)) } +#[utoipa::path( + get, + path = "/v2/admin/notes/{note_id}/provenance", + tag = "admin", + params(("note_id" = Uuid, Path, description = "Note ID.")), + responses( + (status = 200, description = "Note provenance bundle.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 404, description = "Note was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn admin_note_provenance_get( State(state): State, headers: HeaderMap, @@ -1864,6 +2343,18 @@ async fn admin_note_provenance_get( Ok(Json(response)) } +#[utoipa::path( + get, + path = "/v2/admin/events/ingestion-profiles", + tag = "admin", + responses( + (status = 200, description = "Ingestion profile versions.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn admin_ingestion_profiles_list( State(state): State, headers: HeaderMap, @@ -1880,6 +2371,19 @@ async fn admin_ingestion_profiles_list( Ok(Json(response)) } +#[utoipa::path( + post, + path = "/v2/admin/events/ingestion-profiles", + tag = "admin", + request_body = Value, + responses( + (status = 200, description = "Ingestion profile version was created.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn admin_ingestion_profile_create( State(state): State, headers: HeaderMap, @@ -1906,6 +2410,23 @@ async fn admin_ingestion_profile_create( Ok(Json(response)) } +#[utoipa::path( + get, + path = "/v2/admin/events/ingestion-profiles/{profile_id}", + tag = "admin", + params( + ("profile_id" = String, Path, description = "Ingestion profile ID."), + ("version" = Option, Query, description = "Optional profile version."), + ), + responses( + (status = 200, description = "Ingestion profile version.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 404, description = "Profile was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn admin_ingestion_profile_get( State(state): State, headers: HeaderMap, @@ -1936,6 +2457,19 @@ async fn admin_ingestion_profile_get( Ok(Json(response)) } +#[utoipa::path( + get, + path = "/v2/admin/events/ingestion-profiles/{profile_id}/versions", + tag = "admin", + params(("profile_id" = String, Path, description = "Ingestion profile ID.")), + responses( + (status = 200, description = "Versions for one ingestion profile.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn admin_ingestion_profile_versions_list( State(state): State, headers: HeaderMap, @@ -1954,6 +2488,22 @@ async fn admin_ingestion_profile_versions_list( Ok(Json(response)) } +#[utoipa::path( + get, + path = "/v2/admin/events/ingestion-profiles/default", + tag = "admin", + responses( + ( + status = 200, + description = "Default add_event ingestion profile pointer.", + body = AdminIngestionProfileDefaultResponseV2, + ), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn admin_ingestion_profile_default_get( State(state): State, headers: HeaderMap, @@ -1970,6 +2520,24 @@ async fn admin_ingestion_profile_default_get( Ok(Json(response)) } +#[utoipa::path( + put, + path = "/v2/admin/events/ingestion-profiles/default", + tag = "admin", + request_body = AdminIngestionProfileDefaultSetBody, + responses( + ( + status = 200, + description = "Default add_event ingestion profile pointer was updated.", + body = AdminIngestionProfileDefaultResponseV2, + ), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 404, description = "Profile was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn admin_ingestion_profile_default_set( State(state): State, headers: HeaderMap, @@ -1994,12 +2562,37 @@ async fn admin_ingestion_profile_default_set( Ok(Json(response)) } +#[utoipa::path( + post, + path = "/v2/admin/qdrant/rebuild", + tag = "admin", + responses( + (status = 200, description = "Qdrant rebuild report.", body = Value), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn rebuild_qdrant(State(state): State) -> Result, ApiError> { let response = state.service.rebuild_qdrant().await?; Ok(Json(response)) } +#[utoipa::path( + post, + path = "/v2/admin/searches/raw", + tag = "search", + request_body = Value, + responses( + (status = 200, description = "Raw admin search response.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 422, description = "Non-English input rejected.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn searches_raw( State(state): State, headers: HeaderMap, @@ -2068,6 +2661,20 @@ async fn searches_raw( Ok(Json(response)) } +#[utoipa::path( + get, + path = "/v2/admin/traces/{trace_id}", + tag = "admin", + params(("trace_id" = Uuid, Path, description = "Search trace ID.")), + responses( + (status = 200, description = "Search trace bundle without full stage internals.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 404, description = "Trace was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn trace_get( State(state): State, headers: HeaderMap, @@ -2087,6 +2694,27 @@ async fn trace_get( Ok(Json(response)) } +#[utoipa::path( + get, + path = "/v2/admin/traces/recent", + tag = "admin", + params( + ("limit" = Option, Query, description = "Page size."), + ("cursor_created_at" = Option, Query, description = "Created-at page cursor."), + ("cursor_trace_id" = Option, Query, description = "Trace ID page cursor."), + ("agent_id" = Option, Query, description = "Optional trace creator filter."), + ("read_profile" = Option, Query, description = "Optional read profile filter."), + ("created_after" = Option, Query, description = "Strict lower created_at bound."), + ("created_before" = Option, Query, description = "Strict upper created_at bound."), + ), + responses( + (status = 200, description = "Recent search traces.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn trace_recent_list( State(state): State, headers: HeaderMap, @@ -2137,6 +2765,20 @@ async fn trace_recent_list( Ok(Json(response)) } +#[utoipa::path( + get, + path = "/v2/admin/trajectories/{trace_id}", + tag = "admin", + params(("trace_id" = Uuid, Path, description = "Search trace ID.")), + responses( + (status = 200, description = "Search trace retrieval trajectory.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 404, description = "Trace was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn trace_trajectory_get( State(state): State, headers: HeaderMap, @@ -2156,6 +2798,20 @@ async fn trace_trajectory_get( Ok(Json(response)) } +#[utoipa::path( + get, + path = "/v2/admin/trace-items/{item_id}", + tag = "admin", + params(("item_id" = Uuid, Path, description = "Trace item/result handle ID.")), + responses( + (status = 200, description = "Search trace item explain payload.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 404, description = "Trace item was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn trace_item_get( State(state): State, headers: HeaderMap, @@ -2175,6 +2831,25 @@ async fn trace_item_get( Ok(Json(response)) } +#[utoipa::path( + get, + path = "/v2/admin/traces/{trace_id}/bundle", + tag = "admin", + params( + ("trace_id" = Uuid, Path, description = "Search trace ID."), + ("mode" = Option, Query, description = "bounded or full."), + ("stage_items_limit" = Option, Query, description = "Maximum stage items."), + ("candidates_limit" = Option, Query, description = "Maximum candidate snapshot items."), + ), + responses( + (status = 200, description = "Search trace bundle.", body = Value), + (status = 400, description = "Invalid request.", body = ErrorBody), + (status = 401, description = "Authentication required.", body = ErrorBody), + (status = 403, description = "Admin access required.", body = ErrorBody), + (status = 404, description = "Trace was not found.", body = ErrorBody), + (status = 500, description = "Internal error.", body = ErrorBody), + ) +)] async fn trace_bundle_get( State(state): State, headers: HeaderMap, diff --git a/apps/elf-api/tests/http.rs b/apps/elf-api/tests/http.rs index 498f8b6c..cab5ff1a 100644 --- a/apps/elf-api/tests/http.rs +++ b/apps/elf-api/tests/http.rs @@ -13,7 +13,10 @@ use serde_json::Map; use tower::util::ServiceExt as _; use uuid::Uuid; -use elf_api::{routes, state::AppState}; +use elf_api::{ + routes::{self, OPENAPI_JSON_PATH, SCALAR_DOCS_PATH}, + state::AppState, +}; use elf_config::{ Chunking, Config, EmbeddingProviderConfig, Lifecycle, LlmProviderConfig, Memory, Postgres, ProviderConfig, Providers, Qdrant, Ranking, RankingBlend, RankingBlendSegment, @@ -228,6 +231,15 @@ fn dummy_llm_provider() -> LlmProviderConfig { } } +fn assert_openapi_method(spec: &serde_json::Value, path: &str, method: &str) { + let operation = spec + .get("paths") + .and_then(|paths| paths.get(path)) + .and_then(|path_item| path_item.get(method)); + + assert!(operation.is_some(), "Missing OpenAPI operation {method} {path}"); +} + async fn test_env() -> Option<(TestDatabase, String, String)> { let base_dsn = match elf_testkit::env_dsn() { Some(value) => value, @@ -676,6 +688,106 @@ async fn fetch_admin_search_raw_source_ref( item["source_ref"].clone() } +async fn contract_json() -> serde_json::Value { + let app = routes::contract_router::<()>(); + let response = app + .oneshot( + Request::builder() + .uri(OPENAPI_JSON_PATH) + .body(Body::empty()) + .expect("Failed to build OpenAPI request."), + ) + .await + .expect("Failed to call OpenAPI route."); + + assert_eq!(response.status(), StatusCode::OK); + + let body = body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("Failed to read OpenAPI response body."); + + serde_json::from_slice(&body).expect("Failed to parse OpenAPI response.") +} + +#[tokio::test] +async fn openapi_json_route_serves_generated_contract() { + let spec = contract_json().await; + + assert_eq!(spec["info"]["title"], "ELF API"); + assert!(spec.get("request_id").is_none()); + + assert_openapi_method(&spec, "/health", "get"); + assert_openapi_method(&spec, "/v2/notes/ingest", "post"); + assert_openapi_method(&spec, "/v2/events/ingest", "post"); + assert_openapi_method(&spec, "/v2/docs/search/l0", "post"); + assert_openapi_method(&spec, "/v2/searches/{search_id}/notes", "post"); + assert_openapi_method(&spec, "/v2/admin/searches/raw", "post"); + assert_openapi_method(&spec, "/v2/admin/events/ingestion-profiles/default", "get"); + assert_openapi_method(&spec, "/v2/admin/events/ingestion-profiles/default", "put"); +} + +#[tokio::test] +async fn scalar_docs_route_serves_api_reference_html() { + let app = routes::contract_router::<()>(); + let response = app + .oneshot( + Request::builder() + .uri(SCALAR_DOCS_PATH) + .body(Body::empty()) + .expect("Failed to build Scalar docs request."), + ) + .await + .expect("Failed to call Scalar docs route."); + + assert_eq!(response.status(), StatusCode::OK); + + let body = body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("Failed to read Scalar docs response body."); + let html = String::from_utf8(body.to_vec()).expect("Scalar docs response was not UTF-8."); + + assert!(html.contains("@scalar/api-reference")); + assert!(html.contains("/v2/admin/events/ingestion-profiles/default")); +} + +#[tokio::test] +async fn openapi_includes_default_ingestion_profile_get_put_contract() { + let spec = contract_json().await; + let default_path = &spec["paths"]["/v2/admin/events/ingestion-profiles/default"]; + let get_schema_ref = + default_path["get"]["responses"]["200"]["content"]["application/json"]["schema"]["$ref"] + .as_str() + .expect("Missing default profile GET response schema ref."); + let put_request_schema_ref = default_path["put"]["requestBody"]["content"]["application/json"] + ["schema"]["$ref"] + .as_str() + .expect("Missing default profile PUT request schema ref."); + let put_response_schema_ref = + default_path["put"]["responses"]["200"]["content"]["application/json"]["schema"]["$ref"] + .as_str() + .expect("Missing default profile PUT response schema ref."); + + assert!(get_schema_ref.ends_with("/AdminIngestionProfileDefaultResponseV2")); + assert!(put_request_schema_ref.ends_with("/AdminIngestionProfileDefaultSetBody")); + assert!(put_response_schema_ref.ends_with("/AdminIngestionProfileDefaultResponseV2")); + + let schemas = &spec["components"]["schemas"]; + let request_schema = &schemas["AdminIngestionProfileDefaultSetBody"]; + let response_schema = &schemas["AdminIngestionProfileDefaultResponseV2"]; + + assert!(request_schema["properties"].get("profile_id").is_some()); + assert!(request_schema["properties"].get("version").is_some()); + assert!( + request_schema["required"] + .as_array() + .expect("Missing request required fields") + .contains(&serde_json::json!("profile_id")) + ); + assert!(response_schema["properties"].get("profile_id").is_some()); + assert!(response_schema["properties"].get("version").is_some()); + assert!(response_schema["properties"].get("updated_at").is_some()); +} + #[tokio::test] #[ignore = "Requires external Postgres and Qdrant. Set ELF_PG_DSN and ELF_QDRANT_GRPC_URL (or ELF_QDRANT_URL) to run."] async fn sharing_visibility_requires_explicit_project_grant() { diff --git a/apps/elf-eval/src/app.rs b/apps/elf-eval/src/app.rs index 4ce754b6..94bd819d 100644 --- a/apps/elf-eval/src/app.rs +++ b/apps/elf-eval/src/app.rs @@ -18,7 +18,7 @@ use uuid::Uuid; use elf_config::Config; use elf_service::{ ElfService, RankingRequestOverride, SearchIndexItem, SearchIndexResponse, SearchRequest, - search::{self, TraceReplayItem}, + search::{self, TraceReplayContext, TraceReplayItem}, }; use elf_storage::{db::Db, qdrant::QdrantStore}; @@ -1096,7 +1096,7 @@ async fn compare_trace_id( let trace_row = fetch_trace_compare_trace_row(db, trace_id).await?; let candidate_rows = fetch_trace_compare_candidate_rows(db, trace_id).await?; let stage_rows = fetch_trace_compare_stage_rows(db, trace_id).await?; - let context = elf_service::search::TraceReplayContext { + let context = TraceReplayContext { trace_id: trace_row.trace_id, query: trace_row.query.clone(), candidate_count: u32::try_from(trace_row.candidate_count).unwrap_or(0), diff --git a/apps/elf-eval/src/bin/trace_regression_gate.rs b/apps/elf-eval/src/bin/trace_regression_gate.rs index f8599180..44dd93e4 100644 --- a/apps/elf-eval/src/bin/trace_regression_gate.rs +++ b/apps/elf-eval/src/bin/trace_regression_gate.rs @@ -14,7 +14,7 @@ use tracing_subscriber::EnvFilter; use uuid::Uuid; use elf_config::Config; -use elf_service::search; +use elf_service::search::{self, TraceReplayContext}; use elf_storage::db::Db; #[derive(Debug, Parser)] @@ -346,7 +346,7 @@ async fn eval_trace( .created_at .format(&Rfc3339) .map_err(|err| eyre::eyre!("Failed to format created_at: {err}"))?; - let context = elf_service::search::TraceReplayContext { + let context = TraceReplayContext { trace_id: trace_row.trace_id, query: trace_row.query.clone(), candidate_count: u32::try_from(trace_row.candidate_count).unwrap_or(0), diff --git a/apps/elf-worker/src/lib.rs b/apps/elf-worker/src/lib.rs index 6335886f..d8b4bbf3 100644 --- a/apps/elf-worker/src/lib.rs +++ b/apps/elf-worker/src/lib.rs @@ -18,6 +18,7 @@ use elf_storage::{ db::Db, qdrant::{DOCS_SEARCH_FILTER_INDEXES, QdrantStore}, }; +use worker::WorkerState; /// CLI arguments for the worker binary. #[derive(Debug, Parser)] @@ -61,7 +62,7 @@ pub async fn run(args: Args) -> Result<()> { max_tokens: config.chunking.max_tokens, overlap_tokens: config.chunking.overlap_tokens, }; - let state = worker::WorkerState { + let state = WorkerState { db, qdrant, docs_qdrant, diff --git a/docs/guide/getting_started.md b/docs/guide/getting_started.md index b633bd1a..56abb7ea 100644 --- a/docs/guide/getting_started.md +++ b/docs/guide/getting_started.md @@ -53,7 +53,22 @@ cargo run -p elf-api -- -c elf.toml cargo run -p elf-mcp -- -c elf.toml ``` -## 4. Run retrieval evaluation +## 4. Inspect API contract + +After `elf-api` starts, the API process serves: + +- `GET /openapi.json` for the generated OpenAPI contract. +- `GET /docs` for the Scalar API reference UI. + +Use the host and port from `service.http_bind` in `elf.toml`. +For example: + +```sh +curl -fsS http://127.0.0.1:51892/openapi.json +open http://127.0.0.1:51892/docs +``` + +## 5. Run retrieval evaluation Use `elf-eval` with your dataset. @@ -63,7 +78,7 @@ cargo run -p elf-eval -- -c elf.toml -i path/to/eval.json For dataset format and metric details, see `docs/guide/evaluation.md`. -## 5. Development workflow +## 6. Development workflow Use `cargo make` tasks from repository root. diff --git a/packages/elf-config/tests/config_validation.rs b/packages/elf-config/tests/config_validation.rs index ae6ec892..c2b92c42 100644 --- a/packages/elf-config/tests/config_validation.rs +++ b/packages/elf-config/tests/config_validation.rs @@ -13,7 +13,7 @@ use std::{ use toml::Value; -use elf_config::{self, Config, Context, Error}; +use elf_config::{self, Config, Context, Error, MemoryPolicyRule}; const SAMPLE_CONFIG_TEMPLATE_TOML: &str = include_str!("fixtures/sample_config.template.toml"); @@ -515,10 +515,10 @@ fn security_auth_keys_require_known_read_profile() { fn memory_policy_min_confidence_must_be_finite() { let mut cfg = base_config(); - cfg.memory.policy.rules.push(elf_config::MemoryPolicyRule { - min_confidence: Some(f32::NAN), - ..Default::default() - }); + cfg.memory + .policy + .rules + .push(MemoryPolicyRule { min_confidence: Some(f32::NAN), ..Default::default() }); let err = elf_config::validate(&cfg).expect_err("Expected min_confidence validation error."); @@ -535,7 +535,7 @@ fn memory_policy_min_confidence_must_be_in_range() { cfg.memory .policy .rules - .push(elf_config::MemoryPolicyRule { min_confidence: Some(1.01), ..Default::default() }); + .push(MemoryPolicyRule { min_confidence: Some(1.01), ..Default::default() }); let err = elf_config::validate(&cfg).expect_err("Expected min_confidence range validation error."); @@ -551,10 +551,10 @@ fn memory_policy_min_confidence_must_be_in_range() { fn memory_policy_min_importance_must_be_finite() { let mut cfg = base_config(); - cfg.memory.policy.rules.push(elf_config::MemoryPolicyRule { - min_importance: Some(f32::INFINITY), - ..Default::default() - }); + cfg.memory + .policy + .rules + .push(MemoryPolicyRule { min_importance: Some(f32::INFINITY), ..Default::default() }); let err = elf_config::validate(&cfg).expect_err("Expected min_importance validation error."); @@ -571,7 +571,7 @@ fn memory_policy_min_importance_must_be_in_range() { cfg.memory .policy .rules - .push(elf_config::MemoryPolicyRule { min_importance: Some(-0.01), ..Default::default() }); + .push(MemoryPolicyRule { min_importance: Some(-0.01), ..Default::default() }); let err = elf_config::validate(&cfg).expect_err("Expected min_importance range validation error."); @@ -587,10 +587,10 @@ fn memory_policy_min_importance_must_be_in_range() { fn memory_policy_note_type_must_be_known_value() { let mut cfg = base_config(); - cfg.memory.policy.rules.push(elf_config::MemoryPolicyRule { - note_type: Some("unknown".to_string()), - ..Default::default() - }); + cfg.memory + .policy + .rules + .push(MemoryPolicyRule { note_type: Some("unknown".to_string()), ..Default::default() }); let err = elf_config::validate(&cfg).expect_err("Expected note_type validation error."); @@ -606,10 +606,10 @@ fn memory_policy_note_type_must_be_known_value() { fn memory_policy_scope_must_be_allowed() { let mut cfg = base_config(); - cfg.memory.policy.rules.push(elf_config::MemoryPolicyRule { - scope: Some("invalid_scope".to_string()), - ..Default::default() - }); + cfg.memory + .policy + .rules + .push(MemoryPolicyRule { scope: Some("invalid_scope".to_string()), ..Default::default() }); let err = elf_config::validate(&cfg).expect_err("Expected scope validation error."); @@ -639,10 +639,10 @@ fn memory_policy_rule_pairs_must_be_unique() { fn memory_policy_note_type_must_not_be_whitespace_only() { let mut cfg = base_config(); - cfg.memory.policy.rules.push(elf_config::MemoryPolicyRule { - note_type: Some(" ".to_string()), - ..Default::default() - }); + cfg.memory + .policy + .rules + .push(MemoryPolicyRule { note_type: Some(" ".to_string()), ..Default::default() }); let err = elf_config::validate(&cfg).expect_err("Expected whitespace note_type validation error."); @@ -658,10 +658,10 @@ fn memory_policy_note_type_must_not_be_whitespace_only() { fn memory_policy_scope_must_not_be_whitespace_only() { let mut cfg = base_config(); - cfg.memory.policy.rules.push(elf_config::MemoryPolicyRule { - scope: Some(" ".to_string()), - ..Default::default() - }); + cfg.memory + .policy + .rules + .push(MemoryPolicyRule { scope: Some(" ".to_string()), ..Default::default() }); let err = elf_config::validate(&cfg).expect_err("Expected whitespace scope validation error."); diff --git a/packages/elf-service/src/add_event.rs b/packages/elf-service/src/add_event.rs index de806a1d..753fd5f2 100644 --- a/packages/elf-service/src/add_event.rs +++ b/packages/elf-service/src/add_event.rs @@ -8,8 +8,10 @@ use uuid::Uuid; use crate::{ ElfService, Error, InsertVersionArgs, NoteOp, REJECT_EVIDENCE_MISMATCH, - REJECT_WRITE_POLICY_MISMATCH, ResolveUpdateArgs, Result, UpdateDecision, access, - graph_ingestion, ingest_audit, + REJECT_WRITE_POLICY_MISMATCH, ResolveUpdateArgs, Result, UpdateDecision, + access::{self, ORG_PROJECT_ID}, + graph_ingestion, + ingest_audit::{self, IngestAuditArgs}, ingestion_profiles::{self, IngestionProfileRef, IngestionProfileSelector}, structured_fields::{self, StructuredFields}, }; @@ -18,7 +20,7 @@ use elf_domain::{ english_gate, evidence, memory_policy::{self, MemoryPolicyDecision}, ttl, - writegate::{self, WritePolicy, WritePolicyAudit, WritePolicyError}, + writegate::{self, NoteInput, WritePolicy, WritePolicyAudit, WritePolicyError}, }; use elf_storage::models::MemoryNote; @@ -266,7 +268,7 @@ impl ElfService { ) -> Result { let note_data = NoteProcessingData::from_request_and_note(req, ¬e); let effective_project_id = if note_data.scope.trim() == "org_shared" { - access::ORG_PROJECT_ID + ORG_PROJECT_ID } else { req.project_id.as_str() }; @@ -571,7 +573,7 @@ impl ElfService { providers: &self.providers, tenant_id: req.tenant_id.as_str(), project_id: if note_data.scope.trim() == "org_shared" { - access::ORG_PROJECT_ID + ORG_PROJECT_ID } else { req.project_id.as_str() }, @@ -1141,7 +1143,7 @@ fn reject_extracted_note_if_writegate_rejects( scope: &str, text: &str, ) -> Option { - let gate_input = elf_domain::writegate::NoteInput { + let gate_input = NoteInput { note_type: note_type.to_string(), scope: scope.to_string(), text: text.to_string(), @@ -1221,7 +1223,7 @@ async fn record_ingest_decision( graph_present: bool, write_policy_audits: Option>, ) -> Result<()> { - let args = crate::ingest_audit::IngestAuditArgs { + let args = IngestAuditArgs { tenant_id: ctx.tenant_id, project_id: ctx.project_id, agent_id: ctx.agent_id, diff --git a/packages/elf-service/src/add_note.rs b/packages/elf-service/src/add_note.rs index 9344d926..7154bec0 100644 --- a/packages/elf-service/src/add_note.rs +++ b/packages/elf-service/src/add_note.rs @@ -8,7 +8,10 @@ use uuid::Uuid; use crate::{ ElfService, Error, InsertVersionArgs, NoteOp, ResolveUpdateArgs, Result, UpdateDecision, - UpdateDecisionMetadata, access, graph_ingestion, ingest_audit, + UpdateDecisionMetadata, + access::{self, ORG_PROJECT_ID}, + graph_ingestion, + ingest_audit::{self, IngestAuditArgs}, structured_fields::{self, StructuredFields}, }; use elf_config::Config; @@ -16,7 +19,7 @@ use elf_domain::{ english_gate, memory_policy::{self, MemoryPolicyDecision}, ttl, - writegate::{self, WritePolicy, WritePolicyAudit, WritePolicyError}, + writegate::{self, NoteInput, WritePolicy, WritePolicyAudit, WritePolicyError}, }; use elf_storage::models::MemoryNote; @@ -107,7 +110,7 @@ impl ElfService { let embed_version = crate::embedding_version(&self.cfg); let AddNoteRequest { tenant_id, project_id, agent_id, scope, notes } = req; let effective_project_id = - if scope.trim() == "org_shared" { access::ORG_PROJECT_ID } else { project_id.as_str() }; + if scope.trim() == "org_shared" { ORG_PROJECT_ID } else { project_id.as_str() }; let mut results = Vec::with_capacity(notes.len()); for (note_idx, note) in notes.into_iter().enumerate() { @@ -437,7 +440,7 @@ impl ElfService { min_importance: Option, write_policy_audit: Option, ) -> Result<()> { - let decision = crate::ingest_audit::IngestAuditArgs { + let decision = IngestAuditArgs { tenant_id: ctx.tenant_id, project_id: ctx.project_id, agent_id: ctx.agent_id, @@ -894,7 +897,7 @@ fn reject_note_if_writegate_rejects( scope: &str, note: &AddNoteInput, ) -> Option { - let gate_input = elf_domain::writegate::NoteInput { + let gate_input = NoteInput { note_type: note.r#type.clone(), scope: scope.to_string(), text: note.text.clone(), diff --git a/packages/elf-service/src/delete.rs b/packages/elf-service/src/delete.rs index 0570d724..34b2fc7f 100644 --- a/packages/elf-service/src/delete.rs +++ b/packages/elf-service/src/delete.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use time::OffsetDateTime; use uuid::Uuid; -use crate::{ElfService, Error, InsertVersionArgs, NoteOp, Result, access}; +use crate::{ElfService, Error, InsertVersionArgs, NoteOp, Result, access::ORG_PROJECT_ID}; use elf_storage::models::MemoryNote; /// Request payload for note deletion. @@ -54,7 +54,7 @@ FOR UPDATE", .bind(req.note_id) .bind(tenant_id) .bind(project_id) - .bind(access::ORG_PROJECT_ID) + .bind(ORG_PROJECT_ID) .fetch_optional(&mut *tx) .await? .ok_or_else(|| Error::InvalidRequest { message: "Note not found.".to_string() })?; diff --git a/packages/elf-service/src/docs.rs b/packages/elf-service/src/docs.rs index 55196442..ee1fbe4f 100644 --- a/packages/elf-service/src/docs.rs +++ b/packages/elf-service/src/docs.rs @@ -21,7 +21,7 @@ use uuid::Uuid; use crate::{ ElfService, Error, Result, - access::{self, SharedSpaceGrantKey}, + access::{self, ORG_PROJECT_ID, SharedSpaceGrantKey}, search, }; use elf_config::Config; @@ -558,11 +558,8 @@ impl ElfService { let DocsPutRequest { tenant_id, project_id, agent_id, scope, title, source_ref, .. } = req; let chunking_profile = resolve_doc_chunking_profile(doc_type); let tokenizer = load_tokenizer(&self.cfg)?; - let effective_project_id = if scope.trim() == "org_shared" { - crate::access::ORG_PROJECT_ID - } else { - project_id.as_str() - }; + let effective_project_id = + if scope.trim() == "org_shared" { ORG_PROJECT_ID } else { project_id.as_str() }; let content_bytes = content.len(); let content_hash = blake3::hash(content.as_bytes()); let doc_id = Uuid::new_v4(); @@ -688,7 +685,7 @@ LIMIT 1", .bind(req.doc_id) .bind(tenant_id) .bind(project_id) - .bind(crate::access::ORG_PROJECT_ID) + .bind(ORG_PROJECT_ID) .fetch_optional(&self.db.pool) .await?; let Some(row) = row else { @@ -1807,7 +1804,7 @@ fn build_doc_search_filter( if allowed_scopes.iter().any(|scope| scope == "org_shared") { let org_filter = Filter::all([ - Condition::matches("project_id", crate::access::ORG_PROJECT_ID.to_string()), + Condition::matches("project_id", ORG_PROJECT_ID.to_string()), Condition::matches("scope", "org_shared".to_string()), ]); @@ -2164,7 +2161,7 @@ LIMIT 1", .bind(doc_id) .bind(tenant_id) .bind(project_id) - .bind(crate::access::ORG_PROJECT_ID) + .bind(ORG_PROJECT_ID) .fetch_optional(executor) .await?; @@ -2303,7 +2300,7 @@ WHERE c.chunk_id = ANY($1) .bind(tenant_id) .bind(project_id) .bind(status) - .bind(crate::access::ORG_PROJECT_ID) + .bind(ORG_PROJECT_ID) .fetch_all(executor) .await?; let mut map = HashMap::with_capacity(rows.len()); diff --git a/packages/elf-service/src/graph_query.rs b/packages/elf-service/src/graph_query.rs index eca25bd6..f949aa83 100644 --- a/packages/elf-service/src/graph_query.rs +++ b/packages/elf-service/src/graph_query.rs @@ -7,7 +7,11 @@ use sqlx::{FromRow, PgConnection}; use time::OffsetDateTime; use uuid::Uuid; -use crate::{ElfService, Error, Result, access, search}; +use crate::{ + ElfService, Error, Result, + access::{self, ORG_PROJECT_ID}, + search, +}; use elf_storage::{graph, models::GraphEntity}; /// Schema identifier for graph-query responses. @@ -676,7 +680,7 @@ async fn fetch_graph_query_rows( .bind(shared_scope_keys) .bind(limit_plus_one) .bind(GRAPH_QUERY_EVIDENCE_LIMIT) - .bind(crate::access::ORG_PROJECT_ID) + .bind(ORG_PROJECT_ID) .bind(predicate_id) .fetch_all(conn) .await?; diff --git a/packages/elf-service/src/list.rs b/packages/elf-service/src/list.rs index 5f21e7ab..d1e94803 100644 --- a/packages/elf-service/src/list.rs +++ b/packages/elf-service/src/list.rs @@ -8,7 +8,10 @@ use sqlx::{PgPool, QueryBuilder}; use time::OffsetDateTime; use uuid::Uuid; -use crate::{ElfService, Error, Result, access}; +use crate::{ + ElfService, Error, Result, + access::{self, ORG_PROJECT_ID}, +}; use elf_storage::models::MemoryNote; /// Request payload for note listing. @@ -233,7 +236,7 @@ async fn list_notes( builder.push(" AND (project_id = "); builder.push_bind(project_id); builder.push(" OR (project_id = "); - builder.push_bind(access::ORG_PROJECT_ID); + builder.push_bind(ORG_PROJECT_ID); builder.push(" AND scope = "); builder.push_bind("org_shared"); builder.push("))"); diff --git a/packages/elf-service/src/notes.rs b/packages/elf-service/src/notes.rs index 4bad76ab..5b4a2f5d 100644 --- a/packages/elf-service/src/notes.rs +++ b/packages/elf-service/src/notes.rs @@ -8,7 +8,8 @@ use time::OffsetDateTime; use uuid::Uuid; use crate::{ - ElfService, Error, Result, access, + ElfService, Error, Result, + access::{self, ORG_PROJECT_ID}, structured_fields::{self, StructuredFields}, }; use elf_storage::models::MemoryNote; @@ -93,7 +94,7 @@ WHERE note_id = $1 .bind(req.note_id) .bind(tenant_id) .bind(project_id) - .bind(access::ORG_PROJECT_ID) + .bind(ORG_PROJECT_ID) .fetch_optional(&self.db.pool) .await?; let Some(note) = row else { diff --git a/packages/elf-service/src/progressive_search.rs b/packages/elf-service/src/progressive_search.rs index 951a3aa9..c912f84b 100644 --- a/packages/elf-service/src/progressive_search.rs +++ b/packages/elf-service/src/progressive_search.rs @@ -15,7 +15,7 @@ use uuid::Uuid; use crate::{ ElfService, NoteFetchResponse, PayloadLevel, QueryPlan, SearchRequest, SearchTrajectorySummary, - access::{self, SharedSpaceGrantKey}, + access::{self, ORG_PROJECT_ID, SharedSpaceGrantKey}, structured_fields::{self, StructuredFields}, }; use elf_config::Config; @@ -632,7 +632,7 @@ WHERE note_id = ANY($1::uuid[]) .bind(requested_in_session.as_slice()) .bind(session.tenant_id.as_str()) .bind(session.project_id.as_str()) - .bind(access::ORG_PROJECT_ID) + .bind(ORG_PROJECT_ID) .fetch_all(&self.db.pool) .await?; diff --git a/packages/elf-service/src/search.rs b/packages/elf-service/src/search.rs index 10d17392..4fbbc268 100644 --- a/packages/elf-service/src/search.rs +++ b/packages/elf-service/src/search.rs @@ -21,7 +21,11 @@ use sqlx::{FromRow, PgConnection, PgExecutor, PgPool, QueryBuilder, Row}; use time::{Duration, OffsetDateTime}; use uuid::Uuid; -use crate::{ElfService, Result, access, ranking_explain_v2}; +use crate::{ + ElfService, Result, + access::{self, ORG_PROJECT_ID}, + ranking_explain_v2::{self, SEARCH_RANKING_EXPLAIN_SCHEMA_V2, TraceTermsArgs}, +}; use elf_config::{Config, SearchCache}; use elf_domain::english_gate; use elf_storage::{ @@ -3432,7 +3436,7 @@ LIMIT $7", .bind(args.non_private_scopes) .bind(args.vec_text) .bind(args.retrieval_limit) - .bind(access::ORG_PROJECT_ID) + .bind(ORG_PROJECT_ID) .fetch_all(&self.db.pool) .await?; @@ -3476,7 +3480,7 @@ LIMIT $8", .bind(args.non_private_scopes) .bind(args.vec_text) .bind(args.retrieval_limit) - .bind(access::ORG_PROJECT_ID) + .bind(ORG_PROJECT_ID) .fetch_all(&self.db.pool) .await?; @@ -4046,7 +4050,7 @@ ORDER BY c.note_id ASC, e.vec <=> $3::text::vector ASC", let mut scored = Vec::with_capacity(snippet_items.len()); for ((item, rerank_score), rerank_rank) in - snippet_items.into_iter().zip(scores.into_iter()).zip(rerank_ranks.into_iter()) + snippet_items.into_iter().zip(scores).zip(rerank_ranks) { scored.push(score_chunk_candidate(&score_ctx, item, rerank_score, rerank_rank)); } @@ -4600,7 +4604,7 @@ WHERE note_id = ANY($1::uuid[]) .bind(candidate_note_ids) .bind(tenant_id) .bind(project_id) - .bind(access::ORG_PROJECT_ID) + .bind(ORG_PROJECT_ID) .fetch_all(&self.db.pool) .await?; let mut note_meta = HashMap::new(); @@ -4954,7 +4958,7 @@ fn build_search_filter( if allowed_scopes.iter().any(|scope| scope == "org_shared") { let org_filter = Filter::all([ - Condition::matches("project_id", access::ORG_PROJECT_ID.to_string()), + Condition::matches("project_id", ORG_PROJECT_ID.to_string()), Condition::matches("scope", "org_shared".to_string()), ]); @@ -5147,34 +5151,31 @@ fn build_search_item_and_trace_item( matched_fields, args.structured_matches.get(&args.scored_chunk.item.note.note_id), ); - let trace_terms = - ranking_explain_v2::build_trace_terms_v2(ranking_explain_v2::TraceTermsArgs { - cfg: args.cfg, - blend_enabled: args.blend_policy.enabled, - retrieval_normalization: args.blend_policy.retrieval_normalization.as_str(), - rerank_normalization: args.blend_policy.rerank_normalization.as_str(), - blend_retrieval_weight: args.scored_chunk.blend_retrieval_weight, - retrieval_rank: args.scored_chunk.item.retrieval_rank, - retrieval_norm: args.scored_chunk.retrieval_norm, - retrieval_term: args.scored_chunk.retrieval_term, - rerank_score: args.scored_chunk.rerank_score, - rerank_rank: args.scored_chunk.rerank_rank, - rerank_norm: args.scored_chunk.rerank_norm, - rerank_term: args.scored_chunk.rerank_term, - tie_breaker_score: args.scored_chunk.tie_breaker_score, - importance: args.scored_chunk.importance, - age_days: args.scored_chunk.age_days, - scope: args.scored_chunk.item.note.scope.as_str(), - scope_context_boost: args.scored_chunk.scope_context_boost, - deterministic_lexical_overlap_ratio: args - .scored_chunk - .deterministic_lexical_overlap_ratio, - deterministic_lexical_bonus: args.scored_chunk.deterministic_lexical_bonus, - deterministic_hit_count: args.scored_chunk.deterministic_hit_count, - deterministic_last_hit_age_days: args.scored_chunk.deterministic_last_hit_age_days, - deterministic_hit_boost: args.scored_chunk.deterministic_hit_boost, - deterministic_decay_penalty: args.scored_chunk.deterministic_decay_penalty, - }); + let trace_terms = ranking_explain_v2::build_trace_terms_v2(TraceTermsArgs { + cfg: args.cfg, + blend_enabled: args.blend_policy.enabled, + retrieval_normalization: args.blend_policy.retrieval_normalization.as_str(), + rerank_normalization: args.blend_policy.rerank_normalization.as_str(), + blend_retrieval_weight: args.scored_chunk.blend_retrieval_weight, + retrieval_rank: args.scored_chunk.item.retrieval_rank, + retrieval_norm: args.scored_chunk.retrieval_norm, + retrieval_term: args.scored_chunk.retrieval_term, + rerank_score: args.scored_chunk.rerank_score, + rerank_rank: args.scored_chunk.rerank_rank, + rerank_norm: args.scored_chunk.rerank_norm, + rerank_term: args.scored_chunk.rerank_term, + tie_breaker_score: args.scored_chunk.tie_breaker_score, + importance: args.scored_chunk.importance, + age_days: args.scored_chunk.age_days, + scope: args.scored_chunk.item.note.scope.as_str(), + scope_context_boost: args.scored_chunk.scope_context_boost, + deterministic_lexical_overlap_ratio: args.scored_chunk.deterministic_lexical_overlap_ratio, + deterministic_lexical_bonus: args.scored_chunk.deterministic_lexical_bonus, + deterministic_hit_count: args.scored_chunk.deterministic_hit_count, + deterministic_last_hit_age_days: args.scored_chunk.deterministic_last_hit_age_days, + deterministic_hit_boost: args.scored_chunk.deterministic_hit_boost, + deterministic_decay_penalty: args.scored_chunk.deterministic_decay_penalty, + }); let response_terms = ranking_explain_v2::strip_term_inputs(&trace_terms); let relation_context = args.relation_contexts.get(&args.scored_chunk.item.note.note_id).cloned(); @@ -5191,7 +5192,7 @@ fn build_search_item_and_trace_item( matched_fields: matched_fields.clone(), }, ranking: SearchRankingExplain { - schema: ranking_explain_v2::SEARCH_RANKING_EXPLAIN_SCHEMA_V2.to_string(), + schema: SEARCH_RANKING_EXPLAIN_SCHEMA_V2.to_string(), policy_id: args.policy_id.to_string(), final_score: args.scored_chunk.final_score, terms: response_terms, @@ -5202,7 +5203,7 @@ fn build_search_item_and_trace_item( let trace_explain = SearchExplain { r#match: SearchMatchExplain { matched_terms, matched_fields }, ranking: SearchRankingExplain { - schema: ranking_explain_v2::SEARCH_RANKING_EXPLAIN_SCHEMA_V2.to_string(), + schema: SEARCH_RANKING_EXPLAIN_SCHEMA_V2.to_string(), policy_id: args.policy_id.to_string(), final_score: args.scored_chunk.final_score, terms: trace_terms, @@ -5704,7 +5705,7 @@ fn build_replay_items( let mut out = Vec::with_capacity(results.len()); for scored in results { - let terms = ranking_explain_v2::build_trace_terms_v2(ranking_explain_v2::TraceTermsArgs { + let terms = ranking_explain_v2::build_trace_terms_v2(TraceTermsArgs { cfg, blend_enabled: blend_policy.enabled, retrieval_normalization: blend_policy.retrieval_normalization.as_str(), @@ -5732,7 +5733,7 @@ fn build_replay_items( let explain = SearchExplain { r#match: SearchMatchExplain { matched_terms: Vec::new(), matched_fields: Vec::new() }, ranking: SearchRankingExplain { - schema: ranking_explain_v2::SEARCH_RANKING_EXPLAIN_SCHEMA_V2.to_string(), + schema: SEARCH_RANKING_EXPLAIN_SCHEMA_V2.to_string(), policy_id: policy_id.to_string(), final_score: scored.final_score, terms, diff --git a/packages/elf-service/src/sharing.rs b/packages/elf-service/src/sharing.rs index 95311e5d..7687f723 100644 --- a/packages/elf-service/src/sharing.rs +++ b/packages/elf-service/src/sharing.rs @@ -6,7 +6,10 @@ use serde::{Deserialize, Serialize}; use sqlx::FromRow; use uuid::Uuid; -use crate::{ElfService, Error, InsertVersionArgs, access}; +use crate::{ + ElfService, Error, InsertVersionArgs, + access::{self, ORG_PROJECT_ID}, +}; use elf_storage::models::MemoryNote; const PROJECT_SPACE_GRANT_UPSERT_SQL: &str = "\ @@ -270,7 +273,7 @@ FOR UPDATE", .bind(req.note_id) .bind(tenant_id) .bind(project_id) - .bind(access::ORG_PROJECT_ID) + .bind(ORG_PROJECT_ID) .fetch_optional(&mut *tx) .await? .ok_or_else(|| Error::InvalidRequest { message: "Note not found.".to_string() })?; @@ -296,8 +299,7 @@ FOR UPDATE", return Err(Error::ScopeDenied { message: "Scope is not allowed.".to_string() }); } - let target_project_id = - if scope == "org_shared" { access::ORG_PROJECT_ID } else { project_id }; + let target_project_id = if scope == "org_shared" { ORG_PROJECT_ID } else { project_id }; access::ensure_active_project_scope_grant( &mut *tx, @@ -377,7 +379,7 @@ FOR UPDATE", .bind(req.note_id) .bind(tenant_id) .bind(project_id) - .bind(access::ORG_PROJECT_ID) + .bind(ORG_PROJECT_ID) .fetch_optional(&mut *tx) .await? .ok_or_else(|| Error::InvalidRequest { message: "Note not found.".to_string() })?; @@ -401,7 +403,7 @@ FOR UPDATE", let now = time::OffsetDateTime::now_utc(); let prev_snapshot = crate::note_snapshot(¬e); - if note.scope == "org_shared" && note.project_id == access::ORG_PROJECT_ID { + if note.scope == "org_shared" && note.project_id == ORG_PROJECT_ID { note.project_id = project_id.to_string(); } @@ -486,8 +488,7 @@ FOR UPDATE", let grantee_agent_id_ref = grantee_agent_id.as_deref(); let now = time::OffsetDateTime::now_utc(); - let effective_project_id = - if scope == "org_shared" { access::ORG_PROJECT_ID } else { project_id }; + let effective_project_id = if scope == "org_shared" { ORG_PROJECT_ID } else { project_id }; if req.grantee_kind == GranteeKind::Project { self.upsert_project_grant(tenant_id, effective_project_id, scope, agent_id, now) @@ -604,8 +605,7 @@ FOR UPDATE", return Err(Error::ScopeDenied { message: "Scope is not allowed.".to_string() }); } - let effective_project_id = - if scope == "org_shared" { access::ORG_PROJECT_ID } else { project_id }; + let effective_project_id = if scope == "org_shared" { ORG_PROJECT_ID } else { project_id }; let revocation = sqlx::query( "\ UPDATE memory_space_grants @@ -667,8 +667,7 @@ WHERE tenant_id = $1 return Err(Error::ScopeDenied { message: "Scope is not allowed.".to_string() }); } - let effective_project_id = - if scope == "org_shared" { access::ORG_PROJECT_ID } else { project_id }; + let effective_project_id = if scope == "org_shared" { ORG_PROJECT_ID } else { project_id }; #[derive(FromRow)] struct Row { diff --git a/packages/elf-service/src/update.rs b/packages/elf-service/src/update.rs index bc938391..b508a522 100644 --- a/packages/elf-service/src/update.rs +++ b/packages/elf-service/src/update.rs @@ -6,8 +6,11 @@ use sqlx::{Postgres, Transaction}; use time::OffsetDateTime; use uuid::Uuid; -use crate::{ElfService, Error, InsertVersionArgs, NoteOp, Result, access}; -use elf_domain::{english_gate, ttl, writegate}; +use crate::{ElfService, Error, InsertVersionArgs, NoteOp, Result, access::ORG_PROJECT_ID}; +use elf_domain::{ + english_gate, ttl, + writegate::{self, NoteInput}, +}; use elf_storage::models::MemoryNote; /// Request payload for note updates. @@ -79,7 +82,7 @@ impl ElfService { } else { note.text.clone() }; - let gate = elf_domain::writegate::NoteInput { + let gate = NoteInput { note_type: note.r#type.clone(), scope: note.scope.clone(), text: candidate_text, @@ -166,7 +169,7 @@ FOR UPDATE", .bind(note_id) .bind(tenant_id) .bind(project_id) - .bind(access::ORG_PROJECT_ID) + .bind(ORG_PROJECT_ID) .fetch_optional(&mut **tx) .await? .ok_or_else(|| Error::InvalidRequest { message: "Note not found.".to_string() }) diff --git a/packages/elf-service/tests/acceptance/docs_extension_v1.rs b/packages/elf-service/tests/acceptance/docs_extension_v1.rs index 66b417dc..f110596a 100644 --- a/packages/elf-service/tests/acceptance/docs_extension_v1.rs +++ b/packages/elf-service/tests/acceptance/docs_extension_v1.rs @@ -17,7 +17,7 @@ use tokio::{ }; use uuid::Uuid; -use crate::acceptance::{self, SpyExtractor, StubEmbedding, StubRerank}; +use crate::acceptance::{self, SpyExtractor, StubEmbedding, StubRerank, chunking::ChunkingConfig}; use elf_config::EmbeddingProviderConfig; use elf_service::{ AddNoteInput, AddNoteRequest, BoxFuture, DocsExcerptsGetRequest, DocsGetRequest, @@ -27,7 +27,7 @@ use elf_service::{ }; use elf_storage::{db::Db, qdrant::QdrantStore}; use elf_testkit::TestDatabase; -use elf_worker::worker; +use elf_worker::worker::{self, WorkerState}; const TEST_CONTENT: &str = "ELF docs extension v1 stores evidence. Keyword: peregrine.\nSecond sentence for chunking."; @@ -1876,7 +1876,7 @@ async fn assert_doc_excerpt(service: &ElfService, doc_id: Uuid, content_hash: &s async fn spawn_doc_worker(service: &ElfService) -> (JoinHandle<()>, Sender<()>) { let (api_base, shutdown) = start_embed_server().await; - let worker_state = worker::WorkerState { + let worker_state = WorkerState { db: Db::connect(&service.cfg.storage.postgres).await.expect("Failed to connect worker DB."), qdrant: QdrantStore::new(&service.cfg.storage.qdrant) .expect("Failed to build Qdrant store."), @@ -1895,7 +1895,7 @@ async fn spawn_doc_worker(service: &ElfService) -> (JoinHandle<()>, Sender<()>) timeout_ms: 1_000, default_headers: Map::new(), }, - chunking: crate::acceptance::chunking::ChunkingConfig { max_tokens: 64, overlap_tokens: 8 }, + chunking: ChunkingConfig { max_tokens: 64, overlap_tokens: 8 }, tokenizer: build_test_tokenizer(), }; let handle = tokio::spawn(async move { diff --git a/packages/elf-service/tests/acceptance/graph_ingestion.rs b/packages/elf-service/tests/acceptance/graph_ingestion.rs index 0e4596e2..639c9096 100644 --- a/packages/elf-service/tests/acceptance/graph_ingestion.rs +++ b/packages/elf-service/tests/acceptance/graph_ingestion.rs @@ -8,7 +8,7 @@ use sqlx::{FromRow, PgPool}; use time::OffsetDateTime; use uuid::Uuid; -use crate::acceptance; +use crate::acceptance::{self, SpyExtractor, StubEmbedding, StubRerank}; use elf_config::EmbeddingProviderConfig; use elf_domain::memory_policy::MemoryPolicyDecision; use elf_service::{ @@ -384,8 +384,8 @@ async fn add_note_duplicate_fact_attaches_multiple_evidence() { }; let providers = Providers::new( Arc::new(HashEmbedding { vector_dim: 4_096 }), - Arc::new(crate::acceptance::StubRerank), - Arc::new(crate::acceptance::SpyExtractor { + Arc::new(StubRerank), + Arc::new(SpyExtractor { calls: Arc::new(AtomicUsize::new(0)), payload: serde_json::json!({ "notes": [] }), }), @@ -457,9 +457,9 @@ async fn add_note_single_predicate_supersedes_conflicting_fact() { return; }; let providers = Providers::new( - Arc::new(crate::acceptance::StubEmbedding { vector_dim: 4_096 }), - Arc::new(crate::acceptance::StubRerank), - Arc::new(crate::acceptance::SpyExtractor { + Arc::new(StubEmbedding { vector_dim: 4_096 }), + Arc::new(StubRerank), + Arc::new(SpyExtractor { calls: Arc::new(AtomicUsize::new(0)), payload: serde_json::json!({ "notes": [] }), }), @@ -541,9 +541,9 @@ async fn add_note_invalid_relation_rejected_has_field_path() { return; }; let providers = Providers::new( - Arc::new(crate::acceptance::StubEmbedding { vector_dim: 4_096 }), - Arc::new(crate::acceptance::StubRerank), - Arc::new(crate::acceptance::SpyExtractor { + Arc::new(StubEmbedding { vector_dim: 4_096 }), + Arc::new(StubRerank), + Arc::new(SpyExtractor { calls: Arc::new(AtomicUsize::new(0)), payload: serde_json::json!({ "notes": [] }), }), @@ -615,9 +615,9 @@ async fn add_note_persists_graph_relations() { return; }; let providers = Providers::new( - Arc::new(crate::acceptance::StubEmbedding { vector_dim: 4_096 }), - Arc::new(crate::acceptance::StubRerank), - Arc::new(crate::acceptance::SpyExtractor { + Arc::new(StubEmbedding { vector_dim: 4_096 }), + Arc::new(StubRerank), + Arc::new(SpyExtractor { calls: Arc::new(AtomicUsize::new(0)), payload: serde_json::json!({ "notes": [] }), }), @@ -719,12 +719,9 @@ async fn add_event_persists_graph_relations() { }] }); let providers = Providers::new( - Arc::new(crate::acceptance::StubEmbedding { vector_dim: 4_096 }), - Arc::new(crate::acceptance::StubRerank), - Arc::new(crate::acceptance::SpyExtractor { - calls: Arc::new(AtomicUsize::new(0)), - payload: extractor_payload, - }), + Arc::new(StubEmbedding { vector_dim: 4_096 }), + Arc::new(StubRerank), + Arc::new(SpyExtractor { calls: Arc::new(AtomicUsize::new(0)), payload: extractor_payload }), ); let collection = test_db.collection_name("elf_acceptance"); let docs_collection = test_db.collection_name("elf_acceptance_docs"); diff --git a/packages/elf-service/tests/acceptance/outbox_eventual_consistency.rs b/packages/elf-service/tests/acceptance/outbox_eventual_consistency.rs index 50fe9e50..f054ad1d 100644 --- a/packages/elf-service/tests/acceptance/outbox_eventual_consistency.rs +++ b/packages/elf-service/tests/acceptance/outbox_eventual_consistency.rs @@ -20,11 +20,11 @@ use tokio::{ }; use uuid::Uuid; -use crate::acceptance::{self, SpyExtractor, StubEmbedding, StubRerank}; +use crate::acceptance::{self, SpyExtractor, StubEmbedding, StubRerank, chunking::ChunkingConfig}; use elf_config::EmbeddingProviderConfig; use elf_service::{AddNoteInput, AddNoteRequest, ElfService, Providers}; use elf_storage::{db::Db, qdrant::QdrantStore}; -use elf_worker::worker; +use elf_worker::worker::{self, WorkerState}; #[derive(FromRow)] struct OutboxRow { @@ -131,7 +131,7 @@ async fn embed_handler( } async fn spawn_outbox_worker(service: &ElfService, api_base: String) -> JoinHandle<()> { - let worker_state = worker::WorkerState { + let worker_state = WorkerState { db: Db::connect(&service.cfg.storage.postgres).await.expect("Failed to connect worker DB."), qdrant: QdrantStore::new(&service.cfg.storage.qdrant) .expect("Failed to build Qdrant store."), @@ -150,7 +150,7 @@ async fn spawn_outbox_worker(service: &ElfService, api_base: String) -> JoinHand timeout_ms: 1_000, default_headers: Map::new(), }, - chunking: crate::acceptance::chunking::ChunkingConfig { max_tokens: 64, overlap_tokens: 8 }, + chunking: ChunkingConfig { max_tokens: 64, overlap_tokens: 8 }, tokenizer: build_test_tokenizer(), }; diff --git a/packages/elf-service/tests/acceptance/structured_field_retrieval.rs b/packages/elf-service/tests/acceptance/structured_field_retrieval.rs index 0fd069c5..d3103c43 100644 --- a/packages/elf-service/tests/acceptance/structured_field_retrieval.rs +++ b/packages/elf-service/tests/acceptance/structured_field_retrieval.rs @@ -12,7 +12,7 @@ use sqlx::PgExecutor; use time::OffsetDateTime; use uuid::Uuid; -use crate::acceptance; +use crate::acceptance::{self, SpyExtractor, StubEmbedding}; use elf_config::ProviderConfig; use elf_service::{BoxFuture, ElfService, Providers, RerankProvider, Result, SearchRequest}; use elf_storage::qdrant::{BM25_MODEL, BM25_VECTOR_NAME, DENSE_VECTOR_NAME}; @@ -117,9 +117,9 @@ async fn setup_context(test_name: &str) -> Option { return None; }; let providers = Providers::new( - Arc::new(crate::acceptance::StubEmbedding { vector_dim: 4_096 }), + Arc::new(StubEmbedding { vector_dim: 4_096 }), Arc::new(KeywordRerank { keyword: "ZEBRA" }), - Arc::new(crate::acceptance::SpyExtractor { + Arc::new(SpyExtractor { calls: Arc::new(AtomicUsize::new(0)), payload: serde_json::json!({ "notes": [] }), }), From d53eb482fb448e126d8a64403ad4e774d4b61a39 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Mon, 8 Jun 2026 11:31:38 +0800 Subject: [PATCH 2/2] {"schema":"decodex/commit/1","summary":"repair API contract routes and HTTP tests","authority":"XY-790"} --- apps/elf-api/src/routes.rs | 39 +++--- apps/elf-api/tests/http.rs | 255 ++++++++++++++++++++++++++++--------- 2 files changed, 214 insertions(+), 80 deletions(-) diff --git a/apps/elf-api/src/routes.rs b/apps/elf-api/src/routes.rs index 145b4e9f..0afc91b9 100644 --- a/apps/elf-api/src/routes.rs +++ b/apps/elf-api/src/routes.rs @@ -520,24 +520,27 @@ pub fn router(state: AppState) -> Router { .route("/v2/notes/ingest", routing::post(notes_ingest)) .route("/v2/events/ingest", routing::post(events_ingest)) .route("/v2/searches", routing::post(searches_create)) - .route("/v2/searches/:search_id", routing::get(searches_get)) - .route("/v2/searches/:search_id/timeline", routing::get(searches_timeline)) - .route("/v2/searches/:search_id/notes", routing::post(searches_notes)) + .route("/v2/searches/{search_id}", routing::get(searches_get)) + .route("/v2/searches/{search_id}/timeline", routing::get(searches_timeline)) + .route("/v2/searches/{search_id}/notes", routing::post(searches_notes)) .route("/v2/graph/query", routing::post(graph_query)) .route("/v2/notes", routing::get(notes_list)) .route( - "/v2/notes/:note_id", + "/v2/notes/{note_id}", routing::get(notes_get).patch(notes_patch).delete(notes_delete), ) - .route("/v2/notes/:note_id/publish", routing::post(notes_publish)) - .route("/v2/notes/:note_id/unpublish", routing::post(notes_unpublish)) - .route("/v2/spaces/:space/grants", routing::get(space_grants_list).post(space_grant_upsert)) - .route("/v2/spaces/:space/grants/revoke", routing::post(space_grant_revoke)) + .route("/v2/notes/{note_id}/publish", routing::post(notes_publish)) + .route("/v2/notes/{note_id}/unpublish", routing::post(notes_unpublish)) + .route( + "/v2/spaces/{space}/grants", + routing::get(space_grants_list).post(space_grant_upsert), + ) + .route("/v2/spaces/{space}/grants/revoke", routing::post(space_grant_revoke)) .with_state(state.clone()) .layer(DefaultBodyLimit::max(MAX_REQUEST_BYTES)); let docs_router = Router::new() .route("/v2/docs", routing::post(docs_put)) - .route("/v2/docs/:doc_id", routing::get(docs_get)) + .route("/v2/docs/{doc_id}", routing::get(docs_get)) .route("/v2/docs/search/l0", routing::post(docs_search_l0)) .route("/v2/docs/excerpts", routing::post(docs_excerpts_get)) .with_state(state) @@ -561,11 +564,11 @@ pub fn admin_router(state: AppState) -> Router { .put(admin_ingestion_profile_default_set), ) .route( - "/v2/admin/events/ingestion-profiles/:profile_id/versions", + "/v2/admin/events/ingestion-profiles/{profile_id}/versions", routing::get(admin_ingestion_profile_versions_list), ) .route( - "/v2/admin/events/ingestion-profiles/:profile_id", + "/v2/admin/events/ingestion-profiles/{profile_id}", routing::get(admin_ingestion_profile_get), ) .route( @@ -575,20 +578,20 @@ pub fn admin_router(state: AppState) -> Router { .route("/v2/admin/qdrant/rebuild", routing::post(rebuild_qdrant)) .route("/v2/admin/searches/raw", routing::post(searches_raw)) .route("/v2/admin/traces/recent", routing::get(trace_recent_list)) - .route("/v2/admin/traces/:trace_id", routing::get(trace_get)) - .route("/v2/admin/traces/:trace_id/bundle", routing::get(trace_bundle_get)) - .route("/v2/admin/trajectories/:trace_id", routing::get(trace_trajectory_get)) - .route("/v2/admin/trace-items/:item_id", routing::get(trace_item_get)) + .route("/v2/admin/traces/{trace_id}", routing::get(trace_get)) + .route("/v2/admin/traces/{trace_id}/bundle", routing::get(trace_bundle_get)) + .route("/v2/admin/trajectories/{trace_id}", routing::get(trace_trajectory_get)) + .route("/v2/admin/trace-items/{item_id}", routing::get(trace_item_get)) .route("/v2/admin/graph/predicates", routing::get(admin_graph_predicates_list)) .route( - "/v2/admin/graph/predicates/:predicate_id", + "/v2/admin/graph/predicates/{predicate_id}", routing::patch(admin_graph_predicate_patch), ) .route( - "/v2/admin/graph/predicates/:predicate_id/aliases", + "/v2/admin/graph/predicates/{predicate_id}/aliases", routing::post(admin_graph_predicate_alias_add).get(admin_graph_predicate_aliases_list), ) - .route("/v2/admin/notes/:note_id/provenance", routing::get(admin_note_provenance_get)) + .route("/v2/admin/notes/{note_id}/provenance", routing::get(admin_note_provenance_get)) .with_state(state) .layer(DefaultBodyLimit::max(MAX_REQUEST_BYTES)) .layer(middleware::from_fn_with_state(auth_state, admin_auth_middleware)) diff --git a/apps/elf-api/tests/http.rs b/apps/elf-api/tests/http.rs index cab5ff1a..820bebd8 100644 --- a/apps/elf-api/tests/http.rs +++ b/apps/elf-api/tests/http.rs @@ -10,6 +10,7 @@ use axum::{ http::{Request, Response, StatusCode}, }; use serde_json::Map; +use time::{Duration, OffsetDateTime, format_description::well_known::Rfc3339}; use tower::util::ServiceExt as _; use uuid::Uuid; @@ -89,7 +90,7 @@ fn test_config(dsn: String, qdrant_url: String, collection: String) -> Config { log_level: "info".to_string(), }, storage: Storage { - postgres: Postgres { dsn, pool_max_conns: 1 }, + postgres: Postgres { dsn, pool_max_conns: 4 }, qdrant: Qdrant { url: qdrant_url, collection: collection.clone(), @@ -195,11 +196,11 @@ fn test_config(dsn: String, qdrant_url: String, collection: String) -> Config { fn dummy_embedding_provider() -> EmbeddingProviderConfig { EmbeddingProviderConfig { - provider_id: "test".to_string(), + provider_id: "local".to_string(), api_base: "http://127.0.0.1:1".to_string(), api_key: "test-key".to_string(), path: "/".to_string(), - model: "test".to_string(), + model: "local-hash".to_string(), dimensions: 4_096, timeout_ms: 1_000, default_headers: Map::new(), @@ -208,11 +209,11 @@ fn dummy_embedding_provider() -> EmbeddingProviderConfig { fn dummy_provider() -> ProviderConfig { ProviderConfig { - provider_id: "test".to_string(), + provider_id: "local".to_string(), api_base: "http://127.0.0.1:1".to_string(), api_key: "test-key".to_string(), path: "/".to_string(), - model: "test".to_string(), + model: "local-token-overlap".to_string(), timeout_ms: 1_000, default_headers: Map::new(), } @@ -240,6 +241,27 @@ fn assert_openapi_method(spec: &serde_json::Value, path: &str, method: &str) { assert!(operation.is_some(), "Missing OpenAPI operation {method} {path}"); } +fn test_embedding_version(state: &AppState) -> String { + format!( + "{}:{}:{}", + state.service.cfg.providers.embedding.provider_id, + state.service.cfg.providers.embedding.model, + state.service.cfg.storage.qdrant.vector_dim + ) +} + +fn unit_vector_text(dim: usize) -> String { + let mut values = Vec::with_capacity(dim); + + for index in 0..dim { + let value = if index == 0 { "1" } else { "0" }; + + values.push(value); + } + + format!("[{}]", values.join(",")) +} + async fn test_env() -> Option<(TestDatabase, String, String)> { let base_dsn = match elf_testkit::env_dsn() { Some(value) => value, @@ -568,12 +590,18 @@ async fn create_note_for_payload_level_tests( ) .await .expect("Failed to call note ingest."); - - assert_eq!(response.status(), StatusCode::OK); - + let status = response.status(); let body = body::to_bytes(response.into_body(), usize::MAX) .await .expect("Failed to read note ingest response body."); + + assert_eq!( + status, + StatusCode::OK, + "Unexpected note ingest response: {}", + String::from_utf8_lossy(&body) + ); + let json: serde_json::Value = serde_json::from_slice(&body).expect("Failed to parse note ingest response."); let note_id = json["results"] @@ -601,6 +629,143 @@ async fn insert_note_summary_field(state: &AppState, note_id: Uuid, summary: &st .expect("Failed to insert note summary field."); } +async fn seed_raw_search_index(state: &AppState, note_id: Uuid, note_text: &str, field_text: &str) { + let embedding_version = test_embedding_version(state); + let vector_dim = state.service.cfg.storage.qdrant.vector_dim; + let embedding_dim = i32::try_from(vector_dim).expect("Test vector_dim must fit i32."); + let vec_text = unit_vector_text(vector_dim as usize); + let chunk_id = Uuid::new_v4(); + let field_id = Uuid::new_v4(); + + sqlx::query( + "INSERT INTO memory_note_chunks ( + chunk_id, + note_id, + chunk_index, + start_offset, + end_offset, + text, + embedding_version + ) VALUES ($1, $2, $3, $4, $5, $6, $7)", + ) + .bind(chunk_id) + .bind(note_id) + .bind(0_i32) + .bind(0_i32) + .bind(i32::try_from(note_text.len()).expect("Test note text length must fit i32.")) + .bind(note_text) + .bind(embedding_version.as_str()) + .execute(&state.service.db.pool) + .await + .expect("Failed to insert raw search chunk."); + sqlx::query( + "INSERT INTO note_chunk_embeddings (chunk_id, embedding_version, embedding_dim, vec) + VALUES ($1, $2, $3, $4::text::vector)", + ) + .bind(chunk_id) + .bind(embedding_version.as_str()) + .bind(embedding_dim) + .bind(vec_text.as_str()) + .execute(&state.service.db.pool) + .await + .expect("Failed to insert raw search chunk embedding."); + sqlx::query( + "INSERT INTO note_embeddings (note_id, embedding_version, embedding_dim, vec) + VALUES ($1, $2, $3, $4::text::vector) + ON CONFLICT (note_id, embedding_version) DO UPDATE + SET embedding_dim = EXCLUDED.embedding_dim, + vec = EXCLUDED.vec, + created_at = now()", + ) + .bind(note_id) + .bind(embedding_version.as_str()) + .bind(embedding_dim) + .bind(vec_text.as_str()) + .execute(&state.service.db.pool) + .await + .expect("Failed to insert raw search note embedding."); + sqlx::query( + "INSERT INTO memory_note_fields (field_id, note_id, field_kind, item_index, text) + VALUES ($1, $2, $3, $4, $5)", + ) + .bind(field_id) + .bind(note_id) + .bind("summary") + .bind(0_i32) + .bind(field_text) + .execute(&state.service.db.pool) + .await + .expect("Failed to insert raw search field."); + sqlx::query( + "INSERT INTO note_field_embeddings (field_id, embedding_version, embedding_dim, vec) + VALUES ($1, $2, $3, $4::text::vector)", + ) + .bind(field_id) + .bind(embedding_version.as_str()) + .bind(embedding_dim) + .bind(vec_text.as_str()) + .execute(&state.service.db.pool) + .await + .expect("Failed to insert raw search field embedding."); +} + +async fn insert_search_session(state: &AppState, note_id: Uuid, summary: &str) -> Uuid { + let search_session_id = Uuid::new_v4(); + let trace_id = Uuid::new_v4(); + let chunk_id = Uuid::new_v4(); + let now = OffsetDateTime::now_utc(); + let expires_at = now + Duration::hours(1); + let updated_at = now.format(&Rfc3339).expect("Failed to format search session item time."); + let items = serde_json::json!([{ + "rank": 1, + "note_id": note_id, + "chunk_id": chunk_id, + "final_score": 1.0, + "updated_at": updated_at, + "expires_at": null, + "type": "fact", + "key": null, + "scope": "agent_private", + "importance": 0.8, + "confidence": 0.9, + "summary": summary, + }]); + + sqlx::query( + "INSERT INTO search_sessions ( + search_session_id, + trace_id, + tenant_id, + project_id, + agent_id, + read_profile, + query, + mode, + trajectory_summary, + query_plan, + items, + created_at, + expires_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NULL, NULL, $9, $10, $11)", + ) + .bind(search_session_id) + .bind(trace_id) + .bind(TEST_TENANT_ID) + .bind(TEST_PROJECT_ID) + .bind(TEST_AGENT_A) + .bind("private_only") + .bind("payload shaping") + .bind("quick_find") + .bind(items) + .bind(now) + .bind(expires_at) + .execute(&state.service.db.pool) + .await + .expect("Failed to insert search session."); + + search_session_id +} + async fn fetch_search_notes_for_payload_level( app: &Router, search_id: Uuid, @@ -627,12 +792,18 @@ async fn fetch_search_notes_for_payload_level( ) .await .expect("Failed to call search notes."); - - assert_eq!(response.status(), StatusCode::OK); - + let status = response.status(); let body = body::to_bytes(response.into_body(), usize::MAX) .await .expect("Failed to read search notes response body."); + + assert_eq!( + status, + StatusCode::OK, + "Unexpected search notes response: {}", + String::from_utf8_lossy(&body) + ); + let json: serde_json::Value = serde_json::from_slice(&body).expect("Failed to parse search notes response."); @@ -650,6 +821,7 @@ async fn fetch_admin_search_raw_source_ref( payload_level: &str, ) -> serde_json::Value { let payload = serde_json::json!({ + "mode": "quick_find", "query": query, "top_k": 5, "candidate_k": 10, @@ -1514,56 +1686,13 @@ async fn searches_notes_payload_level_shapes_source_ref_and_structured() { } }); let structured_summary = "Compact structured summary used for payload-level l1 and l2 shaping."; - let note_text = "A substantially long payload shaping note used in contract tests for search details output shaping. " - .repeat(6); - let note_id = - create_note_for_payload_level_tests(&app, note_text.as_str(), source_ref.clone()).await; + let note_text = + "A valid payload shaping note used in contract tests for search details output shaping."; + let note_id = create_note_for_payload_level_tests(&app, note_text, source_ref.clone()).await; insert_note_summary_field(&state, note_id, structured_summary).await; - let search_response = app - .clone() - .oneshot( - Request::builder() - .method("POST") - .uri("/v2/searches") - .header("X-ELF-Tenant-Id", TEST_TENANT_ID) - .header("X-ELF-Project-Id", TEST_PROJECT_ID) - .header("X-ELF-Agent-Id", TEST_AGENT_A) - .header("X-ELF-Read-Profile", "private_only") - .header("content-type", "application/json") - .body(Body::from( - serde_json::json!({ - "mode": "quick_find", - "query": "payload shaping", - "top_k": 5, - "candidate_k": 10, - }) - .to_string(), - )) - .expect("Failed to build searches request."), - ) - .await - .expect("Failed to call searches."); - - assert_eq!(search_response.status(), StatusCode::OK); - - let search_body = body::to_bytes(search_response.into_body(), usize::MAX) - .await - .expect("Failed to read searches response body."); - let search_json: serde_json::Value = - serde_json::from_slice(&search_body).expect("Failed to parse searches response."); - let trajectory = &search_json["trajectory_summary"]; - - if !trajectory.is_null() { - assert!(trajectory.is_object()); - assert!(trajectory.get("stages").is_some()); - } - - let search_id = Uuid::parse_str( - search_json["search_id"].as_str().expect("Missing search_id in searches response."), - ) - .expect("Invalid search_id value."); + let search_id = insert_search_session(&state, note_id, structured_summary).await; let notes_l0 = fetch_search_notes_for_payload_level(&app, search_id, note_id, "l0").await; let notes_l1 = fetch_search_notes_for_payload_level(&app, search_id, note_id, "l1").await; let notes_l2 = fetch_search_notes_for_payload_level(&app, search_id, note_id, "l2").await; @@ -1608,9 +1737,8 @@ async fn searches_notes_payload_level_shapes_source_ref_and_structured() { assert!(notes_l1["structured"].is_object()); assert!(notes_l2["structured"].is_object()); assert!(notes_l0_text.len() <= 240); - assert_ne!(notes_l0_text, note_text.as_str()); assert_eq!(notes_l1_text, structured_summary); - assert_eq!(notes_l2_text, note_text.as_str()); + assert_eq!(notes_l2_text, note_text); test_db.cleanup().await.expect("Failed to cleanup test database."); } @@ -1624,7 +1752,7 @@ async fn admin_searches_raw_payload_level_shapes_source_ref() { let config = test_config(test_db.dsn().to_string(), qdrant_url, collection); let state = AppState::new(config).await.expect("Failed to initialize app state."); let app = routes::router(state.clone()); - let admin_app = routes::admin_router(state); + let admin_app = routes::admin_router(state.clone()); let source_ref = serde_json::json!({ "schema": "note_source_ref/v1", "locator": { @@ -1638,7 +1766,10 @@ async fn admin_searches_raw_payload_level_shapes_source_ref() { }); let note_text = "Admin raw search payload shaping contract note. This long note should be indexed."; - let _note_id = create_note_for_payload_level_tests(&app, note_text, source_ref.clone()).await; + let note_id = create_note_for_payload_level_tests(&app, note_text, source_ref.clone()).await; + + seed_raw_search_index(&state, note_id, note_text, "payload shaping").await; + let raw_l0 = fetch_admin_search_raw_source_ref(&admin_app, "payload shaping", "l0").await; let raw_l1 = fetch_admin_search_raw_source_ref(&admin_app, "payload shaping", "l1").await; let raw_l2 = fetch_admin_search_raw_source_ref(&admin_app, "payload shaping", "l2").await;