diff --git a/.gitignore b/.gitignore index 8d2fdc30..e62d7c27 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ control-plane pgedge-control-plane !docker/control-plane e2e/debug + +# Git worktrees (local; never committed) +.worktrees/ diff --git a/api/apiv1/design/database.go b/api/apiv1/design/database.go index 52a237d7..1dd63b1d 100644 --- a/api/apiv1/design/database.go +++ b/api/apiv1/design/database.go @@ -169,10 +169,11 @@ var ServiceSpec = g.Type("ServiceSpec", func() { }) g.Attribute("service_type", g.String, func() { g.Description("The type of service to run.") - g.Enum("mcp", "postgrest", "rag") + g.Enum("mcp", "postgrest", "rag", "lakekeeper") g.Example("mcp") g.Example("postgrest") g.Example("rag") + g.Example("lakekeeper") g.Meta("struct:tag:json", "service_type") }) g.Attribute("version", g.String, func() { diff --git a/api/apiv1/gen/http/control_plane/client/types.go b/api/apiv1/gen/http/control_plane/client/types.go index 4cb91a20..1d69f756 100644 --- a/api/apiv1/gen/http/control_plane/client/types.go +++ b/api/apiv1/gen/http/control_plane/client/types.go @@ -6642,8 +6642,8 @@ func ValidateServiceSpecRequestBody(body *ServiceSpecRequestBody) (err error) { if utf8.RuneCountInString(body.ServiceID) > 36 { err = goa.MergeErrors(err, goa.InvalidLengthError("body.service_id", body.ServiceID, utf8.RuneCountInString(body.ServiceID), 36, false)) } - if !(body.ServiceType == "mcp" || body.ServiceType == "postgrest" || body.ServiceType == "rag") { - err = goa.MergeErrors(err, goa.InvalidEnumValueError("body.service_type", body.ServiceType, []any{"mcp", "postgrest", "rag"})) + if !(body.ServiceType == "mcp" || body.ServiceType == "postgrest" || body.ServiceType == "rag" || body.ServiceType == "lakekeeper") { + err = goa.MergeErrors(err, goa.InvalidEnumValueError("body.service_type", body.ServiceType, []any{"mcp", "postgrest", "rag", "lakekeeper"})) } err = goa.MergeErrors(err, goa.ValidatePattern("body.version", body.Version, "^(\\d+\\.\\d+(\\.\\d+)?|latest)$")) if len(body.HostIds) < 1 { @@ -7448,8 +7448,8 @@ func ValidateServiceSpecRequestBodyRequestBody(body *ServiceSpecRequestBodyReque if utf8.RuneCountInString(body.ServiceID) > 36 { err = goa.MergeErrors(err, goa.InvalidLengthError("body.service_id", body.ServiceID, utf8.RuneCountInString(body.ServiceID), 36, false)) } - if !(body.ServiceType == "mcp" || body.ServiceType == "postgrest" || body.ServiceType == "rag") { - err = goa.MergeErrors(err, goa.InvalidEnumValueError("body.service_type", body.ServiceType, []any{"mcp", "postgrest", "rag"})) + if !(body.ServiceType == "mcp" || body.ServiceType == "postgrest" || body.ServiceType == "rag" || body.ServiceType == "lakekeeper") { + err = goa.MergeErrors(err, goa.InvalidEnumValueError("body.service_type", body.ServiceType, []any{"mcp", "postgrest", "rag", "lakekeeper"})) } err = goa.MergeErrors(err, goa.ValidatePattern("body.version", body.Version, "^(\\d+\\.\\d+(\\.\\d+)?|latest)$")) if len(body.HostIds) < 1 { diff --git a/api/apiv1/gen/http/control_plane/server/types.go b/api/apiv1/gen/http/control_plane/server/types.go index a2ae600e..50017d97 100644 --- a/api/apiv1/gen/http/control_plane/server/types.go +++ b/api/apiv1/gen/http/control_plane/server/types.go @@ -6144,8 +6144,8 @@ func ValidateServiceSpecRequestBody(body *ServiceSpecRequestBody) (err error) { } } if body.ServiceType != nil { - if !(*body.ServiceType == "mcp" || *body.ServiceType == "postgrest" || *body.ServiceType == "rag") { - err = goa.MergeErrors(err, goa.InvalidEnumValueError("body.service_type", *body.ServiceType, []any{"mcp", "postgrest", "rag"})) + if !(*body.ServiceType == "mcp" || *body.ServiceType == "postgrest" || *body.ServiceType == "rag" || *body.ServiceType == "lakekeeper") { + err = goa.MergeErrors(err, goa.InvalidEnumValueError("body.service_type", *body.ServiceType, []any{"mcp", "postgrest", "rag", "lakekeeper"})) } } if body.Version != nil { @@ -6925,8 +6925,8 @@ func ValidateServiceSpecRequestBodyRequestBody(body *ServiceSpecRequestBodyReque } } if body.ServiceType != nil { - if !(*body.ServiceType == "mcp" || *body.ServiceType == "postgrest" || *body.ServiceType == "rag") { - err = goa.MergeErrors(err, goa.InvalidEnumValueError("body.service_type", *body.ServiceType, []any{"mcp", "postgrest", "rag"})) + if !(*body.ServiceType == "mcp" || *body.ServiceType == "postgrest" || *body.ServiceType == "rag" || *body.ServiceType == "lakekeeper") { + err = goa.MergeErrors(err, goa.InvalidEnumValueError("body.service_type", *body.ServiceType, []any{"mcp", "postgrest", "rag", "lakekeeper"})) } } if body.Version != nil { diff --git a/api/apiv1/gen/http/openapi.json b/api/apiv1/gen/http/openapi.json index 2caf49c1..15c86963 100644 --- a/api/apiv1/gen/http/openapi.json +++ b/api/apiv1/gen/http/openapi.json @@ -5273,7 +5273,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -5353,7 +5353,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ] @@ -6017,7 +6017,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -6097,7 +6097,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -6177,7 +6177,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ], @@ -9314,11 +9314,12 @@ "service_type": { "type": "string", "description": "The type of service to run.", - "example": "rag", + "example": "lakekeeper", "enum": [ "mcp", "postgrest", - "rag" + "rag", + "lakekeeper" ] }, "version": { @@ -9406,7 +9407,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, "required": [ diff --git a/api/apiv1/gen/http/openapi.yaml b/api/apiv1/gen/http/openapi.yaml index aeedde4a..cef9c1d1 100644 --- a/api/apiv1/gen/http/openapi.yaml +++ b/api/apiv1/gen/http/openapi.yaml @@ -3770,7 +3770,7 @@ definitions: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -3821,7 +3821,7 @@ definitions: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: type: string @@ -4294,7 +4294,7 @@ definitions: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -4345,7 +4345,7 @@ definitions: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -4396,7 +4396,7 @@ definitions: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: "5" required: @@ -6684,11 +6684,12 @@ definitions: service_type: type: string description: The type of service to run. - example: rag + example: lakekeeper enum: - mcp - postgrest - rag + - lakekeeper version: type: string description: The version of the service (e.g., '1.0.0', '14.5') or the literal 'latest'. @@ -6745,7 +6746,7 @@ definitions: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest required: - service_id diff --git a/api/apiv1/gen/http/openapi3.json b/api/apiv1/gen/http/openapi3.json index 7ec29cbd..98b9441b 100644 --- a/api/apiv1/gen/http/openapi3.json +++ b/api/apiv1/gen/http/openapi3.json @@ -12856,7 +12856,7 @@ }, "port": 0, "service_id": "analytics-service", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -12935,7 +12935,7 @@ }, "port": 0, "service_id": "analytics-service", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -13014,7 +13014,7 @@ }, "port": 0, "service_id": "analytics-service", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ] @@ -13675,7 +13675,7 @@ }, "port": 0, "service_id": "analytics-service", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -13754,7 +13754,7 @@ }, "port": 0, "service_id": "analytics-service", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -13833,7 +13833,7 @@ }, "port": 0, "service_id": "analytics-service", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -13912,7 +13912,7 @@ }, "port": 0, "service_id": "analytics-service", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ], @@ -14352,7 +14352,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -14431,7 +14431,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -14510,7 +14510,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ] @@ -15440,7 +15440,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -15519,7 +15519,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ], @@ -16112,7 +16112,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -16192,7 +16192,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -16272,7 +16272,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -16352,7 +16352,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ] @@ -17016,7 +17016,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -17096,7 +17096,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -17176,7 +17176,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -17256,7 +17256,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ], @@ -18024,7 +18024,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -18104,7 +18104,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -18184,7 +18184,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ] @@ -19023,7 +19023,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -19103,7 +19103,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -19183,7 +19183,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -19263,7 +19263,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ], @@ -19900,7 +19900,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -19980,7 +19980,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -20060,7 +20060,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ] @@ -20791,7 +20791,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -20871,7 +20871,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -20951,7 +20951,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -21031,7 +21031,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ], @@ -21624,7 +21624,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -21704,7 +21704,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -21784,7 +21784,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -21864,7 +21864,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ] @@ -22703,7 +22703,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -22783,7 +22783,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ], @@ -23551,7 +23551,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -23631,7 +23631,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -23711,7 +23711,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ] @@ -24550,7 +24550,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -24630,7 +24630,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -24710,7 +24710,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -24790,7 +24790,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ], @@ -25208,7 +25208,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -25288,7 +25288,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -25368,7 +25368,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -25448,7 +25448,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ] @@ -26112,7 +26112,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -26192,7 +26192,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -26272,7 +26272,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ], @@ -30003,11 +30003,12 @@ "service_type": { "type": "string", "description": "The type of service to run.", - "example": "rag", + "example": "lakekeeper", "enum": [ "mcp", "postgrest", - "rag" + "rag", + "lakekeeper" ] }, "version": { @@ -30094,7 +30095,7 @@ }, "port": 0, "service_id": "analytics-service", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, "required": [ @@ -30173,11 +30174,12 @@ "service_type": { "type": "string", "description": "The type of service to run.", - "example": "rag", + "example": "lakekeeper", "enum": [ "mcp", "postgrest", - "rag" + "rag", + "lakekeeper" ] }, "version": { @@ -30265,7 +30267,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, "required": [ @@ -30344,11 +30346,12 @@ "service_type": { "type": "string", "description": "The type of service to run.", - "example": "rag", + "example": "lakekeeper", "enum": [ "mcp", "postgrest", - "rag" + "rag", + "lakekeeper" ] }, "version": { @@ -30435,7 +30438,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, "required": [ @@ -30516,11 +30519,12 @@ "service_type": { "type": "string", "description": "The type of service to run.", - "example": "rag", + "example": "lakekeeper", "enum": [ "mcp", "postgrest", - "rag" + "rag", + "lakekeeper" ] }, "version": { @@ -30606,7 +30610,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, "required": [ @@ -30686,11 +30690,12 @@ "service_type": { "type": "string", "description": "The type of service to run.", - "example": "rag", + "example": "lakekeeper", "enum": [ "mcp", "postgrest", - "rag" + "rag", + "lakekeeper" ] }, "version": { @@ -30777,7 +30782,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, "required": [ @@ -30857,11 +30862,12 @@ "service_type": { "type": "string", "description": "The type of service to run.", - "example": "rag", + "example": "lakekeeper", "enum": [ "mcp", "postgrest", - "rag" + "rag", + "lakekeeper" ] }, "version": { @@ -30949,7 +30955,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, "required": [ @@ -31029,11 +31035,12 @@ "service_type": { "type": "string", "description": "The type of service to run.", - "example": "rag", + "example": "lakekeeper", "enum": [ "mcp", "postgrest", - "rag" + "rag", + "lakekeeper" ] }, "version": { @@ -31121,7 +31128,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, "required": [ @@ -31201,11 +31208,12 @@ "service_type": { "type": "string", "description": "The type of service to run.", - "example": "rag", + "example": "lakekeeper", "enum": [ "mcp", "postgrest", - "rag" + "rag", + "lakekeeper" ] }, "version": { @@ -31293,7 +31301,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, "required": [ diff --git a/api/apiv1/gen/http/openapi3.yaml b/api/apiv1/gen/http/openapi3.yaml index c9e56044..65da54cb 100644 --- a/api/apiv1/gen/http/openapi3.yaml +++ b/api/apiv1/gen/http/openapi3.yaml @@ -9016,7 +9016,7 @@ components: image: Animi recusandae. port: 0 service_id: analytics-service - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -9066,7 +9066,7 @@ components: image: Animi recusandae. port: 0 service_id: analytics-service - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -9116,7 +9116,7 @@ components: image: Animi recusandae. port: 0 service_id: analytics-service - service_type: rag + service_type: lakekeeper version: latest spock_version: type: string @@ -9586,7 +9586,7 @@ components: image: Animi recusandae. port: 0 service_id: analytics-service - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -9636,7 +9636,7 @@ components: image: Animi recusandae. port: 0 service_id: analytics-service - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -9686,7 +9686,7 @@ components: image: Animi recusandae. port: 0 service_id: analytics-service - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -9736,7 +9736,7 @@ components: image: Animi recusandae. port: 0 service_id: analytics-service - service_type: rag + service_type: lakekeeper version: latest spock_version: "5" required: @@ -10055,7 +10055,7 @@ components: image: Reprehenderit atque aliquid. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -10105,7 +10105,7 @@ components: image: Reprehenderit atque aliquid. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -10155,7 +10155,7 @@ components: image: Reprehenderit atque aliquid. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: type: string @@ -10834,7 +10834,7 @@ components: image: Reprehenderit atque aliquid. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -10884,7 +10884,7 @@ components: image: Reprehenderit atque aliquid. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: "5" required: @@ -11311,7 +11311,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -11362,7 +11362,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -11413,7 +11413,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -11464,7 +11464,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: type: string @@ -11937,7 +11937,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -11988,7 +11988,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -12039,7 +12039,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -12090,7 +12090,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: "5" required: @@ -12644,7 +12644,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -12695,7 +12695,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -12746,7 +12746,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: type: string @@ -13346,7 +13346,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -13397,7 +13397,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -13448,7 +13448,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -13499,7 +13499,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: "5" required: @@ -13964,7 +13964,7 @@ components: image: Et odio. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -14015,7 +14015,7 @@ components: image: Et odio. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -14066,7 +14066,7 @@ components: image: Et odio. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: type: string @@ -14597,7 +14597,7 @@ components: image: Et odio. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -14648,7 +14648,7 @@ components: image: Et odio. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -14699,7 +14699,7 @@ components: image: Et odio. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -14750,7 +14750,7 @@ components: image: Et odio. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: "5" required: @@ -15177,7 +15177,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -15228,7 +15228,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -15279,7 +15279,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -15330,7 +15330,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: type: string @@ -15930,7 +15930,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -15981,7 +15981,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: "5" required: @@ -16535,7 +16535,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -16586,7 +16586,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -16637,7 +16637,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: type: string @@ -17237,7 +17237,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -17288,7 +17288,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -17339,7 +17339,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -17390,7 +17390,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: "5" required: @@ -17690,7 +17690,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -17741,7 +17741,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -17792,7 +17792,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -17843,7 +17843,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: type: string @@ -18316,7 +18316,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -18367,7 +18367,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -18418,7 +18418,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: "5" required: @@ -21114,11 +21114,12 @@ components: service_type: type: string description: The type of service to run. - example: rag + example: lakekeeper enum: - mcp - postgrest - rag + - lakekeeper version: type: string description: The version of the service (e.g., '1.0.0', '14.5') or the literal 'latest'. @@ -21174,7 +21175,7 @@ components: image: Animi recusandae. port: 0 service_id: analytics-service - service_type: rag + service_type: lakekeeper version: latest required: - service_id @@ -21238,11 +21239,12 @@ components: service_type: type: string description: The type of service to run. - example: rag + example: lakekeeper enum: - mcp - postgrest - rag + - lakekeeper version: type: string description: The version of the service (e.g., '1.0.0', '14.5') or the literal 'latest'. @@ -21299,7 +21301,7 @@ components: image: Reprehenderit atque aliquid. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest required: - service_id @@ -21363,11 +21365,12 @@ components: service_type: type: string description: The type of service to run. - example: rag + example: lakekeeper enum: - mcp - postgrest - rag + - lakekeeper version: type: string description: The version of the service (e.g., '1.0.0', '14.5') or the literal 'latest'. @@ -21423,7 +21426,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest required: - service_id @@ -21489,11 +21492,12 @@ components: service_type: type: string description: The type of service to run. - example: rag + example: lakekeeper enum: - mcp - postgrest - rag + - lakekeeper version: type: string description: The version of the service (e.g., '1.0.0', '14.5') or the literal 'latest'. @@ -21548,7 +21552,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest required: - service_id @@ -21613,11 +21617,12 @@ components: service_type: type: string description: The type of service to run. - example: rag + example: lakekeeper enum: - mcp - postgrest - rag + - lakekeeper version: type: string description: The version of the service (e.g., '1.0.0', '14.5') or the literal 'latest'. @@ -21673,7 +21678,7 @@ components: image: Et odio. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest required: - service_id @@ -21738,11 +21743,12 @@ components: service_type: type: string description: The type of service to run. - example: rag + example: lakekeeper enum: - mcp - postgrest - rag + - lakekeeper version: type: string description: The version of the service (e.g., '1.0.0', '14.5') or the literal 'latest'. @@ -21799,7 +21805,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest required: - service_id @@ -21864,11 +21870,12 @@ components: service_type: type: string description: The type of service to run. - example: rag + example: lakekeeper enum: - mcp - postgrest - rag + - lakekeeper version: type: string description: The version of the service (e.g., '1.0.0', '14.5') or the literal 'latest'. @@ -21925,7 +21932,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest required: - service_id @@ -21990,11 +21997,12 @@ components: service_type: type: string description: The type of service to run. - example: rag + example: lakekeeper enum: - mcp - postgrest - rag + - lakekeeper version: type: string description: The version of the service (e.g., '1.0.0', '14.5') or the literal 'latest'. @@ -22051,7 +22059,7 @@ components: image: Incidunt in quas totam. port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest required: - service_id diff --git a/server/internal/api/apiv1/validate.go b/server/internal/api/apiv1/validate.go index 78a040ba..fdfc6317 100644 --- a/server/internal/api/apiv1/validate.go +++ b/server/internal/api/apiv1/validate.go @@ -191,6 +191,9 @@ func validateDatabaseSpec(orchestrator config.Orchestrator, databaseID string, s } seenServiceIDs.Add(string(svc.ServiceID)) + // ColdFront (a lakekeeper service) is single-node only for now. + errs = append(errs, validateColdFrontSingleNode(svc, len(spec.Nodes), svcPath)...) + errs = append(errs, validateServiceSpec(svc, svcPath, false, databaseID, spec.DatabaseUsers, seenNodeNames)...) } } @@ -254,6 +257,9 @@ func validateDatabaseUpdate(old *database.Spec, new *api.DatabaseSpec) error { svcPath := validation.NewPath("services", validation.ArrayIndexElement(i)) isExistingService := existingServiceIDs.Has(string(svc.ServiceID)) + // ColdFront (a lakekeeper service) is single-node only for now. + errs = append(errs, validateColdFrontSingleNode(svc, len(new.Nodes), svcPath)...) + errs = append(errs, validateServiceSpec(svc, svcPath, isExistingService, old.DatabaseID, new.DatabaseUsers, newNodeNames)...) } @@ -357,7 +363,7 @@ func validateServiceSpec(svc *api.ServiceSpec, path validation.Path, isUpdate bo } // Validate service_type allowlist - supportedServiceTypes := validation.NewPath("mcp", "postgrest", "rag") + supportedServiceTypes := validation.NewPath("mcp", "postgrest", "rag", "lakekeeper") if !slices.Contains(supportedServiceTypes, svc.ServiceType) { err := fmt.Errorf("unsupported service type %q (supported: %s)", svc.ServiceType, strings.Join(supportedServiceTypes, ", ")) @@ -394,6 +400,8 @@ func validateServiceSpec(svc *api.ServiceSpec, path validation.Path, isUpdate bo errs = append(errs, validatePostgRESTServiceConfig(svc.Config, path.Append("config"))...) case "rag": errs = append(errs, validateRAGServiceConfig(svc.Config, path.Append("config"), isUpdate)...) + case "lakekeeper": + errs = append(errs, validateLakekeeperServiceConfig(svc.Config, path.Append("config"))...) } // Validate database_connection if provided @@ -518,6 +526,104 @@ func validateRAGServiceConfig(config map[string]any, path validation.Path, isUpd return result } +// coldFrontMultiNodeError is the message returned when a lakekeeper (ColdFront) +// service is requested on a database that spans more than one node. Kept as a +// package-level value so the orchestrator-side guard can return an identical +// message. +const coldFrontMultiNodeError = "coldfront: multi-node ColdFront is not yet supported " + + "(mesh snowflake.node alignment pending); enable ColdFront only on a single-node database" + +// validateColdFrontSingleNode rejects a lakekeeper service on a multi-node +// database. ColdFront's tiering bakery requires +// snowflake.node = hashtext(spock_node_name)&1023, but control-plane currently +// assigns node ordinals rather than reconciling that mesh GUC, so a multi-node +// deployment would silently fail its tiering cron jobs. Fail loudly at +// validation time until the mesh snowflake.node reconciliation lands. +func validateColdFrontSingleNode(svc *api.ServiceSpec, nodeCount int, path validation.Path) []error { + if svc.ServiceType != "lakekeeper" { + return nil + } + if nodeCount <= 1 { + return nil + } + return []error{validation.NewError(errors.New(coldFrontMultiNodeError), path.Append("service_type"))} +} + +// validateLakekeeperServiceConfig checks that the required catalog +// configuration keys are present and non-empty. catalog_db_url is required +// unless catalog_db_create is set, in which case the control plane +// provisions and connects to a managed catalog database itself. An absent +// or empty catalog_db_url in external mode would cause Lakekeeper to start +// with a blank connection string and crash-loop silently; pg_encryption_key +// is always required regardless of mode. We surface these errors at +// spec-validation time instead. +func validateLakekeeperServiceConfig(config map[string]any, path validation.Path) []error { + var errs []error + + catalogDBCreate, _ := config["catalog_db_create"].(bool) + catalogDBURL, _ := config["catalog_db_url"].(string) + if catalogDBURL == "" && !catalogDBCreate { + errs = append(errs, validation.NewError( + errors.New("catalog_db_url is required unless catalog_db_create is set: "+ + "provide the connection URL for an external catalog Postgres, or set "+ + "catalog_db_create for a control-plane-managed catalog"), + path.Append("catalog_db_url"), + )) + } + + pgEncryptionKey, _ := config["pg_encryption_key"].(string) + if pgEncryptionKey == "" { + errs = append(errs, validation.NewError( + errors.New("pg_encryption_key is required: provide the encryption key for Lakekeeper's catalog Postgres"), + path.Append("pg_encryption_key"), + )) + } + + // The warehouse bootstrap and coldfront.set_storage_secret both need the + // object-store coordinates and a provider-specific credential. Fail loud + // here so a database is never left with an unbootstrapped (broken) + // warehouse. Some of these keys are supplied by a consumer-side follow-up. + provider, _ := config["provider"].(string) + switch provider { + case "aws", "azure", "gcs": + case "": + errs = append(errs, validation.NewError( + errors.New("provider is required: one of aws, azure, gcs"), + path.Append("provider"), + )) + default: + errs = append(errs, validation.NewError( + fmt.Errorf("unsupported provider %q: expected one of aws, azure, gcs", provider), + path.Append("provider"), + )) + } + + if warehouse, _ := config["warehouse"].(string); warehouse == "" { + errs = append(errs, validation.NewError( + errors.New("warehouse is required: the warehouse name to create in Lakekeeper"), + path.Append("warehouse"), + )) + } + + if credential, _ := config["credential"].(string); credential == "" { + errs = append(errs, validation.NewError( + errors.New("credential is required: a provider-specific credential JSON string"), + path.Append("credential"), + )) + } + + if provider == "aws" || provider == "gcs" { + if bucket, _ := config["bucket"].(string); bucket == "" { + errs = append(errs, validation.NewError( + errors.New("bucket is required for aws and gcs providers"), + path.Append("bucket"), + )) + } + } + + return errs +} + func validateCPUs(value *string, path validation.Path) []error { var errs []error diff --git a/server/internal/api/apiv1/validate_test.go b/server/internal/api/apiv1/validate_test.go index 5f86959b..324d3168 100644 --- a/server/internal/api/apiv1/validate_test.go +++ b/server/internal/api/apiv1/validate_test.go @@ -1424,6 +1424,187 @@ func TestValidateDatabaseSpec(t *testing.T) { `"spock" must be included in shared_preload_libraries`, }, }, + { + name: "valid single-node ColdFront (lakekeeper)", + spec: &api.DatabaseSpec{ + DatabaseName: "testdb", + PostgresVersion: utils.PointerTo("17.6"), + Nodes: []*api.DatabaseNodeSpec{ + { + Name: "n1", + HostIds: []api.Identifier{api.Identifier("host-1")}, + }, + }, + DatabaseUsers: []*api.DatabaseUserSpec{ + {Username: "app", DbOwner: utils.PointerTo(true)}, + }, + Services: []*api.ServiceSpec{ + { + ServiceID: "coldfront", + ServiceType: "lakekeeper", + Version: "0.9.0", + HostIds: []api.Identifier{"host-1"}, + ConnectAs: "app", + Config: map[string]any{ + "catalog_db_url": "postgres://lk:secret@catalog:5432/lakekeeper?sslmode=disable", + "pg_encryption_key": "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + "provider": "aws", + "warehouse": "analytics", + "credential": `{"aws_access_key_id":"AKIA...","aws_secret_access_key":"..."}`, + "bucket": "coldfront-warehouse", + }, + }, + }, + }, + }, + { + name: "valid single-node ColdFront (lakekeeper) managed catalog", + spec: &api.DatabaseSpec{ + DatabaseName: "testdb", + PostgresVersion: utils.PointerTo("17.6"), + Nodes: []*api.DatabaseNodeSpec{ + { + Name: "n1", + HostIds: []api.Identifier{api.Identifier("host-1")}, + }, + }, + DatabaseUsers: []*api.DatabaseUserSpec{ + {Username: "app", DbOwner: utils.PointerTo(true)}, + }, + Services: []*api.ServiceSpec{ + { + ServiceID: "coldfront", + ServiceType: "lakekeeper", + Version: "0.9.0", + HostIds: []api.Identifier{"host-1"}, + ConnectAs: "app", + Config: map[string]any{ + "catalog_db_create": true, + "pg_encryption_key": "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + "provider": "aws", + "warehouse": "analytics", + "credential": `{"aws_access_key_id":"AKIA...","aws_secret_access_key":"..."}`, + "bucket": "coldfront-warehouse", + }, + }, + }, + }, + }, + { + name: "single-node ColdFront (lakekeeper) missing catalog_db_url is rejected", + spec: &api.DatabaseSpec{ + DatabaseName: "testdb", + PostgresVersion: utils.PointerTo("17.6"), + Nodes: []*api.DatabaseNodeSpec{ + { + Name: "n1", + HostIds: []api.Identifier{api.Identifier("host-1")}, + }, + }, + DatabaseUsers: []*api.DatabaseUserSpec{ + {Username: "app", DbOwner: utils.PointerTo(true)}, + }, + Services: []*api.ServiceSpec{ + { + ServiceID: "coldfront", + ServiceType: "lakekeeper", + Version: "0.9.0", + HostIds: []api.Identifier{"host-1"}, + ConnectAs: "app", + Config: map[string]any{ + "pg_encryption_key": "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + "provider": "aws", + "warehouse": "analytics", + "credential": `{"aws_access_key_id":"AKIA...","aws_secret_access_key":"..."}`, + "bucket": "coldfront-warehouse", + }, + }, + }, + }, + expected: []string{ + "catalog_db_url is required unless catalog_db_create is set", + }, + }, + { + name: "single-node ColdFront (lakekeeper) missing pg_encryption_key is rejected", + spec: &api.DatabaseSpec{ + DatabaseName: "testdb", + PostgresVersion: utils.PointerTo("17.6"), + Nodes: []*api.DatabaseNodeSpec{ + { + Name: "n1", + HostIds: []api.Identifier{api.Identifier("host-1")}, + }, + }, + DatabaseUsers: []*api.DatabaseUserSpec{ + {Username: "app", DbOwner: utils.PointerTo(true)}, + }, + Services: []*api.ServiceSpec{ + { + ServiceID: "coldfront", + ServiceType: "lakekeeper", + Version: "0.9.0", + HostIds: []api.Identifier{"host-1"}, + ConnectAs: "app", + Config: map[string]any{ + // pg_encryption_key is required in both external and + // managed catalog modes; omit it here (managed mode, so + // catalog_db_url is legitimately absent) to prove that + // requirement is independent of the catalog_db_url gate. + "catalog_db_create": true, + "provider": "aws", + "warehouse": "analytics", + "credential": `{"aws_access_key_id":"AKIA...","aws_secret_access_key":"..."}`, + "bucket": "coldfront-warehouse", + }, + }, + }, + }, + expected: []string{ + "pg_encryption_key is required", + }, + }, + { + name: "invalid multi-node ColdFront (lakekeeper) is rejected", + spec: &api.DatabaseSpec{ + DatabaseName: "testdb", + PostgresVersion: utils.PointerTo("17.6"), + Nodes: []*api.DatabaseNodeSpec{ + { + Name: "n1", + HostIds: []api.Identifier{api.Identifier("host-1")}, + }, + { + Name: "n2", + HostIds: []api.Identifier{api.Identifier("host-2")}, + }, + }, + DatabaseUsers: []*api.DatabaseUserSpec{ + {Username: "app", DbOwner: utils.PointerTo(true)}, + }, + Services: []*api.ServiceSpec{ + { + ServiceID: "coldfront", + ServiceType: "lakekeeper", + Version: "0.9.0", + HostIds: []api.Identifier{"host-1"}, + ConnectAs: "app", + Config: map[string]any{ + "catalog_db_url": "postgres://lk:secret@catalog:5432/lakekeeper?sslmode=disable", + "pg_encryption_key": "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + "provider": "aws", + "warehouse": "analytics", + "credential": `{"aws_access_key_id":"AKIA...","aws_secret_access_key":"..."}`, + "bucket": "coldfront-warehouse", + }, + }, + }, + }, + expected: []string{ + "multi-node ColdFront is not yet supported", + "mesh snowflake.node alignment pending", + }, + }, } { t.Run(tc.name, func(t *testing.T) { err := validateDatabaseSpec(config.OrchestratorSwarm, "test-db", tc.spec) diff --git a/server/internal/database/spec.go b/server/internal/database/spec.go index 99b5b930..e45ea835 100644 --- a/server/internal/database/spec.go +++ b/server/internal/database/spec.go @@ -586,6 +586,11 @@ type InstanceSpec struct { OrchestratorOpts *OrchestratorOpts `json:"orchestrator_opts,omitempty"` InPlaceRestore bool `json:"in_place_restore,omitempty"` AllHostIDs []string `json:"all_host_ids"` // All host IDs in the database + // ColdFrontEnabled is set when the database has a lakekeeper (ColdFront) + // service. It makes the instance's postgresql.conf load the coldfront / + // pg_duckdb stack (shared_preload_libraries + the duckdb.* GUCs), which the + // extension requires at server start (findings #6a/#6b). + ColdFrontEnabled bool `json:"coldfront_enabled,omitempty"` } func (s *InstanceSpec) CopySettingsFrom(current *InstanceSpec) { @@ -641,6 +646,7 @@ func (s *InstanceSpec) Clone() *InstanceSpec { NodeSize: s.NodeSize, OrchestratorOpts: s.OrchestratorOpts.Clone(), AllHostIDs: slices.Clone(s.AllHostIDs), + ColdFrontEnabled: s.ColdFrontEnabled, } } @@ -690,6 +696,13 @@ func (s *Spec) NodeInstances() ([]*NodeInstances, error) { return nil, fmt.Errorf("only one user can have db_owner=true, got %d", len(owners)) } + // A lakekeeper (ColdFront) service makes every instance load the coldfront / + // pg_duckdb stack at boot. Computed once here where the service list is in + // scope; carried on each InstanceSpec down to the Patroni config generator. + hasColdFront := slices.ContainsFunc(s.Services, func(svc *ServiceSpec) bool { + return svc.ServiceType == "lakekeeper" + }) + clusterSize := len(s.Nodes) nodes := make([]*NodeInstances, clusterSize) for nodeIdx, node := range s.Nodes { @@ -743,6 +756,7 @@ func (s *Spec) NodeInstances() ([]*NodeInstances, error) { NodeSize: nodeSize, OrchestratorOpts: overridableValue(s.OrchestratorOpts, node.OrchestratorOpts), AllHostIDs: allHostIDs, + ColdFrontEnabled: hasColdFront, } } diff --git a/server/internal/database/spec_test.go b/server/internal/database/spec_test.go index a8451972..5d586bf6 100644 --- a/server/internal/database/spec_test.go +++ b/server/internal/database/spec_test.go @@ -654,3 +654,41 @@ func TestSpec_NodeInstances_PgHbaIdentMerge(t *testing.T) { assert.Nil(t, nodes[0].Instances[0].PgIdentConf) }) } + +func TestSpec_NodeInstances_ColdFrontEnabled(t *testing.T) { + baseSpec := func(services []*database.ServiceSpec) *database.Spec { + return &database.Spec{ + DatabaseID: "test-db", + DatabaseName: "testdb", + PostgresVersion: "17.6", + SpockVersion: "5", + Nodes: []*database.Node{ + {Name: "n1", HostIDs: []string{"host-1", "host-2"}}, + }, + Services: services, + } + } + + t.Run("lakekeeper service sets the flag on every instance", func(t *testing.T) { + s := baseSpec([]*database.ServiceSpec{{ServiceID: "lk", ServiceType: "lakekeeper"}}) + nodes, err := s.NodeInstances() + assert.NoError(t, err) + for _, inst := range nodes[0].Instances { + assert.True(t, inst.ColdFrontEnabled, "instance %s should have ColdFrontEnabled", inst.InstanceID) + } + }) + + t.Run("no lakekeeper service leaves the flag unset", func(t *testing.T) { + s := baseSpec([]*database.ServiceSpec{{ServiceID: "m", ServiceType: "mcp"}}) + nodes, err := s.NodeInstances() + assert.NoError(t, err) + assert.False(t, nodes[0].Instances[0].ColdFrontEnabled) + }) + + t.Run("no services at all leaves the flag unset", func(t *testing.T) { + s := baseSpec(nil) + nodes, err := s.NodeInstances() + assert.NoError(t, err) + assert.False(t, nodes[0].Instances[0].ColdFrontEnabled) + }) +} diff --git a/server/internal/docker/docker.go b/server/internal/docker/docker.go index f4af33f8..dbaa7e55 100644 --- a/server/internal/docker/docker.go +++ b/server/internal/docker/docker.go @@ -589,23 +589,65 @@ func (d *Docker) Shutdown() error { return nil } -// CheckImageExists verifies that img is accessible in its registry by fetching -// its distribution manifest. This is a lightweight registry API call — no image -// layers are downloaded. Returns nil if the image exists, a non-nil error if it -// does not exist or cannot be reached. -func (d *Docker) CheckImageExists(ctx context.Context, img string) error { - encodedAuth := resolveRegistryAuth(img) +// imageInspector is the minimal image-lookup surface CheckImageExists needs: +// a registry distribution-manifest fetch and a local-daemon inspect. It is +// satisfied by *client.Client and lets the registry-then-local fallback logic +// be unit-tested without a live daemon or registry. +type imageInspector interface { + DistributionInspect(ctx context.Context, image, encodedRegistryAuth string) (registry.DistributionInspect, error) + ImageInspectWithRaw(ctx context.Context, imageID string) (types.ImageInspect, []byte, error) +} + +// Timeout budgets for the two image-existence lookups. They are separate so a +// slow-failing registry lookup can never consume the whole budget and leave the +// local-daemon fallback with a dead context (which would report a locally-built +// image as missing — the exact case the fallback exists for). +const ( + registryInspectTimeout = 30 * time.Second + localInspectTimeout = 10 * time.Second +) - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() +// CheckImageExists verifies that img is a usable deployment target. It first +// tries a lightweight registry distribution-manifest fetch (no layers are +// downloaded); if that fails — as it does for a private registry without +// resolvable auth, or an image built locally and never pushed — it falls back +// to inspecting the local Docker daemon's image cache. Returns nil if the image +// is found by either path, a non-nil error only if both fail. +func (d *Docker) CheckImageExists(ctx context.Context, img string) error { + return checkImageExists(ctx, d.client, resolveRegistryAuth(img), img, + registryInspectTimeout, localInspectTimeout) +} + +// checkImageExists implements the registry-then-local-daemon lookup against the +// given inspector. Split out from CheckImageExists so the fallback logic is +// unit-testable via a fake inspector; the two timeouts are parameters so the +// starvation-resistance behaviour can be exercised deterministically. +func checkImageExists(ctx context.Context, inspector imageInspector, encodedAuth, img string, registryTimeout, localTimeout time.Duration) error { + regCtx, cancelReg := context.WithTimeout(ctx, registryTimeout) + defer cancelReg() + _, regErr := inspector.DistributionInspect(regCtx, img, encodedAuth) + if regErr == nil { + return nil + } - _, err := d.client.DistributionInspect(ctx, img, encodedAuth) - if err != nil { - return fmt.Errorf("image %q not found or inaccessible: %w", img, err) + // Registry lookup failed — a private registry without resolvable auth, an + // unreachable registry, or an image built locally and never pushed. Fall + // back to the local Docker daemon's image cache with its own timeout budget, + // independent of however long the registry lookup took. + localCtx, cancelLocal := context.WithTimeout(ctx, localTimeout) + defer cancelLocal() + if _, _, localErr := inspector.ImageInspectWithRaw(localCtx, img); localErr == nil { + return nil } - return nil + + // Surface the registry error: it is the primary lookup path and its message + // is the more actionable of the two for a pushed image. + return fmt.Errorf("image %q not found or inaccessible: %w", img, regErr) } +// ensure *client.Client satisfies imageInspector at compile time. +var _ imageInspector = (*client.Client)(nil) + // resolveRegistryAuth returns a base64-encoded JSON registry.AuthConfig for the // registry that hosts img, loading credentials from the Docker config file. // Returns an empty string (anonymous auth) if credentials cannot be resolved. diff --git a/server/internal/docker/docker_test.go b/server/internal/docker/docker_test.go new file mode 100644 index 00000000..bdacecf6 --- /dev/null +++ b/server/internal/docker/docker_test.go @@ -0,0 +1,88 @@ +package docker + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/registry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeImageInspector is a configurable stand-in for the docker client's +// registry + local image inspection surface, letting checkImageExists be +// unit-tested without a live daemon or registry. +type fakeImageInspector struct { + distErr error // returned by DistributionInspect (registry lookup) + localErr error // returned by ImageInspectWithRaw (local daemon lookup) + + // blockDist makes DistributionInspect block until its context expires and + // then return that context's error, simulating a hung/unreachable registry. + blockDist bool + + distCalled bool + localCalled bool + // localCtxErr records the context error observed by ImageInspectWithRaw at + // call time, so a test can assert the local fallback got a live budget. + localCtxErr error +} + +func (f *fakeImageInspector) DistributionInspect(ctx context.Context, image, encodedRegistryAuth string) (registry.DistributionInspect, error) { + f.distCalled = true + if f.blockDist { + <-ctx.Done() + return registry.DistributionInspect{}, ctx.Err() + } + return registry.DistributionInspect{}, f.distErr +} + +func (f *fakeImageInspector) ImageInspectWithRaw(ctx context.Context, imageID string) (types.ImageInspect, []byte, error) { + f.localCalled = true + f.localCtxErr = ctx.Err() + return types.ImageInspect{}, nil, f.localErr +} + +// Registry lookup succeeds => image is verified, no local fallback needed. +func TestCheckImageExists_RegistryHit(t *testing.T) { + f := &fakeImageInspector{} + require.NoError(t, checkImageExists(context.Background(), f, "", "registry.example.com/img:tag", time.Second, time.Second)) + assert.True(t, f.distCalled, "expected registry inspect to run") + assert.False(t, f.localCalled, "local fallback should not run when the registry lookup succeeds") +} + +// Registry lookup fails but the image is present in the local daemon (a +// locally-built or already-pulled image) => verified via the fallback. +func TestCheckImageExists_RegistryMissLocalHit(t *testing.T) { + f := &fakeImageInspector{distErr: errors.New("no such image in registry")} + require.NoError(t, checkImageExists(context.Background(), f, "", "localbuild:latest", time.Second, time.Second)) + assert.True(t, f.localCalled, "expected local fallback to run when the registry lookup fails") +} + +// Registry lookup fails AND the image is absent locally => error, and it should +// surface the registry error (the primary lookup path). +func TestCheckImageExists_BothMiss(t *testing.T) { + f := &fakeImageInspector{ + distErr: errors.New("registry unreachable"), + localErr: errors.New("no such image"), + } + err := checkImageExists(context.Background(), f, "", "ghost:latest", time.Second, time.Second) + require.Error(t, err) + assert.Contains(t, err.Error(), "ghost:latest") + assert.Contains(t, err.Error(), "registry unreachable") +} + +// A slow/hung registry lookup that consumes its whole budget must NOT starve +// the local-daemon fallback: the local check gets its own independent budget, +// so a locally-built image is still found. Regression guard for the shared- +// context starvation bug. +func TestCheckImageExists_SlowRegistryDoesNotStarveLocal(t *testing.T) { + f := &fakeImageInspector{blockDist: true} + err := checkImageExists(context.Background(), f, "", "localbuild:latest", + 20*time.Millisecond, 2*time.Second) + require.NoError(t, err, "local fallback should still find the image after a slow registry failure") + assert.True(t, f.localCalled, "expected local fallback to run") + assert.NoError(t, f.localCtxErr, "local inspect must receive a live (un-expired) context") +} diff --git a/server/internal/orchestrator/common/patroni_config_generator.go b/server/internal/orchestrator/common/patroni_config_generator.go index ec4932c5..f7b9b94f 100644 --- a/server/internal/orchestrator/common/patroni_config_generator.go +++ b/server/internal/orchestrator/common/patroni_config_generator.go @@ -74,6 +74,12 @@ type PatroniConfigGenerator struct { // SpecParameters are user-specified Postgres parameters that are included // in the database spec. SpecParameters map[string]any `json:"spec_parameters,omitempty"` + // ColdFrontEnabled makes the generated config load the coldfront / pg_duckdb + // stack: pg_duckdb and coldfront are appended to shared_preload_libraries and + // the duckdb.* GUCs are set. Applied after all other parameter sources so it + // wins over both the defaults and any user override (correctness-critical), + // while preserving the existing libraries (incl. spock). + ColdFrontEnabled bool `json:"coldfront_enabled,omitempty"` // TenantID is an optional tenant ID that is associated with this instance. TenantID *string `json:"tenant_id,omitempty"` } @@ -148,6 +154,7 @@ func NewPatroniConfigGenerator(opts PatroniConfigGeneratorOptions) *PatroniConfi PostgresPort: opts.PostgresPort, RestoreCommand: restoreCommand, SpecParameters: opts.Instance.PostgreSQLConf, + ColdFrontEnabled: opts.Instance.ColdFrontEnabled, PgHbaConf: opts.Instance.PgHbaConf, PgIdentConf: opts.Instance.PgIdentConf, TenantID: opts.Instance.TenantID, @@ -229,6 +236,17 @@ func (p *PatroniConfigGenerator) parameters() map[string]any { maps.Copy(parameters, postgres.SnowflakeLolorGUCs(p.NodeOrdinal)) maps.Copy(parameters, p.SpecParameters) + // ColdFront config is applied last so it wins over the defaults and any user + // override. shared_preload_libraries is appended (not replaced) to preserve + // the existing libraries — including spock — and the correctness-critical + // duckdb.* GUCs cannot be overridden by the user. + if p.ColdFrontEnabled { + preload, _ := parameters["shared_preload_libraries"].(string) + parameters["shared_preload_libraries"] = postgres.AppendSharedPreloadLibraries( + preload, "pg_duckdb", "coldfront") + maps.Copy(parameters, postgres.ColdFrontDuckDBGUCs()) + } + return parameters } diff --git a/server/internal/orchestrator/common/patroni_config_generator_coldfront_test.go b/server/internal/orchestrator/common/patroni_config_generator_coldfront_test.go new file mode 100644 index 00000000..5e2f7499 --- /dev/null +++ b/server/internal/orchestrator/common/patroni_config_generator_coldfront_test.go @@ -0,0 +1,83 @@ +package common + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +// splitLibs splits a shared_preload_libraries value into a trimmed set. +func splitLibs(v string) map[string]bool { + set := map[string]bool{} + for lib := range strings.SplitSeq(v, ",") { + if l := strings.TrimSpace(lib); l != "" { + set[l] = true + } + } + return set +} + +// TestParametersColdFrontEnabled verifies the ColdFront boot GUCs (#6a/#6b) are +// applied when the flag is set: pg_duckdb + coldfront are appended to +// shared_preload_libraries (keeping the existing libs incl. spock) and the +// duckdb.* trio is set. +func TestParametersColdFrontEnabled(t *testing.T) { + p := &PatroniConfigGenerator{ + ColdFrontEnabled: true, + MemoryBytes: 8 << 30, + CPUs: 4, + ClusterSize: 3, + } + params := p.parameters() + + libs := splitLibs(params["shared_preload_libraries"].(string)) + // The default libs must survive, and the coldfront stack must be present. + for _, want := range []string{"pg_stat_statements", "snowflake", "spock", "pg_duckdb", "coldfront"} { + assert.Truef(t, libs[want], "shared_preload_libraries missing %q: %v", + want, params["shared_preload_libraries"]) + } + + assert.Equal(t, "/usr/lib/pgedge/coldfront/duckdb-extensions", params["duckdb.extension_directory"]) + assert.Equal(t, "true", params["duckdb.allow_unsigned_extensions"]) + assert.Equal(t, "false", params["duckdb.autoinstall_known_extensions"]) +} + +// TestParametersColdFrontWinsOverUserOverride verifies the coldfront stack is +// appended on top of a user-supplied shared_preload_libraries override (which +// keeps spock per the API validation), rather than being clobbered by it. +func TestParametersColdFrontWinsOverUserOverride(t *testing.T) { + p := &PatroniConfigGenerator{ + ColdFrontEnabled: true, + MemoryBytes: 8 << 30, + CPUs: 4, + ClusterSize: 3, + SpecParameters: map[string]any{ + "shared_preload_libraries": "spock,pg_cron", + }, + } + params := p.parameters() + + libs := splitLibs(params["shared_preload_libraries"].(string)) + for _, want := range []string{"spock", "pg_cron", "pg_duckdb", "coldfront"} { + assert.Truef(t, libs[want], "shared_preload_libraries missing %q: %v", + want, params["shared_preload_libraries"]) + } +} + +// TestParametersColdFrontDisabled verifies a non-ColdFront database is +// untouched: no duckdb.* GUCs and the default shared_preload_libraries. +func TestParametersColdFrontDisabled(t *testing.T) { + p := &PatroniConfigGenerator{ + ColdFrontEnabled: false, + MemoryBytes: 8 << 30, + CPUs: 4, + ClusterSize: 3, + } + params := p.parameters() + + assert.NotContains(t, params, "duckdb.extension_directory") + libs := splitLibs(params["shared_preload_libraries"].(string)) + assert.False(t, libs["pg_duckdb"], "pg_duckdb should not be present when ColdFront is disabled") + assert.False(t, libs["coldfront"], "coldfront should not be present when ColdFront is disabled") +} diff --git a/server/internal/orchestrator/swarm/coldfront_tiering_wiring_test.go b/server/internal/orchestrator/swarm/coldfront_tiering_wiring_test.go new file mode 100644 index 00000000..d13bcddb --- /dev/null +++ b/server/internal/orchestrator/swarm/coldfront_tiering_wiring_test.go @@ -0,0 +1,189 @@ +package swarm + +import ( + "testing" + + "github.com/pgEdge/control-plane/server/internal/database" + "github.com/pgEdge/control-plane/server/internal/resource" + "github.com/pgEdge/control-plane/server/internal/scheduler" +) + +// makeLakekeeperSpecWithStorage extends makeLakekeeperSpec with the storage +// config required to produce tiering schedule resources. +func makeLakekeeperSpecWithStorage() *database.ServiceInstanceSpec { + cfg := map[string]any{ + "catalog_db_url": "postgres://lakekeeper:secret@pg:5432/lk?sslmode=disable", + "pg_encryption_key": "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + "provider": "aws", + "warehouse": "s3://my-bucket/warehouse", + "bucket": "my-bucket", + "region": "us-east-1", + "credential": `{"access_key_id":"AKID","secret_access_key":"SECRET"}`, + } + return &database.ServiceInstanceSpec{ + ServiceInstanceID: "inst-lakekeeper-1", + ServiceSpec: &database.ServiceSpec{ + ServiceID: "lakekeeper-svc", + ServiceType: "lakekeeper", + Version: "0.9.0", + Config: cfg, + }, + DatabaseID: "db-1", + DatabaseName: "testdb", + HostID: "host-1", + NodeName: "n1", + ConnectAsUsername: "app_owner", + } +} + +// TestGenerateLakekeeperInstanceResources_TieringSchedules verifies that when +// a lakekeeper service has storage config (provider + credential), the +// generated resource graph includes ScheduledJobResource entries for the +// archiver, partitioner, and compactor tiering jobs. +func TestGenerateLakekeeperInstanceResources_TieringSchedules(t *testing.T) { + o := newLakekeeperTestOrchestrator(t) + spec := makeLakekeeperSpecWithStorage() + + result, err := o.generateLakekeeperInstanceResources(spec) + if err != nil { + t.Fatalf("generateLakekeeperInstanceResources() unexpected error: %v", err) + } + + // Collect all ScheduledJobResource identifiers from the result. + var scheduledIDs []string + for _, rd := range result.Resources { + if rd.Identifier.Type == scheduler.ResourceTypeScheduledJob { + scheduledIDs = append(scheduledIDs, rd.Identifier.ID) + } + } + + wantWorkflows := []string{ + scheduler.WorkflowColdFrontArchive, + scheduler.WorkflowColdFrontPartition, + scheduler.WorkflowColdFrontCompact, + } + + // Decode each ScheduledJobResource and verify all three workflow types are present. + foundWorkflows := map[string]bool{} + for _, rd := range result.Resources { + if rd.Identifier.Type != scheduler.ResourceTypeScheduledJob { + continue + } + job, decErr := resource.ToResource[*scheduler.ScheduledJobResource](rd) + if decErr != nil { + t.Fatalf("failed to decode ScheduledJobResource %q: %v", rd.Identifier.ID, decErr) + } + foundWorkflows[job.Workflow] = true + } + + for _, wf := range wantWorkflows { + if !foundWorkflows[wf] { + t.Errorf("missing scheduled job for workflow %q; found: %v", wf, scheduledIDs) + } + } +} + +// TestGenerateLakekeeperInstanceResources_TieringDSNUser verifies that the +// connect-as user is threaded into every tiering job's args (as +// local_pg_dsn_user inside service_config), so the binaries authenticate to the +// node's local Postgres as the database owner rather than a hardcoded role. +func TestGenerateLakekeeperInstanceResources_TieringDSNUser(t *testing.T) { + o := newLakekeeperTestOrchestrator(t) + spec := makeLakekeeperSpecWithStorage() + + result, err := o.generateLakekeeperInstanceResources(spec) + if err != nil { + t.Fatalf("generateLakekeeperInstanceResources() unexpected error: %v", err) + } + + var checked int + for _, rd := range result.Resources { + if rd.Identifier.Type != scheduler.ResourceTypeScheduledJob { + continue + } + job, decErr := resource.ToResource[*scheduler.ScheduledJobResource](rd) + if decErr != nil { + t.Fatalf("failed to decode ScheduledJobResource %q: %v", rd.Identifier.ID, decErr) + } + svcCfg, ok := job.Args["service_config"].(map[string]any) + if !ok { + t.Fatalf("job %q: service_config missing or wrong type: %T", rd.Identifier.ID, job.Args["service_config"]) + } + if got := svcCfg["local_pg_dsn_user"]; got != "app_owner" { + t.Errorf("job %q: local_pg_dsn_user = %v, want %q", rd.Identifier.ID, got, "app_owner") + } + checked++ + } + if checked == 0 { + t.Fatal("no tiering ScheduledJobResource found to assert on") + } +} + +// TestGenerateLakekeeperInstanceResources_MigrateAttachesOverlay verifies that +// the per-database overlay is created attachable and the one-shot migrate +// resource carries that overlay's name, so the migrate container can join the +// overlay and resolve the postgres- catalog host in catalog_db_url. +func TestGenerateLakekeeperInstanceResources_MigrateAttachesOverlay(t *testing.T) { + o := newLakekeeperTestOrchestrator(t) + spec := makeLakekeeperSpecWithStorage() + + result, err := o.generateLakekeeperInstanceResources(spec) + if err != nil { + t.Fatalf("generateLakekeeperInstanceResources() unexpected error: %v", err) + } + + const wantNet = "db-1-database" + var foundNet, foundMigrate bool + for _, rd := range result.Resources { + switch rd.Identifier.Type { + case ResourceTypeNetwork: + net, decErr := resource.ToResource[*Network](rd) + if decErr != nil { + t.Fatalf("decode Network: %v", decErr) + } + if net.Name == wantNet { + foundNet = true + if !net.Attachable { + t.Errorf("database overlay %q must be attachable for the migrate one-shot", wantNet) + } + } + case ResourceTypeLakekeeperMigrate: + mig, decErr := resource.ToResource[*LakekeeperMigrateResource](rd) + if decErr != nil { + t.Fatalf("decode LakekeeperMigrateResource: %v", decErr) + } + foundMigrate = true + if mig.DatabaseNetworkName != wantNet { + t.Errorf("migrate DatabaseNetworkName = %q, want %q", mig.DatabaseNetworkName, wantNet) + } + } + } + if !foundNet { + t.Error("database network resource not found in generated graph") + } + if !foundMigrate { + t.Error("lakekeeper migrate resource not found in generated graph") + } +} + +// TestGenerateLakekeeperInstanceResources_TieringSchedules_NoStorage verifies +// that when no storage config is present (provider absent), no tiering +// schedule resources are generated rather than returning an error. +func TestGenerateLakekeeperInstanceResources_TieringSchedules_NoStorage(t *testing.T) { + o := newLakekeeperTestOrchestrator(t) + spec := makeLakekeeperSpec( + "postgres://lakekeeper:secret@pg:5432/lk?sslmode=disable", + "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + ) + + result, err := o.generateLakekeeperInstanceResources(spec) + if err != nil { + t.Fatalf("generateLakekeeperInstanceResources() unexpected error: %v", err) + } + + for _, rd := range result.Resources { + if rd.Identifier.Type == scheduler.ResourceTypeScheduledJob { + t.Errorf("expected no ScheduledJobResource when provider absent, got %q", rd.Identifier.ID) + } + } +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_bootstrap.go b/server/internal/orchestrator/swarm/lakekeeper_bootstrap.go new file mode 100644 index 00000000..816d057d --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_bootstrap.go @@ -0,0 +1,351 @@ +package swarm + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// lakekeeperStorageConfig holds the object-store coordinates and parsed +// credential needed to build a Lakekeeper warehouse storage profile. It is +// derived from the lakekeeper ServiceSpec.Config supplied by the caller. +// +// The Credential map is provider-specific and MUST NOT be logged: it carries +// secret access keys / connection strings. +type lakekeeperStorageConfig struct { + Provider string // "aws" | "azure" | "gcs" + Bucket string + Region string + Endpoint string + Warehouse string + PathPrefix string + // Credential is the parsed provider-specific credential JSON. + // aws: {"access_key_id","secret_access_key"} + // azure: {"connection_string"} + // gcs: {"hmac_access_id","hmac_secret"} + Credential map[string]string +} + +// lakekeeperBootstrapHTTPTimeout bounds each individual REST call to the +// Lakekeeper management/catalog API. +const lakekeeperBootstrapHTTPTimeout = 30 * time.Second + +// parseLakekeeperStorageConfig extracts the object-store configuration and +// parses the provider-specific credential JSON from a lakekeeper +// ServiceSpec.Config map. It fails loud when a required key is missing so that +// a database is never left with an unbootstrapped (broken) warehouse. +// +// Some of these keys are supplied by a consumer-side follow-up; an absent +// required key therefore yields a clear, actionable error rather than a +// silent misconfiguration. +func parseLakekeeperStorageConfig(config map[string]any) (*lakekeeperStorageConfig, error) { + get := func(key string) string { + v, _ := config[key].(string) + return strings.TrimSpace(v) + } + + provider := get("provider") + switch provider { + case "aws", "azure", "gcs": + case "": + return nil, fmt.Errorf("lakekeeper bootstrap: provider is required in config (one of aws, azure, gcs)") + default: + return nil, fmt.Errorf("lakekeeper bootstrap: unsupported provider %q (expected one of aws, azure, gcs)", provider) + } + + warehouse := get("warehouse") + if warehouse == "" { + return nil, fmt.Errorf("lakekeeper bootstrap: warehouse is required in config") + } + + credRaw := get("credential") + if credRaw == "" { + return nil, fmt.Errorf("lakekeeper bootstrap: credential is required in config") + } + var cred map[string]string + if err := json.Unmarshal([]byte(credRaw), &cred); err != nil { + // Deliberately do not include credRaw in the error: it is a secret. + return nil, fmt.Errorf("lakekeeper bootstrap: credential is not valid JSON") + } + + cfg := &lakekeeperStorageConfig{ + Provider: provider, + Bucket: get("bucket"), + Region: get("region"), + Endpoint: get("endpoint"), + Warehouse: warehouse, + PathPrefix: get("path_prefix"), + Credential: cred, + } + + // Provider-specific required fields. + switch provider { + case "aws", "gcs": + if cfg.Bucket == "" { + return nil, fmt.Errorf("lakekeeper bootstrap: bucket is required in config for provider %q", provider) + } + if provider == "aws" { + if cred["access_key_id"] == "" || cred["secret_access_key"] == "" { + return nil, fmt.Errorf("lakekeeper bootstrap: aws credential must contain access_key_id and secret_access_key") + } + } else { // gcs + if cred["hmac_access_id"] == "" || cred["hmac_secret"] == "" { + return nil, fmt.Errorf("lakekeeper bootstrap: gcs credential must contain hmac_access_id and hmac_secret") + } + } + case "azure": + if cfg.Bucket == "" { + return nil, fmt.Errorf("lakekeeper bootstrap: bucket (container/account) is required in config for provider %q", provider) + } + if cred["connection_string"] == "" { + return nil, fmt.Errorf("lakekeeper bootstrap: azure credential must contain connection_string") + } + } + + return cfg, nil +} + +// buildWarehouseRequestBody builds the POST /management/v1/warehouse request +// body for the configured provider. +// +// For aws and gcs an s3 storage profile is used (gcs is reached through its +// S3-compatible HMAC interface). For azure an adls profile is used. +func buildWarehouseRequestBody(cfg *lakekeeperStorageConfig) (map[string]any, error) { + switch cfg.Provider { + case "aws", "gcs": + profile := map[string]any{ + "type": "s3", + "bucket": cfg.Bucket, + "region": cfg.Region, + "sts-enabled": false, + "remote-signing-enabled": false, + } + // Endpoint presence is the discriminator between native cloud AWS and + // an S3-compatible store (GCS via storage.googleapis.com, SeaweedFS, + // etc.). Cloud AWS must use virtual-hosted addressing (flavor "aws", + // path-style-access false); path-style fails on any Region launched + // after 2019. An S3-compatible endpoint uses flavor "s3-compat" with + // path-style addressing. (ColdFront docs/object_store.md, installation.md.) + if cfg.Endpoint != "" { + profile["endpoint"] = cfg.Endpoint + profile["flavor"] = "s3-compat" + profile["path-style-access"] = true + } else { + profile["flavor"] = "aws" + profile["path-style-access"] = false + } + if cfg.PathPrefix != "" { + profile["key-prefix"] = cfg.PathPrefix + } + + var credential map[string]any + if cfg.Provider == "aws" { + credential = map[string]any{ + "type": "s3", + "credential-type": "access-key", + "aws-access-key-id": cfg.Credential["access_key_id"], + "aws-secret-access-key": cfg.Credential["secret_access_key"], + } + } else { // gcs via S3-compatible HMAC + credential = map[string]any{ + "type": "s3", + "credential-type": "access-key", + "aws-access-key-id": cfg.Credential["hmac_access_id"], + "aws-secret-access-key": cfg.Credential["hmac_secret"], + } + } + + return map[string]any{ + "warehouse-name": cfg.Warehouse, + "storage-profile": profile, + "storage-credential": credential, + }, nil + case "azure": + profile := map[string]any{ + "type": "adls", + "account-name": cfg.Region, // account name carried in region for azure + "filesystem": cfg.Bucket, // container + } + if cfg.Endpoint != "" { + profile["endpoint-suffix"] = cfg.Endpoint + } + if cfg.PathPrefix != "" { + profile["path-prefix"] = cfg.PathPrefix + } + credential := map[string]any{ + "type": "az", + "credential-type": "shared-access-key", + "connection-string": cfg.Credential["connection_string"], + } + return map[string]any{ + "warehouse-name": cfg.Warehouse, + "storage-profile": profile, + "storage-credential": credential, + }, nil + default: + return nil, fmt.Errorf("lakekeeper bootstrap: unsupported provider %q", cfg.Provider) + } +} + +// runLakekeeperBootstrap performs the Lakekeeper REST bootstrap sequence +// against baseURL (the Lakekeeper service, e.g. http://:8181) using +// the supplied HTTP client, in order and idempotently: +// +// 1. POST /management/v1/bootstrap {"accept-terms-of-use": true} +// 2. POST /management/v1/warehouse (storage profile + credential) +// 3. extract warehouse-id from the warehouse response (or look it up on +// conflict) +// 4. POST /catalog/v1/{warehouse-id}/namespaces {"namespace":["default"]} +// +// Already-bootstrapped / already-exists responses (4xx conflicts) are treated +// as success so the sequence is safe to re-run. +func runLakekeeperBootstrap(ctx context.Context, client *http.Client, baseURL string, cfg *lakekeeperStorageConfig) error { + baseURL = strings.TrimRight(baseURL, "/") + + // Step 1: bootstrap (accept terms). Idempotent: an already-bootstrapped + // server returns a client error (400/409) which we tolerate. + if err := lakekeeperPost(ctx, client, baseURL+"/management/v1/bootstrap", + map[string]any{"accept-terms-of-use": true}, nil, true); err != nil { + return fmt.Errorf("lakekeeper bootstrap: accept-terms failed: %w", err) + } + + // Step 2: create the warehouse. + body, err := buildWarehouseRequestBody(cfg) + if err != nil { + return err + } + var whResp struct { + WarehouseID string `json:"warehouse-id"` + ID string `json:"id"` + } + created, err := lakekeeperPostDecode(ctx, client, baseURL+"/management/v1/warehouse", body, &whResp) + if err != nil { + return fmt.Errorf("lakekeeper bootstrap: create warehouse failed: %w", err) + } + + // Step 3: resolve the warehouse id. On a fresh create it is in the + // response; on conflict (already exists) we look it up by name. + warehouseID := firstNonEmpty(whResp.WarehouseID, whResp.ID) + if !created || warehouseID == "" { + warehouseID, err = lookupLakekeeperWarehouseID(ctx, client, baseURL, cfg.Warehouse) + if err != nil { + return fmt.Errorf("lakekeeper bootstrap: resolve warehouse id: %w", err) + } + } + if warehouseID == "" { + return fmt.Errorf("lakekeeper bootstrap: could not determine warehouse id for warehouse %q", cfg.Warehouse) + } + + // Step 4: create the default namespace (required for compaction to be able + // to create tables). Idempotent: already-exists is tolerated. + nsURL := fmt.Sprintf("%s/catalog/v1/%s/namespaces", baseURL, warehouseID) + if err := lakekeeperPost(ctx, client, nsURL, + map[string]any{"namespace": []string{"default"}}, nil, true); err != nil { + return fmt.Errorf("lakekeeper bootstrap: create default namespace failed: %w", err) + } + + return nil +} + +// lookupLakekeeperWarehouseID lists warehouses and returns the id of the one +// matching name. Used when the warehouse already exists (create returned a +// conflict). +func lookupLakekeeperWarehouseID(ctx context.Context, client *http.Client, baseURL, name string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, baseURL+"/management/v1/warehouse", nil) + if err != nil { + return "", err + } + req.Header.Set("Accept", "application/json") + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + data, _ := io.ReadAll(resp.Body) + if resp.StatusCode >= 300 { + return "", fmt.Errorf("list warehouses returned status %d", resp.StatusCode) + } + var listResp struct { + Warehouses []struct { + WarehouseID string `json:"warehouse-id"` + ID string `json:"id"` + Name string `json:"name"` + } `json:"warehouses"` + } + if err := json.Unmarshal(data, &listResp); err != nil { + return "", fmt.Errorf("decode warehouse list: %w", err) + } + for _, w := range listResp.Warehouses { + if w.Name == name { + // Prefer the canonical warehouse-id field; fall back to the + // deprecated id, matching the create-response handling. + return firstNonEmpty(w.WarehouseID, w.ID), nil + } + } + return "", nil +} + +// lakekeeperPost issues a POST with a JSON body. When out is non-nil the +// response body is decoded into it. When tolerateConflict is true a 4xx +// response is treated as success (idempotent already-exists), otherwise any +// status >= 300 is an error. +func lakekeeperPost(ctx context.Context, client *http.Client, url string, body any, out any, tolerateConflict bool) error { + _, err := lakekeeperDo(ctx, client, url, body, out, tolerateConflict) + return err +} + +// lakekeeperPostDecode issues a POST that always tolerates conflicts and +// decodes a success response into out. It returns whether the resource was +// newly created (2xx) as opposed to already existing (4xx conflict). +func lakekeeperPostDecode(ctx context.Context, client *http.Client, url string, body any, out any) (created bool, err error) { + return lakekeeperDo(ctx, client, url, body, out, true) +} + +func lakekeeperDo(ctx context.Context, client *http.Client, url string, body any, out any, tolerateConflict bool) (created bool, err error) { + payload, err := json.Marshal(body) + if err != nil { + return false, fmt.Errorf("marshal request body: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload)) + if err != nil { + return false, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + data, _ := io.ReadAll(resp.Body) + + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + if out != nil && len(data) > 0 { + if err := json.Unmarshal(data, out); err != nil { + return true, fmt.Errorf("decode response: %w", err) + } + } + return true, nil + case tolerateConflict && resp.StatusCode >= 400 && resp.StatusCode < 500: + // Already bootstrapped / already exists — idempotent success. + return false, nil + default: + return false, fmt.Errorf("POST %s returned status %d: %s", url, resp.StatusCode, string(data)) + } +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if v != "" { + return v + } + } + return "" +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_bootstrap_resource.go b/server/internal/orchestrator/swarm/lakekeeper_bootstrap_resource.go new file mode 100644 index 00000000..0330af1b --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_bootstrap_resource.go @@ -0,0 +1,167 @@ +package swarm + +import ( + "context" + "fmt" + "net/http" + + "github.com/samber/do" + + "github.com/pgEdge/control-plane/server/internal/docker" + "github.com/pgEdge/control-plane/server/internal/resource" +) + +var _ resource.Resource = (*LakekeeperBootstrapResource)(nil) + +const ResourceTypeLakekeeperBootstrap resource.Type = "swarm.lakekeeper_bootstrap" + +func LakekeeperBootstrapResourceIdentifier(serviceInstanceID string) resource.Identifier { + return resource.Identifier{ + ID: serviceInstanceID, + Type: ResourceTypeLakekeeperBootstrap, + } +} + +// LakekeeperBootstrapResource performs the post-deploy REST bootstrap of a +// Lakekeeper serve instance: it accepts the terms of use, creates the +// configured warehouse (with its object-store storage profile and credential), +// and creates the "default" namespace so that compaction can create tables. +// +// It depends on the serve ServiceInstanceResource, so it only runs after the +// Docker Swarm service is confirmed healthy (WaitForService). It runs on the +// same host as the serve container (HostExecutor) and reaches Lakekeeper at the +// serve container's IP on the Docker default bridge network — the same scheme +// GetInstanceConnectionInfo uses to reach Patroni/Postgres. It does NOT use the +// swarm service name: that resolves only via overlay DNS from inside the overlay, +// which the host-networked CP process is not a member of (finding #14). +// +// A bootstrap FAILURE blocks: a database with an unbootstrapped warehouse is +// broken, so the error is returned from Create (surfaced by the resource +// engine) rather than merely logged in a best-effort PostDeploy hook. +// +// The sequence is idempotent — already-bootstrapped / already-exists responses +// are treated as success — so Create and Update re-run safely. BootstrapDone is +// a sentinel so Refresh can distinguish "never run" from "already applied". +type LakekeeperBootstrapResource struct { + ServiceInstanceID string `json:"service_instance_id"` + HostID string `json:"host_id"` + ServiceName string `json:"service_name"` + Port int `json:"port"` + Config map[string]any `json:"config"` + BootstrapDone bool `json:"bootstrap_done"` +} + +func (r *LakekeeperBootstrapResource) ResourceVersion() string { return "1" } + +func (r *LakekeeperBootstrapResource) DiffIgnore() []string { + // BootstrapDone is runtime state written by Create; exclude it from diffs + // so a completed bootstrap does not trigger spurious updates. + return []string{"/bootstrap_done"} +} + +func (r *LakekeeperBootstrapResource) Identifier() resource.Identifier { + return LakekeeperBootstrapResourceIdentifier(r.ServiceInstanceID) +} + +func (r *LakekeeperBootstrapResource) Executor() resource.Executor { + // Run on the same host as the serve container so we can reach Lakekeeper + // over the bridge network and share its Docker network context. + return resource.HostExecutor(r.HostID) +} + +func (r *LakekeeperBootstrapResource) Dependencies() []resource.Identifier { + // Depend on the serve ServiceInstanceResource so bootstrap only runs after + // the Docker service is confirmed healthy. + return []resource.Identifier{ + ServiceInstanceResourceIdentifier(r.ServiceInstanceID), + } +} + +func (r *LakekeeperBootstrapResource) TypeDependencies() []resource.Type { + return nil +} + +// Refresh returns ErrNotFound until the bootstrap has completed at least once, +// causing the resource engine to call Create. +func (r *LakekeeperBootstrapResource) Refresh(ctx context.Context, rc *resource.Context) error { + if !r.BootstrapDone { + return fmt.Errorf("%w: lakekeeper warehouse bootstrap has not yet run", resource.ErrNotFound) + } + return nil +} + +func (r *LakekeeperBootstrapResource) Create(ctx context.Context, rc *resource.Context) error { + if err := r.bootstrap(ctx, rc); err != nil { + return err + } + r.BootstrapDone = true + return nil +} + +func (r *LakekeeperBootstrapResource) Update(ctx context.Context, rc *resource.Context) error { + // Re-running the bootstrap is safe: the REST sequence is idempotent. + if err := r.bootstrap(ctx, rc); err != nil { + return err + } + r.BootstrapDone = true + return nil +} + +func (r *LakekeeperBootstrapResource) Delete(ctx context.Context, rc *resource.Context) error { + // Deleting the service removes the warehouse configuration; nothing to do. + return nil +} + +func (r *LakekeeperBootstrapResource) bootstrap(ctx context.Context, rc *resource.Context) error { + cfg, err := parseLakekeeperStorageConfig(r.Config) + if err != nil { + return err + } + + host, err := r.serveBridgeHost(ctx, rc) + if err != nil { + return err + } + baseURL := lakekeeperBaseURL(host, r.Port) + + client := &http.Client{Timeout: lakekeeperBootstrapHTTPTimeout} + if err := runLakekeeperBootstrap(ctx, client, baseURL, cfg); err != nil { + return err + } + return nil +} + +// serveBridgeHost returns the serve container's IP on the Docker default bridge +// network, which the host-networked CP process can reach directly — mirroring how +// GetInstanceConnectionInfo reaches Patroni/Postgres. The serve container is +// found by its service instance ID and inspected fresh each run (a rescheduled +// task gets a new bridge IP). +func (r *LakekeeperBootstrapResource) serveBridgeHost(ctx context.Context, rc *resource.Context) (string, error) { + client, err := do.Invoke[*docker.Docker](rc.Injector) + if err != nil { + return "", fmt.Errorf("lakekeeper bootstrap: failed to get docker client: %w", err) + } + c, err := GetServiceContainer(ctx, client, r.ServiceInstanceID) + if err != nil { + return "", fmt.Errorf("lakekeeper bootstrap: failed to find serve container: %w", err) + } + inspect, err := client.ContainerInspect(ctx, c.ID) + if err != nil { + return "", fmt.Errorf("lakekeeper bootstrap: failed to inspect serve container: %w", err) + } + host, err := bridgeIPAddress(inspect) + if err != nil { + return "", fmt.Errorf("lakekeeper bootstrap: serve container %q: %w", c.ID, err) + } + return host, nil +} + +// lakekeeperBaseURL builds the Lakekeeper REST base URL. A zero port falls back +// to the in-container listen port 8181 (reachable on the bridge IP regardless of +// any host-port publishing). +func lakekeeperBaseURL(host string, port int) string { + if port == 0 { + port = 8181 + } + return fmt.Sprintf("http://%s:%d", host, port) +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_bootstrap_resource_test.go b/server/internal/orchestrator/swarm/lakekeeper_bootstrap_resource_test.go new file mode 100644 index 00000000..8a1cfc23 --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_bootstrap_resource_test.go @@ -0,0 +1,48 @@ +package swarm + +import ( + "testing" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/network" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLakekeeperBaseURL(t *testing.T) { + // Default port when unset (0) is the in-container listen port 8181. + assert.Equal(t, "http://172.18.0.5:8181", lakekeeperBaseURL("172.18.0.5", 0)) + // Explicit port is honored. + assert.Equal(t, "http://172.18.0.5:9000", lakekeeperBaseURL("172.18.0.5", 9000)) +} + +func TestBridgeIPAddress(t *testing.T) { + inspect := func(nets map[string]*network.EndpointSettings) types.ContainerJSON { + return types.ContainerJSON{ + NetworkSettings: &types.NetworkSettings{Networks: nets}, + } + } + + t.Run("returns the bridge IP", func(t *testing.T) { + ip, err := bridgeIPAddress(inspect(map[string]*network.EndpointSettings{ + "bridge": {IPAddress: "172.18.0.5"}, + "mydb-database": {IPAddress: "10.0.1.7"}, // overlay IP must not be picked + })) + require.NoError(t, err) + assert.Equal(t, "172.18.0.5", ip) + }) + + t.Run("errors when the bridge network is absent", func(t *testing.T) { + _, err := bridgeIPAddress(inspect(map[string]*network.EndpointSettings{ + "mydb-database": {IPAddress: "10.0.1.7"}, + })) + assert.Error(t, err) + }) + + t.Run("errors when the bridge IP is empty", func(t *testing.T) { + _, err := bridgeIPAddress(inspect(map[string]*network.EndpointSettings{ + "bridge": {IPAddress: ""}, + })) + assert.Error(t, err) + }) +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_bootstrap_test.go b/server/internal/orchestrator/swarm/lakekeeper_bootstrap_test.go new file mode 100644 index 00000000..6a1b591e --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_bootstrap_test.go @@ -0,0 +1,238 @@ +package swarm + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// awsBootstrapConfig is a native cloud-AWS config: no endpoint, so +// virtual-hosted addressing (flavor "aws", path-style-access false). +func awsBootstrapConfig() map[string]any { + return map[string]any{ + "provider": "aws", + "bucket": "my-bucket", + "region": "us-east-1", + "warehouse": "wh1", + "path_prefix": "iceberg", + "credential": `{"access_key_id":"AKIA_TEST","secret_access_key":"SECRET_TEST"}`, + } +} + +// gcsBootstrapConfig is an S3-compatible config (GCS): endpoint present, so +// flavor "s3-compat" and path-style-access true. +func gcsBootstrapConfig() map[string]any { + return map[string]any{ + "provider": "gcs", + "bucket": "my-bucket", + "region": "us", + "endpoint": "https://storage.googleapis.com", + "warehouse": "wh1", + "path_prefix": "iceberg", + "credential": `{"hmac_access_id":"GOOG_ID","hmac_secret":"GOOG_SECRET"}`, + } +} + +func TestParseLakekeeperStorageConfig_FailLoud(t *testing.T) { + tests := []struct { + name string + config map[string]any + errSub string + }{ + {"missing provider", map[string]any{"warehouse": "w", "credential": "{}"}, "provider is required"}, + {"bad provider", map[string]any{"provider": "digitalocean", "warehouse": "w", "credential": "{}"}, "unsupported provider"}, + {"missing warehouse", map[string]any{"provider": "aws", "credential": "{}"}, "warehouse is required"}, + {"missing credential", map[string]any{"provider": "aws", "warehouse": "w"}, "credential is required"}, + {"bad credential json", map[string]any{"provider": "aws", "warehouse": "w", "bucket": "b", "credential": "not-json"}, "not valid JSON"}, + {"missing bucket aws", map[string]any{"provider": "aws", "warehouse": "w", "credential": `{"access_key_id":"a","secret_access_key":"s"}`}, "bucket is required"}, + {"aws missing keys", map[string]any{"provider": "aws", "bucket": "b", "warehouse": "w", "credential": `{}`}, "access_key_id and secret_access_key"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := parseLakekeeperStorageConfig(tc.config) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errSub) + }) + } +} + +func TestParseLakekeeperStorageConfig_AWS(t *testing.T) { + cfg, err := parseLakekeeperStorageConfig(awsBootstrapConfig()) + require.NoError(t, err) + assert.Equal(t, "aws", cfg.Provider) + assert.Equal(t, "my-bucket", cfg.Bucket) + assert.Equal(t, "AKIA_TEST", cfg.Credential["access_key_id"]) +} + +// TestRunLakekeeperBootstrap_OrderAndFlow asserts the four REST calls happen in +// order, the warehouse-id is extracted from the create response and used in the +// namespace URL. +func TestRunLakekeeperBootstrap_OrderAndFlow(t *testing.T) { + var calls []string + var namespaceBody map[string]any + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, r.Method+" "+r.URL.Path) + switch { + case r.URL.Path == "/management/v1/bootstrap": + w.WriteHeader(http.StatusOK) + case r.URL.Path == "/management/v1/warehouse" && r.Method == http.MethodPost: + // verify the storage profile is well-formed for aws + var body map[string]any + require.NoError(t, json.NewDecoder(r.Body).Decode(&body)) + assert.Equal(t, "wh1", body["warehouse-name"]) + profile := body["storage-profile"].(map[string]any) + assert.Equal(t, "s3", profile["type"]) + assert.Equal(t, false, profile["sts-enabled"]) + assert.Equal(t, false, profile["remote-signing-enabled"]) + // Cloud AWS (no endpoint): virtual-hosted addressing. + assert.Equal(t, "aws", profile["flavor"]) + assert.Equal(t, false, profile["path-style-access"]) + assert.NotContains(t, profile, "endpoint") + // key-prefix, not path-prefix. + assert.Equal(t, "iceberg", profile["key-prefix"]) + assert.NotContains(t, profile, "path-prefix") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"warehouse-id":"wh-uuid-123"}`)) + case strings.HasPrefix(r.URL.Path, "/catalog/v1/") && strings.HasSuffix(r.URL.Path, "/namespaces"): + require.NoError(t, json.NewDecoder(r.Body).Decode(&namespaceBody)) + w.WriteHeader(http.StatusOK) + default: + t.Errorf("unexpected call: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + } + })) + defer srv.Close() + + cfg, err := parseLakekeeperStorageConfig(awsBootstrapConfig()) + require.NoError(t, err) + + err = runLakekeeperBootstrap(context.Background(), srv.Client(), srv.URL, cfg) + require.NoError(t, err) + + require.Equal(t, []string{ + "POST /management/v1/bootstrap", + "POST /management/v1/warehouse", + "POST /catalog/v1/wh-uuid-123/namespaces", + }, calls) + assert.Equal(t, []any{"default"}, namespaceBody["namespace"]) +} + +// TestRunLakekeeperBootstrap_AlreadyBootstrapped tolerates a conflict on +// bootstrap and on the warehouse (already exists), looking up the warehouse-id +// via GET, and tolerates a conflict on the namespace. +func TestRunLakekeeperBootstrap_AlreadyBootstrapped(t *testing.T) { + var calls []string + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, r.Method+" "+r.URL.Path) + switch { + case r.URL.Path == "/management/v1/bootstrap": + // already bootstrapped + w.WriteHeader(http.StatusConflict) + case r.URL.Path == "/management/v1/warehouse" && r.Method == http.MethodPost: + // already exists + w.WriteHeader(http.StatusConflict) + case r.URL.Path == "/management/v1/warehouse" && r.Method == http.MethodGet: + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"warehouses":[{"id":"existing-wh","name":"wh1"}]}`)) + case strings.HasSuffix(r.URL.Path, "/namespaces"): + w.WriteHeader(http.StatusConflict) + default: + t.Errorf("unexpected call: %s %s", r.Method, r.URL.Path) + } + })) + defer srv.Close() + + cfg, err := parseLakekeeperStorageConfig(awsBootstrapConfig()) + require.NoError(t, err) + + err = runLakekeeperBootstrap(context.Background(), srv.Client(), srv.URL, cfg) + require.NoError(t, err) + + // The namespace must be created against the looked-up existing-wh id. + require.Equal(t, []string{ + "POST /management/v1/bootstrap", + "POST /management/v1/warehouse", + "GET /management/v1/warehouse", + "POST /catalog/v1/existing-wh/namespaces", + }, calls) +} + +// TestRunLakekeeperBootstrap_ServerErrorSurfaces confirms a genuine 5xx failure +// is surfaced (not swallowed), so an unbootstrapped warehouse blocks. +func TestRunLakekeeperBootstrap_ServerErrorSurfaces(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/management/v1/bootstrap" { + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusInternalServerError) + _, _ = io.WriteString(w, "boom") + })) + defer srv.Close() + + cfg, err := parseLakekeeperStorageConfig(awsBootstrapConfig()) + require.NoError(t, err) + + err = runLakekeeperBootstrap(context.Background(), srv.Client(), srv.URL, cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "create warehouse failed") +} + +func TestBuildWarehouseRequestBody_CloudAWS(t *testing.T) { + cfg, err := parseLakekeeperStorageConfig(awsBootstrapConfig()) + require.NoError(t, err) + + body, err := buildWarehouseRequestBody(cfg) + require.NoError(t, err) + profile := body["storage-profile"].(map[string]any) + assert.Equal(t, "s3", profile["type"]) + assert.Equal(t, "aws", profile["flavor"]) + assert.Equal(t, false, profile["path-style-access"]) + assert.NotContains(t, profile, "endpoint") + assert.Equal(t, "iceberg", profile["key-prefix"]) +} + +func TestBuildWarehouseRequestBody_S3CompatGCS(t *testing.T) { + cfg, err := parseLakekeeperStorageConfig(gcsBootstrapConfig()) + require.NoError(t, err) + + body, err := buildWarehouseRequestBody(cfg) + require.NoError(t, err) + profile := body["storage-profile"].(map[string]any) + assert.Equal(t, "s3", profile["type"]) + // endpoint present => s3-compat, path-style addressing. + assert.Equal(t, "s3-compat", profile["flavor"]) + assert.Equal(t, true, profile["path-style-access"]) + assert.Equal(t, "https://storage.googleapis.com", profile["endpoint"]) + assert.Equal(t, "iceberg", profile["key-prefix"]) + // GCS HMAC creds are carried as S3 access-key credentials. + credential := body["storage-credential"].(map[string]any) + assert.Equal(t, "GOOG_ID", credential["aws-access-key-id"]) +} + +func TestBuildWarehouseRequestBody_Azure(t *testing.T) { + cfg, err := parseLakekeeperStorageConfig(map[string]any{ + "provider": "azure", + "bucket": "container1", + "region": "myaccount", + "warehouse": "wh1", + "credential": `{"connection_string":"DefaultEndpointsProtocol=https;AccountName=x"}`, + }) + require.NoError(t, err) + + body, err := buildWarehouseRequestBody(cfg) + require.NoError(t, err) + profile := body["storage-profile"].(map[string]any) + assert.Equal(t, "adls", profile["type"]) + credential := body["storage-credential"].(map[string]any) + assert.Equal(t, "DefaultEndpointsProtocol=https;AccountName=x", credential["connection-string"]) +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_catalog_db_resource.go b/server/internal/orchestrator/swarm/lakekeeper_catalog_db_resource.go new file mode 100644 index 00000000..ca833df2 --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_catalog_db_resource.go @@ -0,0 +1,199 @@ +package swarm + +import ( + "context" + "fmt" + "net/url" + "unicode/utf8" + + "github.com/pgEdge/control-plane/server/internal/database" + "github.com/pgEdge/control-plane/server/internal/postgres" + "github.com/pgEdge/control-plane/server/internal/resource" +) + +// lakekeeperCatalogDBName derives the managed catalog database name for +// a database. Kept within Postgres's 63-byte identifier limit by +// truncating the database-name prefix, never the suffix. Truncation +// backs off to the last whole UTF-8 rune so it never leaves a partial +// multi-byte rune, which would be an invalid identifier on a UTF8 +// database. +func lakekeeperCatalogDBName(databaseName string) string { + const suffix = "_lakekeeper" + const maxLen = 63 + if limit := maxLen - len(suffix); len(databaseName) > limit { + databaseName = databaseName[:limit] + for len(databaseName) > 0 && !utf8.ValidString(databaseName) { + databaseName = databaseName[:len(databaseName)-1] + } + } + return databaseName + suffix +} + +// buildManagedCatalogDBURL constructs the catalog Postgres URL for a +// control-plane-managed catalog, using the overlay-network host entry +// and the service's connect-as credentials. The result contains a +// password: never log it. +// +// sslmode=prefer matches how the other node-attached services (MCP, RAG) +// connect to Postgres over the swarm overlay (mcp_config.go / rag_config.go): +// use TLS if the server offers it, fall back to plaintext otherwise. It is set +// explicitly rather than relying on the client library's default so both the +// migrate and serve containers agree regardless of their driver defaults. +func buildManagedCatalogDBURL(host database.ServiceHostEntry, username, password, dbName string) string { + u := url.URL{ + Scheme: "postgres", + User: url.UserPassword(username, password), + Host: fmt.Sprintf("%s:%d", host.Host, host.Port), + Path: "/" + dbName, + RawQuery: "sslmode=prefer", + } + return u.String() +} + +var _ resource.Resource = (*LakekeeperCatalogDBResource)(nil) + +const ResourceTypeLakekeeperCatalogDB resource.Type = "swarm.lakekeeper_catalog_db" + +func LakekeeperCatalogDBResourceIdentifier(serviceInstanceID string) resource.Identifier { + return resource.Identifier{ + ID: serviceInstanceID, + Type: ResourceTypeLakekeeperCatalogDB, + } +} + +// LakekeeperCatalogDBResource creates the Postgres database that backs +// the Lakekeeper catalog when the spec sets catalog_db_create. It runs +// idempotently against the node's primary (PrimaryExecutor) after the +// main database is available, and before the Lakekeeper migrate step +// (enforced via LakekeeperMigrateResource.Dependencies). +// +// Delete is a deliberate no-op: the catalog maps every Iceberg table's +// cold data — dropping it makes the cold data unreadable. The database +// rides the node's lifecycle instead. +type LakekeeperCatalogDBResource struct { + ServiceInstanceID string `json:"service_instance_id"` + DatabaseID string `json:"database_id"` + // DatabaseName is the node's main database — the connection target + // from which CREATE DATABASE runs. + DatabaseName string `json:"database_name"` + NodeName string `json:"node_name"` + // CatalogDBName/CatalogDBOwner are derived by the orchestrator + // (lakekeeperCatalogDBName / the service's connect-as user). + CatalogDBName string `json:"catalog_db_name"` + CatalogDBOwner string `json:"catalog_db_owner"` + Created bool `json:"created"` +} + +func (r *LakekeeperCatalogDBResource) ResourceVersion() string { return "1" } + +func (r *LakekeeperCatalogDBResource) DiffIgnore() []string { + return []string{"/created"} +} + +func (r *LakekeeperCatalogDBResource) Identifier() resource.Identifier { + return LakekeeperCatalogDBResourceIdentifier(r.ServiceInstanceID) +} + +func (r *LakekeeperCatalogDBResource) Executor() resource.Executor { + return resource.PrimaryExecutor(r.NodeName) +} + +func (r *LakekeeperCatalogDBResource) Dependencies() []resource.Identifier { + // The main database must exist before we can connect to it and issue + // CREATE DATABASE for the catalog. + return []resource.Identifier{ + database.PostgresDatabaseResourceIdentifier(r.NodeName, r.DatabaseName), + } +} + +func (r *LakekeeperCatalogDBResource) TypeDependencies() []resource.Type { return nil } + +func (r *LakekeeperCatalogDBResource) Refresh(ctx context.Context, rc *resource.Context) error { + if !r.Created { + return fmt.Errorf("%w: lakekeeper catalog database has not yet been created", resource.ErrNotFound) + } + return nil +} + +func (r *LakekeeperCatalogDBResource) Create(ctx context.Context, rc *resource.Context) error { + if err := r.ensure(ctx, rc); err != nil { + return err + } + r.Created = true + return nil +} + +func (r *LakekeeperCatalogDBResource) Update(ctx context.Context, rc *resource.Context) error { + // CREATE DATABASE is conditional and owner alignment is idempotent, + // so re-running is safe. + if err := r.ensure(ctx, rc); err != nil { + return err + } + r.Created = true + return nil +} + +func (r *LakekeeperCatalogDBResource) Delete(ctx context.Context, rc *resource.Context) error { + // Deliberate no-op — see type comment. + return nil +} + +func (r *LakekeeperCatalogDBResource) ensure(ctx context.Context, rc *resource.Context) error { + primary, err := database.GetPrimaryInstance(ctx, rc, r.NodeName) + if err != nil { + return fmt.Errorf("lakekeeper catalog db: failed to get primary instance: %w", err) + } + conn, err := primary.Connection(ctx, rc, r.DatabaseName) + if err != nil { + return fmt.Errorf("lakekeeper catalog db: failed to connect to database %s on node %s: %w", + r.DatabaseName, r.NodeName, err) + } + defer conn.Close(ctx) + + for _, stmt := range ensureCatalogDBStatements(r.CatalogDBName, r.CatalogDBOwner) { + if err := stmt.Exec(ctx, conn); err != nil { + return fmt.Errorf("lakekeeper catalog db %q: %w", r.CatalogDBName, err) + } + } + + // The extension pre-create must run connected to the catalog database + // itself, so open a second connection after create+owner above. + catConn, err := primary.Connection(ctx, rc, r.CatalogDBName) + if err != nil { + return fmt.Errorf("lakekeeper catalog db: failed to connect to catalog database %s: %w", + r.CatalogDBName, err) + } + defer catConn.Close(ctx) + for _, ext := range catalogDBExtensions() { + stmt := postgres.Statement{ + SQL: fmt.Sprintf("CREATE EXTENSION IF NOT EXISTS %s;", postgres.QuoteIdentifier(ext)), + } + if err := stmt.Exec(ctx, catConn); err != nil { + return fmt.Errorf("lakekeeper catalog db %q: create extension %s: %w", + r.CatalogDBName, ext, err) + } + } + return nil +} + +// catalogDBExtensions is the exact set of extensions Lakekeeper's v0.9.0 +// migrations require (CREATE EXTENSION IF NOT EXISTS). All four are TRUSTED on +// stock PG13+, so the owner could install them itself; ensure pre-creates them +// as the system user (belt-and-braces). This is the single source of truth for +// both the ensure step and its test. +func catalogDBExtensions() []string { + return []string{"uuid-ossp", "pgcrypto", "pg_trgm", "btree_gin"} +} + +// ensureCatalogDBStatements returns the idempotent statement sequence: +// create-if-absent, then align ownership so the Lakekeeper migrate step +// (which connects as the owner) can create its schema. +func ensureCatalogDBStatements(dbName, owner string) []postgres.IStatement { + return []postgres.IStatement{ + postgres.CreateDatabase(dbName), + postgres.Statement{ + SQL: fmt.Sprintf("ALTER DATABASE %s OWNER TO %s;", + postgres.QuoteIdentifier(dbName), postgres.QuoteIdentifier(owner)), + }, + } +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_catalog_db_resource_test.go b/server/internal/orchestrator/swarm/lakekeeper_catalog_db_resource_test.go new file mode 100644 index 00000000..2d61540e --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_catalog_db_resource_test.go @@ -0,0 +1,132 @@ +package swarm + +import ( + "context" + "net/url" + "strings" + "testing" + "unicode/utf8" + + "github.com/pgEdge/control-plane/server/internal/database" + "github.com/pgEdge/control-plane/server/internal/postgres" +) + +func TestLakekeeperCatalogDBName(t *testing.T) { + if got := lakekeeperCatalogDBName("mydb"); got != "mydb_lakekeeper" { + t.Fatalf("got %q, want %q", got, "mydb_lakekeeper") + } + // 63-byte identifier cap: a long ASCII name is truncated to exactly the + // limit, prefix trimmed and suffix intact. + long := lakekeeperCatalogDBName(strings.Repeat("a", 70)) + if want := strings.Repeat("a", 52) + "_lakekeeper"; long != want { + t.Fatalf("long name = %q (%d bytes), want %q (%d bytes)", long, len(long), want, len(want)) + } + + // Multi-byte input: truncation must land on a whole-rune boundary, never + // leaving a partial rune (which would be an invalid identifier on a UTF8 + // database). "€" is 3 bytes; 26 of them = 78 bytes, so the 52-byte cut + // falls inside the 18th rune. + multi := lakekeeperCatalogDBName(strings.Repeat("€", 26)) + if len(multi) > 63 { + t.Fatalf("multibyte catalog db name %d bytes, exceeds 63", len(multi)) + } + if !utf8.ValidString(multi) { + t.Fatalf("truncation produced invalid UTF-8: %q", multi) + } + if !strings.HasSuffix(multi, "_lakekeeper") { + t.Fatalf("truncated name lost its suffix: %q", multi) + } + // 17 whole "€" (51 bytes) is the largest that fits under the 52-byte + // prefix limit without splitting a rune. + if want := strings.Repeat("€", 17) + "_lakekeeper"; multi != want { + t.Fatalf("multibyte name = %q, want %q", multi, want) + } +} + +func TestBuildManagedCatalogDBURL(t *testing.T) { + got := buildManagedCatalogDBURL( + database.ServiceHostEntry{Host: "postgres-abc123", Port: 5432}, + "app_ro", "p@ss/word", "mydb_lakekeeper", + ) + want := "postgres://app_ro:p%40ss%2Fword@postgres-abc123:5432/mydb_lakekeeper?sslmode=prefer" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } + // The password contains reserved characters; confirm the URL round-trips + // so the escaping is correct, not merely string-equal to an expectation. + u, err := url.Parse(got) + if err != nil { + t.Fatalf("produced URL does not parse: %v", err) + } + pw, _ := u.User.Password() + if pw != "p@ss/word" { + t.Fatalf("password did not round-trip: got %q, want %q", pw, "p@ss/word") + } + if user := u.User.Username(); user != "app_ro" { + t.Fatalf("username did not round-trip: got %q, want %q", user, "app_ro") + } +} + +func TestEnsureCatalogDBStatements(t *testing.T) { + stmts := ensureCatalogDBStatements("mydb_lakekeeper", "app_ro") + if len(stmts) != 2 { + t.Fatalf("expected 2 statements, got %d", len(stmts)) + } + // First statement: the idempotent conditional CREATE DATABASE. + cond, ok := stmts[0].(postgres.ConditionalStatement) + if !ok { + t.Fatalf("statement 0 is %T, want ConditionalStatement", stmts[0]) + } + // ConditionalStatement.Then is an IStatement (interface) — assert to the + // concrete postgres.Statement before reading .SQL (it has no .SQL field + // through the interface). + then, ok := cond.Then.(postgres.Statement) + if !ok { + t.Fatalf("cond.Then is %T, want postgres.Statement", cond.Then) + } + if !strings.Contains(then.SQL, `CREATE DATABASE "mydb_lakekeeper"`) { + t.Fatalf("unexpected create SQL: %s", then.SQL) + } + // Second statement: owner alignment with quoted identifiers. + own, ok := stmts[1].(postgres.Statement) + if !ok { + t.Fatalf("statement 1 is %T, want Statement", stmts[1]) + } + want := `ALTER DATABASE "mydb_lakekeeper" OWNER TO "app_ro";` + if own.SQL != want { + t.Fatalf("owner SQL = %q, want %q", own.SQL, want) + } +} + +func TestLakekeeperCatalogDBResourceRefresh(t *testing.T) { + r := &LakekeeperCatalogDBResource{} + if err := r.Refresh(context.Background(), nil); err == nil { + t.Fatal("expected ErrNotFound before creation") + } + r.Created = true + if err := r.Refresh(context.Background(), nil); err != nil { + t.Fatalf("expected nil after creation, got %v", err) + } +} + +func TestCatalogDBExtensionNames(t *testing.T) { + // Pin the resource's actual extension list (the one ensure iterates), and + // the quoted-identifier form ("uuid-ossp" needs quoting — it has a hyphen). + // Asserting against catalogDBExtensions() rather than a local literal means + // reordering or editing the real list is caught here. + want := []string{ + `"uuid-ossp"`, + `"pgcrypto"`, + `"pg_trgm"`, + `"btree_gin"`, + } + exts := catalogDBExtensions() + if len(exts) != len(want) { + t.Fatalf("expected %d extensions, got %d", len(want), len(exts)) + } + for i, ext := range exts { + if got := postgres.QuoteIdentifier(ext); got != want[i] { + t.Fatalf("extension %d (%q) quoted = %q, want %q", i, ext, got, want[i]) + } + } +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_coldfront_extension_resource.go b/server/internal/orchestrator/swarm/lakekeeper_coldfront_extension_resource.go new file mode 100644 index 00000000..69c7a9a3 --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_coldfront_extension_resource.go @@ -0,0 +1,145 @@ +package swarm + +import ( + "context" + "fmt" + + "github.com/pgEdge/control-plane/server/internal/database" + "github.com/pgEdge/control-plane/server/internal/postgres" + "github.com/pgEdge/control-plane/server/internal/resource" +) + +var _ resource.Resource = (*LakekeeperColdfrontExtensionResource)(nil) + +const ResourceTypeLakekeeperColdfrontExtension resource.Type = "swarm.lakekeeper_coldfront_extension" + +func LakekeeperColdfrontExtensionResourceIdentifier(serviceInstanceID string) resource.Identifier { + return resource.Identifier{ + ID: serviceInstanceID, + Type: ResourceTypeLakekeeperColdfrontExtension, + } +} + +// LakekeeperColdfrontExtensionResource creates the ColdFront extension in the +// node's application database as part of lakekeeper provisioning. It runs +// idempotently against the node's primary (PrimaryExecutor) after the main +// database is available, and before the storage-secret step (which calls a +// coldfront function and therefore requires the extension). +// +// A lakekeeper service is non-functional without the extension, so this is +// unconditional — there is no meaningful "lakekeeper without coldfront" mode. +// It supersedes the previous assumption (documented on +// LakekeeperStorageSecretResource) that the extension was created elsewhere; +// nothing created it. +// +// pg_duckdb is created explicitly before coldfront (see +// coldfrontExtensionStatements) rather than relying on coldfront.control's +// `requires`, so the step is robust to how the extension is packaged. pg_duckdb +// must be present in shared_preload_libraries for CREATE EXTENSION to succeed; +// the lakekeeper node config adds it (#6b), +// applied at instance start. On a fresh deploy the instance boots with it +// loaded; when a lakekeeper service is added to an existing database, the +// restart that loads it is driven by the standard config-update path +// (PatroniConfig.Update -> restartIfNeeded), and CREATE EXTENSION IF NOT EXISTS +// is idempotent across reconciles. +// +// Delete is a deliberate no-op: dropping the coldfront extension would CASCADE +// to pg_duckdb and make every Iceberg table's cold data unreadable. The +// extension rides the node's lifecycle instead (matching the catalog DB). +type LakekeeperColdfrontExtensionResource struct { + ServiceInstanceID string `json:"service_instance_id"` + DatabaseID string `json:"database_id"` + // DatabaseName is the node's main (application) database — the connection + // target in which the extension is created. + DatabaseName string `json:"database_name"` + NodeName string `json:"node_name"` + Created bool `json:"created"` +} + +func (r *LakekeeperColdfrontExtensionResource) ResourceVersion() string { return "1" } + +func (r *LakekeeperColdfrontExtensionResource) DiffIgnore() []string { + return []string{"/created"} +} + +func (r *LakekeeperColdfrontExtensionResource) Identifier() resource.Identifier { + return LakekeeperColdfrontExtensionResourceIdentifier(r.ServiceInstanceID) +} + +func (r *LakekeeperColdfrontExtensionResource) Executor() resource.Executor { + return resource.PrimaryExecutor(r.NodeName) +} + +func (r *LakekeeperColdfrontExtensionResource) Dependencies() []resource.Identifier { + // The main database must exist before we can connect to it and create the + // extension. + return []resource.Identifier{ + database.PostgresDatabaseResourceIdentifier(r.NodeName, r.DatabaseName), + } +} + +func (r *LakekeeperColdfrontExtensionResource) TypeDependencies() []resource.Type { return nil } + +func (r *LakekeeperColdfrontExtensionResource) Refresh(ctx context.Context, rc *resource.Context) error { + if !r.Created { + return fmt.Errorf("%w: coldfront extension has not yet been created", resource.ErrNotFound) + } + return nil +} + +func (r *LakekeeperColdfrontExtensionResource) Create(ctx context.Context, rc *resource.Context) error { + if err := r.ensure(ctx, rc); err != nil { + return err + } + r.Created = true + return nil +} + +func (r *LakekeeperColdfrontExtensionResource) Update(ctx context.Context, rc *resource.Context) error { + // CREATE EXTENSION IF NOT EXISTS is idempotent, so re-running is safe. + if err := r.ensure(ctx, rc); err != nil { + return err + } + r.Created = true + return nil +} + +func (r *LakekeeperColdfrontExtensionResource) Delete(ctx context.Context, rc *resource.Context) error { + // Deliberate no-op — see type comment. + return nil +} + +func (r *LakekeeperColdfrontExtensionResource) ensure(ctx context.Context, rc *resource.Context) error { + primary, err := database.GetPrimaryInstance(ctx, rc, r.NodeName) + if err != nil { + return fmt.Errorf("coldfront extension: failed to get primary instance: %w", err) + } + conn, err := primary.Connection(ctx, rc, r.DatabaseName) + if err != nil { + return fmt.Errorf("coldfront extension: failed to connect to database %s on node %s: %w", + r.DatabaseName, r.NodeName, err) + } + defer conn.Close(ctx) + + for _, stmt := range coldfrontExtensionStatements() { + if err := stmt.Exec(ctx, conn); err != nil { + return fmt.Errorf("coldfront extension in database %q: %w", r.DatabaseName, err) + } + } + return nil +} + +// coldfrontExtensionStatements returns the idempotent statement sequence that +// creates the ColdFront extension in the application database. pg_duckdb is +// created explicitly first so this does not depend on coldfront.control declaring +// a `requires = 'pg_duckdb'` — the extension needs pg_duckdb present regardless, +// and creating it here is robust to how the extension is packaged. CASCADE is +// kept on the coldfront statement as belt-and-braces for any control-file +// requirement. Both are IF NOT EXISTS for idempotency. This is the single source +// of truth for both the ensure step and its test. +func coldfrontExtensionStatements() []postgres.IStatement { + return []postgres.IStatement{ + postgres.Statement{SQL: "CREATE EXTENSION IF NOT EXISTS pg_duckdb;"}, + postgres.Statement{SQL: "CREATE EXTENSION IF NOT EXISTS coldfront CASCADE;"}, + } +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_coldfront_extension_resource_test.go b/server/internal/orchestrator/swarm/lakekeeper_coldfront_extension_resource_test.go new file mode 100644 index 00000000..1e6da354 --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_coldfront_extension_resource_test.go @@ -0,0 +1,125 @@ +package swarm + +import ( + "context" + "testing" + + "github.com/pgEdge/control-plane/server/internal/database" + "github.com/pgEdge/control-plane/server/internal/postgres" + "github.com/pgEdge/control-plane/server/internal/resource" +) + +func TestColdfrontExtensionStatements(t *testing.T) { + stmts := coldfrontExtensionStatements() + // pg_duckdb MUST be created explicitly and first: the native-packages build + // dropped `requires = 'pg_duckdb'` from coldfront.control, so CASCADE alone + // no longer pulls it. Both are IF NOT EXISTS for idempotency. + want := []string{ + "CREATE EXTENSION IF NOT EXISTS pg_duckdb;", + "CREATE EXTENSION IF NOT EXISTS coldfront CASCADE;", + } + if len(stmts) != len(want) { + t.Fatalf("expected %d statements, got %d", len(want), len(stmts)) + } + for i, w := range want { + stmt, ok := stmts[i].(postgres.Statement) + if !ok { + t.Fatalf("statement %d is %T, want postgres.Statement", i, stmts[i]) + } + if stmt.SQL != w { + t.Fatalf("statement %d SQL = %q, want %q", i, stmt.SQL, w) + } + } +} + +func TestLakekeeperColdfrontExtensionResourceRefresh(t *testing.T) { + r := &LakekeeperColdfrontExtensionResource{} + if err := r.Refresh(context.Background(), nil); err == nil { + t.Fatal("expected ErrNotFound before creation") + } + r.Created = true + if err := r.Refresh(context.Background(), nil); err != nil { + t.Fatalf("expected nil after creation, got %v", err) + } +} + +func TestLakekeeperColdfrontExtensionResourceDependencies(t *testing.T) { + r := &LakekeeperColdfrontExtensionResource{ + NodeName: "n1", + DatabaseName: "mydb", + } + deps := r.Dependencies() + want := database.PostgresDatabaseResourceIdentifier("n1", "mydb") + if len(deps) != 1 || deps[0] != want { + t.Fatalf("Dependencies() = %v, want [%v]", deps, want) + } +} + +// TestGenerateLakekeeperInstanceResources_CreatesColdfrontExtension verifies the +// coldfront extension resource is generated for a lakekeeper service and that +// the storage-secret resource depends on it — closing the gap where the +// storage-secret step assumed the extension existed but nothing created it. +func TestGenerateLakekeeperInstanceResources_CreatesColdfrontExtension(t *testing.T) { + o := newLakekeeperTestOrchestrator(t) + + // Cover both catalog modes: the extension is needed regardless of whether + // the catalog DB is managed or external. + cases := map[string]*database.ServiceInstanceSpec{ + "managed": makeManagedLakekeeperSpec(), + "external": makeLakekeeperSpec("postgres://lakekeeper:secret@pg-host:5432/lakekeeper?sslmode=disable", "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA=="), + } + + for name, spec := range cases { + t.Run(name, func(t *testing.T) { + result, err := o.generateLakekeeperInstanceResources(spec) + if err != nil { + t.Fatalf("generateLakekeeperInstanceResources() unexpected error: %v", err) + } + + extRD := findResourceByType(result.Resources, ResourceTypeLakekeeperColdfrontExtension) + if extRD == nil { + t.Fatal("resource graph missing LakekeeperColdfrontExtensionResource") + } + extRes, err := resource.ToResource[*LakekeeperColdfrontExtensionResource](extRD) + if err != nil { + t.Fatalf("failed to decode LakekeeperColdfrontExtensionResource: %v", err) + } + if extRes.DatabaseName != spec.DatabaseName { + t.Errorf("ext DatabaseName = %q, want %q", extRes.DatabaseName, spec.DatabaseName) + } + if extRes.NodeName != spec.NodeName { + t.Errorf("ext NodeName = %q, want %q", extRes.NodeName, spec.NodeName) + } + + // The extension resource must depend on the app database existing. + dbID := database.PostgresDatabaseResourceIdentifier(spec.NodeName, spec.DatabaseName) + if deps := extRes.Dependencies(); len(deps) != 1 || deps[0] != dbID { + t.Errorf("ext Dependencies() = %v, want [%v]", deps, dbID) + } + + // The storage-secret resource must now depend on the extension + // resource, not merely the database — set_storage_secret calls a + // coldfront function that requires the extension. + secretRD := findResourceByType(result.Resources, ResourceTypeLakekeeperStorageSecret) + if secretRD == nil { + t.Fatal("resource graph missing LakekeeperStorageSecretResource") + } + secretRes, err := resource.ToResource[*LakekeeperStorageSecretResource](secretRD) + if err != nil { + t.Fatalf("failed to decode LakekeeperStorageSecretResource: %v", err) + } + extID := LakekeeperColdfrontExtensionResourceIdentifier(spec.ServiceInstanceID) + foundDep := false + for _, d := range secretRes.Dependencies() { + if d == extID { + foundDep = true + break + } + } + if !foundDep { + t.Errorf("storage-secret Dependencies() missing extension identifier %v; got: %v", + extID, secretRes.Dependencies()) + } + }) + } +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_config_resource.go b/server/internal/orchestrator/swarm/lakekeeper_config_resource.go new file mode 100644 index 00000000..3f4ba168 --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_config_resource.go @@ -0,0 +1,127 @@ +package swarm + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/samber/do" + "github.com/spf13/afero" + + "github.com/pgEdge/control-plane/server/internal/filesystem" + "github.com/pgEdge/control-plane/server/internal/orchestrator/common" + "github.com/pgEdge/control-plane/server/internal/resource" +) + +var _ resource.Resource = (*LakekeeperConfigResource)(nil) + +const ResourceTypeLakekeeperConfig resource.Type = "swarm.lakekeeper_config" + +// lakekeeperReadyFile is the sentinel written on first Create so that +// subsequent reconciliations skip re-running Create unnecessarily. +const lakekeeperReadyFile = ".lakekeeper_ready" + +func LakekeeperConfigResourceIdentifier(serviceInstanceID string) resource.Identifier { + return resource.Identifier{ + ID: serviceInstanceID, + Type: ResourceTypeLakekeeperConfig, + } +} + +// LakekeeperConfigResource manages Lakekeeper's data directory on the host +// filesystem. It follows the same pattern as MCPConfigResource: it ensures +// any host-side artefacts needed by the container are present before the +// container starts. +// +// Lakekeeper is configured entirely via environment variables (LAKEKEEPER__*), +// so Create and Update are lightweight — they just write a sentinel file so +// Refresh can confirm the resource has been applied. Schema migration against +// the external catalog Postgres is handled by LakekeeperMigrateResource, which +// runs the Lakekeeper image with the "migrate" subcommand before the "serve" +// container starts. +type LakekeeperConfigResource struct { + ServiceInstanceID string `json:"service_instance_id"` + ServiceID string `json:"service_id"` + HostID string `json:"host_id"` + DirResourceID string `json:"dir_resource_id"` +} + +func (r *LakekeeperConfigResource) ResourceVersion() string { + return "1" +} + +func (r *LakekeeperConfigResource) DiffIgnore() []string { + return nil +} + +func (r *LakekeeperConfigResource) Identifier() resource.Identifier { + return LakekeeperConfigResourceIdentifier(r.ServiceInstanceID) +} + +func (r *LakekeeperConfigResource) Executor() resource.Executor { + return resource.HostExecutor(r.HostID) +} + +func (r *LakekeeperConfigResource) Dependencies() []resource.Identifier { + return []resource.Identifier{ + filesystem.DirResourceIdentifier(r.DirResourceID), + } +} + +func (r *LakekeeperConfigResource) TypeDependencies() []resource.Type { + return nil +} + +func (r *LakekeeperConfigResource) Refresh(ctx context.Context, rc *resource.Context) error { + fs, err := do.Invoke[afero.Fs](rc.Injector) + if err != nil { + return err + } + + dirPath, err := filesystem.DirResourceFullPath(rc, r.DirResourceID) + if err != nil { + return fmt.Errorf("failed to get service data dir path: %w", err) + } + + // Check for the sentinel file; ErrNotFound here triggers Create. + _, err = common.ReadResourceFile(fs, filepath.Join(dirPath, lakekeeperReadyFile)) + if err != nil { + return fmt.Errorf("failed to read lakekeeper ready sentinel: %w", err) + } + + return nil +} + +func (r *LakekeeperConfigResource) Create(ctx context.Context, rc *resource.Context) error { + fs, err := do.Invoke[afero.Fs](rc.Injector) + if err != nil { + return err + } + + dirPath, err := filesystem.DirResourceFullPath(rc, r.DirResourceID) + if err != nil { + return fmt.Errorf("failed to get service data dir path: %w", err) + } + + // Write the sentinel file so subsequent Refresh calls succeed. + sentinelPath := filepath.Join(dirPath, lakekeeperReadyFile) + if err := afero.WriteFile(fs, sentinelPath, []byte("ok\n"), 0o600); err != nil { + return fmt.Errorf("failed to write lakekeeper ready sentinel: %w", err) + } + + return nil +} + +func (r *LakekeeperConfigResource) Update(ctx context.Context, rc *resource.Context) error { + // No config files to regenerate; Lakekeeper configuration is delivered + // via environment variables set in the container spec. Changes to + // LAKEKEEPER__* env vars force a Swarm service restart automatically + // because the ServiceInstanceSpec resource detects a diff in the desired + // TaskTemplate. + return nil +} + +func (r *LakekeeperConfigResource) Delete(ctx context.Context, rc *resource.Context) error { + // Cleanup is handled by the parent directory resource deletion. + return nil +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_managed_catalog_test.go b/server/internal/orchestrator/swarm/lakekeeper_managed_catalog_test.go new file mode 100644 index 00000000..d72f021e --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_managed_catalog_test.go @@ -0,0 +1,239 @@ +package swarm + +import ( + "strings" + "testing" + + "github.com/pgEdge/control-plane/server/internal/database" + "github.com/pgEdge/control-plane/server/internal/resource" +) + +// makeManagedLakekeeperSpec returns a ServiceInstanceSpec configured for a +// control-plane-managed catalog database (catalog_db_create: true, no +// catalog_db_url). It reuses the same fixture shape as makeLakekeeperSpec, +// adding the fields a managed catalog needs: DatabaseHosts and connect-as +// credentials. +func makeManagedLakekeeperSpec() *database.ServiceInstanceSpec { + spec := makeLakekeeperSpec("", "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==") + spec.ServiceSpec.Config["catalog_db_create"] = true + spec.DatabaseName = "mydb" + spec.NodeName = "n1" + spec.DatabaseHosts = []database.ServiceHostEntry{ + {Host: "postgres-inst1", Port: 5432}, + } + spec.ConnectAsUsername = "app_user" + spec.ConnectAsPassword = "app_password" + return spec +} + +// findResourceByType returns the first ResourceData in resources whose +// Identifier.Type matches t, or nil if none is found. +func findResourceByType(resources []*resource.ResourceData, t resource.Type) *resource.ResourceData { + for _, rd := range resources { + if rd.Identifier.Type == t { + return rd + } + } + return nil +} + +// TestGenerateLakekeeperInstanceResources_ManagedCatalog verifies that when +// catalog_db_create is set, the orchestrator provisions a +// LakekeeperCatalogDBResource, builds the catalog URL itself, threads the +// injected config into every downstream consumer, and wires the migrate +// resource's dependency on the catalog resource. +func TestGenerateLakekeeperInstanceResources_ManagedCatalog(t *testing.T) { + o := newLakekeeperTestOrchestrator(t) + spec := makeManagedLakekeeperSpec() + + result, err := o.generateLakekeeperInstanceResources(spec) + if err != nil { + t.Fatalf("generateLakekeeperInstanceResources() unexpected error: %v", err) + } + + wantCatalogDBName := "mydb_lakekeeper" + wantCatalogDBURL := buildManagedCatalogDBURL( + spec.DatabaseHosts[0], spec.ConnectAsUsername, spec.ConnectAsPassword, wantCatalogDBName) + + // The graph must contain a LakekeeperCatalogDBResource with the derived + // name/owner. + catalogRD := findResourceByType(result.Resources, ResourceTypeLakekeeperCatalogDB) + if catalogRD == nil { + t.Fatal("resource graph missing LakekeeperCatalogDBResource") + } + catalogRes, err := resource.ToResource[*LakekeeperCatalogDBResource](catalogRD) + if err != nil { + t.Fatalf("failed to decode LakekeeperCatalogDBResource: %v", err) + } + if catalogRes.CatalogDBName != wantCatalogDBName { + t.Errorf("CatalogDBName = %q, want %q", catalogRes.CatalogDBName, wantCatalogDBName) + } + if catalogRes.CatalogDBOwner != spec.ConnectAsUsername { + t.Errorf("CatalogDBOwner = %q, want %q", catalogRes.CatalogDBOwner, spec.ConnectAsUsername) + } + + // The migrate resource must carry the built URL and CatalogDBManaged=true, + // and its Dependencies() must include the catalog resource identifier. + migrateRD := findResourceByType(result.Resources, ResourceTypeLakekeeperMigrate) + if migrateRD == nil { + t.Fatal("resource graph missing LakekeeperMigrateResource") + } + migrateRes, err := resource.ToResource[*LakekeeperMigrateResource](migrateRD) + if err != nil { + t.Fatalf("failed to decode LakekeeperMigrateResource: %v", err) + } + if migrateRes.CatalogDBURL != wantCatalogDBURL { + t.Errorf("CatalogDBURL = %q, want %q", migrateRes.CatalogDBURL, wantCatalogDBURL) + } + if !migrateRes.CatalogDBManaged { + t.Error("CatalogDBManaged = false, want true") + } + + catalogID := LakekeeperCatalogDBResourceIdentifier(spec.ServiceInstanceID) + foundDep := false + for _, d := range migrateRes.Dependencies() { + if d == catalogID { + foundDep = true + break + } + } + if !foundDep { + t.Errorf("migrate resource Dependencies() missing catalog identifier %v; got: %v", + catalogID, migrateRes.Dependencies()) + } + + // The ServiceInstanceSpecResource's config copy must carry the built + // catalog_db_url too, proving the serve container env gets the derived + // URL (not the caller-supplied one, since there wasn't one). + specRD := findResourceByType(result.Resources, ResourceTypeServiceInstanceSpec) + if specRD == nil { + t.Fatal("resource graph missing ServiceInstanceSpecResource") + } + specRes, err := resource.ToResource[*ServiceInstanceSpecResource](specRD) + if err != nil { + t.Fatalf("failed to decode ServiceInstanceSpecResource: %v", err) + } + gotURL, _ := specRes.ServiceSpec.Config["catalog_db_url"].(string) + if gotURL != wantCatalogDBURL { + t.Errorf("ServiceInstanceSpecResource.ServiceSpec.Config[catalog_db_url] = %q, want %q", + gotURL, wantCatalogDBURL) + } + + // The bootstrap and storage-secret resources alias the same service Config + // map; the brief names all four consumers as load-bearing, so pin that both + // observe the injected catalog_db_url. Guards against a future regression + // pointing either back at the un-injected spec.ServiceSpec.Config. + bootstrapRD := findResourceByType(result.Resources, ResourceTypeLakekeeperBootstrap) + if bootstrapRD == nil { + t.Fatal("resource graph missing LakekeeperBootstrapResource") + } + bootstrapRes, err := resource.ToResource[*LakekeeperBootstrapResource](bootstrapRD) + if err != nil { + t.Fatalf("failed to decode LakekeeperBootstrapResource: %v", err) + } + if got, _ := bootstrapRes.Config["catalog_db_url"].(string); got != wantCatalogDBURL { + t.Errorf("LakekeeperBootstrapResource.Config[catalog_db_url] = %q, want %q", got, wantCatalogDBURL) + } + + secretRD := findResourceByType(result.Resources, ResourceTypeLakekeeperStorageSecret) + if secretRD == nil { + t.Fatal("resource graph missing LakekeeperStorageSecretResource") + } + secretRes, err := resource.ToResource[*LakekeeperStorageSecretResource](secretRD) + if err != nil { + t.Fatalf("failed to decode LakekeeperStorageSecretResource: %v", err) + } + if got, _ := secretRes.Config["catalog_db_url"].(string); got != wantCatalogDBURL { + t.Errorf("LakekeeperStorageSecretResource.Config[catalog_db_url] = %q, want %q", got, wantCatalogDBURL) + } +} + +// TestGenerateLakekeeperInstanceResources_ExternalCatalogRegression verifies +// that supplying an external catalog_db_url (no catalog_db_create) leaves +// behavior byte-identical to the pre-managed-catalog code path: no catalog +// resource is generated and the migrate resource has no dependencies. +func TestGenerateLakekeeperInstanceResources_ExternalCatalogRegression(t *testing.T) { + o := newLakekeeperTestOrchestrator(t) + const externalURL = "postgres://lakekeeper:secret@pg-host:5432/lakekeeper?sslmode=disable" + spec := makeLakekeeperSpec( + externalURL, + "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + ) + + result, err := o.generateLakekeeperInstanceResources(spec) + if err != nil { + t.Fatalf("generateLakekeeperInstanceResources() unexpected error: %v", err) + } + + if rd := findResourceByType(result.Resources, ResourceTypeLakekeeperCatalogDB); rd != nil { + t.Errorf("resource graph should not contain a LakekeeperCatalogDBResource in external mode; got: %+v", rd) + } + + migrateRD := findResourceByType(result.Resources, ResourceTypeLakekeeperMigrate) + if migrateRD == nil { + t.Fatal("resource graph missing LakekeeperMigrateResource") + } + migrateRes, err := resource.ToResource[*LakekeeperMigrateResource](migrateRD) + if err != nil { + t.Fatalf("failed to decode LakekeeperMigrateResource: %v", err) + } + if migrateRes.CatalogDBManaged { + t.Error("CatalogDBManaged = true, want false in external mode") + } + if deps := migrateRes.Dependencies(); len(deps) != 0 { + t.Errorf("migrate resource Dependencies() = %v, want empty in external mode", deps) + } + // Positive assertion: the caller-supplied URL must pass through unchanged + // (control-plane builds nothing in external mode), both to the migrate + // resource and to the serve container's config. + if migrateRes.CatalogDBURL != externalURL { + t.Errorf("migrate CatalogDBURL = %q, want caller URL %q", migrateRes.CatalogDBURL, externalURL) + } + specRD := findResourceByType(result.Resources, ResourceTypeServiceInstanceSpec) + if specRD == nil { + t.Fatal("resource graph missing ServiceInstanceSpecResource") + } + specRes, err := resource.ToResource[*ServiceInstanceSpecResource](specRD) + if err != nil { + t.Fatalf("failed to decode ServiceInstanceSpecResource: %v", err) + } + if got, _ := specRes.ServiceSpec.Config["catalog_db_url"].(string); got != externalURL { + t.Errorf("ServiceInstanceSpecResource config catalog_db_url = %q, want caller URL %q", got, externalURL) + } +} + +// TestGenerateLakekeeperInstanceResources_ManagedCatalogNoHosts verifies that +// catalog_db_create without any DatabaseHosts fails loudly, naming the +// service ID, rather than producing a resource graph with an empty catalog URL. +func TestGenerateLakekeeperInstanceResources_ManagedCatalogNoHosts(t *testing.T) { + o := newLakekeeperTestOrchestrator(t) + spec := makeManagedLakekeeperSpec() + spec.DatabaseHosts = nil + + _, err := o.generateLakekeeperInstanceResources(spec) + if err == nil { + t.Fatal("expected error for catalog_db_create with no database hosts, got nil") + } + if !strings.Contains(err.Error(), spec.ServiceSpec.ServiceID) { + t.Errorf("error should name the service ID %q, got: %v", spec.ServiceSpec.ServiceID, err) + } +} + +// TestGenerateLakekeeperInstanceResources_FailLoudRegression verifies that +// omitting both catalog_db_url and catalog_db_create still errors, and that +// the updated message documents both the external and managed paths. +func TestGenerateLakekeeperInstanceResources_FailLoudRegression(t *testing.T) { + o := newLakekeeperTestOrchestrator(t) + spec := makeLakekeeperSpec("", "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==") + + _, err := o.generateLakekeeperInstanceResources(spec) + if err == nil { + t.Fatal("expected error for missing catalog_db_url and catalog_db_create, got nil") + } + if !strings.Contains(err.Error(), "catalog_db_url") { + t.Errorf("error should mention catalog_db_url, got: %v", err) + } + if !strings.Contains(err.Error(), "catalog_db_create") { + t.Errorf("error should mention catalog_db_create, got: %v", err) + } +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_migrate_resource.go b/server/internal/orchestrator/swarm/lakekeeper_migrate_resource.go new file mode 100644 index 00000000..b6275e7c --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_migrate_resource.go @@ -0,0 +1,206 @@ +package swarm + +import ( + "bytes" + "context" + "fmt" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" + "github.com/samber/do" + + "github.com/pgEdge/control-plane/server/internal/docker" + "github.com/pgEdge/control-plane/server/internal/resource" +) + +var _ resource.Resource = (*LakekeeperMigrateResource)(nil) + +const ResourceTypeLakekeeperMigrate resource.Type = "swarm.lakekeeper_migrate" + +// lakekeeperMigrateTimeout is the maximum time allowed for the schema migration +// to complete. Migrations run once on the external catalog Postgres and are +// expected to be fast, but a generous timeout is used to tolerate slow cold +// starts or transient network delays. +const lakekeeperMigrateTimeout = 5 * time.Minute + +func LakekeeperMigrateResourceIdentifier(serviceInstanceID string) resource.Identifier { + return resource.Identifier{ + ID: serviceInstanceID, + Type: ResourceTypeLakekeeperMigrate, + } +} + +// LakekeeperMigrateResource runs the Lakekeeper image with the "migrate" +// subcommand as a one-shot Docker container before the "serve" container is +// started. It applies the Iceberg catalog schema to the external Postgres +// database supplied via LAKEKEEPER__PG_DATABASE_URL_{READ,WRITE}. +// +// The resource is idempotent: Lakekeeper's own migrate command is a no-op when +// the schema is already current, so Create and Update both re-run the +// migration safely on every reconciliation cycle. +// +// Lifecycle: +// - Refresh: returns ErrNotFound until migration has completed at least once +// successfully (detected via the sentinel stored in MigratedOnce). This +// causes the resource engine to call Create on the first pass and then +// re-validate on subsequent passes. +// - Create/Update: run the migrate container and wait for exit-0. +// - Delete: no-op; deleting the service removes the catalog DB connection. +type LakekeeperMigrateResource struct { + ServiceInstanceID string `json:"service_instance_id"` + HostID string `json:"host_id"` + Image string `json:"image"` + CatalogDBURL string `json:"catalog_db_url"` + PGEncryptionKey string `json:"pg_encryption_key"` + // DatabaseNetworkName is the per-database overlay the one-shot migrate + // container joins so it can resolve the postgres- alias in + // CatalogDBURL. The overlay is created attachable for this reason. + DatabaseNetworkName string `json:"database_network_name"` + // CatalogDBManaged marks the catalog database as control-plane + // managed (spec key catalog_db_create): migration must wait for the + // LakekeeperCatalogDBResource to create it. + CatalogDBManaged bool `json:"catalog_db_managed"` + // MigratedOnce is set to true after the first successful migration so that + // Refresh can distinguish "never run" from "already applied". + MigratedOnce bool `json:"migrated_once"` +} + +func (r *LakekeeperMigrateResource) ResourceVersion() string { return "1" } + +func (r *LakekeeperMigrateResource) DiffIgnore() []string { + // MigratedOnce is runtime state written by Create; exclude it from diff + // comparisons so that a completed migration does not trigger spurious updates. + return []string{"/migrated_once"} +} + +func (r *LakekeeperMigrateResource) Identifier() resource.Identifier { + return LakekeeperMigrateResourceIdentifier(r.ServiceInstanceID) +} + +func (r *LakekeeperMigrateResource) Executor() resource.Executor { + // The migration container is run on the same host as the serve container + // so that it shares the same Docker daemon and network context. + return resource.HostExecutor(r.HostID) +} + +func (r *LakekeeperMigrateResource) Dependencies() []resource.Identifier { + // When control-plane manages the catalog database, migration must run + // after it exists. For an external catalog there is no resource to + // depend on; the URL is validated at spec time (fail-loud check). + if r.CatalogDBManaged { + return []resource.Identifier{ + LakekeeperCatalogDBResourceIdentifier(r.ServiceInstanceID), + } + } + return nil +} + +func (r *LakekeeperMigrateResource) TypeDependencies() []resource.Type { + return nil +} + +// Refresh returns ErrNotFound when migration has never successfully completed, +// causing the resource engine to call Create. Once MigratedOnce is true the +// resource is considered up-to-date and no further action is taken unless the +// desired state changes (which triggers Update). +func (r *LakekeeperMigrateResource) Refresh(ctx context.Context, rc *resource.Context) error { + if !r.MigratedOnce { + return fmt.Errorf("%w: lakekeeper schema migration has not yet run", resource.ErrNotFound) + } + return nil +} + +func (r *LakekeeperMigrateResource) Create(ctx context.Context, rc *resource.Context) error { + if err := r.runMigrate(ctx, rc); err != nil { + return err + } + r.MigratedOnce = true + return nil +} + +func (r *LakekeeperMigrateResource) Update(ctx context.Context, rc *resource.Context) error { + // Re-running migrate is always safe: Lakekeeper's migrate is idempotent. + if err := r.runMigrate(ctx, rc); err != nil { + return err + } + r.MigratedOnce = true + return nil +} + +func (r *LakekeeperMigrateResource) Delete(ctx context.Context, rc *resource.Context) error { + return nil +} + +// runMigrate starts a one-shot Docker container using the Lakekeeper image +// with the "migrate" subcommand, waits for it to exit with status 0, and +// streams the logs to the resource context logger on failure. +func (r *LakekeeperMigrateResource) runMigrate(ctx context.Context, rc *resource.Context) error { + client, err := do.Invoke[*docker.Docker](rc.Injector) + if err != nil { + return fmt.Errorf("lakekeeper migrate: failed to get docker client: %w", err) + } + + containerName := "lakekeeper-migrate-" + r.ServiceInstanceID + + // Remove any leftover container from a previous failed attempt before + // creating a new one, so the name does not conflict. + _ = client.ContainerRemove(ctx, containerName, container.RemoveOptions{Force: true}) + + // In managed-catalog mode the catalog Postgres is only reachable via the + // per-database overlay (its DSN host is the postgres- alias), so + // the one-shot migrate container must join that (attachable) overlay. In + // external-catalog mode the container keeps default networking so it can + // reach an arbitrary external host. + var netCfg *network.NetworkingConfig + if r.CatalogDBManaged && r.DatabaseNetworkName != "" { + netCfg = &network.NetworkingConfig{ + EndpointsConfig: map[string]*network.EndpointSettings{ + r.DatabaseNetworkName: {}, + }, + } + } + + containerID, err := client.ContainerRun(ctx, docker.ContainerRunOptions{ + Config: &container.Config{ + Image: r.Image, + Cmd: []string{"migrate"}, + Env: []string{ + "LAKEKEEPER__PG_DATABASE_URL_READ=" + r.CatalogDBURL, + "LAKEKEEPER__PG_DATABASE_URL_WRITE=" + r.CatalogDBURL, + "LAKEKEEPER__PG_ENCRYPTION_KEY=" + r.PGEncryptionKey, + }, + }, + Host: &container.HostConfig{ + // AutoRemove is intentionally false so that we can stream logs on + // failure before the container is cleaned up. + AutoRemove: false, + }, + Net: netCfg, + Name: containerName, + }) + if err != nil { + return fmt.Errorf("lakekeeper migrate: failed to start container: %w", err) + } + + // Wait for the migration to complete. + waitErr := client.ContainerWait(ctx, containerID, container.WaitConditionNotRunning, lakekeeperMigrateTimeout) + + // Always remove the container once we have the exit status (or on error). + defer func() { + removeCtx := context.Background() + _ = client.ContainerRemove(removeCtx, containerID, container.RemoveOptions{Force: true}) + }() + + if waitErr != nil { + // Capture logs to help diagnose the failure. + var logBuf bytes.Buffer + _ = client.ContainerLogs(ctx, &logBuf, containerID, container.LogsOptions{ + ShowStdout: true, + ShowStderr: true, + }) + return fmt.Errorf("lakekeeper migrate: migration failed: %w\nlogs:\n%s", waitErr, logBuf.String()) + } + + return nil +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_storage_secret_resource.go b/server/internal/orchestrator/swarm/lakekeeper_storage_secret_resource.go new file mode 100644 index 00000000..5d961658 --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_storage_secret_resource.go @@ -0,0 +1,305 @@ +package swarm + +import ( + "context" + "fmt" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "github.com/pgEdge/control-plane/server/internal/database" + "github.com/pgEdge/control-plane/server/internal/postgres" + "github.com/pgEdge/control-plane/server/internal/resource" +) + +var _ resource.Resource = (*LakekeeperStorageSecretResource)(nil) + +const ResourceTypeLakekeeperStorageSecret resource.Type = "swarm.lakekeeper_storage_secret" + +func LakekeeperStorageSecretResourceIdentifier(serviceInstanceID string) resource.Identifier { + return resource.Identifier{ + ID: serviceInstanceID, + Type: ResourceTypeLakekeeperStorageSecret, + } +} + +// LakekeeperStorageSecretResource configures the ColdFront extension in the +// node's application database: it sets the per-database GUCs the extension's +// attach path reads (coldfront.warehouse / coldfront.lakekeeper_endpoint / +// coldfront.local_pg_dsn) and stores the object-store credential via ColdFront's +// set_storage_secret function, so that the extension can read/write Iceberg data +// in the warehouse's bucket. +// +// It runs once against the node's primary Postgres (PrimaryExecutor, like the +// PostgREST preflight resource) after the coldfront extension is available. It +// depends on the coldfront extension resource for that ordering. +// +// The GUCs are PGC_SUSET, so ALTER DATABASE ... SET by the superuser applies to +// every new session with no restart; the extension re-reads them per statement. +// The set_storage_secret functions upsert and the GUC sets are idempotent, so +// re-running is safe; SecretSet is a sentinel so Refresh can distinguish "never +// run" from "already applied". +// +// The credential is passed to Postgres exclusively as bound query parameters +// and is NEVER logged. +type LakekeeperStorageSecretResource struct { + ServiceInstanceID string `json:"service_instance_id"` + DatabaseID string `json:"database_id"` + DatabaseName string `json:"database_name"` + NodeName string `json:"node_name"` + Config map[string]any `json:"config"` + // LakekeeperEndpoint is the Iceberg REST catalog URL for coldfront's ATTACH, + // including the trailing /catalog path segment the extension requires. Built + // by the orchestrator from the generated service name. + LakekeeperEndpoint string `json:"lakekeeper_endpoint"` + // ConnectAsUsername is the database owner; the coldfront.local_pg_dsn GUC + // connects back to the local database as this user. + ConnectAsUsername string `json:"connect_as_username"` + SecretSet bool `json:"secret_set"` +} + +func (r *LakekeeperStorageSecretResource) ResourceVersion() string { return "1" } + +func (r *LakekeeperStorageSecretResource) DiffIgnore() []string { + return []string{"/secret_set"} +} + +func (r *LakekeeperStorageSecretResource) Identifier() resource.Identifier { + return LakekeeperStorageSecretResourceIdentifier(r.ServiceInstanceID) +} + +func (r *LakekeeperStorageSecretResource) Executor() resource.Executor { + return resource.PrimaryExecutor(r.NodeName) +} + +func (r *LakekeeperStorageSecretResource) Dependencies() []resource.Identifier { + // Depend on the coldfront extension resource so set_storage_secret (a + // coldfront function) has the extension available. That resource depends on + // the database in turn, so this transitively orders after the database too. + return []resource.Identifier{ + LakekeeperColdfrontExtensionResourceIdentifier(r.ServiceInstanceID), + } +} + +func (r *LakekeeperStorageSecretResource) TypeDependencies() []resource.Type { + return nil +} + +func (r *LakekeeperStorageSecretResource) Refresh(ctx context.Context, rc *resource.Context) error { + if !r.SecretSet { + return fmt.Errorf("%w: coldfront storage secret has not yet been set", resource.ErrNotFound) + } + return nil +} + +func (r *LakekeeperStorageSecretResource) Create(ctx context.Context, rc *resource.Context) error { + if err := r.setSecret(ctx, rc); err != nil { + return err + } + r.SecretSet = true + return nil +} + +func (r *LakekeeperStorageSecretResource) Update(ctx context.Context, rc *resource.Context) error { + // set_storage_secret upserts, so re-running is safe. + if err := r.setSecret(ctx, rc); err != nil { + return err + } + r.SecretSet = true + return nil +} + +func (r *LakekeeperStorageSecretResource) Delete(ctx context.Context, rc *resource.Context) error { + return nil +} + +func (r *LakekeeperStorageSecretResource) setSecret(ctx context.Context, rc *resource.Context) error { + cfg, err := parseLakekeeperStorageConfig(r.Config) + if err != nil { + return err + } + + primary, err := database.GetPrimaryInstance(ctx, rc, r.NodeName) + if err != nil { + return fmt.Errorf("coldfront set_storage_secret: failed to get primary instance: %w", err) + } + conn, err := primary.Connection(ctx, rc, r.DatabaseName) + if err != nil { + return fmt.Errorf("coldfront set_storage_secret: failed to connect to database %s on node %s: %w", + r.DatabaseName, r.NodeName, err) + } + defer conn.Close(ctx) + + // Set the per-database coldfront GUCs the extension's attach path reads. + // These are ALTER DATABASE ... SET (DDL, no bound parameters), applied + // before the storage secret so a subsequent session has both in place. + dsn := buildColdfrontLocalPGDSN(r.DatabaseName, r.ConnectAsUsername) + for _, stmt := range buildColdfrontGUCStatements(r.DatabaseName, cfg.Warehouse, r.LakekeeperEndpoint, dsn) { + if err := stmt.Exec(ctx, conn); err != nil { + return fmt.Errorf("coldfront set guc in database %q: %w", r.DatabaseName, err) + } + } + + return execSetStorageSecret(ctx, conn, cfg) +} + +// buildColdfrontLocalPGDSN builds the libpq keyword DSN for the +// coldfront.local_pg_dsn GUC — the loopback connection DuckDB attaches as +// `pglocal` to stream PG-source rows into Iceberg on the write path. Format +// mirrors the tiering DSN (finding #8): a localhost TCP connection as the +// database owner. The sslmode=disable + local-trust assumption is the same +// tracked residual as #8, to revisit with the connectivity/auth work. +func buildColdfrontLocalPGDSN(dbName, user string) string { + return fmt.Sprintf("host=localhost port=5432 user=%s dbname=%s sslmode=disable", user, dbName) +} + +// buildColdfrontGUCStatements returns the ALTER DATABASE ... SET statements for +// the three per-database GUCs the coldfront extension requires. ALTER DATABASE +// SET is DDL and does not accept bound parameters, so values are single-quoted +// with embedded quotes doubled — safe with standard_conforming_strings=on +// (matching the roles.go password-quoting precedent). The database name is +// quoted as an identifier; the GUC names are fixed, trusted literals. +func buildColdfrontGUCStatements(dbName, warehouse, lakekeeperEndpoint, localPGDSN string) []postgres.Statement { + quotedDB := postgres.QuoteIdentifier(dbName) + set := func(guc, value string) postgres.Statement { + return postgres.Statement{ + SQL: fmt.Sprintf("ALTER DATABASE %s SET %s = %s;", + quotedDB, guc, quoteColdfrontGUCLiteral(value)), + } + } + return []postgres.Statement{ + set("coldfront.warehouse", warehouse), + set("coldfront.lakekeeper_endpoint", lakekeeperEndpoint), + set("coldfront.local_pg_dsn", localPGDSN), + } +} + +// quoteColdfrontGUCLiteral wraps a value as a SQL string literal, doubling any +// embedded single quotes. Safe with standard_conforming_strings=on. +func quoteColdfrontGUCLiteral(value string) string { + return "'" + strings.ReplaceAll(value, "'", "''") + "'" +} + +// setStorageSecretExec is the minimal Postgres exec surface needed by +// execSetStorageSecret, satisfied by *pgx.Conn. It lets the SQL selection be +// unit-tested without a live database. +type setStorageSecretExec interface { + Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) +} + +// execSetStorageSecret builds and executes the correct ColdFront function call +// for the configured provider. The credential values are bound as query +// parameters ($1, $2, ...) and never interpolated into the SQL text, so they +// cannot leak via query logging. +func execSetStorageSecret(ctx context.Context, conn setStorageSecretExec, cfg *lakekeeperStorageConfig) error { + sql, args := buildSetStorageSecretSQL(cfg) + if _, err := conn.Exec(ctx, sql, args...); err != nil { + // The error is from Postgres and does not echo bound parameters, so it + // is safe to wrap; do NOT include the credential. + return fmt.Errorf("coldfront set_storage_secret failed for provider %q: %w", cfg.Provider, err) + } + return nil +} + +// buildSetStorageSecretSQL returns the SQL and bound arguments for the +// provider. aws/gcs use coldfront.set_storage_secret; azure uses +// coldfront.set_storage_secret_azure. Credentials are returned as args, never +// embedded in the SQL string. +// +// Endpoint presence is the discriminator, consistent with the warehouse +// storage profile: +// - endpoint ABSENT (cloud AWS): pass endpoint => NULL, which selects +// DuckDB's native per-Region virtual-hosted + HTTPS addressing (required +// for Regions launched after 2019). url_style/use_ssl are left at their +// defaults — they are irrelevant when the endpoint is NULL, and passing a +// non-NULL endpoint here forces path-style and breaks modern Regions with +// HTTP 400 (ColdFront docs/object_store.md). +// - endpoint PRESENT (S3-compatible / GCS): pass the endpoint with +// url_style => 'path' and use_ssl derived from the endpoint scheme +// (https:// => true, http:// => false, schemeless => true). The scheme is +// stripped so the DuckDB secret stores host:port, not a URL. +func buildSetStorageSecretSQL(cfg *lakekeeperStorageConfig) (string, []any) { + switch cfg.Provider { + case "aws": + return buildS3SetStorageSecretSQL( + cfg.Credential["access_key_id"], + cfg.Credential["secret_access_key"], + cfg.Endpoint, + cfg.Region, + ) + case "gcs": + // GCS is only reachable through its S3-compatible HMAC endpoint, so it + // always has an endpoint (default to the canonical host when unset). + endpoint := cfg.Endpoint + if endpoint == "" { + endpoint = "storage.googleapis.com" + } + return buildS3SetStorageSecretSQL( + cfg.Credential["hmac_access_id"], + cfg.Credential["hmac_secret"], + endpoint, + cfg.Region, + ) + case "azure": + return `SELECT coldfront.set_storage_secret_azure(p_connection_string => $1)`, + []any{cfg.Credential["connection_string"]} + default: + // parseLakekeeperStorageConfig has already validated the provider, so + // this branch is unreachable in practice. + return "", nil + } +} + +// buildS3SetStorageSecretSQL builds the set_storage_secret call for an S3 or +// S3-compatible store, driving cloud-vs-s3-compat semantics off endpoint +// presence. keyID/secret are always bound as parameters. +func buildS3SetStorageSecretSQL(keyID, secret, endpoint, region string) (string, []any) { + if endpoint == "" { + // Cloud AWS: endpoint NULL selects native vhost + HTTPS. Do not pass + // url_style/use_ssl — defaults apply and a non-NULL endpoint here is + // exactly the shape the docs warn against. + return `SELECT coldfront.set_storage_secret( + p_key_id => $1, + p_secret => $2, + p_endpoint => NULL, + p_region => $3 + )`, []any{keyID, secret, region} + } + // S3-compatible / GCS: explicit endpoint with path-style. The DuckDB secret + // wants a bare host:port, and use_ssl is driven off the endpoint scheme so + // plain-HTTP stores (MinIO / self-hosted) work, not just HTTPS. + host, useSSL := deriveEndpointSSL(endpoint) + return `SELECT coldfront.set_storage_secret( + p_key_id => $1, + p_secret => $2, + p_endpoint => $3, + p_region => $4, + p_url_style => $5, + p_use_ssl => $6 + )`, []any{keyID, secret, host, region, "path", useSSL} +} + +// deriveEndpointSSL splits an S3 endpoint into its bare host:port form and the +// use_ssl flag implied by its scheme. An https:// endpoint => TLS; an http:// +// endpoint => plaintext; a schemeless endpoint keeps its value and defaults to +// TLS (the safe assumption for a public host such as storage.googleapis.com). +// Surrounding whitespace and a trailing slash are trimmed, and the scheme match +// is case-insensitive, so copy-pasted values like " HTTPS://minio:9000/ " yield +// a clean host:port that DuckDB accepts. +func deriveEndpointSSL(endpoint string) (host string, useSSL bool) { + endpoint = strings.TrimSpace(endpoint) + lower := strings.ToLower(endpoint) + switch { + case strings.HasPrefix(lower, "https://"): + host, useSSL = endpoint[len("https://"):], true + case strings.HasPrefix(lower, "http://"): + host, useSSL = endpoint[len("http://"):], false + default: + host, useSSL = endpoint, true + } + return strings.TrimRight(host, "/"), useSSL +} + +// ensure *pgx.Conn satisfies setStorageSecretExec at compile time. +var _ setStorageSecretExec = (*pgx.Conn)(nil) diff --git a/server/internal/orchestrator/swarm/lakekeeper_storage_secret_resource_test.go b/server/internal/orchestrator/swarm/lakekeeper_storage_secret_resource_test.go new file mode 100644 index 00000000..d1123852 --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_storage_secret_resource_test.go @@ -0,0 +1,266 @@ +package swarm + +import ( + "context" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/pgEdge/control-plane/server/internal/resource" +) + +// fakeExec captures the SQL and args passed to Exec so the test can assert the +// correct function is chosen and the credential is bound (not interpolated). +type fakeExec struct { + sql string + args []any +} + +func (f *fakeExec) Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) { + f.sql = sql + f.args = args + return pgconn.CommandTag{}, nil +} + +// TestBuildSetStorageSecretSQL_CloudAWS: no endpoint => endpoint NULL (native +// vhost+HTTPS), no url_style/use_ssl. Passing a non-NULL endpoint here is the +// shape the docs warn breaks modern Regions with HTTP 400. +func TestBuildSetStorageSecretSQL_CloudAWS(t *testing.T) { + cfg := &lakekeeperStorageConfig{ + Provider: "aws", + Region: "us-east-1", + Credential: map[string]string{ + "access_key_id": "AKIA_TEST", + "secret_access_key": "SECRET_TEST", + }, + } + sql, args := buildSetStorageSecretSQL(cfg) + + assert.Contains(t, sql, "coldfront.set_storage_secret(") + assert.NotContains(t, sql, "set_storage_secret_azure") + // endpoint => NULL literal in SQL; url_style/use_ssl omitted entirely. + assert.Contains(t, sql, "p_endpoint => NULL") + assert.NotContains(t, sql, "p_url_style") + assert.NotContains(t, sql, "p_use_ssl") + // key/secret/region bound as args; no endpoint arg. + assert.Equal(t, []any{"AKIA_TEST", "SECRET_TEST", "us-east-1"}, args) + // Credential must NOT be interpolated into the SQL text. + assert.NotContains(t, sql, "AKIA_TEST") + assert.NotContains(t, sql, "SECRET_TEST") +} + +// TestBuildSetStorageSecretSQL_S3CompatAWS: endpoint present => pass the bare +// host:port (scheme stripped) + path-style, with use_ssl derived from the +// scheme (http:// => false here). +func TestBuildSetStorageSecretSQL_S3CompatAWS(t *testing.T) { + cfg := &lakekeeperStorageConfig{ + Provider: "aws", + Region: "us-east-1", + Endpoint: "http://seaweedfs:8333", + Credential: map[string]string{ + "access_key_id": "AKIA_TEST", + "secret_access_key": "SECRET_TEST", + }, + } + sql, args := buildSetStorageSecretSQL(cfg) + + assert.Contains(t, sql, "coldfront.set_storage_secret(") + assert.Contains(t, sql, "p_url_style => $5") + assert.NotContains(t, sql, "p_endpoint => NULL") + // endpoint present as a bound arg (scheme stripped), path-style, http=>no SSL. + assert.Equal(t, []any{"AKIA_TEST", "SECRET_TEST", "seaweedfs:8333", "us-east-1", "path", false}, args) + assert.NotContains(t, sql, "AKIA_TEST") +} + +// TestBuildSetStorageSecretSQL_GCS: GCS is always S3-compatible, so it uses an +// endpoint (defaulting to the canonical host) with path-style + SSL. +func TestBuildSetStorageSecretSQL_GCS(t *testing.T) { + cfg := &lakekeeperStorageConfig{ + Provider: "gcs", + Region: "us", + Credential: map[string]string{ + "hmac_access_id": "GOOG_ID", + "hmac_secret": "GOOG_SECRET", + }, + } + sql, args := buildSetStorageSecretSQL(cfg) + assert.Contains(t, sql, "coldfront.set_storage_secret(") + assert.NotContains(t, sql, "set_storage_secret_azure") + assert.NotContains(t, sql, "p_endpoint => NULL") + // gcs defaults endpoint to the GCS S3-compatible host and uses path-style. + assert.Contains(t, args, "storage.googleapis.com") + assert.Contains(t, args, "path") + assert.Contains(t, args, "GOOG_ID") + assert.NotContains(t, sql, "GOOG_SECRET") +} + +// TestBuildSetStorageSecretSQL_HTTPEndpoint: a plain-HTTP endpoint (MinIO / +// self-hosted S3) must derive use_ssl=false and be stored WITHOUT the scheme — +// the DuckDB secret wants host:port, not a URL. +func TestBuildSetStorageSecretSQL_HTTPEndpoint(t *testing.T) { + cfg := &lakekeeperStorageConfig{ + Provider: "aws", + Region: "us-east-1", + Endpoint: "http://minio:9000", + Credential: map[string]string{ + "access_key_id": "AKIA_TEST", + "secret_access_key": "SECRET_TEST", + }, + } + sql, args := buildSetStorageSecretSQL(cfg) + + assert.Contains(t, sql, "p_use_ssl => $6") + // Scheme stripped to host:port; use_ssl derived false from http://. + assert.Equal(t, []any{"AKIA_TEST", "SECRET_TEST", "minio:9000", "us-east-1", "path", false}, args) +} + +// TestBuildSetStorageSecretSQL_HTTPSEndpoint: an https endpoint derives +// use_ssl=true and is stored WITHOUT the scheme. +func TestBuildSetStorageSecretSQL_HTTPSEndpoint(t *testing.T) { + cfg := &lakekeeperStorageConfig{ + Provider: "aws", + Region: "eu-west-1", + Endpoint: "https://s3.example.com", + Credential: map[string]string{ + "access_key_id": "AKIA_TEST", + "secret_access_key": "SECRET_TEST", + }, + } + _, args := buildSetStorageSecretSQL(cfg) + + assert.Equal(t, []any{"AKIA_TEST", "SECRET_TEST", "s3.example.com", "eu-west-1", "path", true}, args) +} + +// TestBuildSetStorageSecretSQL_SchemelessEndpoint: an endpoint with no scheme +// (e.g. the GCS canonical host) keeps its value and defaults use_ssl=true. +func TestBuildSetStorageSecretSQL_SchemelessEndpoint(t *testing.T) { + cfg := &lakekeeperStorageConfig{ + Provider: "aws", + Region: "us-east-1", + Endpoint: "objects.internal:9000", + Credential: map[string]string{ + "access_key_id": "AKIA_TEST", + "secret_access_key": "SECRET_TEST", + }, + } + _, args := buildSetStorageSecretSQL(cfg) + + assert.Equal(t, []any{"AKIA_TEST", "SECRET_TEST", "objects.internal:9000", "us-east-1", "path", true}, args) +} + +// TestBuildSetStorageSecretSQL_MessyEndpoint: a copy-pasted endpoint with mixed +// scheme case, surrounding whitespace, and a trailing slash still yields a clean +// host:port with use_ssl derived from the (case-insensitive) scheme. +func TestBuildSetStorageSecretSQL_MessyEndpoint(t *testing.T) { + cfg := &lakekeeperStorageConfig{ + Provider: "aws", + Region: "us-east-1", + Endpoint: " HTTPS://minio.example.com:9000/ ", + Credential: map[string]string{ + "access_key_id": "AKIA_TEST", + "secret_access_key": "SECRET_TEST", + }, + } + _, args := buildSetStorageSecretSQL(cfg) + + assert.Equal(t, []any{"AKIA_TEST", "SECRET_TEST", "minio.example.com:9000", "us-east-1", "path", true}, args) +} + +func TestBuildSetStorageSecretSQL_Azure(t *testing.T) { + cfg := &lakekeeperStorageConfig{ + Provider: "azure", + Credential: map[string]string{ + "connection_string": "DefaultEndpointsProtocol=https;AccountKey=SEKRIT", + }, + } + sql, args := buildSetStorageSecretSQL(cfg) + assert.Contains(t, sql, "coldfront.set_storage_secret_azure(") + assert.NotContains(t, sql, "set_storage_secret(") + require.Len(t, args, 1) + assert.Equal(t, "DefaultEndpointsProtocol=https;AccountKey=SEKRIT", args[0]) + // The connection string must NOT appear in the SQL text. + assert.NotContains(t, sql, "SEKRIT") +} + +func TestExecSetStorageSecret_BindsCredentialAsArgs(t *testing.T) { + cfg := &lakekeeperStorageConfig{ + Provider: "aws", + Region: "us-east-1", + Credential: map[string]string{ + "access_key_id": "AKIA_XYZ", + "secret_access_key": "S3KRET", + }, + } + fe := &fakeExec{} + require.NoError(t, execSetStorageSecret(context.Background(), fe, cfg)) + + // Secret is passed as a bound arg, and the SQL text uses placeholders. + assert.Contains(t, fe.args, "S3KRET") + assert.NotContains(t, fe.sql, "S3KRET") + assert.True(t, strings.Contains(fe.sql, "$1"), "expected parameter placeholders in SQL") +} + +// TestBuildColdfrontLocalPGDSN pins the libpq keyword DSN the coldfront +// extension attaches as `pglocal` (loopback read of PG rows for Iceberg +// writes). Format mirrors the tiering DSN (finding #8): host/port/user/dbname +// with sslmode=disable. The sslmode=disable + local-trust assumption is the +// same tracked residual as #8, to revisit with the connectivity/auth work. +func TestBuildColdfrontLocalPGDSN(t *testing.T) { + got := buildColdfrontLocalPGDSN("mydb", "app_user") + want := "host=localhost port=5432 user=app_user dbname=mydb sslmode=disable" + assert.Equal(t, want, got) +} + +// TestBuildColdfrontGUCStatements pins the three per-database GUCs the +// extension's attach path reads: warehouse (name), lakekeeper_endpoint (MUST +// carry the /catalog path), local_pg_dsn. All are PGC_SUSET so ALTER DATABASE +// SET by the superuser applies to new sessions with no restart. Values are +// single-quoted with embedded quotes doubled (standard_conforming_strings=on), +// matching the roles.go literal-quoting precedent. +func TestBuildColdfrontGUCStatements(t *testing.T) { + stmts := buildColdfrontGUCStatements( + "mydb", + "wh", + "http://svc:8181/catalog", + "host=localhost port=5432 user=app_user dbname=mydb sslmode=disable", + ) + require.Len(t, stmts, 3) + assert.Equal(t, `ALTER DATABASE "mydb" SET coldfront.warehouse = 'wh';`, stmts[0].SQL) + assert.Equal(t, `ALTER DATABASE "mydb" SET coldfront.lakekeeper_endpoint = 'http://svc:8181/catalog';`, stmts[1].SQL) + assert.Equal(t, `ALTER DATABASE "mydb" SET coldfront.local_pg_dsn = 'host=localhost port=5432 user=app_user dbname=mydb sslmode=disable';`, stmts[2].SQL) +} + +// TestBuildColdfrontGUCStatements_QuotesValues verifies a single quote in a +// value (e.g. a warehouse name) is doubled, not broken out of the literal. +func TestBuildColdfrontGUCStatements_QuotesValues(t *testing.T) { + stmts := buildColdfrontGUCStatements("my'db", "wh's", "http://svc:8181/catalog", "dsn") + require.Len(t, stmts, 3) + // Identifier: embedded double-quote handling not exercised here, but the db + // name is quoted as an identifier; a single quote is legal inside it. + assert.Equal(t, `ALTER DATABASE "my'db" SET coldfront.warehouse = 'wh''s';`, stmts[0].SQL) +} + +// TestGenerateLakekeeperInstanceResources_StorageSecretGUCFields verifies the +// orchestrator threads the /catalog endpoint (derived from the generated +// service name) and the connect-as user into the storage-secret resource, so it +// can set the coldfront GUCs. +func TestGenerateLakekeeperInstanceResources_StorageSecretGUCFields(t *testing.T) { + o := newLakekeeperTestOrchestrator(t) + spec := makeManagedLakekeeperSpec() + + result, err := o.generateLakekeeperInstanceResources(spec) + require.NoError(t, err) + + secretRD := findResourceByType(result.Resources, ResourceTypeLakekeeperStorageSecret) + require.NotNil(t, secretRD) + secretRes, err := resource.ToResource[*LakekeeperStorageSecretResource](secretRD) + require.NoError(t, err) + + serviceName := ServiceInstanceName(spec.DatabaseID, spec.ServiceSpec.ServiceID, spec.HostID) + wantEndpoint := "http://" + serviceName + ":8181/catalog" + assert.Equal(t, wantEndpoint, secretRes.LakekeeperEndpoint) + assert.Equal(t, spec.ConnectAsUsername, secretRes.ConnectAsUsername) +} diff --git a/server/internal/orchestrator/swarm/manifest_loader.go b/server/internal/orchestrator/swarm/manifest_loader.go index 0f1a0adf..6e8c0ecc 100644 --- a/server/internal/orchestrator/swarm/manifest_loader.go +++ b/server/internal/orchestrator/swarm/manifest_loader.go @@ -41,10 +41,11 @@ func manifestCachePath(dir, url string) string { // versionManifestImages is the typed images block inside version-manifest.json. type versionManifestImages struct { - Postgres []manifestPostgresEntry `json:"postgres"` - PostgREST []manifestServiceEntry `json:"postgrest"` - MCP []manifestServiceEntry `json:"mcp"` - RAG []manifestServiceEntry `json:"rag"` + Postgres []manifestPostgresEntry `json:"postgres"` + PostgREST []manifestServiceEntry `json:"postgrest"` + MCP []manifestServiceEntry `json:"mcp"` + RAG []manifestServiceEntry `json:"rag"` + Lakekeeper []manifestServiceEntry `json:"lakekeeper"` } // versionManifest is the top-level structure of version-manifest.json. @@ -109,12 +110,12 @@ func withEmbeddedFallback() manifestLoaderOption { // (always succeeds; panics only if the embedded JSON is corrupt, which // indicates a broken build). type ManifestLoader struct { - cfg config.Config - logger zerolog.Logger - cachePath string - httpClient *http.Client - tickerC <-chan time.Time // nil → use default hourly ticker; injectable for tests - embeddedFallback bool // set when using the default URL; injectable for tests + cfg config.Config + logger zerolog.Logger + cachePath string + httpClient *http.Client + tickerC <-chan time.Time // nil → use default hourly ticker; injectable for tests + embeddedFallback bool // set when using the default URL; injectable for tests mu sync.RWMutex versions *Versions @@ -414,6 +415,7 @@ func buildServiceVersions(cfg config.Config, mf *versionManifest) (*ServiceVersi {"postgrest", mf.Images.PostgREST}, {"mcp", mf.Images.MCP}, {"rag", mf.Images.RAG}, + {"lakekeeper", mf.Images.Lakekeeper}, } { var defaultImage, lastImage *ServiceImage for _, e := range s.entries { diff --git a/server/internal/orchestrator/swarm/network.go b/server/internal/orchestrator/swarm/network.go index d25e6022..a3892ed1 100644 --- a/server/internal/orchestrator/swarm/network.go +++ b/server/internal/orchestrator/swarm/network.go @@ -37,6 +37,20 @@ type Network struct { NetworkID string `json:"network_id"` Subnet netip.Prefix `json:"subnet"` Gateway netip.Addr `json:"gateway"` + // Attachable allows standalone (non-service) containers to join this + // overlay network. Required for one-shot containers (e.g. the lakekeeper + // migrate container) that must resolve the postgres- overlay + // alias. + // + // NOTE: this is applied at CREATE only. Docker cannot toggle Attachable on + // an existing network, and Refresh does not read the live value back, so a + // network that already exists as non-attachable will NOT be reconciled to + // attachable — the lakekeeper migrate step would still fail for a database + // whose overlay predates this field (e.g. adding a catalog_db_create + // lakekeeper service to a pre-existing database). Handling that case + // (recreate-on-mismatch or a loud error) is tracked with the #11 + // permanent-form decision (attachable overlay vs. migrate-as-swarm-service). + Attachable bool `json:"attachable"` } func (n *Network) ResourceVersion() string { @@ -136,8 +150,9 @@ func (n *Network) Create(ctx context.Context, rc *resource.Context) error { } gateway := subnet.Addr().Next() networkID, err := client.NetworkCreate(ctx, n.Name, network.CreateOptions{ - Scope: n.Scope, - Driver: n.Driver, + Scope: n.Scope, + Driver: n.Driver, + Attachable: n.Attachable, IPAM: &network.IPAM{ Config: []network.IPAMConfig{ { diff --git a/server/internal/orchestrator/swarm/orchestrator.go b/server/internal/orchestrator/swarm/orchestrator.go index 2be09fab..88aee9bd 100644 --- a/server/internal/orchestrator/swarm/orchestrator.go +++ b/server/internal/orchestrator/swarm/orchestrator.go @@ -379,10 +379,11 @@ func (o *Orchestrator) instanceResources(spec *database.InstanceSpec, scripts da // instance will output this same network. They'll get deduplicated when we // add them to the state. databaseNetwork := &Network{ - Scope: "swarm", - Driver: OverlayDriver, - Name: fmt.Sprintf("%s-database", spec.DatabaseID), - Allocator: o.dbNetworkAllocator, + Scope: "swarm", + Driver: OverlayDriver, + Name: fmt.Sprintf("%s-database", spec.DatabaseID), + Allocator: o.dbNetworkAllocator, + Attachable: true, } // directory resources @@ -631,6 +632,8 @@ func (o *Orchestrator) GenerateServiceInstanceResources(spec *database.ServiceIn return o.generateMCPInstanceResources(spec) case "rag": return o.generateRAGInstanceResources(spec) + case "lakekeeper": + return o.generateLakekeeperInstanceResources(spec) default: return nil, fmt.Errorf("service type %q instance generation is not yet supported", spec.ServiceSpec.ServiceType) } @@ -655,10 +658,11 @@ func (o *Orchestrator) generateMCPInstanceResources(spec *database.ServiceInstan // Database network (shared with postgres instances) databaseNetwork := &Network{ - Scope: "swarm", - Driver: OverlayDriver, - Name: fmt.Sprintf("%s-database", spec.DatabaseID), - Allocator: o.dbNetworkAllocator, + Scope: "swarm", + Driver: OverlayDriver, + Name: fmt.Sprintf("%s-database", spec.DatabaseID), + Allocator: o.dbNetworkAllocator, + Attachable: true, } // Service data directory resource (host-side bind mount directory) @@ -818,6 +822,293 @@ func (o *Orchestrator) generateMCPInstanceResources(spec *database.ServiceInstan return o.buildServiceInstanceResources(spec, orchestratorResources) } +// generateLakekeeperInstanceResources returns the resources needed for one +// Lakekeeper service instance (Apache Iceberg REST catalog). +func (o *Orchestrator) generateLakekeeperInstanceResources(spec *database.ServiceInstanceSpec) (*database.ServiceInstanceResources, error) { + // Reject ColdFront on a multi-node database. ColdFront's tiering bakery + // requires snowflake.node = hashtext(spock_node_name)&1023, but + // control-plane currently assigns node ordinals rather than reconciling + // that mesh GUC, so a multi-node deployment would silently fail its tiering + // cron jobs (a clean failure — tiering errors only, no corruption). The API + // validation layer (validateColdFrontSingleNode) rejects this earlier; this + // is a defence-in-depth guard for callers that bypass it. The message is + // duplicated verbatim from apiv1.coldFrontMultiNodeError to avoid an import + // cycle. Remove once mesh snowflake.node reconciliation lands. + if len(spec.DatabaseNodes) > 1 { + return nil, fmt.Errorf("coldfront: multi-node ColdFront is not yet supported " + + "(mesh snowflake.node alignment pending); enable ColdFront only on a single-node database") + } + + // Get service image. + serviceImage, err := o.resolveServiceImage(spec) + if err != nil { + return nil, fmt.Errorf("failed to get service image: %w", err) + } + + // Validate compatibility with database version. + if spec.PgEdgeVersion != nil { + if err := serviceImage.ValidateCompatibility( + spec.PgEdgeVersion.PostgresVersion, + spec.PgEdgeVersion.SpockVersion, + ); err != nil { + return nil, fmt.Errorf("service %q version %q is not compatible with this database: %w", + spec.ServiceSpec.ServiceType, spec.ServiceSpec.Version, err) + } + } + + // Catalog database: either the caller supplies a reachable external + // catalog URL, or (catalog_db_create) control-plane provisions a + // catalog database on the node's primary and constructs the URL + // itself from deploy-time facts (overlay host + connect-as + // credentials) — no caller can know these at spec-build time. + serviceConfig := spec.ServiceSpec.Config + catalogDBCreate, _ := serviceConfig["catalog_db_create"].(bool) + var lakekeeperCatalogDBRes *LakekeeperCatalogDBResource + if catalogDBCreate { + if len(spec.DatabaseHosts) == 0 { + return nil, fmt.Errorf( + "lakekeeper service %q: catalog_db_create requires at least one database host", + spec.ServiceSpec.ServiceID, + ) + } + catalogDBName := lakekeeperCatalogDBName(spec.DatabaseName) + lakekeeperCatalogDBRes = &LakekeeperCatalogDBResource{ + ServiceInstanceID: spec.ServiceInstanceID, + DatabaseID: spec.DatabaseID, + DatabaseName: spec.DatabaseName, + NodeName: spec.NodeName, + CatalogDBName: catalogDBName, + CatalogDBOwner: spec.ConnectAsUsername, + } + serviceConfig = maps.Clone(serviceConfig) + serviceConfig["catalog_db_url"] = buildManagedCatalogDBURL( + spec.DatabaseHosts[0], spec.ConnectAsUsername, spec.ConnectAsPassword, catalogDBName) + } + + // Fail loudly if the catalog Postgres connection details are absent. An + // empty catalog_db_url or pg_encryption_key would cause Lakekeeper to + // start with blank env vars and crash-loop silently; returning a clear + // error at plan time is far more helpful. + catalogDBURL, _ := serviceConfig["catalog_db_url"].(string) + if catalogDBURL == "" { + return nil, fmt.Errorf( + "lakekeeper service %q: catalog_db_url is required in config; "+ + "provide the connection URL for an external catalog Postgres, "+ + "or set catalog_db_create for a control-plane-managed catalog", + spec.ServiceSpec.ServiceID, + ) + } + pgEncryptionKey, _ := serviceConfig["pg_encryption_key"].(string) + if pgEncryptionKey == "" { + return nil, fmt.Errorf( + "lakekeeper service %q: pg_encryption_key is required in config", + spec.ServiceSpec.ServiceID, + ) + } + + // In managed mode, thread the injected Config through every downstream + // consumer. ServiceSpec is a pointer and Config is a reference type, so + // build a shallow copy carrying the cloned map rather than mutating the + // caller's spec in place. + specForResources := spec.ServiceSpec + if catalogDBCreate { + patched := *spec.ServiceSpec // shallow struct copy + patched.Config = serviceConfig + specForResources = &patched + } + + // Database network (shared with Postgres instances). + databaseNetwork := &Network{ + Scope: "swarm", + Driver: OverlayDriver, + Name: fmt.Sprintf("%s-database", spec.DatabaseID), + Allocator: o.dbNetworkAllocator, + Attachable: true, + } + + // Service data directory (host-side bind mount). Lakekeeper runs as root + // (UID 0) in the official image, so no ownership override is needed here. + dataDirID := spec.ServiceInstanceID + "-data" + dataDir := &filesystem.DirResource{ + ID: dataDirID, + HostID: spec.HostID, + Path: filepath.Join(o.cfg.DataDir, "services", spec.ServiceInstanceID), + } + + // Lakekeeper config resource — writes the sentinel file and acts as a + // placeholder for future config artefacts. + lakekeeperConfigRes := &LakekeeperConfigResource{ + ServiceInstanceID: spec.ServiceInstanceID, + ServiceID: spec.ServiceSpec.ServiceID, + HostID: spec.HostID, + DirResourceID: dataDirID, + } + + // Migrate resource — runs the Lakekeeper image with the "migrate" + // subcommand as a one-shot Docker container to apply the Iceberg catalog + // schema to the external catalog Postgres. Must complete before the + // "serve" container starts (enforced via ServiceInstanceSpecResource + // dependencies for lakekeeper). + lakekeeperMigrateRes := &LakekeeperMigrateResource{ + ServiceInstanceID: spec.ServiceInstanceID, + HostID: spec.HostID, + Image: serviceImage.Tag, + CatalogDBURL: catalogDBURL, + PGEncryptionKey: pgEncryptionKey, + CatalogDBManaged: catalogDBCreate, + DatabaseNetworkName: fmt.Sprintf("%s-database", spec.DatabaseID), + } + + // Service instance spec resource — holds the computed Docker Swarm service spec. + serviceName := ServiceInstanceName(spec.DatabaseID, spec.ServiceSpec.ServiceID, spec.HostID) + serviceInstanceSpec := &ServiceInstanceSpecResource{ + ServiceInstanceID: spec.ServiceInstanceID, + ServiceSpec: specForResources, + DatabaseID: spec.DatabaseID, + DatabaseName: spec.DatabaseName, + HostID: spec.HostID, + ServiceName: serviceName, + Hostname: serviceName, + CohortMemberID: o.swarmNodeID, + ServiceImage: serviceImage, + DatabaseNetworkID: databaseNetwork.Name, + DatabaseHosts: spec.DatabaseHosts, + TargetSessionAttrs: spec.TargetSessionAttrs, + Port: spec.Port, + DataDirID: dataDirID, + } + + // Service instance resource (actual Docker Swarm service). + serviceInstance := &ServiceInstanceResource{ + ServiceInstanceID: spec.ServiceInstanceID, + DatabaseID: spec.DatabaseID, + ServiceName: serviceName, + ServiceID: spec.ServiceSpec.ServiceID, + ServiceSpecID: spec.ServiceSpec.ServiceID, + ServiceType: spec.ServiceSpec.ServiceType, + HostID: spec.HostID, + } + + // Bootstrap resource — after the serve container is healthy, creates the + // warehouse (with its storage profile and credential) and the default + // namespace via Lakekeeper's REST API. Depends on serviceInstance so it + // only runs once the Docker service is confirmed healthy. A failure blocks: + // an unbootstrapped warehouse is a broken database. + lakekeeperBootstrapRes := &LakekeeperBootstrapResource{ + ServiceInstanceID: spec.ServiceInstanceID, + HostID: spec.HostID, + ServiceName: serviceName, + Port: utils.FromPointer(spec.Port), + Config: specForResources.Config, + } + + // Coldfront extension resource — creates the coldfront extension (CASCADE + // pulls pg_duckdb) in the node's application database. A lakekeeper service + // is non-functional without it, so this is unconditional. Runs on the node's + // primary after the database is available and before the storage-secret step, + // which calls a coldfront function. + lakekeeperColdfrontExtRes := &LakekeeperColdfrontExtensionResource{ + ServiceInstanceID: spec.ServiceInstanceID, + DatabaseID: spec.DatabaseID, + DatabaseName: spec.DatabaseName, + NodeName: spec.NodeName, + } + + // Storage secret resource — sets the per-database coldfront GUCs and stores + // the object-store credential via ColdFront's set_storage_secret. Runs on the + // node's primary after the coldfront extension is available. The GUC endpoint + // is the catalog root: the generated service name plus the /catalog path the + // extension's Iceberg ATTACH requires. + lakekeeperPort := utils.FromPointer(spec.Port) + if lakekeeperPort == 0 { + lakekeeperPort = 8181 + } + lakekeeperGUCEndpoint := fmt.Sprintf("http://%s:%d/catalog", serviceName, lakekeeperPort) + lakekeeperStorageSecretRes := &LakekeeperStorageSecretResource{ + ServiceInstanceID: spec.ServiceInstanceID, + DatabaseID: spec.DatabaseID, + DatabaseName: spec.DatabaseName, + NodeName: spec.NodeName, + Config: specForResources.Config, + LakekeeperEndpoint: lakekeeperGUCEndpoint, + ConnectAsUsername: spec.ConnectAsUsername, + } + + orchestratorResources := []resource.Resource{ + databaseNetwork, + dataDir, + lakekeeperConfigRes, + } + if lakekeeperCatalogDBRes != nil { + orchestratorResources = append(orchestratorResources, lakekeeperCatalogDBRes) + } + orchestratorResources = append(orchestratorResources, + lakekeeperMigrateRes, + serviceInstanceSpec, + serviceInstance, + lakekeeperBootstrapRes, + lakekeeperColdfrontExtRes, + lakekeeperStorageSecretRes, + ) + + // Append tiering schedule resources when storage config is present. If the + // provider key is absent (not yet configured), no schedules are registered. + // Cron defaults: archiver hourly, partitioner every 6h, compactor daily. + // Override via service_config keys archiver_cron / partitioner_cron / compactor_cron. + if _, hasProvider := serviceConfig["provider"]; hasProvider { + // Bare endpoint (no /catalog path): the tiering binaries build their own + // REST paths. This differs from the coldfront.lakekeeper_endpoint GUC, + // which needs the /catalog catalog root. + lakekeeperEndpoint := fmt.Sprintf("http://%s:%d", serviceName, lakekeeperPort) + + // Build the args that the scheduled-job executor will decode. The + // connect-as user is carried alongside the derived endpoint so the tiering + // binaries authenticate to the node's local Postgres as the database's + // owner rather than a hardcoded "coldfront" role. + serviceConfigCopy := maps.Clone(serviceConfig) + serviceConfigCopy["lakekeeper_endpoint"] = lakekeeperEndpoint + serviceConfigCopy["local_pg_dsn_user"] = spec.ConnectAsUsername + + tieringArgs := map[string]interface{}{ + "database_id": spec.DatabaseID, + "node_name": spec.NodeName, + "service_id": spec.ServiceSpec.ServiceID, + "service_config": serviceConfigCopy, + "database_name": spec.DatabaseName, + } + + getCron := func(key, defaultExpr string) string { + if v, ok := serviceConfig[key].(string); ok && v != "" { + return v + } + return defaultExpr + } + + tierings := []struct { + suffix string + workflow string + cron string + cronKey string + }{ + {"archiver", scheduler.WorkflowColdFrontArchive, "0 * * * *", "archiver_cron"}, + {"partitioner", scheduler.WorkflowColdFrontPartition, "0 */6 * * *", "partitioner_cron"}, + {"compactor", scheduler.WorkflowColdFrontCompact, "0 2 * * *", "compactor_cron"}, + } + for _, t := range tierings { + jobID := fmt.Sprintf("coldfront-%s-%s-%s", t.suffix, spec.DatabaseID, spec.NodeName) + orchestratorResources = append(orchestratorResources, scheduler.NewScheduledJobResource( + jobID, + getCron(t.cronKey, t.cron), + t.workflow, + tieringArgs, + nil, + )) + } + } + + return o.buildServiceInstanceResources(spec, orchestratorResources) +} + // buildServiceInstanceResources converts a slice of resources into a // ServiceInstanceResources, shared by all service type generators. func (o *Orchestrator) buildServiceInstanceResources(spec *database.ServiceInstanceSpec, orchestratorResources []resource.Resource) (*database.ServiceInstanceResources, error) { @@ -869,10 +1160,11 @@ func (o *Orchestrator) generateRAGInstanceResources(spec *database.ServiceInstan // Database network (shared with postgres instances). databaseNetwork := &Network{ - Scope: "swarm", - Driver: OverlayDriver, - Name: fmt.Sprintf("%s-database", spec.DatabaseID), - Allocator: o.dbNetworkAllocator, + Scope: "swarm", + Driver: OverlayDriver, + Name: fmt.Sprintf("%s-database", spec.DatabaseID), + Allocator: o.dbNetworkAllocator, + Attachable: true, } orchestratorResources := []resource.Resource{databaseNetwork} @@ -979,9 +1271,9 @@ func (o *Orchestrator) GetInstanceConnectionInfo(ctx context.Context, if err != nil { return nil, fmt.Errorf("failed to inspect postgres container: %w", err) } - bridge, ok := inspect.NetworkSettings.Networks["bridge"] - if !ok { - return nil, fmt.Errorf("no bridge network found for postgres container %q", container.ID) + bridgeIP, err := bridgeIPAddress(inspect) + if err != nil { + return nil, fmt.Errorf("postgres container %q: %w", container.ID, err) } dbPort, err := nat.NewPort("tcp", strconv.Itoa(PostgresContainerPort)) if err != nil { @@ -1001,7 +1293,7 @@ func (o *Orchestrator) GetInstanceConnectionInfo(ctx context.Context, } return &database.ConnectionInfo{ - AdminHost: bridge.IPAddress, + AdminHost: bridgeIP, AdminPort: PostgresContainerPort, PeerHost: fmt.Sprintf("%s.%s-database", inspect.Config.Hostname, databaseID), PeerPort: PostgresContainerPort, diff --git a/server/internal/orchestrator/swarm/orchestrator_test.go b/server/internal/orchestrator/swarm/orchestrator_test.go index 503bfc80..9ba8b855 100644 --- a/server/internal/orchestrator/swarm/orchestrator_test.go +++ b/server/internal/orchestrator/swarm/orchestrator_test.go @@ -4,6 +4,10 @@ import ( "fmt" "strings" "testing" + + "github.com/pgEdge/control-plane/server/internal/config" + "github.com/pgEdge/control-plane/server/internal/database" + "github.com/pgEdge/control-plane/server/internal/resource" ) func TestServiceInstanceName(t *testing.T) { @@ -89,3 +93,204 @@ func TestServiceInstanceName(t *testing.T) { }) } + +// newLakekeeperTestOrchestrator returns an Orchestrator wired for unit tests +// of lakekeeper resource generation. It uses the zero-value Docker client +// (unavailable in unit tests) but sets up the serviceVersions and config so +// that generateLakekeeperInstanceResources can be exercised without a real +// Docker daemon. +func newLakekeeperTestOrchestrator(t *testing.T) *Orchestrator { + cfg := config.Config{ + DataDir: "/var/lib/pgedge", + DockerSwarm: config.DockerSwarm{ + ImageRepositoryHost: "ghcr.io/pgedge", + }, + } + return &Orchestrator{ + cfg: cfg, + serviceVersions: newTestServiceVersions(t, cfg), + swarmNodeID: "test-swarm-node", + // dbNetworkAllocator is the zero value — generateLakekeeperInstanceResources + // only stores it in the Network resource; it does not dereference it. + dbNetworkAllocator: Allocator{}, + } +} + +// makeLakekeeperSpec returns a minimal ServiceInstanceSpec for a lakekeeper +// service, pre-populated with both required config keys. +func makeLakekeeperSpec(catalogDBURL, pgEncryptionKey string) *database.ServiceInstanceSpec { + cfg := map[string]any{} + if catalogDBURL != "" { + cfg["catalog_db_url"] = catalogDBURL + } + if pgEncryptionKey != "" { + cfg["pg_encryption_key"] = pgEncryptionKey + } + return &database.ServiceInstanceSpec{ + ServiceInstanceID: "inst-lakekeeper-1", + ServiceSpec: &database.ServiceSpec{ + ServiceID: "lakekeeper", + ServiceType: "lakekeeper", + Version: "0.9.0", + Config: cfg, + }, + DatabaseID: "db-1", + DatabaseName: "testdb", + HostID: "host-1", + } +} + +// TestGenerateLakekeeperInstanceResources_MissingCatalogURL verifies that the +// resource generator fails loudly when catalog_db_url is absent, rather than +// producing a resource graph with blank env vars that would crash-loop the container. +func TestGenerateLakekeeperInstanceResources_MissingCatalogURL(t *testing.T) { + o := newLakekeeperTestOrchestrator(t) + + t.Run("missing catalog_db_url returns error", func(t *testing.T) { + spec := makeLakekeeperSpec("", "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==") + _, err := o.generateLakekeeperInstanceResources(spec) + if err == nil { + t.Fatal("expected error for missing catalog_db_url, got nil") + } + if !strings.Contains(err.Error(), "catalog_db_url") { + t.Errorf("error should mention catalog_db_url, got: %v", err) + } + }) + + t.Run("missing pg_encryption_key returns error", func(t *testing.T) { + spec := makeLakekeeperSpec("postgres://lakekeeper:secret@pg-host:5432/lakekeeper?sslmode=disable", "") + _, err := o.generateLakekeeperInstanceResources(spec) + if err == nil { + t.Fatal("expected error for missing pg_encryption_key, got nil") + } + if !strings.Contains(err.Error(), "pg_encryption_key") { + t.Errorf("error should mention pg_encryption_key, got: %v", err) + } + }) + + t.Run("both missing returns error", func(t *testing.T) { + spec := makeLakekeeperSpec("", "") + _, err := o.generateLakekeeperInstanceResources(spec) + if err == nil { + t.Fatal("expected error for missing config keys, got nil") + } + }) +} + +// TestGenerateLakekeeperInstanceResources_MultiNodeRejected verifies that the +// orchestrator refuses to generate resources for a lakekeeper (ColdFront) +// service on a database that spans more than one node. This is a +// defence-in-depth guard behind the API-layer validateColdFrontSingleNode +// check; multi-node ColdFront is unsupported until mesh snowflake.node +// reconciliation lands. +func TestGenerateLakekeeperInstanceResources_MultiNodeRejected(t *testing.T) { + o := newLakekeeperTestOrchestrator(t) + + makeMultiNode := func(nodeCount int) *database.ServiceInstanceSpec { + spec := makeLakekeeperSpec( + "postgres://lakekeeper:secret@pg-host:5432/lakekeeper?sslmode=disable", + "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + ) + spec.DatabaseNodes = make([]*database.NodeInstances, nodeCount) + for i := range spec.DatabaseNodes { + spec.DatabaseNodes[i] = &database.NodeInstances{ + NodeName: fmt.Sprintf("n%d", i+1), + } + } + return spec + } + + t.Run("two nodes returns error", func(t *testing.T) { + _, err := o.generateLakekeeperInstanceResources(makeMultiNode(2)) + if err == nil { + t.Fatal("expected error for multi-node ColdFront, got nil") + } + if !strings.Contains(err.Error(), "multi-node ColdFront is not yet supported") { + t.Errorf("error should mention multi-node ColdFront, got: %v", err) + } + }) + + t.Run("single node is accepted", func(t *testing.T) { + if _, err := o.generateLakekeeperInstanceResources(makeMultiNode(1)); err != nil { + t.Fatalf("single-node ColdFront should be accepted, got: %v", err) + } + }) +} + +// TestGenerateLakekeeperInstanceResources_ResourceGraph verifies that the +// generated resource graph includes a migrate resource and that the +// ServiceInstanceSpec resource depends on it (ordering guarantee). +func TestGenerateLakekeeperInstanceResources_ResourceGraph(t *testing.T) { + o := newLakekeeperTestOrchestrator(t) + + spec := makeLakekeeperSpec( + "postgres://lakekeeper:secret@pg-host:5432/lakekeeper?sslmode=disable", + "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + ) + + result, err := o.generateLakekeeperInstanceResources(spec) + if err != nil { + t.Fatalf("generateLakekeeperInstanceResources() unexpected error: %v", err) + } + + // Collect resource identifiers from the generated graph. + var resourceTypes []string + for _, rd := range result.Resources { + resourceTypes = append(resourceTypes, string(rd.Identifier.Type)) + } + + // The migrate resource must be present. + migrateType := string(ResourceTypeLakekeeperMigrate) + found := false + for _, rt := range resourceTypes { + if rt == migrateType { + found = true + break + } + } + if !found { + t.Errorf("resource graph missing %q; got: %v", migrateType, resourceTypes) + } + + // The ServiceInstanceSpec must declare the migrate resource as a dependency + // (enforcing migrate-before-serve ordering). + migrateID := LakekeeperMigrateResourceIdentifier(spec.ServiceInstanceID) + specID := ServiceInstanceSpecResourceIdentifier(spec.ServiceInstanceID) + + // Decode the ServiceInstanceSpecResource from the resource data so we can + // inspect its Dependencies(). + var specRD *resource.ResourceData + for _, rd := range result.Resources { + if rd.Identifier == specID { + specRD = rd + break + } + } + if specRD == nil { + t.Fatalf("ServiceInstanceSpecResource not found in resource graph; got: %v", resourceTypes) + } + + specRes, err := resource.ToResource[*ServiceInstanceSpecResource](specRD) + if err != nil { + t.Fatalf("failed to decode ServiceInstanceSpecResource: %v", err) + } + + deps := specRes.Dependencies() + depIDs := make([]string, len(deps)) + for i, d := range deps { + depIDs[i] = fmt.Sprintf("%s/%s", d.Type, d.ID) + } + + migrateIDStr := fmt.Sprintf("%s/%s", migrateID.Type, migrateID.ID) + foundDep := false + for _, d := range depIDs { + if d == migrateIDStr { + foundDep = true + break + } + } + if !foundDep { + t.Errorf("ServiceInstanceSpecResource.Dependencies() missing migrate resource %q; got: %v", + migrateIDStr, depIDs) + } +} diff --git a/server/internal/orchestrator/swarm/resources.go b/server/internal/orchestrator/swarm/resources.go index d943daef..fbd95c47 100644 --- a/server/internal/orchestrator/swarm/resources.go +++ b/server/internal/orchestrator/swarm/resources.go @@ -20,4 +20,10 @@ func RegisterResourceTypes(registry *resource.Registry) { resource.RegisterResourceType[*RAGPreflightResource](registry, ResourceTypeRAGPreflightResource) resource.RegisterResourceType[*RAGServiceKeysResource](registry, ResourceTypeRAGServiceKeys) resource.RegisterResourceType[*RAGConfigResource](registry, ResourceTypeRAGConfig) + resource.RegisterResourceType[*LakekeeperConfigResource](registry, ResourceTypeLakekeeperConfig) + resource.RegisterResourceType[*LakekeeperMigrateResource](registry, ResourceTypeLakekeeperMigrate) + resource.RegisterResourceType[*LakekeeperBootstrapResource](registry, ResourceTypeLakekeeperBootstrap) + resource.RegisterResourceType[*LakekeeperStorageSecretResource](registry, ResourceTypeLakekeeperStorageSecret) + resource.RegisterResourceType[*LakekeeperCatalogDBResource](registry, ResourceTypeLakekeeperCatalogDB) + resource.RegisterResourceType[*LakekeeperColdfrontExtensionResource](registry, ResourceTypeLakekeeperColdfrontExtension) } diff --git a/server/internal/orchestrator/swarm/service_images_test.go b/server/internal/orchestrator/swarm/service_images_test.go index b9652cb4..e4d215aa 100644 --- a/server/internal/orchestrator/swarm/service_images_test.go +++ b/server/internal/orchestrator/swarm/service_images_test.go @@ -37,6 +37,13 @@ func TestGetServiceImage(t *testing.T) { wantTag: "ghcr.io/pgedge/postgrest:14.5", wantErr: false, }, + { + name: "valid lakekeeper 0.9.0", + serviceType: "lakekeeper", + version: "0.9.0", + wantTag: "quay.io/lakekeeper/catalog:v0.9.0", + wantErr: false, + }, { name: "unsupported service type", serviceType: "unknown", diff --git a/server/internal/orchestrator/swarm/service_instance_spec.go b/server/internal/orchestrator/swarm/service_instance_spec.go index 6597c4ba..fc5e2a42 100644 --- a/server/internal/orchestrator/swarm/service_instance_spec.go +++ b/server/internal/orchestrator/swarm/service_instance_spec.go @@ -35,10 +35,10 @@ type ServiceInstanceSpecResource struct { CohortMemberID string `json:"cohort_member_id"` ServiceImage *ServiceImage `json:"service_image"` DatabaseNetworkID string `json:"database_network_id"` - DatabaseHosts []database.ServiceHostEntry `json:"database_hosts"` // Ordered Postgres host:port entries - TargetSessionAttrs string `json:"target_session_attrs"` // libpq target_session_attrs - Port *int `json:"port"` // Service published port (optional, 0 = random) - DataDirID string `json:"data_dir_id"` // DirResource ID for the service data directory + DatabaseHosts []database.ServiceHostEntry `json:"database_hosts"` // Ordered Postgres host:port entries + TargetSessionAttrs string `json:"target_session_attrs"` // libpq target_session_attrs + Port *int `json:"port"` // Service published port (optional, 0 = random) + DataDirID string `json:"data_dir_id"` // DirResource ID for the service data directory KBDirPath string `json:"kb_dir_path,omitempty"` // Host-side KB directory for bind mount (MCP only, KB enabled) Spec swarm.ServiceSpec `json:"spec"` } @@ -81,6 +81,14 @@ func (s *ServiceInstanceSpecResource) Dependencies() []resource.Identifier { RAGConfigResourceIdentifier(s.ServiceInstanceID), RAGServiceKeysResourceIdentifier(s.ServiceInstanceID), ) + case "lakekeeper": + deps = append(deps, + LakekeeperConfigResourceIdentifier(s.ServiceInstanceID), + // The migrate resource must complete before the serve container + // starts, so that the catalog schema exists before Lakekeeper + // attempts to use it. + LakekeeperMigrateResourceIdentifier(s.ServiceInstanceID), + ) default: log.Warn().Str("service_type", s.ServiceSpec.ServiceType).Msg("unknown service type in dependencies") } diff --git a/server/internal/orchestrator/swarm/service_spec.go b/server/internal/orchestrator/swarm/service_spec.go index 0a8ebe38..385c3453 100644 --- a/server/internal/orchestrator/swarm/service_spec.go +++ b/server/internal/orchestrator/swarm/service_spec.go @@ -24,6 +24,9 @@ const ragContainerUID = 1001 // See: https://github.com/PostgREST/postgrest/blob/main/Dockerfile (USER 1000) const postgrestContainerUID = 1000 +// lakekeeperListenPort is the port Lakekeeper listens on inside the container. +const lakekeeperListenPort = 8181 + // Shared health check timing for all service container types. const ( serviceHealthCheckStartPeriod = 30 * time.Second @@ -136,8 +139,14 @@ func ServiceContainerSpec(opts *ServiceContainerSpecOptions) (swarm.ServiceSpec, // Get container image (already resolved in ServiceImage) image := opts.ServiceImage.Tag - // Build port configuration (expose 8080 for HTTP API) - ports := buildServicePortConfig(opts.Port) + // Determine target port: most services use 8080, Lakekeeper uses 8181. + containerPort := 8080 + if opts.ServiceSpec.ServiceType == "lakekeeper" { + containerPort = lakekeeperListenPort + } + + // Build port configuration + ports := buildServicePortConfig(opts.Port, containerPort) // Build resource limits var resources *swarm.ResourceRequirements @@ -231,6 +240,41 @@ func ServiceContainerSpec(opts *ServiceContainerSpecOptions) (swarm.ServiceSpec, if opts.KeysPath != "" { mounts = append(mounts, docker.BuildMount(opts.KeysPath, "/app/keys", true)) } + case "lakekeeper": + // Lakekeeper is an Apache Iceberg REST catalog backed by an external + // Postgres instance. Connection details are supplied by the caller via + // ServiceSpec.Config. The LAKEKEEPER__ env vars are the idiomatic + // configuration mechanism for this service. + // Both catalog_db_url and pg_encryption_key are validated at spec time + // (validateLakekeeperServiceConfig / generateLakekeeperInstanceResources), + // so they will be non-empty here during normal operation. + catalogDBURL, _ := opts.ServiceSpec.Config["catalog_db_url"].(string) + pgEncryptionKey, _ := opts.ServiceSpec.Config["pg_encryption_key"].(string) + // The lakekeeper image ENTRYPOINT is the lakekeeper binary itself, so + // "serve" must be an ARG appended to it (Swarm ContainerSpec.Command + // would REPLACE the entrypoint → exec "serve" not found). + args = []string{"serve"} + env = []string{ + "LAKEKEEPER__PG_DATABASE_URL_READ=" + catalogDBURL, + "LAKEKEEPER__PG_DATABASE_URL_WRITE=" + catalogDBURL, + "LAKEKEEPER__PG_ENCRYPTION_KEY=" + pgEncryptionKey, + fmt.Sprintf("LAKEKEEPER__LISTEN_PORT=%d", lakekeeperListenPort), + } + healthcheck = &container.HealthConfig{ + // "healthcheck" is a SUBCOMMAND of the lakekeeper binary, not a + // standalone executable, and the image is distroless (no shell), so + // the healthcheck must invoke the binary by its absolute path. + Test: []string{"CMD", "/home/nonroot/lakekeeper", "healthcheck"}, + StartPeriod: serviceHealthCheckStartPeriod, + Interval: serviceHealthCheckInterval, + Timeout: serviceHealthCheckTimeout, + Retries: serviceHealthCheckRetries, + } + if opts.DataPath != "" { + mounts = []mount.Mount{ + docker.BuildMount(opts.DataPath, "/app/data", false), + } + } default: return swarm.ServiceSpec{}, fmt.Errorf("unsupported service type: %q", opts.ServiceSpec.ServiceType) } @@ -268,11 +312,12 @@ func ServiceContainerSpec(opts *ServiceContainerSpecOptions) (swarm.ServiceSpec, } // buildServicePortConfig builds port configuration for service containers. -// Exposes port 8080 for the HTTP API. +// targetPort is the port the service listens on inside the container (typically 8080, +// but 8181 for Lakekeeper). // If port is nil, no port is published. -// If port is non-nil and > 0, publish on that specific port. +// If port is non-nil and > 0, publish on that specific host port. // If port is non-nil and == 0, let Docker assign a random port. -func buildServicePortConfig(port *int) []swarm.PortConfig { +func buildServicePortConfig(port *int, targetPort int) []swarm.PortConfig { if port == nil { // Do not expose any port if not specified return nil @@ -280,7 +325,7 @@ func buildServicePortConfig(port *int) []swarm.PortConfig { config := swarm.PortConfig{ PublishMode: swarm.PortConfigPublishModeHost, - TargetPort: 8080, + TargetPort: uint32(targetPort), Name: "http", Protocol: swarm.PortConfigProtocolTCP, } diff --git a/server/internal/orchestrator/swarm/service_spec_test.go b/server/internal/orchestrator/swarm/service_spec_test.go index e7957f93..6eec2efd 100644 --- a/server/internal/orchestrator/swarm/service_spec_test.go +++ b/server/internal/orchestrator/swarm/service_spec_test.go @@ -274,7 +274,8 @@ func TestBuildServicePortConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ports := buildServicePortConfig(tt.port) + // Pass the standard target port (8080) for these generic tests. + ports := buildServicePortConfig(tt.port, 8080) if len(ports) != tt.wantPortCount { t.Fatalf("got %d ports, want %d", len(ports), tt.wantPortCount) @@ -636,6 +637,89 @@ func TestServiceContainerSpec_MCP_KBMount(t *testing.T) { } } +// --- Lakekeeper container spec tests --- + +func makeLakekeeperSpecOpts() *ServiceContainerSpecOptions { + return &ServiceContainerSpecOptions{ + ServiceSpec: &database.ServiceSpec{ + ServiceID: "lakekeeper", + ServiceType: "lakekeeper", + Config: map[string]interface{}{ + "catalog_db_url": "postgres://lakekeeper:secret@pg-host1:5432/lakekeeper?sslmode=disable", + "pg_encryption_key": "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + }, + }, + ServiceInstanceID: "db1-lakekeeper-host1", + DatabaseID: "db1", + DatabaseName: "testdb", + HostID: "host1", + ServiceName: "db1-lakekeeper-host1", + Hostname: "lakekeeper-host1", + CohortMemberID: "node-123", + ServiceImage: &ServiceImage{Tag: "quay.io/lakekeeper/catalog:v0.9.0"}, + DatabaseNetworkID: "db1-database", + DataPath: "/var/lib/pgedge/services/db1-lakekeeper-host1", + } +} + +func TestServiceContainerSpec_Lakekeeper_Command(t *testing.T) { + spec, err := ServiceContainerSpec(makeLakekeeperSpecOpts()) + require.NoError(t, err) + + // The lakekeeper image ENTRYPOINT is the lakekeeper binary, so "serve" must + // be an Arg appended to it, NOT a Command — a Swarm ContainerSpec.Command + // REPLACES the entrypoint, which would try to exec "serve" (not found). + cs := spec.TaskTemplate.ContainerSpec + assert.Empty(t, cs.Command, "Command must be empty so the image entrypoint (lakekeeper binary) is used") + assert.Equal(t, []string{"serve"}, cs.Args) +} + +func TestServiceContainerSpec_Lakekeeper_EnvVars(t *testing.T) { + spec, err := ServiceContainerSpec(makeLakekeeperSpecOpts()) + require.NoError(t, err) + + envMap := make(map[string]string) + for _, e := range spec.TaskTemplate.ContainerSpec.Env { + parts := strings.SplitN(e, "=", 2) + if len(parts) == 2 { + envMap[parts[0]] = parts[1] + } + } + + require.Contains(t, envMap, "LAKEKEEPER__PG_DATABASE_URL_READ") + require.Contains(t, envMap, "LAKEKEEPER__PG_DATABASE_URL_WRITE") + require.Contains(t, envMap, "LAKEKEEPER__PG_ENCRYPTION_KEY") + assert.Equal(t, "8181", envMap["LAKEKEEPER__LISTEN_PORT"]) + assert.Equal(t, "postgres://lakekeeper:secret@pg-host1:5432/lakekeeper?sslmode=disable", envMap["LAKEKEEPER__PG_DATABASE_URL_READ"]) + assert.Equal(t, "postgres://lakekeeper:secret@pg-host1:5432/lakekeeper?sslmode=disable", envMap["LAKEKEEPER__PG_DATABASE_URL_WRITE"]) + assert.Equal(t, "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", envMap["LAKEKEEPER__PG_ENCRYPTION_KEY"]) +} + +func TestServiceContainerSpec_Lakekeeper_Healthcheck(t *testing.T) { + spec, err := ServiceContainerSpec(makeLakekeeperSpecOpts()) + require.NoError(t, err) + + hc := spec.TaskTemplate.ContainerSpec.Healthcheck + require.NotNil(t, hc, "healthcheck must be set for lakekeeper") + // "healthcheck" is a subcommand of the (distroless) lakekeeper binary, so it + // must be invoked by absolute path — not exec'd as a standalone "healthcheck". + assert.Equal(t, []string{"CMD", "/home/nonroot/lakekeeper", "healthcheck"}, hc.Test) +} + +func TestServiceContainerSpec_Lakekeeper_Port8181(t *testing.T) { + opts := makeLakekeeperSpecOpts() + port := 8181 + opts.Port = &port + + spec, err := ServiceContainerSpec(opts) + require.NoError(t, err) + + ports := spec.EndpointSpec.Ports + require.Len(t, ports, 1) + assert.Equal(t, uint32(8181), ports[0].TargetPort, "Lakekeeper target port must be 8181") + assert.Equal(t, uint32(8181), ports[0].PublishedPort) +} + func TestServiceContainerSpec_MCPHasConfigVersionEnv(t *testing.T) { // The MCP config lives in a bind-mounted config.yaml. Editing it is invisible // to Swarm, and SIGHUP does not re-initialize the knowledgebase, so a KB config diff --git a/server/internal/orchestrator/swarm/utils.go b/server/internal/orchestrator/swarm/utils.go index 009d6f1f..c008caea 100644 --- a/server/internal/orchestrator/swarm/utils.go +++ b/server/internal/orchestrator/swarm/utils.go @@ -39,6 +39,24 @@ func GetPostgresContainer(ctx context.Context, dockerClient *docker.Docker, inst return matches[0], nil } +// bridgeIPAddress returns a container's IP on the Docker default `bridge` +// network. This is the address a host-networked Control Plane process uses to +// reach services running inside database containers (Postgres/Patroni, and the +// lakekeeper serve API). The swarm service NAME is deliberately not used: it +// resolves only via overlay DNS from inside the overlay, which CP is not a +// member of. Every database container is attached to the bridge for exactly +// this reason. +func bridgeIPAddress(inspect types.ContainerJSON) (string, error) { + if inspect.NetworkSettings == nil { + return "", fmt.Errorf("container has no network settings") + } + bridge, ok := inspect.NetworkSettings.Networks["bridge"] + if !ok || bridge == nil || bridge.IPAddress == "" { + return "", fmt.Errorf("no bridge network IP found for container") + } + return bridge.IPAddress, nil +} + func GetServiceContainer(ctx context.Context, dockerClient *docker.Docker, serviceInstanceID string) (types.Container, error) { matches, err := dockerClient.ContainerList(ctx, container.ListOptions{ Filters: filters.NewArgs( diff --git a/server/internal/orchestrator/swarm/version-manifest.json b/server/internal/orchestrator/swarm/version-manifest.json index 9236175a..f6380095 100644 --- a/server/internal/orchestrator/swarm/version-manifest.json +++ b/server/internal/orchestrator/swarm/version-manifest.json @@ -117,6 +117,14 @@ "stability": "stable", "default": true } + ], + "lakekeeper": [ + { + "version": "0.9.0", + "image": "quay.io/lakekeeper/catalog:v0.9.0", + "stability": "stable", + "default": true + } ] } } diff --git a/server/internal/postgres/gucs.go b/server/internal/postgres/gucs.go index 6f2e6f1e..0d80e12a 100644 --- a/server/internal/postgres/gucs.go +++ b/server/internal/postgres/gucs.go @@ -2,8 +2,54 @@ package postgres import ( "math" + "strings" ) +// coldFrontDuckDBExtensionDir is where the pgedge-coldfront-duckdb-extensions +// package installs the patched DuckDB iceberg extensions. CP points pg_duckdb at +// it via a GUC rather than a $PGDATA copy. Hardcoding the package path mirrors +// how CP hardcodes the /usr/bin path for the tiering binaries (finding #15). +const coldFrontDuckDBExtensionDir = "/usr/lib/pgedge/coldfront/duckdb-extensions" + +// ColdFrontDuckDBGUCs returns the static duckdb.* GUCs a ColdFront data node +// needs (finding #6a). They are boot-time settings: extension_directory is read +// at server start. autoinstall_known_extensions MUST be false — if true, DuckDB +// silently downloads the UNPATCHED upstream iceberg extension, which 409s under +// concurrent cold writes. These values are correctness-critical and set by CP so +// no consumer or image can get them wrong. +func ColdFrontDuckDBGUCs() map[string]any { + return map[string]any{ + "duckdb.extension_directory": coldFrontDuckDBExtensionDir, + "duckdb.allow_unsigned_extensions": "true", + "duckdb.autoinstall_known_extensions": "false", + } +} + +// AppendSharedPreloadLibraries appends libraries to a comma-separated +// shared_preload_libraries value, preserving the existing entries and order and +// skipping any that are already present. Dedup keeps reconciles stable: the same +// desired set yields the same string, so an idempotent re-apply does not look +// like a change and does not trigger a restart. Entry whitespace is trimmed. +func AppendSharedPreloadLibraries(current string, add ...string) string { + var libs []string + seen := map[string]bool{} + appendLib := func(lib string) { + lib = strings.TrimSpace(lib) + if lib == "" || seen[lib] { + return + } + seen[lib] = true + libs = append(libs, lib) + } + for lib := range strings.SplitSeq(current, ",") { + appendLib(lib) + } + for _, lib := range add { + appendLib(lib) + } + return strings.Join(libs, ",") +} + func DefaultGUCs() map[string]any { return map[string]any{ "archive_command": "/bin/true", diff --git a/server/internal/postgres/gucs_test.go b/server/internal/postgres/gucs_test.go index 2a8559fe..adf44bb0 100644 --- a/server/internal/postgres/gucs_test.go +++ b/server/internal/postgres/gucs_test.go @@ -178,3 +178,38 @@ func TestDefaultTunableGUCs(t *testing.T) { }) } } + +func TestColdFrontDuckDBGUCs(t *testing.T) { + g := postgres.ColdFrontDuckDBGUCs() + assert.Equal(t, "/usr/lib/pgedge/coldfront/duckdb-extensions", g["duckdb.extension_directory"]) + // allow_unsigned_extensions must be true (the patched iceberg build is + // unsigned); autoinstall_known_extensions MUST be false or DuckDB silently + // downloads the UNPATCHED upstream iceberg (409s under concurrent writes). + assert.Equal(t, "true", g["duckdb.allow_unsigned_extensions"]) + assert.Equal(t, "false", g["duckdb.autoinstall_known_extensions"]) + assert.Len(t, g, 3) +} + +func TestAppendSharedPreloadLibraries(t *testing.T) { + // Appends to the default, preserving order incl. spock. + assert.Equal(t, + "pg_stat_statements,snowflake,spock,pg_duckdb,coldfront", + postgres.AppendSharedPreloadLibraries("pg_stat_statements,snowflake,spock", "pg_duckdb", "coldfront"), + ) + // Dedup: already-present libs are not doubled (keeps reconciles stable so a + // restart isn't triggered every update). + assert.Equal(t, + "spock,pg_duckdb,coldfront", + postgres.AppendSharedPreloadLibraries("spock,pg_duckdb", "pg_duckdb", "coldfront"), + ) + // Whitespace around entries is trimmed. + assert.Equal(t, + "spock,pg_duckdb,coldfront", + postgres.AppendSharedPreloadLibraries(" spock , pg_duckdb ", "coldfront"), + ) + // Empty current value. + assert.Equal(t, + "pg_duckdb,coldfront", + postgres.AppendSharedPreloadLibraries("", "pg_duckdb", "coldfront"), + ) +} diff --git a/server/internal/scheduler/coldfront_tiering_executor_test.go b/server/internal/scheduler/coldfront_tiering_executor_test.go new file mode 100644 index 00000000..f2674dbb --- /dev/null +++ b/server/internal/scheduler/coldfront_tiering_executor_test.go @@ -0,0 +1,119 @@ +package scheduler + +import ( + "context" + "testing" +) + +// TestColdFrontWorkflowConstants verifies that the three ColdFront tiering +// workflow name constants are defined and non-empty. +func TestColdFrontWorkflowConstants(t *testing.T) { + cases := []struct { + name string + value string + }{ + {"WorkflowColdFrontArchive", WorkflowColdFrontArchive}, + {"WorkflowColdFrontPartition", WorkflowColdFrontPartition}, + {"WorkflowColdFrontCompact", WorkflowColdFrontCompact}, + } + for _, tc := range cases { + if tc.value == "" { + t.Errorf("%s is empty", tc.name) + } + } +} + +// testWorkflowExecutor is a fake WorkflowExecutor that records the names of +// workflows dispatched to it, for use in executor dispatch tests. +type testWorkflowExecutor struct { + dispatched []string + returnErr error +} + +func (e *testWorkflowExecutor) Execute(_ context.Context, workflowName string, _ map[string]interface{}) error { + e.dispatched = append(e.dispatched, workflowName) + return e.returnErr +} + +// TestExecuteColdFrontWorkflowsDispatch_NoUnknownWorkflow verifies that calling +// Execute with each of the three ColdFront workflow names does NOT return an +// "unknown workflow" error. We use a fake WorkflowExecutor to bypass the real +// database/workflow services, which would require a full DI container. +// +// This tests dispatch routing only, not the full execution path. +func TestExecuteColdFrontWorkflowsDispatch_NoUnknownWorkflow(t *testing.T) { + // The DefaultWorkflowExecutor switch delegates to runColdFrontTiering, + // which calls dbSvc — wiring around that requires a real service. Instead + // we assert the dispatch via the public Execute interface using a + // stand-alone fake executor. + fake := &testWorkflowExecutor{} + + for _, wf := range []string{ + WorkflowColdFrontArchive, + WorkflowColdFrontPartition, + WorkflowColdFrontCompact, + } { + fake.dispatched = nil + if err := fake.Execute(context.Background(), wf, nil); err != nil { + t.Errorf("fake Execute(%q) returned error: %v", wf, err) + } + } + + // Verify the workflow-name constants are distinct from WorkflowCreatePgBackRestBackup. + for _, wf := range []string{ + WorkflowColdFrontArchive, + WorkflowColdFrontPartition, + WorkflowColdFrontCompact, + } { + if wf == WorkflowCreatePgBackRestBackup { + t.Errorf("ColdFront workflow %q must not equal WorkflowCreatePgBackRestBackup", wf) + } + } +} + +// TestDefaultWorkflowExecutor_ColdFrontCasesNotDefault verifies that the +// Execute method's switch statement routes ColdFront workflow names to +// something other than the "unknown workflow" default case. We probe this by +// checking the error message: a nil-pointer panic (before reaching default) +// still proves the case was entered. We capture it via recover. +func TestDefaultWorkflowExecutor_ColdFrontCasesNotDefault(t *testing.T) { + executor := &DefaultWorkflowExecutor{ + workflowSvc: nil, + dbSvc: nil, + } + + for _, wf := range []string{ + WorkflowColdFrontArchive, + WorkflowColdFrontPartition, + WorkflowColdFrontCompact, + } { + t.Run(wf, func(t *testing.T) { + args := map[string]interface{}{ + "database_id": "db-1", + "node_name": "n1", + "service_id": "svc-1", + "service_config": map[string]interface{}{}, + "database_name": "mydb", + } + err := safeExecute(executor, wf, args) + if err != nil && err.Error() == "unknown workflow: "+wf { + t.Errorf("workflow %q fell through to default case", wf) + } + // Any other error (including a recovered panic) means we dispatched correctly. + }) + } +} + +// safeExecute calls executor.Execute and recovers any panic, returning it as +// an error string. A nil-pointer panic from a nil service proves the case was +// dispatched (not a fall-through to "unknown workflow"). +func safeExecute(e *DefaultWorkflowExecutor, wf string, args map[string]interface{}) (retErr error) { + defer func() { + if r := recover(); r != nil { + // Panic happened inside the case branch — dispatch confirmed. + // Return nil to indicate "not unknown workflow". + retErr = nil + } + }() + return e.Execute(context.Background(), wf, args) +} diff --git a/server/internal/scheduler/scheduled_job_executor.go b/server/internal/scheduler/scheduled_job_executor.go index 776087e4..4f8fcaf9 100644 --- a/server/internal/scheduler/scheduled_job_executor.go +++ b/server/internal/scheduler/scheduled_job_executor.go @@ -65,6 +65,13 @@ func (e *DefaultWorkflowExecutor) Execute(ctx context.Context, workflowName stri ) return err + case WorkflowColdFrontArchive: + return e.runColdFrontTiering(ctx, "archiver", args) + case WorkflowColdFrontPartition: + return e.runColdFrontTiering(ctx, "partitioner", args) + case WorkflowColdFrontCompact: + return e.runColdFrontTiering(ctx, "compactor", args) + default: return fmt.Errorf("unknown workflow: %s", workflowName) } @@ -87,3 +94,51 @@ type CreatePgBackRestBackupScheduleInput struct { NodeName string `json:"node_name"` Type string `json:"type"` } + +// RunColdFrontTieringScheduleInput carries the arguments stored in +// ScheduledJobResource.Args for the three ColdFront tiering jobs. +type RunColdFrontTieringScheduleInput struct { + DatabaseID string `json:"database_id"` + NodeName string `json:"node_name"` + ServiceID string `json:"service_id"` + ServiceConfig map[string]any `json:"service_config"` + DatabaseName string `json:"database_name"` +} + +func (e *DefaultWorkflowExecutor) runColdFrontTiering(ctx context.Context, binary string, args map[string]interface{}) error { + var input RunColdFrontTieringScheduleInput + if err := decodeArgs(args, &input); err != nil { + return err + } + + db, err := e.dbSvc.GetDatabase(ctx, input.DatabaseID) + if err != nil { + return fmt.Errorf("failed to fetch database: %w", err) + } + if !database.DatabaseStateModifiable(db.State) { + return fmt.Errorf("database is not in a modifiable state: %s", db.State) + } + + node, err := db.Spec.Node(input.NodeName) + if err != nil { + return fmt.Errorf("invalid node name %q: %w", input.NodeName, err) + } + + instances := make([]*workflows.InstanceHost, 0, len(node.HostIDs)) + for _, hostID := range node.HostIDs { + instances = append(instances, &workflows.InstanceHost{ + InstanceID: database.InstanceIDFor(hostID, input.DatabaseID, input.NodeName), + HostID: hostID, + }) + } + + _, err = e.workflowSvc.RunColdFrontTiering(ctx, binary, &workflows.ColdFrontTieringInput{ + DatabaseID: input.DatabaseID, + NodeName: input.NodeName, + ServiceID: input.ServiceID, + ServiceConfig: input.ServiceConfig, + DatabaseName: input.DatabaseName, + Instances: instances, + }) + return err +} diff --git a/server/internal/scheduler/types.go b/server/internal/scheduler/types.go index 26076add..165e57f3 100644 --- a/server/internal/scheduler/types.go +++ b/server/internal/scheduler/types.go @@ -14,6 +14,10 @@ const ( JobStatusFailed = "failed" WorkflowCreatePgBackRestBackup = "CreatePgBackRestBackup" + + WorkflowColdFrontArchive = "ColdFrontArchive" + WorkflowColdFrontPartition = "ColdFrontPartition" + WorkflowColdFrontCompact = "ColdFrontCompact" ) type StoredScheduledJob struct { diff --git a/server/internal/task/task.go b/server/internal/task/task.go index 7c01077e..ab88fddb 100644 --- a/server/internal/task/task.go +++ b/server/internal/task/task.go @@ -41,6 +41,7 @@ const ( TypeFailover Type = "failover" TypeRemoveHost Type = "remove_host" TypeUpgrade Type = "upgrade" + TypeTiering Type = "tiering" ) type Status string diff --git a/server/internal/workflows/activities/activities.go b/server/internal/workflows/activities/activities.go index f434339e..f88e4bc5 100644 --- a/server/internal/workflows/activities/activities.go +++ b/server/internal/workflows/activities/activities.go @@ -47,6 +47,7 @@ func (a *Activities) Register(work *worker.Worker) error { work.RegisterActivity(a.UpdatePlannedInstanceStates), work.RegisterActivity(a.UpdateTask), work.RegisterActivity(a.ValidateInstanceSpecs), + work.RegisterActivity(a.RunColdFrontBinary), } return errors.Join(errs...) } diff --git a/server/internal/workflows/activities/coldfront_tiering.go b/server/internal/workflows/activities/coldfront_tiering.go new file mode 100644 index 00000000..bb5cf9d1 --- /dev/null +++ b/server/internal/workflows/activities/coldfront_tiering.go @@ -0,0 +1,335 @@ +package activities + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "strings" + + "github.com/cschleiden/go-workflows/activity" + "github.com/cschleiden/go-workflows/workflow" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" + "github.com/samber/do" + "gopkg.in/yaml.v3" + + "github.com/pgEdge/control-plane/server/internal/docker" + "github.com/pgEdge/control-plane/server/internal/utils" +) + +// coldFrontArchiverBinary is the name of the archiver binary. The benign-empty +// classification is scoped to this binary only (see runColdFrontBinary). +const coldFrontArchiverBinary = "archiver" + +// coldFrontBinDir is where the tiering binaries (archiver/partitioner/compactor) +// live in the data-node image. The pgedge-coldfront package installs them to +// /usr/bin (RPM %{_bindir}), matching CP's other externally-installed binaries +// (pgbackrest/patroni default to /usr/bin too). +const coldFrontBinDir = "/usr/bin" + +// buildTieringCommand builds the `sh -c` argv that writes the base64-encoded +// config to a temp file inside the container and runs the tiering binary against +// it. base64 is used so the config never has to be shell-quoted. The binary is +// resolved under coldFrontBinDir. +func buildTieringCommand(encodedConfig, configPath, binary string) []string { + binaryPath := coldFrontBinDir + "/" + binary + return []string{ + "sh", "-c", + fmt.Sprintf("printf '%%s' '%s' | base64 -d > %s && %s --config %s", + encodedConfig, configPath, binaryPath, configPath), + } +} + +// coldFrontStorageConfig holds the parsed object-store coordinates extracted +// from a lakekeeper ServiceSpec.Config. The Credential field MUST NOT be logged. +type coldFrontStorageConfig struct { + Provider string + Warehouse string + Bucket string + Region string + Endpoint string + PathPrefix string + Credential map[string]string +} + +// parseColdFrontStorageConfig extracts storage config from a lakekeeper +// ServiceSpec.Config map. Returns nil (no error) if the provider key is +// absent — callers treat that as "no storage configured yet". +func parseColdFrontStorageConfig(config map[string]any) (*coldFrontStorageConfig, error) { + get := func(key string) string { + v, _ := config[key].(string) + return strings.TrimSpace(v) + } + + provider := get("provider") + if provider == "" { + return nil, nil + } + switch provider { + case "aws", "azure", "gcs": + default: + return nil, fmt.Errorf("coldfront: unsupported provider %q", provider) + } + + credRaw := get("credential") + var cred map[string]string + if credRaw != "" { + if err := json.Unmarshal([]byte(credRaw), &cred); err != nil { + return nil, fmt.Errorf("coldfront: credential is not valid JSON") + } + } + + return &coldFrontStorageConfig{ + Provider: provider, + Warehouse: get("warehouse"), + Bucket: get("bucket"), + Region: get("region"), + Endpoint: get("endpoint"), + PathPrefix: get("path_prefix"), + Credential: cred, + }, nil +} + +// buildColdFrontConfigYAML renders the YAML configuration for the archiver, +// partitioner, or compactor binary. The table list is intentionally omitted — +// the binaries resolve which tables to process from the DB registry +// (coldfront.partition_config). Credentials are written to the YAML but the +// caller must ensure the file is ephemeral and the content is never logged. +// +// dsnUser is the connect-as user the binary should authenticate as against the +// node's local Postgres; it falls back to "coldfront" when empty so the DSN is +// always well-formed. +func buildColdFrontConfigYAML(cfg coldFrontStorageConfig, dbName, lakekeeperEndpoint, dsnUser string) ([]byte, error) { + if dsnUser == "" { + dsnUser = "coldfront" + } + m := map[string]any{ + "postgres": map[string]any{ + "dsn": fmt.Sprintf("host=localhost port=5432 user=%s dbname=%s sslmode=disable", dsnUser, dbName), + }, + "iceberg": map[string]any{ + "warehouse": cfg.Warehouse, + "lakekeeper_endpoint": lakekeeperEndpoint, + "namespace": "default", + }, + } + + switch cfg.Provider { + case "aws", "gcs": + keyID := cfg.Credential["access_key_id"] + secret := cfg.Credential["secret_access_key"] + if cfg.Provider == "gcs" { + keyID = cfg.Credential["hmac_access_id"] + secret = cfg.Credential["hmac_secret"] + } + s3cfg := map[string]any{ + "access_key_id": keyID, + "secret_access_key": secret, + "bucket": cfg.Bucket, + "region": cfg.Region, + } + if cfg.Endpoint != "" { + s3cfg["endpoint"] = cfg.Endpoint + } + if cfg.PathPrefix != "" { + s3cfg["path_prefix"] = cfg.PathPrefix + } + m["s3"] = s3cfg + case "azure": + m["azure"] = map[string]any{ + "connection_string": cfg.Credential["connection_string"], + } + } + + return yaml.Marshal(m) +} + +// isBenignArchiverEmpty reports whether the given binary's output indicates +// that no tables have been registered yet. This is a normal, non-error +// condition when a database has just been created and nothing has been marked +// for tiering. +// +// This classification is deliberately scoped to the ARCHIVER only: the +// partitioner and compactor must NEVER have failures masked this way. +// +// FRAGILE INTERIM: the archiver logs "no tables configured" via log.Fatalf, +// which exits with code 1 — the SAME exit code as a genuine fatal error. There +// is therefore no distinct exit code to key on today, so a substring match on +// the binary's log text is the only available signal. A robust fix needs a +// ColdFront upstream change to emit a dedicated benign exit code (tracked as a +// cross-team follow-up); until then, changes to the archiver's log wording will +// silently break this detection. +func isBenignArchiverEmpty(binary, output string) bool { + if binary != coldFrontArchiverBinary { + return false + } + return strings.Contains(strings.ToLower(output), "no tables configured") +} + +// tieringExecer is the minimal exec surface the tiering activity needs: run a +// command in a container and report its exit code, combined output, and any +// transport-level error. It is satisfied by *docker.Docker via +// dockerTieringExecer and lets the exit-code + benign-classification behaviour +// be unit-tested with a fake. +type tieringExecer interface { + Exec(ctx context.Context, containerID string, cmd []string) (exitCode int, output string, err error) +} + +// dockerTieringExecer adapts *docker.Docker to the tieringExecer interface. +type dockerTieringExecer struct { + docker *docker.Docker +} + +func (d dockerTieringExecer) Exec(ctx context.Context, containerID string, cmd []string) (int, string, error) { + var buf bytes.Buffer + // docker.Docker.Exec returns a non-nil error wrapping "command failed with + // exit code N" for a non-zero exit. We normalise that to an explicit exit + // code so the classification logic does not depend on error-string parsing. + err := d.docker.Exec(ctx, &buf, containerID, cmd) + output := buf.String() + if err != nil { + // A non-zero exit is reported as an error by docker.Docker.Exec. We + // cannot recover the precise code from the wrapped error, so we report + // a sentinel non-zero (1) which is sufficient for classification: the + // only exit code we treat specially (benign archiver-empty) is itself + // exit 1 and is distinguished by output text, not by the code. + return 1, output, err + } + return 0, output, nil +} + +// runColdFrontBinary executes the tiering binary via the supplied execer and +// classifies the result. It returns nil when the run succeeded OR when it was a +// benign archiver-empty run; it returns a non-nil error for a genuine failure. +// The caller (the workflow) maps a nil result to task success and a non-nil +// result to task failure, so this function encodes the full success/fail/benign +// decision. Credentials in the config are never included in the returned error. +func runColdFrontBinary(ctx context.Context, execer tieringExecer, containerID, binary string, cmd []string) error { + exitCode, output, execErr := execer.Exec(ctx, containerID, cmd) + if exitCode == 0 && execErr == nil { + return nil + } + if isBenignArchiverEmpty(binary, output) { + // No tables registered yet: nothing to tier. Recorded as success. + return nil + } + if execErr != nil { + return fmt.Errorf("coldfront %s exited with error: %w\noutput:\n%s", binary, execErr, output) + } + return fmt.Errorf("coldfront %s exited with code %d\noutput:\n%s", binary, exitCode, output) +} + +// RunColdFrontBinaryInput holds the parameters for a single tiering binary run. +type RunColdFrontBinaryInput struct { + DatabaseID string `json:"database_id"` + NodeName string `json:"node_name"` + InstanceID string `json:"instance_id"` + ServiceConfig map[string]any `json:"service_config"` + DatabaseName string `json:"database_name"` + Binary string `json:"binary"` +} + +type RunColdFrontBinaryOutput struct{} + +// ExecuteRunColdFrontBinary dispatches the RunColdFrontBinary activity to the +// given host's workflow queue. +func (a *Activities) ExecuteRunColdFrontBinary( + ctx workflow.Context, + hostID string, + input *RunColdFrontBinaryInput, +) workflow.Future[*RunColdFrontBinaryOutput] { + options := workflow.ActivityOptions{ + Queue: utils.HostQueue(hostID), + RetryOptions: workflow.RetryOptions{ + MaxAttempts: 1, + }, + } + return workflow.ExecuteActivity[*RunColdFrontBinaryOutput](ctx, options, a.RunColdFrontBinary, input) +} + +// RunColdFrontBinary executes a single-pass ColdFront tiering binary +// (archiver, partitioner, or compactor) inside the primary node's Postgres +// container via docker exec. The binary's config is written to a temporary +// file inside the container using base64 to avoid shell injection. Exit codes +// are captured: a "no tables configured" non-zero exit from the archiver is +// treated as benign (nothing to tier yet). +func (a *Activities) RunColdFrontBinary(ctx context.Context, input *RunColdFrontBinaryInput) (*RunColdFrontBinaryOutput, error) { + logger := activity.Logger(ctx).With( + "database_id", input.DatabaseID, + "instance_id", input.InstanceID, + "binary", input.Binary, + ) + logger.Info("running coldfront tiering binary") + + storageCfg, err := parseColdFrontStorageConfig(input.ServiceConfig) + if err != nil { + return nil, fmt.Errorf("coldfront %s: invalid storage config: %w", input.Binary, err) + } + if storageCfg == nil { + logger.Warn("no storage provider configured; skipping coldfront run") + return &RunColdFrontBinaryOutput{}, nil + } + + dockerClient, err := do.Invoke[*docker.Docker](a.Injector) + if err != nil { + return nil, fmt.Errorf("coldfront %s: failed to get docker client: %w", input.Binary, err) + } + + // The lakekeeper endpoint is supplied in the service config (baked into the + // scheduled-job args at reconciliation time as http://:). + lakekeeperEndpoint := "" + if ep, ok := input.ServiceConfig["lakekeeper_endpoint"].(string); ok && ep != "" { + lakekeeperEndpoint = ep + } + + // The connect-as user is likewise baked into the service config at + // reconciliation time (from spec.ConnectAsUsername). buildColdFrontConfigYAML + // falls back to "coldfront" when it is absent. + dsnUser := "" + if u, ok := input.ServiceConfig["local_pg_dsn_user"].(string); ok { + dsnUser = u + } + if dsnUser == "" { + // The orchestrator always injects the connect-as user, so an empty value + // signals a misconfiguration. buildColdFrontConfigYAML still falls back to + // "coldfront" to stay functional, but surface it rather than silently + // reverting to the hardcode this fix removed. + logger.Warn("no connect-as user in tiering config; falling back to coldfront DSN user") + } + + configYAML, err := buildColdFrontConfigYAML(*storageCfg, input.DatabaseName, lakekeeperEndpoint, dsnUser) + if err != nil { + return nil, fmt.Errorf("coldfront %s: failed to render config: %w", input.Binary, err) + } + + // Locate the primary's Postgres container on this host via instance ID label. + pgContainers, err := dockerClient.ContainerList(ctx, container.ListOptions{ + Filters: filters.NewArgs( + filters.Arg("label", fmt.Sprintf("pgedge.instance.id=%s", input.InstanceID)), + filters.Arg("label", "pgedge.component=postgres"), + ), + }) + if err != nil { + return nil, fmt.Errorf("coldfront %s: failed to list containers for instance %s: %w", + input.Binary, input.InstanceID, err) + } + if len(pgContainers) == 0 { + return nil, fmt.Errorf("coldfront %s: no postgres container found for instance %s", + input.Binary, input.InstanceID) + } + pgContainer := pgContainers[0] + + // Write the config file into the container using base64 to avoid any shell + // quoting or injection issues, then run the binary. + encoded := base64.StdEncoding.EncodeToString(configYAML) + cmd := buildTieringCommand(encoded, "/tmp/coldfront-config.yaml", input.Binary) + + if err := runColdFrontBinary(ctx, dockerTieringExecer{docker: dockerClient}, pgContainer.ID, input.Binary, cmd); err != nil { + return nil, err + } + + logger.Info("coldfront tiering binary completed successfully") + return &RunColdFrontBinaryOutput{}, nil +} diff --git a/server/internal/workflows/activities/coldfront_tiering_test.go b/server/internal/workflows/activities/coldfront_tiering_test.go new file mode 100644 index 00000000..d17e9719 --- /dev/null +++ b/server/internal/workflows/activities/coldfront_tiering_test.go @@ -0,0 +1,246 @@ +package activities + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/pgEdge/control-plane/server/internal/task" +) + +// TestBuildColdFrontConfig verifies that the config YAML renderer produces +// the correct structure for each provider and never embeds credentials in +// plain text in a way that could slip into a log (smoke-test only — the real +// credential check is that the field exists with the expected value). +func TestBuildColdFrontConfig(t *testing.T) { + cases := []struct { + name string + cfg coldFrontStorageConfig + dbName string + endpoint string + wantKey string + }{ + { + name: "aws", + cfg: coldFrontStorageConfig{ + Provider: "aws", + Warehouse: "s3://my-bucket/warehouse", + Bucket: "my-bucket", + Region: "us-east-1", + Credential: map[string]string{ + "access_key_id": "AKID", + "secret_access_key": "SECRET", + }, + }, + dbName: "mydb", + wantKey: "access_key_id", + }, + { + name: "azure", + cfg: coldFrontStorageConfig{ + Provider: "azure", + Warehouse: "abfss://container@account.dfs.core.windows.net", + Bucket: "container", + Credential: map[string]string{ + "connection_string": "DefaultEndpointsProtocol=https;...", + }, + }, + dbName: "mydb", + wantKey: "connection_string", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + yaml, err := buildColdFrontConfigYAML(tc.cfg, tc.dbName, "lakekeeper-svc:8181", "coldfront") + if err != nil { + t.Fatalf("buildColdFrontConfigYAML returned error: %v", err) + } + if len(yaml) == 0 { + t.Fatal("empty config YAML") + } + content := string(yaml) + if !strings.Contains(content, "postgres:") { + t.Error("missing postgres section") + } + if !strings.Contains(content, "iceberg:") { + t.Error("missing iceberg section") + } + if strings.Contains(content, "tables:") { + t.Error("config must NOT contain archiver.tables — tables come from DB registry") + } + }) + } +} + +// TestBuildColdFrontConfigDSNUser verifies the tiering DSN uses the supplied +// connect-as user, and falls back to "coldfront" when none is provided. +func TestBuildColdFrontConfigDSNUser(t *testing.T) { + cfg := coldFrontStorageConfig{ + Provider: "aws", + Warehouse: "s3://my-bucket/warehouse", + Bucket: "my-bucket", + Region: "us-east-1", + Credential: map[string]string{ + "access_key_id": "AKID", + "secret_access_key": "SECRET", + }, + } + + t.Run("explicit user", func(t *testing.T) { + yaml, err := buildColdFrontConfigYAML(cfg, "mydb", "lakekeeper-svc:8181", "app_owner") + if err != nil { + t.Fatalf("buildColdFrontConfigYAML returned error: %v", err) + } + if !strings.Contains(string(yaml), "user=app_owner ") { + t.Errorf("expected DSN to use connect-as user app_owner, got:\n%s", yaml) + } + }) + + t.Run("empty user falls back to coldfront", func(t *testing.T) { + yaml, err := buildColdFrontConfigYAML(cfg, "mydb", "lakekeeper-svc:8181", "") + if err != nil { + t.Fatalf("buildColdFrontConfigYAML returned error: %v", err) + } + if !strings.Contains(string(yaml), "user=coldfront ") { + t.Errorf("expected DSN to fall back to coldfront user, got:\n%s", yaml) + } + }) +} + +// TestBuildTieringCommand_UsesUsrBin verifies the tiering binary is invoked from +// /usr/bin (where the pgedge-coldfront package installs it), not the legacy +// /usr/local/bin, and that the config file is written and passed via --config. +func TestBuildTieringCommand_UsesUsrBin(t *testing.T) { + cmd := buildTieringCommand("QkFTRTY0", "/tmp/coldfront-config.yaml", "archiver") + + if len(cmd) != 3 || cmd[0] != "sh" || cmd[1] != "-c" { + t.Fatalf("expected [sh -c