diff --git a/README.md b/README.md index f4675f4..0c1dfb7 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,19 @@ shared across its API keys; excess requests return 429 with Live entries are never evicted to admit new requests. Metrics bind to loopback in Nitro builds and retain their internal-network listener in ordinary builds. +A confidential request may set `"require_attested_inference": true` in its +encrypted envelope, next to `schema_version`, `request_id`, `issued_at` and +`request`. The enclave then lets only attested code read the prompt: itself +and a provider whose attestation it verifies before sending anything (today, +Tinfoil). It refuses routers such as `nexus/auto`, which send the prompt to +the Router, skips PII masking, which sends it to the analyzer, and never falls +back to a provider without that verification. Nor does it store a Tinfoil +proof of the request: the client keeps its own. A request it can't serve that +way returns 422 `attested_inference_unavailable` and nothing is sent upstream. +The field is inside the encrypted body, so nothing between the client and the +enclave can add or remove it, and a Gateway that predates it rejects the +envelope rather than ignoring it. + PII masking covers conversation text, tool results, assistant reasoning, tool-call argument string values, and `user`. Tool definitions, names, IDs, schemas, metadata, provider options, and image/audio content are not scanned. Keep diff --git a/apps/gateway/internal/adapters/http/handlers/confidential_chat_completions_handler.go b/apps/gateway/internal/adapters/http/handlers/confidential_chat_completions_handler.go index f7a08bb..1ac3afe 100644 --- a/apps/gateway/internal/adapters/http/handlers/confidential_chat_completions_handler.go +++ b/apps/gateway/internal/adapters/http/handlers/confidential_chat_completions_handler.go @@ -49,6 +49,11 @@ type confidentialEnvelope struct { RequestID string `json:"request_id"` IssuedAt string `json:"issued_at"` Request json.RawMessage `json:"request"` + // RequireAttestedInference asks that only attested code read the prompt: + // this enclave and a provider it verifies by attestation. See + // domain.WithAttestedInference. Gateways that predate the field reject + // the envelope as unknown, so asking for it never silently does nothing. + RequireAttestedInference bool `json:"require_attested_inference,omitempty"` } // NewConfidentialChatCompletionsHandler creates a strict encrypted handler. @@ -195,6 +200,9 @@ func (h *ConfidentialChatCompletionsHandler) handleDecrypted(w http.ResponseWrit // The downstream handler rechecks authentication and remains responsible for // OpenAI validation, generation, and streaming. Only replace the // decrypted envelope with its exact nested OpenAI request object. + if envelope.RequireAttestedInference { + r = r.WithContext(domain.WithAttestedInference(r.Context())) + } r.Body = io.NopCloser(bytes.NewReader(envelope.Request)) r.ContentLength = int64(len(envelope.Request)) r.Header.Del("Content-Length") diff --git a/apps/gateway/internal/adapters/http/handlers/confidential_chat_completions_handler_test.go b/apps/gateway/internal/adapters/http/handlers/confidential_chat_completions_handler_test.go index 41c71db..6b42acb 100644 --- a/apps/gateway/internal/adapters/http/handlers/confidential_chat_completions_handler_test.go +++ b/apps/gateway/internal/adapters/http/handlers/confidential_chat_completions_handler_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/binary" "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" @@ -267,6 +268,78 @@ func TestConfidentialChatCompletionsHandler_RejectsInvalidFreshnessAndReplay(t * } } +// The attested-inference requirement travels inside the encrypted envelope, +// so only the client that encrypted the request can set it, and the +// downstream handler sees it only when it was asked for. +func TestConfidentialChatCompletionsHandler_CarriesAttestedInferenceRequirement(t *testing.T) { + request := json.RawMessage(`{"model":"private/model","messages":[]}`) + envelopeWith := func(field string) []byte { + return []byte(fmt.Sprintf(`{"schema_version":%d,"request_id":%q,"issued_at":%q,"request":%s%s}`, + confidentialEnvelopeSchema, uuid.NewString(), time.Now().UTC().Format(time.RFC3339Nano), request, field)) + } + tests := []struct { + name string + envelope []byte + wantStatus int + wantRequired bool + }{ + {name: "required", envelope: envelopeWith(`,"require_attested_inference":true`), wantStatus: http.StatusOK, wantRequired: true}, + {name: "explicitly not required", envelope: envelopeWith(`,"require_attested_inference":false`), wantStatus: http.StatusOK}, + {name: "absent", envelope: envelopeWith(""), wantStatus: http.StatusOK}, + {name: "not a boolean", envelope: envelopeWith(`,"require_attested_inference":"yes"`), wantStatus: http.StatusBadRequest}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + calls, required := 0, false + var body []byte + handler, clientIdentity := newConfidentialTestHandler(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + required = domain.AttestedInferenceRequired(r.Context()) + body, _ = io.ReadAll(r.Body) + _, _ = io.WriteString(w, "ok") + })) + encrypted, requestContext, _ := newEncryptedConfidentialRequest(t, clientIdentity, test.envelope) + recorder := httptest.NewRecorder() + + handler.Handle(recorder, encrypted) + _ = decryptConfidentialResponse(t, recorder, requestContext) + + if recorder.Code != test.wantStatus { + t.Fatalf("status = %d, want %d", recorder.Code, test.wantStatus) + } + if test.wantStatus != http.StatusOK { + if calls != 0 { + t.Fatalf("downstream calls = %d, want 0", calls) + } + return + } + if required != test.wantRequired { + t.Fatalf("attested inference required = %v, want %v", required, test.wantRequired) + } + if !bytes.Equal(body, request) { + t.Fatalf("downstream body = %s, want only the nested request %s", body, request) + } + }) + } +} + +// A plaintext request can't carry the requirement: the header is ignored and +// the endpoint refuses plaintext before any handler runs. +func TestConfidentialChatCompletionsHandler_PlaintextCannotRequestAttestedInference(t *testing.T) { + calls := 0 + handler, _ := newConfidentialTestHandler(t, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { calls++ })) + request := httptest.NewRequest(http.MethodPost, "/v1/confidential/chat/completions", + strings.NewReader(`{"schema_version":1,"require_attested_inference":true}`)) + request.Header.Set("Authorization", "Bearer nexus-test-key") + recorder := httptest.NewRecorder() + + handler.Handle(recorder, request) + + if recorder.Code != http.StatusBadRequest || calls != 0 { + t.Fatalf("status = %d, downstream calls = %d; want 400 and none", recorder.Code, calls) + } +} + func TestConfidentialChatCompletionsHandler_RejectsTamperingAndStaleKey(t *testing.T) { tests := []struct { name string diff --git a/apps/gateway/internal/adapters/providers/attested_transport_test.go b/apps/gateway/internal/adapters/providers/attested_transport_test.go new file mode 100644 index 0000000..ffaca6e --- /dev/null +++ b/apps/gateway/internal/adapters/providers/attested_transport_test.go @@ -0,0 +1,28 @@ +package providers_test + +import ( + "testing" + "time" + + "github.com/dappnode/dappnode-nexus-gateway/apps/gateway/internal/adapters/providers/openai" + "github.com/dappnode/dappnode-nexus-gateway/apps/gateway/internal/adapters/providers/tinfoil" + "github.com/dappnode/dappnode-nexus-gateway/apps/gateway/internal/adapters/providers/typesafe" + "github.com/dappnode/dappnode-nexus-gateway/apps/gateway/internal/application/ports" +) + +// Attested inference sends a prompt only to adapters that claim an attested +// transport, so the claim must be exactly where verification happens. +func TestOnlyTinfoilClaimsAnAttestedTransport(t *testing.T) { + adapters := map[string]ports.Provider{ + "tinfoil": tinfoil.NewAdapter(time.Second), + "openai": openai.NewAdapter(time.Second), + "typesafe": typesafe.NewAdapter(time.Second), + } + for name, adapter := range adapters { + attested, ok := adapter.(ports.AttestedProvider) + got := ok && attested.AttestedTransport() + if want := name == "tinfoil"; got != want { + t.Errorf("%s adapter attested transport = %v, want %v", name, got, want) + } + } +} diff --git a/apps/gateway/internal/adapters/providers/tinfoil/adapter.go b/apps/gateway/internal/adapters/providers/tinfoil/adapter.go index 583ae81..5e347cd 100644 --- a/apps/gateway/internal/adapters/providers/tinfoil/adapter.go +++ b/apps/gateway/internal/adapters/providers/tinfoil/adapter.go @@ -62,6 +62,11 @@ func NewAdapterWithFactory(timeout time.Duration, factory VerifiedClientFactory, return &Adapter{timeout: timeout, factory: factory, logger: l} } +// AttestedTransport reports that every request is sent only through a client +// that has verified the Tinfoil enclave's attestation, over EHBP to the key +// that attestation binds, and that verification failure sends nothing. +func (a *Adapter) AttestedTransport() bool { return true } + func missingProviderCredentialError() *domain.GatewayError { return domain.ErrInternal("an internal error occurred").WithMeta( "provider", providerName, diff --git a/apps/gateway/internal/application/ports/provider.go b/apps/gateway/internal/application/ports/provider.go index 2a20051..0f8f559 100644 --- a/apps/gateway/internal/application/ports/provider.go +++ b/apps/gateway/internal/application/ports/provider.go @@ -18,6 +18,16 @@ type Provider interface { Complete(ctx context.Context, raw []byte, model domain.PublicModel) ([]byte, *domain.TinfoilTransportProof, error) } +// AttestedProvider is a provider adapter that sends a request only after it +// has verified the provider's hardware attestation, encrypted to a key bound +// to that attestation, so the provider's operator can't read the prompt +// either. It must fail closed: no verification, no request. Requests that +// require attested inference (domain.AttestedInferenceRequired) are sent only +// to providers whose adapter reports true. +type AttestedProvider interface { + AttestedTransport() bool +} + // ImageProvider generates images through the OpenAI images API. A provider // adapter that doesn't implement it can't serve image models. type ImageProvider interface { diff --git a/apps/gateway/internal/application/services/attested_inference_test.go b/apps/gateway/internal/application/services/attested_inference_test.go new file mode 100644 index 0000000..f8d79e3 --- /dev/null +++ b/apps/gateway/internal/application/services/attested_inference_test.go @@ -0,0 +1,351 @@ +package services + +import ( + "context" + "errors" + "io" + "strings" + "testing" + + "github.com/dappnode/dappnode-nexus-gateway/apps/gateway/internal/application/ports" + "github.com/dappnode/dappnode-nexus-gateway/pkg/domain" +) + +// attestedStubProvider is a provider whose adapter verifies its upstream's +// attestation, like the Tinfoil adapter. withoutProof models an adapter that +// sends a request but returns no proof of the transport it used. +type attestedStubProvider struct { + stubProvider + withoutProof bool +} + +func (*attestedStubProvider) AttestedTransport() bool { return true } + +func (p *attestedStubProvider) proof() *domain.TinfoilTransportProof { + if p.withoutProof { + return nil + } + return &domain.TinfoilTransportProof{Provider: "attested", Status: domain.ProofStatusVerified} +} + +func (p *attestedStubProvider) Complete(ctx context.Context, raw []byte, model domain.PublicModel) ([]byte, *domain.TinfoilTransportProof, error) { + body, _, err := p.stubProvider.Complete(ctx, raw, model) + if err != nil { + return nil, nil, err + } + return body, p.proof(), nil +} + +func (p *attestedStubProvider) Stream(ctx context.Context, raw []byte, model domain.PublicModel) (ports.ProviderStream, error) { + stream, err := p.stubProvider.Stream(ctx, raw, model) + if err != nil { + return ports.ProviderStream{}, err + } + stream.Proof = p.proof() + return stream, nil +} + +// closeTrackingBody records whether the gateway closed an upstream stream. +type closeTrackingBody struct { + io.Reader + closed bool +} + +func (b *closeTrackingBody) Close() error { + b.closed = true + return nil +} + +type attestedTestSetup struct { + svc *GenerateService + meter *stubUsageMeter + router *stubRouterClient + pii *fakePIIFilter + attested *attestedStubProvider + plain *stubProvider +} + +// newAttestedTestService serves "private/model" from an attested provider +// (falling back to a plain one), "open/model" from a plain provider, and the +// router "nexus/auto". The API key asks for PII masking. +func newAttestedTestService() *attestedTestSetup { + setup := &attestedTestSetup{ + meter: &stubUsageMeter{}, + router: &stubRouterClient{decision: domain.RouteDecision{PublicModelID: "private/model"}}, + pii: &fakePIIFilter{enabled: true}, + attested: &attestedStubProvider{}, + plain: &stubProvider{}, + } + private := domain.PublicModel{ + PublicModelID: "private/model", + ProviderModelID: "attested-model", + ProviderConfig: domain.ProviderConfig{ProviderName: "attested"}, + Fallback: &domain.ProviderTarget{ProviderModelID: "plain-model", ProviderConfig: domain.ProviderConfig{ProviderName: "plain"}}, + SupportsChatCompletions: true, + MaxContextWindow: 1000, + MaxOutputTokens: 100, + } + open := domain.PublicModel{ + PublicModelID: "open/model", + ProviderModelID: "plain-model", + ProviderConfig: domain.ProviderConfig{ProviderName: "plain"}, + SupportsChatCompletions: true, + MaxContextWindow: 1000, + MaxOutputTokens: 100, + } + setup.svc = NewGenerateService( + &stubAuthService{authCtx: domain.AuthContext{ + Account: domain.Account{ID: "acc1", Status: domain.AccountStatusActive}, + APIKey: domain.APIKey{ID: "key1", Active: true, PIIMode: domain.APIKeyPIIModeBalanced}, + }}, + &stubModelCatalog{ + models: map[string]domain.PublicModel{"private/model": private, "open/model": open}, + routers: map[string]domain.RouterEntry{"nexus/auto": {RouterID: "nexus/auto", Active: true}}, + }, + setup.router, + &stubProviderRegistry{providers: map[string]ports.Provider{ + "attested": setup.attested, + "plain": setup.plain, + }}, + setup.meter, + setup.pii, + stubLogger{}, + ) + return setup +} + +const attestedTestPrompt = `{"model":"private/model","messages":[{"role":"user","content":"Email Jane at jane@example.com"}]}` + +func proxyAttested(t *testing.T, setup *attestedTestSetup, endpoint, model string, stream bool) (*ProxyCall, error) { + t.Helper() + ctx := domain.WithAttestedInference(context.Background()) + return setup.svc.Proxy(ctx, endpoint, []byte(attestedTestPrompt), domain.GenerateRequest{PublicModelID: model, Stream: stream}, "sk-test") +} + +func requireAttestedInferenceUnavailable(t *testing.T, err error) { + t.Helper() + var gwErr *domain.GatewayError + if !errors.As(err, &gwErr) || gwErr.Code != domain.ErrCodeAttestedInferenceUnavailable { + t.Fatalf("error = %v, want %s", err, domain.ErrCodeAttestedInferenceUnavailable) + } + if gwErr.HTTPStatus != 422 || domain.FaultOf(err) != domain.FaultCaller { + t.Fatalf("status %d fault %s, want 422 caller", gwErr.HTTPStatus, domain.FaultOf(err)) + } +} + +func TestAttestedInference_SendsToAttestedProviderWithoutPIIAnalyzer(t *testing.T) { + setup := newAttestedTestService() + + call, err := proxyAttested(t, setup, domain.EndpointChatCompletions, "private/model", false) + if err != nil { + t.Fatalf("Proxy returned error: %v", err) + } + if setup.attested.calls != 1 || setup.plain.calls != 0 { + t.Fatalf("provider calls = attested %d, plain %d; want 1, 0", setup.attested.calls, setup.plain.calls) + } + if setup.pii.calls != 0 { + t.Fatalf("PII analyzer calls = %d, want 0: the analyzer is not attested code", setup.pii.calls) + } + if setup.attested.lastRaw != attestedTestPrompt { + t.Fatalf("provider body = %s, want the prompt unchanged", setup.attested.lastRaw) + } + if !strings.Contains(string(call.Body), `"content":"ok"`) { + t.Fatalf("response = %s", call.Body) + } + if setup.meter.successCalls != 1 { + t.Fatalf("success calls = %d, want 1", setup.meter.successCalls) + } +} + +// recordingProofs records the Tinfoil proofs the gateway stores. +type recordingProofs struct{ stored int } + +func (r *recordingProofs) UpsertTinfoilTransportProof(context.Context, domain.TinfoilTransportProof) error { + r.stored++ + return nil +} + +func (r *recordingProofs) GetTinfoilTransportProof(context.Context, string, string) (domain.TinfoilTransportProof, error) { + return domain.TinfoilTransportProof{}, nil +} + +func TestAttestedInference_StoresNoServerProof(t *testing.T) { + for _, attested := range []bool{true, false} { + setup := newAttestedTestService() + proofs := &recordingProofs{} + setup.svc.SetTinfoilProofRepository(proofs) + catalog := setup.svc.catalog.(*stubModelCatalog) + model := catalog.models["private/model"] + model.ProofMode = domain.ProofModeTinfoilAttestedTransport + model.ProviderConfig.ProviderName = "tinfoil" // Tinfoil proof mode requires the Tinfoil provider. + catalog.models["private/model"] = model + setup.svc.registry = &stubProviderRegistry{providers: map[string]ports.Provider{"tinfoil": setup.attested, "plain": setup.plain}} + + ctx := context.Background() + if attested { + ctx = domain.WithAttestedInference(ctx) + } + if _, err := setup.svc.Proxy(ctx, domain.EndpointChatCompletions, []byte(attestedTestPrompt), domain.GenerateRequest{PublicModelID: "private/model"}, "sk-test"); err != nil { + t.Fatalf("Proxy returned error: %v", err) + } + if want := map[bool]int{true: 0, false: 1}[attested]; proofs.stored != want { + t.Fatalf("attested=%v: stored proofs = %d, want %d", attested, proofs.stored, want) + } + } +} + +func TestAttestedInference_WithoutTheRequirementMasksAsBefore(t *testing.T) { + setup := newAttestedTestService() + + _, err := setup.svc.Proxy(context.Background(), domain.EndpointChatCompletions, []byte(attestedTestPrompt), + domain.GenerateRequest{PublicModelID: "open/model"}, "sk-test") + if err != nil { + t.Fatalf("Proxy returned error: %v", err) + } + if setup.pii.calls == 0 || setup.plain.calls != 1 { + t.Fatalf("PII calls = %d, plain calls = %d; want masking and the plain provider", setup.pii.calls, setup.plain.calls) + } +} + +func TestAttestedInference_RefusesUnattestedProviderBeforeHoldingCredit(t *testing.T) { + setup := newAttestedTestService() + + _, err := proxyAttested(t, setup, domain.EndpointChatCompletions, "open/model", false) + requireAttestedInferenceUnavailable(t, err) + if setup.plain.calls != 0 || setup.attested.calls != 0 { + t.Fatalf("provider calls = plain %d, attested %d; want none", setup.plain.calls, setup.attested.calls) + } + if setup.meter.reserveCalls != 0 { + t.Fatalf("reserve calls = %d, want 0", setup.meter.reserveCalls) + } + if setup.pii.calls != 0 { + t.Fatalf("PII analyzer calls = %d, want 0", setup.pii.calls) + } +} + +func TestAttestedInference_NeverCallsTheRouter(t *testing.T) { + setup := newAttestedTestService() + + _, err := proxyAttested(t, setup, domain.EndpointChatCompletions, "nexus/auto", false) + requireAttestedInferenceUnavailable(t, err) + if setup.router.calls != 0 { + t.Fatalf("router calls = %d, want 0: the Router reads the prompt outside the enclave", setup.router.calls) + } + if setup.attested.calls != 0 || setup.meter.reserveCalls != 0 { + t.Fatalf("provider calls = %d, reserve calls = %d; want none", setup.attested.calls, setup.meter.reserveCalls) + } +} + +func TestAttestedInference_UnknownModelStaysUnsupported(t *testing.T) { + setup := newAttestedTestService() + + _, err := proxyAttested(t, setup, domain.EndpointChatCompletions, "missing/model", false) + var gwErr *domain.GatewayError + if !errors.As(err, &gwErr) || gwErr.Code != domain.ErrCodeUnsupportedModel { + t.Fatalf("error = %v, want unsupported_model", err) + } +} + +func TestAttestedInference_DoesNotFallBackToUnattestedProvider(t *testing.T) { + setup := newAttestedTestService() + primaryErr := domain.ErrProviderUnavailable("attested") + setup.attested.err = primaryErr + + _, err := proxyAttested(t, setup, domain.EndpointChatCompletions, "private/model", false) + if err != primaryErr { + t.Fatalf("error = %v, want the primary provider's error", err) + } + if setup.plain.calls != 0 { + t.Fatalf("plain fallback calls = %d, want 0", setup.plain.calls) + } + if setup.meter.failureCalls != 1 || setup.meter.successCalls != 0 { + t.Fatalf("metering = %d failures, %d successes; want the hold released once", setup.meter.failureCalls, setup.meter.successCalls) + } +} + +func TestAttestedInference_FallsBackToAnotherAttestedProvider(t *testing.T) { + setup := newAttestedTestService() + setup.attested.err = domain.ErrProviderUnavailable("attested") + second := &attestedStubProvider{} + setup.svc.registry = &stubProviderRegistry{providers: map[string]ports.Provider{ + "attested": setup.attested, + "plain": second, // The fallback's provider name now resolves to an attested adapter. + }} + + if _, err := proxyAttested(t, setup, domain.EndpointChatCompletions, "private/model", false); err != nil { + t.Fatalf("Proxy returned error: %v", err) + } + if setup.attested.calls != 1 || second.calls != 1 { + t.Fatalf("calls = primary %d, fallback %d; want 1 each", setup.attested.calls, second.calls) + } +} + +func TestAttestedInference_DiscardsAnswerWithoutTransportProof(t *testing.T) { + setup := newAttestedTestService() + setup.attested.withoutProof = true + + call, err := proxyAttested(t, setup, domain.EndpointChatCompletions, "private/model", false) + if err == nil || call != nil { + t.Fatalf("Proxy = %v, %v; want an error and no answer", call, err) + } + if setup.plain.calls != 0 { + t.Fatalf("plain fallback calls = %d, want 0", setup.plain.calls) + } + if setup.meter.successCalls != 0 || setup.meter.failureCalls != 1 { + t.Fatalf("metering = %d successes, %d failures; want the hold released", setup.meter.successCalls, setup.meter.failureCalls) + } +} + +func TestAttestedInference_DiscardsStreamWithoutTransportProof(t *testing.T) { + setup := newAttestedTestService() + setup.attested.withoutProof = true + body := &closeTrackingBody{Reader: strings.NewReader("data: {\"choices\":[{\"delta\":{\"content\":\"secret\"}}]}\n\n")} + setup.svc.registry = &stubProviderRegistry{providers: map[string]ports.Provider{ + "attested": &streamBodyProvider{attestedStubProvider: setup.attested, body: body}, + "plain": setup.plain, + }} + + call, err := proxyAttested(t, setup, domain.EndpointChatCompletions, "private/model", true) + if err == nil || call != nil { + t.Fatalf("Proxy = %v, %v; want an error and no stream", call, err) + } + if !body.closed { + t.Fatal("upstream stream was left open") + } +} + +func TestAttestedInference_StreamsFromAttestedProvider(t *testing.T) { + setup := newAttestedTestService() + setup.attested.sse = "data: {\"id\":\"r1\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hi\"}}]}\n\ndata: {\"id\":\"r1\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\ndata: [DONE]\n\n" + + call, err := proxyAttested(t, setup, domain.EndpointChatCompletions, "private/model", true) + if err != nil { + t.Fatalf("Proxy returned error: %v", err) + } + if call.Stream == nil || call.Stream.proof == nil { + t.Fatalf("stream = %+v, want a stream carrying its transport proof", call.Stream) + } + call.Stream.Close() +} + +func TestAttestedInference_RefusesOtherEndpoints(t *testing.T) { + for _, endpoint := range []string{domain.EndpointSystemOne, domain.EndpointImageGenerations} { + setup := newAttestedTestService() + _, err := proxyAttested(t, setup, endpoint, "private/model", false) + requireAttestedInferenceUnavailable(t, err) + if setup.attested.calls != 0 || setup.meter.reserveCalls != 0 { + t.Fatalf("%s: provider calls = %d, reserve calls = %d; want none", endpoint, setup.attested.calls, setup.meter.reserveCalls) + } + } +} + +// streamBodyProvider streams a fixed body, to observe whether it is closed. +type streamBodyProvider struct { + *attestedStubProvider + body io.ReadCloser +} + +func (p *streamBodyProvider) Stream(context.Context, []byte, domain.PublicModel) (ports.ProviderStream, error) { + p.calls++ + return ports.ProviderStream{Body: p.body, Proof: p.proof()}, nil +} diff --git a/apps/gateway/internal/application/services/generate_service.go b/apps/gateway/internal/application/services/generate_service.go index a64b5a7..3430128 100644 --- a/apps/gateway/internal/application/services/generate_service.go +++ b/apps/gateway/internal/application/services/generate_service.go @@ -147,6 +147,11 @@ func (s *GenerateService) resolveModel(ctx context.Context, req domain.GenerateR } return domain.PublicModel{}, req, routerErr } + // The Router reads the prompt to choose a model, and it runs outside the + // enclave, so attested inference never calls it. + if domain.AttestedInferenceRequired(ctx) { + return domain.PublicModel{}, req, domain.ErrAttestedInferenceUnavailable(requestedModelID) + } if s.router == nil { return domain.PublicModel{}, req, domain.ErrInternal("an internal error occurred").WithMeta( "dependency", "router", @@ -334,6 +339,12 @@ func (s *GenerateService) storeTinfoilProof(ctx context.Context, auth domain.Aut if s == nil || s.tinfoilProofs == nil || model.EffectiveProofMode() != domain.ProofModeTinfoilAttestedTransport { return } + // The client of an attested-inference request verifies this enclave + // itself and keeps its own proof; a server copy would only be a lasting + // record of who used a private model when. + if domain.AttestedInferenceRequired(ctx) { + return + } if result.TinfoilProof == nil { if s.logger != nil { s.logger.Warn("Tinfoil proof mode enabled but provider returned no proof evidence", diff --git a/apps/gateway/internal/application/services/proxy.go b/apps/gateway/internal/application/services/proxy.go index 6fc176a..2e0ac2d 100644 --- a/apps/gateway/internal/application/services/proxy.go +++ b/apps/gateway/internal/application/services/proxy.go @@ -72,6 +72,15 @@ func (s *GenerateService) Proxy(ctx context.Context, endpoint string, raw []byte return nil, reject(err, nil) } } + // Only the confidential chat endpoint can set this; see + // domain.WithAttestedInference. + attested := domain.AttestedInferenceRequired(ctx) + if attested && endpoint != domain.EndpointChatCompletions { + err := domain.ErrAttestedInferenceUnavailable(req.PublicModelID) + s.recordTerminalOutcome(metrics.OutcomeError, endpoint, req, nil, time.Since(start).Milliseconds()) + s.recordError(endpoint, req, nil, metrics.PhaseRequest, err) + return nil, reject(err, nil) + } var model domain.PublicModel execReq := req if endpoint == domain.EndpointSystemOne { @@ -91,14 +100,28 @@ func (s *GenerateService) Proxy(ctx context.Context, endpoint string, raw []byte s.recordFailure(ctx, nil, &authCtx, endpoint, &execReq, &model, err, nil, time.Since(start).Milliseconds()) return nil, reject(err, &model) } - masked, mapping, err := s.maskBody(ctx, raw, authCtx.APIKey.PIIMode, endpoint) - if err != nil { - err = clientGone(ctx, err) + // Refused before any credit is held; attempt checks every target again. + if attested && !s.attestedTarget(model.ProviderConfig.ProviderName) { + err := domain.ErrAttestedInferenceUnavailable(execReq.PublicModelID).WithMeta("provider", model.ProviderConfig.ProviderName) s.recordTerminalOutcome(metrics.OutcomeError, endpoint, execReq, &model, time.Since(start).Milliseconds()) s.recordError(endpoint, execReq, &model, metrics.PhaseRequest, err) s.recordFailure(ctx, nil, &authCtx, endpoint, &execReq, &model, err, nil, time.Since(start).Milliseconds()) return nil, reject(err, &model) } + // Masking sends the prompt to the PII analyzer, which is not attested code, + // so attested inference skips it: only attested code reads that prompt. + masked, mapping := raw, (*domain.PIIMapping)(nil) + if !attested { + var err error + masked, mapping, err = s.maskBody(ctx, raw, authCtx.APIKey.PIIMode, endpoint) + if err != nil { + err = clientGone(ctx, err) + s.recordTerminalOutcome(metrics.OutcomeError, endpoint, execReq, &model, time.Since(start).Milliseconds()) + s.recordError(endpoint, execReq, &model, metrics.PhaseRequest, err) + s.recordFailure(ctx, nil, &authCtx, endpoint, &execReq, &model, err, nil, time.Since(start).Milliseconds()) + return nil, reject(err, &model) + } + } // The hold covers the prompt as sent upstream, after masking. if tokens, ok := estimateInputTokens(endpoint, masked); ok { execReq.EstimatedInputTokens = &tokens @@ -150,6 +173,10 @@ func (s *GenerateService) Proxy(ctx context.Context, endpoint string, raw []byte return nil, domain.ErrProviderUnavailable(target.ProviderConfig.ProviderName).WithMeta( "fault", domain.FaultNexus, "cause", domain.CauseProviderNotRegistered) } + if attested && !attestedTransport(provider) { + return nil, domain.ErrAttestedInferenceUnavailable(execReq.PublicModelID).WithMeta( + "provider", target.ProviderConfig.ProviderName) + } upstreamStart := time.Now() if !execReq.Stream { body, proof, err := provider.Complete(ctx, masked, target) @@ -157,6 +184,9 @@ func (s *GenerateService) Proxy(ctx context.Context, endpoint string, raw []byte if err != nil { return nil, err } + if attested && proof == nil { + return nil, errMissingTransportProof(target) + } var rewritten []byte var o outcome if endpoint == domain.EndpointSystemOne { @@ -176,6 +206,10 @@ func (s *GenerateService) Proxy(ctx context.Context, endpoint string, raw []byte if err != nil { return nil, err } + if attested && resp.Proof == nil { + resp.Body.Close() + return nil, errMissingTransportProof(target) + } stream := newProxyStream(resp.Body, target.PublicModelID) stream.proof = resp.Proof stream.hideUsage = !wantsUsage(raw) @@ -193,7 +227,8 @@ func (s *GenerateService) Proxy(ctx context.Context, endpoint string, raw []byte call, err := attempt(model) executed := model if err != nil && clientCtx.Err() == nil && shouldTryFallback(ctx, err, model.Fallback) && - withProviderTarget(model, *model.Fallback).SupportsEndpoint(endpoint) { + withProviderTarget(model, *model.Fallback).SupportsEndpoint(endpoint) && + (!attested || s.attestedTarget(model.Fallback.ProviderConfig.ProviderName)) { executed = withProviderTarget(model, *model.Fallback) s.logFallback(requestID, model, executed, sanitizeErrorWithPIIMapping(err, mapping)) metrics.RecordFallback(endpoint, model.PublicModelID, model.ProviderConfig.ProviderName, executed.ProviderConfig.ProviderName, err) @@ -215,6 +250,28 @@ func (s *GenerateService) Proxy(ctx context.Context, endpoint string, raw []byte return call, nil } +// attestedTransport reports whether a provider adapter verifies its upstream's +// attestation before sending anything; see ports.AttestedProvider. +func attestedTransport(provider ports.Provider) bool { + attested, ok := provider.(ports.AttestedProvider) + return ok && attested.AttestedTransport() +} + +// attestedTarget reports whether the named provider is served by an adapter +// with an attested transport. +func (s *GenerateService) attestedTarget(providerName string) bool { + provider, err := s.registry.GetProvider(providerName) + return err == nil && attestedTransport(provider) +} + +// errMissingTransportProof is an attested adapter that returned no proof of +// the transport it used. Attested inference fails closed on it, so the answer +// is discarded rather than delivered without evidence. +func errMissingTransportProof(target domain.PublicModel) *domain.GatewayError { + return domain.ErrProviderUnavailable(target.ProviderConfig.ProviderName).WithMeta( + "fault", domain.FaultNexus, "reason", "attested provider returned no transport proof") +} + // outcome is what the gateway read from a proxied response. type outcome struct { providerID string diff --git a/pkg/domain/attested_inference.go b/pkg/domain/attested_inference.go new file mode 100644 index 0000000..3643729 --- /dev/null +++ b/pkg/domain/attested_inference.go @@ -0,0 +1,37 @@ +package domain + +import ( + "context" + "fmt" +) + +// ErrCodeAttestedInferenceUnavailable refuses a request that asked for +// attested inference when its model can't be served that way. +const ErrCodeAttestedInferenceUnavailable = "attested_inference_unavailable" + +type attestedInferenceKey struct{} + +// WithAttestedInference marks a request whose prompt may be read only by +// attested code: the Gateway enclave itself and a provider whose hardware +// attestation the Gateway verifies before sending it anything. Such a request +// is never routed by the Router, never sent to the PII analyzer, and never +// sent to a provider without that verification, primary or fallback. +// +// Only the confidential endpoint sets it, from a field inside the decrypted +// request, so nothing outside the enclave can add or remove it. +func WithAttestedInference(ctx context.Context) context.Context { + return context.WithValue(ctx, attestedInferenceKey{}, true) +} + +// AttestedInferenceRequired reports whether WithAttestedInference marked ctx. +func AttestedInferenceRequired(ctx context.Context) bool { + required, _ := ctx.Value(attestedInferenceKey{}).(bool) + return required +} + +// ErrAttestedInferenceUnavailable is a request that asked for attested +// inference on a model or route that can't provide it. Nothing was sent. +func ErrAttestedInferenceUnavailable(model string) *GatewayError { + return &GatewayError{HTTPStatus: 422, Type: ErrTypeInvalidRequest, Code: ErrCodeAttestedInferenceUnavailable, + Message: fmt.Sprintf("Model '%s' can't be served so that only attested code reads the prompt. Nothing was sent.", model)} +} diff --git a/pkg/domain/error_class.go b/pkg/domain/error_class.go index 84f63cd..2ecf17a 100644 --- a/pkg/domain/error_class.go +++ b/pkg/domain/error_class.go @@ -65,6 +65,8 @@ var faultByCode = map[string]string{ ErrCodeProviderRejectedRequest: FaultUnclear, ErrCodeProviderConfiguration: FaultNexus, ErrCodeInternalError: FaultNexus, + + ErrCodeAttestedInferenceUnavailable: FaultCaller, } // FaultOf classifies an error. A "fault" in the metadata wins over the code,