From a729941dd6d344df19d94b76614709cdf5259d6a Mon Sep 17 00:00:00 2001 From: hcastc00 Date: Mon, 16 Nov 2026 13:41:32 +0100 Subject: [PATCH] feat: enforce runtime guest output token limits --- .../adapters/metering/runtime_client.go | 18 ++++++++++-------- .../adapters/metering/runtime_client_test.go | 5 ++++- .../application/services/output_limit_test.go | 15 +++++++++++++++ .../internal/application/services/proxy.go | 6 +++++- pkg/domain/account.go | 2 ++ 5 files changed, 36 insertions(+), 10 deletions(-) diff --git a/apps/gateway/internal/adapters/metering/runtime_client.go b/apps/gateway/internal/adapters/metering/runtime_client.go index 8b80fea..13f2b26 100644 --- a/apps/gateway/internal/adapters/metering/runtime_client.go +++ b/apps/gateway/internal/adapters/metering/runtime_client.go @@ -30,10 +30,11 @@ func (c *Client) AuthenticateAPIKey(ctx context.Context, rawKey string) (domain. } return domain.AuthContext{ Account: domain.Account{ - ID: response.Account.ID, - ExternalCustomerID: response.Account.ExternalCustomerID, - Status: response.Account.Status, - IsGuest: response.Account.IsGuest, + ID: response.Account.ID, + ExternalCustomerID: response.Account.ExternalCustomerID, + Status: response.Account.Status, + IsGuest: response.Account.IsGuest, + GuestMaxOutputTokens: response.Account.GuestMaxOutputTokens, }, APIKey: domain.APIKey{ ID: response.APIKey.ID, @@ -433,10 +434,11 @@ type authenticateResponse struct { } type runtimeAccount struct { - ID string `json:"id"` - ExternalCustomerID *string `json:"external_customer_id,omitempty"` - Status string `json:"status"` - IsGuest bool `json:"is_guest"` + ID string `json:"id"` + ExternalCustomerID *string `json:"external_customer_id,omitempty"` + Status string `json:"status"` + IsGuest bool `json:"is_guest"` + GuestMaxOutputTokens int `json:"guest_max_output_tokens,omitempty"` } type runtimeAPIKey struct { diff --git a/apps/gateway/internal/adapters/metering/runtime_client_test.go b/apps/gateway/internal/adapters/metering/runtime_client_test.go index 5f20ce8..7648fe1 100644 --- a/apps/gateway/internal/adapters/metering/runtime_client_test.go +++ b/apps/gateway/internal/adapters/metering/runtime_client_test.go @@ -60,7 +60,7 @@ func TestRuntimeClientAuthenticateReadsTheGuestFlag(t *testing.T) { account string want bool }{ - {`{"id":"acc-1","status":"active","is_guest":true}`, true}, + {`{"id":"acc-1","status":"active","is_guest":true,"guest_max_output_tokens":4096}`, true}, {`{"id":"acc-1","status":"active","is_guest":false}`, false}, {`{"id":"acc-1","status":"active"}`, false}, } { @@ -72,6 +72,9 @@ func TestRuntimeClientAuthenticateReadsTheGuestFlag(t *testing.T) { if err != nil { t.Fatalf("AuthenticateAPIKey: %v", err) } + if tc.want && auth.Account.GuestMaxOutputTokens != 4096 { + t.Fatalf("runtime guest output cap = %d, want 4096", auth.Account.GuestMaxOutputTokens) + } if auth.Account.IsGuest != tc.want { t.Fatalf("account %s: IsGuest = %v, want %v", tc.account, auth.Account.IsGuest, tc.want) } diff --git a/apps/gateway/internal/application/services/output_limit_test.go b/apps/gateway/internal/application/services/output_limit_test.go index 5ef3fba..c586560 100644 --- a/apps/gateway/internal/application/services/output_limit_test.go +++ b/apps/gateway/internal/application/services/output_limit_test.go @@ -40,6 +40,21 @@ func sentField(t *testing.T, raw, field string) any { return body[field] } +func TestProxy_GuestRuntimeOutputLimitOverridesDeploymentFallback(t *testing.T) { + for _, limit := range []int{25, 100} { + meter, provider := &stubUsageMeter{}, &stubProvider{} + svc := newGuestTestService(meter, provider, true) + svc.auth.(*stubAuthService).authCtx.Account.GuestMaxOutputTokens = limit + proxyChat(t, svc, `{"model":"model","messages":[{"role":"user","content":"hi"}]}`) + if got := sentField(t, provider.lastRaw, "max_completion_tokens"); got != float64(limit) { + t.Fatalf("output limit = %v, want %d", got, limit) + } + if got := meter.lastReserveReq.MaxOutputTokens; got == nil || *got != limit { + t.Fatalf("reservation does not use runtime limit %d", limit) + } + } +} + // A guest's chat request reaches the provider with an output limit no larger // than the guest limit (50 here), and Metering holds credit for that limit. // Other accounts' requests keep their own limit, or none. diff --git a/apps/gateway/internal/application/services/proxy.go b/apps/gateway/internal/application/services/proxy.go index 83a63d4..6fc176a 100644 --- a/apps/gateway/internal/application/services/proxy.go +++ b/apps/gateway/internal/application/services/proxy.go @@ -59,7 +59,11 @@ func (s *GenerateService) Proxy(ctx context.Context, endpoint string, raw []byte if authCtx.Account.IsGuest { err := s.checkGuestModel(ctx, endpoint, req.PublicModelID) if err == nil { - raw, req, err = capOutput(raw, req, s.guestMaxOutputTokens) + limit := authCtx.Account.GuestMaxOutputTokens + if limit <= 0 { + limit = s.guestMaxOutputTokens + } + raw, req, err = capOutput(raw, req, limit) } if err != nil { err = clientGone(ctx, err) diff --git a/pkg/domain/account.go b/pkg/domain/account.go index 43fc994..6ab0952 100644 --- a/pkg/domain/account.go +++ b/pkg/domain/account.go @@ -34,6 +34,8 @@ type Account struct { // login, funded with a small credit. Its chat requests get a smaller output // limit than other accounts' (see services.capOutput). IsGuest bool + // GuestMaxOutputTokens comes from runtime admin settings; zero uses the deployment fallback. + GuestMaxOutputTokens int } // IsActive returns true if the account status is "active".