Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions apps/gateway/internal/adapters/metering/runtime_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,11 @@ func (c *Client) AuthenticateAPIKey(ctx context.Context, rawKey string) (domain.
}
return domain.AuthContext{
Account: domain.Account{
ID: response.Account.ID,
ExternalCustomerID: response.Account.ExternalCustomerID,
Status: response.Account.Status,
IsGuest: response.Account.IsGuest,
ID: response.Account.ID,
ExternalCustomerID: response.Account.ExternalCustomerID,
Status: response.Account.Status,
IsGuest: response.Account.IsGuest,
GuestMaxOutputTokens: response.Account.GuestMaxOutputTokens,
},
APIKey: domain.APIKey{
ID: response.APIKey.ID,
Expand Down Expand Up @@ -433,10 +434,11 @@ type authenticateResponse struct {
}

type runtimeAccount struct {
ID string `json:"id"`
ExternalCustomerID *string `json:"external_customer_id,omitempty"`
Status string `json:"status"`
IsGuest bool `json:"is_guest"`
ID string `json:"id"`
ExternalCustomerID *string `json:"external_customer_id,omitempty"`
Status string `json:"status"`
IsGuest bool `json:"is_guest"`
GuestMaxOutputTokens int `json:"guest_max_output_tokens,omitempty"`
}

type runtimeAPIKey struct {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ func TestRuntimeClientAuthenticateReadsTheGuestFlag(t *testing.T) {
account string
want bool
}{
{`{"id":"acc-1","status":"active","is_guest":true}`, true},
{`{"id":"acc-1","status":"active","is_guest":true,"guest_max_output_tokens":4096}`, true},
{`{"id":"acc-1","status":"active","is_guest":false}`, false},
{`{"id":"acc-1","status":"active"}`, false},
} {
Expand All @@ -72,6 +72,9 @@ func TestRuntimeClientAuthenticateReadsTheGuestFlag(t *testing.T) {
if err != nil {
t.Fatalf("AuthenticateAPIKey: %v", err)
}
if tc.want && auth.Account.GuestMaxOutputTokens != 4096 {
t.Fatalf("runtime guest output cap = %d, want 4096", auth.Account.GuestMaxOutputTokens)
}
if auth.Account.IsGuest != tc.want {
t.Fatalf("account %s: IsGuest = %v, want %v", tc.account, auth.Account.IsGuest, tc.want)
}
Expand Down
15 changes: 15 additions & 0 deletions apps/gateway/internal/application/services/output_limit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,21 @@ func sentField(t *testing.T, raw, field string) any {
return body[field]
}

func TestProxy_GuestRuntimeOutputLimitOverridesDeploymentFallback(t *testing.T) {
for _, limit := range []int{25, 100} {
meter, provider := &stubUsageMeter{}, &stubProvider{}
svc := newGuestTestService(meter, provider, true)
svc.auth.(*stubAuthService).authCtx.Account.GuestMaxOutputTokens = limit
proxyChat(t, svc, `{"model":"model","messages":[{"role":"user","content":"hi"}]}`)
if got := sentField(t, provider.lastRaw, "max_completion_tokens"); got != float64(limit) {
t.Fatalf("output limit = %v, want %d", got, limit)
}
if got := meter.lastReserveReq.MaxOutputTokens; got == nil || *got != limit {
t.Fatalf("reservation does not use runtime limit %d", limit)
}
}
}

// A guest's chat request reaches the provider with an output limit no larger
// than the guest limit (50 here), and Metering holds credit for that limit.
// Other accounts' requests keep their own limit, or none.
Expand Down
6 changes: 5 additions & 1 deletion apps/gateway/internal/application/services/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,11 @@ func (s *GenerateService) Proxy(ctx context.Context, endpoint string, raw []byte
if authCtx.Account.IsGuest {
err := s.checkGuestModel(ctx, endpoint, req.PublicModelID)
if err == nil {
raw, req, err = capOutput(raw, req, s.guestMaxOutputTokens)
limit := authCtx.Account.GuestMaxOutputTokens
if limit <= 0 {
limit = s.guestMaxOutputTokens
}
raw, req, err = capOutput(raw, req, limit)
}
if err != nil {
err = clientGone(ctx, err)
Expand Down
2 changes: 2 additions & 0 deletions pkg/domain/account.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ type Account struct {
// login, funded with a small credit. Its chat requests get a smaller output
// limit than other accounts' (see services.capOutput).
IsGuest bool
// GuestMaxOutputTokens comes from runtime admin settings; zero uses the deployment fallback.
GuestMaxOutputTokens int
}

// IsActive returns true if the account status is "active".
Expand Down
Loading