From 32aa7c5b7e3fc4401588561f33197605230054ba Mon Sep 17 00:00:00 2001
From: adababys <8872412+adababys@users.noreply.github.com>
Date: Wed, 28 Oct 2026 17:15:35 +0800
Subject: [PATCH 1/2] feat(proxy): request-outcome metric distinguishing
 routing/upstream/client-closed
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The reverse proxy (shared by client-proxy and the orchestrator sandbox proxy)
had no request-outcome metric — only connection gauges. There was no way to
tell, on a dashboard, whether a proxied request failed because the destination
could not be resolved (500), the node was unreachable (502), the client went
away (499), or the sandbox was not found. During a node scale-down this is the
difference between stale routing record and real routing bug, and it could
only be recovered by scraping logs.

Add a request-outcome counter, recorded at the points where a request's fate is
decided:
- handler() pre-routing terminal branches: client_closed (ctx ended before
  routing), sandbox_not_found, too_many_connections, upstream_unreachable
  (owner acquire failed), routing_failed (unexpected resolve error).
- pool ErrorHandler: client_closed (context.Canceled / ECONNRESET / EOF) vs
  upstream_unreachable (everything else), and routing_failed when the
  destination is missing from context.
- ModifyResponse: ok (the upstream answered).

The shared proxy stays telemetry-agnostic: it classifies and calls an
OnRequestOutcome(ctx, outcome) callback (mirroring the existing
ConnectionLimitConfig callbacks). Each service owns its counter and name:
client_proxy.request.outcome and orchestrator.proxy.request.outcome, labeled
outcome = ok | routing_failed | upstream_unreachable | client_closed |
sandbox_not_found | too_many_connections.

Fixes #3757
---
 packages/client-proxy/internal/proxy/proxy.go |  12 +-
 packages/orchestrator/pkg/proxy/proxy.go      |  12 +-
 packages/orchestrator/pkg/proxy/proxy_test.go |   4 +-
 .../pkg/proxy/shutdown_h2c_test.go            |   2 +-
 .../orchestrator/pkg/proxy/shutdown_test.go   |   2 +-
 packages/shared/pkg/proxy/cors_proxy_test.go  |   2 +-
 packages/shared/pkg/proxy/full_duplex_test.go |   4 +-
 packages/shared/pkg/proxy/handler.go          |   7 +
 packages/shared/pkg/proxy/outcome_test.go     | 151 ++++++++++++++++++
 packages/shared/pkg/proxy/pool/client.go      |  28 ++++
 packages/shared/pkg/proxy/pool/pool.go        |  40 ++++-
 packages/shared/pkg/proxy/pool/pool_test.go   |  16 +-
 packages/shared/pkg/proxy/proxy.go            |   2 +
 packages/shared/pkg/proxy/proxy_test.go       |   7 +-
 packages/shared/pkg/proxy/shutdown_test.go    |   2 +-
 packages/shared/pkg/telemetry/meters.go       |  17 ++
 16 files changed, 286 insertions(+), 22 deletions(-)
 create mode 100644 packages/shared/pkg/proxy/outcome_test.go

diff --git a/packages/client-proxy/internal/proxy/proxy.go b/packages/client-proxy/internal/proxy/proxy.go
index d7093c14e4..797066bcd3 100644
--- a/packages/client-proxy/internal/proxy/proxy.go
+++ b/packages/client-proxy/internal/proxy/proxy.go
@@ -12,6 +12,7 @@ import (
 	"strings"
 	"time"
 
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
 	"go.uber.org/zap"
 	"google.golang.org/grpc/codes"
@@ -195,6 +196,14 @@ func NewClientProxy(meterProvider metric.MeterProvider, serviceName string, port
 		resumeCloser = flights
 	}
 
+	outcomeCounter, err := telemetry.GetCounter(meter, telemetry.ClientProxyRequestOutcomeCounterName)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error registering client proxy request outcome metric (%s): %w", telemetry.ClientProxyRequestOutcomeCounterName, err)
+	}
+	recordOutcome := func(ctx context.Context, outcome pool.RequestOutcome) {
+		outcomeCounter.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", string(outcome))))
+	}
+
 	getTargetFromRequest := reverseproxy.GetTargetFromRequest()
 	proxy := reverseproxy.New(
 		port,
@@ -291,9 +300,10 @@ func NewClientProxy(meterProvider metric.MeterProvider, serviceName string, port
 		},
 		nil,
 		false,
+		recordOutcome,
 	)
 
-	_, err := telemetry.GetObservableUpDownCounter(
+	_, err = telemetry.GetObservableUpDownCounter(
 		meter, telemetry.ClientProxyPoolConnectionsMeterCounterName, func(_ context.Context, observer metric.Int64Observer) error {
 			observer.Observe(proxy.CurrentPoolConnections())
 
diff --git a/packages/orchestrator/pkg/proxy/proxy.go b/packages/orchestrator/pkg/proxy/proxy.go
index 714de1249e..fa80aabbd7 100644
--- a/packages/orchestrator/pkg/proxy/proxy.go
+++ b/packages/orchestrator/pkg/proxy/proxy.go
@@ -13,6 +13,7 @@ import (
 	"sync"
 	"time"
 
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
 	"go.uber.org/zap"
 
@@ -116,6 +117,14 @@ func NewSandboxProxy(meterProvider metric.MeterProvider, port uint16, sandboxes
 		OnConnectionBlocked:  metrics.RecordConnectionBlocked,
 	}
 
+	outcomeCounter, err := telemetry.GetCounter(meter, telemetry.OrchestratorProxyRequestOutcomeCounterName)
+	if err != nil {
+		return nil, fmt.Errorf("error registering orchestrator proxy request outcome metric (%s): %w", telemetry.OrchestratorProxyRequestOutcomeCounterName, err)
+	}
+	recordOutcome := func(ctx context.Context, outcome pool.RequestOutcome) {
+		outcomeCounter.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", string(outcome))))
+	}
+
 	sandboxProxy := &SandboxProxy{limiter: limiter, sandboxes: sandboxes, owners: make(map[string]*pool.Owner), retirement: retirementTimeout}
 	proxy := reverseproxy.New(
 		port,
@@ -128,9 +137,10 @@ func NewSandboxProxy(meterProvider metric.MeterProvider, port uint16, sandboxes
 		// because the servers inside of the sandbox can be unstable (restarts),
 		// and we are also on the same host, so the overhead is minimal.
 		true,
+		recordOutcome,
 	)
 
-	_, err := telemetry.GetObservableUpDownCounter(meter, telemetry.OrchestratorProxyServerConnectionsMeterCounterName, func(_ context.Context, observer metric.Int64Observer) error {
+	_, err = telemetry.GetObservableUpDownCounter(meter, telemetry.OrchestratorProxyServerConnectionsMeterCounterName, func(_ context.Context, observer metric.Int64Observer) error {
 		observer.Observe(proxy.CurrentServerConnections())
 
 		return nil
diff --git a/packages/orchestrator/pkg/proxy/proxy_test.go b/packages/orchestrator/pkg/proxy/proxy_test.go
index 6d94fa2cd7..d2c9786445 100644
--- a/packages/orchestrator/pkg/proxy/proxy_test.go
+++ b/packages/orchestrator/pkg/proxy/proxy_test.go
@@ -142,7 +142,7 @@ func TestSandboxProxyRefusesFullRouterIngress(t *testing.T) {
 
 			sandboxes := sandbox.NewSandboxesMap()
 			p := &SandboxProxy{sandboxes: sandboxes, owners: make(map[string]*pool.Owner), limiter: connlimit.NewConnectionLimiter(), retirement: retirementTimeout}
-			p.proxy = reverseproxy.New(0, 1, time.Minute, p.resolveDestination, nil, true)
+			p.proxy = reverseproxy.New(0, 1, time.Minute, p.resolveDestination, nil, true, nil)
 			defer p.proxy.Close()
 			frontend := httptest.NewServer(p.proxy.Handler)
 			defer frontend.Close()
@@ -207,7 +207,7 @@ func TestSandboxProxyResetAndLifecycleRetirement(t *testing.T) {
 		}
 
 		return 8
-	}}, true)
+	}}, true, nil)
 	defer p.proxy.Close()
 	hold := func() *limitGate {
 		g := &limitGate{reachedCh: make(chan struct{}), releaseCh: make(chan struct{})}
diff --git a/packages/orchestrator/pkg/proxy/shutdown_h2c_test.go b/packages/orchestrator/pkg/proxy/shutdown_h2c_test.go
index 69d6618c2e..4788904685 100644
--- a/packages/orchestrator/pkg/proxy/shutdown_h2c_test.go
+++ b/packages/orchestrator/pkg/proxy/shutdown_h2c_test.go
@@ -54,7 +54,7 @@ func TestSandboxProxyCloseInterruptsStalledH2CUpload(t *testing.T) {
 					Limiter:              connlimit.NewConnectionLimiter(),
 					GetMaxLimit:          func(context.Context) int { return 10 },
 					OnConnectionReleased: func(context.Context, int64) { close(handlerDone) },
-				}, true)}
+				}, true, nil)}
 			var listenConfig net.ListenConfig
 			listener, err := listenConfig.Listen(t.Context(), "tcp", "127.0.0.1:0")
 			require.NoError(t, err)
diff --git a/packages/orchestrator/pkg/proxy/shutdown_test.go b/packages/orchestrator/pkg/proxy/shutdown_test.go
index c6338d168a..29d6d1eb5b 100644
--- a/packages/orchestrator/pkg/proxy/shutdown_test.go
+++ b/packages/orchestrator/pkg/proxy/shutdown_test.go
@@ -62,7 +62,7 @@ func TestSandboxProxyCloseInterruptsStalledUpload(t *testing.T) {
 					Limiter:              connlimit.NewConnectionLimiter(),
 					GetMaxLimit:          func(context.Context) int { return 10 },
 					OnConnectionReleased: func(context.Context, int64) { close(handlerDone) },
-				}, true)}
+				}, true, nil)}
 			var listenConfig net.ListenConfig
 			listener, err := listenConfig.Listen(t.Context(), "tcp", "127.0.0.1:0")
 			require.NoError(t, err)
diff --git a/packages/shared/pkg/proxy/cors_proxy_test.go b/packages/shared/pkg/proxy/cors_proxy_test.go
index ad039f609b..5e8ba85e47 100644
--- a/packages/shared/pkg/proxy/cors_proxy_test.go
+++ b/packages/shared/pkg/proxy/cors_proxy_test.go
@@ -313,7 +313,7 @@ func TestNoRawHTTPErrorInProxyPackage(t *testing.T) {
 
 func TestRetiredOwnerAnswersPreflight(t *testing.T) {
 	t.Parallel()
-	p := pool.New(4, 1, time.Minute, false)
+	p := pool.New(4, 1, time.Minute, false, nil)
 	target := &url.URL{Scheme: "http", Host: "127.0.0.1:1"}
 	d, err := destinationFor(target, "retired")(nil)
 	require.NoError(t, err)
diff --git a/packages/shared/pkg/proxy/full_duplex_test.go b/packages/shared/pkg/proxy/full_duplex_test.go
index bc87bbcf00..831c8868ad 100644
--- a/packages/shared/pkg/proxy/full_duplex_test.go
+++ b/packages/shared/pkg/proxy/full_duplex_test.go
@@ -169,7 +169,7 @@ func startProxyWithIdleTimeout(t *testing.T, getDestination func(r *http.Request
 	limits := &ConnectionLimitConfig{Limiter: limiter, GetMaxLimit: func(context.Context) int { return -1 }}
 	serverLog := &lockedBuffer{}
 	port := listener.Addr().(*net.TCPAddr).Port
-	proxy := New(uint16(port), 1, idleTimeout, getDestination, limits, false)
+	proxy := New(uint16(port), 1, idleTimeout, getDestination, limits, false, nil)
 	proxy.IdleTimeout = 2 * idleTimeout
 	proxy.ErrorLog = log.New(serverLog, "", 0)
 	go func() {
@@ -733,7 +733,7 @@ func TestHandlerEnablesFullDuplex(t *testing.T) {
 	backendURL, err := url.Parse(backend.URL)
 	require.NoError(t, err)
 
-	p := pool.New(maxClientConns, 1, time.Second, false)
+	p := pool.New(maxClientConns, 1, time.Second, false, nil)
 
 	recorder := &fullDuplexRecorder{ResponseRecorder: httptest.NewRecorder()}
 	req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "http://49983-sbx-1-00000000.localhost/", nil)
diff --git a/packages/shared/pkg/proxy/handler.go b/packages/shared/pkg/proxy/handler.go
index 243421b1f4..cb32661466 100644
--- a/packages/shared/pkg/proxy/handler.go
+++ b/packages/shared/pkg/proxy/handler.go
@@ -53,6 +53,7 @@ func handler(p *pool.ProxyPool, getDestination func(r *http.Request) (*pool.Dest
 		// request ended; a client that is gone reads nothing either way.
 		if ctxErr := ctx.Err(); err != nil && ctxErr != nil && errors.Is(err, ctxErr) {
 			l.Debug(ctx, "request ended before it was routed", zap.String("host", r.Host))
+			p.RecordOutcome(ctx, pool.OutcomeClientClosed)
 			cors.Error(w, "Request ended before it was routed", http.StatusRequestTimeout)
 
 			return
@@ -97,6 +98,8 @@ func handler(p *pool.ProxyPool, getDestination func(r *http.Request) (*pool.Dest
 				zap.String("host", r.Host),
 				logger.WithSandboxID(notFoundErr.SandboxId))
 
+			p.RecordOutcome(ctx, pool.OutcomeSandboxNotFound)
+
 			err := template.
 				NewSandboxNotFoundError(notFoundErr.SandboxId, r.Host).
 				HandleError(w, r)
@@ -250,6 +253,7 @@ func handler(p *pool.ProxyPool, getDestination func(r *http.Request) (*pool.Dest
 
 		if err != nil {
 			l.Error(ctx, "failed to route request", zap.Error(err), zap.String("host", r.Host))
+			p.RecordOutcome(ctx, pool.OutcomeRoutingFailed)
 			cors.Error(w, fmt.Sprintf("Unexpected error when routing request: %s", err), http.StatusInternalServerError)
 
 			return
@@ -279,6 +283,7 @@ func handler(p *pool.ProxyPool, getDestination func(r *http.Request) (*pool.Dest
 			if uploading {
 				closeUnreadBody(w)
 			}
+			p.RecordOutcome(ctx, pool.OutcomeUpstreamUnreachable)
 			cors.Error(w, "Failed to route request to sandbox", http.StatusBadGateway)
 
 			return
@@ -322,6 +327,8 @@ func handler(p *pool.ProxyPool, getDestination func(r *http.Request) (*pool.Dest
 					connLimitConfig.OnConnectionBlocked(ctx)
 				}
 
+				p.RecordOutcome(ctx, pool.OutcomeTooManyConnections)
+
 				// The sandbox is reachable, but the proxy is refusing to dial it,
 				// so the proxy is the only party that can answer this preflight.
 				if cors.HandlePreflight(w, r) {
diff --git a/packages/shared/pkg/proxy/outcome_test.go b/packages/shared/pkg/proxy/outcome_test.go
new file mode 100644
index 0000000000..7ad4ec83ff
--- /dev/null
+++ b/packages/shared/pkg/proxy/outcome_test.go
@@ -0,0 +1,151 @@
+package proxy
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/e2b-dev/infra/packages/shared/pkg/logger"
+	"github.com/e2b-dev/infra/packages/shared/pkg/proxy/pool"
+)
+
+// outcomeRecorder collects the outcomes the proxy reports, concurrency-safe.
+type outcomeRecorder struct {
+	mu       sync.Mutex
+	outcomes []pool.RequestOutcome
+}
+
+func (o *outcomeRecorder) add(_ context.Context, outcome pool.RequestOutcome) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	o.outcomes = append(o.outcomes, outcome)
+}
+
+func (o *outcomeRecorder) last() (pool.RequestOutcome, bool) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	if len(o.outcomes) == 0 {
+		return "", false
+	}
+
+	return o.outcomes[len(o.outcomes)-1], true
+}
+
+// startOutcomeProxy stands up a proxy whose destination resolver returns
+// resolve(), recording every request outcome into rec. Returns the proxy URL.
+func startOutcomeProxy(t *testing.T, rec *outcomeRecorder, resolve func() (*pool.Destination, error)) string {
+	t.Helper()
+
+	var lisCfg net.ListenConfig
+	proxyListener, err := lisCfg.Listen(t.Context(), "tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	proxyAddr := proxyListener.Addr().(*net.TCPAddr)
+	proxyPort := uint16(proxyAddr.Port) //nolint:gosec // test-only port fits
+
+	proxy := New(proxyPort, 1, time.Second, func(_ *http.Request) (*pool.Destination, error) {
+		return resolve()
+	}, nil, false, rec.add)
+
+	go func() {
+		serveErr := proxy.Serve(proxyListener)
+		assert.ErrorIs(t, serveErr, http.ErrServerClosed)
+	}()
+	t.Cleanup(func() { assert.NoError(t, proxy.Close()) })
+
+	return fmt.Sprintf("http://%s", proxyAddr.String())
+}
+
+func TestProxyRecordsOutcome_OK(t *testing.T) {
+	t.Parallel()
+
+	var lisCfg net.ListenConfig
+	backendListener, err := lisCfg.Listen(t.Context(), "tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	backendPort := uint64(backendListener.Addr().(*net.TCPAddr).Port) //nolint:gosec // test-only
+	backendURL, err := url.Parse(fmt.Sprintf("http://%s", backendListener.Addr().String()))
+	require.NoError(t, err)
+
+	backend := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	})}
+	go func() { assert.ErrorIs(t, backend.Serve(backendListener), http.ErrServerClosed) }()
+	t.Cleanup(func() { assert.NoError(t, backend.Close()) })
+
+	rec := &outcomeRecorder{}
+	proxyURL := startOutcomeProxy(t, rec, func() (*pool.Destination, error) {
+		return &pool.Destination{
+			Url:           backendURL,
+			SandboxId:     "sbx-ok",
+			SandboxPort:   backendPort,
+			ConnectionKey: "ok",
+			RequestLogger: logger.L(),
+		}, nil
+	})
+
+	resp, err := http.Get(proxyURL) //nolint:noctx // test
+	require.NoError(t, err)
+	_ = resp.Body.Close()
+
+	require.Eventually(t, func() bool {
+		o, ok := rec.last()
+
+		return ok && o == pool.OutcomeOK
+	}, 2*time.Second, 10*time.Millisecond, "expected an ok outcome")
+}
+
+func TestProxyRecordsOutcome_UpstreamUnreachable(t *testing.T) {
+	t.Parallel()
+
+	// Port 1 is reliably closed: the dial fails, so the request is
+	// upstream-unreachable, not a routing failure.
+	deadURL, err := url.Parse("http://127.0.0.1:1")
+	require.NoError(t, err)
+
+	rec := &outcomeRecorder{}
+	proxyURL := startOutcomeProxy(t, rec, func() (*pool.Destination, error) {
+		return &pool.Destination{
+			Url:           deadURL,
+			SandboxId:     "sbx-dead",
+			SandboxPort:   1,
+			ConnectionKey: "dead",
+			RequestLogger: logger.L(),
+		}, nil
+	})
+
+	resp, err := http.Get(proxyURL) //nolint:noctx // test
+	require.NoError(t, err)
+	_ = resp.Body.Close()
+
+	require.Eventually(t, func() bool {
+		o, ok := rec.last()
+
+		return ok && o == pool.OutcomeUpstreamUnreachable
+	}, 2*time.Second, 10*time.Millisecond, "expected an upstream_unreachable outcome")
+}
+
+func TestProxyRecordsOutcome_SandboxNotFound(t *testing.T) {
+	t.Parallel()
+
+	rec := &outcomeRecorder{}
+	proxyURL := startOutcomeProxy(t, rec, func() (*pool.Destination, error) {
+		return nil, NewErrSandboxNotFound("sbx-missing")
+	})
+
+	resp, err := http.Get(proxyURL) //nolint:noctx // test
+	require.NoError(t, err)
+	_ = resp.Body.Close()
+
+	require.Eventually(t, func() bool {
+		o, ok := rec.last()
+
+		return ok && o == pool.OutcomeSandboxNotFound
+	}, 2*time.Second, 10*time.Millisecond, "expected a sandbox_not_found outcome")
+}
diff --git a/packages/shared/pkg/proxy/pool/client.go b/packages/shared/pkg/proxy/pool/client.go
index 820aea82e7..0b5883118c 100644
--- a/packages/shared/pkg/proxy/pool/client.go
+++ b/packages/shared/pkg/proxy/pool/client.go
@@ -4,11 +4,13 @@ import (
 	"context"
 	"crypto/tls"
 	"errors"
+	"io"
 	"log"
 	"net"
 	"net/http"
 	"net/http/httputil"
 	"sync/atomic"
+	"syscall"
 	"time"
 
 	"go.uber.org/zap"
@@ -46,7 +48,13 @@ func newProxyClient(
 	l *log.Logger,
 	disableKeepAlives bool,
 	insecureSkipTLSVerify bool,
+	onOutcome OutcomeRecorder,
 ) *ProxyClient {
+	recordOutcome := func(ctx context.Context, outcome RequestOutcome) {
+		if onOutcome != nil {
+			onOutcome(ctx, outcome)
+		}
+	}
 	activeConnections := smap.New[*tracking.Connection]()
 
 	transport := &http.Transport{
@@ -156,6 +164,7 @@ func newProxyClient(
 			t, ok := pc.getDestination(r)
 			if !ok {
 				logger.L().Error(ctx, "proxy request without sandbox received error", zap.Error(err))
+				recordOutcome(ctx, OutcomeRoutingFailed)
 				cors.Error(w, "Failed to route request to sandbox", http.StatusInternalServerError)
 
 				return
@@ -163,11 +172,19 @@ func newProxyClient(
 
 			if r.Host == "" { // kept around for historical reasons, unsure of usefulness. todo: find out if this is useful.
 				t.RequestLogger.Error(ctx, "error handler called from rewrite because of missing DestinationContext", zap.Error(err))
+				recordOutcome(ctx, OutcomeRoutingFailed)
 				cors.Error(w, "Failed to route request to sandbox", http.StatusInternalServerError)
 
 				return
 			}
 
+			// A client that went away mid-request (cancel, reset, EOF) is not a
+			// server fault: classify it apart from a genuine upstream failure so
+			// the two are separable on dashboards. Matches the Warn/Error split below.
+			clientGone := errors.Is(err, context.Canceled) ||
+				errors.Is(err, syscall.ECONNRESET) ||
+				errors.Is(err, io.EOF)
+
 			if err != nil {
 				if t.SandboxPort == uint64(consts.DefaultEnvdServerPort) {
 					if errors.Is(err, context.Canceled) {
@@ -180,6 +197,12 @@ func newProxyClient(
 				}
 			}
 
+			if clientGone {
+				recordOutcome(ctx, OutcomeClientClosed)
+			} else {
+				recordOutcome(ctx, OutcomeUpstreamUnreachable)
+			}
+
 			if t.DefaultToPortError {
 				err = template.
 					NewPortClosedError(t.SandboxId, r.Host, t.SandboxPort).
@@ -204,6 +227,11 @@ func newProxyClient(
 				return nil
 			}
 
+			// The upstream answered and the response is being forwarded: the
+			// request succeeded as far as the proxy is concerned, whatever the
+			// backend's own status code.
+			recordOutcome(ctx, OutcomeOK)
+
 			if r.StatusCode >= 500 {
 				t.RequestLogger.Warn(
 					ctx,
diff --git a/packages/shared/pkg/proxy/pool/pool.go b/packages/shared/pkg/proxy/pool/pool.go
index 3154423a74..b9a5c7e6ba 100644
--- a/packages/shared/pkg/proxy/pool/pool.go
+++ b/packages/shared/pkg/proxy/pool/pool.go
@@ -22,6 +22,33 @@ const hostConnectionSplit = 4
 // by different sandboxes cause failed connections.
 const ClientProxyConnectionKey = "client-proxy"
 
+// RequestOutcome names how a proxied request ended. It is reported through the
+// OnOutcome callback so each service can count it on its own meter. The shared
+// proxy stays telemetry-agnostic: it only classifies, the caller counts.
+type RequestOutcome string
+
+const (
+	// OutcomeOK: the upstream answered and the response was forwarded.
+	OutcomeOK RequestOutcome = "ok"
+	// OutcomeRoutingFailed: the request could not be routed to a destination
+	// (no destination resolved, or an input the proxy rejected). Answered 4xx/500.
+	OutcomeRoutingFailed RequestOutcome = "routing_failed"
+	// OutcomeUpstreamUnreachable: the destination resolved but the proxy could
+	// not reach it (dial/transport failure, or a closed sandbox port). Answered 502.
+	OutcomeUpstreamUnreachable RequestOutcome = "upstream_unreachable"
+	// OutcomeClientClosed: the client went away before the proxy answered
+	// (context cancelled, connection reset, EOF). Nobody is left to read a page.
+	OutcomeClientClosed RequestOutcome = "client_closed"
+	// OutcomeSandboxNotFound: no sandbox matched the request.
+	OutcomeSandboxNotFound RequestOutcome = "sandbox_not_found"
+	// OutcomeTooManyConnections: the per-sandbox connection limit shed the request.
+	OutcomeTooManyConnections RequestOutcome = "too_many_connections"
+)
+
+// OutcomeRecorder is the callback the proxy calls once per request with its
+// terminal outcome. Nil is valid and means "do not record".
+type OutcomeRecorder func(ctx context.Context, outcome RequestOutcome)
+
 type ProxyPool struct {
 	clients               atomic.Int64
 	pool                  *smap.Map[*ProxyClient]
@@ -31,15 +58,25 @@ type ProxyPool struct {
 	totalConnsCounter     atomic.Uint64
 	currentConnsCounter   atomic.Int64
 	disableKeepAlives     bool
+	onOutcome             OutcomeRecorder
 }
 
-func New(maxClientConns int, maxConnectionAttempts int, idleTimeout time.Duration, disableKeepAlives bool) *ProxyPool {
+func New(maxClientConns int, maxConnectionAttempts int, idleTimeout time.Duration, disableKeepAlives bool, onOutcome OutcomeRecorder) *ProxyPool {
 	return &ProxyPool{
 		pool:                  smap.New[*ProxyClient](),
 		maxClientConns:        maxClientConns,
 		maxConnectionAttempts: maxConnectionAttempts,
 		idleTimeout:           idleTimeout,
 		disableKeepAlives:     disableKeepAlives,
+		onOutcome:             onOutcome,
+	}
+}
+
+// RecordOutcome reports a request outcome through the configured recorder, if
+// any. Safe to call with a nil recorder.
+func (p *ProxyPool) RecordOutcome(ctx context.Context, outcome RequestOutcome) {
+	if p.onOutcome != nil {
+		p.onOutcome(ctx, outcome)
 	}
 }
 
@@ -103,6 +140,7 @@ func (p *ProxyPool) newClient(ctx context.Context, d *Destination, owner *Owner)
 		stdLogger,
 		p.disableKeepAlives,
 		d.InsecureSkipTLSVerify,
+		p.onOutcome,
 	)
 }
 
diff --git a/packages/shared/pkg/proxy/pool/pool_test.go b/packages/shared/pkg/proxy/pool/pool_test.go
index 66f0c1fad2..525d748491 100644
--- a/packages/shared/pkg/proxy/pool/pool_test.go
+++ b/packages/shared/pkg/proxy/pool/pool_test.go
@@ -45,7 +45,7 @@ func TestPoolReusesClientAcrossInsecureSkipTLSVerify(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 
-			p := New(4, 1, time.Second, false)
+			p := New(4, 1, time.Second, false, nil)
 
 			client := p.Get(t.Context(), testDestination("lifecycle", tt.first))
 			require.NotNil(t, client)
@@ -59,7 +59,7 @@ func TestPoolReusesClientAcrossInsecureSkipTLSVerify(t *testing.T) {
 func TestPoolBuildsOneClientPerConnectionKey(t *testing.T) {
 	t.Parallel()
 
-	p := New(4, 1, time.Second, false)
+	p := New(4, 1, time.Second, false, nil)
 
 	first := p.Get(t.Context(), testDestination("lifecycle-a", true))
 	second := p.Get(t.Context(), testDestination("lifecycle-b", false))
@@ -74,7 +74,7 @@ func TestPoolBuildsOneClientPerConnectionKey(t *testing.T) {
 // register it. Cleanup must wait for that socket, not just its registry.
 func TestOwnerJoinsLateDial(t *testing.T) {
 	t.Parallel()
-	p := New(4, 1, time.Second, false)
+	p := New(4, 1, time.Second, false, nil)
 	d := testDestination("lifetime", false)
 	d.Owner = p.NewOwner()
 	client := p.Get(t.Context(), d)
@@ -133,7 +133,7 @@ func TestOwnerJoinsLateDial(t *testing.T) {
 
 func TestOwnerCancelsReadinessRetry(t *testing.T) {
 	t.Parallel()
-	p := New(4, 5, time.Second, false)
+	p := New(4, 5, time.Second, false, nil)
 	d := testDestination("retry", false)
 	d.Owner = p.NewOwner()
 	client := p.Get(t.Context(), d)
@@ -161,7 +161,7 @@ func TestOwnerCancelsReadinessRetry(t *testing.T) {
 
 func TestOwnerStopsTLSHandshake(t *testing.T) {
 	t.Parallel()
-	p := New(4, 1, time.Second, false)
+	p := New(4, 1, time.Second, false, nil)
 	listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0")
 	require.NoError(t, err)
 	defer listener.Close()
@@ -214,7 +214,7 @@ func (c resetErrorConn) Close() error {
 
 func TestResetErrorDoesNotRetainSettledOwner(t *testing.T) {
 	t.Parallel()
-	p := New(4, 1, time.Second, false)
+	p := New(4, 1, time.Second, false, nil)
 	d := testDestination("reset-error", false)
 	d.Owner = p.NewOwner()
 	client := p.Get(t.Context(), d)
@@ -241,7 +241,7 @@ func TestResetErrorDoesNotRetainSettledOwner(t *testing.T) {
 // to be refused through; that client is not counted as a pool member.
 func TestClientBuiltAfterStopIsNotCounted(t *testing.T) {
 	t.Parallel()
-	p := New(4, 1, time.Second, false)
+	p := New(4, 1, time.Second, false, nil)
 	d := testDestination("late", false)
 	d.Owner = p.NewOwner()
 	require.NoError(t, d.Owner.Close(t.Context()))
@@ -254,7 +254,7 @@ func TestClientBuiltAfterStopIsNotCounted(t *testing.T) {
 // that the eventual settlement does not give back.
 func TestRetriedCloseLeavesNothingBehind(t *testing.T) {
 	t.Parallel()
-	p := New(4, 1, time.Second, false)
+	p := New(4, 1, time.Second, false, nil)
 	d := testDestination("retried", false)
 	d.Owner = p.NewOwner()
 	client := p.Get(t.Context(), d)
diff --git a/packages/shared/pkg/proxy/proxy.go b/packages/shared/pkg/proxy/proxy.go
index 092e5e1868..a8fc76bade 100644
--- a/packages/shared/pkg/proxy/proxy.go
+++ b/packages/shared/pkg/proxy/proxy.go
@@ -53,12 +53,14 @@ func New(
 	getDestination func(r *http.Request) (*pool.Destination, error),
 	connLimitConfig *ConnectionLimitConfig,
 	disableKeepAlives bool,
+	onOutcome pool.OutcomeRecorder,
 ) *Proxy {
 	p := pool.New(
 		maxClientConns,
 		int(maxConnectionAttempts),
 		idleTimeout,
 		disableKeepAlives,
+		onOutcome,
 	)
 
 	proxy := &Proxy{
diff --git a/packages/shared/pkg/proxy/proxy_test.go b/packages/shared/pkg/proxy/proxy_test.go
index 0d4547a8ca..b8fef3343a 100644
--- a/packages/shared/pkg/proxy/proxy_test.go
+++ b/packages/shared/pkg/proxy/proxy_test.go
@@ -171,6 +171,7 @@ func newTestProxyWithConnLimit(t *testing.T, getDestination func(r *http.Request
 		getDestination,
 		connLimitConfig,
 		false,
+		nil,
 	)
 
 	// Start the proxy server
@@ -628,7 +629,7 @@ func TestProxyAnswersRequestTimeoutWhenTheRequestEndedBeforeRouting(t *testing.T
 
 	proxy := New(0, SandboxProxyRetries, time.Second, func(r *http.Request) (*pool.Destination, error) {
 		return nil, fmt.Errorf("resolving: %w", r.Context().Err())
-	}, nil, false)
+	}, nil, false, nil)
 	t.Cleanup(func() { _ = proxy.Close() })
 
 	ctx, cancel := context.WithCancel(t.Context())
@@ -1389,7 +1390,7 @@ func TestChangeResponseHeader(t *testing.T) {
 			IncludeSandboxIdInProxyErrorLogger: true,
 			MaskRequestHost:                    new(maskedHost),
 		}, nil
-	}, nil, false)
+	}, nil, false, nil)
 
 	go func() {
 		err := proxy.Serve(proxyListener)
@@ -1668,7 +1669,7 @@ func TestSharedIngressPolicy(t *testing.T) {
 				r.Header.Set("Origin", "https://caller.example")
 				r.Header.Set("Access-Control-Request-Method", "POST")
 				recorder := httptest.NewRecorder()
-				p := pool.New(4, 1, time.Second, false)
+				p := pool.New(4, 1, time.Second, false, nil)
 				handler(p, resolver, nil).ServeHTTP(recorder, r)
 				assert.GreaterOrEqual(t, recorder.Code, 200)
 				assert.Less(t, recorder.Code, 300)
diff --git a/packages/shared/pkg/proxy/shutdown_test.go b/packages/shared/pkg/proxy/shutdown_test.go
index 9741f19df1..9c5891623f 100644
--- a/packages/shared/pkg/proxy/shutdown_test.go
+++ b/packages/shared/pkg/proxy/shutdown_test.go
@@ -226,7 +226,7 @@ func TestProxyShutdownReleasesFailedH2CPreface(t *testing.T) {
 
 			proxy := New(0, SandboxProxyRetries, time.Minute, func(*http.Request) (*pool.Destination, error) {
 				return nil, errors.New("unexpected proxy request")
-			}, nil, false)
+			}, nil, false, nil)
 			hijacked := make(chan struct{})
 			proxy.ConnState = func(_ net.Conn, state http.ConnState) {
 				if state == http.StateHijacked {
diff --git a/packages/shared/pkg/telemetry/meters.go b/packages/shared/pkg/telemetry/meters.go
index a9b6cf0497..a82e4a4557 100644
--- a/packages/shared/pkg/telemetry/meters.go
+++ b/packages/shared/pkg/telemetry/meters.go
@@ -417,6 +417,19 @@ const (
 	// ClientProxyResumeParkingRejectedCounterName counts callers shed with a
 	// 503 because the parking lot was full when their flight first retried.
 	ClientProxyResumeParkingRejectedCounterName CounterType = "client_proxy.resume.parking.rejected"
+
+	// ClientProxyRequestOutcomeCounterName counts proxied requests by how they
+	// ended (outcome): ok | routing_failed | upstream_unreachable |
+	// client_closed | sandbox_not_found | too_many_connections. routing_failed
+	// vs upstream_unreachable separates a 500 (no destination) from a 502
+	// (destination resolved, node unreachable) without scraping logs; a spike
+	// in upstream_unreachable is the signal for stale routing records pointing
+	// at a drained node.
+	ClientProxyRequestOutcomeCounterName CounterType = "client_proxy.request.outcome"
+	// OrchestratorProxyRequestOutcomeCounterName is the orchestrator sandbox
+	// proxy's equivalent of ClientProxyRequestOutcomeCounterName, same outcome
+	// values; upstream_unreachable here is a closed sandbox port / unreachable guest.
+	OrchestratorProxyRequestOutcomeCounterName CounterType = "orchestrator.proxy.request.outcome"
 )
 
 const (
@@ -881,6 +894,8 @@ var counterDesc = map[CounterType]string{
 
 	ClientProxyResumeFlightCounterName:          "Callers of the per-sandbox resume flight at the client proxy (role=leader|joiner|none, result=served|error|canceled)",
 	ClientProxyResumeParkingRejectedCounterName: "Requests shed with a 503 because the client proxy parking lot was full at their flight's first retry",
+	ClientProxyRequestOutcomeCounterName:        "Client proxy requests by terminal outcome (ok | routing_failed | upstream_unreachable | client_closed | sandbox_not_found | too_many_connections).",
+	OrchestratorProxyRequestOutcomeCounterName:  "Orchestrator sandbox proxy requests by terminal outcome (ok | routing_failed | upstream_unreachable | client_closed | sandbox_not_found | too_many_connections).",
 
 	ApiOrchestratorPlacementsWithoutHugepageCeiling: "Hugepage sandboxes placed under a hugepage limit on a node the limit could not bound, by node and reason (no_capacity | no_page_size).",
 }
@@ -942,6 +957,8 @@ var counterUnits = map[CounterType]string{
 	TCPFirewallErrorsTotal:                       "{error}",
 	ClientProxyResumeFlightCounterName:           "{request}",
 	ClientProxyResumeParkingRejectedCounterName:  "{request}",
+	ClientProxyRequestOutcomeCounterName:         "{request}",
+	OrchestratorProxyRequestOutcomeCounterName:   "{request}",
 	TCPFirewallDecisionsTotal:                    "{decision}",
 
 	OrchestratorTemplateCacheAncestorResolutionsCounterName: "{ancestor}",

From 39deace8263ddfd760c5a7fd611ce03008ddef7a Mon Sep 17 00:00:00 2001
From: adababys <8872412+adababys@users.noreply.github.com>
Date: Wed, 28 Oct 2026 20:17:36 +0800
Subject: [PATCH 2/2] feat(proxy): label request-outcome metric by destination
 node
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The request-outcome counter shows that upstream-unreachable dials are
happening but not which node they target. During a scale-down a drained
node's routing record stays live (TTL = sandbox_max_length_in_hours,
deleted only on a clean MarkStopping), so client-proxy keeps resolving
sandboxes to it: resolution succeeds, and the failure surfaces only at dial
time. Without a node label the dashboard shows a 502 spike but not the
offending node.

Attribute each outcome to its destination node:
- add Node to pool.Destination; client-proxy sets it to the resolved
  SandboxInfo.OrchestratorID (worker instance, bounded cardinality — the
  same identity the hugepage-ceiling metric labels by).
- extend OnRequestOutcome to carry the node; the pool's ErrorHandler /
  ModifyResponse read it off the Destination, pre-routing handler branches
  pass  (recorded as node=unknown).
- client-proxy labels client_proxy.request.outcome by outcome + node; the
  orchestrator sandbox proxy leaves Node empty (its node is itself), so its
  metric is unchanged.

So client_proxy.request.outcome{outcome="upstream_unreachable",node=...}
isolates a drained/stale node directly.

Fixes #3759
---
 packages/client-proxy/internal/proxy/proxy.go | 13 ++++++++++--
 packages/orchestrator/pkg/proxy/proxy.go      |  4 +++-
 packages/shared/pkg/proxy/handler.go          | 10 +++++-----
 packages/shared/pkg/proxy/outcome_test.go     | 20 ++++++++++++++++++-
 packages/shared/pkg/proxy/pool/client.go      | 14 ++++++-------
 packages/shared/pkg/proxy/pool/destination.go |  6 ++++++
 packages/shared/pkg/proxy/pool/pool.go        | 15 ++++++++------
 packages/shared/pkg/telemetry/meters.go       |  2 +-
 8 files changed, 61 insertions(+), 23 deletions(-)

diff --git a/packages/client-proxy/internal/proxy/proxy.go b/packages/client-proxy/internal/proxy/proxy.go
index 797066bcd3..45e56820e3 100644
--- a/packages/client-proxy/internal/proxy/proxy.go
+++ b/packages/client-proxy/internal/proxy/proxy.go
@@ -200,8 +200,14 @@ func NewClientProxy(meterProvider metric.MeterProvider, serviceName string, port
 	if err != nil {
 		return nil, nil, fmt.Errorf("error registering client proxy request outcome metric (%s): %w", telemetry.ClientProxyRequestOutcomeCounterName, err)
 	}
-	recordOutcome := func(ctx context.Context, outcome pool.RequestOutcome) {
-		outcomeCounter.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", string(outcome))))
+	recordOutcome := func(ctx context.Context, outcome pool.RequestOutcome, node string) {
+		if node == "" {
+			node = "unknown"
+		}
+		outcomeCounter.Add(ctx, 1, metric.WithAttributes(
+			attribute.String("outcome", string(outcome)),
+			attribute.String("node", node),
+		))
 	}
 
 	getTargetFromRequest := reverseproxy.GetTargetFromRequest()
@@ -289,6 +295,9 @@ func NewClientProxy(meterProvider metric.MeterProvider, serviceName string, port
 				SandboxPort:   port,
 				ConnectionKey: pool.ClientProxyConnectionKey,
 				Url:           url,
+				// Label outcomes by the worker instance so an unreachable-upstream
+				// spike points straight at a drained/stale node.
+				Node: route.OrchestratorID,
 				MaskRequestHost: clientProxyMaskRequestHost(
 					ctx,
 					featureFlagsClient,
diff --git a/packages/orchestrator/pkg/proxy/proxy.go b/packages/orchestrator/pkg/proxy/proxy.go
index fa80aabbd7..749c7923da 100644
--- a/packages/orchestrator/pkg/proxy/proxy.go
+++ b/packages/orchestrator/pkg/proxy/proxy.go
@@ -121,7 +121,9 @@ func NewSandboxProxy(meterProvider metric.MeterProvider, port uint16, sandboxes
 	if err != nil {
 		return nil, fmt.Errorf("error registering orchestrator proxy request outcome metric (%s): %w", telemetry.OrchestratorProxyRequestOutcomeCounterName, err)
 	}
-	recordOutcome := func(ctx context.Context, outcome pool.RequestOutcome) {
+	// node is unused here: the orchestrator sandbox proxy's node is itself, so
+	// its destinations carry no Node and the label would always be "unknown".
+	recordOutcome := func(ctx context.Context, outcome pool.RequestOutcome, _ string) {
 		outcomeCounter.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", string(outcome))))
 	}
 
diff --git a/packages/shared/pkg/proxy/handler.go b/packages/shared/pkg/proxy/handler.go
index cb32661466..514d7cb57b 100644
--- a/packages/shared/pkg/proxy/handler.go
+++ b/packages/shared/pkg/proxy/handler.go
@@ -53,7 +53,7 @@ func handler(p *pool.ProxyPool, getDestination func(r *http.Request) (*pool.Dest
 		// request ended; a client that is gone reads nothing either way.
 		if ctxErr := ctx.Err(); err != nil && ctxErr != nil && errors.Is(err, ctxErr) {
 			l.Debug(ctx, "request ended before it was routed", zap.String("host", r.Host))
-			p.RecordOutcome(ctx, pool.OutcomeClientClosed)
+			p.RecordOutcome(ctx, pool.OutcomeClientClosed, "")
 			cors.Error(w, "Request ended before it was routed", http.StatusRequestTimeout)
 
 			return
@@ -98,7 +98,7 @@ func handler(p *pool.ProxyPool, getDestination func(r *http.Request) (*pool.Dest
 				zap.String("host", r.Host),
 				logger.WithSandboxID(notFoundErr.SandboxId))
 
-			p.RecordOutcome(ctx, pool.OutcomeSandboxNotFound)
+			p.RecordOutcome(ctx, pool.OutcomeSandboxNotFound, "")
 
 			err := template.
 				NewSandboxNotFoundError(notFoundErr.SandboxId, r.Host).
@@ -253,7 +253,7 @@ func handler(p *pool.ProxyPool, getDestination func(r *http.Request) (*pool.Dest
 
 		if err != nil {
 			l.Error(ctx, "failed to route request", zap.Error(err), zap.String("host", r.Host))
-			p.RecordOutcome(ctx, pool.OutcomeRoutingFailed)
+			p.RecordOutcome(ctx, pool.OutcomeRoutingFailed, "")
 			cors.Error(w, fmt.Sprintf("Unexpected error when routing request: %s", err), http.StatusInternalServerError)
 
 			return
@@ -283,7 +283,7 @@ func handler(p *pool.ProxyPool, getDestination func(r *http.Request) (*pool.Dest
 			if uploading {
 				closeUnreadBody(w)
 			}
-			p.RecordOutcome(ctx, pool.OutcomeUpstreamUnreachable)
+			p.RecordOutcome(ctx, pool.OutcomeUpstreamUnreachable, d.Node)
 			cors.Error(w, "Failed to route request to sandbox", http.StatusBadGateway)
 
 			return
@@ -327,7 +327,7 @@ func handler(p *pool.ProxyPool, getDestination func(r *http.Request) (*pool.Dest
 					connLimitConfig.OnConnectionBlocked(ctx)
 				}
 
-				p.RecordOutcome(ctx, pool.OutcomeTooManyConnections)
+				p.RecordOutcome(ctx, pool.OutcomeTooManyConnections, d.Node)
 
 				// The sandbox is reachable, but the proxy is refusing to dial it,
 				// so the proxy is the only party that can answer this preflight.
diff --git a/packages/shared/pkg/proxy/outcome_test.go b/packages/shared/pkg/proxy/outcome_test.go
index 7ad4ec83ff..791b607e88 100644
--- a/packages/shared/pkg/proxy/outcome_test.go
+++ b/packages/shared/pkg/proxy/outcome_test.go
@@ -21,12 +21,14 @@ import (
 type outcomeRecorder struct {
 	mu       sync.Mutex
 	outcomes []pool.RequestOutcome
+	nodes    []string
 }
 
-func (o *outcomeRecorder) add(_ context.Context, outcome pool.RequestOutcome) {
+func (o *outcomeRecorder) add(_ context.Context, outcome pool.RequestOutcome, node string) {
 	o.mu.Lock()
 	defer o.mu.Unlock()
 	o.outcomes = append(o.outcomes, outcome)
+	o.nodes = append(o.nodes, node)
 }
 
 func (o *outcomeRecorder) last() (pool.RequestOutcome, bool) {
@@ -39,6 +41,16 @@ func (o *outcomeRecorder) last() (pool.RequestOutcome, bool) {
 	return o.outcomes[len(o.outcomes)-1], true
 }
 
+func (o *outcomeRecorder) lastNode() (string, bool) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	if len(o.nodes) == 0 {
+		return "", false
+	}
+
+	return o.nodes[len(o.nodes)-1], true
+}
+
 // startOutcomeProxy stands up a proxy whose destination resolver returns
 // resolve(), recording every request outcome into rec. Returns the proxy URL.
 func startOutcomeProxy(t *testing.T, rec *outcomeRecorder, resolve func() (*pool.Destination, error)) string {
@@ -87,6 +99,7 @@ func TestProxyRecordsOutcome_OK(t *testing.T) {
 			SandboxPort:   backendPort,
 			ConnectionKey: "ok",
 			RequestLogger: logger.L(),
+			Node:          "node-42",
 		}, nil
 	})
 
@@ -99,6 +112,11 @@ func TestProxyRecordsOutcome_OK(t *testing.T) {
 
 		return ok && o == pool.OutcomeOK
 	}, 2*time.Second, 10*time.Millisecond, "expected an ok outcome")
+
+	// The destination's node is carried through to the recorder.
+	node, ok := rec.lastNode()
+	require.True(t, ok)
+	assert.Equal(t, "node-42", node)
 }
 
 func TestProxyRecordsOutcome_UpstreamUnreachable(t *testing.T) {
diff --git a/packages/shared/pkg/proxy/pool/client.go b/packages/shared/pkg/proxy/pool/client.go
index 0b5883118c..8f3965bfad 100644
--- a/packages/shared/pkg/proxy/pool/client.go
+++ b/packages/shared/pkg/proxy/pool/client.go
@@ -50,9 +50,9 @@ func newProxyClient(
 	insecureSkipTLSVerify bool,
 	onOutcome OutcomeRecorder,
 ) *ProxyClient {
-	recordOutcome := func(ctx context.Context, outcome RequestOutcome) {
+	recordOutcome := func(ctx context.Context, outcome RequestOutcome, node string) {
 		if onOutcome != nil {
-			onOutcome(ctx, outcome)
+			onOutcome(ctx, outcome, node)
 		}
 	}
 	activeConnections := smap.New[*tracking.Connection]()
@@ -164,7 +164,7 @@ func newProxyClient(
 			t, ok := pc.getDestination(r)
 			if !ok {
 				logger.L().Error(ctx, "proxy request without sandbox received error", zap.Error(err))
-				recordOutcome(ctx, OutcomeRoutingFailed)
+				recordOutcome(ctx, OutcomeRoutingFailed, "")
 				cors.Error(w, "Failed to route request to sandbox", http.StatusInternalServerError)
 
 				return
@@ -172,7 +172,7 @@ func newProxyClient(
 
 			if r.Host == "" { // kept around for historical reasons, unsure of usefulness. todo: find out if this is useful.
 				t.RequestLogger.Error(ctx, "error handler called from rewrite because of missing DestinationContext", zap.Error(err))
-				recordOutcome(ctx, OutcomeRoutingFailed)
+				recordOutcome(ctx, OutcomeRoutingFailed, t.Node)
 				cors.Error(w, "Failed to route request to sandbox", http.StatusInternalServerError)
 
 				return
@@ -198,9 +198,9 @@ func newProxyClient(
 			}
 
 			if clientGone {
-				recordOutcome(ctx, OutcomeClientClosed)
+				recordOutcome(ctx, OutcomeClientClosed, t.Node)
 			} else {
-				recordOutcome(ctx, OutcomeUpstreamUnreachable)
+				recordOutcome(ctx, OutcomeUpstreamUnreachable, t.Node)
 			}
 
 			if t.DefaultToPortError {
@@ -230,7 +230,7 @@ func newProxyClient(
 			// The upstream answered and the response is being forwarded: the
 			// request succeeded as far as the proxy is concerned, whatever the
 			// backend's own status code.
-			recordOutcome(ctx, OutcomeOK)
+			recordOutcome(ctx, OutcomeOK, t.Node)
 
 			if r.StatusCode >= 500 {
 				t.RequestLogger.Warn(
diff --git a/packages/shared/pkg/proxy/pool/destination.go b/packages/shared/pkg/proxy/pool/destination.go
index 4406b24357..f2ef11424f 100644
--- a/packages/shared/pkg/proxy/pool/destination.go
+++ b/packages/shared/pkg/proxy/pool/destination.go
@@ -24,6 +24,12 @@ type Destination struct {
 	// Embedders should pick a value that is unique per lifecycle.
 	ConnectionKey                      string
 	IncludeSandboxIdInProxyErrorLogger bool
+	// Node identifies the destination node (e.g. the orchestrator instance
+	// serving the sandbox), used only to label the request-outcome metric so an
+	// unreachable-upstream spike can be attributed to a single node. Empty when
+	// the embedder has no meaningful node identity (the orchestrator sandbox
+	// proxy, whose node is itself); recorded as "unknown" then.
+	Node string
 	// MaskRequestHost is used to mask the request host.
 	MaskRequestHost *string
 	// InsecureSkipTLSVerify is read once, when the pool builds the client for
diff --git a/packages/shared/pkg/proxy/pool/pool.go b/packages/shared/pkg/proxy/pool/pool.go
index b9a5c7e6ba..7c11edad5d 100644
--- a/packages/shared/pkg/proxy/pool/pool.go
+++ b/packages/shared/pkg/proxy/pool/pool.go
@@ -46,8 +46,10 @@ const (
 )
 
 // OutcomeRecorder is the callback the proxy calls once per request with its
-// terminal outcome. Nil is valid and means "do not record".
-type OutcomeRecorder func(ctx context.Context, outcome RequestOutcome)
+// terminal outcome and the destination node it was routed to (empty when no
+// destination was resolved or the embedder has no node identity). Nil is valid
+// and means "do not record".
+type OutcomeRecorder func(ctx context.Context, outcome RequestOutcome, node string)
 
 type ProxyPool struct {
 	clients               atomic.Int64
@@ -72,11 +74,12 @@ func New(maxClientConns int, maxConnectionAttempts int, idleTimeout time.Duratio
 	}
 }
 
-// RecordOutcome reports a request outcome through the configured recorder, if
-// any. Safe to call with a nil recorder.
-func (p *ProxyPool) RecordOutcome(ctx context.Context, outcome RequestOutcome) {
+// RecordOutcome reports a request outcome (with the destination node, which may
+// be empty) through the configured recorder, if any. Safe to call with a nil
+// recorder.
+func (p *ProxyPool) RecordOutcome(ctx context.Context, outcome RequestOutcome, node string) {
 	if p.onOutcome != nil {
-		p.onOutcome(ctx, outcome)
+		p.onOutcome(ctx, outcome, node)
 	}
 }
 
diff --git a/packages/shared/pkg/telemetry/meters.go b/packages/shared/pkg/telemetry/meters.go
index a82e4a4557..2d2dcb2032 100644
--- a/packages/shared/pkg/telemetry/meters.go
+++ b/packages/shared/pkg/telemetry/meters.go
@@ -894,7 +894,7 @@ var counterDesc = map[CounterType]string{
 
 	ClientProxyResumeFlightCounterName:          "Callers of the per-sandbox resume flight at the client proxy (role=leader|joiner|none, result=served|error|canceled)",
 	ClientProxyResumeParkingRejectedCounterName: "Requests shed with a 503 because the client proxy parking lot was full at their flight's first retry",
-	ClientProxyRequestOutcomeCounterName:        "Client proxy requests by terminal outcome (ok | routing_failed | upstream_unreachable | client_closed | sandbox_not_found | too_many_connections).",
+	ClientProxyRequestOutcomeCounterName:        "Client proxy requests by terminal outcome (ok | routing_failed | upstream_unreachable | client_closed | sandbox_not_found | too_many_connections) and destination node (orchestrator instance, or unknown).",
 	OrchestratorProxyRequestOutcomeCounterName:  "Orchestrator sandbox proxy requests by terminal outcome (ok | routing_failed | upstream_unreachable | client_closed | sandbox_not_found | too_many_connections).",
 
 	ApiOrchestratorPlacementsWithoutHugepageCeiling: "Hugepage sandboxes placed under a hugepage limit on a node the limit could not bound, by node and reason (no_capacity | no_page_size).",