Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 23 additions & 8 deletions cmd/api/api/instances.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,25 @@ import (
"github.com/samber/lo"
)

// toDomainNetworkEgress maps the OpenAPI egress policy to the domain type.
// Normalization and validation happen in the instances package.
func toDomainNetworkEgress(egress *oapi.CreateInstanceRequestNetworkEgress) *instances.NetworkEgressPolicy {
if egress == nil {
return nil
}
enabled := egress.Enabled != nil && *egress.Enabled
policy := &instances.NetworkEgressPolicy{Enabled: enabled}
if egress.Proxy != nil {
policy.Proxy = instances.EgressProxyMode(*egress.Proxy)
}
if egress.Enforcement != nil && egress.Enforcement.Mode != nil {
policy.EnforcementMode = instances.EgressEnforcementMode(*egress.Enforcement.Mode)
} else if enabled {
policy.EnforcementMode = instances.EgressEnforcementModeAll
}
return policy
}

// ListInstances lists instances, optionally filtered by state and/or tags.
func (s *ApiService) ListInstances(ctx context.Context, request oapi.ListInstancesRequestObject) (oapi.ListInstancesResponseObject, error) {
log := logger.FromContext(ctx)
Expand Down Expand Up @@ -140,14 +159,8 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
networkEnabled = *request.Body.Network.Enabled
}
var networkEgress *instances.NetworkEgressPolicy
if request.Body.Network != nil && request.Body.Network.Egress != nil {
enabled := request.Body.Network.Egress.Enabled != nil && *request.Body.Network.Egress.Enabled
networkEgress = &instances.NetworkEgressPolicy{Enabled: enabled}
if request.Body.Network.Egress.Enforcement != nil && request.Body.Network.Egress.Enforcement.Mode != nil {
networkEgress.EnforcementMode = instances.EgressEnforcementMode(*request.Body.Network.Egress.Enforcement.Mode)
} else if enabled {
networkEgress.EnforcementMode = instances.EgressEnforcementModeAll
}
if request.Body.Network != nil {
networkEgress = toDomainNetworkEgress(request.Body.Network.Egress)
}
var credentials map[string]instances.CredentialPolicy
if request.Body.Credentials != nil {
Expand Down Expand Up @@ -1039,6 +1052,8 @@ func (s *ApiService) UpdateInstance(ctx context.Context, request oapi.UpdateInst
HealthCheck: healthCheck,
RestartPolicy: restartPolicy,
RestartPolicySet: request.Body.RestartPolicy != nil,
NetworkEgress: toDomainNetworkEgress(request.Body.Egress),
NetworkEgressSet: request.Body.Egress != nil,
})
if err != nil {
switch {
Expand Down
6 changes: 5 additions & 1 deletion lib/egressproxy/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ That means the VM can make authenticated outbound requests without ever receivin
- Per instance, `network.egress.enforcement.mode` controls host-side direct egress blocking:
- `all` (default when proxy is enabled): reject direct non-proxy TCP egress from the VM TAP interface.
- `http_https_only`: reject direct TCP egress only on destination ports `80` and `443`.
- `all_traffic`: reject direct TCP and UDP egress. This also blocks QUIC and direct DNS, so the guest must resolve names through its proxy.
- Per instance, `network.egress.proxy` selects the host-side proxy behavior:
- `mitm` (default): the full mediated path described in this document.
- `none`: enforcement only. The host applies the TAP-level egress rules but does not run the MITM proxy, inject `HTTP_PROXY`/`HTTPS_PROXY` into the guest, or install the proxy CA. Intended for guests configured to reach an external proxy at the network gateway. Credentials require `mitm`. Enforcement-only policies can also be set or cleared on a live instance via `PATCH /instances/{id}` (`egress`); `mitm` policies cannot, since their guest-visible configuration is fixed at boot.
- Inside the VM, each credential key is rewritten to `mock-<CREDENTIAL_NAME>` (for example `mock-OUTBOUND_OPENAI_KEY`).
- Header injection is applied to HTTPS requests only after MITM decryption.
- For HTTPS egress, the proxy validates upstream TLS certificates with the host trust store before forwarding.
Expand Down Expand Up @@ -64,7 +68,7 @@ Operationally, this is intended for key rotation, revocation/reissue flows, and
- Real secret values are persisted in the normal instance `env` metadata, which is already host-side state.
- TLS interception requires guest trust of the proxy CA; hypeman installs this CA in the guest when proxy mode is enabled.
- Egress enforcement is applied per instance TAP device and removed when the instance stops/standbys/deletes.
- Enforcement intentionally targets TCP egress only. DNS/other non-TCP traffic is not rewritten and is not blocked by `all` mode.
- The `all` and `http_https_only` modes target TCP egress only; DNS/other non-TCP traffic is not rewritten and not blocked. Use `all_traffic` to also block direct UDP egress.

## Limits of enforcement

Expand Down
14 changes: 14 additions & 0 deletions lib/egressproxy/enforce.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package egressproxy

// ApplyEnforcement installs host-side egress enforcement for an instance TAP
// device without registering the instance with the MITM proxy service. Used
// for enforcement-only egress policies (network.egress.proxy=none).
func ApplyEnforcement(instanceID, tapDevice, gatewayIP string, blockAllTCPEgress, blockUDPEgress bool) error {
return applyEgressEnforcement(instanceID, tapDevice, gatewayIP, blockAllTCPEgress, blockUDPEgress)
}

// RemoveEnforcement removes any host-side egress enforcement rules for an
// instance. Safe to call for instances that never had enforcement applied.
func RemoveEnforcement(instanceID string) error {
return removeEgressEnforcement(instanceID)
}
32 changes: 21 additions & 11 deletions lib/egressproxy/enforce_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,34 +13,46 @@ const (
enforcementSuffixPort80 = "80"
enforcementSuffixPort443 = "443"
enforcementSuffixAllTCP = "all-tcp"
enforcementSuffixAllUDP = "all-udp"
)

func applyEgressEnforcement(instanceID, tapDevice, gatewayIP string, proxyPort int, blockAllTCPEgress bool) error {
if instanceID == "" || tapDevice == "" || gatewayIP == "" || proxyPort <= 0 {
func applyEgressEnforcement(instanceID, tapDevice, gatewayIP string, blockAllTCPEgress, blockUDPEgress bool) error {
if instanceID == "" || tapDevice == "" || gatewayIP == "" {
return fmt.Errorf("invalid egress enforcement inputs")
}

comment80 := enforcementComment(instanceID, enforcementSuffixPort80)
comment443 := enforcementComment(instanceID, enforcementSuffixPort443)
commentAllTCP := enforcementComment(instanceID, enforcementSuffixAllTCP)
commentAllUDP := enforcementComment(instanceID, enforcementSuffixAllUDP)

// Clean old rules first so updates are idempotent across restarts and mode changes.
_ = removeRuleByComment(comment80)
_ = removeRuleByComment(comment443)
_ = removeRuleByComment(commentAllTCP)
_ = removeRuleByComment(commentAllUDP)

if blockUDPEgress {
if err := insertRejectAllRule(tapDevice, gatewayIP, "udp", commentAllUDP); err != nil {
return fmt.Errorf("insert all-udp egress enforcement: %w", err)
}
}

if blockAllTCPEgress {
if err := insertRejectAllTCPRule(tapDevice, gatewayIP, commentAllTCP); err != nil {
if err := insertRejectAllRule(tapDevice, gatewayIP, "tcp", commentAllTCP); err != nil {
_ = removeRuleByComment(commentAllUDP)
return fmt.Errorf("insert all-tcp egress enforcement: %w", err)
}
return nil
}

if err := insertRejectRule(tapDevice, gatewayIP, 80, comment80); err != nil {
_ = removeRuleByComment(commentAllUDP)
return fmt.Errorf("insert port 80 egress enforcement: %w", err)
}
if err := insertRejectRule(tapDevice, gatewayIP, 443, comment443); err != nil {
_ = removeRuleByComment(comment80)
_ = removeRuleByComment(commentAllUDP)
return fmt.Errorf("insert port 443 egress enforcement: %w", err)
}

Expand All @@ -51,12 +63,10 @@ func removeEgressEnforcement(instanceID string) error {
if instanceID == "" {
return nil
}
comment80 := enforcementComment(instanceID, enforcementSuffixPort80)
comment443 := enforcementComment(instanceID, enforcementSuffixPort443)
commentAllTCP := enforcementComment(instanceID, enforcementSuffixAllTCP)
_ = removeRuleByComment(comment80)
_ = removeRuleByComment(comment443)
_ = removeRuleByComment(commentAllTCP)
_ = removeRuleByComment(enforcementComment(instanceID, enforcementSuffixPort80))
_ = removeRuleByComment(enforcementComment(instanceID, enforcementSuffixPort443))
_ = removeRuleByComment(enforcementComment(instanceID, enforcementSuffixAllTCP))
_ = removeRuleByComment(enforcementComment(instanceID, enforcementSuffixAllUDP))
return nil
}

Expand All @@ -76,11 +86,11 @@ func insertRejectRule(tapDevice, gatewayIP string, port int, comment string) err
return nil
}

func insertRejectAllTCPRule(tapDevice, gatewayIP, comment string) error {
func insertRejectAllRule(tapDevice, gatewayIP, protocol, comment string) error {
cmd := exec.Command(
"iptables", "-I", "FORWARD", "1",
"-i", tapDevice,
"-p", "tcp",
"-p", protocol,
"!", "-d", gatewayIP,
"-m", "comment", "--comment", comment,
"-j", "REJECT",
Expand Down
3 changes: 1 addition & 2 deletions lib/egressproxy/enforce_other.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@

package egressproxy

func applyEgressEnforcement(instanceID, tapDevice, gatewayIP string, proxyPort int, blockAllTCPEgress bool) error {
_ = blockAllTCPEgress
func applyEgressEnforcement(instanceID, tapDevice, gatewayIP string, blockAllTCPEgress, blockUDPEgress bool) error {
return nil
}

Expand Down
9 changes: 6 additions & 3 deletions lib/egressproxy/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ func (s *Service) RegisterInstance(ctx context.Context, gatewayIP string, cfg In
log := s.loggerForContext(ctx)
result := "error"
var opErr error
enforcementMode := enforcementModeLabel(cfg.BlockAllTCPEgress)
enforcementMode := enforcementModeLabel(cfg.BlockAllTCPEgress, cfg.BlockUDPEgress)
ctx, span := s.startControlPlaneSpan(ctx, "EgressProxy.RegisterInstance",
attribute.String("operation", "register"),
attribute.String("enforcement_mode", enforcementMode),
Expand All @@ -212,7 +212,7 @@ func (s *Service) RegisterInstance(ctx context.Context, gatewayIP string, cfg In
return GuestConfig{}, err
}

if err := applyEgressEnforcement(cfg.InstanceID, cfg.TAPDevice, gatewayIP, s.listenPort, cfg.BlockAllTCPEgress); err != nil {
if err := applyEgressEnforcement(cfg.InstanceID, cfg.TAPDevice, gatewayIP, cfg.BlockAllTCPEgress, cfg.BlockUDPEgress); err != nil {
log.WarnContext(ctx, "failed to apply egress proxy enforcement", "instance_id", cfg.InstanceID, "error", err)
opErr = err
return GuestConfig{}, err
Expand Down Expand Up @@ -651,7 +651,10 @@ func (s *Service) finishControlPlaneSpan(span trace.Span, result string, err err
span.End()
}

func enforcementModeLabel(blockAllTCPEgress bool) string {
func enforcementModeLabel(blockAllTCPEgress, blockUDPEgress bool) string {
if blockAllTCPEgress && blockUDPEgress {
return "all_traffic"
}
if blockAllTCPEgress {
return "all"
}
Expand Down
1 change: 1 addition & 0 deletions lib/egressproxy/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ type InstanceConfig struct {
SourceIP string
TAPDevice string
BlockAllTCPEgress bool
BlockUDPEgress bool
HeaderInjectRules []HeaderInjectRuleConfig
}

Expand Down
8 changes: 8 additions & 0 deletions lib/instances/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -586,6 +586,11 @@ func validateCreateRequest(req *CreateInstanceRequest) error {
return err
}
req.NetworkEgress.EnforcementMode = mode
proxyMode, err := normalizeEgressProxyMode(req.NetworkEgress.Proxy)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Enforcement-only create rollback leak

High Severity

With network.egress.proxy=none, maybeRegisterEgressProxy installs host iptables rules but returns a nil guest config, preventing cleanup registration. If instance creation or startup fails, these rules are not removed, leaving stale FORWARD rules on the host's TAP interface.

Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit 67806b6. Configure here.

if err != nil {
return err
}
req.NetworkEgress.Proxy = proxyMode
}
normalizedCredentials, err := normalizeCredentialPolicies(req.Credentials)
if err != nil {
Expand All @@ -596,6 +601,9 @@ func validateCreateRequest(req *CreateInstanceRequest) error {
if req.NetworkEgress == nil || !req.NetworkEgress.Enabled {
return fmt.Errorf("%w: credentials require network.egress.enabled=true", ErrInvalidRequest)
}
if req.NetworkEgress.Proxy == EgressProxyModeNone {
return fmt.Errorf("%w: credentials require network.egress.proxy=mitm", ErrInvalidRequest)
}
if err := validateCredentialEnvBindings(normalizedCredentials, req.Env); err != nil {
return err
}
Expand Down
103 changes: 103 additions & 0 deletions lib/instances/create_egress_proxy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,109 @@ func TestValidateCreateRequest_AllowsHTTPHTTPSOnlyEgressMode(t *testing.T) {
assert.Equal(t, EgressEnforcementModeHTTPHTTPSOnly, cfg.EnforcementMode)
}

func TestValidateCreateRequest_NormalizesEgressProxyMode(t *testing.T) {
t.Parallel()

cfg := &NetworkEgressPolicy{Enabled: true}
req := CreateInstanceRequest{
Name: "test-egress",
Image: "docker.io/library/alpine:latest",
NetworkEnabled: true,
NetworkEgress: cfg,
}

err := validateCreateRequest(&req)
require.NoError(t, err)
assert.Equal(t, EgressProxyModeMITM, cfg.Proxy)
}

func TestValidateCreateRequest_RejectsInvalidEgressProxyMode(t *testing.T) {
t.Parallel()

req := CreateInstanceRequest{
Name: "test-egress",
Image: "docker.io/library/alpine:latest",
NetworkEnabled: true,
NetworkEgress: &NetworkEgressPolicy{
Enabled: true,
Proxy: EgressProxyMode("bogus"),
},
}

err := validateCreateRequest(&req)
require.Error(t, err)
assert.ErrorIs(t, err, ErrInvalidRequest)
assert.Contains(t, err.Error(), "invalid network.egress.proxy")
}

func TestValidateCreateRequest_AllowsEnforcementOnlyEgress(t *testing.T) {
t.Parallel()

cfg := &NetworkEgressPolicy{
Enabled: true,
Proxy: EgressProxyModeNone,
EnforcementMode: EgressEnforcementModeAllTraffic,
}
req := CreateInstanceRequest{
Name: "test-egress",
Image: "docker.io/library/alpine:latest",
NetworkEnabled: true,
NetworkEgress: cfg,
}

err := validateCreateRequest(&req)
require.NoError(t, err)
assert.Equal(t, EgressProxyModeNone, cfg.Proxy)
assert.Equal(t, EgressEnforcementModeAllTraffic, cfg.EnforcementMode)
}

func TestValidateCreateRequest_CredentialsRequireMITMProxy(t *testing.T) {
t.Parallel()

req := CreateInstanceRequest{
Name: "test-egress",
Image: "docker.io/library/alpine:latest",
NetworkEnabled: true,
Env: map[string]string{"OUTBOUND_OPENAI_KEY": "real"},
NetworkEgress: &NetworkEgressPolicy{
Enabled: true,
Proxy: EgressProxyModeNone,
},
Credentials: map[string]CredentialPolicy{
"OUTBOUND_OPENAI_KEY": {
Source: CredentialSource{Env: "OUTBOUND_OPENAI_KEY"},
Inject: []CredentialInjectRule{{
As: CredentialInjectAs{Header: "Authorization", Format: "Bearer ${value}"},
}},
},
},
}

err := validateCreateRequest(&req)
require.Error(t, err)
assert.ErrorIs(t, err, ErrInvalidRequest)
assert.Contains(t, err.Error(), "credentials require network.egress.proxy=mitm")
}

func TestEgressBlockFlags(t *testing.T) {
t.Parallel()

cases := []struct {
mode EgressEnforcementMode
blockAllTCP bool
blockUDP bool
}{
{EgressEnforcementModeAll, true, false},
{EgressEnforcementModeHTTPHTTPSOnly, false, false},
{EgressEnforcementModeAllTraffic, true, true},
}
for _, tc := range cases {
blockAllTCP, blockUDP := egressBlockFlags(tc.mode)
assert.Equal(t, tc.blockAllTCP, blockAllTCP, "mode %s", tc.mode)
assert.Equal(t, tc.blockUDP, blockUDP, "mode %s", tc.mode)
}
}

func TestValidateCreateRequest_NormalizesCredentialsInPlace(t *testing.T) {
t.Parallel()

Expand Down
Loading
Loading