diff --git a/internal/purchase/ladder_visibility.go b/internal/purchase/ladder_visibility.go new file mode 100644 index 00000000..1c5ec710 --- /dev/null +++ b/internal/purchase/ladder_visibility.go @@ -0,0 +1,296 @@ +package purchase + +import ( + "context" + "errors" + "fmt" + "slices" + "time" + + "github.com/LeanerCloud/cloud-commitments-go/pkg/ladder" + "github.com/LeanerCloud/cloud-commitments-go/pkg/logging" + "github.com/LeanerCloud/cloud-commitments-platform/internal/config" + "github.com/aws/aws-sdk-go-v2/aws" + sdksp "github.com/aws/aws-sdk-go-v2/service/savingsplans" + sptypes "github.com/aws/aws-sdk-go-v2/service/savingsplans/types" +) + +// ladderVisibleSPStates is the set of Savings Plan states a commitment must be +// in to count as visible. It MUST equal what the pinned AWS ladder passes to +// DescribeSavingsPlans when it builds ExistingUSDPerHour for GetLayerStates +// (providers/aws/ladder/adapters.go spListerStates, unexported, so it cannot be +// imported). TestLadderVisibleStatesMatchGetLayerStates pins the equivalence +// against the real AWSLadder. If this set were wider than the ladder's, a +// tranche would complete while netting still cannot see it and the next run +// would over-buy. +var ladderVisibleSPStates = []sptypes.SavingsPlanState{ + sptypes.SavingsPlanStateActive, + sptypes.SavingsPlanStatePaymentPending, +} + +// ladderDescribeBatch is how many plan IDs go into one DescribeSavingsPlans call. +const ladderDescribeBatch = 100 + +// SPDescriber is the one Savings Plans API call the visibility prefetch needs. +// *savingsplans.Client satisfies it. +type SPDescriber interface { + DescribeSavingsPlans(ctx context.Context, in *sdksp.DescribeSavingsPlansInput, optFns ...func(*sdksp.Options)) (*sdksp.DescribeSavingsPlansOutput, error) +} + +// LadderSPDescriberFactory returns a describer authenticated for one cloud +// account and region. +type LadderSPDescriberFactory func(ctx context.Context, cloudAccountID, region string) (SPDescriber, error) + +// LadderSPDescriber is the production factory: the same per-account credential +// resolution the purchase itself used. +func (m *Manager) LadderSPDescriber(ctx context.Context, cloudAccountID, region string) (SPDescriber, error) { + account, err := m.config.GetCloudAccount(ctx, cloudAccountID) + if err != nil { + return nil, fmt.Errorf("load cloud account %s: %w", cloudAccountID, err) + } + if account == nil { + return nil, fmt.Errorf("cloud account %s not found", cloudAccountID) + } + provCfg, err := m.resolveAccountProvider(ctx, *account) + if err != nil { + return nil, err + } + if provCfg.AWSCredentialsProvider == nil { + return nil, fmt.Errorf("cloud account %s resolved without AWS credentials", cloudAccountID) + } + return sdksp.NewFromConfig(aws.Config{Region: region, Credentials: provCfg.AWSCredentialsProvider}), nil +} + +type ladderVisibility struct { + err error + visible bool +} + +// prefetchedVisibility is a pure lookup over results fetched before the +// reconciler opens any transaction. Unknown executions fail closed. +type prefetchedVisibility map[string]ladderVisibility + +func (p prefetchedVisibility) CommitmentVisible(_ context.Context, _ config.LadderTrancheDB, executionID string) (bool, error) { + v, ok := p[executionID] + if !ok { + return false, fmt.Errorf("execution %s was not prefetched", executionID) + } + return v.visible, v.err +} + +// expectedPlan is one commitment a completed execution should have created. +type expectedPlan struct { + id, planType, region string +} + +// spPlanTypeForLayer is the SavingsPlanType GetLayerStates sums into a layer. +func spPlanTypeForLayer(l ladder.LayerType) (sptypes.SavingsPlanType, error) { + switch l { + case ladder.LayerComputeSP: + return sptypes.SavingsPlanTypeCompute, nil + case ladder.LayerEC2InstanceSP: + return sptypes.SavingsPlanTypeEc2Instance, nil + } + return "", fmt.Errorf("layer %q is not a savings plan layer", l) +} + +// spCountsAsExisting mirrors the ladder's ExistingUSDPerHour predicate for one +// plan: state in ladderVisibleSPStates, plan type equal to the layer's, and an +// EC2Instance plan only in the ladder's region (Compute plans are global). +func spCountsAsExisting(sp sptypes.SavingsPlan, e expectedPlan) bool { + return slices.Contains(ladderVisibleSPStates, sp.State) && + string(sp.SavingsPlanType) == e.planType && + (sp.SavingsPlanType != sptypes.SavingsPlanTypeEc2Instance || aws.ToString(sp.Region) == e.region) +} + +type ladderFetchGroup struct { + account, region string + execs map[string][]expectedPlan // execution ID -> its plans +} + +// PrefetchLadderVisibility resolves, outside any transaction, whether the +// commitments of each fired tranche's completed execution are visible. It +// makes no provider call when nothing is fired or no execution has completed. +// Any lookup or API error is recorded against the affected executions and +// logged; nothing is ever reported visible on an error. +func PrefetchLadderVisibility(ctx context.Context, store interface { + GetExecutionByID(ctx context.Context, id string) (*config.PurchaseExecution, error) +}, factory LadderSPDescriberFactory, fired []config.LadderTrancheDB) prefetchedVisibility { + out := prefetchedVisibility{} + groups := map[string]*ladderFetchGroup{} + for i := range fired { + tr := fired[i] + if tr.ExecutionID == nil { + continue + } + exec, err := chainTail(ctx, store, *tr.ExecutionID) + if err == nil && exec != nil && exec.Status == config.StatusCompleted { + err = addExpected(groups, exec, tr) + } + if err != nil { + logging.Errorf("ladder tranche %s: visibility prefetch failed: %v", tr.ID, err) + if exec != nil { + out[exec.ExecutionID] = ladderVisibility{err: err} + } + } + } + for _, g := range groups { + fetchGroup(ctx, factory, g, out) + } + return out +} + +func chainTail(ctx context.Context, store interface { + GetExecutionByID(ctx context.Context, id string) (*config.PurchaseExecution, error) +}, id string) (*config.PurchaseExecution, error) { + var exec *config.PurchaseExecution + for range config.LadderChainMaxDepth { + var err error + if exec, err = store.GetExecutionByID(ctx, id); err != nil { + return nil, err + } + if exec == nil { + return nil, fmt.Errorf("execution %s not found", id) + } + if exec.RetryExecutionID == nil { + return exec, nil + } + id = *exec.RetryExecutionID + } + return exec, fmt.Errorf("retry chain longer than %d", config.LadderChainMaxDepth) +} + +func addExpected(groups map[string]*ladderFetchGroup, exec *config.PurchaseExecution, tr config.LadderTrancheDB) error { + planType, err := spPlanTypeForLayer(tr.LayerType) + if err != nil { + return err + } + account, err := SingleCloudAccountIDFromRecs(exec.Recommendations) + if err != nil { + return err + } + if exec.CloudAccountID != nil { + account = exec.CloudAccountID + } + var plans []expectedPlan + region := "" + for i := range exec.Recommendations { + r := &exec.Recommendations[i] + if !r.Selected { + continue + } + if r.PurchaseID == "" { + return fmt.Errorf("completed execution %s has a selected recommendation without a commitment ID", exec.ExecutionID) + } + plans = append(plans, expectedPlan{id: r.PurchaseID, planType: string(planType), region: r.Region}) + region = r.Region + } + if account == nil || len(plans) == 0 { + return fmt.Errorf("completed execution %s has no account or commitment to look up", exec.ExecutionID) + } + key := *account + "/" + region + g := groups[key] + if g == nil { + g = &ladderFetchGroup{account: *account, region: region, execs: map[string][]expectedPlan{}} + groups[key] = g + } + g.execs[exec.ExecutionID] = plans + return nil +} + +// fetchGroup issues one batched per-ID describe per (account, region) and +// marks an execution visible only when every one of its plans is returned and +// satisfies spCountsAsExisting. +func fetchGroup(ctx context.Context, factory LadderSPDescriberFactory, g *ladderFetchGroup, out prefetchedVisibility) { + fail := func(err error) { + logging.Errorf("ladder visibility lookup for account %s region %s failed: %v", g.account, g.region, err) + for id := range g.execs { + out[id] = ladderVisibility{err: err} + } + } + var ids []string + for _, plans := range g.execs { + for _, p := range plans { + if !slices.Contains(ids, p.id) { + ids = append(ids, p.id) + } + } + } + desc, err := factory(ctx, g.account, g.region) + if err != nil { + fail(err) + return + } + found := map[string]sptypes.SavingsPlan{} + for chunk := range slices.Chunk(ids, ladderDescribeBatch) { + if err := describeIDs(ctx, desc, chunk, found); err != nil { + fail(err) + return + } + } + for id, plans := range g.execs { + out[id] = ladderVisibility{visible: allCountAsExisting(plans, found)} + } +} + +func allCountAsExisting(plans []expectedPlan, found map[string]sptypes.SavingsPlan) bool { + for _, p := range plans { + sp, ok := found[p.id] + if !ok || !spCountsAsExisting(sp, p) { + return false + } + } + return true +} + +func describeIDs(ctx context.Context, desc SPDescriber, ids []string, found map[string]sptypes.SavingsPlan) error { + var token *string + seen := map[string]bool{} + for { + res, err := desc.DescribeSavingsPlans(ctx, &sdksp.DescribeSavingsPlansInput{ + SavingsPlanIds: ids, States: ladderVisibleSPStates, NextToken: token, + }) + if err != nil { + return fmt.Errorf("DescribeSavingsPlans: %w", err) + } + for i := range res.SavingsPlans { + found[aws.ToString(res.SavingsPlans[i].SavingsPlanId)] = res.SavingsPlans[i] + } + tok := aws.ToString(res.NextToken) + if tok == "" { + return nil + } + if seen[tok] { + return errors.New("DescribeSavingsPlans returned a repeated pagination token") + } + seen[tok] = true + token = res.NextToken + } +} + +// LadderReconcileDeps is what one scheduled reconcile run needs. +type LadderReconcileDeps struct { + Store ladderReconcileDB + SP LadderSPDescriberFactory + Now func() time.Time +} + +type ladderReconcileDB interface { + LadderReconcileStore + GetExecutionByID(ctx context.Context, id string) (*config.PurchaseExecution, error) +} + +// RunLadderReconcile is the entry point for the future scheduled reconcile +// task: prefetch visibility, then ReconcileLadderTranches against the map. +// Nothing schedules it yet. With no fired tranche it touches no provider API. +func RunLadderReconcile(ctx context.Context, d LadderReconcileDeps) (LadderReconcileResult, error) { + fired, err := d.Store.ListFiredLadderTranches(ctx, ladderReconcileBatch) + if err != nil { + return LadderReconcileResult{}, err + } + if len(fired) == 0 { + return LadderReconcileResult{}, nil + } + vis := PrefetchLadderVisibility(ctx, d.Store, d.SP, fired) + return ReconcileLadderTranches(ctx, d.Store, vis, d.Now()) +} diff --git a/internal/purchase/ladder_visibility_test.go b/internal/purchase/ladder_visibility_test.go new file mode 100644 index 00000000..3e7fd91a --- /dev/null +++ b/internal/purchase/ladder_visibility_test.go @@ -0,0 +1,227 @@ +package purchase + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/LeanerCloud/cloud-commitments-go/pkg/ladder" + "github.com/LeanerCloud/cloud-commitments-platform/internal/config" + "github.com/aws/aws-sdk-go-v2/aws" + sdksp "github.com/aws/aws-sdk-go-v2/service/savingsplans" + sptypes "github.com/aws/aws-sdk-go-v2/service/savingsplans/types" + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// visFake is both the reconcile store and the SP API. inTx lets the API assert +// it is never reached while a reconcile transaction is open. +type visFake struct { + mu sync.Mutex + inTx bool + callsInTx int + calls int + factoryHits int + factoryAccts []string + apiErr error + plans []sptypes.SavingsPlan + fired []config.LadderTrancheDB + execs map[string]*config.PurchaseExecution + completed []string + lastStates []sptypes.SavingsPlanState +} + +func (f *visFake) ListFiredLadderTranches(context.Context, int) ([]config.LadderTrancheDB, error) { + return f.fired, nil +} +func (f *visFake) WithTx(_ context.Context, fn func(pgx.Tx) error) error { + f.inTx = true + defer func() { f.inTx = false }() + return fn(nil) +} +func (f *visFake) LockLadderTrancheChainTx(_ context.Context, _ pgx.Tx, id string) ([]config.LadderChainNode, error) { + for _, tr := range f.fired { + if tr.ID == id { + return []config.LadderChainNode{{ExecutionID: *tr.ExecutionID, Status: f.execs[*tr.ExecutionID].Status, CreatedAt: time.Unix(0, 0)}}, nil + } + } + return nil, config.ErrLadderTrancheNotFired +} +func (f *visFake) TransitionLadderTrancheTx(_ context.Context, _ pgx.Tx, id string, _, to ladder.TrancheStatus, _ string) (bool, error) { + if to == ladder.TrancheStatusCompleted { + f.completed = append(f.completed, id) + } + return true, nil +} +func (f *visFake) GetExecutionByID(_ context.Context, id string) (*config.PurchaseExecution, error) { + return f.execs[id], nil +} +func (f *visFake) DescribeSavingsPlans(_ context.Context, in *sdksp.DescribeSavingsPlansInput, _ ...func(*sdksp.Options)) (*sdksp.DescribeSavingsPlansOutput, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls++ + f.lastStates = in.States + if f.inTx { + f.callsInTx++ + } + if f.apiErr != nil { + return nil, f.apiErr + } + return &sdksp.DescribeSavingsPlansOutput{SavingsPlans: f.plans}, nil +} +func (f *visFake) factory(_ context.Context, acct, _ string) (SPDescriber, error) { + f.factoryHits++ + f.factoryAccts = append(f.factoryAccts, acct) + return f, nil +} + +func (f *visFake) addTranche(id, acct, spID string, layer ladder.LayerType) { + execID := "exec-" + id + f.fired = append(f.fired, config.LadderTrancheDB{ID: id, ExecutionID: &execID, LayerType: layer, Status: ladder.TrancheStatusFired}) + if f.execs == nil { + f.execs = map[string]*config.PurchaseExecution{} + } + f.execs[execID] = &config.PurchaseExecution{ + ExecutionID: execID, Status: config.StatusCompleted, CloudAccountID: aws.String(acct), + Recommendations: []config.RecommendationRecord{{Selected: true, PurchaseID: spID, Region: "us-east-1"}}, + } +} + +func sp(id string, typ sptypes.SavingsPlanType, state sptypes.SavingsPlanState, region string) sptypes.SavingsPlan { + p := sptypes.SavingsPlan{SavingsPlanId: aws.String(id), SavingsPlanType: typ, State: state} + if region != "" { + p.Region = aws.String(region) + } + return p +} + +func (f *visFake) run(t *testing.T) LadderReconcileResult { + t.Helper() + res, err := RunLadderReconcile(context.Background(), LadderReconcileDeps{Store: f, SP: f.factory, Now: time.Now}) + require.NoError(t, err) + return res +} + +func TestRunLadderReconcile_NoFiredMakesNoProviderCall(t *testing.T) { + f := &visFake{} + res := f.run(t) + assert.Zero(t, res.Checked) + assert.Zero(t, f.calls) + assert.Zero(t, f.factoryHits) +} + +func TestRunLadderReconcile_VisibleCompletesWithProviderCallsOutsideTx(t *testing.T) { + f := &visFake{} + f.addTranche("t1", "acct-a", "sp-1", ladder.LayerComputeSP) + f.plans = []sptypes.SavingsPlan{sp("sp-1", sptypes.SavingsPlanTypeCompute, sptypes.SavingsPlanStateActive, "")} + res := f.run(t) + assert.Equal(t, []string{"t1"}, f.completed) + assert.Equal(t, 1, res.Completed) + assert.Equal(t, 1, f.calls) + assert.Zero(t, f.callsInTx, "provider must not be called inside the locked transaction") +} + +func TestRunLadderReconcile_StaysFiredWhenNotVisible(t *testing.T) { + cases := map[string]sptypes.SavingsPlan{ + "queued": sp("sp-1", sptypes.SavingsPlanTypeCompute, sptypes.SavingsPlanStateQueued, ""), + "pending-return": sp("sp-1", sptypes.SavingsPlanTypeCompute, sptypes.SavingsPlanStatePendingReturn, ""), + "wrong plan type": sp("sp-1", sptypes.SavingsPlanTypeEc2Instance, sptypes.SavingsPlanStateActive, "us-east-1"), + "absent": sp("other", sptypes.SavingsPlanTypeCompute, sptypes.SavingsPlanStateActive, ""), + } + for name, plan := range cases { + t.Run(name, func(t *testing.T) { + f := &visFake{plans: []sptypes.SavingsPlan{plan}} + f.addTranche("t1", "acct-a", "sp-1", ladder.LayerComputeSP) + res := f.run(t) + assert.Empty(t, f.completed) + assert.Equal(t, 1, res.Unresolved) + }) + } + t.Run("ec2 instance sp in another region", func(t *testing.T) { + f := &visFake{plans: []sptypes.SavingsPlan{sp("sp-1", sptypes.SavingsPlanTypeEc2Instance, sptypes.SavingsPlanStateActive, "eu-west-1")}} + f.addTranche("t1", "acct-a", "sp-1", ladder.LayerEC2InstanceSP) + f.run(t) + assert.Empty(t, f.completed) + }) +} + +func TestRunLadderReconcile_APIErrorFailsClosed(t *testing.T) { + f := &visFake{apiErr: errors.New("throttled")} + f.addTranche("t1", "acct-a", "sp-1", ladder.LayerComputeSP) + res := f.run(t) + assert.Empty(t, f.completed) + assert.Equal(t, 1, res.Unresolved) +} + +func TestRunLadderReconcile_BatchesPerAccountAndRegion(t *testing.T) { + f := &visFake{plans: []sptypes.SavingsPlan{ + sp("sp-1", sptypes.SavingsPlanTypeCompute, sptypes.SavingsPlanStateActive, ""), + sp("sp-2", sptypes.SavingsPlanTypeCompute, sptypes.SavingsPlanStatePaymentPending, ""), + sp("sp-3", sptypes.SavingsPlanTypeCompute, sptypes.SavingsPlanStateActive, ""), + }} + f.addTranche("t1", "acct-a", "sp-1", ladder.LayerComputeSP) + f.addTranche("t2", "acct-a", "sp-2", ladder.LayerComputeSP) + f.addTranche("t3", "acct-b", "sp-3", ladder.LayerComputeSP) + f.run(t) + assert.Equal(t, 2, f.calls, "one call per account, not per tranche") + assert.ElementsMatch(t, []string{"t1", "t2", "t3"}, f.completed) +} + +func TestRunLadderReconcile_StaleUnresolvedIsCounted(t *testing.T) { + f := &visFake{apiErr: errors.New("boom")} + f.addTranche("t1", "acct-a", "sp-1", ladder.LayerComputeSP) + res := f.run(t) + assert.Equal(t, 1, res.Stale, "chain head is epoch, far older than LadderFiredStaleAfter") +} + +// TestLadderVisiblePredicateMatchesUpstream pins the predicate to the pinned +// module's, without any network access. The module exports no symbol for it, +// so the table is copied from providers/aws/ladder at the go.mod pin: +// - states: adapters.go:39-42 spListerStates (active, payment-pending only; +// queued is excluded on purpose, adapters.go:30-37) +// - EC2Instance plans only in the ladder region, Compute global: +// adapters.go:123-133 appendRegionScopedSPs +// - plans are summed into the layer of their own plan type: +// layer_states.go:244-251 sumSPHourlyCost +// +// TODO: export the set from cloud-commitments-go and import it here. +func TestLadderVisiblePredicateMatchesUpstream(t *testing.T) { + const region = "us-east-1" + cases := []struct { + name string + state sptypes.SavingsPlanState + typ sptypes.SavingsPlanType + plan sptypes.SavingsPlanType + reg string + want bool + }{ + {"compute active", sptypes.SavingsPlanStateActive, sptypes.SavingsPlanTypeCompute, sptypes.SavingsPlanTypeCompute, "", true}, + {"compute payment-pending", sptypes.SavingsPlanStatePaymentPending, sptypes.SavingsPlanTypeCompute, sptypes.SavingsPlanTypeCompute, "", true}, + {"compute queued", sptypes.SavingsPlanStateQueued, sptypes.SavingsPlanTypeCompute, sptypes.SavingsPlanTypeCompute, "", false}, + {"compute pending-return", sptypes.SavingsPlanStatePendingReturn, sptypes.SavingsPlanTypeCompute, sptypes.SavingsPlanTypeCompute, "", false}, + {"compute retired", sptypes.SavingsPlanStateRetired, sptypes.SavingsPlanTypeCompute, sptypes.SavingsPlanTypeCompute, "", false}, + {"ec2 same region", sptypes.SavingsPlanStateActive, sptypes.SavingsPlanTypeEc2Instance, sptypes.SavingsPlanTypeEc2Instance, region, true}, + {"ec2 other region", sptypes.SavingsPlanStateActive, sptypes.SavingsPlanTypeEc2Instance, sptypes.SavingsPlanTypeEc2Instance, "eu-west-1", false}, + {"type differs from layer", sptypes.SavingsPlanStateActive, sptypes.SavingsPlanTypeCompute, sptypes.SavingsPlanTypeEc2Instance, region, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := spCountsAsExisting(sp("x", c.typ, c.state, c.reg), expectedPlan{id: "x", planType: string(c.plan), region: region}) + assert.Equal(t, c.want, got) + }) + } + assert.ElementsMatch(t, []sptypes.SavingsPlanState{sptypes.SavingsPlanStateActive, sptypes.SavingsPlanStatePaymentPending}, ladderVisibleSPStates) +} + +func TestRunLadderReconcile_PartialVisibilityStaysFired(t *testing.T) { + f := &visFake{plans: []sptypes.SavingsPlan{sp("sp-1", sptypes.SavingsPlanTypeCompute, sptypes.SavingsPlanStateActive, "")}} + f.addTranche("t1", "acct-a", "sp-1", ladder.LayerComputeSP) + f.execs["exec-t1"].Recommendations = append(f.execs["exec-t1"].Recommendations, + config.RecommendationRecord{Selected: true, PurchaseID: "sp-2", Region: "us-east-1"}) + f.run(t) + assert.Empty(t, f.completed) + assert.Equal(t, []string{"acct-a"}, f.factoryAccts) +} diff --git a/internal/purchase/main_nonet_test.go b/internal/purchase/main_nonet_test.go new file mode 100644 index 00000000..57f93350 --- /dev/null +++ b/internal/purchase/main_nonet_test.go @@ -0,0 +1,50 @@ +package purchase + +import ( + "fmt" + "net" + "os" + "sync/atomic" + "testing" +) + +// TestMain routes every proxy-aware HTTP client in this test binary (Go's +// default transport and the AWS SDK clients, which honor HTTP(S)_PROXY) into a +// local listener that counts and drops connections. If any non-loopback +// connection is attempted the run fails after the tests finish. Loopback +// traffic (httptest) bypasses the proxy and is unaffected. Not covered: code +// that dials directly (raw net.Dial, gRPC) or builds a transport with no proxy. +func TestMain(m *testing.M) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + fmt.Fprintln(os.Stderr, "no-network guard: listen:", err) + os.Exit(1) + } + var attempts atomic.Int64 + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + attempts.Add(1) + _ = c.Close() + } + }() + proxy := "http://" + ln.Addr().String() + for k, v := range map[string]string{ + "HTTP_PROXY": proxy, "HTTPS_PROXY": proxy, "http_proxy": proxy, "https_proxy": proxy, + "NO_PROXY": "127.0.0.1,localhost", "no_proxy": "127.0.0.1,localhost", + "AWS_EC2_METADATA_DISABLED": "true", + "AWS_ACCESS_KEY_ID": "test", "AWS_SECRET_ACCESS_KEY": "test", + } { + _ = os.Setenv(k, v) + } + code := m.Run() + _ = ln.Close() + if n := attempts.Load(); n > 0 { + fmt.Fprintf(os.Stderr, "FAIL: purchase tests attempted %d outbound network connection(s)\n", n) + code = 1 + } + os.Exit(code) +}