From 081b297efde120155fad7ca9a3b5ed4cfbb428e2 Mon Sep 17 00:00:00 2001 From: Vladimir Shchukin Date: Mon, 14 Sep 2026 14:27:57 -0400 Subject: [PATCH 1/4] Add transmission attempt gas limit assertion with user provided value --- chain_capabilities/evm/actions/actions.go | 2 + .../evm/actions/write_report.go | 164 ++++++++++--- .../evm/actions/write_report_test.go | 221 ++++++++++++++++-- chain_capabilities/evm/config/config.go | 8 +- .../evm/internal/contracts/cre_forwarder.go | 40 +++- .../evm/monitoring/log_trigger.pb.go | 2 +- chain_capabilities/evm/monitoring/messages.go | 30 ++- chain_capabilities/evm/monitoring/metrics.go | 27 +++ .../evm/monitoring/monitoring_test.go | 1 + .../evm/monitoring/processor.go | 5 + .../evm/monitoring/read_actions.pb.go | 2 +- .../evm/monitoring/write_report.pb.go | 121 ++++++++-- .../evm/monitoring/write_report.proto | 8 + 13 files changed, 544 insertions(+), 87 deletions(-) diff --git a/chain_capabilities/evm/actions/actions.go b/chain_capabilities/evm/actions/actions.go index 2678c2b22..2e5562169 100644 --- a/chain_capabilities/evm/actions/actions.go +++ b/chain_capabilities/evm/actions/actions.go @@ -47,6 +47,7 @@ type EVM struct { keystoneForwarderAddress common.Address forwarderClient contracts.CREForwarderClient ReceiverGasMinimum uint64 + forwarderGasOverhead uint64 LookbackBlocks uint64 lggr logger.SugaredLogger @@ -80,6 +81,7 @@ func NewEVM(cfg config.Config, evmService types.EVMService, lggr logger.Logger, keystoneForwarderAddress: keystoneForwarderAddress, forwarderClient: kfc, ReceiverGasMinimum: cfg.ReceiverGasMinimum, + forwarderGasOverhead: contracts.ForwarderGasOverhead(cfg.ForwarderGasOverheadMargin), lggr: logger.Sugared(lggr), beholderProcessor: beholderProcessor, messageBuilder: messageBuilder, diff --git a/chain_capabilities/evm/actions/write_report.go b/chain_capabilities/evm/actions/write_report.go index bb0fe9ce8..b962df130 100644 --- a/chain_capabilities/evm/actions/write_report.go +++ b/chain_capabilities/evm/actions/write_report.go @@ -40,9 +40,10 @@ var ErrUnexpectedSuccessfulTransmission = errors.New("unexpected successful tran type WriteReport struct { types.EVMService - forwarderClient contracts.CREForwarderClient - ReceiverGasMinimum uint64 - chainSelector uint64 + forwarderClient contracts.CREForwarderClient + ReceiverGasMinimum uint64 + forwarderGasOverhead uint64 + chainSelector uint64 lggr logger.SugaredLogger beholderProcessor beholder.ProtoProcessor @@ -84,10 +85,11 @@ func (e *EVM) WriteReport(ctx context.Context, metadata capabilities.RequestMeta func (e *EVM) executeWriteReport(ctx context.Context, request *evm.WriteReportRequest, metadata capabilities.RequestMetadata, telemetryContext monitoring.TelemetryContext) (*evm.WriteReportReply, capabilities.ResponseMetadata, error) { wr := &WriteReport{ - EVMService: e.EVMService, - forwarderClient: e.forwarderClient, - ReceiverGasMinimum: e.ReceiverGasMinimum, - chainSelector: e.chainSelector, + EVMService: e.EVMService, + forwarderClient: e.forwarderClient, + ReceiverGasMinimum: e.ReceiverGasMinimum, + forwarderGasOverhead: e.forwarderGasOverhead, + chainSelector: e.chainSelector, lggr: e.messageBuilder.RequestLggr(e.lggr, telemetryContext), beholderProcessor: e.beholderProcessor, @@ -125,6 +127,7 @@ func (e *WriteReport) executeWriteReport(ctx context.Context, request *evm.Write e.lggr = e.lggr.With(transmissionID.LogAttrs()...) ctx = contexts.WithChainSelector(ctx, e.chainSelector) + userProvidedGas := request.GasConfig != nil && request.GasConfig.GasLimit != 0 if request.GasConfig == nil || request.GasConfig.GasLimit == 0 { request.GasConfig = &evm.GasConfig{} request.GasConfig.GasLimit, err = e.txGasLimit.Limit(ctx) @@ -180,24 +183,16 @@ func (e *WriteReport) executeWriteReport(ctx context.Context, request *evm.Write } return reply, capabilities.ResponseMetadata{}, nil case contracts.TransmissionStateFailed: - hadEnoughGas, calculatedReceiverGasBudget := e.attemptHadEnoughGas(request, transmissionInfo) - if hadEnoughGas { - txHash, err := txHashRetriever.GetFailedTransmissionHash(ctx) - if err != nil { - if errors.Is(err, ErrUnexpectedSuccessfulTransmission) { - monitoring.LogAndEmitError(ctx, e.lggr, e.beholderProcessor, e.messageBuilder.BuildWriteReportInvalidTransmissionState(telemetryContext, request, transmissionInfo, "WriteReport unexpected successful transmission", err.Error())) - } else { - e.lggr.Errorw("Returning without a transmission attempt - prior transmission failed with sufficient gas, but failed to retrieve its tx hash", "error", err.Error(), "receiverGasBudget", calculatedReceiverGasBudget, "transmissionReceiverGasBudget", transmissionInfo.GasLimit) - } - return nil, capabilities.ResponseMetadata{}, err - } - + decision, err := e.assessFailedTransmission(ctx, request, transmissionInfo, txHashRetriever, telemetryContext, userProvidedGas) + if err != nil { + return nil, capabilities.ResponseMetadata{}, err + } + if !decision.retry { e.lggr.Infow("Returning without a transmission attempt - prior transmission failed with sufficient gas", - "txHash", common.Bytes2Hex(txHash[:]), - "receiverGasBudget", calculatedReceiverGasBudget, + "receiverGasBudget", decision.receiverGasBudget, "transmissionReceiverGasBudget", transmissionInfo.GasLimit, ) - reply, err := e.buildRevertReplyFromTx(ctx, *txHash, transmissionInfo, transmissionID) + reply, err := e.buildRevertReplyFromTx(ctx, decision.priorTxHash, transmissionInfo, transmissionID) if err != nil { // The receiver reverted despite sufficient gas (user contract fault); surface the // reason even if the receipt/fee lookup failed, as a user error. @@ -206,7 +201,7 @@ func (e *WriteReport) executeWriteReport(ctx context.Context, request *evm.Write return reply, capabilities.ResponseMetadata{}, nil } monitoring.LogAndEmitSuccess(ctx, "Retrying a failed transmission after prior attempt had insufficient receiver gas", e.lggr, e.beholderProcessor, - e.messageBuilder.BuildWriteReportInsufficientGasRetry(telemetryContext, request, calculatedReceiverGasBudget, transmissionInfo.GasLimit, queuePosition)) + e.messageBuilder.BuildWriteReportInsufficientGasRetry(telemetryContext, request, decision.receiverGasBudget, transmissionInfo.GasLimit, queuePosition)) default: errorMsg := getInvalidStateErrorMessage(transmissionInfo.State) monitoring.LogAndEmitError(ctx, e.lggr, e.beholderProcessor, e.messageBuilder.BuildWriteReportInvalidTransmissionState(telemetryContext, request, transmissionInfo, "WriteReport invalid transmission state", errorMsg)) @@ -364,9 +359,10 @@ func (e *WriteReport) pollTransmissionInfo( case contracts.TransmissionStateSucceeded, contracts.TransmissionStateInvalidReceiver: return lastValidInfo, nil case contracts.TransmissionStateFailed: - hadEnoughGas, calculatedReceiverGasBudget := e.attemptHadEnoughGas(request, lastValidInfo) - // none of the previous nodes will try to resend this transmission, so we can stop polling early - if hadEnoughGas { + // Cheap estimate check while polling; the authoritative decision is made after + // polling completes (see assessFailedTransmission). + if e.recordedBudgetExceedsEstimate(request, lastValidInfo) { + // none of the previous nodes will try to resend this transmission, so we can stop polling early return lastValidInfo, nil } _, count, err := txHashRetriever.GetFailedTransmissionHashWithCount(ctx) @@ -376,7 +372,6 @@ func (e *WriteReport) pollTransmissionInfo( e.lggr.Infow("Stopping poll - all prior nodes in queue finished their transmission attempts", "queuePosition", queuePosition, "failedAttemptCount", count, - "calculatedReceiverGasBudget", calculatedReceiverGasBudget, "transmissionReceiverGasBudget", lastValidInfo.GasLimit, ) return lastValidInfo, nil @@ -433,18 +428,111 @@ func getInvalidStateErrorMessage(state contracts.TransmissionState) string { return fmt.Sprintf("unexpected transmission state: %v", state) } -// attemptHadEnoughGas reports whether a prior failed transmission used sufficient receiver gas, -// meaning a resubmit would not help (e.g. receiver contract revert). -// The second return value is the receiver gas budget derived from the request. -func (e *WriteReport) attemptHadEnoughGas(request *evm.WriteReportRequest, info contracts.TransmissionInfo) (bool, uint64) { - receiverGasBudget := e.ReceiverGasMinimum + contracts.ForwarderContractLogicGasCost +// failedTransmissionDecision is the outcome of assessing a failed transmission: either the +// failure is genuine (retry=false, priorTxHash identifies the failed attempt) or a retry can +// deliver more receiver gas than the failed attempt got (retry=true). +type failedTransmissionDecision struct { + retry bool + // priorTxHash is the failed attempt's tx hash, set when retry=false. + priorTxHash evmtypes.Hash + // receiverGasBudget is the offchain-derived receiver gas budget, for telemetry. + receiverGasBudget uint64 +} + +// assessFailedTransmission decides whether a failed transmission should be retried. +// +// The verdict answers one question: would our resubmission deliver more receiver gas than +// the failed attempt got? A retry sends the identical report through the identical forwarder, +// so the forwarder overhead cancels out and the answer reduces to comparing gas limits. +// +// When the user provided the gas limit, the comparison is trustless on both sides: the prior +// tx's actual onchain gas limit (consensus data the previous submitter cannot misreport) +// against the user's requested limit (what every honest node would have submitted). Any +// inequality — lower or higher — is anomalous and emits a warn log plus a gas mismatch metric. +// +// When the gas limit was node-derived, or the prior tx's gas cannot be fetched, the comparison +// falls back to the forwarder-recorded receiver budget against our offchain estimate. +func (e *WriteReport) assessFailedTransmission(ctx context.Context, request *evm.WriteReportRequest, transmissionInfo contracts.TransmissionInfo, txHashRetriever TxHashRetriever, telemetryContext monitoring.TelemetryContext, userProvidedGas bool) (failedTransmissionDecision, error) { + receiverGasBudget := e.estimateReceiverGasBudget(request) + + priorTxHash, err := txHashRetriever.GetFailedTransmissionHash(ctx) + if err != nil { + if errors.Is(err, ErrUnexpectedSuccessfulTransmission) { + monitoring.LogAndEmitError(ctx, e.lggr, e.beholderProcessor, e.messageBuilder.BuildWriteReportInvalidTransmissionState(telemetryContext, request, transmissionInfo, "WriteReport unexpected successful transmission", err.Error())) + } else { + e.lggr.Errorw("Failed to retrieve the prior failed transmission's tx hash", "error", err.Error(), "receiverGasBudget", receiverGasBudget, "transmissionReceiverGasBudget", transmissionInfo.GasLimit) + } + return failedTransmissionDecision{}, err + } + + if userProvidedGas { + priorTxGas, gasErr := e.fetchTxGasLimit(ctx, *priorTxHash) + if gasErr == nil { + requestedGasLimit := request.GasConfig.GetGasLimit() + if priorTxGas != requestedGasLimit { + e.warnGasMismatch(ctx, telemetryContext, request, *priorTxHash, requestedGasLimit, priorTxGas) + } + return failedTransmissionDecision{ + retry: priorTxGas < requestedGasLimit, + priorTxHash: *priorTxHash, + receiverGasBudget: receiverGasBudget, + }, nil + } + e.lggr.Debugw("Failed to fetch the prior failed transmission tx's gas limit, falling back to estimate comparison", "error", gasErr) + } + + // Node-derived gas, or the prior tx's gas could not be fetched: compare the forwarder- + // recorded receiver budget against the offchain estimate. A nil recorded budget is retryable. + if transmissionInfo.GasLimit == nil || transmissionInfo.GasLimit.Uint64() <= receiverGasBudget { + return failedTransmissionDecision{retry: true, receiverGasBudget: receiverGasBudget}, nil + } + return failedTransmissionDecision{priorTxHash: *priorTxHash, receiverGasBudget: receiverGasBudget}, nil +} + +// estimateReceiverGasBudget derives the receiver gas budget offchain: the requested gas +// limit minus the forwarder's gas overhead, or the configured receiver gas minimum when no +// explicit limit was provided. +func (e *WriteReport) estimateReceiverGasBudget(request *evm.WriteReportRequest) uint64 { + receiverGasBudget := e.ReceiverGasMinimum + e.forwarderGasOverhead if request.GasConfig != nil && request.GasConfig.GasLimit > receiverGasBudget { - receiverGasBudget = request.GasConfig.GasLimit - contracts.ForwarderContractLogicGasCost + receiverGasBudget = request.GasConfig.GasLimit - e.forwarderGasOverhead + } + return receiverGasBudget +} + +// recordedBudgetExceedsEstimate reports whether the forwarder-recorded receiver budget +// exceeds our offchain estimate, i.e. the failed attempt plausibly had enough gas. Used only +// as the polling early-exit heuristic; a nil recorded budget counts as not enough. +func (e *WriteReport) recordedBudgetExceedsEstimate(request *evm.WriteReportRequest, transmissionInfo contracts.TransmissionInfo) bool { + if transmissionInfo.GasLimit == nil { + return false } - if info.GasLimit == nil { - return false, receiverGasBudget + return transmissionInfo.GasLimit.Uint64() > e.estimateReceiverGasBudget(request) +} + +// warnGasMismatch reports a prior tx whose onchain gas limit differs from the requested one. +func (e *WriteReport) warnGasMismatch(ctx context.Context, telemetryContext monitoring.TelemetryContext, request *evm.WriteReportRequest, txHash evmtypes.Hash, requestedGasLimit, actualTxGasLimit uint64) { + e.lggr.Warnw("Gas mismatch: prior transmission tx gas limit does not match the requested gas limit", + "txHash", common.Bytes2Hex(txHash[:]), + "priorTxGasLimit", actualTxGasLimit, + "requestedGasLimit", requestedGasLimit, + ) + monitoring.EmitInitiated(ctx, e.lggr, e.beholderProcessor, + e.messageBuilder.BuildWriteReportGasMismatch(telemetryContext, request, common.Bytes2Hex(txHash[:]), requestedGasLimit, actualTxGasLimit)) +} + +// fetchTxGasLimit reads a transaction's gas limit from chain. +func (e *WriteReport) fetchTxGasLimit(ctx context.Context, txHash evmtypes.Hash) (uint64, error) { + tx, err := capcommon.WithQuickRetry(ctx, e.lggr, func(ctx context.Context) (*evmtypes.Transaction, error) { + return e.GetTransactionByHash(ctx, evmtypes.GetTransactionByHashRequest{ + Hash: txHash, + IsExternal: false, // gas limit is needed for the retry decision, not user output + }) + }) + if err != nil { + return 0, err } - return info.GasLimit.Uint64() > receiverGasBudget, receiverGasBudget + return tx.Gas, nil } func getTransmissionID(workflowExecutionID string, request *evm.WriteReportRequest) (contracts.TransmissionID, error) { @@ -597,8 +685,8 @@ func (e *EVM) validateInputsAndReportMetadata(requestMetadata capabilities.Reque return fmt.Errorf("workflowID in the report does not match WorkflowID in the request metadata. Report WorkflowID: %s, request WorkflowID: %s", reportMetadata.WorkflowID, requestMetadata.WorkflowID) } - if request.GasConfig != nil && request.GasConfig.GasLimit != 0 && request.GasConfig.GasLimit < e.ReceiverGasMinimum+contracts.ForwarderContractLogicGasCost { - return fmt.Errorf("gas limit is %d, which is lower than minimum gas limit of: %d, for unbounded gas leave the gas limit as nil or 0", request.GasConfig.GasLimit, e.ReceiverGasMinimum+contracts.ForwarderContractLogicGasCost) + if request.GasConfig != nil && request.GasConfig.GasLimit != 0 && request.GasConfig.GasLimit < e.ReceiverGasMinimum+e.forwarderGasOverhead { + return fmt.Errorf("gas limit is %d, which is lower than minimum gas limit of: %d, for unbounded gas leave the gas limit as nil or 0", request.GasConfig.GasLimit, e.ReceiverGasMinimum+e.forwarderGasOverhead) } return nil diff --git a/chain_capabilities/evm/actions/write_report_test.go b/chain_capabilities/evm/actions/write_report_test.go index e77ca0a5d..637802d25 100644 --- a/chain_capabilities/evm/actions/write_report_test.go +++ b/chain_capabilities/evm/actions/write_report_test.go @@ -107,6 +107,156 @@ func expectSuccessfulTransmissionEvent(t *testing.T, mockForwarderClient *mocks. Return([]*evmtypes.Log{{TxHash: txHash, Data: successLogData(), BlockNumber: big.NewInt(100)}}, nil) } +// TestAssessFailedTransmission_TxGasCheck guards the tx-gas check fix (report 91254). When +// the gas limit is user-provided, the retry decision compares the prior tx's actual onchain +// gas limit against the requested gas limit — both trustless — instead of the forwarder- +// recorded budget against our offchain estimate. A retry delivers more receiver gas iff the +// prior tx used less gas than the request, since the forwarder overhead is identical between +// attempts. The estimate comparison remains for node-derived gas and as fallback when the +// prior tx cannot be fetched. +func TestAssessFailedTransmission_TxGasCheck(t *testing.T) { + t.Parallel() + + overhead := contracts.ForwarderGasOverhead(0) + + newWriteReport := func() *WriteReport { + return &WriteReport{ + ReceiverGasMinimum: ConfiguredReceiverGasMinimum, + forwarderGasOverhead: overhead, + lggr: logger.Sugared(logger.Test(t)), + beholderProcessor: test.NopBeholderProcessor{}, + messageBuilder: monitoring.NewMessageBuilder(types.ChainInfo{}, capabilities.CapabilityInfo{}, ""), + } + } + // writeReport + retriever whose prior failed tx carried priorTxGas; tx fetch never fails. + newWriteReportWithPriorTxGas := func(t *testing.T, priorTxGas uint64) (*WriteReport, TxHashRetriever) { + t.Helper() + mockForwarderClient := mocks.NewCREForwarderClient(t) + priorTxHash := evmtypes.Hash(test.RandomBytes(32)) + mockForwarderClient.EXPECT(). + GetReportProcessedEvents(mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return([]*evmtypes.Log{{TxHash: priorTxHash, Data: failedLogData(), BlockNumber: big.NewInt(100)}}, nil) + mockEVMService := mocks2.NewEVMService(t) + mockEVMService.EXPECT(). + GetTransactionByHash(mock.Anything, evmtypes.GetTransactionByHashRequest{Hash: priorTxHash, IsExternal: false}). + Return(&evmtypes.Transaction{Gas: priorTxGas}, nil) + wr := newWriteReport() + wr.EVMService = mockEVMService + return wr, NewTxHashRetriever(mockForwarderClient, logger.Test(t), newWriteReportTestFixture(t).transmissionID) + } + // writeReport + retriever whose prior failed tx exists but its gas cannot be fetched. + newWriteReportWithUnfetchableTxGas := func(t *testing.T) (*WriteReport, TxHashRetriever) { + t.Helper() + mockForwarderClient := mocks.NewCREForwarderClient(t) + priorTxHash := evmtypes.Hash(test.RandomBytes(32)) + mockForwarderClient.EXPECT(). + GetReportProcessedEvents(mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return([]*evmtypes.Log{{TxHash: priorTxHash, Data: failedLogData(), BlockNumber: big.NewInt(100)}}, nil) + mockEVMService := mocks2.NewEVMService(t) + mockEVMService.EXPECT(). + GetTransactionByHash(mock.Anything, evmtypes.GetTransactionByHashRequest{Hash: priorTxHash, IsExternal: false}). + Return(nil, errors.New("tx not found")) + wr := newWriteReport() + wr.EVMService = mockEVMService + return wr, NewTxHashRetriever(mockForwarderClient, logger.Test(t), newWriteReportTestFixture(t).transmissionID) + } + + t.Run("prior tx used less gas than requested => retry (attack case)", func(t *testing.T) { + t.Parallel() + + requestedGasLimit := uint64(500_000) + wr, retriever := newWriteReportWithPriorTxGas(t, 380_000) + request := &evm.WriteReportRequest{GasConfig: &evm.GasConfig{GasLimit: requestedGasLimit}} + decision, err := wr.assessFailedTransmission(t.Context(), request, contracts.TransmissionInfo{ + State: contracts.TransmissionStateFailed, + GasLimit: big.NewInt(309_432), // attacker's tx recorded budget + }, retriever, monitoring.TelemetryContext{}, true) + require.NoError(t, err) + require.True(t, decision.retry) + require.Equal(t, requestedGasLimit-overhead, decision.receiverGasBudget) + }) + + t.Run("prior tx used exactly the requested gas => no retry (genuine revert)", func(t *testing.T) { + t.Parallel() + + requestedGasLimit := uint64(500_000) + wr, retriever := newWriteReportWithPriorTxGas(t, requestedGasLimit) + request := &evm.WriteReportRequest{GasConfig: &evm.GasConfig{GasLimit: requestedGasLimit}} + decision, err := wr.assessFailedTransmission(t.Context(), request, contracts.TransmissionInfo{ + State: contracts.TransmissionStateFailed, + GasLimit: big.NewInt(int64(requestedGasLimit - 50_000)), + }, retriever, monitoring.TelemetryContext{}, true) + require.NoError(t, err) + require.False(t, decision.retry) + }) + + t.Run("prior tx used more than requested => no retry (retry would deliver less)", func(t *testing.T) { + t.Parallel() + + requestedGasLimit := uint64(500_000) + wr, retriever := newWriteReportWithPriorTxGas(t, requestedGasLimit+100_000) + request := &evm.WriteReportRequest{GasConfig: &evm.GasConfig{GasLimit: requestedGasLimit}} + decision, err := wr.assessFailedTransmission(t.Context(), request, contracts.TransmissionInfo{ + State: contracts.TransmissionStateFailed, + GasLimit: big.NewInt(int64(requestedGasLimit)), + }, retriever, monitoring.TelemetryContext{}, true) + require.NoError(t, err) + require.False(t, decision.retry) + }) + + t.Run("tx gas fetch fails => falls back to estimate comparison", func(t *testing.T) { + t.Parallel() + + requestedGasLimit := uint64(500_000) + wr, retriever := newWriteReportWithUnfetchableTxGas(t) + request := &evm.WriteReportRequest{GasConfig: &evm.GasConfig{GasLimit: requestedGasLimit}} + // Recorded budget (450k) > estimate (requested - overhead) => enough gas, no retry. + decision, err := wr.assessFailedTransmission(t.Context(), request, contracts.TransmissionInfo{ + State: contracts.TransmissionStateFailed, + GasLimit: big.NewInt(450_000), + }, retriever, monitoring.TelemetryContext{}, true) + require.NoError(t, err) + require.False(t, decision.retry) + require.Equal(t, requestedGasLimit-overhead, decision.receiverGasBudget) + }) + + t.Run("node-derived gas => estimate comparison as before", func(t *testing.T) { + t.Parallel() + + requestedGasLimit := uint64(500_000) + wr := newWriteReport() + // Prior failed tx exists (hash is resolved) but its gas is never fetched for + // node-derived gas — the estimate comparison decides. + mockForwarderClient := mocks.NewCREForwarderClient(t) + priorTxHash := evmtypes.Hash(test.RandomBytes(32)) + mockForwarderClient.EXPECT(). + GetReportProcessedEvents(mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return([]*evmtypes.Log{{TxHash: priorTxHash, Data: failedLogData(), BlockNumber: big.NewInt(100)}}, nil) + retriever := NewTxHashRetriever(mockForwarderClient, logger.Test(t), newWriteReportTestFixture(t).transmissionID) + request := &evm.WriteReportRequest{GasConfig: &evm.GasConfig{GasLimit: requestedGasLimit}} + // Recorded budget (450k) > estimate => enough gas, no retry — unchanged behavior. + decision, err := wr.assessFailedTransmission(t.Context(), request, contracts.TransmissionInfo{ + State: contracts.TransmissionStateFailed, + GasLimit: big.NewInt(450_000), + }, retriever, monitoring.TelemetryContext{}, false) + require.NoError(t, err) + require.False(t, decision.retry) + }) + + t.Run("nil recorded gas limit but tx carried requested gas => no retry", func(t *testing.T) { + t.Parallel() + + requestedGasLimit := uint64(500_000) + wr, retriever := newWriteReportWithPriorTxGas(t, requestedGasLimit) + request := &evm.WriteReportRequest{GasConfig: &evm.GasConfig{GasLimit: requestedGasLimit}} + decision, err := wr.assessFailedTransmission(t.Context(), request, contracts.TransmissionInfo{ + State: contracts.TransmissionStateFailed, + }, retriever, monitoring.TelemetryContext{}, true) + require.NoError(t, err) + require.False(t, decision.retry) + }) +} + func expectReceiptFetchFailure(t *testing.T, evmServiceMock *mocks2.EVMService, txHash evmtypes.Hash) error { t.Helper() @@ -460,7 +610,7 @@ func TestWriteReport_InputValidation(t *testing.T) { reportMetadata := createTestReportMetadata() encodedReportMetadata, _ := reportMetadata.Encode() - belowMinimum := ConfiguredReceiverGasMinimum + contracts.ForwarderContractLogicGasCost - 1 + belowMinimum := ConfiguredReceiverGasMinimum + contracts.ForwarderGasOverhead(0) - 1 _, err := service.WriteReport(ctx, createTestRequestMetadata(reportMetadata), &evm.WriteReportRequest{ Receiver: testutils.NewAddress().Bytes(), @@ -750,7 +900,7 @@ func TestWriteReport_ExecuteWriteReport(t *testing.T) { // Make request gas big enough so code uses: // txGasLimit = requestGasLimit - overhead desiredReceiverGas := uint64(EnoughReceiverGas * 100) - writeReportGasLimit := contracts.ForwarderContractLogicGasCost + desiredReceiverGas + writeReportGasLimit := contracts.ForwarderGasOverhead(0) + desiredReceiverGas fixture.request.GasConfig = &evm.GasConfig{GasLimit: writeReportGasLimit} // We want to hit: @@ -797,7 +947,7 @@ func TestWriteReport_ExecuteWriteReport(t *testing.T) { // so executeWriteReport uses: // txGasLimit = requestGasLimit - overhead desiredReceiverGas := uint64(EnoughReceiverGas * 100) - writeReportGasLimit := contracts.ForwarderContractLogicGasCost + desiredReceiverGas + writeReportGasLimit := contracts.ForwarderGasOverhead(0) + desiredReceiverGas fixture.request.GasConfig = &evm.GasConfig{GasLimit: writeReportGasLimit} // Previously attempted and failed, but transmission recorded gasLimit is strictly greater than @@ -817,6 +967,11 @@ func TestWriteReport_ExecuteWriteReport(t *testing.T) { txHash := evmtypes.Hash(test.RandomBytes(32)) expectFailedTransmissionEvent(t, mockForwarderClient, txHash) + // Prior tx carried exactly the requested gas limit => genuine revert, no retry. + evmServiceMock.EXPECT(). + GetTransactionByHash(mock.Anything, evmtypes.GetTransactionByHashRequest{Hash: txHash, IsExternal: false}). + Return(&evmtypes.Transaction{Gas: writeReportGasLimit}, nil) + receipt := evmtypes.Receipt{ Status: uint64(contracts.TransmissionStateSucceeded), TxHash: txHash, @@ -886,7 +1041,7 @@ func TestWriteReport_ExecuteWriteReport(t *testing.T) { ) desiredReceiverGas := uint64(EnoughReceiverGas * 100) - writeReportGasLimit := contracts.ForwarderContractLogicGasCost + desiredReceiverGas + writeReportGasLimit := contracts.ForwarderGasOverhead(0) + desiredReceiverGas mockForwarderClient. On("GetTransmissionInfo", mock.Anything, transmissionID). @@ -903,6 +1058,11 @@ func TestWriteReport_ExecuteWriteReport(t *testing.T) { GetReportProcessedEvents(mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return([]*evmtypes.Log{{TxHash: txHash, Data: failedLogData(), BlockNumber: big.NewInt(100)}}, nil) + // Prior tx carried exactly the requested gas limit => genuine revert, no retry. + evmServiceMock.EXPECT(). + GetTransactionByHash(mock.Anything, evmtypes.GetTransactionByHashRequest{Hash: txHash, IsExternal: false}). + Return(&evmtypes.Transaction{Gas: writeReportGasLimit}, nil) + receipt := evmtypes.Receipt{ Status: 1, TxHash: txHash, @@ -957,7 +1117,7 @@ func TestWriteReport_ExecuteWriteReport(t *testing.T) { // Also make request gas large enough that code uses: // txGasLimit = requestGasLimit - overhead desiredReceiverGasMin := uint64(EnoughReceiverGas * 100) - writeReportGasLimit := contracts.ForwarderContractLogicGasCost + desiredReceiverGasMin + writeReportGasLimit := contracts.ForwarderGasOverhead(0) + desiredReceiverGasMin fixture.request.GasConfig = &evm.GasConfig{GasLimit: writeReportGasLimit} // Previously attempted and failed with too-low transmission gas => should retry. @@ -972,6 +1132,17 @@ func TestWriteReport_ExecuteWriteReport(t *testing.T) { }, nil). Once() + // The prior failed tx used less gas than requested => retry path. One-shot: the + // post-retry success lookup must fall through to the success event expectation below. + priorFailedTxHash := evmtypes.Hash(test.RandomBytes(32)) + mockForwarderClient.EXPECT(). + GetReportProcessedEvents(mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return([]*evmtypes.Log{{TxHash: priorFailedTxHash, Data: failedLogData(), BlockNumber: big.NewInt(100)}}, nil). + Times(1) + evmServiceMock.EXPECT(). + GetTransactionByHash(mock.Anything, evmtypes.GetTransactionByHashRequest{Hash: priorFailedTxHash, IsExternal: false}). + Return(&evmtypes.Transaction{Gas: writeReportGasLimit - 1}, nil) + txHash := evmtypes.Hash(test.RandomBytes(32)) mockForwarderClient. On("InvokeOnReport", mock.Anything, fixture.receiver, fixture.request.Report, nonNilPositiveGasCfgMatcher()). @@ -1266,7 +1437,7 @@ func TestWriteReport_ExecuteWriteReport(t *testing.T) { equalWriteReportReply(t, &evm.WriteReportReply{ TxStatus: evm.TxStatus_TX_STATUS_SUCCESS, - TxHash: receipt.TxHash[:], // MUST be the log hash + TxHash: receipt.TxHash[:], ReceiverContractExecutionStatus: evm.ReceiverContractExecutionStatus_RECEIVER_CONTRACT_EXECUTION_STATUS_SUCCESS.Enum(), TransactionFee: pb.NewBigIntFromInt(big.NewInt(txFee)), }, txResult.Response) @@ -1431,14 +1602,14 @@ func TestWriteReport_ExecuteWriteReport(t *testing.T) { receiverAddress := testutils.NewAddress() signedReport, capabilitiesMetadata, transmissionID := createReportAndMetadataForQueuePosition( t, - &service.transmissionScheduler, + &scheduler, receiverAddress.Bytes(), queuePosition, ) // Make request gas large enough so we compute txGasLimit = requestGasLimit - overhead desiredReceiverGas := uint64(EnoughReceiverGas * 100) - writeReportGasLimit := contracts.ForwarderContractLogicGasCost + desiredReceiverGas + writeReportGasLimit := contracts.ForwarderGasOverhead(0) + desiredReceiverGas writeReportRequest := &evm.WriteReportRequest{ Receiver: receiverAddress.Bytes(), @@ -1482,11 +1653,20 @@ func TestWriteReport_ExecuteWriteReport(t *testing.T) { Once() var receiptTxHash evmtypes.Hash + // Earliest failed log is the prior attempt's tx; the tx-gas check fetches it. + var priorFailedTxHash evmtypes.Hash if queuePosition == 0 { receiptTxHash = latestTxHash + priorFailedTxHash = latestTxHash + mockForwarderClient.EXPECT(). + GetReportProcessedEvents(mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return([]*evmtypes.Log{ + {TxHash: latestTxHash, Data: failedLogData(), BlockNumber: big.NewInt(100)}, + }, nil) } else { originalFailedTxHash := evmtypes.Hash(test.RandomBytes(32)) receiptTxHash = originalFailedTxHash + priorFailedTxHash = originalFailedTxHash mockForwarderClient.EXPECT(). GetReportProcessedEvents(mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return([]*evmtypes.Log{ @@ -1495,6 +1675,11 @@ func TestWriteReport_ExecuteWriteReport(t *testing.T) { }, nil) } + // Prior tx used less gas than requested => retry path. + evmServiceMock.EXPECT(). + GetTransactionByHash(mock.Anything, evmtypes.GetTransactionByHashRequest{Hash: priorFailedTxHash, IsExternal: false}). + Return(&evmtypes.Transaction{Gas: writeReportGasLimit - 1}, nil) + receipt := evmtypes.Receipt{ Status: 1, TxHash: receiptTxHash, @@ -1519,10 +1704,6 @@ func TestWriteReport_ExecuteWriteReport(t *testing.T) { require.NotNil(t, txResult.Response.ReceiverContractExecutionStatus) require.Equal(t, evm.ReceiverContractExecutionStatus_RECEIVER_CONTRACT_EXECUTION_STATUS_REVERTED.Enum(), txResult.Response.ReceiverContractExecutionStatus.Enum()) evmtest.ValidateMeteringWriteReport(t, txResult.ResponseMetadata, 1, "0.0000000000000003", "300") - - if queuePosition == 0 { - mockForwarderClient.AssertNotCalled(t, "GetReportProcessedEvents", mock.Anything, mock.Anything, mock.Anything, mock.Anything) - } }) } }) @@ -1654,7 +1835,7 @@ func TestPollTransmissionInfo_QueuePositionScenarios(t *testing.T) { desiredReceiverGas := uint64(EnoughReceiverGas * 100) request.GasConfig = &evm.GasConfig{ - GasLimit: contracts.ForwarderContractLogicGasCost + desiredReceiverGas, + GasLimit: contracts.ForwarderGasOverhead(0) + desiredReceiverGas, } mockForwarderClient. @@ -1685,7 +1866,7 @@ func TestPollTransmissionInfo_QueuePositionScenarios(t *testing.T) { desiredReceiverGas := uint64(EnoughReceiverGas * 100) request.GasConfig = &evm.GasConfig{ - GasLimit: contracts.ForwarderContractLogicGasCost + desiredReceiverGas, + GasLimit: contracts.ForwarderGasOverhead(0) + desiredReceiverGas, } failedInfo := contracts.TransmissionInfo{ @@ -1722,7 +1903,7 @@ func TestPollTransmissionInfo_QueuePositionScenarios(t *testing.T) { desiredReceiverGas := uint64(EnoughReceiverGas * 100) request.GasConfig = &evm.GasConfig{ - GasLimit: contracts.ForwarderContractLogicGasCost + desiredReceiverGas, + GasLimit: contracts.ForwarderGasOverhead(0) + desiredReceiverGas, } mockForwarderClient. @@ -2188,7 +2369,8 @@ func TestWriteReport_RevertReceiptFetchFailsReturnsUserError(t *testing.T) { t.Helper() desiredReceiverGas := uint64(EnoughReceiverGas * 100) - fixture.request.GasConfig = &evm.GasConfig{GasLimit: contracts.ForwarderContractLogicGasCost + desiredReceiverGas} + writeReportGasLimit := contracts.ForwarderGasOverhead(0) + desiredReceiverGas + fixture.request.GasConfig = &evm.GasConfig{GasLimit: writeReportGasLimit} mockForwarderClient. On("GetTransmissionInfo", mock.Anything, mock.Anything). Return(contracts.TransmissionInfo{ @@ -2199,6 +2381,10 @@ func TestWriteReport_RevertReceiptFetchFailsReturnsUserError(t *testing.T) { }, nil). Once() expectFailedTransmissionEvent(t, mockForwarderClient, txHash) + // Prior tx carried exactly the requested gas limit => genuine revert, no retry. + evmServiceMock.EXPECT(). + GetTransactionByHash(mock.Anything, evmtypes.GetTransactionByHashRequest{Hash: txHash, IsExternal: false}). + Return(&evmtypes.Transaction{Gas: writeReportGasLimit}, nil) return expectReceiptFetchFailure(t, evmServiceMock, txHash) }, run: writeReportCall, @@ -2307,6 +2493,7 @@ func createMocksAndCapability(t *testing.T, lggr logger.Logger) (*mocks2.EVMServ lggr: logger.Sugared(lggr), EVMService: mockEVMService, ReceiverGasMinimum: ConfiguredReceiverGasMinimum, + forwarderGasOverhead: contracts.ForwarderGasOverhead(0), chainSelector: 1, beholderProcessor: test.NopBeholderProcessor{}, messageBuilder: monitoring.NewMessageBuilder(types.ChainInfo{}, capabilities.CapabilityInfo{}, ""), @@ -2333,6 +2520,7 @@ func createMocksAndCapabilityWithScheduler( lggr: logger.Sugared(lggr), EVMService: evmService, ReceiverGasMinimum: ConfiguredReceiverGasMinimum, + forwarderGasOverhead: contracts.ForwarderGasOverhead(0), chainSelector: 1, beholderProcessor: test.NopBeholderProcessor{}, messageBuilder: monitoring.NewMessageBuilder(types.ChainInfo{}, capabilities.CapabilityInfo{}, ""), @@ -2445,6 +2633,7 @@ func setupPollTransmissionInfoForQueuePosition( wr := &WriteReport{ forwarderClient: mockForwarderClient, ReceiverGasMinimum: ConfiguredReceiverGasMinimum, + forwarderGasOverhead: contracts.ForwarderGasOverhead(0), lggr: logger.Sugared(testLogger), beholderProcessor: test.NopBeholderProcessor{}, messageBuilder: monitoring.NewMessageBuilder(types.ChainInfo{}, capabilities.CapabilityInfo{}, ""), diff --git a/chain_capabilities/evm/config/config.go b/chain_capabilities/evm/config/config.go index 4e898a8ed..335fa6291 100644 --- a/chain_capabilities/evm/config/config.go +++ b/chain_capabilities/evm/config/config.go @@ -11,7 +11,13 @@ type Config struct { CREForwarderAddress string `json:"creForwarderAddress"` ForwarderLookbackBlocks int64 `json:"forwarderLookbackBlocks"` // defines how many blocks back to search for the ReportProcessed event (default 100). // The minimum amount of gas that the receiver contract must get to process the forwarder report. This is the default value used when the user doesn't specify a gas limit when invoking WriteReport. - ReceiverGasMinimum uint64 `json:"receiverGasMinimum"` + ReceiverGasMinimum uint64 `json:"receiverGasMinimum"` + // Safety margin, in gas, added on top of the forwarder contract's internal gas reservation + // when deriving the receiver gas budget offchain. It covers pre-route consumption not visible + // in the contract constants (tx intrinsic cost, calldata, ecrecover per signature, storage). + // 0 means "use the default" (see contracts.DefaultForwarderGasOverheadMargin). Set per chain + // when the chain's gas metering or report/signature sizes differ materially from mainnet. + ForwarderGasOverheadMargin uint64 `json:"forwarderGasOverheadMargin"` NodeAddress string `json:"nodeAddress"` ObservationPollerWorkersCount uint `json:"observationPollerWorkersCount"` ObservationPollPeriod time.Duration `json:"observationPollPeriod"` diff --git a/chain_capabilities/evm/internal/contracts/cre_forwarder.go b/chain_capabilities/evm/internal/contracts/cre_forwarder.go index 501a0a866..0f97d06d8 100644 --- a/chain_capabilities/evm/internal/contracts/cre_forwarder.go +++ b/chain_capabilities/evm/internal/contracts/cre_forwarder.go @@ -80,17 +80,41 @@ func (ti TransmissionInfo) LogAttrs() []any { return attrs } -// The gas cost of the forwarder contract logic, including state updates and event emission. -// This is a rough estimate and should be updated if the forwarder contract logic changes. -// PLEX-1524 - Make the forwarder contract logic gas cost limit configurable +// Gas accounting constants mirroring KeystoneForwarder.sol (cre/src/v1). +// Keep in sync with the contract; see ForwarderGasOverhead for how they combine. const ( - // ForwarderContractLogicGasCost is at minimum 100k, but often goes up by several x*10%. - // Overshoot it by double to make sure that we don't resend txs that had enough gas leftover based on transmission gas info. - ForwarderContractLogicGasCost = 200_000 - LatestBlock = -2 // PLEX-1524 - Use constant defined by EVM types once it's ready. - DefaultLookbackBlocks = 100 + // InternalGasRequirementsAfterReport is the forwarder's reservation for storing the + // transmission result after the receiver call (INTERNAL_GAS_REQUIREMENTS_AFTER_REPORT). + InternalGasRequirementsAfterReport uint64 = 5_000 + // InternalGasRequirements is the forwarder's total internal reservation, subtracted from + // gasleft() before the receiver gas budget is recorded (INTERNAL_GAS_REQUIREMENTS). + InternalGasRequirements uint64 = 25_000 + InternalGasRequirementsAfterReport + // MinimumGasLimit is the forwarder's routing floor: route() reverts the whole tx when the + // recorded receiver budget would fall below it (MINIMUM_GAS_LIMIT). Any transmission that + // routed at all therefore records at least this much gas. + MinimumGasLimit uint64 = InternalGasRequirements + 30_000*3 + 10_000 + // DefaultForwarderGasOverheadMargin is the default safety margin added on top of the + // forwarder's internal reservation to cover the pre-route consumption that is not visible + // in the contract constants: tx intrinsic cost, calldata (report + signatures), ecrecover + // per signature, storage reads and the external this.route() call. Measured at ~40k gas + // for production reports on mainnet; operators may override it per chain via config. + DefaultForwarderGasOverheadMargin uint64 = 40_000 + + LatestBlock = -2 // PLEX-1524 - Use constant defined by EVM types once it's ready. + DefaultLookbackBlocks = 100 ) +// ForwarderGasOverhead returns the total gas consumed between the gas limit set on the +// transmission tx and the receiver gas budget the forwarder records onchain: the contract's +// internal reservation plus a margin covering pre-route consumption. margin is the per-chain +// configurable safety margin; 0 selects the default. +func ForwarderGasOverhead(margin uint64) uint64 { + if margin == 0 { + margin = DefaultForwarderGasOverheadMargin + } + return InternalGasRequirements + margin +} + func NewCREForwarderCodec() (CREForwarderCodec, error) { ABI, err := forwarder.KeystoneForwarderMetaData.GetAbi() if err != nil { diff --git a/chain_capabilities/evm/monitoring/log_trigger.pb.go b/chain_capabilities/evm/monitoring/log_trigger.pb.go index a925538dd..210a762bd 100644 --- a/chain_capabilities/evm/monitoring/log_trigger.pb.go +++ b/chain_capabilities/evm/monitoring/log_trigger.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.10 +// protoc-gen-go v1.36.11 // protoc v5.29.3 // source: chain_capabilities/evm/monitoring/log_trigger.proto diff --git a/chain_capabilities/evm/monitoring/messages.go b/chain_capabilities/evm/monitoring/messages.go index f70e19b8b..709bfb389 100644 --- a/chain_capabilities/evm/monitoring/messages.go +++ b/chain_capabilities/evm/monitoring/messages.go @@ -56,15 +56,22 @@ func (m *MessageBuilder) BuildWriteReportInitiated(tc TelemetryContext, req *evm } func convertWriteReportRequest(req *evmcap.WriteReportRequest) *WriteReportRequest { - return &WriteReportRequest{ - Receiver: req.Receiver, - Report: &ReportResponse{ + if req == nil { + return nil + } + report := &ReportResponse{} + if req.Report != nil { + report = &ReportResponse{ ConfigDigest: req.Report.ConfigDigest, SeqNr: req.Report.SeqNr, ReportContext: req.Report.ReportContext, RawReport: req.Report.RawReport, Sigs: convertAttributedSignature(req.Report.Sigs), - }, + } + } + return &WriteReportRequest{ + Receiver: req.Receiver, + Report: report, GasConfig: &GasConfig{ GasLimit: req.GasConfig.GetGasLimit(), }, @@ -162,6 +169,21 @@ func (m *MessageBuilder) BuildWriteReportInsufficientGasRetry( } } +func (m *MessageBuilder) BuildWriteReportGasMismatch( + tc TelemetryContext, + req *evmcap.WriteReportRequest, + txHash string, + expectedTxGasLimit, actualTxGasLimit uint64, +) Message { + return &WriteReportGasMismatch{ + Req: convertWriteReportRequest(req), + TxHash: txHash, + ExpectedTxGasLimit: expectedTxGasLimit, + ActualTxGasLimit: actualTxGasLimit, + ExecutionContext: m.BuildExecutionContext(tc), + } +} + func (m *MessageBuilder) BuildLogTriggerInitiated(tc TelemetryContext, req *evmcap.FilterLogTriggerRequest) *LogTriggerInitiated { return &LogTriggerInitiated{Req: logTriggerRequestToMonitoring(req), ExecutionContext: m.BuildExecutionContext(tc)} } diff --git a/chain_capabilities/evm/monitoring/metrics.go b/chain_capabilities/evm/monitoring/metrics.go index 71dbd3ad2..88b0f8f0f 100644 --- a/chain_capabilities/evm/monitoring/metrics.go +++ b/chain_capabilities/evm/monitoring/metrics.go @@ -101,6 +101,9 @@ type Metrics struct { WriteReportInsufficientGasRetry struct { basic commoncapbeholder.MetricsCapBasic } + WriteReportGasMismatch struct { + basic commoncapbeholder.MetricsCapBasic + } LogTriggerSuccess struct { basic commoncapbeholder.MetricsCapBasic } @@ -217,6 +220,11 @@ func NewMetrics() (Metrics, error) { if err != nil { return Metrics{}, fmt.Errorf("failed to create write report insufficient gas retry metric: %w", err) } + wrGasMismatch := commoncapbeholder.NewMetricsInfoCapBasic(ns("write_report_gas_mismatch"), commonbeholder.ToSchemaFullName(&WriteReportGasMismatch{})) + m.WriteReportGasMismatch.basic, err = commoncapbeholder.NewMetricsCapBasic(wrGasMismatch) + if err != nil { + return Metrics{}, fmt.Errorf("failed to create write report gas mismatch metric: %w", err) + } // -- LogTrigger -- ltSuccess := commoncapbeholder.NewMetricsInfoCapBasic(ns("log_trigger_success"), commonbeholder.ToSchemaFullName(&LogTriggerSuccess{})) @@ -384,6 +392,12 @@ func (m *Metrics) OnWriteReportInsufficientGasRetry(ctx context.Context, msg *Wr return nil } +func (m *Metrics) OnWriteReportGasMismatch(ctx context.Context, msg *WriteReportGasMismatch) error { + start, emit := msg.ExecutionContext.MetaCapabilityTimestampStart, msg.ExecutionContext.MetaCapabilityTimestampEmit + m.WriteReportGasMismatch.basic.RecordEmit(ctx, start, emit, msg.MetricAttributes()...) + return nil +} + // -- LogTrigger -- func (m *Metrics) OnLogTriggerSuccess(ctx context.Context, msg *LogTriggerSuccess) error { @@ -658,6 +672,19 @@ func (r *WriteReportInsufficientGasRetry) MetricAttributes() []attribute.KeyValu return r.ExecutionContext.MetricsAttributes() } +func (r *WriteReportGasMismatch) LogAttributes() []attribute.KeyValue { + return append([]attribute.KeyValue{ + attribute.String("receiver", getReceiver(r.Req.GetReceiver())), + attribute.String("tx_hash", r.GetTxHash()), + attribute.Int64("expected_tx_gas_limit", int64(r.GetExpectedTxGasLimit())), //nolint:gosec // G115: EVM gas fits int64 for logging + attribute.Int64("actual_tx_gas_limit", int64(r.GetActualTxGasLimit())), //nolint:gosec // G115: EVM gas fits int64 for logging + }, r.ExecutionContext.LogAttributes()...) +} + +func (r *WriteReportGasMismatch) MetricAttributes() []attribute.KeyValue { + return r.ExecutionContext.MetricsAttributes() +} + func (r *LogTriggerSuccess) LogAttributes() []attribute.KeyValue { return append([]attribute.KeyValue{ attribute.String("trigger_id", r.GetTriggerID()), diff --git a/chain_capabilities/evm/monitoring/monitoring_test.go b/chain_capabilities/evm/monitoring/monitoring_test.go index d2db4387d..6535203fe 100644 --- a/chain_capabilities/evm/monitoring/monitoring_test.go +++ b/chain_capabilities/evm/monitoring/monitoring_test.go @@ -85,6 +85,7 @@ func TestProcessor_Process_SuccessMessages(t *testing.T) { {"CallContractSuccess", &monitoring.CallContractSuccess{ExecutionContext: &capmonitoring.ExecutionContext{}}}, {"WriteReportSuccess", &monitoring.WriteReportSuccess{ExecutionContext: &capmonitoring.ExecutionContext{}}}, {"WriteReportInsufficientGasRetry", &monitoring.WriteReportInsufficientGasRetry{ExecutionContext: &capmonitoring.ExecutionContext{}}}, + {"WriteReportGasMismatch", &monitoring.WriteReportGasMismatch{ExecutionContext: &capmonitoring.ExecutionContext{}}}, {"LogTriggerSuccess", &monitoring.LogTriggerSuccess{ExecutionContext: &capmonitoring.ExecutionContext{}}}, {"FilterLogsSuccess", &monitoring.FilterLogsSuccess{ExecutionContext: &capmonitoring.ExecutionContext{}}}, {"BalanceAtSuccess", &monitoring.BalanceAtSuccess{ExecutionContext: &capmonitoring.ExecutionContext{}}}, diff --git a/chain_capabilities/evm/monitoring/processor.go b/chain_capabilities/evm/monitoring/processor.go index 099eff880..d6325bcd4 100644 --- a/chain_capabilities/evm/monitoring/processor.go +++ b/chain_capabilities/evm/monitoring/processor.go @@ -85,6 +85,11 @@ func (p *processor) Process(ctx context.Context, m proto.Message, attrKVs ...any if err := p.metrics.OnWriteReportInsufficientGasRetry(ctx, msg); err != nil { return fmt.Errorf("failed to publish WriteReportInsufficientGasRetry metrics: %w", err) } + case *WriteReportGasMismatch: + p.logMessage(msg) + if err := p.metrics.OnWriteReportGasMismatch(ctx, msg); err != nil { + return fmt.Errorf("failed to publish WriteReportGasMismatch metrics: %w", err) + } // -- LogTrigger -- case *LogTriggerSuccess: p.logMessage(msg) diff --git a/chain_capabilities/evm/monitoring/read_actions.pb.go b/chain_capabilities/evm/monitoring/read_actions.pb.go index 46889330a..6b58256bc 100644 --- a/chain_capabilities/evm/monitoring/read_actions.pb.go +++ b/chain_capabilities/evm/monitoring/read_actions.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.10 +// protoc-gen-go v1.36.11 // protoc v5.29.3 // source: chain_capabilities/evm/monitoring/read_actions.proto diff --git a/chain_capabilities/evm/monitoring/write_report.pb.go b/chain_capabilities/evm/monitoring/write_report.pb.go index 18b41433b..17bce9d4a 100644 --- a/chain_capabilities/evm/monitoring/write_report.pb.go +++ b/chain_capabilities/evm/monitoring/write_report.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.10 +// protoc-gen-go v1.36.11 // protoc v5.29.3 // source: chain_capabilities/evm/monitoring/write_report.proto @@ -890,6 +890,82 @@ func (x *WriteReportInsufficientGasRetry) GetExecutionContext() *monitoring.Exec return nil } +type WriteReportGasMismatch struct { + state protoimpl.MessageState `protogen:"open.v1"` + Req *WriteReportRequest `protobuf:"bytes,1,opt,name=req,proto3" json:"req,omitempty"` + ExpectedTxGasLimit uint64 `protobuf:"varint,2,opt,name=expected_tx_gas_limit,json=expectedTxGasLimit,proto3" json:"expected_tx_gas_limit,omitempty"` + ActualTxGasLimit uint64 `protobuf:"varint,3,opt,name=actual_tx_gas_limit,json=actualTxGasLimit,proto3" json:"actual_tx_gas_limit,omitempty"` + TxHash string `protobuf:"bytes,4,opt,name=tx_hash,json=txHash,proto3" json:"tx_hash,omitempty"` + ExecutionContext *monitoring.ExecutionContext `protobuf:"bytes,20,opt,name=execution_context,json=executionContext,proto3" json:"execution_context,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WriteReportGasMismatch) Reset() { + *x = WriteReportGasMismatch{} + mi := &file_chain_capabilities_evm_monitoring_write_report_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WriteReportGasMismatch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WriteReportGasMismatch) ProtoMessage() {} + +func (x *WriteReportGasMismatch) ProtoReflect() protoreflect.Message { + mi := &file_chain_capabilities_evm_monitoring_write_report_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WriteReportGasMismatch.ProtoReflect.Descriptor instead. +func (*WriteReportGasMismatch) Descriptor() ([]byte, []int) { + return file_chain_capabilities_evm_monitoring_write_report_proto_rawDescGZIP(), []int{13} +} + +func (x *WriteReportGasMismatch) GetReq() *WriteReportRequest { + if x != nil { + return x.Req + } + return nil +} + +func (x *WriteReportGasMismatch) GetExpectedTxGasLimit() uint64 { + if x != nil { + return x.ExpectedTxGasLimit + } + return 0 +} + +func (x *WriteReportGasMismatch) GetActualTxGasLimit() uint64 { + if x != nil { + return x.ActualTxGasLimit + } + return 0 +} + +func (x *WriteReportGasMismatch) GetTxHash() string { + if x != nil { + return x.TxHash + } + return "" +} + +func (x *WriteReportGasMismatch) GetExecutionContext() *monitoring.ExecutionContext { + if x != nil { + return x.ExecutionContext + } + return nil +} + var File_chain_capabilities_evm_monitoring_write_report_proto protoreflect.FileDescriptor const file_chain_capabilities_evm_monitoring_write_report_proto_rawDesc = "" + @@ -960,6 +1036,12 @@ const file_chain_capabilities_evm_monitoring_write_report_proto_rawDesc = "" + "\x13receiver_gas_budget\x18\x02 \x01(\x04R\x11receiverGasBudget\x12G\n" + " transmission_receiver_gas_budget\x18\x03 \x01(\x04R\x1dtransmissionReceiverGasBudget\x12%\n" + "\x0equeue_position\x18\x04 \x01(\x05R\rqueuePosition\x12I\n" + + "\x11execution_context\x18\x14 \x01(\v2\x1c.monitoring.ExecutionContextR\x10executionContext\"\x9c\x02\n" + + "\x16WriteReportGasMismatch\x12<\n" + + "\x03req\x18\x01 \x01(\v2*.chain_capabilities.evm.WriteReportRequestR\x03req\x121\n" + + "\x15expected_tx_gas_limit\x18\x02 \x01(\x04R\x12expectedTxGasLimit\x12-\n" + + "\x13actual_tx_gas_limit\x18\x03 \x01(\x04R\x10actualTxGasLimit\x12\x17\n" + + "\atx_hash\x18\x04 \x01(\tR\x06txHash\x12I\n" + "\x11execution_context\x18\x14 \x01(\v2\x1c.monitoring.ExecutionContextR\x10executionContextBLZJgithub.com/smartcontractkit/capabilities/chain_capabilities/evm;monitoringb\x06proto3" var ( @@ -974,7 +1056,7 @@ func file_chain_capabilities_evm_monitoring_write_report_proto_rawDescGZIP() []b return file_chain_capabilities_evm_monitoring_write_report_proto_rawDescData } -var file_chain_capabilities_evm_monitoring_write_report_proto_msgTypes = make([]protoimpl.MessageInfo, 13) +var file_chain_capabilities_evm_monitoring_write_report_proto_msgTypes = make([]protoimpl.MessageInfo, 14) var file_chain_capabilities_evm_monitoring_write_report_proto_goTypes = []any{ (*WriteReportInitiated)(nil), // 0: chain_capabilities.evm.WriteReportInitiated (*WriteReportSuccess)(nil), // 1: chain_capabilities.evm.WriteReportSuccess @@ -989,33 +1071,36 @@ var file_chain_capabilities_evm_monitoring_write_report_proto_goTypes = []any{ (*AttributedSignature)(nil), // 10: chain_capabilities.evm.AttributedSignature (*WriteReportSuccessfulEarlyReturn)(nil), // 11: chain_capabilities.evm.WriteReportSuccessfulEarlyReturn (*WriteReportInsufficientGasRetry)(nil), // 12: chain_capabilities.evm.WriteReportInsufficientGasRetry - (*monitoring.ExecutionContext)(nil), // 13: monitoring.ExecutionContext + (*WriteReportGasMismatch)(nil), // 13: chain_capabilities.evm.WriteReportGasMismatch + (*monitoring.ExecutionContext)(nil), // 14: monitoring.ExecutionContext } var file_chain_capabilities_evm_monitoring_write_report_proto_depIdxs = []int32{ 7, // 0: chain_capabilities.evm.WriteReportInitiated.req:type_name -> chain_capabilities.evm.WriteReportRequest - 13, // 1: chain_capabilities.evm.WriteReportInitiated.execution_context:type_name -> monitoring.ExecutionContext + 14, // 1: chain_capabilities.evm.WriteReportInitiated.execution_context:type_name -> monitoring.ExecutionContext 7, // 2: chain_capabilities.evm.WriteReportSuccess.req:type_name -> chain_capabilities.evm.WriteReportRequest - 13, // 3: chain_capabilities.evm.WriteReportSuccess.execution_context:type_name -> monitoring.ExecutionContext + 14, // 3: chain_capabilities.evm.WriteReportSuccess.execution_context:type_name -> monitoring.ExecutionContext 7, // 4: chain_capabilities.evm.WriteReportError.req:type_name -> chain_capabilities.evm.WriteReportRequest - 13, // 5: chain_capabilities.evm.WriteReportError.execution_context:type_name -> monitoring.ExecutionContext + 14, // 5: chain_capabilities.evm.WriteReportError.execution_context:type_name -> monitoring.ExecutionContext 7, // 6: chain_capabilities.evm.WriteReportTxFeeCalculationError.req:type_name -> chain_capabilities.evm.WriteReportRequest - 13, // 7: chain_capabilities.evm.WriteReportTxFeeCalculationError.execution_context:type_name -> monitoring.ExecutionContext + 14, // 7: chain_capabilities.evm.WriteReportTxFeeCalculationError.execution_context:type_name -> monitoring.ExecutionContext 7, // 8: chain_capabilities.evm.WriteReportInvalidTransmissionState.req:type_name -> chain_capabilities.evm.WriteReportRequest - 13, // 9: chain_capabilities.evm.WriteReportInvalidTransmissionState.execution_context:type_name -> monitoring.ExecutionContext + 14, // 9: chain_capabilities.evm.WriteReportInvalidTransmissionState.execution_context:type_name -> monitoring.ExecutionContext 7, // 10: chain_capabilities.evm.WriteReportDuplicateTx.req:type_name -> chain_capabilities.evm.WriteReportRequest - 13, // 11: chain_capabilities.evm.WriteReportDuplicateTx.execution_context:type_name -> monitoring.ExecutionContext - 13, // 12: chain_capabilities.evm.TransmissionSchedulerNodeNotFoundInDon.execution_context:type_name -> monitoring.ExecutionContext + 14, // 11: chain_capabilities.evm.WriteReportDuplicateTx.execution_context:type_name -> monitoring.ExecutionContext + 14, // 12: chain_capabilities.evm.TransmissionSchedulerNodeNotFoundInDon.execution_context:type_name -> monitoring.ExecutionContext 9, // 13: chain_capabilities.evm.WriteReportRequest.report:type_name -> chain_capabilities.evm.ReportResponse 8, // 14: chain_capabilities.evm.WriteReportRequest.gas_config:type_name -> chain_capabilities.evm.GasConfig 10, // 15: chain_capabilities.evm.ReportResponse.sigs:type_name -> chain_capabilities.evm.AttributedSignature - 13, // 16: chain_capabilities.evm.WriteReportSuccessfulEarlyReturn.execution_context:type_name -> monitoring.ExecutionContext + 14, // 16: chain_capabilities.evm.WriteReportSuccessfulEarlyReturn.execution_context:type_name -> monitoring.ExecutionContext 7, // 17: chain_capabilities.evm.WriteReportInsufficientGasRetry.req:type_name -> chain_capabilities.evm.WriteReportRequest - 13, // 18: chain_capabilities.evm.WriteReportInsufficientGasRetry.execution_context:type_name -> monitoring.ExecutionContext - 19, // [19:19] is the sub-list for method output_type - 19, // [19:19] is the sub-list for method input_type - 19, // [19:19] is the sub-list for extension type_name - 19, // [19:19] is the sub-list for extension extendee - 0, // [0:19] is the sub-list for field type_name + 14, // 18: chain_capabilities.evm.WriteReportInsufficientGasRetry.execution_context:type_name -> monitoring.ExecutionContext + 7, // 19: chain_capabilities.evm.WriteReportGasMismatch.req:type_name -> chain_capabilities.evm.WriteReportRequest + 14, // 20: chain_capabilities.evm.WriteReportGasMismatch.execution_context:type_name -> monitoring.ExecutionContext + 21, // [21:21] is the sub-list for method output_type + 21, // [21:21] is the sub-list for method input_type + 21, // [21:21] is the sub-list for extension type_name + 21, // [21:21] is the sub-list for extension extendee + 0, // [0:21] is the sub-list for field type_name } func init() { file_chain_capabilities_evm_monitoring_write_report_proto_init() } @@ -1030,7 +1115,7 @@ func file_chain_capabilities_evm_monitoring_write_report_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_chain_capabilities_evm_monitoring_write_report_proto_rawDesc), len(file_chain_capabilities_evm_monitoring_write_report_proto_rawDesc)), NumEnums: 0, - NumMessages: 13, + NumMessages: 14, NumExtensions: 0, NumServices: 0, }, diff --git a/chain_capabilities/evm/monitoring/write_report.proto b/chain_capabilities/evm/monitoring/write_report.proto index 358e711e5..3d24c552b 100644 --- a/chain_capabilities/evm/monitoring/write_report.proto +++ b/chain_capabilities/evm/monitoring/write_report.proto @@ -93,3 +93,11 @@ message WriteReportInsufficientGasRetry { int32 queue_position = 4; monitoring.ExecutionContext execution_context = 20; } + +message WriteReportGasMismatch { + WriteReportRequest req = 1; + uint64 expected_tx_gas_limit = 2; + uint64 actual_tx_gas_limit = 3; + string tx_hash = 4; + monitoring.ExecutionContext execution_context = 20; +} From 1d10a07e75a03cbafe5b22908f0e5f491e1bc4f0 Mon Sep 17 00:00:00 2001 From: Vladimir Shchukin Date: Mon, 14 Sep 2026 15:16:24 -0400 Subject: [PATCH 2/4] run generate --- chain_capabilities/evm/monitoring/log_trigger.pb.go | 2 +- chain_capabilities/evm/monitoring/read_actions.pb.go | 2 +- chain_capabilities/evm/monitoring/write_report.pb.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/chain_capabilities/evm/monitoring/log_trigger.pb.go b/chain_capabilities/evm/monitoring/log_trigger.pb.go index 210a762bd..a925538dd 100644 --- a/chain_capabilities/evm/monitoring/log_trigger.pb.go +++ b/chain_capabilities/evm/monitoring/log_trigger.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.11 +// protoc-gen-go v1.36.10 // protoc v5.29.3 // source: chain_capabilities/evm/monitoring/log_trigger.proto diff --git a/chain_capabilities/evm/monitoring/read_actions.pb.go b/chain_capabilities/evm/monitoring/read_actions.pb.go index 6b58256bc..46889330a 100644 --- a/chain_capabilities/evm/monitoring/read_actions.pb.go +++ b/chain_capabilities/evm/monitoring/read_actions.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.11 +// protoc-gen-go v1.36.10 // protoc v5.29.3 // source: chain_capabilities/evm/monitoring/read_actions.proto diff --git a/chain_capabilities/evm/monitoring/write_report.pb.go b/chain_capabilities/evm/monitoring/write_report.pb.go index 17bce9d4a..8bd221d30 100644 --- a/chain_capabilities/evm/monitoring/write_report.pb.go +++ b/chain_capabilities/evm/monitoring/write_report.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.11 +// protoc-gen-go v1.36.10 // protoc v5.29.3 // source: chain_capabilities/evm/monitoring/write_report.proto From de3d230da8509400cb9d27f452eb946d1c1291e6 Mon Sep 17 00:00:00 2001 From: Vladimir Shchukin Date: Tue, 15 Sep 2026 17:01:15 -0400 Subject: [PATCH 3/4] fmt --- chain_capabilities/evm/actions/write_report.go | 1 - 1 file changed, 1 deletion(-) diff --git a/chain_capabilities/evm/actions/write_report.go b/chain_capabilities/evm/actions/write_report.go index 6ffdc699b..96cf472e5 100644 --- a/chain_capabilities/evm/actions/write_report.go +++ b/chain_capabilities/evm/actions/write_report.go @@ -657,7 +657,6 @@ func (e *EVM) validateInputsAndReportMetadata(requestMetadata capabilities.Reque return err } - if request.GasConfig != nil && request.GasConfig.GasLimit != 0 && request.GasConfig.GasLimit < e.ReceiverGasMinimum+e.forwarderGasOverhead { return fmt.Errorf("gas limit is %d, which is lower than minimum gas limit of: %d, for unbounded gas leave the gas limit as nil or 0", request.GasConfig.GasLimit, e.ReceiverGasMinimum+e.forwarderGasOverhead) } From 7a3e5d3100571b24c52a15cdee067fb3191d0e14 Mon Sep 17 00:00:00 2001 From: Vladimir Shchukin Date: Fri, 2 Oct 2026 17:25:33 -0400 Subject: [PATCH 4/4] removed extra atributes --- chain_capabilities/evm/internal/contracts/cre_forwarder.go | 4 ---- chain_capabilities/evm/monitoring/metrics.go | 3 --- 2 files changed, 7 deletions(-) diff --git a/chain_capabilities/evm/internal/contracts/cre_forwarder.go b/chain_capabilities/evm/internal/contracts/cre_forwarder.go index 0f97d06d8..55a11ed76 100644 --- a/chain_capabilities/evm/internal/contracts/cre_forwarder.go +++ b/chain_capabilities/evm/internal/contracts/cre_forwarder.go @@ -89,10 +89,6 @@ const ( // InternalGasRequirements is the forwarder's total internal reservation, subtracted from // gasleft() before the receiver gas budget is recorded (INTERNAL_GAS_REQUIREMENTS). InternalGasRequirements uint64 = 25_000 + InternalGasRequirementsAfterReport - // MinimumGasLimit is the forwarder's routing floor: route() reverts the whole tx when the - // recorded receiver budget would fall below it (MINIMUM_GAS_LIMIT). Any transmission that - // routed at all therefore records at least this much gas. - MinimumGasLimit uint64 = InternalGasRequirements + 30_000*3 + 10_000 // DefaultForwarderGasOverheadMargin is the default safety margin added on top of the // forwarder's internal reservation to cover the pre-route consumption that is not visible // in the contract constants: tx intrinsic cost, calldata (report + signatures), ecrecover diff --git a/chain_capabilities/evm/monitoring/metrics.go b/chain_capabilities/evm/monitoring/metrics.go index 88b0f8f0f..32b1ce26f 100644 --- a/chain_capabilities/evm/monitoring/metrics.go +++ b/chain_capabilities/evm/monitoring/metrics.go @@ -675,9 +675,6 @@ func (r *WriteReportInsufficientGasRetry) MetricAttributes() []attribute.KeyValu func (r *WriteReportGasMismatch) LogAttributes() []attribute.KeyValue { return append([]attribute.KeyValue{ attribute.String("receiver", getReceiver(r.Req.GetReceiver())), - attribute.String("tx_hash", r.GetTxHash()), - attribute.Int64("expected_tx_gas_limit", int64(r.GetExpectedTxGasLimit())), //nolint:gosec // G115: EVM gas fits int64 for logging - attribute.Int64("actual_tx_gas_limit", int64(r.GetActualTxGasLimit())), //nolint:gosec // G115: EVM gas fits int64 for logging }, r.ExecutionContext.LogAttributes()...) }