From c1f85a2ce33e8bd6c96a6e021a38ab8e0e3103c8 Mon Sep 17 00:00:00 2001 From: Mateusz Galazyn Date: Thu, 21 May 2026 08:21:21 +0200 Subject: [PATCH 01/40] Update cardano-rpc-11.0, cardano-api-11.3, cardano-cli-11.1, plutus-ledger-api-1.65 --- bench/plutus-scripts-bench/plutus-scripts-bench.cabal | 8 ++++---- bench/tx-generator/tx-generator.cabal | 4 ++-- cabal.project | 2 +- cardano-node-chairman/cardano-node-chairman.cabal | 2 +- cardano-node/cardano-node.cabal | 4 ++-- cardano-submit-api/cardano-submit-api.cabal | 4 ++-- cardano-testnet/cardano-testnet.cabal | 4 ++-- flake.lock | 6 +++--- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/bench/plutus-scripts-bench/plutus-scripts-bench.cabal b/bench/plutus-scripts-bench/plutus-scripts-bench.cabal index b0812288828..1333026f4af 100644 --- a/bench/plutus-scripts-bench/plutus-scripts-bench.cabal +++ b/bench/plutus-scripts-bench/plutus-scripts-bench.cabal @@ -82,10 +82,10 @@ library -- IOG dependencies -------------------------- build-depends: - , cardano-api ^>=11.0 - , plutus-ledger-api ^>=1.63 - , plutus-tx ^>=1.63 - , plutus-tx-plugin ^>=1.63 + , cardano-api ^>=11.3 + , plutus-ledger-api ^>=1.65 + , plutus-tx ^>=1.65 + , plutus-tx-plugin ^>=1.65 ------------------------ -- Non-IOG dependencies diff --git a/bench/tx-generator/tx-generator.cabal b/bench/tx-generator/tx-generator.cabal index ef230e3e003..5ebadb43f08 100644 --- a/bench/tx-generator/tx-generator.cabal +++ b/bench/tx-generator/tx-generator.cabal @@ -109,9 +109,9 @@ library , attoparsec-aeson , base16-bytestring , bytestring - , cardano-api ^>= 11.0 + , cardano-api ^>= 11.3 , cardano-binary - , cardano-cli ^>= 11.0 + , cardano-cli ^>= 11.1 , cardano-crypto-class , cardano-crypto-wrapper , cardano-data diff --git a/cabal.project b/cabal.project index c818a53ab0c..ccbde969df7 100644 --- a/cabal.project +++ b/cabal.project @@ -14,7 +14,7 @@ repository cardano-haskell-packages -- you need to run if you change them index-state: , hackage.haskell.org 2026-04-17T09:20:55Z - , cardano-haskell-packages 2026-05-02T16:21:41Z + , cardano-haskell-packages 2026-05-27T09:43:46Z active-repositories: , :rest diff --git a/cardano-node-chairman/cardano-node-chairman.cabal b/cardano-node-chairman/cardano-node-chairman.cabal index 09db99a60e3..fe0eaccea9c 100644 --- a/cardano-node-chairman/cardano-node-chairman.cabal +++ b/cardano-node-chairman/cardano-node-chairman.cabal @@ -86,5 +86,5 @@ test-suite chairman-tests ghc-options: -threaded -rtsopts "-with-rtsopts=-N -T" build-tool-depends: cardano-node:cardano-node - , cardano-cli:cardano-cli ^>= 11.0 + , cardano-cli:cardano-cli ^>= 11.1 , cardano-node-chairman:cardano-node-chairman diff --git a/cardano-node/cardano-node.cabal b/cardano-node/cardano-node.cabal index 2614d7d7884..6f1441ca94e 100644 --- a/cardano-node/cardano-node.cabal +++ b/cardano-node/cardano-node.cabal @@ -123,7 +123,7 @@ library , async , base16-bytestring , bytestring - , cardano-api ^>= 11.0 + , cardano-api ^>= 11.3 , cardano-data , cardano-crypto-class ^>=2.3 , cardano-crypto-wrapper @@ -141,7 +141,7 @@ library , cardano-prelude , cardano-protocol-tpraos >= 1.4 , cardano-slotting >= 0.2 - , cardano-rpc ^>= 10.2 + , cardano-rpc ^>= 11.0 , cborg ^>= 0.2.4 , containers , contra-tracer diff --git a/cardano-submit-api/cardano-submit-api.cabal b/cardano-submit-api/cardano-submit-api.cabal index 6ef4ff984f7..a7dbb612790 100644 --- a/cardano-submit-api/cardano-submit-api.cabal +++ b/cardano-submit-api/cardano-submit-api.cabal @@ -39,9 +39,9 @@ library , aeson , async , bytestring - , cardano-api ^>= 11.0 + , cardano-api ^>= 11.3 , cardano-binary - , cardano-cli ^>= 11.0 + , cardano-cli ^>= 11.1 , cardano-crypto-class ^>=2.3 , containers , ekg-core diff --git a/cardano-testnet/cardano-testnet.cabal b/cardano-testnet/cardano-testnet.cabal index 3d24101a0ad..77a0582ef02 100644 --- a/cardano-testnet/cardano-testnet.cabal +++ b/cardano-testnet/cardano-testnet.cabal @@ -41,8 +41,8 @@ library , annotated-exception , ansi-terminal , bytestring - , cardano-api ^>= 11.0 - , cardano-cli:{cardano-cli, cardano-cli-test-lib} ^>= 11.0 + , cardano-api ^>= 11.3 + , cardano-cli:{cardano-cli, cardano-cli-test-lib} ^>= 11.1 , cardano-crypto-class ^>=2.3 , cardano-crypto-wrapper , cardano-git-rev ^>= 0.2.2 diff --git a/flake.lock b/flake.lock index 07c24c46639..3be1a1293b2 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "CHaP": { "flake": false, "locked": { - "lastModified": 1777742585, - "narHash": "sha256-ZzXz2vOhqethlqPgBExPXEnKWvaTbidsIxh5MGv+pwE=", + "lastModified": 1779876270, + "narHash": "sha256-FA9E1EaQvPITpO/8weQyi7p3KHgyNb9GiwM6F96Aoeo=", "owner": "intersectmbo", "repo": "cardano-haskell-packages", - "rev": "e8a483522ee73c8c9493ea6055553e5c2532e66b", + "rev": "cb63b6483a5d6ce36fb07815736315bd4408162e", "type": "github" }, "original": { From a330e0484bb9ef445ffb2f3e1514e34feeb26398 Mon Sep 17 00:00:00 2001 From: Mateusz Galazyn Date: Mon, 16 Mar 2026 22:59:50 +0100 Subject: [PATCH 02/40] cardano-testnet | Add chainpoint check to gRPC test --- .../src/Cardano/Node/Protocol/Shelley.hs | 1 + .../src/Testnet/Process/Cli/SPO.hs | 1 + .../src/Testnet/Property/Assert.hs | 1 + .../Cardano/Testnet/Test/Rpc/Query.hs | 33 ++++++++++++++++--- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/cardano-node/src/Cardano/Node/Protocol/Shelley.hs b/cardano-node/src/Cardano/Node/Protocol/Shelley.hs index 22ccebee181..a816b8e91ed 100644 --- a/cardano-node/src/Cardano/Node/Protocol/Shelley.hs +++ b/cardano-node/src/Cardano/Node/Protocol/Shelley.hs @@ -26,6 +26,7 @@ module Cardano.Node.Protocol.Shelley import Cardano.Api hiding (FileError) import qualified Cardano.Api as Api +import Cardano.Api.Experimental.Certificate (OperationalCertificate (..), getHotKey) import qualified Cardano.Crypto.Hash.Class as Crypto import Cardano.Ledger.BaseTypes (ProtVer (..), natVersion) diff --git a/cardano-testnet/src/Testnet/Process/Cli/SPO.hs b/cardano-testnet/src/Testnet/Process/Cli/SPO.hs index 703ff345b65..6b976ff5a17 100644 --- a/cardano-testnet/src/Testnet/Process/Cli/SPO.hs +++ b/cardano-testnet/src/Testnet/Process/Cli/SPO.hs @@ -16,6 +16,7 @@ module Testnet.Process.Cli.SPO ) where import Cardano.Api hiding (cardanoEra) +import Cardano.Api.Experimental.Certificate (PoolId) import qualified Cardano.Api.Ledger as L import qualified Cardano.Ledger.Shelley.LedgerState as L diff --git a/cardano-testnet/src/Testnet/Property/Assert.hs b/cardano-testnet/src/Testnet/Property/Assert.hs index 2c0e6a0afd1..2e610847b86 100644 --- a/cardano-testnet/src/Testnet/Property/Assert.hs +++ b/cardano-testnet/src/Testnet/Property/Assert.hs @@ -16,6 +16,7 @@ module Testnet.Property.Assert import Cardano.Api hiding (Value) +import Cardano.Api.Experimental.Certificate (PoolId) import Prelude hiding (lines) diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/Query.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/Query.hs index 316917617ea..c21ee856091 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/Query.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/Query.hs @@ -29,15 +29,20 @@ import Cardano.Testnet import Prelude import Control.Exception +import Control.Monad import qualified Data.ByteString.Short as SBS import Data.Default.Class import qualified Data.Map.Strict as M +import Data.Time.Clock.POSIX (utcTimeToPOSIXSeconds) +import Data.Word (Word64) +import GHC.Exts (toList) import Lens.Micro import Testnet.Components.Query import Testnet.Process.Run import Testnet.Property.Util (integrationRetryWorkspace) import Testnet.Start.Types +import Testnet.Types (nodeConnectionInfo) import Hedgehog import qualified Hedgehog as H @@ -57,7 +62,7 @@ hprop_rpc_query_pparams = integrationRetryWorkspace 2 "rpc-query-pparams" $ \tem creationOptions = def{creationEra = AnyShelleyBasedEra sbe} runtimeOptions = def{runtimeEnableRpc = RpcEnabled} - TestnetRuntime + tr@TestnetRuntime { testnetMagic , configurationFile , testnetNodes = node0@TestnetNode{nodeSprocket} : _ @@ -80,6 +85,20 @@ hprop_rpc_query_pparams = integrationRetryWorkspace 2 "rpc-query-pparams" $ \tem ChainTipAtGenesis -> H.failure -- impossible ChainTip (SlotNo slot) (HeaderHash hash) (BlockNo blockNo) -> pure (slot, SBS.fromShort hash, blockNo) + ----------------------------------- + -- Compute expected tip timestamp + ----------------------------------- + connectionInfo <- nodeConnectionInfo tr 0 + (systemStart, eraHistory) <- + (H.leftFail <=< H.leftFailM) . H.evalIO $ + executeLocalStateQueryExpr connectionInfo VolatileTip $ do + ss <- querySystemStart + eh <- queryEraHistory + pure $ (,) <$> ss <*> eh + expectedTimestampMs :: Word64 <- H.leftFail $ do + utcTime <- slotToUTCTime systemStart eraHistory (SlotNo slot) + pure . round $ utcTimeToPOSIXSeconds utcTime * 1000 + -------------- -- RPC queries -------------- @@ -90,7 +109,10 @@ hprop_rpc_query_pparams = integrationRetryWorkspace 2 "rpc-query-pparams" $ \tem Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf U5c.QueryService "readParams")) req utxos' <- do - let req = Rpc.defMessage + let req = Rpc.defMessage & U5c.keys .~ + [ def & U5c.hash .~ serialiseToRawBytes tid & U5c.index .~ fromIntegral tix + | (TxIn tid (TxIx tix), _) <- toList utxos + ] Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf U5c.QueryService "readUtxos")) req pure (pparams', utxos') @@ -100,7 +122,7 @@ hprop_rpc_query_pparams = integrationRetryWorkspace 2 "rpc-query-pparams" $ \tem pparamsResponse ^. U5c.ledgerTip . U5c.slot === slot pparamsResponse ^. U5c.ledgerTip . U5c.hash === blockHash pparamsResponse ^. U5c.ledgerTip . U5c.height === blockNo - pparamsResponse ^. U5c.ledgerTip . U5c.timestamp === 0 -- not possible to implement at this moment + H.assertWithinTolerance (pparamsResponse ^. U5c.ledgerTip . U5c.timestamp) expectedTimestampMs 1000 -- https://docs.cardano.org/about-cardano/explore-more/parameter-guide let chainParams = pparamsResponse ^. U5c.values . U5c.cardano @@ -108,9 +130,10 @@ hprop_rpc_query_pparams = integrationRetryWorkspace 2 "rpc-query-pparams" $ \tem pparams ^. L.ppCoinsPerUTxOByteL . to L.unCoinPerByte . to L.fromCompact . to L.unCoin ===^ chainParams ^. U5c.coinsPerUtxoByte . to utxoRpcBigIntToInteger pparams ^. L.ppMaxTxSizeL === chainParams ^. U5c.maxTxSize . to fromIntegral - pparams ^. L.ppTxFeeFixedL ===^ chainParams ^. U5c.minFeeCoefficient . to (fmap L.Coin . utxoRpcBigIntToInteger) - pparams ^. L.ppTxFeePerByteL . to L.unCoinPerByte . to L.fromCompact . to L.unCoin + pparams ^. L.ppTxFeeFixedL . to L.unCoin ===^ chainParams ^. U5c.minFeeConstant . to utxoRpcBigIntToInteger + pparams ^. L.ppTxFeePerByteL . to L.unCoinPerByte . to L.fromCompact . to L.unCoin + ===^ chainParams ^. U5c.minFeeCoefficient . to utxoRpcBigIntToInteger pparams ^. L.ppMaxBBSizeL === chainParams ^. U5c.maxBlockBodySize . to fromIntegral pparams ^. L.ppMaxBHSizeL === chainParams ^. U5c.maxBlockHeaderSize . to fromIntegral pparams ^. L.ppKeyDepositL ===^ chainParams ^. U5c.stakeKeyDeposit . to (fmap L.Coin . utxoRpcBigIntToInteger) From 0515727b75cef16d5c91c6621afa2a8a78b507f7 Mon Sep 17 00:00:00 2001 From: Mateusz Galazyn Date: Thu, 21 May 2026 08:27:30 +0200 Subject: [PATCH 03/40] cardano-testnet | Fix ProposeNewConstitution flaky test --- .gitignore | 2 + .../Test/Gov/ProposeNewConstitution.hs | 142 ++++++++++-------- .../files/golden/queries/govStateOut.json | 56 +++---- .../queries/protocolParametersFileOut.json | 14 +- .../golden/queries/protocolParametersOut.txt | 14 +- 5 files changed, 127 insertions(+), 101 deletions(-) diff --git a/.gitignore b/.gitignore index fa56a0bd882..617e5c62f91 100644 --- a/.gitignore +++ b/.gitignore @@ -77,3 +77,5 @@ cardano-tracer/cardano-tracer-test .idea/ .codex + +.serena/ diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitution.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitution.hs index 0e8df21398f..d1693a00c73 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitution.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitution.hs @@ -232,69 +232,29 @@ hprop_ledger_events_propose_new_constitution = integrationRetryWorkspace 2 "prop retryUntilJustM epochStateView (WaitForEpochs $ EpochInterval 1) $ maybeExtractGovernanceActionIndex governanceActionTxId <$> getEpochState epochStateView - -- Proposal was successfully submitted, now we vote on the proposal and confirm it was ratified - voteFiles <- generateVoteFiles execConfig work "vote-files" - governanceActionTxId governanceActionIndex - [(defaultDRepKeyPair idx, vote) | (vote, idx) <- allVotes] - - -- Submit votes - voteTxBodyFp <- createVotingTxBody execConfig epochStateView sbe work "vote-tx-body" - voteFiles wallet0 - - let signingKeys = Some <$> (paymentKeyInfoPair wallet0:(defaultDRepKeyPair . snd <$> allVotes)) - voteTxFp <- signTx execConfig cEra gov "signed-vote-tx" voteTxBodyFp signingKeys - - submitTx execConfig cEra voteTxFp - - waitForGovActionVotes epochStateView (EpochInterval 1) - - txId <- H.noteShowM $ retrieveTransactionId execConfig signedProposalTx - - -- Count votes before checking for ratification. It may happen that the proposal gets removed after - -- ratification because of a long waiting time, so we won't be able to access votes. - govState <- getGovState epochStateView ceo - govActionState <- H.headM $ govState ^. L.cgsProposalsL . L.pPropsL . to toList - let votes = govActionState ^. L.gasDRepVotesL . to toList - - length (filter ((== L.VoteYes) . snd) votes) === 4 - length (filter ((== L.VoteNo) . snd) votes) === 3 - length (filter ((== L.Abstain) . snd) votes) === 2 - length votes === fromIntegral numVotes - - -- We check that constitution was successfully ratified - void . H.leftFailM . H.evalIO . runExceptT $ - foldEpochState - configurationFile - socketPath - FullValidation - (EpochNo 10) - () - (\epochState _ _ -> foldBlocksCheckConstitutionWasRatified constitutionHash constitutionScriptHash epochState) - - proposalsJSON :: Aeson.Value <- execCliStdoutToJson execConfig - [ eraName, "query", "proposals", "--governance-action-tx-id", prettyShow txId - , "--governance-action-index", "0" - ] + -- Query proposals via CLI before voting to verify proposal structure. + -- This is race-free: the proposal cannot be ratified before votes are cast. + -- Retry until the DRep pulsing snapshot (used by `query proposals`) is refreshed + -- with the newly submitted proposal. The current proposals map is updated immediately, but the + -- pulsing snapshot only picks up new proposals at epoch boundaries. + (proposalsJSON, proposalsArray) <- + retryUntilJustM epochStateView (WaitForEpochs $ EpochInterval 2) $ do + json :: Aeson.Value <- execCliStdoutToJson execConfig + [ eraName, "query", "proposals", "--governance-action-tx-id", prettyShow governanceActionTxId + , "--governance-action-index", "0" + ] + pure $ do + arr <- json ^? Aeson._Array + guard (length arr == 1) + pure (json, arr) -- Display JSON returned in case of failure H.note_ $ Text.unpack . decodeUtf8 $ prettyPrintJSON proposalsJSON - - -- Check that the proposals array has only one element and fetch it - proposalsArray <- H.evalMaybe $ proposalsJSON ^? Aeson._Array - length proposalsArray === 1 let proposal = proposalsArray Vector.! 0 -- Check TxId returned is the same as the one we used proposalsTxId <- H.evalMaybe $ proposal ^? Aeson.key "actionId" . Aeson.key "txId" . Aeson._String - proposalsTxId === Text.pack (prettyShow txId) - - -- Check that committeeVotes is an empty object - proposalsCommitteeVotes <- H.evalMaybe $ proposal ^? Aeson.key "committeeVotes" . Aeson._Object - proposalsCommitteeVotes === mempty - - -- Check that dRepVotes has the expected number of votes - proposalsDRepVotes <- H.evalMaybe $ proposal ^? Aeson.key "dRepVotes" . Aeson._Object - length proposalsDRepVotes === numVotes + proposalsTxId === Text.pack (prettyShow governanceActionTxId) -- Fetch proposalProcedure and anchor proposalsProcedure <- H.evalMaybe $ proposal ^? Aeson.key "proposalProcedure" @@ -334,9 +294,73 @@ hprop_ledger_events_propose_new_constitution = integrationRetryWorkspace 2 "prop proposalsTag <- H.evalMaybe $ proposalsProcedure ^? Aeson.key "govAction" . Aeson.key "tag" . Aeson._String proposalsTag === "NewConstitution" - -- Check the stake pool votes are empty - proposalsStakePoolVotes <- H.evalMaybe $ proposal ^? Aeson.key "stakePoolVotes" . Aeson._Object - proposalsStakePoolVotes === mempty + -- Proposal was successfully submitted, now we vote on the proposal and confirm it was ratified + voteFiles <- generateVoteFiles execConfig work "vote-files" + governanceActionTxId governanceActionIndex + [(defaultDRepKeyPair idx, vote) | (vote, idx) <- allVotes] + + -- Submit votes + voteTxBodyFp <- createVotingTxBody execConfig epochStateView sbe work "vote-tx-body" + voteFiles wallet0 + + let signingKeys = Some <$> (paymentKeyInfoPair wallet0:(defaultDRepKeyPair . snd <$> allVotes)) + voteTxFp <- signTx execConfig cEra gov "signed-vote-tx" voteTxBodyFp signingKeys + + submitTx execConfig cEra voteTxFp + + waitForGovActionVotes epochStateView (EpochInterval 1) + + -- Count votes before checking for ratification. It may happen that the proposal gets removed after + -- ratification because of a long waiting time, so we won't be able to access votes. + govState <- getGovState epochStateView ceo + govActionState <- H.headM $ govState ^. L.cgsProposalsL . L.pPropsL . to toList + let votes = govActionState ^. L.gasDRepVotesL . to toList + + length (filter ((== L.VoteYes) . snd) votes) === 4 + length (filter ((== L.VoteNo) . snd) votes) === 3 + length (filter ((== L.Abstain) . snd) votes) === 2 + length votes === fromIntegral numVotes + + -- Query proposals via CLI to verify vote counts are reported correctly. + -- The proposal may have been ratified at an epoch boundary between the ledger check above and this + -- CLI query, in which case the proposal is removed from gov-state and the query returns []. + -- The pulsing snapshot may also not yet include the votes even though they are in the ledger state. + -- The ledger-level vote checks above already verified correctness, so we skip gracefully in both cases. + votedProposalsJSON :: Aeson.Value <- execCliStdoutToJson execConfig + [ eraName, "query", "proposals", "--governance-action-tx-id", prettyShow governanceActionTxId + , "--governance-action-index", "0" + ] + + H.note_ $ Text.unpack . decodeUtf8 $ prettyPrintJSON votedProposalsJSON + + votedProposalsArray <- H.evalMaybe $ votedProposalsJSON ^? Aeson._Array + unless (null votedProposalsArray) $ do + length votedProposalsArray === 1 + let votedProposal = votedProposalsArray Vector.! 0 + + -- Check that dRepVotes has the expected number of votes + proposalsDRepVotes <- H.evalMaybe $ votedProposal ^? Aeson.key "dRepVotes" . Aeson._Object + -- Skip if the pulsing snapshot has not yet refreshed with votes + unless (null proposalsDRepVotes) $ do + length proposalsDRepVotes === numVotes + + -- Check that committeeVotes is an empty object + proposalsCommitteeVotes <- H.evalMaybe $ votedProposal ^? Aeson.key "committeeVotes" . Aeson._Object + proposalsCommitteeVotes === mempty + + -- Check the stake pool votes are empty + proposalsStakePoolVotes <- H.evalMaybe $ votedProposal ^? Aeson.key "stakePoolVotes" . Aeson._Object + proposalsStakePoolVotes === mempty + + -- We check that constitution was successfully ratified + void . H.leftFailM . H.evalIO . runExceptT $ + foldEpochState + configurationFile + socketPath + FullValidation + (EpochNo 10) + () + (\epochState _ _ -> foldBlocksCheckConstitutionWasRatified constitutionHash constitutionScriptHash epochState) foldBlocksCheckConstitutionWasRatified :: String -- submitted constitution hash diff --git a/cardano-testnet/test/cardano-testnet-test/files/golden/queries/govStateOut.json b/cardano-testnet/test/cardano-testnet-test/files/golden/queries/govStateOut.json index ec713c7fd2a..8984e8ce8d2 100644 --- a/cardano-testnet/test/cardano-testnet-test/files/golden/queries/govStateOut.json +++ b/cardano-testnet/test/cardano-testnet-test/files/golden/queries/govStateOut.json @@ -414,7 +414,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -424,9 +424,9 @@ 42921, 4, 2, - 30623, - 28755, - 75, + 24548, + 29498, + 38, 1, 898148, 27279, @@ -479,7 +479,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -495,7 +495,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -506,7 +506,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -1092,7 +1092,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -1102,9 +1102,9 @@ 42921, 4, 2, - 30623, - 28755, - 75, + 24548, + 29498, + 38, 1, 898148, 27279, @@ -1157,7 +1157,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -1173,7 +1173,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -1184,7 +1184,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -1759,7 +1759,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -1769,9 +1769,9 @@ 42921, 4, 2, - 30623, - 28755, - 75, + 24548, + 29498, + 38, 1, 898148, 27279, @@ -1824,7 +1824,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -1840,7 +1840,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -1851,7 +1851,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -2423,7 +2423,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -2433,9 +2433,9 @@ 42921, 4, 2, - 30623, - 28755, - 75, + 24548, + 29498, + 38, 1, 898148, 27279, @@ -2488,7 +2488,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -2504,7 +2504,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -2515,7 +2515,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, diff --git a/cardano-testnet/test/cardano-testnet-test/files/golden/queries/protocolParametersFileOut.json b/cardano-testnet/test/cardano-testnet-test/files/golden/queries/protocolParametersFileOut.json index e96183cc046..f8156095026 100644 --- a/cardano-testnet/test/cardano-testnet-test/files/golden/queries/protocolParametersFileOut.json +++ b/cardano-testnet/test/cardano-testnet-test/files/golden/queries/protocolParametersFileOut.json @@ -403,7 +403,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -413,9 +413,9 @@ 42921, 4, 2, - 30623, - 28755, - 75, + 24548, + 29498, + 38, 1, 898148, 27279, @@ -468,7 +468,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -484,7 +484,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -495,7 +495,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, diff --git a/cardano-testnet/test/cardano-testnet-test/files/golden/queries/protocolParametersOut.txt b/cardano-testnet/test/cardano-testnet-test/files/golden/queries/protocolParametersOut.txt index e96183cc046..f8156095026 100644 --- a/cardano-testnet/test/cardano-testnet-test/files/golden/queries/protocolParametersOut.txt +++ b/cardano-testnet/test/cardano-testnet-test/files/golden/queries/protocolParametersOut.txt @@ -403,7 +403,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -413,9 +413,9 @@ 42921, 4, 2, - 30623, - 28755, - 75, + 24548, + 29498, + 38, 1, 898148, 27279, @@ -468,7 +468,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -484,7 +484,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, @@ -495,7 +495,7 @@ 7305, -900, 1716, - 960, + 549, 57, 85848, 0, From 3407a73e09621e087621da7b113d323368f4ce7b Mon Sep 17 00:00:00 2001 From: Mateusz Galazyn Date: Thu, 21 May 2026 08:28:14 +0200 Subject: [PATCH 04/40] cardano-testnet | Add searchUtxos gRPC test --- .../src/Cardano/Node/Tracing/Tracers/Rpc.hs | 11 + cardano-testnet/cardano-testnet.cabal | 2 + .../Cardano/Testnet/Test/Rpc/SearchUtxos.hs | 219 ++++++++++++++++++ .../Cardano/Testnet/Test/Rpc/Transaction.hs | 47 ++-- .../cardano-testnet-test.hs | 2 + .../files/calculatePlutusScriptCost.json | 2 +- .../files/golden/queries/govStateOut.json | 56 ++--- .../queries/protocolParametersFileOut.json | 14 +- .../golden/queries/protocolParametersOut.txt | 14 +- 9 files changed, 299 insertions(+), 68 deletions(-) create mode 100644 cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/SearchUtxos.hs diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/Rpc.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/Rpc.hs index d81458625fb..94fb782be12 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/Rpc.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/Rpc.hs @@ -35,6 +35,10 @@ instance LogFormatting TraceRpc where [ "queryName" .= String "ReadUtxos" , spanToObject s ] + TraceRpcQuerySearchUtxosSpan s -> + [ "queryName" .= String "SearchUtxos" + , spanToObject s + ] TraceRpcSubmit submitTrace -> ["kind" .= String "SubmitService"] <> case submitTrace of @@ -50,6 +54,7 @@ instance LogFormatting TraceRpc where -- query names here are taken from UTXORPC spec: https://utxorpc.org/query/intro/#operations TraceRpcQuery (TraceRpcQueryParamsSpan (SpanBegin _)) -> [CounterM "rpc.request.QueryService.ReadParams" Nothing] TraceRpcQuery (TraceRpcQueryReadUtxosSpan (SpanBegin _)) -> [CounterM "rpc.request.QueryService.ReadUtxos" Nothing] + TraceRpcQuery (TraceRpcQuerySearchUtxosSpan (SpanBegin _)) -> [CounterM "rpc.request.QueryService.SearchUtxos" Nothing] TraceRpcSubmit (TraceRpcSubmitSpan (SpanBegin _)) -> [CounterM "rpc.request.SubmitService.SubmitTx" Nothing] _ -> [] @@ -63,6 +68,7 @@ instance MetaTrace TraceRpc where : case queryTrace of TraceRpcQueryParamsSpan _ -> ["ReadParams", "Span"] TraceRpcQueryReadUtxosSpan _ -> ["ReadUtxos", "Span"] + TraceRpcQuerySearchUtxosSpan _ -> ["SearchUtxos", "Span"] TraceRpcSubmit submitTrace -> "SubmitService" : case submitTrace of @@ -76,6 +82,7 @@ instance MetaTrace TraceRpc where ["Error"] -> Just Debug -- those are normal operation errors, like request errors, hide them by default ["QueryService", "ReadParams", "Span"] -> Just Debug ["QueryService", "ReadUtxos", "Span"] -> Just Debug + ["QueryService", "SearchUtxos", "Span"] -> Just Debug ["SubmitService", "SubmitTx", "Span"] -> Just Debug ["SubmitService", "N2cConnectionError"] -> Just Warning -- this is a more serious error, this shouldn't happen ["SubmitService", "TxDecodingError"] -> Just Debug -- request error @@ -87,6 +94,7 @@ instance MetaTrace TraceRpc where ["Error"] -> Just "Normal operation errors such as request errors. Those are not harmful to the RPC server itself." ["QueryService", "ReadParams", "Span"] -> Just "Span for the ReadParams UTXORPC method." ["QueryService", "ReadUtxos", "Span"] -> Just "Span for the ReadUtxos UTXORPC method." + ["QueryService", "SearchUtxos", "Span"] -> Just "Span for the SearchUtxos UTXORPC method." ["SubmitService", "SubmitTx", "Span"] -> Just "Span for the SubmitTx UTXORPC method." ["SubmitService", "N2cConnectionError"] -> Just @@ -100,6 +108,8 @@ instance MetaTrace TraceRpc where [("rpc.request.QueryService.ReadParams", "Span for the ReadParams UTXORPC method.")] ["QueryService", "ReadUtxos", "Span"] -> [("rpc.request.QueryService.ReadUtxos", "Span for the ReadUtxos UTXORPC method.")] + ["QueryService", "SearchUtxos", "Span"] -> + [("rpc.request.QueryService.SearchUtxos", "Span for the SearchUtxos UTXORPC method.")] ["SubmitService", "SubmitTx", "Span"] -> [("rpc.request.SubmitService.SubmitTx", "Span for the SubmitTx UTXORPC method.")] _ -> [] @@ -110,6 +120,7 @@ instance MetaTrace TraceRpc where , ["Error"] , ["QueryService", "ReadParams", "Span"] , ["QueryService", "ReadUtxos", "Span"] + , ["QueryService", "SearchUtxos", "Span"] , ["SubmitService", "SubmitTx", "Span"] , ["SubmitService", "N2cConnectionError"] , ["SubmitService", "TxDecodingError"] diff --git a/cardano-testnet/cardano-testnet.cabal b/cardano-testnet/cardano-testnet.cabal index 77a0582ef02..766c485b639 100644 --- a/cardano-testnet/cardano-testnet.cabal +++ b/cardano-testnet/cardano-testnet.cabal @@ -239,6 +239,7 @@ test-suite cardano-testnet-test Cardano.Testnet.Test.Gov.TreasuryGrowth Cardano.Testnet.Test.Gov.TreasuryWithdrawal Cardano.Testnet.Test.Rpc.Query + Cardano.Testnet.Test.Rpc.SearchUtxos Cardano.Testnet.Test.Rpc.Transaction Cardano.Testnet.Test.Misc Cardano.Testnet.Test.Node.Shutdown @@ -275,6 +276,7 @@ test-suite cardano-testnet-test , directory , exceptions , filepath + , grpc-spec , hedgehog , hedgehog-extras , http-conduit diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/SearchUtxos.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/SearchUtxos.hs new file mode 100644 index 00000000000..d6974a19925 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/SearchUtxos.hs @@ -0,0 +1,219 @@ +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE GADTs #-} +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedLists #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} + +module Cardano.Testnet.Test.Rpc.SearchUtxos + ( hprop_rpc_search_utxos + ) +where + +import Cardano.Api +import qualified Cardano.Api.Experimental as Exp +import qualified Cardano.Api.Experimental.Tx as Exp +import qualified Cardano.Api.Ledger as L + +import Cardano.Rpc.Client (Proto) +import qualified Cardano.Rpc.Client as Rpc +import qualified Cardano.Rpc.Proto.Api.UtxoRpc.Query as U5c hiding (cardano) +import qualified Cardano.Rpc.Proto.Api.UtxoRpc.Query as UtxoRpc +import qualified Cardano.Rpc.Proto.Api.UtxoRpc.Submit as U5c +import qualified Cardano.Rpc.Proto.Api.UtxoRpc.Submit as UtxoRpc +import Cardano.Rpc.Server.Internal.UtxoRpc.Predicate (serialisePaymentCredential) +import Cardano.Rpc.Server.Internal.UtxoRpc.Type +import Cardano.Testnet + +import Prelude + +import Control.Exception (try) +import Control.Monad.Trans.Control (liftBaseOp) +import Data.ByteString (ByteString) +import Data.Default.Class +import GHC.Stack +import Lens.Micro +import Network.GRPC.Spec (GrpcError (..), GrpcException (..)) + +import Testnet.Components.Query (TestnetWaitPeriod (..), getEpochStateView, retryUntilM) +import Testnet.Property.Util (integrationRetryWorkspace) +import Testnet.Types + +import Hedgehog +import qualified Hedgehog as H +import qualified Hedgehog.Extras.Test.Base as H +import qualified Hedgehog.Extras.Test.TestWatchdog as H + +-- | E2E test for the SearchUtxos gRPC method. +-- +-- Spins up a testnet, submits a transaction to create UTxOs at a known address, +-- waits for them to appear in the UTxO set, then exercises SearchUtxos with +-- exact-address and payment-credential predicates. +-- +-- Run with: +-- @TASTY_PATTERN='/RPC SearchUtxos/' cabal test cardano-testnet-test@ +hprop_rpc_search_utxos :: Property +hprop_rpc_search_utxos = integrationRetryWorkspace 2 "rpc-search-utxos" $ \tempAbsBasePath' -> H.runWithDefaultWatchdog_ $ do + conf <- mkConf tempAbsBasePath' + let era = Exp.ConwayEra + sbe = convert era + creationOptions = def{creationEra = AnyShelleyBasedEra sbe} + runtimeOptions = def{runtimeEnableRpc = RpcEnabled} + addressInEra = asAddressInEra sbe + + TestnetRuntime + { configurationFile + , testnetNodes = node0 : _ + , wallets = wallet0@(PaymentKeyInfo _ addressText0) : (PaymentKeyInfo _ addressText1) : _ + } <- + createAndRunTestnet creationOptions runtimeOptions conf + + epochStateView <- getEpochStateView configurationFile $ nodeSocketPath node0 + rpcSocket <- H.note . unFile $ nodeRpcSocketPath node0 + + H.noteShow_ addressText0 + address0 <- H.nothingFail $ deserialiseAddress addressInEra addressText0 + + H.noteShow_ addressText1 + address1 <- H.nothingFail $ deserialiseAddress addressInEra addressText1 + + wit0 :: ShelleyWitnessSigningKey <- + H.leftFailM . H.evalIO $ + readFileTextEnvelopeAnyOf + [FromSomeType asType WitnessGenesisUTxOKey] + (signingKey $ paymentKeyInfoPair wallet0) + + let rpcServer = Rpc.ServerUnix rpcSocket + + ---------------------- + -- Build and submit tx + ---------------------- + (pparamsResponse, initialSearch) <- H.noteShowM . H.evalIO . Rpc.withConnection def rpcServer $ \conn -> do + pparams' <- + Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf UtxoRpc.QueryService "readParams")) def + + search' <- + Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf UtxoRpc.QueryService "searchUtxos")) $ + def & U5c.predicate .~ addressPredicate address0 + pure (pparams', search') + + pparams <- H.leftFail $ utxoRpcPParamsToProtocolParams era $ pparamsResponse ^. U5c.values . U5c.cardano + + txOut0 : _ <- H.noteShow $ initialSearch ^. U5c.items + txIn0 <- txoRefToTxIn $ txOut0 ^. U5c.txoRef + + outputCoin <- H.leftFail $ txOut0 ^. U5c.cardano . U5c.coin . to utxoRpcBigIntToInteger + let amount = 200_000_000 + fee = 500 + change = outputCoin - amount - fee + mkOut ledgerAddress coin = Exp.obtainCommonConstraints era $ + Exp.TxOut $ L.mkBasicTxOut ledgerAddress $ L.inject $ L.Coin coin + content = + Exp.defaultTxBodyContent + & Exp.setTxIns [(txIn0, Exp.AnyKeyWitnessPlaceholder)] + & Exp.setTxFee (L.Coin fee) + & Exp.setTxOuts [mkOut (toShelleyAddr address1) amount, mkOut (toShelleyAddr address0) change] + & Exp.setTxProtocolParams pparams + + unsignedTx <- H.leftFail $ Exp.makeUnsignedTx era content + let keyWit = Exp.makeKeyWitness era unsignedTx wit0 + Exp.SignedTx signedLedgerTx = Exp.signTx era [] [keyWit] unsignedTx + + liftBaseOp (Rpc.withConnection def rpcServer) $ \conn -> do + _submitResponse <- H.noteShowM . H.evalIO $ + Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf UtxoRpc.SubmitService "submitTx")) $ + def & U5c.tx .~ (def & U5c.raw .~ serialiseToRawBytes (Exp.SignedTx signedLedgerTx)) + + ------------------------------------------- + -- Wait for UTxOs to appear at address1 + ------------------------------------------- + H.note_ $ "Wait for 2 UTxOs at address " <> show addressText1 + utxosAtAddress1 <- retryUntilM epochStateView (WaitForBlocks 10) + (do searchResult <- H.evalIO $ + Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf UtxoRpc.QueryService "searchUtxos")) $ + def & U5c.predicate .~ addressPredicate address1 + pure $ searchResult ^. U5c.items + ) + (\xs -> length xs == 2) + + ------------------------------------------- + -- Test 1: exact address predicate returns correct amounts + ------------------------------------------- + H.note_ "Test 1: Verify exact address search returns correct coin values" + let outputAmounts = map (^. U5c.cardano . U5c.coin) utxosAtAddress1 + H.assertWith outputAmounts $ elem (inject amount) + + ------------------------------------------- + -- Test 2: payment credential predicate + ------------------------------------------- + H.note_ "Test 2: Verify exact address + payment credential predicate matches same UTxOs" + paymentCredBytes <- case address1 of + AddressInEra ShelleyAddressInEra{} (ShelleyAddress _ payCred _) -> + pure $ serialisePaymentCredential $ fromShelleyPaymentCredential payCred + _ -> do + H.note_ "Expected a Shelley address" + H.failure + let paymentPredicate :: Proto UtxoRpc.UtxoPredicate + paymentPredicate = + def + & U5c.match + .~ ( def + & U5c.cardano + .~ (def & U5c.address .~ (def & U5c.paymentPart .~ paymentCredBytes)) + ) + payCredSearch <- H.noteShowM . H.evalIO $ + Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf UtxoRpc.QueryService "searchUtxos")) $ + def & U5c.predicate .~ paymentPredicate + + let payCredUtxos = payCredSearch ^. U5c.items + H.assertWith payCredUtxos $ \xs -> length xs == 2 + + ------------------------------------------- + -- Test 3: search with invalid address is rejected + ------------------------------------------- + H.note_ "Test 3: Verify search with invalid address returns GrpcInvalidArgument" + let bogusAddressPredicate :: Proto UtxoRpc.UtxoPredicate + bogusAddressPredicate = + def + & U5c.match + .~ ( def + & U5c.cardano + .~ (def & U5c.address .~ (def & U5c.exactAddress .~ "\x00\x01\x02\x03")) + ) + bogusResult <- H.evalIO . try @GrpcException $ + Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf UtxoRpc.QueryService "searchUtxos")) $ + def & U5c.predicate .~ bogusAddressPredicate + + case bogusResult of + Left err -> grpcError err === GrpcInvalidArgument + Right _ -> do + H.note_ "Expected GrpcInvalidArgument but search succeeded" + H.failure + + ------------------------------------------- + -- Test 4: search without predicate returns all UTxOs + ------------------------------------------- + H.note_ "Test 4: Verify search without predicate returns all UTxOs" + allUtxosSearch <- H.noteShowM . H.evalIO $ + Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf UtxoRpc.QueryService "searchUtxos")) def + + H.assertWith (allUtxosSearch ^. U5c.items) $ \xs -> length xs > 2 + +asAddressInEra :: ShelleyBasedEra era -> AsType (AddressInEra era) +asAddressInEra s = shelleyBasedEraConstraints s $ AsAddressInEra asType + +txoRefToTxIn :: (HasCallStack, MonadTest m) => Proto UtxoRpc.TxoRef -> m TxIn +txoRefToTxIn r = withFrozenCallStack $ do + txId' <- H.leftFail $ deserialiseFromRawBytes AsTxId $ r ^. U5c.hash + pure $ TxIn txId' (TxIx . fromIntegral $ r ^. U5c.index) + +addressPredicate :: IsCardanoEra era => AddressInEra era -> Proto UtxoRpc.UtxoPredicate +addressPredicate address = + def + & U5c.match + .~ ( def + & U5c.cardano + .~ (def & U5c.address .~ (def & U5c.exactAddress .~ serialiseToRawBytes address)) + ) diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/Transaction.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/Transaction.hs index ea88dfe501c..a447f7c4a16 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/Transaction.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/Transaction.hs @@ -18,7 +18,7 @@ import qualified Cardano.Api.Ledger as L import Cardano.Rpc.Client (Proto) import qualified Cardano.Rpc.Client as Rpc -import qualified Cardano.Rpc.Proto.Api.UtxoRpc.Query as U5c hiding (cardano, items, tx) +import qualified Cardano.Rpc.Proto.Api.UtxoRpc.Query as U5c hiding (cardano) import qualified Cardano.Rpc.Proto.Api.UtxoRpc.Query as UtxoRpc import qualified Cardano.Rpc.Proto.Api.UtxoRpc.Submit as U5c import qualified Cardano.Rpc.Proto.Api.UtxoRpc.Submit as UtxoRpc @@ -27,10 +27,8 @@ import Cardano.Testnet import Prelude -import Control.Monad import Control.Monad.Trans.Control (liftBaseOp) import Data.Default.Class -import qualified Data.Text.Encoding as T import GHC.Stack import Lens.Micro @@ -43,8 +41,6 @@ import qualified Hedgehog as H import qualified Hedgehog.Extras.Test.Base as H import qualified Hedgehog.Extras.Test.TestWatchdog as H -import RIO (ByteString) - -- | Run with: -- @TASTY_PATTERN='/RPC Transaction Submit/' cabal test cardano-testnet-test@ hprop_rpc_transaction :: Property @@ -84,21 +80,18 @@ hprop_rpc_transaction = integrationRetryWorkspace 2 "rpc-tx" $ \tempAbsBasePath' -- RPC queries -------------- let rpcServer = Rpc.ServerUnix rpcSocket - (pparamsResponse, utxosResponse) <- H.noteShowM . H.evalIO . Rpc.withConnection def rpcServer $ \conn -> do - pparams' <- do - let req = def - Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf UtxoRpc.QueryService "readParams")) req + (pparamsResponse, searchResponse) <- H.noteShowM . H.evalIO . Rpc.withConnection def rpcServer $ \conn -> do + pparams' <- + Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf UtxoRpc.QueryService "readParams")) def - utxos' <- do - let req = def -- & # U5c.keys .~ [T.encodeUtf8 addressText0] - Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf UtxoRpc.QueryService "readUtxos")) req - pure (pparams', utxos') + search' <- + Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf UtxoRpc.QueryService "searchUtxos")) $ + def & U5c.predicate .~ addressPredicate address0 + pure (pparams', search') pparams <- H.leftFail $ utxoRpcPParamsToProtocolParams era $ pparamsResponse ^. U5c.values . U5c.cardano - txOut0 : _ <- H.noteShowM . flip filterM (utxosResponse ^. U5c.items) $ \utxo -> do - utxoAddress <- deserialiseAddressBs addressInEra $ utxo ^. U5c.cardano . U5c.address - pure $ address0 == utxoAddress + txOut0 : _ <- H.noteShow $ searchResponse ^. U5c.items txIn0 <- txoRefToTxIn $ txOut0 ^. U5c.txoRef outputCoin <- H.leftFail $ txOut0 ^. U5c.cardano . U5c.coin . to utxoRpcBigIntToInteger @@ -119,7 +112,7 @@ hprop_rpc_transaction = integrationRetryWorkspace 2 "rpc-tx" $ \tempAbsBasePath' Exp.SignedTx signedLedgerTx = Exp.signTx era [] [keyWit] unsignedTx txId' <- H.noteShow . Exp.obtainCommonConstraints era . TxId $ Exp.hashTxBody (signedLedgerTx ^. L.bodyTxL) - H.noteShowPretty_ utxosResponse + H.noteShowPretty_ searchResponse liftBaseOp (Rpc.withConnection def rpcServer) $ \conn -> do submitResponse <- H.noteShowM . H.evalIO $ @@ -131,14 +124,12 @@ hprop_rpc_transaction = integrationRetryWorkspace 2 "rpc-tx" $ \tempAbsBasePath' H.note_ "Ensure that submitTx returns the same transaction ID as the locally computed signed transaction ID" txId' === submittedTxId - -- TODO use searchUtxos when available H.note_ $ "Ensure that there are 2 UTXOs in the address " <> show addressText1 utxosForAddress <- retryUntilM epochStateView (WaitForBlocks 10) - (do utxos <- H.evalIO $ - Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf UtxoRpc.QueryService "readUtxos")) def - flip filterM (utxos ^. U5c.items) $ \utxo -> do - utxoAddress <- deserialiseAddressBs addressInEra $ utxo ^. U5c.cardano . U5c.address - pure $ address1 == utxoAddress + (do searchResult <- H.evalIO $ + Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf UtxoRpc.QueryService "searchUtxos")) $ + def & U5c.predicate .~ addressPredicate address1 + pure $ searchResult ^. U5c.items ) (\xs -> length xs == 2) @@ -154,5 +145,11 @@ txoRefToTxIn r = withFrozenCallStack $ do txId' <- H.leftFail $ deserialiseFromRawBytes AsTxId $ r ^. U5c.hash pure $ TxIn txId' (TxIx . fromIntegral $ r ^. U5c.index) -deserialiseAddressBs :: (MonadTest m, SerialiseAddress c) => AsType c -> ByteString -> m c -deserialiseAddressBs addressInEra = H.nothingFail . deserialiseAddress addressInEra <=< H.leftFail . T.decodeUtf8' +addressPredicate :: IsCardanoEra era => AddressInEra era -> Proto UtxoRpc.UtxoPredicate +addressPredicate address = + def + & U5c.match + .~ ( def + & U5c.cardano + .~ (def & U5c.address .~ (def & U5c.exactAddress .~ serialiseToRawBytes address)) + ) diff --git a/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs b/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs index 49fe74b79a4..f3774bb4206 100644 --- a/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs +++ b/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs @@ -36,6 +36,7 @@ import qualified Cardano.Testnet.Test.MainnetParams import qualified Cardano.Testnet.Test.Node.Shutdown import qualified Cardano.Testnet.Test.Parser import qualified Cardano.Testnet.Test.Rpc.Query +import qualified Cardano.Testnet.Test.Rpc.SearchUtxos import qualified Cardano.Testnet.Test.Rpc.Transaction import qualified Cardano.Testnet.Test.RunTestnet import qualified Cardano.Testnet.Test.SanityCheck @@ -147,6 +148,7 @@ tests = do ] , T.testGroup "RPC" [ ignoreOnWindows "RPC Query Protocol Params" Cardano.Testnet.Test.Rpc.Query.hprop_rpc_query_pparams + , ignoreOnWindows "RPC SearchUtxos" Cardano.Testnet.Test.Rpc.SearchUtxos.hprop_rpc_search_utxos , ignoreOnWindows "RPC Transaction Submit" Cardano.Testnet.Test.Rpc.Transaction.hprop_rpc_transaction ] , T.testGroup "NodesWithOptions parser" diff --git a/cardano-testnet/test/cardano-testnet-test/files/calculatePlutusScriptCost.json b/cardano-testnet/test/cardano-testnet-test/files/calculatePlutusScriptCost.json index b2d1ef63f2c..98a7e4de9bd 100644 --- a/cardano-testnet/test/cardano-testnet-test/files/calculatePlutusScriptCost.json +++ b/cardano-testnet/test/cardano-testnet-test/files/calculatePlutusScriptCost.json @@ -7,4 +7,4 @@ "lovelaceCost": 34, "scriptHash": "186e32faa80a26810392fda6d559c7ed4721a65ce1c9d4ef3e1c87b4" } -] +] \ No newline at end of file diff --git a/cardano-testnet/test/cardano-testnet-test/files/golden/queries/govStateOut.json b/cardano-testnet/test/cardano-testnet-test/files/golden/queries/govStateOut.json index 8984e8ce8d2..ec713c7fd2a 100644 --- a/cardano-testnet/test/cardano-testnet-test/files/golden/queries/govStateOut.json +++ b/cardano-testnet/test/cardano-testnet-test/files/golden/queries/govStateOut.json @@ -414,7 +414,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -424,9 +424,9 @@ 42921, 4, 2, - 24548, - 29498, - 38, + 30623, + 28755, + 75, 1, 898148, 27279, @@ -479,7 +479,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -495,7 +495,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -506,7 +506,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -1092,7 +1092,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -1102,9 +1102,9 @@ 42921, 4, 2, - 24548, - 29498, - 38, + 30623, + 28755, + 75, 1, 898148, 27279, @@ -1157,7 +1157,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -1173,7 +1173,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -1184,7 +1184,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -1759,7 +1759,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -1769,9 +1769,9 @@ 42921, 4, 2, - 24548, - 29498, - 38, + 30623, + 28755, + 75, 1, 898148, 27279, @@ -1824,7 +1824,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -1840,7 +1840,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -1851,7 +1851,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -2423,7 +2423,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -2433,9 +2433,9 @@ 42921, 4, 2, - 24548, - 29498, - 38, + 30623, + 28755, + 75, 1, 898148, 27279, @@ -2488,7 +2488,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -2504,7 +2504,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -2515,7 +2515,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, diff --git a/cardano-testnet/test/cardano-testnet-test/files/golden/queries/protocolParametersFileOut.json b/cardano-testnet/test/cardano-testnet-test/files/golden/queries/protocolParametersFileOut.json index f8156095026..e96183cc046 100644 --- a/cardano-testnet/test/cardano-testnet-test/files/golden/queries/protocolParametersFileOut.json +++ b/cardano-testnet/test/cardano-testnet-test/files/golden/queries/protocolParametersFileOut.json @@ -403,7 +403,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -413,9 +413,9 @@ 42921, 4, 2, - 24548, - 29498, - 38, + 30623, + 28755, + 75, 1, 898148, 27279, @@ -468,7 +468,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -484,7 +484,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -495,7 +495,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, diff --git a/cardano-testnet/test/cardano-testnet-test/files/golden/queries/protocolParametersOut.txt b/cardano-testnet/test/cardano-testnet-test/files/golden/queries/protocolParametersOut.txt index f8156095026..e96183cc046 100644 --- a/cardano-testnet/test/cardano-testnet-test/files/golden/queries/protocolParametersOut.txt +++ b/cardano-testnet/test/cardano-testnet-test/files/golden/queries/protocolParametersOut.txt @@ -403,7 +403,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -413,9 +413,9 @@ 42921, 4, 2, - 24548, - 29498, - 38, + 30623, + 28755, + 75, 1, 898148, 27279, @@ -468,7 +468,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -484,7 +484,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, @@ -495,7 +495,7 @@ 7305, -900, 1716, - 549, + 960, 57, 85848, 0, From 8563b3a40e9047ea25c6fe1bd8a2eeb72142cb35 Mon Sep 17 00:00:00 2001 From: Mateusz Galazyn Date: Tue, 5 May 2026 16:15:47 +0200 Subject: [PATCH 05/40] cardano-testnet | Add gRPC evalTx test --- .../src/Cardano/Node/Tracing/Tracers/Rpc.hs | 13 + cardano-testnet/cardano-testnet.cabal | 1 + .../Cardano/Testnet/Test/Cli/Transaction.hs | 1 - .../Cardano/Testnet/Test/Rpc/Eval.hs | 418 ++++++++++++++++++ .../Cardano/Testnet/Test/Rpc/SearchUtxos.hs | 25 +- .../cardano-testnet-test.hs | 2 + 6 files changed, 447 insertions(+), 13 deletions(-) create mode 100644 cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/Eval.hs diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/Rpc.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/Rpc.hs index 94fb782be12..f9781f5efe6 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/Rpc.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/Rpc.hs @@ -46,6 +46,8 @@ instance LogFormatting TraceRpc where TraceRpcSubmitTxDecodingError _ -> [] TraceRpcSubmitTxValidationError _ -> [] TraceRpcSubmitSpan s -> [spanToObject s] + TraceRpcEvalTxDecodingError _ -> [] + TraceRpcEvalTxSpan s -> [spanToObject s] forHuman = docToText . pretty @@ -56,6 +58,7 @@ instance LogFormatting TraceRpc where TraceRpcQuery (TraceRpcQueryReadUtxosSpan (SpanBegin _)) -> [CounterM "rpc.request.QueryService.ReadUtxos" Nothing] TraceRpcQuery (TraceRpcQuerySearchUtxosSpan (SpanBegin _)) -> [CounterM "rpc.request.QueryService.SearchUtxos" Nothing] TraceRpcSubmit (TraceRpcSubmitSpan (SpanBegin _)) -> [CounterM "rpc.request.SubmitService.SubmitTx" Nothing] + TraceRpcSubmit (TraceRpcEvalTxSpan (SpanBegin _)) -> [CounterM "rpc.request.SubmitService.EvalTx" Nothing] _ -> [] instance MetaTrace TraceRpc where @@ -76,6 +79,8 @@ instance MetaTrace TraceRpc where TraceRpcSubmitTxDecodingError _ -> ["TxDecodingError"] TraceRpcSubmitTxValidationError _ -> ["TxValidationError"] TraceRpcSubmitSpan _ -> ["SubmitTx", "Span"] + TraceRpcEvalTxDecodingError _ -> ["EvalTxDecodingError"] + TraceRpcEvalTxSpan _ -> ["EvalTx", "Span"] severityFor (Namespace _ nsInner) _ = case nsInner of ["FatalError"] -> Just Error -- RPC server startup errors @@ -84,9 +89,11 @@ instance MetaTrace TraceRpc where ["QueryService", "ReadUtxos", "Span"] -> Just Debug ["QueryService", "SearchUtxos", "Span"] -> Just Debug ["SubmitService", "SubmitTx", "Span"] -> Just Debug + ["SubmitService", "EvalTx", "Span"] -> Just Debug ["SubmitService", "N2cConnectionError"] -> Just Warning -- this is a more serious error, this shouldn't happen ["SubmitService", "TxDecodingError"] -> Just Debug -- request error ["SubmitService", "TxValidationError"] -> Just Debug -- request error + ["SubmitService", "EvalTxDecodingError"] -> Just Debug -- request error _ -> Nothing documentFor (Namespace _ nsInner) = case nsInner of @@ -96,11 +103,13 @@ instance MetaTrace TraceRpc where ["QueryService", "ReadUtxos", "Span"] -> Just "Span for the ReadUtxos UTXORPC method." ["QueryService", "SearchUtxos", "Span"] -> Just "Span for the SearchUtxos UTXORPC method." ["SubmitService", "SubmitTx", "Span"] -> Just "Span for the SubmitTx UTXORPC method." + ["SubmitService", "EvalTx", "Span"] -> Just "Span for the EvalTx UTXORPC method." ["SubmitService", "N2cConnectionError"] -> Just "Node connection error. This should not happen, as this means that there is an issue in cardano-rpc configuration." ["SubmitService", "TxDecodingError"] -> Just "A regular request error, when submitted transaction decoding fails." ["SubmitService", "TxValidationError"] -> Just "A regular request error, when submitted transaction is invalid." + ["SubmitService", "EvalTxDecodingError"] -> Just "A regular request error, when evalTx transaction decoding fails." _ -> Nothing metricsDocFor (Namespace _ nsInner) = case nsInner of @@ -112,6 +121,8 @@ instance MetaTrace TraceRpc where [("rpc.request.QueryService.SearchUtxos", "Span for the SearchUtxos UTXORPC method.")] ["SubmitService", "SubmitTx", "Span"] -> [("rpc.request.SubmitService.SubmitTx", "Span for the SubmitTx UTXORPC method.")] + ["SubmitService", "EvalTx", "Span"] -> + [("rpc.request.SubmitService.EvalTx", "Span for the EvalTx UTXORPC method.")] _ -> [] allNamespaces = @@ -122,9 +133,11 @@ instance MetaTrace TraceRpc where , ["QueryService", "ReadUtxos", "Span"] , ["QueryService", "SearchUtxos", "Span"] , ["SubmitService", "SubmitTx", "Span"] + , ["SubmitService", "EvalTx", "Span"] , ["SubmitService", "N2cConnectionError"] , ["SubmitService", "TxDecodingError"] , ["SubmitService", "TxValidationError"] + , ["SubmitService", "EvalTxDecodingError"] ] -- helper functions diff --git a/cardano-testnet/cardano-testnet.cabal b/cardano-testnet/cardano-testnet.cabal index 766c485b639..f939a8be8d1 100644 --- a/cardano-testnet/cardano-testnet.cabal +++ b/cardano-testnet/cardano-testnet.cabal @@ -238,6 +238,7 @@ test-suite cardano-testnet-test Cardano.Testnet.Test.Gov.TreasuryDonation Cardano.Testnet.Test.Gov.TreasuryGrowth Cardano.Testnet.Test.Gov.TreasuryWithdrawal + Cardano.Testnet.Test.Rpc.Eval Cardano.Testnet.Test.Rpc.Query Cardano.Testnet.Test.Rpc.SearchUtxos Cardano.Testnet.Test.Rpc.Transaction diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/Transaction.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/Transaction.hs index 46265346eaa..af9d4b781d0 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/Transaction.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/Transaction.hs @@ -13,7 +13,6 @@ import qualified Cardano.Api.Ledger as L import Cardano.CLI.Type.Common import Cardano.Crypto.Hash.Class (hashToStringAsHex) -import qualified Cardano.Ledger.Core as L import Cardano.Testnet import Prelude diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/Eval.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/Eval.hs new file mode 100644 index 00000000000..60b2f0ee22b --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/Eval.hs @@ -0,0 +1,418 @@ +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} + +-- | Run with: +-- @TASTY_PATTERN='/RPC Eval Tx/' cabal test cardano-testnet-test@ +module Cardano.Testnet.Test.Rpc.Eval + ( hprop_rpc_eval_tx + ) +where + +import Cardano.Api +import qualified Cardano.Api.Experimental as Exp +import qualified Cardano.Api.Ledger as L + +import Cardano.Rpc.Client (Proto (..)) +import qualified Cardano.Rpc.Client as Rpc +import qualified Cardano.Rpc.Proto.Api.UtxoRpc.Query as U5c hiding (cardano) +import qualified Cardano.Rpc.Proto.Api.UtxoRpc.Submit as U5c +import Cardano.Rpc.Server.Internal.UtxoRpc.Type (utxoRpcBigIntToInteger) +import Cardano.Testnet + +import Prelude + +import Control.Monad (void) +import Control.Monad.Catch (MonadCatch) +import Control.Monad.Trans.Control (MonadBaseControl, liftBaseOp) +import Data.Default.Class +import qualified Data.Map.Strict as Map +import qualified Data.Text as T +import GHC.Stack (HasCallStack, withFrozenCallStack) +import Lens.Micro +import System.FilePath (()) + +import Testnet.Components.Query (TestnetWaitPeriod (..), findLargestUtxoForPaymentKey, + findLargestUtxoWithAddress, getEpochStateView, retryUntilJustM) +import Testnet.Defaults (plutusV3Script) +import Testnet.Process.Run (execCli', mkExecConfig) +import Testnet.Property.Util (integrationRetryWorkspace) +import Testnet.Start.Types (anyEraToString) +import Testnet.Types + +import Hedgehog +import qualified Hedgehog as H +import qualified Hedgehog.Extras.Test.Base as H +import qualified Hedgehog.Extras.Test.File as H +import qualified Hedgehog.Extras.Test.TestWatchdog as H + +-- | Evaluate a Plutus V3 spending transaction via the gRPC evalTx endpoint and +-- verify that the response contains a valid fee, non-zero execution units, one +-- redeemer, and no errors. +hprop_rpc_eval_tx :: Property +hprop_rpc_eval_tx = integrationRetryWorkspace 2 "rpc-eval-tx" $ \tempAbsBasePath' -> H.runWithDefaultWatchdog_ $ do + conf@Conf{tempAbsPath} <- mkConf tempAbsBasePath' + let tempAbsPath' = unTmpAbsPath tempAbsPath + work <- H.createDirectoryIfMissing $ tempAbsPath' "work" + + let tempBaseAbsPath = makeTmpBaseAbsPath $ TmpAbsolutePath tempAbsPath' + era = Exp.ConwayEra + sbe = convert era + anyEra = AnyCardanoEra $ toCardanoEra sbe + creationOptions = def{creationEra = AnyShelleyBasedEra sbe} + runtimeOptions = def{runtimeEnableRpc = RpcEnabled} + + TestnetRuntime + { configurationFile + , testnetMagic + , testnetNodes = node : _ + , wallets = wallet0 : wallet1 : _ + } <- + createAndRunTestnet creationOptions runtimeOptions conf + + poolSprocket <- H.noteShow $ nodeSprocket node + execConfig <- mkExecConfig tempBaseAbsPath poolSprocket testnetMagic + epochStateView <- getEpochStateView configurationFile $ nodeSocketPath node + rpcSocket <- H.note . unFile $ nodeRpcSocketPath node + + let rpcServer = Rpc.ServerUnix rpcSocket + utxoSKeyFile = signingKeyFp $ paymentKeyInfoPair wallet0 + utxoSKeyFile1 = signingKeyFp $ paymentKeyInfoPair wallet1 + + ------------------------------------ + -- Write Plutus V3 always-succeeds script + ------------------------------------ + plutusScriptFile <- H.note $ work "always-succeeds.plutusV3" + H.writeFile plutusScriptFile $ T.unpack plutusV3Script + + plutusSpendingScriptAddr <- + execCli' + execConfig + [ "latest" + , "address" + , "build" + , "--payment-script-file" , plutusScriptFile + ] + + scriptDatumHash <- + filter (/= '\n') + <$> execCli' + execConfig + [ "latest" + , "transaction" + , "hash-script-data" + , "--script-data-value" + , "0" + ] + + -- Send ADA to the script address with a datum hash, creating a script-locked + -- UTxO that the spending transaction can later reference with a redeemer. + ------------------------------------ + -- 1. Fund the script address + ------------------------------------ + txinFund <- findLargestUtxoForPaymentKey epochStateView sbe wallet0 + + let fundTxBody = work "fund-script-tx-body" + fundTx = work "fund-script-tx" + + void $ + execCli' + execConfig + [ anyEraToString anyEra + , "transaction" + , "build" + , "--change-address" , T.unpack $ paymentKeyInfoAddr wallet0 + , "--tx-in" , T.unpack $ renderTxIn txinFund + , "--tx-out" , plutusSpendingScriptAddr <> "+" <> show @Int 5_000_000 + , "--tx-out-datum-hash" , scriptDatumHash + , "--out-file" , fundTxBody + ] + + void $ + execCli' + execConfig + [ "latest" + , "transaction" + , "sign" + , "--tx-body-file" , fundTxBody + , "--signing-key-file" , utxoSKeyFile + , "--out-file" , fundTx + ] + + ------------------------------------ + -- 1b. EvalTx on the funding tx (no scripts) + ------------------------------------ + (fundLedgerTx, fundTxEval) <- evalTxFile sbe rpcServer fundTx + + let fundCliFee = fundLedgerTx ^. L.bodyTxL . L.feeTxBodyL + + H.note_ "EvalTx minimum fee should not exceed the CLI-computed fee" + fundEvalFee <- H.leftFail $ fundTxEval ^. U5c.fee . to utxoRpcBigIntToInteger + H.assertWith fundEvalFee (<= L.unCoin fundCliFee) + + H.note_ "No execution units for a plain key-witnessed transaction" + fundTxEval ^. U5c.exUnits . U5c.steps === 0 + fundTxEval ^. U5c.exUnits . U5c.memory === 0 + + H.note_ "No redeemers for a plain key-witnessed transaction" + fundTxEval ^. U5c.redeemers === [] + + H.note_ "No evaluation errors" + fundTxEval ^. U5c.errors === [] + + void $ + execCli' + execConfig + [ "latest" + , "transaction" + , "submit" + , "--tx-file" , fundTx + ] + + ------------------------------------ + -- 2. Wait for the script UTxO, find collateral + ------------------------------------ + plutusScriptTxIn <- + fmap fst . retryUntilJustM epochStateView (WaitForBlocks 3) $ + findLargestUtxoWithAddress epochStateView sbe $ + T.pack plutusSpendingScriptAddr + + txinCollateral <- findLargestUtxoForPaymentKey epochStateView sbe wallet1 + + ------------------------------------ + -- 3. Build and sign the spending tx + ------------------------------------ + let spendTxBody = work "spend-script-tx-body" + spendTx = work "spend-script-tx" + + void $ + execCli' + execConfig + [ anyEraToString anyEra + , "transaction" + , "build" + , "--change-address" , T.unpack $ paymentKeyInfoAddr wallet1 + , "--tx-in-collateral" , T.unpack $ renderTxIn txinCollateral + , "--tx-in" , T.unpack $ renderTxIn plutusScriptTxIn + , "--tx-in-script-file" , plutusScriptFile + , "--tx-in-datum-value" , "0" + , "--tx-in-redeemer-value" , "0" + , "--out-file" , spendTxBody + ] + + void $ + execCli' + execConfig + [ "latest" + , "transaction" + , "sign" + , "--tx-body-file" , spendTxBody + , "--signing-key-file" , utxoSKeyFile1 + , "--out-file" , spendTx + ] + + ------------------------------------ + -- 4. EvalTx on the spending tx (Plutus V3 always-succeeds) + ------------------------------------ + (ledgerTx, txEval) <- evalTxFile sbe rpcServer spendTx + + let cliFee = ledgerTx ^. L.bodyTxL . L.feeTxBodyL + + H.note_ "EvalTx minimum fee should not exceed the CLI-computed fee" + evalFee <- H.leftFail $ txEval ^. U5c.fee . to utxoRpcBigIntToInteger + H.assertWith evalFee (<= L.unCoin cliFee) + + H.note_ "Execution units should match the transaction" + (_, L.ExUnits txMem txSteps) <- H.headM . Map.elems . L.unRedeemers $ ledgerTx ^. L.witsTxL . L.rdmrsTxWitsL + txEval ^. U5c.exUnits . U5c.steps === fromIntegral txSteps + txEval ^. U5c.exUnits . U5c.memory === fromIntegral txMem + + H.note_ "One redeemer for the spend purpose at index 0" + let redeemers = txEval ^. U5c.redeemers + length redeemers === 1 + redeemer0 <- H.headM redeemers + redeemer0 ^. U5c.purpose === Proto U5c.REDEEMER_PURPOSE_SPEND + redeemer0 ^. U5c.index === 0 + redeemer0 ^. U5c.exUnits . U5c.steps === fromIntegral txSteps + redeemer0 ^. U5c.exUnits . U5c.memory === fromIntegral txMem + + H.note_ "No evaluation errors" + txEval ^. U5c.errors === [] + + ------------------------------------ + -- 5. Failure path: always-fails script + ------------------------------------ + let failScript = PlutusScript PlutusScriptV1 $ examplePlutusScriptAlwaysFails WitCtxTxIn + failScriptFile <- H.note $ work "always-fails.plutusV1" + H.leftFailM . H.evalIO $ + writeFileTextEnvelope (File failScriptFile) Nothing failScript + + failScriptAddr <- + execCli' + execConfig + [ "latest" + , "address" + , "build" + , "--payment-script-file" , failScriptFile + ] + + txinFund2 <- findLargestUtxoForPaymentKey epochStateView sbe wallet0 + + let fundFailTxBody = work "fund-fail-script-tx-body" + fundFailTx = work "fund-fail-script-tx" + + void $ + execCli' + execConfig + [ anyEraToString anyEra + , "transaction" + , "build" + , "--change-address" , T.unpack $ paymentKeyInfoAddr wallet0 + , "--tx-in" , T.unpack $ renderTxIn txinFund2 + , "--tx-out" , failScriptAddr <> "+" <> show @Int 5_000_000 + , "--tx-out-datum-hash" , scriptDatumHash + , "--out-file" , fundFailTxBody + ] + + void $ + execCli' + execConfig + [ "latest" + , "transaction" + , "sign" + , "--tx-body-file" , fundFailTxBody + , "--signing-key-file" , utxoSKeyFile + , "--out-file" , fundFailTx + ] + + void $ + execCli' + execConfig + [ "latest" + , "transaction" + , "submit" + , "--tx-file" , fundFailTx + ] + + failScriptTxIn <- + fmap fst . retryUntilJustM epochStateView (WaitForBlocks 3) $ + findLargestUtxoWithAddress epochStateView sbe $ + T.pack failScriptAddr + + txinCollateral2 <- findLargestUtxoForPaymentKey epochStateView sbe wallet1 + + protocolParamsFile <- H.note $ work "protocol-params.json" + void $ + execCli' + execConfig + [ "latest" + , "query" + , "protocol-parameters" + , "--out-file" , protocolParamsFile + ] + + -- Use build-raw because `transaction build` would reject the always-fails script. + let failSpendTxBody = work "fail-spend-tx-body" + failSpendTx = work "fail-spend-tx" + + void $ + execCli' + execConfig + [ anyEraToString anyEra + , "transaction" + , "build-raw" + , "--tx-in" , T.unpack $ renderTxIn failScriptTxIn + , "--tx-in-collateral" , T.unpack $ renderTxIn txinCollateral2 + , "--tx-in-script-file" , failScriptFile + , "--tx-in-datum-value" , "0" + , "--tx-in-redeemer-value" , "0" + , "--tx-in-execution-units" , "(10000000000,10000000)" + , "--tx-out" , T.unpack (paymentKeyInfoAddr wallet1) <> "+4700000" + , "--fee" , "300000" + , "--protocol-params-file" , protocolParamsFile + , "--out-file" , failSpendTxBody + ] + + void $ + execCli' + execConfig + [ "latest" + , "transaction" + , "sign" + , "--tx-body-file" , failSpendTxBody + , "--signing-key-file" , utxoSKeyFile1 + , "--out-file" , failSpendTx + ] + + ------------------------------------ + -- 5b. EvalTx on the always-fails spending tx + ------------------------------------ + (_, failTxEval) <- evalTxFile sbe rpcServer failSpendTx + + H.note_ "Evaluation errors for always-fails script" + failTxEval ^. U5c.errors /== [] + + ------------------------------------ + -- 6. Unbalanced key-witnessed tx + ------------------------------------ + txinUnbal <- findLargestUtxoForPaymentKey epochStateView sbe wallet0 + + let unbalTxBody = work "unbal-tx-body" + unbalTx = work "unbal-tx" + + void $ + execCli' + execConfig + [ anyEraToString anyEra + , "transaction" + , "build-raw" + , "--tx-in" , T.unpack $ renderTxIn txinUnbal + , "--tx-out" , T.unpack (paymentKeyInfoAddr wallet0) <> "+1000000" + , "--fee" , "200000" + , "--out-file" , unbalTxBody + ] + + void $ + execCli' + execConfig + [ "latest" + , "transaction" + , "sign" + , "--tx-body-file" , unbalTxBody + , "--signing-key-file" , utxoSKeyFile + , "--out-file" , unbalTx + ] + + (_, unbalTxEval) <- evalTxFile sbe rpcServer unbalTx + + H.note_ "Balance error for unbalanced transaction" + unbalTxEval ^. U5c.errors /== [] + unbalError <- H.headM $ unbalTxEval ^. U5c.errors + H.assertWith (unbalError ^. U5c.msg) $ T.isInfixOf "not balanced" + + H.note_ "Non-zero fee is still returned" + unbalEvalFee <- H.leftFail $ unbalTxEval ^. U5c.fee . to utxoRpcBigIntToInteger + H.assertWith unbalEvalFee (> 0) + +-- | Read a signed transaction from a file and evaluate it via the gRPC evalTx +-- endpoint, returning the ledger transaction and the TxEval result. +evalTxFile + :: forall era m + . (HasCallStack, MonadBaseControl IO m, MonadCatch m, MonadIO m, MonadTest m) + => ShelleyBasedEra era + -> Rpc.Server + -> FilePath + -> m (L.Tx L.TopTx (ShelleyLedgerEra era), Proto U5c.TxEval) +evalTxFile sbe' rpcServer txFile = withFrozenCallStack $ shelleyBasedEraConstraints sbe' $ do + textEnvelope <- H.leftFailM . H.evalIO $ readTextEnvelopeFromFile txFile + ShelleyTx _ ledgerTx <- H.leftFail $ deserialiseFromTextEnvelope @(Tx era) textEnvelope + let request = def & U5c.tx . U5c.raw .~ textEnvelopeRawCBOR textEnvelope + (response :: Proto U5c.EvalTxResponse) <- + liftBaseOp (Rpc.withConnection def rpcServer) $ \conn -> + H.noteShowPrettyM . H.evalIO $ + Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf U5c.SubmitService "evalTx")) request + pure (ledgerTx, response ^. U5c.report . U5c.cardano) diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/SearchUtxos.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/SearchUtxos.hs index d6974a19925..0ab696971a2 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/SearchUtxos.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/SearchUtxos.hs @@ -146,22 +146,22 @@ hprop_rpc_search_utxos = integrationRetryWorkspace 2 "rpc-search-utxos" $ \tempA H.assertWith outputAmounts $ elem (inject amount) ------------------------------------------- - -- Test 2: payment credential predicate + -- Test 2: exact address + payment credential predicate ------------------------------------------- H.note_ "Test 2: Verify exact address + payment credential predicate matches same UTxOs" - paymentCredBytes <- case address1 of - AddressInEra ShelleyAddressInEra{} (ShelleyAddress _ payCred _) -> - pure $ serialisePaymentCredential $ fromShelleyPaymentCredential payCred - _ -> do - H.note_ "Expected a Shelley address" - H.failure - let paymentPredicate :: Proto UtxoRpc.UtxoPredicate + let paymentCredBytes :: ByteString + paymentCredBytes = case address1 of + AddressInEra ShelleyAddressInEra{} (ShelleyAddress _ payCred _) -> + serialisePaymentCredential $ fromShelleyPaymentCredential payCred + _ -> error "Expected a Shelley address" + paymentPredicate :: Proto UtxoRpc.UtxoPredicate paymentPredicate = def & U5c.match .~ ( def & U5c.cardano - .~ (def & U5c.address .~ (def & U5c.paymentPart .~ paymentCredBytes)) + .~ (def & U5c.address .~ (def & U5c.exactAddress .~ serialiseToRawBytes address1 + & U5c.paymentPart .~ paymentCredBytes)) ) payCredSearch <- H.noteShowM . H.evalIO $ Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf UtxoRpc.QueryService "searchUtxos")) $ @@ -193,11 +193,12 @@ hprop_rpc_search_utxos = integrationRetryWorkspace 2 "rpc-search-utxos" $ \tempA H.failure ------------------------------------------- - -- Test 4: search without predicate returns all UTxOs + -- Test 4: combined address predicate returns UTxOs from both addresses ------------------------------------------- - H.note_ "Test 4: Verify search without predicate returns all UTxOs" + H.note_ "Test 4: Verify anyOf predicate with both addresses returns all UTxOs" allUtxosSearch <- H.noteShowM . H.evalIO $ - Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf UtxoRpc.QueryService "searchUtxos")) def + Rpc.nonStreaming conn (Rpc.rpc @(Rpc.Protobuf UtxoRpc.QueryService "searchUtxos")) $ + def & U5c.predicate .~ (def & U5c.anyOf .~ [addressPredicate address0, addressPredicate address1]) H.assertWith (allUtxosSearch ^. U5c.items) $ \xs -> length xs > 2 diff --git a/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs b/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs index f3774bb4206..c1d7ab52310 100644 --- a/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs +++ b/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs @@ -35,6 +35,7 @@ import qualified Cardano.Testnet.Test.Gov.TreasuryWithdrawal as Gov import qualified Cardano.Testnet.Test.MainnetParams import qualified Cardano.Testnet.Test.Node.Shutdown import qualified Cardano.Testnet.Test.Parser +import qualified Cardano.Testnet.Test.Rpc.Eval import qualified Cardano.Testnet.Test.Rpc.Query import qualified Cardano.Testnet.Test.Rpc.SearchUtxos import qualified Cardano.Testnet.Test.Rpc.Transaction @@ -150,6 +151,7 @@ tests = do [ ignoreOnWindows "RPC Query Protocol Params" Cardano.Testnet.Test.Rpc.Query.hprop_rpc_query_pparams , ignoreOnWindows "RPC SearchUtxos" Cardano.Testnet.Test.Rpc.SearchUtxos.hprop_rpc_search_utxos , ignoreOnWindows "RPC Transaction Submit" Cardano.Testnet.Test.Rpc.Transaction.hprop_rpc_transaction + , ignoreOnWindows "RPC Eval Tx" Cardano.Testnet.Test.Rpc.Eval.hprop_rpc_eval_tx ] , T.testGroup "NodesWithOptions parser" [ H.testPropertyNamed "Roundtrip" (fromString "prop_parseNodeSpecs_roundtrip") From 178f66b94df3b475d69a46ae80bc7f36a3c1ecec Mon Sep 17 00:00:00 2001 From: Mateusz Galazyn Date: Wed, 27 May 2026 13:16:17 +0200 Subject: [PATCH 06/40] cardano-testnet | Rewrite TxReferenceInputDatum using experimental api --- .../src/Cardano/TxSubmit/Web.hs | 7 +- ...lazyn_bump_cardano_api_cardano_cli_chap.md | 7 + .../src/Testnet/Process/Cli/DRep.hs | 2 +- .../src/Testnet/Process/Cli/Transaction.hs | 2 +- .../Testnet/Test/Api/TxReferenceInputDatum.hs | 140 +++++++++++------- 5 files changed, 103 insertions(+), 55 deletions(-) create mode 100644 cardano-testnet/changelog.d/20260527_120000_mgalazyn_bump_cardano_api_cardano_cli_chap.md diff --git a/cardano-submit-api/src/Cardano/TxSubmit/Web.hs b/cardano-submit-api/src/Cardano/TxSubmit/Web.hs index 430bee24bbf..dbcc78c84b1 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit/Web.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit/Web.hs @@ -17,8 +17,8 @@ import Cardano.Api (AllegraEra, AnyCardanoEra (AnyCardanoEra), AsType IsCardanoEra (..), LocalNodeConnectInfo (LocalNodeConnectInfo, localConsensusModeParams, localNodeNetworkId, localNodeSocketPath), NetworkId, SerialiseAsCBOR (..), ShelleyBasedEra (..), ShelleyEra, SocketPath, - ToJSON, Tx, TxId (..), TxInMode (TxInMode), TxValidationErrorInCardanoMode (..), - getTxBody, getTxId, submitTxToNodeLocal) + ToJSON, Tx (Tx), TxId (..), TxInMode (TxInMode), + TxValidationErrorInCardanoMode (..), getTxId, submitTxToNodeLocal) import qualified Cardano.Api import Cardano.Binary (DecoderError (..)) @@ -144,7 +144,8 @@ txSubmitPost trace p@(CardanoModeParams cModeParams) networkId socketPath txByte case res of Cardano.Api.TxSubmitSuccess -> do liftIO $ T.putStrLn "Transaction successfully submitted." - return $ getTxId (getTxBody tx) + let Tx txBody _ = tx + return $ getTxId txBody Cardano.Api.TxSubmitFail e -> left $ TxCmdTxSubmitValidationError e Cardano.Api.TxSubmitError e -> diff --git a/cardano-testnet/changelog.d/20260527_120000_mgalazyn_bump_cardano_api_cardano_cli_chap.md b/cardano-testnet/changelog.d/20260527_120000_mgalazyn_bump_cardano_api_cardano_cli_chap.md new file mode 100644 index 00000000000..18cee6700a1 --- /dev/null +++ b/cardano-testnet/changelog.d/20260527_120000_mgalazyn_bump_cardano_api_cardano_cli_chap.md @@ -0,0 +1,7 @@ + +### Maintenance + +- Bump `cardano-api` to `^>= 11.3` +- Bump `cardano-cli` to `^>= 11.1` +- Bump CHaP index-state to `2026-05-27` + diff --git a/cardano-testnet/src/Testnet/Process/Cli/DRep.hs b/cardano-testnet/src/Testnet/Process/Cli/DRep.hs index c7c4ac72ded..d2a7c481163 100644 --- a/cardano-testnet/src/Testnet/Process/Cli/DRep.hs +++ b/cardano-testnet/src/Testnet/Process/Cli/DRep.hs @@ -17,7 +17,7 @@ module Testnet.Process.Cli.DRep , makeActivityChangeProposal ) where -import Cardano.Api hiding (Certificate, TxBody, txId) +import Cardano.Api hiding (TxBody, txId) import Cardano.Api.Experimental (Some (..)) import Cardano.Api.Ledger (EpochInterval (EpochInterval, unEpochInterval)) diff --git a/cardano-testnet/src/Testnet/Process/Cli/Transaction.hs b/cardano-testnet/src/Testnet/Process/Cli/Transaction.hs index 5ecbed60686..6a4c3984feb 100644 --- a/cardano-testnet/src/Testnet/Process/Cli/Transaction.hs +++ b/cardano-testnet/src/Testnet/Process/Cli/Transaction.hs @@ -17,7 +17,7 @@ module Testnet.Process.Cli.Transaction ) where -import Cardano.Api hiding (Certificate, TxBody) +import Cardano.Api hiding (TxBody) import Cardano.Api.Experimental (Some (..)) import Prelude diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Api/TxReferenceInputDatum.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Api/TxReferenceInputDatum.hs index ba614728b7e..c6204ca365c 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Api/TxReferenceInputDatum.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Api/TxReferenceInputDatum.hs @@ -4,6 +4,7 @@ {-# LANGUAGE OverloadedLists #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} module Cardano.Testnet.Test.Api.TxReferenceInputDatum ( hprop_tx_refin_datum @@ -11,10 +12,13 @@ module Cardano.Testnet.Test.Api.TxReferenceInputDatum where import Cardano.Api hiding (txId) +import qualified Cardano.Api.Experimental as Exp +import qualified Cardano.Api.Experimental.Tx as Exp import qualified Cardano.Api.Ledger as L import qualified Cardano.Api.Network as Net import qualified Cardano.Api.UTxO as Utxo +import Cardano.Ledger.Plutus.Data (hashData) import Cardano.Testnet import Prelude @@ -25,6 +29,7 @@ import Data.List (isInfixOf) import qualified Data.Map.Strict as M import Data.Maybe import Data.Set (Set) +import qualified Data.Set as Set import GHC.Exts (IsList (..)) import GHC.Stack import Lens.Micro @@ -53,9 +58,10 @@ hprop_tx_refin_datum = integrationRetryWorkspace 2 "api-tx-refin-dat" $ \tempAbs conf@Conf{tempAbsPath} <- mkConf tempAbsBasePath' let tempAbsPath' = unTmpAbsPath tempAbsPath - let ceo = ConwayEraOnwardsConway + let era = Exp.ConwayEra + sbe = convert era + ceo = convert era beo = convert ceo - sbe = convert ceo eraProxy = proxyToAsType Proxy creationOptions = def{creationEra = AnyShelleyBasedEra sbe} @@ -115,6 +121,7 @@ hprop_tx_refin_datum = integrationRetryWorkspace 2 "api-tx-refin-dat" $ \tempAbs -- prepare txout let txOutValue = lovelaceToTxOutValue sbe 100_000_000 + txOuts :: [TxOut CtxTx ConwayEra] txOuts = [ TxOut addr1 txOutValue txDatum1 ReferenceScriptNone , TxOut addr1 txOutValue txDatum2 ReferenceScriptNone @@ -122,28 +129,40 @@ hprop_tx_refin_datum = integrationRetryWorkspace 2 "api-tx-refin-dat" $ \tempAbs ] -- build a transaction + expTxOuts = map (Exp.TxOut . toShelleyTxOut sbe . toCtxUTxOTxOut) txOuts + -- toCtxUTxOTxOut strips the TxOutSupplementalDatum marker, so we must pass + -- supplemental datums explicitly + supplementalDatums = Exp.obtainCommonConstraints era $ M.fromList + [ let ledgerData = toAlonzoData @(ShelleyLedgerEra ConwayEra) sd + in (hashData ledgerData, ledgerData) + | sd <- [scriptData3] + ] content = - defaultTxBodyContent sbe - & setTxIns [(txIn, pure $ KeyWitness KeyWitnessForSpending)] - & setTxOuts txOuts - & setTxProtocolParams (pure $ pure pparams) + Exp.defaultTxBodyContent + & Exp.setTxIns [(txIn, Exp.AnyKeyWitnessPlaceholder)] + & Exp.setTxOuts expTxOuts + & Exp.setTxProtocolParams (unLedgerProtocolParameters pparams) + & Exp.setTxSupplementalDatums supplementalDatums utxo <- findAllUtxos epochStateView sbe + let ledgerUtxo = Utxo.toShelleyUTxO sbe utxo - BalancedTxBody _ txBody@(ShelleyTxBody _ lbody _ (TxBodyScriptData _ (L.TxDats datums) _) _ _) _ fee <- + (unsignedTx@(Exp.UnsignedTx ledgerTx), _finalContent) <- H.leftFail $ - makeTransactionBodyAutoBalance - sbe + Exp.makeTransactionBodyAutoBalance @ConwayEra systemStart epochInfo - pparams + (unLedgerProtocolParameters pparams) mempty mempty mempty - utxo + ledgerUtxo content addr0 Nothing -- keys override + + let lbody = ledgerTx ^. L.bodyTxL + fee = Exp.getUnsignedTxFee unsignedTx H.noteShow_ fee H.noteShowPretty_ lbody @@ -151,17 +170,20 @@ hprop_tx_refin_datum = integrationRetryWorkspace 2 "api-tx-refin-dat" $ \tempAbs -- sanity check that the integrity hash was calculated lbody ^. L.scriptIntegrityHashTxBodyL /== L.SNothing - let bodyScriptData = fromList . map fromAlonzoData $ M.elems datums :: Set HashableScriptData + let L.TxDats datums = ledgerTx ^. L.witsTxL . L.datsTxWitsL + bodyScriptData = fromList . map fromAlonzoData $ M.elems datums :: Set HashableScriptData -- Only supplemental datums are included here [ scriptData3 ] === bodyScriptData - let tx = signShelleyTransaction sbe txBody [wit0] - txId <- H.noteShow . getTxId $ getTxBody tx + let keyWit = Exp.makeKeyWitness era unsignedTx wit0 + Exp.SignedTx signedLedgerTx = Exp.signTx era [] [keyWit] unsignedTx + txId <- H.noteShow . Exp.obtainCommonConstraints era . TxId $ Exp.hashTxBody (signedLedgerTx ^. L.bodyTxL) - H.noteShowPretty_ tx + let signedTx = ShelleyTx sbe signedLedgerTx + H.noteShowPretty_ signedTx - expectTxSubmissionSuccess =<< submitTx sbe connectionInfo tx + expectTxSubmissionSuccess =<< submitTx sbe connectionInfo signedTx -- wait till transaction gets included in the block txUtxo <- retryUntilM epochStateView (WaitForBlocks 5) @@ -196,33 +218,42 @@ hprop_tx_refin_datum = integrationRetryWorkspace 2 "api-tx-refin-dat" $ \tempAbs -- manually balance txOutValue = lovelaceToTxOutValue sbe (100_000_000 - txFee) txOut = TxOut addr0 txOutValue txDatum ReferenceScriptNone - -- add actual datum values for the two reference inputs - txInsReference = TxInsReference beo [txIn1, txIn3] $ pure [scriptData1, scriptData3] + -- add actual datum values for the two reference inputs via supplemental datums + ledgerPparams = unLedgerProtocolParameters pparams + supplementalDatums = Exp.obtainCommonConstraints era $ M.fromList + [ let ledgerData = toAlonzoData @(ShelleyLedgerEra ConwayEra) sd + in (hashData ledgerData, ledgerData) + | sd <- [scriptData1, scriptData3, scriptData4] + ] let content = - defaultTxBodyContent sbe - & setTxIns [(txIn2, pure $ KeyWitness KeyWitnessForSpending)] - & setTxInsReference txInsReference - & setTxFee (TxFeeExplicit sbe txFee) - & setTxOuts [txOut] - & setTxProtocolParams (pure $ pure pparams) - - txBody@(ShelleyTxBody _ lbody _ (TxBodyScriptData _ (L.TxDats datums) _) _ _) <- - H.leftFail $ createTransactionBody sbe content - - let bodyScriptData = fromList . map fromAlonzoData $ M.elems datums :: Set HashableScriptData + Exp.defaultTxBodyContent + & Exp.setTxIns [(txIn2, Exp.AnyKeyWitnessPlaceholder)] + & Exp.setTxInsReference (Exp.TxInsReference [txIn1, txIn3] Set.empty) + & Exp.setTxFee txFee + & Exp.setTxOuts [Exp.TxOut . toShelleyTxOut sbe $ toCtxUTxOTxOut txOut] + & Exp.setTxProtocolParams ledgerPparams + & Exp.setTxSupplementalDatums supplementalDatums + + unsignedTx@(Exp.UnsignedTx ledgerTx) <- + H.leftFail $ Exp.makeUnsignedTx era content + + let lbody = ledgerTx ^. L.bodyTxL + L.TxDats datums = ledgerTx ^. L.witsTxL . L.datsTxWitsL + bodyScriptData = fromList . map fromAlonzoData $ M.elems datums :: Set HashableScriptData -- only hashes (1 & 3) and supplemental (4) are present here [scriptData1, scriptData3, scriptData4] === bodyScriptData - H.noteShowPretty_ txBody + H.noteShowPretty_ lbody -- make sure that the script integrity hash was calculated lbody ^. L.scriptIntegrityHashTxBodyL /== L.SNothing - let tx = signShelleyTransaction sbe txBody [wit1] - txId <- H.noteShow . getTxId $ getTxBody tx + let keyWit = Exp.makeKeyWitness era unsignedTx wit1 + Exp.SignedTx signedLedgerTx = Exp.signTx era [] [keyWit] unsignedTx + txId <- H.noteShow . Exp.obtainCommonConstraints era . TxId $ Exp.hashTxBody (signedLedgerTx ^. L.bodyTxL) - expectTxSubmissionSuccess =<< submitTx sbe connectionInfo tx + expectTxSubmissionSuccess =<< submitTx sbe connectionInfo (ShelleyTx sbe signedLedgerTx) -- wait till transaction gets included in the block txUtxo <- retryUntilM epochStateView (WaitForBlocks 5) @@ -244,34 +275,43 @@ hprop_tx_refin_datum = integrationRetryWorkspace 2 "api-tx-refin-dat" $ \tempAbs txOutValue = lovelaceToTxOutValue sbe (99_999_500 - txFee) txOut = TxOut addr0 txOutValue TxOutDatumNone ReferenceScriptNone -- add one reference input with datum hash and its datum, and one superfluous datum - txInsReference = TxInsReference beo [txIn1] $ pure [scriptData1, scriptData3] + ledgerPparams3 = unLedgerProtocolParameters pparams + supplementalDatums3 = Exp.obtainCommonConstraints era $ M.fromList + [ let ledgerData = toAlonzoData @(ShelleyLedgerEra ConwayEra) sd + in (hashData ledgerData, ledgerData) + | sd <- [scriptData1, scriptData3] + ] let content = - defaultTxBodyContent sbe - & setTxIns [(tx2In1, pure $ KeyWitness KeyWitnessForSpending)] - & setTxInsReference txInsReference - & setTxFee (TxFeeExplicit sbe txFee) - & setTxOuts [txOut] - & setTxProtocolParams (pure $ pure pparams) - - txBody@(ShelleyTxBody _ lbody _ (TxBodyScriptData _ (L.TxDats datums) _) _ _) <- - H.leftFail $ createTransactionBody sbe content - - let bodyScriptData = fromList . map fromAlonzoData $ M.elems datums :: Set HashableScriptData + Exp.defaultTxBodyContent + & Exp.setTxIns [(tx2In1, Exp.AnyKeyWitnessPlaceholder)] + & Exp.setTxInsReference (Exp.TxInsReference [txIn1] Set.empty) + & Exp.setTxFee txFee + & Exp.setTxOuts [Exp.TxOut . toShelleyTxOut sbe $ toCtxUTxOTxOut txOut] + & Exp.setTxProtocolParams ledgerPparams3 + & Exp.setTxSupplementalDatums supplementalDatums3 + + unsignedTx3@(Exp.UnsignedTx ledgerTx3) <- + H.leftFail $ Exp.makeUnsignedTx era content + + let lbody = ledgerTx3 ^. L.bodyTxL + L.TxDats datums = ledgerTx3 ^. L.witsTxL . L.datsTxWitsL + bodyScriptData = fromList . map fromAlonzoData $ M.elems datums :: Set HashableScriptData -- all hashes of datums supplied to reference inputs (1 & 3) are present here [scriptData1, scriptData3] === bodyScriptData - H.noteShowPretty_ txBody + H.noteShowPretty_ lbody -- make sure that the script integrity hash was calculated lbody ^. L.scriptIntegrityHashTxBodyL /== L.SNothing - let tx = signShelleyTransaction sbe txBody [wit0] - -- H.noteShowPretty_ tx - H.noteShow_ . getTxId $ getTxBody tx + let keyWit = Exp.makeKeyWitness era unsignedTx3 wit0 + Exp.SignedTx signedLedgerTx = Exp.signTx era [] [keyWit] unsignedTx3 + Exp.obtainCommonConstraints era $ + H.noteShow_ . TxId $ Exp.hashTxBody (signedLedgerTx ^. L.bodyTxL) -- transaction contains not allowed supplemental datum, submission has to fail - submitTx sbe connectionInfo tx >>= \case + submitTx sbe connectionInfo (ShelleyTx sbe signedLedgerTx) >>= \case Right () -> do H.note_ "Transaction submission succeeded, but it should fail!" H.failure From 63dcf3fa6840e0843fd3683eb7284f88d1cb37d7 Mon Sep 17 00:00:00 2001 From: Mateusz Galazyn Date: Thu, 28 May 2026 12:06:34 +0200 Subject: [PATCH 07/40] cardano-testnet | Fix golden tests --- .../test/cardano-testnet-golden/files/golden/version_cmd.cli | 4 ++-- .../cardano-testnet-test/files/calculatePlutusScriptCost.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/version_cmd.cli b/cardano-testnet/test/cardano-testnet-golden/files/golden/version_cmd.cli index 47df628e4e8..f701358c870 100644 --- a/cardano-testnet/test/cardano-testnet-golden/files/golden/version_cmd.cli +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/version_cmd.cli @@ -1,2 +1,2 @@ -built against cardano-api 11.0.0.0 -built against cardano-cli 11.0.0.0 +built against cardano-api 11.3.0.0 +built against cardano-cli 11.1.0.0 diff --git a/cardano-testnet/test/cardano-testnet-test/files/calculatePlutusScriptCost.json b/cardano-testnet/test/cardano-testnet-test/files/calculatePlutusScriptCost.json index 98a7e4de9bd..b2d1ef63f2c 100644 --- a/cardano-testnet/test/cardano-testnet-test/files/calculatePlutusScriptCost.json +++ b/cardano-testnet/test/cardano-testnet-test/files/calculatePlutusScriptCost.json @@ -7,4 +7,4 @@ "lovelaceCost": 34, "scriptHash": "186e32faa80a26810392fda6d559c7ed4721a65ce1c9d4ef3e1c87b4" } -] \ No newline at end of file +] From 9b7a065b69c80d625a064fa803400d2cdecaf96a Mon Sep 17 00:00:00 2001 From: Mateusz Galazyn Date: Thu, 28 May 2026 09:38:29 +0200 Subject: [PATCH 08/40] Update tx-generator to not use deprecated cardano-api api --- .../Benchmarking/GeneratorTx/SizedMetadata.hs | 38 ++++++++------ .../GeneratorTx/SubmissionClient.hs | 16 +++--- .../src/Cardano/Benchmarking/Script/Core.hs | 11 ++-- .../src/Cardano/TxGenerator/Genesis.hs | 41 ++++++++++----- .../src/Cardano/TxGenerator/Tx.hs | 52 ++++++++++++------- bench/tx-generator/tx-generator.cabal | 11 +--- 6 files changed, 103 insertions(+), 66 deletions(-) diff --git a/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/SizedMetadata.hs b/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/SizedMetadata.hs index 768081fcaf0..eb6610306b4 100644 --- a/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/SizedMetadata.hs +++ b/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/SizedMetadata.hs @@ -3,11 +3,14 @@ {-# LANGUAGE GADTs #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} + module Cardano.Benchmarking.GeneratorTx.SizedMetadata where import Cardano.Api +import Cardano.Ledger.BaseTypes (maybeToStrictMaybe) +import qualified Cardano.Ledger.Core as L import Cardano.TxGenerator.Utils import Prelude @@ -16,6 +19,7 @@ import qualified Data.ByteString as BS import Data.Function ((&)) import qualified Data.Map.Strict as Map import Data.Word (Word64) +import Lens.Micro ((.~), (^.)) maxMapSize :: Int @@ -53,7 +57,7 @@ prop_mapCostsMary = measureMapCosts AsMaryEra == assumeMapCosts AsMaryE prop_mapCostsAlonzo = measureMapCosts AsAlonzoEra == assumeMapCosts AsAlonzoEra prop_mapCostsBabbage = measureMapCosts AsBabbageEra == assumeMapCosts AsBabbageEra prop_mapCostsConway = measureMapCosts AsConwayEra == assumeMapCosts AsConwayEra -prop_mapCostsDijkstra = measureMapCosts AsDijkstraEra == assumeMapCosts AsDijkstraEra +prop_mapCostsDijkstra = measureMapCosts AsDijkstraEra == assumeMapCosts AsDijkstraEra assumeMapCosts :: forall era . IsShelleyBasedEra era => AsType era -> [Int] assumeMapCosts _proxy = stepFunction [ @@ -113,21 +117,25 @@ measureBSCosts era = map (metadataSize era . Just . bsMetadata) [0..maxBSSize] metadataSize :: forall era . IsShelleyBasedEra era => AsType era -> Maybe TxMetadata -> Int metadataSize p m = dummyTxSize p m - dummyTxSize p Nothing -dummyTxSizeInEra :: IsShelleyBasedEra era => TxMetadataInEra era -> Int -dummyTxSizeInEra metadata = case createTransactionBody shelleyBasedEra dummyTx of - Right b -> BS.length $ serialiseToCBOR b - Left err -> error $ "metaDataSize " ++ show err +dummyTxSizeInEra :: forall era. IsShelleyBasedEra era => TxMetadataInEra era -> Int +dummyTxSizeInEra metadata = + BS.length $ serialiseToCBOR dummyTx where - dummyTx = defaultTxBodyContent shelleyBasedEra - & setTxIns - [ ( mkTxIn "dbaff4e270cfb55612d9e2ac4658a27c79da4a5271c6f90853042d1403733810#0" - , BuildTxWith $ KeyWitness KeyWitnessForSpending - ) - ] - & setTxFee (mkTxFee 0) - & setTxValidityLowerBound TxValidityNoLowerBound - & setTxValidityUpperBound (mkTxValidityUpperBound 0) - & setTxMetadata metadata + sbe = shelleyBasedEra @era + txInputs = + [ ( mkTxIn "dbaff4e270cfb55612d9e2ac4658a27c79da4a5271c6f90853042d1403733810#0" + , BuildTxWith $ KeyWitness KeyWitnessForSpending + ) + ] + txAuxData = toAuxiliaryData sbe metadata TxAuxScriptsNone + ledgerTxBody = + mkCommonTxBody sbe txInputs [] (mkTxFee 0) TxWithdrawalsNone txAuxData + & invalidHereAfterTxBodyL sbe .~ convValidityUpperBound sbe (mkTxValidityUpperBound 0) + dummyTx :: Tx era + dummyTx = shelleyBasedEraConstraints sbe $ + ShelleyTx sbe $ + L.mkBasicTx (ledgerTxBody ^. txBodyL) + & L.auxDataTxL .~ maybeToStrictMaybe txAuxData dummyTxSize :: forall era . IsShelleyBasedEra era => AsType era -> Maybe TxMetadata -> Int dummyTxSize _p m = (dummyTxSizeInEra @era) $ metadataInEra m diff --git a/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/SubmissionClient.hs b/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/SubmissionClient.hs index 312466573ad..824e648eb20 100644 --- a/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/SubmissionClient.hs +++ b/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/SubmissionClient.hs @@ -101,7 +101,7 @@ txSubmissionClient tr bmtr initialTxSource endOfProtocolCallback = fail (T.unpack err) let (stillUnacked, acked) = L.splitAtEnd ack unAcked let newStats = stats { stsAcked = stsAcked stats + Ack ack } - traceWith bmtr $ SubmissionClientDiscardAcknowledged (getTxId . getTxBody <$> acked) + traceWith bmtr $ SubmissionClientDiscardAcknowledged (txIdFromTx <$> acked) return (txSource, UnAcked stillUnacked, newStats) queueNewTxs :: [Tx era] -> LocalState era -> LocalState era @@ -130,8 +130,8 @@ txSubmissionClient tr bmtr initialTxSource endOfProtocolCallback = let stateC@(_, UnAcked outs , stats) = queueNewTxs newTxs stateB traceWith tr $ idListTrace (ToAnnce newTxs) blocking - traceWith bmtr $ SubmissionClientReplyTxIds (getTxId . getTxBody <$> newTxs) - traceWith bmtr $ SubmissionClientUnAcked (getTxId . getTxBody <$> outs) + traceWith bmtr $ SubmissionClientReplyTxIds (txIdFromTx <$> newTxs) + traceWith bmtr $ SubmissionClientUnAcked (txIdFromTx <$> outs) case blocking of SingBlocking -> case NE.nonEmpty newTxs of @@ -155,12 +155,12 @@ txSubmissionClient tr bmtr initialTxSource endOfProtocolCallback = reqTxIds = fmap fromGenTxId txIds traceWith tr $ ReqTxs (length reqTxIds) let UnAcked ua = unAcked - uaIds = getTxId . getTxBody <$> ua - (toSend, _retained) = L.partition ((`L.elem` reqTxIds) . getTxId . getTxBody) ua + uaIds = txIdFromTx <$> ua + (toSend, _retained) = L.partition ((`L.elem` reqTxIds) . txIdFromTx) ua missIds = reqTxIds L.\\ uaIds traceWith tr $ TxList (length toSend) - traceWith bmtr $ SubmissionClientUnAcked (getTxId . getTxBody <$> ua) + traceWith bmtr $ SubmissionClientUnAcked (txIdFromTx <$> ua) traceWith bmtr $ TraceBenchTxSubServReq reqTxIds unless (L.null missIds) $ traceWith bmtr $ TraceBenchTxSubServUnav missIds @@ -190,6 +190,10 @@ txSubmissionClient tr bmtr initialTxSource endOfProtocolCallback = fromGenTxId (Block.GenTxIdConway (Mempool.ShelleyTxId i)) = fromShelleyTxId i fromGenTxId _ = error "TODO: fix incomplete match" + txIdFromTx :: Tx era -> TxId + txIdFromTx (ShelleyTx sbe tx) = + shelleyBasedEraConstraints sbe $ fromShelleyTxId $ Ledger.txIdTxBody (tx ^. Ledger.bodyTxL) + tokIsBlocking :: SingBlockingStyle a -> Bool tokIsBlocking = \case SingBlocking -> True diff --git a/bench/tx-generator/src/Cardano/Benchmarking/Script/Core.hs b/bench/tx-generator/src/Cardano/Benchmarking/Script/Core.hs index 1b345952511..b13774896b4 100644 --- a/bench/tx-generator/src/Cardano/Benchmarking/Script/Core.hs +++ b/bench/tx-generator/src/Cardano/Benchmarking/Script/Core.hs @@ -38,6 +38,7 @@ import Cardano.Benchmarking.Version as Version import Cardano.Benchmarking.Wallet as Wallet import qualified Cardano.Ledger.Coin as L import qualified Cardano.Ledger.Core as Ledger +import Cardano.Ledger.Tools (estimateMinFeeTx) import Cardano.Logging hiding (LocalSocket) import Cardano.TxGenerator.Fund as Fund import qualified Cardano.TxGenerator.FundQueue as FundQueue @@ -353,10 +354,12 @@ evalGenerator generator txParams@TxGenTxParams{txParamFee = fee} era = do Right tx -> do let txSize = txSizeInBytes tx - txFeeEstimate = case toLedgerPParams shelleyBasedEra protocolParameters of - Left{} -> Nothing - Right ledgerPParams -> Just $ - evaluateTransactionFee shelleyBasedEra ledgerPParams (getTxBody tx) (fromIntegral $ inputs + 1) 0 0 -- 1 key witness per tx input + 1 collateral + txFeeEstimate = case tx of + ShelleyTx sbe ledgerTx -> shelleyBasedEraConstraints sbe $ + case toLedgerPParams sbe protocolParameters of + Left{} -> Nothing + Right ledgerPParams -> Just $ + estimateMinFeeTx ledgerPParams ledgerTx (inputs + 1) 0 0 -- 1 key witness per tx input + 1 collateral traceDebug $ "Projected Tx size in bytes: " ++ show txSize traceDebug $ "Projected Tx fee in Coin: " ++ show txFeeEstimate -- TODO: possibly emit a warning when (Just txFeeEstimate) is lower than specified by config in TxGenTxParams.txFee diff --git a/bench/tx-generator/src/Cardano/TxGenerator/Genesis.hs b/bench/tx-generator/src/Cardano/TxGenerator/Genesis.hs index 6ab5e091806..c35bac56c7d 100644 --- a/bench/tx-generator/src/Cardano/TxGenerator/Genesis.hs +++ b/bench/tx-generator/src/Cardano/TxGenerator/Genesis.hs @@ -4,6 +4,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} + {- HLINT ignore "Use map with tuple-section" -} -- | This module provides means to secure funds that are given in genesis. @@ -21,16 +22,20 @@ where import Cardano.Api hiding (ShelleyGenesis) import qualified Cardano.Ledger.Coin as L +import qualified Cardano.Ledger.Core as Ledger +import Cardano.Ledger.Keys.WitVKey (WitVKey (WitVKey)) import Cardano.Ledger.Shelley.API (Addr (..)) import Cardano.TxGenerator.Fund import Cardano.TxGenerator.Types import Cardano.TxGenerator.Utils import Ouroboros.Consensus.Shelley.Node (validateGenesis) -import Data.Bifunctor (bimap, second) +import Data.Bifunctor (second) import Data.Function ((&)) import Data.List (find) import qualified Data.ListMap as ListMap (toList) +import qualified Data.Set as Set +import Lens.Micro ((.~), (^.)) genesisValidate :: ShelleyGenesis -> Either String () @@ -105,12 +110,16 @@ genesisExpenditure networkId inputKey addr value fee ttl outputKey pseudoTxIn = genesisTxInput networkId inputKey fund tx = FundInEra { - _fundTxIn = TxIn (getTxId $ getTxBody tx) (TxIx 0) + _fundTxIn = TxIn (txIdFromTx tx) (TxIx 0) , _fundWitness = KeyWitness KeyWitnessForSpending , _fundVal = value , _fundSigningKey = Just outputKey } + txIdFromTx :: Tx era -> TxId + txIdFromTx (ShelleyTx sbe' tx') = + shelleyBasedEraConstraints sbe' $ fromShelleyTxId $ Ledger.txIdTxBody (tx' ^. Ledger.bodyTxL) + mkGenesisTransaction :: forall era . IsShelleyBasedEra era => SigningKey GenesisUTxOKey @@ -119,18 +128,24 @@ mkGenesisTransaction :: forall era . -> [TxIn] -> [TxOut CtxTx era] -> Either TxGenError (Tx era) -mkGenesisTransaction key ttl fee txins txouts - = bimap - ApiError - (\b -> signShelleyTransaction (shelleyBasedEra @era) b [WitnessGenesisUTxOKey key]) - (createTransactionBody (shelleyBasedEra @era) txBodyContent) +mkGenesisTransaction key ttl fee txins txouts = + shelleyBasedEraConstraints sbe $ + let txInputs = zip txins $ repeat $ BuildTxWith $ KeyWitness KeyWitnessForSpending + ledgerTxBody = + mkCommonTxBody sbe txInputs txouts (mkTxFee fee) TxWithdrawalsNone Nothing + & invalidHereAfterTxBodyL sbe .~ convValidityUpperBound sbe (mkTxValidityUpperBound ttl) + rawBody = ledgerTxBody ^. txBodyL + unsignedLedgerTx = Ledger.mkBasicTx rawBody + txHash = Ledger.extractHash $ Ledger.hashAnnotated rawBody + shelleySigningKey = toShelleySigningKey (WitnessGenesisUTxOKey key) + witVKey = WitVKey + (getShelleyKeyWitnessVerificationKey shelleySigningKey) + (makeShelleySignature txHash shelleySigningKey) + signedLedgerTx = unsignedLedgerTx + & Ledger.witsTxL .~ (Ledger.mkBasicTxWits & Ledger.addrTxWitsL .~ Set.singleton witVKey) + in Right $ ShelleyTx sbe signedLedgerTx where - txBodyContent = defaultTxBodyContent shelleyBasedEra - & setTxIns (zip txins $ repeat $ BuildTxWith $ KeyWitness KeyWitnessForSpending) - & setTxOuts txouts - & setTxFee (mkTxFee fee) - & setTxValidityLowerBound TxValidityNoLowerBound - & setTxValidityUpperBound (mkTxValidityUpperBound ttl) + sbe = shelleyBasedEra @era castKey :: SigningKey PaymentKey -> SigningKey GenesisUTxOKey castKey (PaymentSigningKey skey) = GenesisUTxOSigningKey skey diff --git a/bench/tx-generator/src/Cardano/TxGenerator/Tx.hs b/bench/tx-generator/src/Cardano/TxGenerator/Tx.hs index d1431f4ae9d..86eda440cf8 100644 --- a/bench/tx-generator/src/Cardano/TxGenerator/Tx.hs +++ b/bench/tx-generator/src/Cardano/TxGenerator/Tx.hs @@ -1,7 +1,7 @@ {-# LANGUAGE GADTs #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} -{-# LANGUAGE TypeApplications #-} + module Cardano.TxGenerator.Tx (module Cardano.TxGenerator.Tx) @@ -9,15 +9,20 @@ module Cardano.TxGenerator.Tx import Cardano.Api hiding (txId) +import Cardano.Ledger.BaseTypes (maybeToStrictMaybe) import qualified Cardano.Ledger.Coin as L +import qualified Cardano.Ledger.Core as Ledger +import Cardano.Ledger.Keys.WitVKey (WitVKey (WitVKey)) import Cardano.TxGenerator.Fund import Cardano.TxGenerator.Types import Cardano.TxGenerator.UTxO (ToUTxOList) -import Data.Bifunctor (bimap, second) +import Data.Bifunctor (second) import qualified Data.ByteString as BS (length) import Data.Function ((&)) import Data.Maybe (mapMaybe) +import qualified Data.Set as Set +import Lens.Micro ((.~), (^.)) -- | 'CreateAndStore' is meant to represent building a transaction @@ -165,22 +170,33 @@ genTx :: forall era. () -> TxFee era -> TxMetadataInEra era -> TxGenerator era -genTx sbe ledgerParameters (collateral, collFunds) fee metadata inFunds outputs - = bimap - ApiError - (\b -> (signShelleyTransaction (shelleyBasedEra @era) b $ map WitnessPaymentKey allKeys, getTxId b)) - (createTransactionBody (shelleyBasedEra @era) txBodyContent) - where - allKeys = mapMaybe getFundKey $ inFunds ++ collFunds - txBodyContent = defaultTxBodyContent sbe - & setTxIns (map (\f -> (getFundTxIn f, BuildTxWith $ getFundWitness f)) inFunds) - & setTxInsCollateral collateral - & setTxOuts outputs - & setTxFee fee - & setTxValidityLowerBound TxValidityNoLowerBound - & setTxValidityUpperBound (defaultTxValidityUpperBound sbe) - & setTxMetadata metadata - & setTxProtocolParams (BuildTxWith (Just ledgerParameters)) +genTx sbe _ledgerParameters (collateral, collFunds) fee metadata inFunds outputs = + shelleyBasedEraConstraints sbe $ do + let allKeys = mapMaybe getFundKey $ inFunds ++ collFunds + setCollateral = case collateral of + TxInsCollateralNone -> id + TxInsCollateral eon _ -> collateralInputsTxBodyL eon .~ convCollateralTxIns collateral + txInputs = map (\f -> (getFundTxIn f, BuildTxWith $ getFundWitness f)) inFunds + txAuxData = toAuxiliaryData sbe metadata TxAuxScriptsNone + ledgerTxBody = + mkCommonTxBody sbe txInputs outputs fee TxWithdrawalsNone txAuxData + & invalidHereAfterTxBodyL sbe .~ convValidityUpperBound sbe (defaultTxValidityUpperBound sbe) + & setCollateral + rawBody = ledgerTxBody ^. txBodyL + unsignedLedgerTx = Ledger.mkBasicTx rawBody + txHash = Ledger.extractHash $ Ledger.hashAnnotated rawBody + witVKeys = Set.fromList + [ WitVKey + (getShelleyKeyWitnessVerificationKey sk) + (makeShelleySignature txHash sk) + | sk <- map (toShelleySigningKey . WitnessPaymentKey) allKeys + ] + signedLedgerTx = unsignedLedgerTx + & Ledger.witsTxL .~ (Ledger.mkBasicTxWits & Ledger.addrTxWitsL .~ witVKeys) + & Ledger.auxDataTxL .~ maybeToStrictMaybe txAuxData + tx = ShelleyTx sbe signedLedgerTx + txId = fromShelleyTxId $ Ledger.txIdTxBody rawBody + Right (tx, txId) txSizeInBytes :: forall era. IsShelleyBasedEra era => diff --git a/bench/tx-generator/tx-generator.cabal b/bench/tx-generator/tx-generator.cabal index 5ebadb43f08..3345b5d4fc1 100644 --- a/bench/tx-generator/tx-generator.cabal +++ b/bench/tx-generator/tx-generator.cabal @@ -113,28 +113,20 @@ library , cardano-binary , cardano-cli ^>= 11.1 , cardano-crypto-class - , cardano-crypto-wrapper , cardano-data , cardano-diffusion ^>= 1.0 , cardano-git-rev ^>= 0.2.2 - , cardano-ledger-alonzo , cardano-ledger-api - , cardano-ledger-byron , cardano-ledger-core , cardano-node , cardano-prelude - , cardano-strict-containers >=0.1 , contra-tracer , cborg >= 0.2.2 && < 0.3 , containers - , constraints-extras , directory , dlist , extra , filepath - , formatting - , generic-monoid - , ghc-prim , io-classes:{io-classes, strict-stm} , microlens , mtl @@ -158,7 +150,6 @@ library , trace-forward , transformers , transformers-except - , unordered-containers , yaml -- Needed by "Cardano.Api.Internal.ProtocolParameters" port. , either @@ -195,12 +186,12 @@ executable calibrate-script , aeson , aeson-pretty , bytestring + , cardano-api , containers , directory , extra , filepath , optparse-applicative - , cardano-api , text , transformers , transformers-except From 6a6b6cc52de7609565e6ea2b02b754210e110e9a Mon Sep 17 00:00:00 2001 From: Mateusz Galazyn Date: Thu, 28 May 2026 12:50:59 +0200 Subject: [PATCH 09/40] Limit devshells in github actions to only required packages --- .github/workflows/actionlint.yml | 2 ++ .github/workflows/check-changelog.yml | 8 +++++--- .github/workflows/shellcheck.yml | 2 ++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index 1970fb42d6a..d9026e86d66 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -23,6 +23,8 @@ jobs: extra-substituters = https://cache.iog.io/ # Make the Nix environment available to next steps - uses: rrbutani/use-nix-shell-action@f97339023a09121113e5a58ad88fe0e9fde3406b # v1 + with: + flakes: nixpkgs#shellcheck,nixpkgs#actionlint - name: actionlint run: | diff --git a/.github/workflows/check-changelog.yml b/.github/workflows/check-changelog.yml index c2b31c444ca..d30b20536c5 100644 --- a/.github/workflows/check-changelog.yml +++ b/.github/workflows/check-changelog.yml @@ -42,9 +42,11 @@ jobs: extra-trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= extra-substituters = https://cache.iog.io/ - - name: Check scriv fragments are correct + - uses: rrbutani/use-nix-shell-action@v1 if: steps.filter.outputs.cardano == 'true' - uses: rrbutani/use-nix-shell-action@f97339023a09121113e5a58ad88fe0e9fde3406b # v1 with: - script: cd cardano-testnet && scriv collect --version "CI-CHECK" --keep + flakes: nixpkgs#scriv + - name: Check scriv fragments are correct + if: steps.filter.outputs.cardano == 'true' + run: cd cardano-testnet && scriv collect --version "CI-CHECK" --keep diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index bdac554824b..911e274412c 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -25,6 +25,8 @@ jobs: extra-substituters = https://cache.iog.io/ # Make the Nix environment available to next steps - uses: rrbutani/use-nix-shell-action@f97339023a09121113e5a58ad88fe0e9fde3406b # v1 + with: + flakes: nixpkgs#shellcheck - name: shellcheck run: | for file in $(git ls-files "*.sh") From 96b63d60a60873b195541fa67614a7c1c0c5b408 Mon Sep 17 00:00:00 2001 From: Mateusz Galazyn Date: Thu, 28 May 2026 13:15:45 +0200 Subject: [PATCH 10/40] Disable haddock for cardano-api, because of tyConStupidTheta error on GHC 9.6.7 --- nix/haskell.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nix/haskell.nix b/nix/haskell.nix index eb2dbf88ce3..d086e60638f 100644 --- a/nix/haskell.nix +++ b/nix/haskell.nix @@ -137,6 +137,8 @@ let package-keys = ["plutus-tx-plugin"]; packages.plutus-tx-plugin.components.library.platforms = with lib.platforms; [ linux darwin ]; + # GHC 9.6.7 haddock panics on TopTx type family (tyConStupidTheta) + packages.cardano-api.components.library.doHaddock = false; packages.fs-api.components.library.doHaddock = false; packages.cardano-ledger-allegra.components.library.doHaddock = false; packages.cardano-ledger-alonzo.components.library.doHaddock = false; From a8f73c27ec551a3c9593080b9cb2ccc9b3b5279f Mon Sep 17 00:00:00 2001 From: Adithya Kumar Date: Sun, 17 May 2026 13:34:10 +0530 Subject: [PATCH 11/40] Add the required instances and fix compilation (cherry picked from commit 17ff148370c7a8530c0b53b76444876a7e590745) --- cabal.project | 10 ++ .../Cardano/Node/Configuration/LedgerDB.hs | 13 +- .../src/Cardano/Node/Configuration/POM.hs | 47 +++-- cardano-node/src/Cardano/Node/Run.hs | 6 +- .../src/Cardano/Node/Tracing/Tracers.hs | 19 ++ .../Cardano/Node/Tracing/Tracers/ChainDB.hs | 165 ++++++++++-------- .../Cardano/Node/Tracing/Tracers/Consensus.hs | 132 ++++++++++++++ .../Node/Tracing/Tracers/LedgerMetrics.hs | 4 +- .../Node/Tracing/Tracers/NodeToClient.hs | 71 ++++++++ .../Cardano/Node/Tracing/Tracers/Startup.hs | 1 + 10 files changed, 373 insertions(+), 95 deletions(-) diff --git a/cabal.project b/cabal.project index ccbde969df7..27c27df50ec 100644 --- a/cabal.project +++ b/cabal.project @@ -50,6 +50,16 @@ packages: bench/trace-schemas/scripts/schema-gen trace-resources trace-forward + ouroboros-consensus + ouroboros-network/ouroboros-network + ouroboros-network/cardano-diffusion + ouroboros-network/cardano-ping + ouroboros-network/monoidal-synchronisation + ouroboros-network/network-mux + ekg-forward + hermod-tracing/trace-dispatcher + cardano-api/cardano-api + cardano-api/cardano-rpc -- Needed when cross compiling extra-packages: alex, dmq-node >= 0.4.2.0 diff --git a/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs b/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs index 2c60b7e9d87..16bc69e893e 100644 --- a/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs +++ b/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs @@ -31,6 +31,8 @@ import Data.Proxy import System.FilePath import System.Random (StdGen) +import Ouroboros.Consensus.Ledger.Basics (LedgerState) + -- | Choose the LedgerDB Backend -- -- As of UTxO-HD, the LedgerDB now uses either an in-memory backend or LMDB to @@ -73,8 +75,7 @@ noDeprecatedOptions = DeprecatedOptions [] data LedgerDbConfiguration = LedgerDbConfiguration - NumOfDiskSnapshots - SnapshotInterval + SnapshotPolicyArgs QueryBatchSize LedgerDbSelectorFlag DeprecatedOptions @@ -139,10 +140,16 @@ defaultLMDBLimits = LMDB.LMDBLimits { defaultLMDBPath :: FilePath -> FilePath defaultLMDBPath = ( "lmdb") -selectorToArgs :: forall blk. (LedgerSupportsProtocol blk, LedgerSupportsLedgerDB blk) => LedgerDbSelectorFlag -> FilePath -> StdGen -> (LedgerDbBackendArgs IO blk, StdGen) +selectorToArgs :: + forall blk. + ( LedgerSupportsProtocol blk + , LedgerDbSerialiseConstraints blk + , CanUpgradeLedgerTables LedgerState blk + ) => LedgerDbSelectorFlag -> FilePath -> StdGen -> (LedgerDbBackendArgs IO blk, StdGen) selectorToArgs V2InMemory _ = InMemory.mkInMemoryArgs selectorToArgs (V1LMDB ff fp l mxReaders) fastStoragePath = LMDB.mkLMDBArgs + (Proxy @blk) ff (fromMaybe (defaultLMDBPath fastStoragePath) fp) ( maybe id (\overrideMaxReaders lim -> lim{LMDB.lmdbMaxReaders = overrideMaxReaders}) mxReaders $ diff --git a/cardano-node/src/Cardano/Node/Configuration/POM.hs b/cardano-node/src/Cardano/Node/Configuration/POM.hs index 72a7fc94ff3..19217c0f7e5 100644 --- a/cardano-node/src/Cardano/Node/Configuration/POM.hs +++ b/cardano-node/src/Cardano/Node/Configuration/POM.hs @@ -45,8 +45,10 @@ import Ouroboros.Consensus.Node (NodeDatabasePaths (..)) import Ouroboros.Consensus.Node.Genesis (GenesisConfig, GenesisConfigFlags, defaultGenesisConfigFlags, mkGenesisConfig) import Ouroboros.Consensus.Storage.LedgerDB.Args (QueryBatchSize (..)) -import Ouroboros.Consensus.Storage.LedgerDB.Snapshots (NumOfDiskSnapshots (..), - SnapshotInterval (..)) +import Ouroboros.Consensus.Util.Args (OverrideOrDefault (..)) +import Ouroboros.Consensus.Storage.LedgerDB.Snapshots ( + SnapshotFrequency (..), SnapshotFrequencyArgs (..), SnapshotPolicyArgs (..), + defaultSnapshotPolicyArgs, NumOfDiskSnapshots (..)) import Ouroboros.Consensus.Storage.LedgerDB.V1.Args (FlushFrequency (..)) import Ouroboros.Network.Diffusion.Configuration as Configuration import qualified Ouroboros.Network.Diffusion.Configuration as Ouroboros @@ -74,6 +76,8 @@ import System.Random (randomIO) import Generic.Data (gmappend) import Generic.Data.Orphans () +import Cardano.Ledger.BaseTypes.NonZero (nonZero) + -- | Isomorphic to a `Maybe DiffTime`, but expresses what `Nothing` means, in -- this case that we want to /NOT/ override the default timeout. data TimeoutOverride = NoTimeoutOverride | TimeoutOverride DiffTime @@ -484,8 +488,11 @@ instance FromJSON PartialNodeConfiguration where Nothing -> return Nothing parseLedgerDbConfig v = do - let snapInterval x = fmap (RequestedSnapshotInterval . secondsToDiffTime) <$> x .:? "SnapshotInterval" - snapNum x = fmap RequestedNumOfDiskSnapshots <$> x .:? "NumOfDiskSnapshots" + let snapInterval x = do + si <- x .:? "SnapshotInterval" + when (any (<= 0) si) $ fail $ "Non-positive SnapshotInterval: " <> show si + pure $ Override <$> (si >>= nonZero) + snapNum x = fmap (Override . NumOfDiskSnapshots) <$> x .:? "NumOfDiskSnapshots" mTopLevelSnapInterval <- snapInterval v mTopLevelSnapNum <- snapNum v @@ -499,12 +506,22 @@ instance FromJSON PartialNodeConfiguration where mLedgerDB <- v .:? "LedgerDB" case mLedgerDB of Nothing -> do - let si = fromMaybe DefaultSnapshotInterval mTopLevelSnapInterval - sn = fromMaybe DefaultNumOfDiskSnapshots mTopLevelSnapNum - return $ Just $ LedgerDbConfiguration sn si DefaultQueryBatchSize V2InMemory deprecatedOpts + let si = fromMaybe UseDefault mTopLevelSnapInterval + sn = fromMaybe UseDefault mTopLevelSnapNum + sf = SnapshotFrequencyArgs { + sfaInterval = si + , sfaOffset = UseDefault + , sfaRateLimit = UseDefault + , sfaDelaySnapshotRange = UseDefault + } + spArgs = SnapshotPolicyArgs (SnapshotFrequency sf) sn + return $ Just $ LedgerDbConfiguration spArgs DefaultQueryBatchSize V2InMemory deprecatedOpts + Just ledgerDB -> flip (withObject "LedgerDB") ledgerDB $ \o -> do - ldbSnapInterval <- (getLast . (Last mTopLevelSnapInterval <>) . Last <$> snapInterval o) .!= DefaultSnapshotInterval - ldbSnapNum <- (getLast . (Last mTopLevelSnapNum <>) . Last <$> snapNum o) .!= DefaultNumOfDiskSnapshots + ldbSnapInterval <- (getLast . (Last mTopLevelSnapInterval <>) . Last <$> snapInterval o) .!= UseDefault + ldbSnapNum <- (getLast . (Last mTopLevelSnapNum <>) . Last <$> snapNum o) .!= UseDefault + ldbSnapOffset <- (fmap Override <$> o .:? "SlotOffset") .!= UseDefault + ldbSnapRateLimit<- (fmap (Override . secondsToDiffTime) <$> o .:? "RateLimit") .!= UseDefault qsize <- (fmap RequestedQueryBatchSize <$> o .:? "QueryBatchSize") .!= DefaultQueryBatchSize backend <- o .:? "Backend" .!= "V2InMemory" selector <- case backend of @@ -519,7 +536,14 @@ instance FromJSON PartialNodeConfiguration where lsmPath :: Maybe FilePath <- o .:? "LSMDatabasePath" pure $ V2LSM lsmPath _ -> fail $ "Malformed LedgerDB Backend: " <> backend - pure $ Just $ LedgerDbConfiguration ldbSnapNum ldbSnapInterval qsize selector deprecatedOpts + let sf = SnapshotFrequencyArgs { + sfaInterval = ldbSnapInterval + , sfaOffset = ldbSnapOffset + , sfaRateLimit = ldbSnapRateLimit + , sfaDelaySnapshotRange = UseDefault + } + spArgs = SnapshotPolicyArgs (SnapshotFrequency sf) ldbSnapNum + pure $ Just $ LedgerDbConfiguration spArgs qsize selector deprecatedOpts parseByronProtocol v = do primary <- v .:? "ByronGenesisFile" @@ -683,8 +707,7 @@ defaultPartialNodeConfiguration = , pncLedgerDbConfig = Last $ Just $ LedgerDbConfiguration - DefaultNumOfDiskSnapshots - DefaultSnapshotInterval + defaultSnapshotPolicyArgs DefaultQueryBatchSize V2InMemory noDeprecatedOptions diff --git a/cardano-node/src/Cardano/Node/Run.hs b/cardano-node/src/Cardano/Node/Run.hs index f4f6d43e365..10c77266977 100644 --- a/cardano-node/src/Cardano/Node/Run.hs +++ b/cardano-node/src/Cardano/Node/Run.hs @@ -561,15 +561,11 @@ handleSimpleNode blockType runP tracers nc networkMagic onKernel = do Just version_ -> Map.takeWhileAntitone (<= version_) LedgerDbConfiguration - snapInterval - numSnaps + snapshotPolicyArgs queryBatchSize ldbBackend deprecatedOpts = ncLedgerDbConfig nc - snapshotPolicyArgs :: SnapshotPolicyArgs - snapshotPolicyArgs = SnapshotPolicyArgs numSnaps snapInterval - -------------------------------------------------------------------------------- -- SIGHUP Handlers -------------------------------------------------------------------------------- diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers.hs index cbf985df114..e8f7e14cb79 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers.hs @@ -354,6 +354,13 @@ mkConsensusTracers configReflection trBase trForward mbTrEKG _trDataPoint trConf !txCountersTracer <- mkCardanoTracer trBase trForward mbTrEKG ["txCounters", "Remote"] + + !txPerasCertIn <- mkCardanoTracer trBase trForward mbTrEKG ["Peras", "Cert", "Inbound"] + !txPerasCertOut <- mkCardanoTracer trBase trForward mbTrEKG ["Peras", "Cert", "Outbound"] + !txPerasVoteIn <- mkCardanoTracer trBase trForward mbTrEKG ["Peras", "Vote", "Inbound"] + !txPerasVoteOut <- mkCardanoTracer trBase trForward mbTrEKG ["Peras", "Vote", "Outbound"] + + configureTracers configReflection trConfig [txCountersTracer] pure $ Consensus.Tracers @@ -408,6 +415,10 @@ mkConsensusTracers configReflection trBase trForward mbTrEKG _trDataPoint trConf traceWith txLogicTracer , Consensus.txCountersTracer = Tracer $ traceWith txCountersTracer + , Consensus.perasCertDiffusionInboundTracer = Tracer $ traceWith txPerasCertIn + , Consensus.perasCertDiffusionOutboundTracer = Tracer $ traceWith txPerasCertOut + , Consensus.perasVoteDiffusionInboundTracer = Tracer $ traceWith txPerasVoteIn + , Consensus.perasVoteDiffusionOutboundTracer = Tracer $ traceWith txPerasVoteOut } mkNodeToClientTracers :: forall blk. @@ -505,6 +516,10 @@ mkNodeToNodeTracers configReflection trBase trForward mbTrEKG _trDataPoint trCon !txLogicTracer <- mkCardanoTracer trBase trForward mbTrEKG ["txLogic", "Remote"] + + !txPerasCertDiffusion <- mkCardanoTracer trBase trForward mbTrEKG ["Peras", "Cert", "Inbound"] + !txPerasVoteDiffusion <- mkCardanoTracer trBase trForward mbTrEKG ["Peras", "Vote", "Inbound"] + configureTracers configReflection trConfig [txLogicTracer] pure $ NtN.Tracers @@ -524,6 +539,10 @@ mkNodeToNodeTracers configReflection trBase trForward mbTrEKG _trDataPoint trCon traceWith peerSharingTracer , NtN.tTxLogicTracer = Tracer $ traceWith txLogicTracer + , NtN.tPerasCertDiffusionTracer = Tracer $ + traceWith txPerasCertDiffusion + , NtN.tPerasVoteDiffusionTracer = Tracer $ + traceWith txPerasVoteDiffusion } mkDiffusionTracers :: diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs index 1427af94e67..43be89de859 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs @@ -48,13 +48,14 @@ import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.BackingStore.Impl.LMDB import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.Backend as V2 import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.InMemory as InMemory import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.LSM as LSM -import qualified Ouroboros.Consensus.Storage.PerasCertDB.Impl as PerasCertDB import qualified Ouroboros.Consensus.Storage.VolatileDB as VolDB import Ouroboros.Consensus.TypeFamilyWrappers import Ouroboros.Consensus.Util.Condense (condense) import Ouroboros.Consensus.Util.Enclose import qualified Ouroboros.Network.AnchoredFragment as AF import Ouroboros.Network.Block (MaxSlotNo (..)) +import qualified Ouroboros.Consensus.Storage.PerasVoteDB as PerasVoteDB +import qualified Ouroboros.Consensus.Storage.PerasCertDB as PerasCertDB import Data.Aeson (Object, ToJSON, Value (Object, String), object, toJSON, (.=)) import qualified Data.ByteString.Base16 as B16 @@ -104,6 +105,7 @@ instance ( LogFormatting (Header blk) ) => LogFormatting (ChainDB.TraceEvent blk) where forHuman ChainDB.TraceLastShutdownUnclean = "ChainDB is not clean. Validating all immutable chunks" + forHuman (ChainDB.TracePerasVoteDbEvent v) = forHuman v forHuman (ChainDB.TraceAddBlockEvent v) = forHuman v forHuman (ChainDB.TraceFollowerEvent v) = forHuman v forHuman (ChainDB.TraceCopyToImmutableDBEvent v) = forHuman v @@ -130,6 +132,8 @@ instance ( LogFormatting (Header blk) RisingEdge -> "risingEdge" .= True FallingEdgeWith pt -> "fallingEdge" .= forMachine dtal pt ] + forMachine details (ChainDB.TracePerasVoteDbEvent v) = + forMachine details v forMachine details (ChainDB.TraceAddBlockEvent v) = forMachine details v forMachine details (ChainDB.TraceFollowerEvent v) = @@ -158,6 +162,7 @@ instance ( LogFormatting (Header blk) asMetrics ChainDB.TraceLastShutdownUnclean = [] asMetrics (ChainDB.TraceChainSelStarvationEvent _) = [] + asMetrics (ChainDB.TracePerasVoteDbEvent v) = asMetrics v asMetrics (ChainDB.TraceAddBlockEvent v) = asMetrics v asMetrics (ChainDB.TraceFollowerEvent v) = asMetrics v asMetrics (ChainDB.TraceCopyToImmutableDBEvent v) = asMetrics v @@ -177,6 +182,8 @@ instance MetaTrace (ChainDB.TraceEvent blk) where Namespace [] ["LastShutdownUnclean"] namespaceFor ChainDB.TraceChainSelStarvationEvent{} = Namespace [] ["ChainSelStarvationEvent"] + namespaceFor (ChainDB.TracePerasVoteDbEvent ev) = + nsPrependInner "PerasVoteDbEvent" (namespaceFor ev) namespaceFor (ChainDB.TraceAddBlockEvent ev) = nsPrependInner "AddBlockEvent" (namespaceFor ev) namespaceFor (ChainDB.TraceFollowerEvent ev) = @@ -1642,6 +1649,77 @@ instance MetaTrace (ChainDB.UnknownRange blk) where , Namespace [] ["ForkTooOld"] ] +-- -------------------------------------------------------------------------------- +-- -- Peras +-- -------------------------------------------------------------------------------- + +instance MetaTrace (PerasVoteDB.TraceEvent blk) where + namespaceFor (PerasVoteDB.AddVote {}) = Namespace [] ["AddVote"] + namespaceFor (PerasVoteDB.GarbageCollected {}) = Namespace [] ["GarbageCollected"] + allNamespaces = + [ Namespace [] ["AddVote"] + , Namespace [] ["GarbageCollected"] + ] + + severityFor _ _ = Just Info + privacyFor _ _ = Just Public + detailsFor _ _ = Just DNormal + + documentFor (Namespace _ ["AddVote"]) = Just "AddVote" + documentFor (Namespace _ ["GarbageCollected"]) = Just "GarbageCollected" + documentFor _ = Nothing + +instance LogFormatting (PerasVoteDB.TraceEvent blk) where + forHuman (PerasVoteDB.AddVote {}) = "PerasVoteDB.AddVote" + forHuman (PerasVoteDB.GarbageCollected {}) = "PerasVoteDB.GarbageCollected" + + forMachine _dtal (PerasVoteDB.AddVote cert _ _) = + mconcat + [ "kind" .= String "AddVote" + , "cert" .= String (Text.pack $ show cert) + ] + forMachine _dtal (PerasVoteDB.GarbageCollected slotNo) = + mconcat + [ "kind" .= String "GarbageCollected" + , "slotNo" .= String (Text.pack $ show slotNo) + ] + + asMetrics _ = [] + +instance MetaTrace (PerasCertDB.TraceEvent blk) where + namespaceFor (PerasCertDB.AddCert _ _ _) = Namespace [] ["AddCert"] + namespaceFor (PerasCertDB.GarbageCollected _) = Namespace [] ["GarbageCollected"] + allNamespaces = + [ Namespace [] ["AddCert"] + , Namespace [] ["GarbageCollected"] + ] + + severityFor _ _ = Just Info + privacyFor _ _ = Just Public + detailsFor _ _ = Just DNormal + + documentFor (Namespace _ ["AddCert"]) = Just "AddCert" + documentFor (Namespace _ ["GarbageCollected"]) = Just "GarbageCollected" + documentFor _ = Nothing + +instance LogFormatting (PerasCertDB.TraceEvent blk) where + forHuman (PerasCertDB.AddCert _ _ _) = "PerasCertDB.AddCert" + forHuman (PerasCertDB.GarbageCollected _slotNo) = "PerasCertDB.GarbageCollected" + + forMachine _dtal (PerasCertDB.AddCert cert _ _) = + mconcat + [ "kind" .= String "AddCert" + , "cert" .= String (Text.pack $ show cert) + ] + forMachine _dtal (PerasCertDB.GarbageCollected slotNo) = + mconcat + [ "kind" .= String "GarbageCollected" + , "slotNo" .= String (Text.pack $ show slotNo) + ] + + asMetrics _ = [] + + -- -------------------------------------------------------------------------------- -- -- LedgerDB.TraceEvent -- -------------------------------------------------------------------------------- @@ -1712,6 +1790,8 @@ instance MetaTrace (LedgerDB.TraceEvent blk) where instance ( StandardHash blk , ConvertRawHash blk) => LogFormatting (LedgerDB.TraceSnapshotEvent blk) where + forHuman (LedgerDB.SnapshotRequestDelayed {}) = "LedgerDB.SnapshotRequestDelayed" + forHuman (LedgerDB.SnapshotRequestCompleted) = "LedgerDB.SnapshotRequestCompleted" forHuman (LedgerDB.TookSnapshot snap pt RisingEdge) = Text.unwords [ "Taking ledger snapshot" , showT snap @@ -1750,6 +1830,13 @@ instance ( StandardHash blk " Snapshot was created for a different backend. Convert it with `snapshot-converter`." _ -> "" + -- TODO: Create a proper log for SnapshotRequestDelayed + forMachine _ (LedgerDB.SnapshotRequestDelayed _ _ _) = + mconcat [ "kind" .= String "SnapshotRequestDelayed" + ] + forMachine _ (LedgerDB.SnapshotRequestCompleted) = + mconcat [ "kind" .= String "SnapshotRequestCompleted" + ] forMachine dtals (LedgerDB.TookSnapshot snap pt enclosedTiming) = mconcat [ "kind" .= String "TookSnapshot" , "snapshot" .= forMachine dtals snap @@ -1765,10 +1852,14 @@ instance ( StandardHash blk , "failure" .= show failure ] instance MetaTrace (LedgerDB.TraceSnapshotEvent blk) where + namespaceFor LedgerDB.SnapshotRequestDelayed {} = Namespace [] ["SnapshotRequestDelayed"] + namespaceFor LedgerDB.SnapshotRequestCompleted {} = Namespace [] ["SnapshotRequestCompleted"] namespaceFor LedgerDB.TookSnapshot {} = Namespace [] ["TookSnapshot"] namespaceFor LedgerDB.DeletedSnapshot {} = Namespace [] ["DeletedSnapshot"] namespaceFor LedgerDB.InvalidSnapshot {} = Namespace [] ["InvalidSnapshot"] + severityFor (Namespace _ ["SnapshotRequestDelayed"]) _ = Just Info + severityFor (Namespace _ ["SnapshotRequestCompleted"]) _ = Just Info severityFor (Namespace _ ["TookSnapshot"]) _ = Just Info severityFor (Namespace _ ["DeletedSnapshot"]) _ = Just Debug severityFor (Namespace _ ["InvalidSnapshot"]) _ = Just Error @@ -3088,30 +3179,6 @@ instance (Show (PBFT.PBftVerKeyHash c)) , "numForged" .= numForged ] --- PerasCertDB.TraceEvent instances -instance LogFormatting (PerasCertDB.TraceEvent blk) where - forHuman (PerasCertDB.AddedPerasCert _cert _peer) = "Added Peras certificate to database" - forHuman (PerasCertDB.IgnoredCertAlreadyInDB _cert _peer) = "Ignored Peras certificate already in database" - forHuman PerasCertDB.OpenedPerasCertDB = "Opened Peras certificate database" - forHuman PerasCertDB.ClosedPerasCertDB = "Closed Peras certificate database" - forHuman (PerasCertDB.AddingPerasCert _cert _peer) = "Adding Peras certificate to database" - - forMachine _dtal (PerasCertDB.AddedPerasCert cert _peer) = - mconcat ["kind" .= String "AddedPerasCert", - "cert" .= String (Text.pack $ show cert)] - forMachine _dtal (PerasCertDB.IgnoredCertAlreadyInDB cert _peer) = - mconcat ["kind" .= String "IgnoredCertAlreadyInDB", - "cert" .= String (Text.pack $ show cert)] - forMachine _dtal PerasCertDB.OpenedPerasCertDB = - mconcat ["kind" .= String "OpenedPerasCertDB"] - forMachine _dtal PerasCertDB.ClosedPerasCertDB = - mconcat ["kind" .= String "ClosedPerasCertDB"] - forMachine _dtal (PerasCertDB.AddingPerasCert cert _peer) = - mconcat ["kind" .= String "AddingPerasCert", - "cert" .= String (Text.pack $ show cert)] - - asMetrics _ = [] - -- ChainDB.TraceAddPerasCertEvent instances instance ConvertRawHash blk => LogFormatting (ChainDB.TraceAddPerasCertEvent blk) where forHuman (ChainDB.AddedPerasCertToQueue roundNo boostedBlock _queueSize) = @@ -3168,54 +3235,6 @@ instance ConvertRawHash blk => LogFormatting (ChainDB.TraceAddPerasCertEvent blk asMetrics _ = [] --- PerasCertDB.TraceEvent MetaTrace instance -instance MetaTrace (PerasCertDB.TraceEvent blk) where - namespaceFor (PerasCertDB.AddedPerasCert _ _) = - Namespace [] ["AddedPerasCert"] - namespaceFor (PerasCertDB.IgnoredCertAlreadyInDB _ _) = - Namespace [] ["IgnoredCertAlreadyInDB"] - namespaceFor PerasCertDB.OpenedPerasCertDB = - Namespace [] ["OpenedPerasCertDB"] - namespaceFor PerasCertDB.ClosedPerasCertDB = - Namespace [] ["ClosedPerasCertDB"] - namespaceFor (PerasCertDB.AddingPerasCert _ _) = - Namespace [] ["AddingPerasCert"] - - severityFor (Namespace _ ["AddedPerasCert"]) _ = Just Info - severityFor (Namespace _ ["IgnoredCertAlreadyInDB"]) _ = Just Info - severityFor (Namespace _ ["OpenedPerasCertDB"]) _ = Just Info - severityFor (Namespace _ ["ClosedPerasCertDB"]) _ = Just Info - severityFor (Namespace _ ["AddingPerasCert"]) _ = Just Debug - severityFor _ _ = Nothing - - privacyFor (Namespace _ ["AddedPerasCert"]) _ = Just Public - privacyFor (Namespace _ ["IgnoredCertAlreadyInDB"]) _ = Just Public - privacyFor (Namespace _ ["OpenedPerasCertDB"]) _ = Just Public - privacyFor (Namespace _ ["ClosedPerasCertDB"]) _ = Just Public - privacyFor (Namespace _ ["AddingPerasCert"]) _ = Just Public - privacyFor _ _ = Nothing - - detailsFor (Namespace _ ["AddedPerasCert"]) _ = Just DNormal - detailsFor (Namespace _ ["IgnoredCertAlreadyInDB"]) _ = Just DNormal - detailsFor (Namespace _ ["OpenedPerasCertDB"]) _ = Just DNormal - detailsFor (Namespace _ ["ClosedPerasCertDB"]) _ = Just DNormal - detailsFor (Namespace _ ["AddingPerasCert"]) _ = Just DDetailed - detailsFor _ _ = Nothing - - documentFor (Namespace _ ["AddedPerasCert"]) = Just "Certificate added to Peras certificate database" - documentFor (Namespace _ ["IgnoredCertAlreadyInDB"]) = Just "Certificate ignored as it was already in the database" - documentFor (Namespace _ ["OpenedPerasCertDB"]) = Just "Peras certificate database opened" - documentFor (Namespace _ ["ClosedPerasCertDB"]) = Just "Peras certificate database closed" - documentFor (Namespace _ ["AddingPerasCert"]) = Just "Adding certificate to Peras certificate database" - documentFor _ = Nothing - - allNamespaces = - [Namespace [] ["AddedPerasCert"], - Namespace [] ["IgnoredCertAlreadyInDB"], - Namespace [] ["OpenedPerasCertDB"], - Namespace [] ["ClosedPerasCertDB"], - Namespace [] ["AddingPerasCert"]] - -- ChainDB.TraceAddPerasCertEvent MetaTrace instance instance MetaTrace (ChainDB.TraceAddPerasCertEvent blk) where namespaceFor ChainDB.AddedPerasCertToQueue{} = Namespace [] ["AddedPerasCertToQueue"] diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/Consensus.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/Consensus.hs index 36ae550c0f2..edb2462b0d0 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/Consensus.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/Consensus.hs @@ -84,6 +84,8 @@ import qualified Data.Text as Text import Data.Time (NominalDiffTime) import Data.Word (Word32, Word64) import Network.TypedProtocol.Core +import Ouroboros.Consensus.MiniProtocol.ObjectDiffusion.Inbound (TraceObjectDiffusionInbound (..)) +import Ouroboros.Consensus.MiniProtocol.ObjectDiffusion.Outbound (TraceObjectDiffusionOutbound (..)) enclosingValue :: ToJSON a => Enclosing' a -> Value enclosingValue RisingEdge = object [ "edge" .= String "Starting" ] @@ -1040,6 +1042,7 @@ instance ( HasHeader blk instance MetaTrace SanityCheckIssue where namespaceFor InconsistentSecurityParam {} = Namespace [] ["SanityCheckIssue"] + namespaceFor _ = Namespace [] ["SanityCheckIssue"] severityFor (Namespace _ ["SanityCheckIssue"]) _ = Just Error severityFor _ _ = Nothing @@ -1054,8 +1057,12 @@ instance LogFormatting SanityCheckIssue where mconcat [ "kind" .= String "InconsistentSecurityParam" , "error" .= String (Text.pack $ show e) ] + forMachine _ _ = + mconcat [ "kind" .= String "SnapshotIssue" + ] forHuman (InconsistentSecurityParam e) = "Configuration contains multiple security parameters: " <> Text.pack (show e) + forHuman _ = "SnapshotIssue" -------------------------------------------------------------------------------- -- TxSubmissionServer Tracer @@ -2308,3 +2315,128 @@ instance MetaTrace KESAgentClientTrace where allNamespaces = Namespace [] ["KESAgentClientException"] : fmap nsCast (allNamespaces :: [Namespace Agent.ServiceClientTrace]) + +-------------------------------------------------------------------------------- +-- Peras +-------------------------------------------------------------------------------- + +-- TODO: Move this to a proper place. A lot of this is duplicated in the +-- ToObject instance. This is likely in an incorrect place. Fix +-- duplication. +instance LogFormatting (TraceObjectDiffusionInbound objectId object) where + forMachine _ (TraceObjectDiffusionInboundCollectedObjects payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionInboundCollectedObjects" + , "payload" .= String (Text.pack . show $ payload) + ] + forMachine _ (TraceObjectDiffusionInboundAddedObjects payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionInboundAddedObjects" + , "payload" .= String (Text.pack . show $ payload) + ] + forMachine _ (TraceObjectDiffusionInboundRecvControlMessage payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionInboundRecvControlMessage" + , "payload" .= String (Text.pack . show $ payload) + ] + forMachine _ (TraceObjectDiffusionInboundCanRequestMoreObjects payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionInboundCanRequestMoreObjects" + , "payload" .= String (Text.pack . show $ payload) + ] + forMachine _ (TraceObjectDiffusionInboundCannotRequestMoreObjects payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionInboundCannotRequestMoreObjects" + , "payload" .= String (Text.pack . show $ payload) + ] + +instance MetaTrace (TraceObjectDiffusionInbound objectId object) where + namespaceFor (TraceObjectDiffusionInboundCollectedObjects _) = + Namespace [] ["TraceObjectDiffusionInboundCollectedObjects"] + namespaceFor (TraceObjectDiffusionInboundAddedObjects _) = + Namespace [] ["TraceObjectDiffusionInboundAddedObjects"] + namespaceFor (TraceObjectDiffusionInboundRecvControlMessage _) = + Namespace [] ["TraceObjectDiffusionInboundRecvControlMessage"] + namespaceFor (TraceObjectDiffusionInboundCanRequestMoreObjects _) = + Namespace [] ["TraceObjectDiffusionInboundCanRequestMoreObjects"] + namespaceFor (TraceObjectDiffusionInboundCannotRequestMoreObjects _) = + Namespace [] ["TraceObjectDiffusionInboundCannotRequestMoreObjects"] + + severityFor (Namespace [] ["TraceObjectDiffusionInboundCollectedObjects"]) _ = Just Info + severityFor (Namespace [] ["TraceObjectDiffusionInboundAddedObjects"]) _ = Just Info + severityFor (Namespace [] ["TraceObjectDiffusionInboundRecvControlMessage"]) _ = Just Info + severityFor (Namespace [] ["TraceObjectDiffusionInboundCanRequestMoreObjects"]) _ = Just Info + severityFor (Namespace [] ["TraceObjectDiffusionInboundCannotRequestMoreObjects"]) _ = Just Info + severityFor _ _ = Nothing + + documentFor _ = Nothing + + allNamespaces = + [ Namespace [] ["TraceObjectDiffusionInboundCollectedObjects"] + , Namespace [] ["TraceObjectDiffusionInboundAddedObjects"] + , Namespace [] ["TraceObjectDiffusionInboundRecvControlMessage"] + , Namespace [] ["TraceObjectDiffusionInboundCanRequestMoreObjects"] + , Namespace [] ["TraceObjectDiffusionInboundCannotRequestMoreObjects"] + ] + +instance + ( Show objectId + , Show object + ) => + LogFormatting (TraceObjectDiffusionOutbound objectId object) + where + forMachine _ (TraceObjectDiffusionOutboundRecvMsgRequestObjectIds payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionOutboundRecvMsgRequestObjectIds" + , "payload" .= String (Text.pack . show $ payload) + ] + forMachine _ (TraceObjectDiffusionOutboundSendMsgReplyObjectIds payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionOutboundSendMsgReplyObjectIds" + , "payload" .= String (Text.pack . show $ payload) + ] + forMachine _ (TraceObjectDiffusionOutboundRecvMsgRequestObjects payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionOutboundRecvMsgRequestObjects" + , "payload" .= String (Text.pack . show $ payload) + ] + forMachine _ (TraceObjectDiffusionOutboundSendMsgReplyObjects payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionOutboundSendMsgReplyObjects" + , "payload" .= String (Text.pack . show $ payload) + ] + forMachine _ TraceObjectDiffusionOutboundTerminated = + mconcat + [ "kind" .= String "TraceObjectDiffusionOutboundTerminated" + ] + + +instance MetaTrace (TraceObjectDiffusionOutbound objectId object) where + namespaceFor (TraceObjectDiffusionOutboundRecvMsgRequestObjectIds _) = + Namespace [] ["TraceObjectDiffusionOutboundRecvMsgRequestObjectIds"] + namespaceFor (TraceObjectDiffusionOutboundSendMsgReplyObjectIds _) = + Namespace [] ["TraceObjectDiffusionOutboundSendMsgReplyObjectIds"] + namespaceFor (TraceObjectDiffusionOutboundRecvMsgRequestObjects _) = + Namespace [] ["TraceObjectDiffusionOutboundRecvMsgRequestObjects"] + namespaceFor (TraceObjectDiffusionOutboundSendMsgReplyObjects _) = + Namespace [] ["TraceObjectDiffusionOutboundSendMsgReplyObjects"] + namespaceFor TraceObjectDiffusionOutboundTerminated = + Namespace [] ["TraceObjectDiffusionOutboundTerminated"] + + + severityFor (Namespace [] ["TraceObjectDiffusionOutboundRecvMsgRequestObjectIds"]) _ = Just Info + severityFor (Namespace [] ["TraceObjectDiffusionOutboundSendMsgReplyObjectIds"]) _ = Just Info + severityFor (Namespace [] ["TraceObjectDiffusionOutboundRecvMsgRequestObjects"]) _ = Just Info + severityFor (Namespace [] ["TraceObjectDiffusionOutboundSendMsgReplyObjects"]) _ = Just Info + severityFor (Namespace [] ["TraceObjectDiffusionOutboundTerminated"]) _ = Just Info + severityFor _ _ = Nothing + + documentFor _ = Nothing + + allNamespaces = + [ Namespace [] ["TraceObjectDiffusionOutboundRecvMsgRequestObjectIds"] + , Namespace [] ["TraceObjectDiffusionOutboundSendMsgReplyObjectIds"] + , Namespace [] ["TraceObjectDiffusionOutboundRecvMsgRequestObjects"] + , Namespace [] ["TraceObjectDiffusionOutboundSendMsgReplyObjects"] + , Namespace [] ["TraceObjectDiffusionOutboundTerminated"] + ] diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/LedgerMetrics.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/LedgerMetrics.hs index 6f2e0820ff4..a94086dbc4c 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/LedgerMetrics.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/LedgerMetrics.hs @@ -40,7 +40,7 @@ import GHC.Conc (labelThread, myThreadId) startLedgerMetricsTracer :: forall blk - . IsLedger (LedgerState blk) + . IsLedger LedgerState blk => LedgerQueries blk => AF.HasHeader (Header blk) => AF.HasHeader blk @@ -93,7 +93,7 @@ data LedgerMetrics = } traceLedgerMetrics :: - ( IsLedger (LedgerState blk) + ( IsLedger LedgerState blk , LedgerQueries blk , AF.HasHeader blk , AF.HasHeader (Header blk)) diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/NodeToClient.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/NodeToClient.hs index ff105fbc036..e634e504d2f 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/NodeToClient.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/NodeToClient.hs @@ -17,6 +17,8 @@ import qualified Ouroboros.Network.Protocol.LocalStateQuery.Type as LSQ import qualified Ouroboros.Network.Protocol.LocalTxMonitor.Type as LTM import qualified Ouroboros.Network.Protocol.LocalTxSubmission.Type as LTS import Ouroboros.Network.Tracing () +import Ouroboros.Network.Protocol.ObjectDiffusion.Type (ObjectDiffusion) +import qualified Ouroboros.Network.Protocol.ObjectDiffusion.Type as OD import Data.Aeson (Value (String), (.=)) import Data.Text (Text, pack) @@ -466,3 +468,72 @@ instance MetaTrace (Stateful.AnyMessage (LSQ.LocalStateQuery blk pt (Query blk)) , Namespace [] ["ReAcquire"] , Namespace [] ["Done"] ] + +-- -------------------------------------------------------------------------------- +-- -- TObjectDiffusion Tracer +-- -------------------------------------------------------------------------------- + +instance LogFormatting (Simple.AnyMessage (ObjectDiffusion objectId object)) where + forMachine _dtal (Simple.AnyMessageAndAgency stok OD.MsgInit {}) = + mconcat [ "kind" .= String "MsgInit" + , "agency" .= String (pack $ show stok) + ] + forMachine _dtal (Simple.AnyMessageAndAgency stok OD.MsgRequestObjectIds {}) = + mconcat [ "kind" .= String "MsgRequestObjectIds" + , "agency" .= String (pack $ show stok) + ] + forMachine _dtal (Simple.AnyMessageAndAgency stok OD.MsgReplyObjectIds {}) = + mconcat [ "kind" .= String "MsgReplyObjectIds" + , "agency" .= String (pack $ show stok) + ] + forMachine _dtal (Simple.AnyMessageAndAgency stok OD.MsgRequestObjects {}) = + mconcat [ "kind" .= String "MsgRequestObjects" + , "agency" .= String (pack $ show stok) + ] + forMachine _dtal (Simple.AnyMessageAndAgency stok OD.MsgReplyObjects {}) = + mconcat [ "kind" .= String "MsgReplyObjects" + , "agency" .= String (pack $ show stok) + ] + forMachine _dtal (Simple.AnyMessageAndAgency stok OD.MsgDone {}) = + mconcat [ "kind" .= String "MsgDone" + , "agency" .= String (pack $ show stok) + ] + +instance MetaTrace (Simple.AnyMessage (ObjectDiffusion objectId object)) where + namespaceFor (Simple.AnyMessageAndAgency _agency OD.MsgInit {}) = + Namespace [] ["Init"] + namespaceFor (Simple.AnyMessageAndAgency _agency OD.MsgRequestObjectIds {}) = + Namespace [] ["RequestObjectIds"] + namespaceFor (Simple.AnyMessageAndAgency _agency OD.MsgReplyObjectIds {}) = + Namespace [] ["ReplyObjectIds"] + namespaceFor (Simple.AnyMessageAndAgency _agency OD.MsgRequestObjects {}) = + Namespace [] ["RequestObjects"] + namespaceFor (Simple.AnyMessageAndAgency _agency OD.MsgReplyObjects {}) = + Namespace [] ["ReplyObjects"] + namespaceFor (Simple.AnyMessageAndAgency _agency OD.MsgDone {}) = + Namespace [] ["Done"] + + severityFor (Namespace [] ["Init"]) _ = Just Info + severityFor (Namespace [] ["RequestObjectIds"]) _ = Just Info + severityFor (Namespace [] ["ReplyObjectIds"]) _ = Just Info + severityFor (Namespace [] ["RequestObjects"]) _ = Just Info + severityFor (Namespace [] ["ReplyObjects"]) _ = Just Info + severityFor (Namespace [] ["Done"]) _ = Just Info + severityFor _ _ = Nothing + + documentFor (Namespace [] ["Init"]) = Just "" + documentFor (Namespace [] ["RequestObjectIds"]) = Just "" + documentFor (Namespace [] ["ReplyObjectIds"]) = Just "" + documentFor (Namespace [] ["RequestObjects"]) = Just "" + documentFor (Namespace [] ["ReplyObjects"]) = Just "" + documentFor (Namespace [] ["Done"]) = Just "" + documentFor _ = Nothing + + allNamespaces = + [ Namespace [] ["Init"] + , Namespace [] ["RequestObjectIds"] + , Namespace [] ["ReplyObjectIds"] + , Namespace [] ["RequestObjects"] + , Namespace [] ["ReplyObjects"] + , Namespace [] ["Done"] + ] diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/Startup.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/Startup.hs index c8eefaef1cf..7402b91814d 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/Startup.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/Startup.hs @@ -512,6 +512,7 @@ nodeToNodeVersionToInt :: NodeToNodeVersion -> Int nodeToNodeVersionToInt = \case NodeToNodeV_14 -> 14 NodeToNodeV_15 -> 15 + NodeToNodeV_16 -> 16 -- | Pretty print 'StartupInfoTrace' -- From 6a43d6ef5fb1461a2391215a8a77ec7b3d492396 Mon Sep 17 00:00:00 2001 From: Adithya Kumar Date: Mon, 15 Jun 2026 02:12:12 +0530 Subject: [PATCH 12/40] Remove LMDB (cherry picked from commit 2733a4aace131a2a5e765980a8a04580a0d35ec0) --- cardano-node/cardano-node.cabal | 2 +- .../Cardano/Node/Configuration/LedgerDB.hs | 71 +---- .../src/Cardano/Node/Configuration/POM.hs | 7 - .../Cardano/Node/Tracing/Tracers/ChainDB.hs | 279 +----------------- 4 files changed, 4 insertions(+), 355 deletions(-) diff --git a/cardano-node/cardano-node.cabal b/cardano-node/cardano-node.cabal index 6f1441ca94e..7caad0b2d54 100644 --- a/cardano-node/cardano-node.cabal +++ b/cardano-node/cardano-node.cabal @@ -164,7 +164,7 @@ library , network-mux >= 0.8 , nothunks , optparse-applicative - , ouroboros-consensus:{ouroboros-consensus, lmdb, lsm, cardano, diffusion, protocol} ^>= 3.0.1 + , ouroboros-consensus:{ouroboros-consensus, lsm, cardano, diffusion, protocol} ^>= 3.0.1 , ouroboros-network:{api, ouroboros-network, orphan-instances, framework, protocols, framework-tracing, tracing} ^>= 1.1 , cardano-diffusion:{api, cardano-diffusion, tracing, orphan-instances} ^>=1.0 , prettyprinter diff --git a/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs b/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs index 16bc69e893e..2b69f695a5a 100644 --- a/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs +++ b/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs @@ -6,6 +6,7 @@ {-# LANGUAGE TypeApplications #-} {-# OPTIONS_GHC -Wno-orphans #-} +{-# OPTIONS_GHC -Wno-unused-top-binds #-} module Cardano.Node.Configuration.LedgerDB ( DeprecatedOptions (..), @@ -20,8 +21,6 @@ import Ouroboros.Consensus.Ledger.SupportsProtocol import Ouroboros.Consensus.Storage.LedgerDB.API import Ouroboros.Consensus.Storage.LedgerDB.Args import Ouroboros.Consensus.Storage.LedgerDB.Snapshots -import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.Args as V1 -import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.BackingStore.Impl.LMDB as LMDB import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.InMemory as InMemory import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.LSM as LSM @@ -45,18 +44,7 @@ import Ouroboros.Consensus.Ledger.Basics (LedgerState) -- -- - 'V2LSM': Uses the LSM backend. data LedgerDbSelectorFlag = - V1LMDB - V1.FlushFrequency - -- ^ The frequency at which changes are flushed to the disk. - (Maybe FilePath) - -- ^ Path for the live tables. If not provided the default will be used - -- (@/lmdb@). - (Maybe Gigabytes) - -- ^ A map size can be specified, this is the maximum disk space the LMDB - -- database can fill. If not provided, the default of 16GB will be used. - (Maybe Int) - -- ^ An override to the max number of readers. - | V2InMemory + V2InMemory | V2LSM (Maybe FilePath) -- ^ Maybe a custom path to the LSM database. If not provided the default @@ -90,53 +78,6 @@ newtype Gigabytes = Gigabytes Int toBytes :: Gigabytes -> Int toBytes (Gigabytes x) = x * 1024 * 1024 * 1024 --- | Recommended settings for the LMDB backing store. --- --- === @'lmdbMapSize'@ --- The default @'LMDBLimits'@ uses an @'lmdbMapSize'@ of @1024 * 1024 * 1024 * 16@ --- bytes, or 16 Gigabytes. @'lmdbMapSize'@ sets the size of the memory map --- that is used internally by the LMDB backing store, and is also the --- maximum size of the on-disk database. 16 GB should be sufficient for the --- medium term, i.e., it is sufficient until a more performant alternative to --- the LMDB backing store is implemented, which will probably replace the LMDB --- backing store altogether. --- --- Note(jdral): It is recommended not to set the @'lmdbMapSize'@ to a value --- that is much smaller than 16 GB through manual configuration: the node will --- die with a fatal error as soon as the database size exceeds the --- @'lmdbMapSize'@. If this fatal error were to occur, we would expect that --- the node can continue normal operation if it is restarted with a higher --- @'lmdbMapSize'@ configured. Nonetheless, this situation should be avoided. --- --- === @'lmdbMaxDatabases'@ --- The @'lmdbMaxDatabases'@ is set to 10, which means that the LMDB backing --- store will allow up @<= 10@ internal databases. We say /internal/ --- databases, since they are not exposed outside the backing store interface, --- such that from the outside view there is just one /logical/ database. --- Two of these internal databases are reserved for normal operation of the --- backing store, while the remaining databases will be used to store ledger --- tables. At the moment, there is at most one ledger table that will be --- stored in an internal database: the UTxO. Nonetheless, we set --- @'lmdbMaxDatabases'@ to @10@ in order to future-proof these limits. --- --- === @'lmdbMaxReaders'@ --- The @'lmdbMaxReaders'@ limit sets the maximum number of threads that can --- read from the LMDB database. Currently, there should only be a single reader --- active. Again, we set @'lmdbMaxReaders'@ to @16@ in order to future-proof --- these limits. --- --- === References --- For more information about LMDB limits, one should inspect: --- * The @lmdb-simple@ and @haskell-lmdb@ forked repositories. --- * The official LMDB API documentation at --- . -defaultLMDBLimits :: LMDB.LMDBLimits -defaultLMDBLimits = LMDB.LMDBLimits { - LMDB.lmdbMapSize = 16 * 1024 * 1024 * 1024 - , LMDB.lmdbMaxDatabases = 10 - , LMDB.lmdbMaxReaders = 16 - } - defaultLMDBPath :: FilePath -> FilePath defaultLMDBPath = ( "lmdb") @@ -147,12 +88,4 @@ selectorToArgs :: , CanUpgradeLedgerTables LedgerState blk ) => LedgerDbSelectorFlag -> FilePath -> StdGen -> (LedgerDbBackendArgs IO blk, StdGen) selectorToArgs V2InMemory _ = InMemory.mkInMemoryArgs -selectorToArgs (V1LMDB ff fp l mxReaders) fastStoragePath = - LMDB.mkLMDBArgs - (Proxy @blk) - ff - (fromMaybe (defaultLMDBPath fastStoragePath) fp) - ( maybe id (\overrideMaxReaders lim -> lim{LMDB.lmdbMaxReaders = overrideMaxReaders}) mxReaders $ - maybe id (\ll lim -> lim{LMDB.lmdbMapSize = toBytes ll}) l defaultLMDBLimits - ) selectorToArgs (V2LSM fp) fastStoragePath = LSM.mkLSMArgsIO (Proxy @blk) (fromMaybe "lsm" fp) fastStoragePath diff --git a/cardano-node/src/Cardano/Node/Configuration/POM.hs b/cardano-node/src/Cardano/Node/Configuration/POM.hs index 19217c0f7e5..09590a9b929 100644 --- a/cardano-node/src/Cardano/Node/Configuration/POM.hs +++ b/cardano-node/src/Cardano/Node/Configuration/POM.hs @@ -49,7 +49,6 @@ import Ouroboros.Consensus.Util.Args (OverrideOrDefault (..)) import Ouroboros.Consensus.Storage.LedgerDB.Snapshots ( SnapshotFrequency (..), SnapshotFrequencyArgs (..), SnapshotPolicyArgs (..), defaultSnapshotPolicyArgs, NumOfDiskSnapshots (..)) -import Ouroboros.Consensus.Storage.LedgerDB.V1.Args (FlushFrequency (..)) import Ouroboros.Network.Diffusion.Configuration as Configuration import qualified Ouroboros.Network.Diffusion.Configuration as Ouroboros import qualified Ouroboros.Network.Mux as Mux @@ -525,12 +524,6 @@ instance FromJSON PartialNodeConfiguration where qsize <- (fmap RequestedQueryBatchSize <$> o .:? "QueryBatchSize") .!= DefaultQueryBatchSize backend <- o .:? "Backend" .!= "V2InMemory" selector <- case backend of - "V1LMDB" -> do - flush <- (fmap RequestedFlushFrequency <$> o .:? "FlushFrequency") .!= DefaultFlushFrequency - mapSize :: Maybe Gigabytes <- o .:? "MapSize" - lmdbPath :: Maybe FilePath <- o .:? "LiveTablesPath" - mxReaders :: Maybe Int <- o .:? "MaxReaders" - return $ V1LMDB flush lmdbPath mapSize mxReaders "V2InMemory" -> return V2InMemory "V2LSM" -> do lsmPath :: Maybe FilePath <- o .:? "LSMDatabasePath" diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs index 43be89de859..5954059c499 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs @@ -43,8 +43,6 @@ import Ouroboros.Consensus.Storage.ImmutableDB.Chunks.Internal (chunkN import qualified Ouroboros.Consensus.Storage.ImmutableDB.Impl.Types as ImmDB import qualified Ouroboros.Consensus.Storage.LedgerDB as LedgerDB import qualified Ouroboros.Consensus.Storage.LedgerDB.Snapshots as LedgerDB -import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.BackingStore as V1 -import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.BackingStore.Impl.LMDB as LMDB import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.Backend as V2 import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.InMemory as InMemory import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.LSM as LSM @@ -2109,303 +2107,28 @@ instance MetaTrace LedgerDB.TraceForkerEvent where -------------------------------------------------------------------------------- instance LogFormatting LedgerDB.FlavorImplSpecificTrace where - forMachine dtal (LedgerDB.FlavorImplSpecificTraceV1 ev) = forMachine dtal ev forMachine dtal (LedgerDB.FlavorImplSpecificTraceV2 ev) = forMachine dtal ev - forHuman (LedgerDB.FlavorImplSpecificTraceV1 ev) = forHuman ev forHuman (LedgerDB.FlavorImplSpecificTraceV2 ev) = forHuman ev instance MetaTrace LedgerDB.FlavorImplSpecificTrace where - namespaceFor (LedgerDB.FlavorImplSpecificTraceV1 ev) = - nsPrependInner "V1" (namespaceFor ev) namespaceFor (LedgerDB.FlavorImplSpecificTraceV2 ev) = nsPrependInner "V2" (namespaceFor ev) - severityFor (Namespace out ("V1" : tl)) Nothing = - severityFor (Namespace out tl :: Namespace V1.SomeBackendTrace) Nothing - severityFor (Namespace out ("V1" : tl)) (Just (LedgerDB.FlavorImplSpecificTraceV1 ev)) = - severityFor (Namespace out tl :: Namespace V1.SomeBackendTrace) (Just ev) severityFor (Namespace out ("V2" : tl)) Nothing = severityFor (Namespace out tl :: Namespace V2.LedgerDBV2Trace) Nothing severityFor (Namespace out ("V2" : tl)) (Just (LedgerDB.FlavorImplSpecificTraceV2 ev)) = severityFor (Namespace out tl :: Namespace V2.LedgerDBV2Trace) (Just ev) severityFor _ _ = Nothing - documentFor (Namespace out ("V1" : tl)) = - documentFor (Namespace out tl :: Namespace V1.SomeBackendTrace) documentFor (Namespace out ("V2" : tl)) = documentFor (Namespace out tl :: Namespace V2.LedgerDBV2Trace) documentFor _ = Nothing allNamespaces = - map (nsPrependInner "V1") - (allNamespaces :: [Namespace V1.SomeBackendTrace]) - ++ map (nsPrependInner "V2") + map (nsPrependInner "V2") (allNamespaces :: [Namespace V2.LedgerDBV2Trace]) --------------------------------------------------------------------------------- --- V1 --------------------------------------------------------------------------------- - -unwrapV1Trace :: forall a backend. Typeable backend => (V1.Trace LMDB.LMDB -> a) -> V1.Trace backend -> a -unwrapV1Trace g ev = - case cast @(V1.Trace backend) @(V1.Trace LMDB.LMDB) ev of - Just t -> g t - _ -> error "blah" - -instance LogFormatting V1.SomeBackendTrace where - forMachine dtal (V1.SomeBackendTrace ev) = - unwrapV1Trace (forMachine dtal) ev - - forHuman (V1.SomeBackendTrace ev) = - unwrapV1Trace forHuman ev - -instance MetaTrace V1.SomeBackendTrace where - namespaceFor (V1.SomeBackendTrace ev) = - unwrapV1Trace (nsPrependInner "LMDB" . namespaceFor) ev - - severityFor (Namespace out ("LMDB" : tl)) (Just (V1.SomeBackendTrace ev)) = - unwrapV1Trace (severityFor (Namespace out tl :: Namespace (V1.Trace LMDB.LMDB)) . Just) ev - severityFor (Namespace _ ("LMDB" : _)) Nothing = - Just Debug - severityFor _ _ = Nothing - - documentFor (Namespace _ ("LMDB" : _)) = - Just "An LMDB trace" - documentFor _ = Nothing - - allNamespaces = - map (nsPrependInner "LMDB") - (allNamespaces :: [Namespace (V1.Trace LMDB.LMDB)]) - -instance LogFormatting (V1.Trace LMDB.LMDB) where - forMachine _dtal (LMDB.OnDiskBackingStoreInitialise limits) = - mconcat [ "kind" .= String "LMDBBackingStoreInitialise", "limits" .= showT limits ] - forMachine dtal (LMDB.OnDiskBackingStoreTrace ev) = forMachine dtal ev - - forHuman (LMDB.OnDiskBackingStoreInitialise limits) = "Initializing LMDB backing store with limits " <> showT limits - forHuman (LMDB.OnDiskBackingStoreTrace ev) = forHuman ev - -instance MetaTrace (V1.Trace LMDB.LMDB) where - namespaceFor LMDB.OnDiskBackingStoreInitialise{} = - Namespace [] ["Initialise"] - namespaceFor (LMDB.OnDiskBackingStoreTrace ev) = - nsPrependInner "BackingStoreEvent" (namespaceFor ev) - - severityFor (Namespace _ ("Initialise" : _)) _ = Just Debug - severityFor (Namespace out ("BackingStoreEvent" : tl)) Nothing = - severityFor (Namespace out tl :: Namespace V1.BackingStoreTrace) Nothing - severityFor (Namespace out ("BackingStoreEvent" : tl)) (Just (LMDB.OnDiskBackingStoreTrace ev)) = - severityFor (Namespace out tl :: Namespace V1.BackingStoreTrace) (Just ev) - severityFor _ _ = Nothing - - documentFor (Namespace _ ("Initialise" : _)) = Just - "Backing store is being initialised" - documentFor (Namespace out ("BackingStoreEvent" : tl)) = - documentFor (Namespace out tl :: Namespace V1.BackingStoreTrace) - documentFor _ = Nothing - - allNamespaces = - Namespace [] ["Initialise"] - : map (nsPrependInner "BackingStoreEvent") - (allNamespaces :: [Namespace V1.BackingStoreTrace]) - -instance LogFormatting V1.BackingStoreTrace where - forMachine _dtals V1.BSOpening = mempty - forMachine _dtals (V1.BSOpened p) = - maybe mempty (\p' -> mconcat [ "path" .= showT p' ]) p - forMachine _dtals (V1.BSInitialisingFromCopy p) = - mconcat [ "path" .= showT p ] - forMachine _dtals (V1.BSInitialisedFromCopy p) = - mconcat [ "path" .= showT p ] - forMachine _dtals (V1.BSInitialisingFromValues sl) = - mconcat [ "slot" .= showT sl ] - forMachine _dtals (V1.BSInitialisedFromValues sl) = - mconcat [ "slot" .= showT sl ] - forMachine _dtals V1.BSClosing = mempty - forMachine _dtals V1.BSAlreadyClosed = mempty - forMachine _dtals V1.BSClosed = mempty - forMachine _dtals (V1.BSCopying p) = - mconcat [ "path" .= showT p ] - forMachine _dtals (V1.BSCopied p) = - mconcat [ "path" .= showT p ] - forMachine _dtals V1.BSCreatingValueHandle = mempty - forMachine _dtals V1.BSCreatedValueHandle = mempty - forMachine _dtals (V1.BSWriting s) = - mconcat [ "slot" .= showT s ] - forMachine _dtals (V1.BSWritten s1 s2) = - mconcat [ "old" .= showT s1, "new" .= showT s2 ] - forMachine _dtals (V1.BSValueHandleTrace i _ev) = - maybe mempty (\i' -> mconcat ["idx" .= showT i']) i -instance LogFormatting V1.BackingStoreValueHandleTrace where - forMachine _dtals V1.BSVHClosing = mempty - forMachine _dtals V1.BSVHAlreadyClosed = mempty - forMachine _dtals V1.BSVHClosed = mempty - forMachine _dtals V1.BSVHRangeReading = mempty - forMachine _dtals V1.BSVHRangeRead = mempty - forMachine _dtals V1.BSVHReading = mempty - forMachine _dtals V1.BSVHRead = mempty - forMachine _dtals V1.BSVHStatting = mempty - forMachine _dtals V1.BSVHStatted = mempty - -instance MetaTrace V1.BackingStoreTrace where - namespaceFor V1.BSOpening = Namespace [] ["Opening"] - namespaceFor V1.BSOpened{} = Namespace [] ["Opened"] - namespaceFor V1.BSInitialisingFromCopy{} = - Namespace [] ["InitialisingFromCopy"] - namespaceFor V1.BSInitialisedFromCopy{} = - Namespace [] ["InitialisedFromCopy"] - namespaceFor V1.BSInitialisingFromValues{} = - Namespace [] ["InitialisingFromValues"] - namespaceFor V1.BSInitialisedFromValues{} = - Namespace [] ["InitialisedFromValues"] - namespaceFor V1.BSClosing = Namespace [] ["Closing"] - namespaceFor V1.BSAlreadyClosed = Namespace [] ["AlreadyClosed"] - namespaceFor V1.BSClosed = Namespace [] ["Closed"] - namespaceFor V1.BSCopying{} = Namespace [] ["Copying"] - namespaceFor V1.BSCopied{} = Namespace [] ["Copied"] - namespaceFor V1.BSCreatingValueHandle = Namespace [] ["CreatingValueHandle"] - namespaceFor V1.BSCreatedValueHandle = Namespace [] ["CreatedValueHandle"] - namespaceFor (V1.BSValueHandleTrace _ bsValueHandleTrace) = - nsPrependInner "ValueHandleTrace" (namespaceFor bsValueHandleTrace) - namespaceFor V1.BSWriting{} = Namespace [] ["Writing"] - namespaceFor V1.BSWritten{} = Namespace [] ["Written"] - - severityFor (Namespace _ ("Opening" : _)) _ = Just Debug - severityFor (Namespace _ ("Opened" : _)) _ = Just Debug - severityFor (Namespace _ ("InitialisingFromCopy" : _)) _ = Just Debug - severityFor (Namespace _ ("InitialisedFromCopy" : _)) _ = Just Debug - severityFor (Namespace _ ("InitialisingFromValues" : _)) _ = Just Debug - severityFor (Namespace _ ("InitialisedFromValues" : _)) _ = Just Debug - severityFor (Namespace _ ("Closing" : _)) _ = Just Debug - severityFor (Namespace _ ("AlreadyClosed" : _)) _ = Just Debug - severityFor (Namespace _ ("Closed" : _)) _ = Just Debug - severityFor (Namespace _ ("Copying" : _)) _ = Just Debug - severityFor (Namespace _ ("Copied" : _)) _ = Just Debug - severityFor (Namespace _ ("CreatingValueHandle" : _)) _ = Just Debug - severityFor (Namespace _ ("CreatedValueHandle" : _)) _ = Just Debug - severityFor (Namespace out ("ValueHandleTrace" : t1)) Nothing = - severityFor - (Namespace out t1 :: Namespace V1.BackingStoreValueHandleTrace) - Nothing - severityFor - (Namespace out ("ValueHandleTrace" : t1)) - (Just (V1.BSValueHandleTrace _ bsValueHandleTrace)) = - severityFor - (Namespace out t1 :: Namespace V1.BackingStoreValueHandleTrace) - (Just bsValueHandleTrace) - severityFor (Namespace _ ("Writing" : _)) _ = Just Debug - severityFor (Namespace _ ("Written" : _)) _ = Just Debug - severityFor _ _ = Nothing - - documentFor (Namespace _ ("Opening" : _ )) = Just - "Opening backing store" - documentFor (Namespace _ ("Opened" : _ )) = Just - "Backing store opened" - documentFor (Namespace _ ("InitialisingFromCopy" : _ )) = Just - "Initialising backing store from copy" - documentFor (Namespace _ ("InitialisedFromCopy" : _ )) = Just - "Backing store initialised from copy" - documentFor (Namespace _ ("InitialisingFromValues" : _ )) = Just - "Initialising backing store from values" - documentFor (Namespace _ ("InitialisedFromValues" : _ )) = Just - "Backing store initialised from values" - documentFor (Namespace _ ("Closing" : _ )) = Just - "Closing backing store" - documentFor (Namespace _ ("AlreadyClosed" : _ )) = Just - "Backing store is already closed" - documentFor (Namespace _ ("Closed" : _ )) = Just - "Backing store closed" - documentFor (Namespace _ ("Copying" : _ )) = Just - "Copying backing store" - documentFor (Namespace _ ("Copied" : _ )) = Just - "Backing store copied" - documentFor (Namespace _ ("CreatingValueHandle" : _ )) = Just - "Creating value handle for backing store" - documentFor (Namespace _ ("CreatedValueHandle" : _ )) = Just - "Value handle for backing store created" - documentFor (Namespace out ("ValueHandleTrace" : t1 )) = - documentFor (Namespace out t1 :: Namespace V1.BackingStoreValueHandleTrace) - documentFor (Namespace _ ("Writing" : _ )) = Just - "Writing backing store" - documentFor (Namespace _ ("Written" : _ )) = Just - "Backing store written" - documentFor _ = Nothing - - allNamespaces = - [ Namespace [] ["Opening"] - , Namespace [] ["Opened"] - , Namespace [] ["InitialisingFromCopy"] - , Namespace [] ["InitialisedFromCopy"] - , Namespace [] ["InitialisingFromValues"] - , Namespace [] ["InitialisedFromValues"] - , Namespace [] ["Closing"] - , Namespace [] ["AlreadyClosed"] - , Namespace [] ["Closed"] - , Namespace [] ["Copying"] - , Namespace [] ["Copied"] - , Namespace [] ["CreatingValueHandle"] - , Namespace [] ["CreatedValueHandle"] - , Namespace [] ["Writing"] - , Namespace [] ["Written"] - ] ++ map (nsPrependInner "ValueHandleTrace") - (allNamespaces :: [Namespace V1.BackingStoreValueHandleTrace]) - - -instance MetaTrace V1.BackingStoreValueHandleTrace where - namespaceFor V1.BSVHClosing = Namespace [] ["Closing"] - namespaceFor V1.BSVHAlreadyClosed = Namespace [] ["AlreadyClosed"] - namespaceFor V1.BSVHClosed = Namespace [] ["Closed"] - namespaceFor V1.BSVHRangeReading = Namespace [] ["RangeReading"] - namespaceFor V1.BSVHRangeRead = Namespace [] ["RangeRead"] - namespaceFor V1.BSVHReading = Namespace [] ["Reading"] - namespaceFor V1.BSVHRead = Namespace [] ["Read"] - namespaceFor V1.BSVHStatting = Namespace [] ["Statting"] - namespaceFor V1.BSVHStatted = Namespace [] ["Statted"] - - severityFor (Namespace _ ("Closing" : _ )) _ = Just Debug - severityFor (Namespace _ ("AlreadyClosed" : _ )) _ = Just Debug - severityFor (Namespace _ ("Closed" : _ )) _ = Just Debug - severityFor (Namespace _ ("RangeReading" : _ )) _ = Just Debug - severityFor (Namespace _ ("RangeRead" : _ )) _ = Just Debug - severityFor (Namespace _ ("Reading" : _ )) _ = Just Debug - severityFor (Namespace _ ("Read" : _ )) _ = Just Debug - severityFor (Namespace _ ("Statting" : _ )) _ = Just Debug - severityFor (Namespace _ ("Statted" : _ )) _ = Just Debug - severityFor _ _ = Nothing - - documentFor (Namespace _ ("Closing" : _ )) = Just - "Closing backing store value handle" - documentFor (Namespace _ ("AlreadyClosed" : _ )) = Just - "Backing store value handle already clsoed" - documentFor (Namespace _ ("Closed" : _ )) = Just - "Backing store value handle closed" - documentFor (Namespace _ ("RangeReading" : _ )) = Just - "Reading range for backing store value handle" - documentFor (Namespace _ ("RangeRead" : _ )) = Just - "Range for backing store value handle read" - documentFor (Namespace _ ("Reading" : _ )) = Just - "Reading backing store value handle" - documentFor (Namespace _ ("Read" : _ )) = Just - "Backing store value handle read" - documentFor (Namespace _ ("Statting" : _ )) = Just - "Statting backing store value handle" - documentFor (Namespace _ ("Statted" : _ )) = Just - "Backing store value handle statted" - documentFor _ = Nothing - - allNamespaces = - [ Namespace [] ["Closing"] - , Namespace [] ["AlreadyClosed"] - , Namespace [] ["Closed"] - , Namespace [] ["RangeReading"] - , Namespace [] ["RangeRead"] - , Namespace [] ["Reading"] - , Namespace [] ["Read"] - , Namespace [] ["Statting"] - , Namespace [] ["Statted"] - ] - {------------------------------------------------------------------------------- V2 -------------------------------------------------------------------------------} From ffe9249320dbf5a484eba3ea6690a7477e78bbf1 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Wed, 17 Jun 2026 21:35:28 +0200 Subject: [PATCH 13/40] First draft --- .../Benchmarking/GeneratorTx/NodeToNode.hs | 11 +- .../Cardano/Benchmarking/OuroborosImports.hs | 18 +- .../src/Cardano/Benchmarking/Script/Action.hs | 2 +- .../src/Cardano/Benchmarking/Script/Core.hs | 7 +- .../Cardano/TxGenerator/Setup/NixService.hs | 10 +- .../Cardano/TxGenerator/Setup/NodeConfig.hs | 2 +- bench/tx-generator/tx-generator.cabal | 2 +- cabal.project | 114 ++++++++++-- cardano-node-chairman/app/Cardano/Chairman.hs | 2 +- .../app/Cardano/Chairman/Commands/Run.hs | 20 +-- .../cardano-node-chairman.cabal | 2 +- cardano-node/cardano-node.cabal | 9 +- .../Cardano/Node/Configuration/LedgerDB.hs | 2 +- .../src/Cardano/Node/Protocol/Cardano.hs | 10 +- .../src/Cardano/Node/Protocol/Conway.hs | 1 + .../src/Cardano/Node/Protocol/Shelley.hs | 9 + .../src/Cardano/Node/Protocol/Types.hs | 2 +- cardano-node/src/Cardano/Node/Run.hs | 16 +- cardano-node/src/Cardano/Node/Startup.hs | 44 +++-- .../src/Cardano/Node/Tracing/Consistency.hs | 1 - .../src/Cardano/Node/Tracing/Documentation.hs | 3 - .../src/Cardano/Node/Tracing/Era/Shelley.hs | 13 +- .../src/Cardano/Node/Tracing/Tracers.hs | 168 +++++++++--------- .../Cardano/Node/Tracing/Tracers/Startup.hs | 4 +- cardano-submit-api/cardano-submit-api.cabal | 8 +- cardano-testnet/cardano-testnet.cabal | 13 +- .../src/Testnet/Components/Query.hs | 12 +- cardano-testnet/src/Testnet/Defaults.hs | 1 + cardano-testnet/src/Testnet/Filepath.hs | 13 ++ cardano-testnet/src/Testnet/Ping.hs | 145 ++------------- .../src/Testnet/Process/Cli/SPO.hs | 2 + cardano-testnet/src/Testnet/Runtime.hs | 3 + cardano-testnet/src/Testnet/Start/Cardano.hs | 2 + .../Cardano/Testnet/Test/FoldEpochState.hs | 7 +- .../Cardano/Testnet/Test/Gov/InfoAction.hs | 3 + .../Test/Gov/ProposeNewConstitution.hs | 20 ++- .../Test/Gov/ProposeNewConstitutionSPO.hs | 4 +- .../Testnet/Test/Gov/TreasuryGrowth.hs | 7 +- .../Testnet/Test/Gov/TreasuryWithdrawal.hs | 7 +- .../Cardano/Testnet/Test/SanityCheck.hs | 3 + .../src/Cardano/Tracer/Acceptors/Client.hs | 6 +- .../src/Cardano/Tracer/Acceptors/Server.hs | 4 +- .../test/Cardano/Tracer/Test/Forwarder.hs | 6 +- flake.lock | 12 +- trace-forward/src/Trace/Forward/Forwarding.hs | 6 +- 45 files changed, 398 insertions(+), 358 deletions(-) diff --git a/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/NodeToNode.hs b/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/NodeToNode.hs index f2030e39ae6..492f6c47f5c 100644 --- a/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/NodeToNode.hs +++ b/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/NodeToNode.hs @@ -40,6 +40,7 @@ import Ouroboros.Network.Mux (MiniProtocolCb (..), OuroborosApplicatio import Ouroboros.Network.PeerSelection.PeerSharing (PeerSharing (..)) import Ouroboros.Network.PeerSelection.PeerSharing.Codec (decodeRemoteAddress, encodeRemoteAddress) +import Ouroboros.Network.PerasSupport (PerasSupport (..)) import Ouroboros.Network.Protocol.BlockFetch.Client (BlockFetchClient (..), blockFetchClientPeer) import Ouroboros.Network.Protocol.Handshake.Version (simpleSingletonVersions) @@ -112,7 +113,7 @@ benchmarkConnectTxSubmit EnvConsts { .. } handshakeTracer submissionTracer codec supportedVers = supportedNodeToNodeVersions (Proxy @blk) myCodecs :: Codecs blk NtN.RemoteAddress DeserialiseFailure IO ByteString ByteString ByteString ByteString ByteString ByteString - ByteString + ByteString ByteString ByteString myCodecs = defaultCodecs codecConfig blkN2nVer encodeRemoteAddress decodeRemoteAddress n2nVer peerMultiplex :: NtN.Versions NodeToNodeVersion NtN.NodeToNodeVersionData @@ -129,10 +130,11 @@ benchmarkConnectTxSubmit EnvConsts { .. } handshakeTracer submissionTracer codec , NtN.diffusionMode = NtN.InitiatorOnlyDiffusionMode , NtN.peerSharing = ownPeerSharing , NtN.query = False + , NtN.perasSupport = PerasUnsupported }) $ \n2nData -> mkApp $ - NtN.nodeToNodeProtocols NtN.defaultMiniProtocolParameters + NtN.nodeToNodeProtocols mempty NtN.defaultMiniProtocolParameters NtN.NodeToNodeProtocols { NtN.chainSyncProtocol = InitiatorProtocolOnly $ MiniProtocolCb $ \_ctx channel -> runPeer @@ -160,6 +162,11 @@ benchmarkConnectTxSubmit EnvConsts { .. } handshakeTracer submissionTracer codec (cPeerSharingCodec myCodecs) channel (peerSharingClientPeer peerSharingClientNull) + -- TODO Peras is not supported here + , NtN.perasCertDiffusionProtocol = InitiatorProtocolOnly $ MiniProtocolCb $ \_ctx _channel -> + error "tx-generator: Peras cert diffusion is unsupported" + , NtN.perasVoteDiffusionProtocol = InitiatorProtocolOnly $ MiniProtocolCb $ \_ctx _channel -> + error "tx-generator: Peras vote diffusion is unsupported" } n2nVer n2nData diff --git a/bench/tx-generator/src/Cardano/Benchmarking/OuroborosImports.hs b/bench/tx-generator/src/Cardano/Benchmarking/OuroborosImports.hs index abd5a10c54f..77d54cad06b 100644 --- a/bench/tx-generator/src/Cardano/Benchmarking/OuroborosImports.hs +++ b/bench/tx-generator/src/Cardano/Benchmarking/OuroborosImports.hs @@ -38,21 +38,19 @@ import Prelude type CardanoBlock = Consensus.CardanoBlock StandardCrypto -toProtocolInfo :: SomeConsensusProtocol -> ProtocolInfo CardanoBlock -toProtocolInfo (SomeConsensusProtocol CardanoBlockType info) = fst $ protocolInfo @IO info +toProtocolInfo :: SomeConsensusProtocol -> IO (ProtocolInfo CardanoBlock) +toProtocolInfo (SomeConsensusProtocol CardanoBlockType info) = fst <$> protocolInfo @IO info toProtocolInfo _ = error "toProtocolInfo unknown protocol" -protocolToTopLevelConfig :: SomeConsensusProtocol -> TopLevelConfig CardanoBlock -protocolToTopLevelConfig ptcl = pInfoConfig - where - ProtocolInfo {pInfoConfig} = toProtocolInfo ptcl +protocolToTopLevelConfig :: SomeConsensusProtocol -> IO (TopLevelConfig CardanoBlock) +protocolToTopLevelConfig ptcl = pInfoConfig <$> toProtocolInfo ptcl -protocolToCodecConfig :: SomeConsensusProtocol -> CodecConfig CardanoBlock -protocolToCodecConfig = configCodec . protocolToTopLevelConfig +protocolToCodecConfig :: SomeConsensusProtocol -> IO (CodecConfig CardanoBlock) +protocolToCodecConfig = fmap configCodec . protocolToTopLevelConfig -protocolToNetworkId :: SomeConsensusProtocol -> NetworkId +protocolToNetworkId :: SomeConsensusProtocol -> IO NetworkId protocolToNetworkId ptcl - = Testnet $ getNetworkMagic $ configBlock $ protocolToTopLevelConfig ptcl + = Testnet . getNetworkMagic . configBlock <$> protocolToTopLevelConfig ptcl makeLocalConnectInfo :: NetworkId -> SocketPath -> LocalNodeConnectInfo makeLocalConnectInfo networkId socketPath diff --git a/bench/tx-generator/src/Cardano/Benchmarking/Script/Action.hs b/bench/tx-generator/src/Cardano/Benchmarking/Script/Action.hs index 3435fbddeb9..0c992938ee0 100644 --- a/bench/tx-generator/src/Cardano/Benchmarking/Script/Action.hs +++ b/bench/tx-generator/src/Cardano/Benchmarking/Script/Action.hs @@ -69,8 +69,8 @@ startProtocol configFile tracerSocket = do setEnvGenesis $ getGenesis protocol iomgr <- askIOManager + networkId <- liftIO $ protocolToNetworkId protocol let - networkId = protocolToNetworkId protocol tracerSocket' = (,,) iomgr networkId `fmap` tracerSocket setEnvNetworkId networkId diff --git a/bench/tx-generator/src/Cardano/Benchmarking/Script/Core.hs b/bench/tx-generator/src/Cardano/Benchmarking/Script/Core.hs index b13774896b4..b26d5964203 100644 --- a/bench/tx-generator/src/Cardano/Benchmarking/Script/Core.hs +++ b/bench/tx-generator/src/Cardano/Benchmarking/Script/Core.hs @@ -57,7 +57,7 @@ import Prelude import Control.Concurrent (threadDelay) import Control.Monad import Control.Monad.Trans.RWS.Strict (ask) -import "contra-tracer" Control.Tracer (Tracer (..)) +import "contra-tracer" Control.Tracer (mkTracer) import Data.ByteString.Lazy.Char8 as BSL (writeFile) import Data.Ratio ((%)) import qualified Data.Text as Text (unpack) @@ -137,11 +137,12 @@ getConnectClient = do protocol <- getEnvProtocol void $ return $ btSubmission2_ tracers envConsts <- lift ask + codecConfig <- liftIO $ protocolToCodecConfig protocol return $ benchmarkConnectTxSubmit envConsts - (Tracer $ traceWith (btConnect_ tracers)) + (mkTracer $ traceWith (btConnect_ tracers)) mempty -- (btSubmission2_ tracers) - (protocolToCodecConfig protocol) + codecConfig networkMagic waitBenchmark :: ActionM () waitBenchmark = do diff --git a/bench/tx-generator/src/Cardano/TxGenerator/Setup/NixService.hs b/bench/tx-generator/src/Cardano/TxGenerator/Setup/NixService.hs index 8b83528f49a..2e1c48c2852 100644 --- a/bench/tx-generator/src/Cardano/TxGenerator/Setup/NixService.hs +++ b/bench/tx-generator/src/Cardano/TxGenerator/Setup/NixService.hs @@ -27,7 +27,7 @@ import Cardano.Api (AnyCardanoEra, mapFile) import Cardano.CLI.Type.Common (FileDirection (..), SigningKeyFile) import qualified Cardano.Ledger.Coin as L import Cardano.Node.Configuration.NodeAddress (NodeAddress' (..), - NodeHostIPv4Address (..), NodeIPv4Address) + NodeIPv4Address) import Cardano.Node.Types (AdjustFilePaths (..)) import Cardano.TxGenerator.Internal.Orphans () import Cardano.TxGenerator.Types @@ -74,22 +74,20 @@ data NodeDescription = instance FromJSON NodeDescription where parseJSON = withObject "NodeDescription" \v -> do - unNodeHostIPv4Address + naHostAddress <- v .: "addr" Key "addr" naPort <- fmap toEnum $ v .: "port" Key "port" - let naHostAddress = NodeHostIPv4Address {..} - ndAddr = NodeAddress {..} + let ndAddr = NodeAddress {..} ndName <- v .:? "name" Key "name" .!= show ndAddr pure $ NodeDescription {..} instance ToJSON NodeDescription where toJSON NodeDescription {ndAddr, ndName} = object [ "name" .= ndName - , "addr" .= unNodeHostIPv4Address + , "addr" .= naHostAddress , "port" .= fromEnum naPort ] where _addr@NodeAddress {naHostAddress, naPort} = ndAddr - _hostAddr@NodeHostIPv4Address {unNodeHostIPv4Address} = naHostAddress -- Long GC pauses on target nodes can trigger spurious MVar deadlock diff --git a/bench/tx-generator/src/Cardano/TxGenerator/Setup/NodeConfig.hs b/bench/tx-generator/src/Cardano/TxGenerator/Setup/NodeConfig.hs index fab37d0fe8b..8e78199c68b 100644 --- a/bench/tx-generator/src/Cardano/TxGenerator/Setup/NodeConfig.hs +++ b/bench/tx-generator/src/Cardano/TxGenerator/Setup/NodeConfig.hs @@ -34,7 +34,7 @@ getGenesis :: SomeConsensusProtocol -> ShelleyGenesis getGenesis (SomeConsensusProtocol CardanoBlockType proto) = getConst $ Ledger.tcShelleyGenesisL Const transCfg where - ProtocolInfoArgsCardano Consensus.CardanoProtocolParams + ProtocolInfoArgsCardano _ Consensus.CardanoProtocolParams { Consensus.cardanoLedgerTransitionConfig = transCfg } = proto diff --git a/bench/tx-generator/tx-generator.cabal b/bench/tx-generator/tx-generator.cabal index 3345b5d4fc1..acabbb3d6d7 100644 --- a/bench/tx-generator/tx-generator.cabal +++ b/bench/tx-generator/tx-generator.cabal @@ -134,7 +134,7 @@ library , network-mux , optparse-applicative , ouroboros-consensus:{ouroboros-consensus, cardano, diffusion} >= 3.0.1 - , ouroboros-network:{api, framework, framework-tracing, ouroboros-network, protocols} >= 1.1 + , ouroboros-network:{api, framework, tracing, ouroboros-network, protocols} >= 1.1 , plutus-ledger-api , plutus-tx , random diff --git a/cabal.project b/cabal.project index 27c27df50ec..ae1112f8238 100644 --- a/cabal.project +++ b/cabal.project @@ -13,8 +13,8 @@ repository cardano-haskell-packages -- See CONTRIBUTING for information about these, including some Nix commands -- you need to run if you change them index-state: - , hackage.haskell.org 2026-04-17T09:20:55Z - , cardano-haskell-packages 2026-05-27T09:43:46Z + , hackage.haskell.org 2026-06-10T14:43:25Z + , cardano-haskell-packages 2026-06-10T14:16:00Z active-repositories: , :rest @@ -50,16 +50,6 @@ packages: bench/trace-schemas/scripts/schema-gen trace-resources trace-forward - ouroboros-consensus - ouroboros-network/ouroboros-network - ouroboros-network/cardano-diffusion - ouroboros-network/cardano-ping - ouroboros-network/monoidal-synchronisation - ouroboros-network/network-mux - ekg-forward - hermod-tracing/trace-dispatcher - cardano-api/cardano-api - cardano-api/cardano-rpc -- Needed when cross compiling extra-packages: alex, dmq-node >= 0.4.2.0 @@ -96,3 +86,103 @@ allow-newer: -- IMPORTANT -- Do NOT add more source-repository-package stanzas here unless they are strictly -- temporary! Please read the section in CONTRIBUTING about updating dependencies. + + +source-repository-package + type: git + location: https://github.com/IntersectMBO/cardano-api.git + tag: 500e283a93fcb4c82b03febf243cb976431a613f + --sha256: sha256-aHpjdMQhjRpe0w5Fb9a5LeVGcO44VAl928CS7HoYB0Q= + subdir: + cardano-api + cardano-rpc + +source-repository-package + type: git + location: https://github.com/IntersectMBO/cardano-cli.git + tag: 9678a844fd2b53bc3777906d61b53fe0ff93e8ed + --sha256: sha256-0Eo3txVuqkmvZdsyalKbS+bnmFzevG4O99CavSjo7SM= + subdir: + cardano-cli + +source-repository-package + type: git + location: https://github.com/IntersectMBO/cardano-ledger.git + tag: 3f879bb37df4738ed8211e500c7d180443cfcbe4 + --sha256: sha256-uLjiIHiU1SzAmoKs+rynQphc3FUYXKeJLlOnp87uNdg= + subdir: + eras/allegra/impl + eras/alonzo/impl + eras/babbage/impl + eras/byron/chain/executable-spec + eras/byron/crypto + eras/byron/ledger/executable-spec + eras/byron/ledger/impl + eras/conway/impl + eras/dijkstra/impl + eras/mary/impl + eras/shelley-ma/test-suite + eras/shelley/impl + eras/shelley/test-suite + libs/cardano-data + libs/cardano-ledger-api + libs/cardano-ledger-binary + libs/cardano-ledger-core + libs/cardano-protocol-tpraos + libs/non-integral + libs/small-steps + libs/vector-map + +source-repository-package + type: git + location: https://github.com/f-f/kes-agent.git + tag: fdb4f4db05e3744ed413f83477020fdf43cf32a2 + --sha256: sha256-eyQc8Dk7+upSRQvH5eXZuj6asYhOLsH59ABJZDyvQ6I= + subdir: + kes-agent + kes-agent-crypto + +source-repository-package + type: git + location: https://github.com/IntersectMBO/ouroboros-consensus.git + tag: 0411b4d50dc62cab07bbbf75805cf585a7a1f8e7 + --sha256: sha256-DNnGHdo+oQDBbHzAl6UZ/VcPoS4TxGrC9eCOiMxgc8A= + subdir: + . + +source-repository-package + type: git + location: https://github.com/IntersectMBO/ouroboros-network.git + tag: 1881340e26ca98b3ee2b89ea536348406b79e051 + --sha256: sha256-O1Th+YI6LZqCLM6n75tVV+3InA4W8sC0yHEi09u8Fyw= + subdir: + ./cardano-diffusion + ./monoidal-synchronisation + ./network-mux + ./ouroboros-network + +source-repository-package + type: git + location: https://github.com/f-f/dmq-node.git + tag: b7d59834e52d46229bb3ff6e5595509eede730dd + --sha256: sha256-8GKAaNPRZkKsk10Emokrlnc0sQOx1aHUnggqlMdY0nw= + subdir: + dmq-node + +source-repository-package + type: git + location: https://github.com/IntersectMBO/plutus.git + tag: 5b401b3ba95d2b8ecf9454aca7bf704bfd69f13c + --sha256: sha256-1kIRfqJGitFLdg77kYrmA8P8gDDmnldRLqI1iBDSVRU= + subdir: + plutus-core + plutus-ledger-api + plutus-tx + plutus-tx-plugin + plutus-metatheory + +source-repository-package + type: git + location: https://github.com/f-f/ekg-forward + tag: b24b3aba2806ce223c62f8ce3e267ec92dcc52e2 + --sha256: sha256-s5Hxxm04HmFVmdBjAnFEsJEhTqr5Z/uiB4K1s2VaVwE= diff --git a/cardano-node-chairman/app/Cardano/Chairman.hs b/cardano-node-chairman/app/Cardano/Chairman.hs index b43842cf160..0d5e7453ca5 100644 --- a/cardano-node-chairman/app/Cardano/Chairman.hs +++ b/cardano-node-chairman/app/Cardano/Chairman.hs @@ -272,7 +272,7 @@ runChairman tracer networkId runningTime socketPaths cModeParams secParam = do , localNodeSocketPath = socketPath } chairmanChainSyncClient = LocalChainSyncClient $ - chainSyncClient (showTracing tracer) socketPath chainsVar secParam + chainSyncClient (show >$< tracer) socketPath chainsVar secParam protocolsInMode = LocalNodeClientProtocols { localChainSyncClient = chairmanChainSyncClient , localTxSubmissionClient = Nothing diff --git a/cardano-node-chairman/app/Cardano/Chairman/Commands/Run.hs b/cardano-node-chairman/app/Cardano/Chairman/Commands/Run.hs index 66b645adf46..a7e68fdd855 100644 --- a/cardano-node-chairman/app/Cardano/Chairman/Commands/Run.hs +++ b/cardano-node-chairman/app/Cardano/Chairman/Commands/Run.hs @@ -24,7 +24,7 @@ import Ouroboros.Consensus.Config.SupportsNode import Ouroboros.Consensus.Node.ProtocolInfo import Control.Monad.Class.MonadTime.SI (DiffTime) -import Control.Tracer (Tracer (..), stdoutTracer) +import Control.Tracer (Tracer, mkTracer, stdoutTracer, traceWith) import Data.Monoid (Last (..)) import qualified Data.Time.Clock as DTC import Options.Applicative @@ -113,14 +113,14 @@ run RunOpts Left err -> putStrLn (docToString $ prettyError err) >> exitFailure Right p -> pure p - let (k , nId) = case p of - SomeConsensusProtocol _ runP -> - let ProtocolInfo { pInfoConfig } = fst $ Api.protocolInfo @IO runP - in ( Consensus.configSecurityParam pInfoConfig - , fromNetworkMagic . getNetworkMagic $ Consensus.configBlock pInfoConfig - ) + (k , nId) <- case p of + SomeConsensusProtocol _ runP -> do + ProtocolInfo { pInfoConfig } <- fst <$> Api.protocolInfo @IO runP + pure ( Consensus.configSecurityParam pInfoConfig + , fromNetworkMagic . getNetworkMagic $ Consensus.configBlock pInfoConfig + ) - consensusModeParams = getConsensusMode k ptclConfig + let consensusModeParams = getConsensusMode k ptclConfig chairmanTest (timed stdoutTracer) @@ -146,10 +146,10 @@ run RunOpts getLast pncProtocolConfig timed :: Tracer IO a -> Tracer IO a -timed (Tracer runTracer) = Tracer $ \a -> do +timed tr = mkTracer $ \a -> do ts <- DTC.getCurrentTime IO.putStr ("[" <> show ts <> "] ") - runTracer a + traceWith tr a cmdRun :: Mod CommandFields (IO ()) cmdRun = command "run" $ flip info idm $ run <$> parseRunOpts diff --git a/cardano-node-chairman/cardano-node-chairman.cabal b/cardano-node-chairman/cardano-node-chairman.cabal index fe0eaccea9c..501e44cab1e 100644 --- a/cardano-node-chairman/cardano-node-chairman.cabal +++ b/cardano-node-chairman/cardano-node-chairman.cabal @@ -67,7 +67,7 @@ test-suite chairman-tests build-depends: , cardano-api , cardano-testnet - , cardano-crypto-class ^>=2.3 + , cardano-crypto-class ^>=2.5 , data-default-class , filepath , hedgehog diff --git a/cardano-node/cardano-node.cabal b/cardano-node/cardano-node.cabal index 7caad0b2d54..a4c2c092439 100644 --- a/cardano-node/cardano-node.cabal +++ b/cardano-node/cardano-node.cabal @@ -125,7 +125,7 @@ library , bytestring , cardano-api ^>= 11.3 , cardano-data - , cardano-crypto-class ^>=2.3 + , cardano-crypto-class ^>=2.5 , cardano-crypto-wrapper , cardano-git-rev ^>=0.2.2 , cardano-ledger-alonzo @@ -144,18 +144,19 @@ library , cardano-rpc ^>= 11.0 , cborg ^>= 0.2.4 , containers - , contra-tracer + , contra-tracer >= 0.2.1 , data-default-class , deepseq , directory , dns , ekg-core , filepath + , fs-api , generic-data , hashable , hostname , io-classes:{io-classes,strict-stm,si-timers} ^>= 1.8 - , kes-agent ^>=1.2 + , kes-agent ^>=1.3 , microlens , mmap , network-mux @@ -165,7 +166,7 @@ library , nothunks , optparse-applicative , ouroboros-consensus:{ouroboros-consensus, lsm, cardano, diffusion, protocol} ^>= 3.0.1 - , ouroboros-network:{api, ouroboros-network, orphan-instances, framework, protocols, framework-tracing, tracing} ^>= 1.1 + , ouroboros-network:{api, ouroboros-network, orphan-instances, framework, protocols, tracing} ^>= 1.1 , cardano-diffusion:{api, cardano-diffusion, tracing, orphan-instances} ^>=1.0 , prettyprinter , prettyprinter-ansi-terminal diff --git a/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs b/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs index 2b69f695a5a..276c27aadaa 100644 --- a/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs +++ b/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs @@ -88,4 +88,4 @@ selectorToArgs :: , CanUpgradeLedgerTables LedgerState blk ) => LedgerDbSelectorFlag -> FilePath -> StdGen -> (LedgerDbBackendArgs IO blk, StdGen) selectorToArgs V2InMemory _ = InMemory.mkInMemoryArgs -selectorToArgs (V2LSM fp) fastStoragePath = LSM.mkLSMArgsIO (Proxy @blk) (fromMaybe "lsm" fp) fastStoragePath +selectorToArgs (V2LSM fp) fastStoragePath = LSM.mkLSMArgsIO (Proxy @blk) (fromMaybe "lsm" fp) Nothing fastStoragePath diff --git a/cardano-node/src/Cardano/Node/Protocol/Cardano.hs b/cardano-node/src/Cardano/Node/Protocol/Cardano.hs index 9e5598b6b50..0904cd4ff50 100644 --- a/cardano-node/src/Cardano/Node/Protocol/Cardano.hs +++ b/cardano-node/src/Cardano/Node/Protocol/Cardano.hs @@ -39,6 +39,10 @@ import Ouroboros.Consensus.HardFork.Combinator.Condense () import Prelude import Data.Function ((&)) +import System.FilePath (takeDirectory) +import System.FS.API (SomeHasFS (..)) +import System.FS.API.Types (MountPoint (MountPoint)) +import System.FS.IO (ioHasFS) ------------------------------------------------------------------------------ -- Real Cardano protocol @@ -147,8 +151,12 @@ mkSomeConsensusProtocolCardano NodeByronProtocolConfiguration { firstExceptT CardanoProtocolInstantiationCheckpointsReadError $ readCheckpointsMap checkpointsConfiguration + -- Filesystem rooted at the Shelley genesis directory, used by the ledger to + -- read initial funds/staking injected from genesis (testnets only). + let shelleyGenesisFS = SomeHasFS $ ioHasFS $ MountPoint $ takeDirectory $ unGenesisFile npcShelleyGenesisFile + return $! - SomeConsensusProtocol CardanoBlockType $ ProtocolInfoArgsCardano $ Consensus.CardanoProtocolParams { + SomeConsensusProtocol CardanoBlockType $ ProtocolInfoArgsCardano shelleyGenesisFS $ Consensus.CardanoProtocolParams { Consensus.byronProtocolParams = Consensus.ProtocolParamsByron { byronGenesis = byronGenesis, diff --git a/cardano-node/src/Cardano/Node/Protocol/Conway.hs b/cardano-node/src/Cardano/Node/Protocol/Conway.hs index ef75e1c0c49..c64f5981a52 100644 --- a/cardano-node/src/Cardano/Node/Protocol/Conway.hs +++ b/cardano-node/src/Cardano/Node/Protocol/Conway.hs @@ -98,6 +98,7 @@ emptyConwayGenesis cm = , cgCommittee = DefaultClass.def , cgDelegs = mempty , cgInitialDReps = mempty + , cgExtraConfig = SNothing } diff --git a/cardano-node/src/Cardano/Node/Protocol/Shelley.hs b/cardano-node/src/Cardano/Node/Protocol/Shelley.hs index a816b8e91ed..a8b930ae882 100644 --- a/cardano-node/src/Cardano/Node/Protocol/Shelley.hs +++ b/cardano-node/src/Cardano/Node/Protocol/Shelley.hs @@ -53,6 +53,10 @@ import qualified Data.Aeson as Aeson import qualified Data.ByteString as BS import qualified Data.Text as T import System.Directory (getFileSize) +import System.FS.API (SomeHasFS (..)) +import System.FS.API.Types (MountPoint (MountPoint)) +import System.FS.IO (ioHasFS) +import System.FilePath (takeDirectory) import qualified System.IO.MMap as MMap @@ -82,7 +86,12 @@ mkSomeConsensusProtocolShelley NodeShelleyProtocolConfiguration { leaderCredentials <- firstExceptT PraosLeaderCredentialsError $ readLeaderCredentials files + -- Filesystem rooted at the Shelley genesis directory, used by the ledger to + -- read initial funds/staking injected from genesis (testnets only). + let shelleyGenesisFS = SomeHasFS $ ioHasFS $ MountPoint $ takeDirectory $ unGenesisFile npcShelleyGenesisFile + return $ SomeConsensusProtocol Api.ShelleyBlockType $ Api.ProtocolInfoArgsShelley + shelleyGenesisFS genesis Consensus.ProtocolParamsShelleyBased { shelleyBasedInitialNonce = genesisHashToPraosNonce genesisHash, diff --git a/cardano-node/src/Cardano/Node/Protocol/Types.hs b/cardano-node/src/Cardano/Node/Protocol/Types.hs index 1bf0d8860ca..42fa4d46ef2 100644 --- a/cardano-node/src/Cardano/Node/Protocol/Types.hs +++ b/cardano-node/src/Cardano/Node/Protocol/Types.hs @@ -50,5 +50,5 @@ data SomeConsensusProtocol where , Api.FromCBOR (HeaderHash blk) ) => Api.BlockType blk - -> Api.ProtocolInfoArgs blk + -> Api.ProtocolInfoArgs IO blk -> SomeConsensusProtocol diff --git a/cardano-node/src/Cardano/Node/Run.hs b/cardano-node/src/Cardano/Node/Run.hs index 10c77266977..479a177b734 100644 --- a/cardano-node/src/Cardano/Node/Run.hs +++ b/cardano-node/src/Cardano/Node/Run.hs @@ -238,15 +238,15 @@ handleNodeWithTracers -> SomeConsensusProtocol -> IO () handleNodeWithTracers cmdPc nc p@(SomeConsensusProtocol blockType runP) = do - let ProtocolInfo{pInfoConfig} = fst $ Api.protocolInfo @IO runP - networkMagic :: Api.NetworkMagic = getNetworkMagic $ Consensus.configBlock pInfoConfig + (ProtocolInfo{pInfoConfig}, mkBlockForging) <- Api.protocolInfo @IO runP + let networkMagic :: Api.NetworkMagic = getNetworkMagic $ Consensus.configBlock pInfoConfig -- This IORef contains node kernel structure which holds node kernel. -- Used for ledger queries and peer connection status. nodeKernelData <- mkNodeKernelData let fp = maybe "No file path found!" unConfigPath (getLast (pncConfigFile cmdPc)) - blockForging <- snd (Api.protocolInfo runP) nullTracer + blockForging <- mkBlockForging nullTracer tracers <- initTraceDispatcher nc @@ -302,7 +302,7 @@ handleSimpleNode ( Api.Protocol IO blk ) => Api.BlockType blk - -> Api.ProtocolInfoArgs blk + -> Api.ProtocolInfoArgs IO blk -> Tracers RemoteAddress LocalAddress blk IO -> NodeConfiguration -> NetworkMagic @@ -323,7 +323,7 @@ handleSimpleNode blockType runP tracers nc networkMagic onKernel = do traceWith (startupTracer tracers) StartupDBValidation - let pInfo = fst $ Api.protocolInfo @IO runP + pInfo <- fst <$> Api.protocolInfo @IO runP (publicIPv4SocketOrAddr, publicIPv6SocketOrAddr, localSocketOrPath) <- do result <- runExceptT (gatherConfiguredSockets $ ncSocketConfig nc) @@ -403,7 +403,8 @@ handleSimpleNode blockType runP tracers nc networkMagic onKernel = do } , rnNodeKernelHook = \registry nodeKernel -> do -- set the initial block forging - blockForging <- snd (Api.protocolInfo runP) (Consensus.kesAgentTracer $ consensusTracers tracers) + (_, mkBlockForging) <- Api.protocolInfo runP + blockForging <- mkBlockForging (Consensus.kesAgentTracer $ consensusTracers tracers) unless (ncStartAsNonProducingNode nc) $ setBlockForging nodeKernel blockForging @@ -634,7 +635,8 @@ updateBlockForging startupTracer kesAgentTracer blockType nodeKernel nc = do case Api.reflBlockType blockType blockType' of Just Refl -> do -- TODO: check if runP' has changed - blockForging <- snd (Api.protocolInfo runP') kesAgentTracer + (_, mkBlockForging) <- Api.protocolInfo runP' + blockForging <- mkBlockForging kesAgentTracer traceWith startupTracer (BlockForgingUpdate (if null blockForging then DisabledBlockForging diff --git a/cardano-node/src/Cardano/Node/Startup.hs b/cardano-node/src/Cardano/Node/Startup.hs index af9b9edee9f..a876e4bc919 100644 --- a/cardano-node/src/Cardano/Node/Startup.hs +++ b/cardano-node/src/Cardano/Node/Startup.hs @@ -211,6 +211,27 @@ prepareNodeInfo -> IO NodeInfo prepareNodeInfo nc (SomeConsensusProtocol whichP pForInfo) tc nodeStartTime = do nodeName <- prepareNodeName + cfg <- pInfoConfig . fst <$> Api.protocolInfo @IO pForInfo + let getSystemStartByron = WCT.getSystemStart . getSystemStart . configBlock $ cfg + systemStartTime :: UTCTime + systemStartTime = + case whichP of + Api.ByronBlockType -> + getSystemStartByron + Api.ShelleyBlockType -> + let DegenLedgerConfig cfgShelley = configLedger cfg + in getSystemStartShelley cfgShelley + Api.CardanoBlockType -> + let CardanoLedgerConfig _ cfgShelley cfgAllegra cfgMary cfgAlonzo cfgBabbage cfgConway cfgDijkstra = configLedger cfg + in minimum [ getSystemStartByron + , getSystemStartShelley cfgShelley + , getSystemStartShelley cfgAllegra + , getSystemStartShelley cfgMary + , getSystemStartShelley cfgAlonzo + , getSystemStartShelley cfgBabbage + , getSystemStartShelley cfgConway + , getSystemStartShelley cfgDijkstra + ] return $ NodeInfo { niName = nodeName , niProtocol = pack . show . ncProtocol $ nc @@ -220,29 +241,6 @@ prepareNodeInfo nc (SomeConsensusProtocol whichP pForInfo) tc nodeStartTime = do , niSystemStartTime = systemStartTime } where - cfg = pInfoConfig $ fst $ Api.protocolInfo @IO pForInfo - - systemStartTime :: UTCTime - systemStartTime = - case whichP of - Api.ByronBlockType -> - getSystemStartByron - Api.ShelleyBlockType -> - let DegenLedgerConfig cfgShelley = configLedger cfg - in getSystemStartShelley cfgShelley - Api.CardanoBlockType -> - let CardanoLedgerConfig _ cfgShelley cfgAllegra cfgMary cfgAlonzo cfgBabbage cfgConway cfgDijkstra = configLedger cfg - in minimum [ getSystemStartByron - , getSystemStartShelley cfgShelley - , getSystemStartShelley cfgAllegra - , getSystemStartShelley cfgMary - , getSystemStartShelley cfgAlonzo - , getSystemStartShelley cfgBabbage - , getSystemStartShelley cfgConway - , getSystemStartShelley cfgDijkstra - ] - - getSystemStartByron = WCT.getSystemStart . getSystemStart . configBlock $ cfg getSystemStartShelley = sgSystemStart . shelleyLedgerGenesis . shelleyLedgerConfig prepareNodeName = diff --git a/cardano-node/src/Cardano/Node/Tracing/Consistency.hs b/cardano-node/src/Cardano/Node/Tracing/Consistency.hs index b611c07fc90..addffca92e7 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Consistency.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Consistency.hs @@ -93,7 +93,6 @@ import qualified Ouroboros.Network.Protocol.LocalTxSubmission.Type as LTS import Ouroboros.Network.Protocol.TxSubmission2.Type (TxSubmission2) import qualified Ouroboros.Network.Server as Server (Trace (..)) import Ouroboros.Network.Snocket (LocalAddress (..)) -import Ouroboros.Network.Tracing.PeerSelection () import Ouroboros.Network.TxSubmission.Inbound.V2 (TraceTxSubmissionInbound) import Ouroboros.Network.TxSubmission.Outbound (TraceTxSubmissionOutbound) diff --git a/cardano-node/src/Cardano/Node/Tracing/Documentation.hs b/cardano-node/src/Cardano/Node/Tracing/Documentation.hs index e5b0a998ad6..b51824643c4 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Documentation.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Documentation.hs @@ -20,9 +20,6 @@ module Cardano.Node.Tracing.Documentation , docTracersFirstPhase ) where -import Ouroboros.Network.Tracing.TxSubmission.Inbound () -import Ouroboros.Network.Tracing.TxSubmission.Outbound () -import Ouroboros.Network.Tracing.PeerSelection () import Cardano.Network.Tracing.PeerSelection () import Cardano.Network.Tracing.PeerSelectionCounters () import Cardano.Git.Rev (gitRev) diff --git a/cardano-node/src/Cardano/Node/Tracing/Era/Shelley.hs b/cardano-node/src/Cardano/Node/Tracing/Era/Shelley.hs index b048c027c0a..f9d0efe92f9 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Era/Shelley.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Era/Shelley.hs @@ -54,7 +54,6 @@ import Cardano.Protocol.TPraos.Rules.Overlay import Cardano.Protocol.TPraos.Rules.Prtcl (PrtclPredicateFailure (OverlayFailure, UpdnFailure), PrtlSeqFailure (WrongBlockNoPrtclSeq, WrongBlockSequencePrtclSeq, WrongSlotIntervalPrtclSeq)) -import Cardano.Protocol.TPraos.Rules.Tickn (TicknPredicateFailure) import Cardano.Protocol.TPraos.Rules.Updn (UpdnPredicateFailure) import Cardano.Slotting.Block (BlockNo (..)) import Ouroboros.Consensus.Ledger.SupportsMempool (txId) @@ -64,11 +63,14 @@ import Ouroboros.Consensus.Protocol.TPraos (TPraosCannotForge (..)) import Ouroboros.Consensus.Shelley.Ledger hiding (TxId) import qualified Ouroboros.Consensus.Shelley.Ledger as Consensus import Ouroboros.Consensus.Shelley.Ledger.Inspect -import qualified Ouroboros.Consensus.Shelley.Protocol.Praos as Praos +import qualified Ouroboros.Consensus.Shelley.Protocol.EnvelopeChecks as Praos + (EnvelopeError (..)) import Ouroboros.Consensus.Util.Condense (condense) import Ouroboros.Network.Block (SlotNo (..), blockHash, blockNo, blockSlot) import Ouroboros.Network.Point (WithOrigin, withOriginToMaybe) +import Control.DeepSeq (NFData) + import Data.Aeson (ToJSON (..), Value (..), (.=)) import qualified Data.Aeson.Key as Aeson (fromText) import qualified Data.Aeson.Types as Aeson @@ -252,6 +254,7 @@ instance , LogFormatting (PredicateFailure (ShelleyUTXO era)) , LogFormatting (PredicateFailure (ShelleyUTXOW era)) , LogFormatting (PredicateFailure (Ledger.EraRule "BBODY" era)) + , NFData (PredicateFailure (Ledger.EraRule "BBODY" era)) ) => LogFormatting (BlockTransitionError era) where forMachine dtal (BlockTransitionError fs) = mconcat [ "kind" .= String "BlockTransitionError" @@ -767,10 +770,6 @@ instance LogFormatting (ShelleyPoolPredFailure era) where ] -instance LogFormatting TicknPredicateFailure where - forMachine _dtal x = case x of {} -- no constructors - - instance ( Ledger.Crypto crypto ) => LogFormatting (PrtclPredicateFailure crypto) where @@ -1325,7 +1324,7 @@ instance LogFormatting (Praos.PraosCannotForge crypto) where , "opCertStartingKesPeriod" .= kesPeriodValue startingKesPeriod ] -instance LogFormatting Praos.PraosEnvelopeError where +instance LogFormatting Praos.EnvelopeError where forMachine _ err' = case err' of Praos.ObsoleteNode maxPtclVersionFromPparams blkHeaderPtclVersion -> diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers.hs index e8f7e14cb79..e0d88261a9d 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers.hs @@ -60,7 +60,7 @@ import qualified Ouroboros.Network.Diffusion as Diffusion import Codec.CBOR.Read (DeserialiseFailure) import Control.Monad (unless) -import "contra-tracer" Control.Tracer (Tracer (..)) +import "contra-tracer" Control.Tracer (mkTracer) import Cardano.Network.OrphanInstances () import Data.Aeson (ToJSON (..)) import Data.Proxy (Proxy (..)) @@ -175,26 +175,26 @@ mkDispatchTracers nodeKernel trBase trForward mbTrEKG trDataPoint trConfig p = d pure Tracers { - chainDBTracer = Tracer (traceWith chainDBTr') - <> Tracer (traceWith replayBlockTr') - <> Tracer (SR.traceNodeStateChainDB p nodeStateDP) + chainDBTracer = mkTracer (traceWith chainDBTr') + <> mkTracer (traceWith replayBlockTr') + <> mkTracer (SR.traceNodeStateChainDB p nodeStateDP) , consensusTracers = consensusTr - , churnModeTracer = Tracer (traceWith churnModeTr) + , churnModeTracer = mkTracer (traceWith churnModeTr) , nodeToClientTracers = nodeToClientTr , nodeToNodeTracers = nodeToNodeTr , diffusionTracers = diffusionTr - , startupTracer = Tracer (traceWith startupTr) - <> Tracer (SR.traceNodeStateStartup nodeStateDP) - , shutdownTracer = Tracer (traceWith shutdownTr) - <> Tracer (SR.traceNodeStateShutdown nodeStateDP) - , nodeInfoTracer = Tracer (traceWith nodeInfoDP) - , nodeStartupInfoTracer = Tracer (traceWith nodeStartupInfoDP) - , nodeStateTracer = Tracer (traceWith stateTr) - <> Tracer (traceWith nodeStateDP) - , nodeVersionTracer = Tracer (traceWith nodeVersionTr) - , resourcesTracer = Tracer (traceWith resourcesTr) - , ledgerMetricsTracer = Tracer (traceWith ledgerMetricsTr) - , rpcTracer = Tracer (traceWith rpcTr) + , startupTracer = mkTracer (traceWith startupTr) + <> mkTracer (SR.traceNodeStateStartup nodeStateDP) + , shutdownTracer = mkTracer (traceWith shutdownTr) + <> mkTracer (SR.traceNodeStateShutdown nodeStateDP) + , nodeInfoTracer = mkTracer (traceWith nodeInfoDP) + , nodeStartupInfoTracer = mkTracer (traceWith nodeStartupInfoDP) + , nodeStateTracer = mkTracer (traceWith stateTr) + <> mkTracer (traceWith nodeStateDP) + , nodeVersionTracer = mkTracer (traceWith nodeVersionTr) + , resourcesTracer = mkTracer (traceWith resourcesTr) + , ledgerMetricsTracer = mkTracer (traceWith ledgerMetricsTr) + , rpcTracer = mkTracer (traceWith rpcTr) } mkConsensusTracers :: forall blk. @@ -364,61 +364,61 @@ mkConsensusTracers configReflection trBase trForward mbTrEKG _trDataPoint trConf configureTracers configReflection trConfig [txCountersTracer] pure $ Consensus.Tracers - { Consensus.chainSyncClientTracer = Tracer $ + { Consensus.chainSyncClientTracer = mkTracer $ traceWith chainSyncClientTr - , Consensus.chainSyncServerHeaderTracer = Tracer $ + , Consensus.chainSyncServerHeaderTracer = mkTracer $ traceWith chainSyncServerHeaderTr <> traceWith chainSyncServerHeaderMetricsTr - , Consensus.chainSyncServerBlockTracer = Tracer $ + , Consensus.chainSyncServerBlockTracer = mkTracer $ traceWith chainSyncServerBlockTr - , Consensus.consensusSanityCheckTracer = Tracer $ + , Consensus.consensusSanityCheckTracer = mkTracer $ traceWith consensusSanityCheckTr - , Consensus.blockFetchDecisionTracer = Tracer $ + , Consensus.blockFetchDecisionTracer = mkTracer $ traceWith blockFetchDecisionTr - , Consensus.blockFetchClientTracer = Tracer $ + , Consensus.blockFetchClientTracer = mkTracer $ traceWith blockFetchClientTr <> traceWith blockFetchClientMetricsTr - , Consensus.blockFetchServerTracer = Tracer $ + , Consensus.blockFetchServerTracer = mkTracer $ traceWith blockFetchServerTr <> traceWith servedBlockLatestTr - , Consensus.forgeStateInfoTracer = Tracer $ + , Consensus.forgeStateInfoTracer = mkTracer $ traceWith (traceAsKESInfo (Proxy @blk) forgeKESInfoTr) - , Consensus.gddTracer = Tracer $ + , Consensus.gddTracer = mkTracer $ traceWith consensusGddTr - , Consensus.txInboundTracer = Tracer $ + , Consensus.txInboundTracer = mkTracer $ traceWith txInboundTr - , Consensus.txOutboundTracer = Tracer $ + , Consensus.txOutboundTracer = mkTracer $ traceWith txOutboundTr - , Consensus.localTxSubmissionServerTracer = Tracer $ + , Consensus.localTxSubmissionServerTracer = mkTracer $ traceWith localTxSubmissionServerTr - , Consensus.mempoolTracer = Tracer $ + , Consensus.mempoolTracer = mkTracer $ traceWith mempoolTr , Consensus.forgeTracer = - Tracer (\(Consensus.TraceLabelCreds _ x) -> traceWith forgeTr x) + mkTracer (\(Consensus.TraceLabelCreds _ x) -> traceWith forgeTr x) <> - Tracer (\(Consensus.TraceLabelCreds _ x) -> traceWith forgeStatsTr x) - , Consensus.blockchainTimeTracer = Tracer $ + mkTracer (\(Consensus.TraceLabelCreds _ x) -> traceWith forgeStatsTr x) + , Consensus.blockchainTimeTracer = mkTracer $ traceWith blockchainTimeTr - , Consensus.keepAliveClientTracer = Tracer $ + , Consensus.keepAliveClientTracer = mkTracer $ traceWith keepAliveClientTr - , Consensus.consensusErrorTracer = Tracer $ + , Consensus.consensusErrorTracer = mkTracer $ traceWith consensusStartupErrorTr . ConsensusStartupException - , Consensus.gsmTracer = Tracer $ + , Consensus.gsmTracer = mkTracer $ traceWith consensusGsmTr - , Consensus.csjTracer = Tracer $ + , Consensus.csjTracer = mkTracer $ traceWith consensusCsjTr - , Consensus.dbfTracer = Tracer $ + , Consensus.dbfTracer = mkTracer $ traceWith consensusDbfTr - , Consensus.kesAgentTracer = Tracer $ + , Consensus.kesAgentTracer = mkTracer $ traceWith consensusKesAgentTr - , Consensus.txLogicTracer = Tracer $ + , Consensus.txLogicTracer = mkTracer $ traceWith txLogicTracer - , Consensus.txCountersTracer = Tracer $ + , Consensus.txCountersTracer = mkTracer $ traceWith txCountersTracer - , Consensus.perasCertDiffusionInboundTracer = Tracer $ traceWith txPerasCertIn - , Consensus.perasCertDiffusionOutboundTracer = Tracer $ traceWith txPerasCertOut - , Consensus.perasVoteDiffusionInboundTracer = Tracer $ traceWith txPerasVoteIn - , Consensus.perasVoteDiffusionOutboundTracer = Tracer $ traceWith txPerasVoteOut + , Consensus.perasCertDiffusionInboundTracer = mkTracer $ traceWith txPerasCertIn + , Consensus.perasCertDiffusionOutboundTracer = mkTracer $ traceWith txPerasCertOut + , Consensus.perasVoteDiffusionInboundTracer = mkTracer $ traceWith txPerasVoteIn + , Consensus.perasVoteDiffusionOutboundTracer = mkTracer $ traceWith txPerasVoteOut } mkNodeToClientTracers :: forall blk. @@ -456,13 +456,13 @@ mkNodeToClientTracers configReflection trBase trForward mbTrEKG _trDataPoint trC configureTracers configReflection trConfig [stateQueryTr] pure $ NtC.Tracers - { NtC.tChainSyncTracer = Tracer $ + { NtC.tChainSyncTracer = mkTracer $ traceWith chainSyncTr - , NtC.tTxMonitorTracer = Tracer $ + , NtC.tTxMonitorTracer = mkTracer $ traceWith txMonitorTr - , NtC.tTxSubmissionTracer = Tracer $ + , NtC.tTxSubmissionTracer = mkTracer $ traceWith txSubmissionTr - , NtC.tStateQueryTracer = Tracer $ + , NtC.tStateQueryTracer = mkTracer $ traceWith stateQueryTr } @@ -523,25 +523,25 @@ mkNodeToNodeTracers configReflection trBase trForward mbTrEKG _trDataPoint trCon configureTracers configReflection trConfig [txLogicTracer] pure $ NtN.Tracers - { NtN.tChainSyncTracer = Tracer $ + { NtN.tChainSyncTracer = mkTracer $ traceWith chainSyncTracer - , NtN.tChainSyncSerialisedTracer = Tracer $ + , NtN.tChainSyncSerialisedTracer = mkTracer $ traceWith chainSyncSerialisedTr - , NtN.tBlockFetchTracer = Tracer $ + , NtN.tBlockFetchTracer = mkTracer $ traceWith blockFetchTr - , NtN.tBlockFetchSerialisedTracer = Tracer $ + , NtN.tBlockFetchSerialisedTracer = mkTracer $ traceWith blockFetchSerialisedTr - , NtN.tTxSubmission2Tracer = Tracer $ + , NtN.tTxSubmission2Tracer = mkTracer $ traceWith txSubmission2Tracer - , NtN.tKeepAliveTracer = Tracer $ + , NtN.tKeepAliveTracer = mkTracer $ traceWith keepAliveTracer - , NtN.tPeerSharingTracer = Tracer $ + , NtN.tPeerSharingTracer = mkTracer $ traceWith peerSharingTracer - , NtN.tTxLogicTracer = Tracer $ + , NtN.tTxLogicTracer = mkTracer $ traceWith txLogicTracer - , NtN.tPerasCertDiffusionTracer = Tracer $ + , NtN.tPerasCertDiffusionTracer = mkTracer $ traceWith txPerasCertDiffusion - , NtN.tPerasVoteDiffusionTracer = Tracer $ + , NtN.tPerasVoteDiffusionTracer = mkTracer $ traceWith txPerasVoteDiffusion } @@ -687,54 +687,54 @@ mkDiffusionTracers configReflection trBase trForward mbTrEKG _trDataPoint trConf configureTracers configReflection trConfig [dtDnsTr] pure $ Diffusion.Tracers - { Diffusion.dtMuxTracer = Tracer $ + { Diffusion.dtMuxTracer = mkTracer $ traceWith dtMuxTr - , Diffusion.dtChannelTracer = Tracer $ + , Diffusion.dtChannelTracer = mkTracer $ traceWith dtChannelTracer - , Diffusion.dtBearerTracer = Tracer $ + , Diffusion.dtBearerTracer = mkTracer $ traceWith dtBearerTracer - , Diffusion.dtHandshakeTracer = Tracer $ + , Diffusion.dtHandshakeTracer = mkTracer $ traceWith dtHandshakeTracer - , Diffusion.dtLocalMuxTracer = Tracer $ + , Diffusion.dtLocalMuxTracer = mkTracer $ traceWith dtLocalMuxTr - , Diffusion.dtLocalChannelTracer = Tracer $ + , Diffusion.dtLocalChannelTracer = mkTracer $ traceWith dtLocalChannelTracer - , Diffusion.dtLocalBearerTracer = Tracer $ + , Diffusion.dtLocalBearerTracer = mkTracer $ traceWith dtLocalBearerTracer - , Diffusion.dtLocalHandshakeTracer = Tracer $ + , Diffusion.dtLocalHandshakeTracer = mkTracer $ traceWith dtLocalHandshakeTracer - , Diffusion.dtDiffusionTracer = Tracer $ + , Diffusion.dtDiffusionTracer = mkTracer $ traceWith dtDiffusionInitializationTr - , Diffusion.dtTraceLocalRootPeersTracer = Tracer $ + , Diffusion.dtTraceLocalRootPeersTracer = mkTracer $ traceWith localRootPeersTr - , Diffusion.dtTracePublicRootPeersTracer = Tracer $ + , Diffusion.dtTracePublicRootPeersTracer = mkTracer $ traceWith publicRootPeersTr - , Diffusion.dtTracePeerSelectionTracer = Tracer $ + , Diffusion.dtTracePeerSelectionTracer = mkTracer $ traceWith peerSelectionTr - , Diffusion.dtDebugPeerSelectionTracer = Tracer $ + , Diffusion.dtDebugPeerSelectionTracer = mkTracer $ traceWith debugPeerSelectionTr - , Diffusion.dtTracePeerSelectionCounters = Tracer $ + , Diffusion.dtTracePeerSelectionCounters = mkTracer $ traceWith peerSelectionCountersTr - , Diffusion.dtPeerSelectionActionsTracer = Tracer $ + , Diffusion.dtPeerSelectionActionsTracer = mkTracer $ traceWith peerSelectionActionsTr - , Diffusion.dtConnectionManagerTracer = Tracer $ + , Diffusion.dtConnectionManagerTracer = mkTracer $ traceWith connectionManagerTr - , Diffusion.dtConnectionManagerTransitionTracer = Tracer $ + , Diffusion.dtConnectionManagerTransitionTracer = mkTracer $ traceWith connectionManagerTransitionsTr - , Diffusion.dtServerTracer = Tracer $ + , Diffusion.dtServerTracer = mkTracer $ traceWith serverTr - , Diffusion.dtInboundGovernorTracer = Tracer $ + , Diffusion.dtInboundGovernorTracer = mkTracer $ traceWith inboundGovernorTr - , Diffusion.dtLocalInboundGovernorTracer = Tracer $ + , Diffusion.dtLocalInboundGovernorTracer = mkTracer $ traceWith localInboundGovernorTr - , Diffusion.dtInboundGovernorTransitionTracer = Tracer $ + , Diffusion.dtInboundGovernorTransitionTracer = mkTracer $ traceWith inboundGovernorTransitionsTr - , Diffusion.dtLocalConnectionManagerTracer = Tracer $ + , Diffusion.dtLocalConnectionManagerTracer = mkTracer $ traceWith localConnectionManagerTr - , Diffusion.dtLocalServerTracer = Tracer $ + , Diffusion.dtLocalServerTracer = mkTracer $ traceWith localServerTr - , Diffusion.dtTraceLedgerPeersTracer = Tracer $ + , Diffusion.dtTraceLedgerPeersTracer = mkTracer $ traceWith dtLedgerPeersTr - , Diffusion.dtDnsTracer = Tracer $ + , Diffusion.dtDnsTracer = mkTracer $ traceWith dtDnsTr } diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/Startup.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/Startup.hs index 7402b91814d..3b513420fd0 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/Startup.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/Startup.hs @@ -67,8 +67,8 @@ getStartupInfo -> IO [StartupTrace blk] getStartupInfo nc (SomeConsensusProtocol whichP pForInfo) fp = do nodeStartTime <- getCurrentTime - let cfg = pInfoConfig $ fst $ Api.protocolInfo @IO pForInfo - basicInfoCommon = BICommon $ BasicInfoCommon { + cfg <- pInfoConfig . fst <$> Api.protocolInfo @IO pForInfo + let basicInfoCommon = BICommon $ BasicInfoCommon { biProtocol = pack . show $ ncProtocol nc , biVersion = pack . showVersion $ version , biCommit = $(gitRev) diff --git a/cardano-submit-api/cardano-submit-api.cabal b/cardano-submit-api/cardano-submit-api.cabal index a7dbb612790..9d8f950df0d 100644 --- a/cardano-submit-api/cardano-submit-api.cabal +++ b/cardano-submit-api/cardano-submit-api.cabal @@ -42,13 +42,13 @@ library , cardano-api ^>= 11.3 , cardano-binary , cardano-cli ^>= 11.1 - , cardano-crypto-class ^>=2.3 + , cardano-crypto-class ^>=2.5 , containers , ekg-core , http-media , mtl , network - , optparse-applicative-fork + , optparse-applicative , ouroboros-consensus:cardano , ouroboros-network:{protocols} , prometheus >= 2.2.4 @@ -84,7 +84,7 @@ executable cardano-submit-api hs-source-dirs: app ghc-options: -threaded -rtsopts "-with-rtsopts=-T -I0" build-depends: base - , optparse-applicative-fork + , optparse-applicative , cardano-cli , cardano-crypto-class , cardano-git-rev ^>=0.2.2 @@ -99,4 +99,4 @@ test-suite unit main-is: test.hs hs-source-dirs: test build-depends: base - , cardano-crypto-class ^>=2.3 + , cardano-crypto-class ^>=2.5 diff --git a/cardano-testnet/cardano-testnet.cabal b/cardano-testnet/cardano-testnet.cabal index f939a8be8d1..0f4d1f2976e 100644 --- a/cardano-testnet/cardano-testnet.cabal +++ b/cardano-testnet/cardano-testnet.cabal @@ -43,7 +43,7 @@ library , bytestring , cardano-api ^>= 11.3 , cardano-cli:{cardano-cli, cardano-cli-test-lib} ^>= 11.1 - , cardano-crypto-class ^>=2.3 + , cardano-crypto-class ^>=2.5 , cardano-crypto-wrapper , cardano-git-rev ^>= 0.2.2 , cardano-ledger-alonzo @@ -57,13 +57,12 @@ library , cardano-ledger-dijkstra , cardano-ledger-shelley , cardano-node - , cardano-ping ^>= 0.10 + , cardano-diffusion:ping ^>= 1.0 , cardano-prelude , cardano-rpc , contra-tracer , containers , data-default-class - , cborg , containers , contra-tracer , data-default-class @@ -72,6 +71,7 @@ library , exceptions , extra , filepath + , fs-api ^>= 0.4 , hedgehog , hedgehog-extras ^>= 0.10 , http-conduit @@ -82,8 +82,7 @@ library , mono-traversable , mtl , network - , network-mux - , optparse-applicative-fork + , optparse-applicative , parsec , ouroboros-network:{api, framework, ouroboros-network} ^>= 1.1 , cardano-diffusion:{api, cardano-diffusion} ^>= 1.0 @@ -154,10 +153,10 @@ executable cardano-testnet main-is: cardano-testnet.hs - build-depends: cardano-crypto-class ^>=2.3 + build-depends: cardano-crypto-class ^>=2.5 , cardano-cli , cardano-testnet - , optparse-applicative-fork + , optparse-applicative ghc-options: -threaded -rtsopts "-with-rtsopts=-N -T" diff --git a/cardano-testnet/src/Testnet/Components/Query.hs b/cardano-testnet/src/Testnet/Components/Query.hs index 8f085bd662f..7b688fe17ca 100644 --- a/cardano-testnet/src/Testnet/Components/Query.hs +++ b/cardano-testnet/src/Testnet/Components/Query.hs @@ -80,6 +80,7 @@ import GHC.Exts (IsList (..)) import GHC.Stack import Lens.Micro (Lens', to, (^.)) +import Testnet.Filepath (mkNodeConfigFs) import Testnet.Process.RunIO (liftIOAnnotated) import Testnet.Property.Assert import Testnet.Runtime @@ -102,9 +103,11 @@ waitUntilEpoch -> EpochNo -- ^ Desired epoch -> m EpochNo -- ^ The epoch number reached waitUntilEpoch nodeConfigFile socketPath desiredEpoch = withFrozenCallStack $ do - result <- H.evalIO . runExceptT $ - foldEpochState - nodeConfigFile socketPath QuickValidation desiredEpoch () (\_ _ _ -> pure ConditionNotMet) + result <- H.evalIO $ do + fs <- mkNodeConfigFs nodeConfigFile + runExceptT $ + foldEpochState + fs nodeConfigFile socketPath QuickValidation desiredEpoch () (\_ _ _ -> pure ConditionNotMet) case result of Left (FoldBlocksApplyBlockError (TerminationEpochReached epochNo)) -> pure epochNo @@ -388,7 +391,8 @@ getEpochStateView getEpochStateView nodeConfigFile socketPath = withFrozenCallStack $ do esv <- H.evalIO $ EpochStateView <$> newTVarIO (Left EpochStateNotInitialised) <*> newTVarIO 0 _ <- asyncRegister_ $ do - result <- runExceptT $ foldEpochState nodeConfigFile socketPath QuickValidation (EpochNo maxBound) () + fs <- mkNodeConfigFs nodeConfigFile + result <- runExceptT $ foldEpochState fs nodeConfigFile socketPath QuickValidation (EpochNo maxBound) () $ \epochState slotNumber blockNumber -> do liftIOAnnotated . atomically $ writeEpochStateView esv $ Right (epochState, slotNumber, blockNumber) pure ConditionNotMet diff --git a/cardano-testnet/src/Testnet/Defaults.hs b/cardano-testnet/src/Testnet/Defaults.hs index e6e743fb51c..d3bd2f95b05 100644 --- a/cardano-testnet/src/Testnet/Defaults.hs +++ b/cardano-testnet/src/Testnet/Defaults.hs @@ -190,6 +190,7 @@ defaultConwayGenesis = do , cgCommittee = DefaultClass.def , cgDelegs = mempty , cgInitialDReps = mempty + , cgExtraConfig = SNothing } -- | The only era supported by cardano-testnet for the moment. diff --git a/cardano-testnet/src/Testnet/Filepath.hs b/cardano-testnet/src/Testnet/Filepath.hs index 59c5771ab56..ab87ce9c1e1 100644 --- a/cardano-testnet/src/Testnet/Filepath.hs +++ b/cardano-testnet/src/Testnet/Filepath.hs @@ -9,19 +9,27 @@ module Testnet.Filepath , makeSocketDir , makeSprocket , makeTmpBaseAbsPath + , mkNodeConfigFs ) where import Prelude import Data.String (IsString (..)) +import System.Directory (makeAbsolute) import System.FilePath import Hedgehog.Extras.Stock.IO.Network.Sprocket (Sprocket (..)) import RIO (Display (..)) +import Cardano.Api (File (..)) + import Cardano.Node.Testnet.Paths (defaultSocketDir) +import System.FS.API (SomeHasFS (..)) +import System.FS.API.Types (MountPoint (MountPoint)) +import System.FS.IO (ioHasFS) + makeSprocket :: TmpAbsolutePath @@ -51,3 +59,8 @@ makeTmpBaseAbsPath (TmpAbsolutePath fp) = addTrailingPathSeparator $ takeDirecto makeLogDir :: TmpAbsolutePath -> FilePath makeLogDir (TmpAbsolutePath fp) = addTrailingPathSeparator $ fp "logs" + +mkNodeConfigFs :: File content direction -> IO (SomeHasFS IO) +mkNodeConfigFs configFile = do + configDir <- takeDirectory <$> makeAbsolute (unFile configFile) + pure $ SomeHasFS (ioHasFS (MountPoint configDir)) diff --git a/cardano-testnet/src/Testnet/Ping.hs b/cardano-testnet/src/Testnet/Ping.hs index b2d2824de09..02df9d68830 100644 --- a/cardano-testnet/src/Testnet/Ping.hs +++ b/cardano-testnet/src/Testnet/Ping.hs @@ -1,9 +1,4 @@ -{-# LANGUAGE BangPatterns #-} -{-# LANGUAGE LambdaCase #-} {-# LANGUAGE NamedFieldPuns #-} -{-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE RankNTypes #-} -{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} module Testnet.Ping @@ -12,36 +7,22 @@ module Testnet.Ping , waitForSprocket , waitForPortClosed , TestnetMagic - , PingClientError(..) + , CNP.PingClientException ) where -import Cardano.Api (Error (..)) +import qualified Cardano.Network.Ping as CNP -import Cardano.Network.Ping (HandshakeFailure, NodeVersion (..), handshakeDec, - handshakeReq, isSameVersionAndMagic, supportedNodeToClientVersions) - -import qualified Codec.CBOR.Read as CBOR import Control.Exception.Safe -import Control.Monad -import Control.Monad.Class.MonadTime.SI (Time) +import Control.Monad (when) import qualified Control.Monad.Class.MonadTimer.SI as MT import Control.Monad.IO.Class import qualified Control.Retry as R import Control.Tracer (nullTracer) -import qualified Data.ByteString.Lazy as LBS import Data.Either import Data.IORef -import qualified Data.List as L import Data.Word (Word32) -import qualified Network.Mux as Mux -import Network.Mux.Bearer (MakeBearer (..), makeSocketBearer) -import Network.Mux.Timeout (TimeoutFn, withTimeoutSerial) -import Network.Mux.Types (MiniProtocolDir (InitiatorDir), MiniProtocolNum (..), - RemoteClockModel (RemoteClockModel), SDU (..), SDUHeader (..)) -import qualified Network.Mux.Types as Mux -import Network.Socket (AddrInfo (..), PortNumber, StructLinger (..)) +import Network.Socket (AddrInfo (..), PortNumber) import qualified Network.Socket as Socket -import Prettyprinter import Testnet.Process.RunIO (liftIOAnnotated) @@ -50,93 +31,25 @@ import qualified Hedgehog.Extras.Stock.IO.Network.Sprocket as IO type TestnetMagic = Word32 --- | Mini protocol number. We're only sending ping, so 0. -handshakeNum :: MiniProtocolNum -handshakeNum = MiniProtocolNum 0 - --- | Timeout for reading a multiplexer service data unit, in seconds. -sduTimeout :: MT.DiffTime -sduTimeout = 30 - --- | Perform handshake query to obtain supported version numbers by node. -doHandshakeQuery :: Bool -doHandshakeQuery = True - -- | Ping the node once pingNode :: MonadIO m => TestnetMagic -- ^ testnet magic -> IO.Sprocket -- ^ node sprocket - -> m (Either PingClientError ()) -- ^ '()' means success -pingNode networkMagic sprocket = liftIOAnnotated $ bracket - (Socket.socket (Socket.addrFamily peer) Socket.Stream Socket.defaultProtocol) - Socket.close - (\sd -> handle (pure . Left . PceException) $ withTimeoutSerial $ \timeoutfn -> do - when (Socket.addrFamily peer /= Socket.AF_UNIX) $ do - Socket.setSocketOption sd Socket.NoDelay 1 - Socket.setSockOpt sd Socket.Linger - StructLinger - { sl_onoff = 1 - , sl_linger = 0 - } - - Socket.connect sd (Socket.addrAddress peer) - peerStr <- peerString - - bearer <- getBearer makeSocketBearer sduTimeout sd Nothing - - let versions = supportedNodeToClientVersions networkMagic - !_ <- Mux.write bearer nullTracer timeoutfn $ wrap handshakeNum InitiatorDir (handshakeReq versions doHandshakeQuery) - (msg, !_) <- nextMsg bearer timeoutfn handshakeNum - - pure $ case CBOR.deserialiseFromBytes handshakeDec msg of - Left err -> Left $ PceDecodingError peerStr err - Right (_, Left err) -> Left $ PceProtocolError peerStr err - Right (_, Right recVersions) - | areVersionsAccepted versions recVersions -> pure () - | otherwise -> Left $ PceVersionNegotiationError peerStr versions recVersions - ) + -> m (Either CNP.PingClientException ()) -- ^ 'Right ()' means success +pingNode networkMagic sprocket = + liftIOAnnotated $ + CNP.pingClient nullTracer nullTracer (pingOpts networkMagic) (sprocketToAddrInfo sprocket) where - peer = sprocketToAddrInfo sprocket :: AddrInfo - - -- | Wrap a message in a mux service data unit. - wrap :: MiniProtocolNum -> MiniProtocolDir -> LBS.ByteString -> SDU - wrap mhNum mhDir msBlob = SDU - { msHeader = SDUHeader - { mhTimestamp = RemoteClockModel 0 - , mhNum - , mhDir - , mhLength = fromIntegral $ LBS.length msBlob - } - , msBlob + pingOpts magic = CNP.PingOpts + { CNP.pingOptsCount = 1 + , CNP.pingOptsMagic = CNP.NetworkMagic magic + , CNP.pingOptsJson = CNP.AsText + , CNP.pingOptsQuiet = True + , CNP.pingOptsMode = CNP.PingMode + , CNP.pingOptsSRVPrefix = "_cardano._tcp" + , CNP.pingOptsColor = CNP.ColorAuto } - areVersionsAccepted :: [NodeVersion] -> [NodeVersion] -> Bool - areVersionsAccepted accVersions recVersions = - let intersects = L.intersectBy isSameVersionAndMagic recVersions accVersions in - not $ null intersects - - peerString :: IO String - peerString = - case Socket.addrFamily peer of - Socket.AF_UNIX -> pure . show $ Socket.addrAddress peer - _ -> do - (Just host, Just port) <- - Socket.getNameInfo - [Socket.NI_NUMERICHOST, Socket.NI_NUMERICSERV] - True True (Socket.addrAddress peer) - pure $ host <> ":" <> port - - -- | Fetch next message from mux bearer. Ignores messages not matching handshake protocol number. - nextMsg :: Mux.Bearer IO -- ^ a mux bearer - -> TimeoutFn IO -- ^ timeout function, for reading messages - -> MiniProtocolNum -- ^ handshake protocol number - -> IO (LBS.ByteString, Time) -- ^ raw message and timestamp - nextMsg bearer timeoutfn ptclNum = do - (sdu, t_e) <- Mux.read bearer nullTracer timeoutfn - if mhNum (msHeader sdu) == ptclNum - then pure (msBlob sdu, t_e) - else nextMsg bearer timeoutfn ptclNum - -- | Wait for 'sprocket' to become ready. Periodically tries to connect to 'sprocket', with the provided interval. -- If there was no success within 'timeout' period, return the last exception thrown during a connection -- attempt. @@ -185,29 +98,3 @@ waitForPortClosed timeout interval portNumber = liftIOAnnotated $ do let retryPolicy = R.constantDelay (round @Double $ realToFrac interval) <> R.limitRetries (ceiling $ toRational timeout / toRational interval) fmap not . R.retrying retryPolicy (const pure) $ \_ -> liftIOAnnotated (IO.isPortOpen (fromIntegral portNumber)) - -data PingClientError - = PceDecodingError - !String -- ^ peer string - !CBOR.DeserialiseFailure -- ^ deserialization exception - | PceProtocolError - !String -- ^ peer string - !HandshakeFailure -- ^ handshake exception - | PceVersionNegotiationError - !String -- ^ peer string - ![NodeVersion] -- ^ requested versions - ![NodeVersion] -- ^ received node versions - | PceException - !SomeException - -instance Error PingClientError where - prettyError = \case - PceDecodingError peerStr exception -> pretty peerStr <+> "Decoding error:" <+> pretty (displayException exception) - PceProtocolError peerStr exception -> pretty peerStr <+> "Protocol error:" <+> viaShow exception - PceVersionNegotiationError peerStr requestedVersions receivedVersions -> vsep - [ pretty peerStr <+> "Version negotiation error: No overlapping versions with" <+> viaShow requestedVersions - , "Received versions:" <+> viaShow receivedVersions - ] - PceException exception -> "An unknown exception occurred:" <+> pretty (displayException exception) - - diff --git a/cardano-testnet/src/Testnet/Process/Cli/SPO.hs b/cardano-testnet/src/Testnet/Process/Cli/SPO.hs index 6b976ff5a17..65fe2375332 100644 --- a/cardano-testnet/src/Testnet/Process/Cli/SPO.hs +++ b/cardano-testnet/src/Testnet/Process/Cli/SPO.hs @@ -101,7 +101,9 @@ checkStakeKeyRegistered tempAbsP nodeConfigFile sPath terminationEpoch execConfi sAddr <- case deserialiseAddress AsStakeAddress $ Text.pack stakeAddr of Just sAddr -> return sAddr Nothing -> H.failWithCustom GHC.callStack Nothing $ "Invalid stake address: " <> stakeAddr + fs <- liftIO $ mkNodeConfigFs nodeConfigFile result <- runExceptT $ foldEpochState + fs nodeConfigFile sPath QuickValidation diff --git a/cardano-testnet/src/Testnet/Runtime.hs b/cardano-testnet/src/Testnet/Runtime.hs index e271f264d42..cb0dd06bdd5 100644 --- a/cardano-testnet/src/Testnet/Runtime.hs +++ b/cardano-testnet/src/Testnet/Runtime.hs @@ -490,8 +490,11 @@ startLedgerNewEpochStateLogging testnetRuntime tmpWorkspace = withFrozenCallStac Just (sprocket, _) -> H.sprocketSystemName sprocket Nothing -> throwString "No testnet sprocket available" + fs <- liftIOAnnotated $ mkNodeConfigFs (configurationFile testnetRuntime) + void $ asyncRegister_ . runExceptT $ foldEpochState + fs (configurationFile testnetRuntime) (Api.File socketPath) Api.QuickValidation diff --git a/cardano-testnet/src/Testnet/Start/Cardano.hs b/cardano-testnet/src/Testnet/Start/Cardano.hs index c4df60856cb..2cb2272536d 100644 --- a/cardano-testnet/src/Testnet/Start/Cardano.hs +++ b/cardano-testnet/src/Testnet/Start/Cardano.hs @@ -433,8 +433,10 @@ cardanoTestnet -> TestnetNode -> m () waitForBlockThrow timeoutSeconds nodeConfigFile node@TestnetNode{nodeName} = do + fs <- liftIO $ mkNodeConfigFs nodeConfigFile result <- timeout (timeoutSeconds * 1_000_000) $ runExceptT . foldEpochState + fs nodeConfigFile (nodeSocketPath node) QuickValidation diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/FoldEpochState.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/FoldEpochState.hs index 5968465b287..49ced0b4ce8 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/FoldEpochState.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/FoldEpochState.hs @@ -16,6 +16,7 @@ import Data.Default.Class import qualified System.Directory as IO import System.FilePath (()) +import Testnet.Filepath (mkNodeConfigFs) import Testnet.Property.Util (integrationRetryWorkspace) import Hedgehog ((===)) @@ -49,7 +50,9 @@ prop_foldEpochState = integrationRetryWorkspace 2 "foldEpochState" $ \tempAbsBas then pure ConditionMet else pure ConditionNotMet - (_, nums) <- H.leftFailM $ H.evalIO $ runExceptT $ - Api.foldEpochState configurationFile (Api.File socketPathAbs) Api.QuickValidation (EpochNo maxBound) [] handler + (_, nums) <- H.leftFailM $ H.evalIO $ do + fs <- mkNodeConfigFs configurationFile + runExceptT $ + Api.foldEpochState fs configurationFile (Api.File socketPathAbs) Api.QuickValidation (EpochNo maxBound) [] handler length nums === 10 diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/InfoAction.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/InfoAction.hs index b4d17b8703b..75b4548d405 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/InfoAction.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/InfoAction.hs @@ -34,6 +34,7 @@ import System.FilePath (()) import Test.Cardano.CLI.Hash (serveFilesWhile) import Testnet.Components.Query import Testnet.Defaults +import Testnet.Filepath (mkNodeConfigFs) import Testnet.Process.Cli.Keys import Testnet.Process.Cli.SPO (createStakeKeyRegistrationCertificate) import Testnet.Process.Cli.Transaction (retrieveTransactionId) @@ -246,8 +247,10 @@ hprop_ledger_events_info_action = integrationRetryWorkspace 2 "info-hash" $ \tem ] -- We check that info action was successfully ratified + fs <- evalIO $ mkNodeConfigFs configurationFile !meInfoRatified <- H.timeout 120_000_000 $ runExceptT $ foldBlocks + fs configurationFile socketPath FullValidation diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitution.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitution.hs index d1693a00c73..f8d517a1871 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitution.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitution.hs @@ -41,6 +41,7 @@ import Testnet.Components.Configuration import Testnet.Components.Query import Testnet.Defaults import Testnet.EpochStateProcessing (unsafeEraFromSbe, waitForGovActionVotes) +import Testnet.Filepath (mkNodeConfigFs) import Testnet.Process.Cli.DRep import Testnet.Process.Cli.Keys import Testnet.Process.Cli.SPO (createStakeKeyRegistrationCertificate) @@ -353,14 +354,17 @@ hprop_ledger_events_propose_new_constitution = integrationRetryWorkspace 2 "prop proposalsStakePoolVotes === mempty -- We check that constitution was successfully ratified - void . H.leftFailM . H.evalIO . runExceptT $ - foldEpochState - configurationFile - socketPath - FullValidation - (EpochNo 10) - () - (\epochState _ _ -> foldBlocksCheckConstitutionWasRatified constitutionHash constitutionScriptHash epochState) + void . H.leftFailM . H.evalIO $ do + fs <- mkNodeConfigFs configurationFile + runExceptT $ + foldEpochState + fs + configurationFile + socketPath + FullValidation + (EpochNo 10) + () + (\epochState _ _ -> foldBlocksCheckConstitutionWasRatified constitutionHash constitutionScriptHash epochState) foldBlocksCheckConstitutionWasRatified :: String -- submitted constitution hash diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitutionSPO.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitutionSPO.hs index 7add69da333..45b1572b5db 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitutionSPO.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitutionSPO.hs @@ -30,6 +30,7 @@ import System.FilePath (()) import Testnet.Components.Query import Testnet.Defaults import Testnet.EpochStateProcessing (unsafeEraFromSbe) +import Testnet.Filepath (mkNodeConfigFs) import Testnet.Process.Cli.DRep import Testnet.Process.Cli.Keys import qualified Testnet.Process.Cli.SPO as SPO @@ -182,7 +183,8 @@ getConstitutionProposal -> EpochNo -- ^ The termination epoch: the constitution proposal must be found *before* this epoch -> m (Maybe L.GovActionId) getConstitutionProposal nodeConfigFile socketPath maxEpoch = do - result <- H.evalIO . runExceptT $ foldEpochState nodeConfigFile socketPath QuickValidation maxEpoch Nothing + fs <- H.evalIO $ mkNodeConfigFs nodeConfigFile + result <- H.evalIO . runExceptT $ foldEpochState fs nodeConfigFile socketPath QuickValidation maxEpoch Nothing $ \(AnyNewEpochState actualEra newEpochState _) _slotNb _blockNb -> obtainCommonConstraints (unsafeEraFromSbe actualEra) $ do let proposals = newEpochState diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/TreasuryGrowth.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/TreasuryGrowth.hs index ae7507f9124..fbaede662e5 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/TreasuryGrowth.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/TreasuryGrowth.hs @@ -23,6 +23,7 @@ import Lens.Micro ((^.)) import qualified System.Directory as IO import System.FilePath (()) +import Testnet.Filepath (mkNodeConfigFs) import Testnet.Process.Run (execCli', mkExecConfig) import Testnet.Property.Util (integrationRetryWorkspace) import Testnet.Start.Types @@ -58,8 +59,10 @@ prop_check_if_treasury_is_growing = integrationRetryWorkspace 2 "growing-treasur execConfig <- mkExecConfig tempBaseAbsPath poolSprocket1 testnetMagic pure (execConfig, socketPathAbs) - (_condition, treasuryValues) <- H.leftFailM . H.evalIO . runExceptT $ - Api.foldEpochState configurationFile socketPathAbs Api.QuickValidation (EpochNo 10) M.empty handler + (_condition, treasuryValues) <- H.leftFailM . H.evalIO $ do + fs <- mkNodeConfigFs configurationFile + runExceptT $ + Api.foldEpochState fs configurationFile socketPathAbs Api.QuickValidation (EpochNo 10) M.empty handler H.note_ $ "treasury for last 5 epochs: " <> show treasuryValues let treasuriesSortedByEpoch = diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/TreasuryWithdrawal.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/TreasuryWithdrawal.hs index a4ae043d71b..ac94cb48ca3 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/TreasuryWithdrawal.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/TreasuryWithdrawal.hs @@ -41,6 +41,7 @@ import Test.Cardano.CLI.Hash (serveFilesWhile) import Testnet.Components.Query import Testnet.Defaults import Testnet.EpochStateProcessing (unsafeEraFromSbe) +import Testnet.Filepath (mkNodeConfigFs) import Testnet.Process.Cli.Keys (cliStakeAddressKeyGen) import Testnet.Process.Cli.SPO (createStakeKeyRegistrationCertificate) import Testnet.Process.Cli.Transaction (retrieveTransactionId) @@ -271,7 +272,8 @@ getAnyWithdrawals -> EpochNo -> m (Maybe (Map (Credential Staking) Coin)) getAnyWithdrawals nodeConfigFile socketPath maxEpoch = withFrozenCallStack $ do - fmap snd . H.leftFailM . evalIO . runExceptT $ foldEpochState nodeConfigFile socketPath FullValidation maxEpoch Nothing + fs <- evalIO $ mkNodeConfigFs nodeConfigFile + fmap snd . H.leftFailM . evalIO . runExceptT $ foldEpochState fs nodeConfigFile socketPath FullValidation maxEpoch Nothing $ \(AnyNewEpochState actualEra newEpochState _) _ _ -> obtainCommonConstraints (unsafeEraFromSbe actualEra) $ do let withdrawals = newEpochState @@ -296,7 +298,8 @@ getTreasuryWithdrawalProposal -> EpochNo -- ^ The termination epoch: the withdrawal proposal must be found *before* this epoch -> m (Maybe L.GovActionId) getTreasuryWithdrawalProposal nodeConfigFile socketPath maxEpoch = withFrozenCallStack $ do - fmap snd . H.leftFailM . evalIO . runExceptT $ foldEpochState nodeConfigFile socketPath QuickValidation maxEpoch Nothing + fs <- evalIO $ mkNodeConfigFs nodeConfigFile + fmap snd . H.leftFailM . evalIO . runExceptT $ foldEpochState fs nodeConfigFile socketPath QuickValidation maxEpoch Nothing $ \(AnyNewEpochState actualEra newEpochState _) _ _ -> obtainCommonConstraints (unsafeEraFromSbe actualEra) $ do let proposals = newEpochState diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/SanityCheck.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/SanityCheck.hs index 5beec8c1999..88f403e17dd 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/SanityCheck.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/SanityCheck.hs @@ -23,6 +23,7 @@ import Data.Time.Clock import GHC.Conc (ThreadStatus (..), threadStatus) import GHC.Stack +import Testnet.Filepath (mkNodeConfigFs) import Testnet.Property.Util (integrationRetryWorkspace) import Testnet.Runtime import Testnet.Start.Types @@ -67,8 +68,10 @@ hprop_ledger_events_sanity_check = integrationRetryWorkspace 2 "ledger-events-sa H.note_ $ "Abs path: " <> tempAbsBasePath' H.note_ $ "Socketpath: " <> unFile socketPath + fs <- evalIO $ mkNodeConfigFs configurationFile !ret <- runExceptT $ handleIOExceptionsWith IOE $ evalIO $ runExceptT $ foldBlocks + fs configurationFile socketPath FullValidation diff --git a/cardano-tracer/src/Cardano/Tracer/Acceptors/Client.hs b/cardano-tracer/src/Cardano/Tracer/Acceptors/Client.hs index 997fb031b57..6fc787bf5bc 100644 --- a/cardano-tracer/src/Cardano/Tracer/Acceptors/Client.hs +++ b/cardano-tracer/src/Cardano/Tracer/Acceptors/Client.hs @@ -20,7 +20,7 @@ import Ouroboros.Network.Mux (MiniProtocol (..), MiniProtocolLimits (. MiniProtocolNum (..), OuroborosApplication (..), OuroborosApplicationWithMinimalCtx, RunMiniProtocol (..), miniProtocolLimits, miniProtocolNum, miniProtocolRun) -import Ouroboros.Network.Protocol.Handshake.Codec (cborTermVersionDataCodec, +import Ouroboros.Network.Protocol.Handshake.Codec (mkVersionedCodecCBORTerm, codecHandshake, noTimeLimitsHandshake, timeLimitsHandshake) import Ouroboros.Network.Protocol.Handshake.Type (Handshake) import Ouroboros.Network.Protocol.Handshake.Version (acceptableVersion, queryVersion, @@ -144,7 +144,7 @@ doConnectToForwarderLocal snocket address netMagic timeLimits app = do args = ConnectToArgs { ctaHandshakeCodec = codecHandshake forwardingVersionCodec, ctaHandshakeTimeLimits = timeLimits, - ctaVersionDataCodec = cborTermVersionDataCodec forwardingCodecCBORTerm, + ctaVersionDataCodec = mkVersionedCodecCBORTerm forwardingCodecCBORTerm, ctaConnectTracers = nullNetworkConnectTracers, ctaHandshakeCallbacks = HandshakeCallbacks acceptableVersion queryVersion } @@ -178,7 +178,7 @@ doConnectToForwarderSocket snocket address netMagic timeLimits app = do args = ConnectToArgs { ctaHandshakeCodec = codecHandshake forwardingVersionCodec, ctaHandshakeTimeLimits = timeLimits, - ctaVersionDataCodec = cborTermVersionDataCodec forwardingCodecCBORTerm, + ctaVersionDataCodec = mkVersionedCodecCBORTerm forwardingCodecCBORTerm, ctaConnectTracers = nullNetworkConnectTracers, ctaHandshakeCallbacks = HandshakeCallbacks acceptableVersion queryVersion } diff --git a/cardano-tracer/src/Cardano/Tracer/Acceptors/Server.hs b/cardano-tracer/src/Cardano/Tracer/Acceptors/Server.hs index 18b2b373835..84f3d47a2c6 100644 --- a/cardano-tracer/src/Cardano/Tracer/Acceptors/Server.hs +++ b/cardano-tracer/src/Cardano/Tracer/Acceptors/Server.hs @@ -130,7 +130,7 @@ doListenToForwarderLocal snocket address netMagic timeLimits app = do haHandshakeTracer = nullTracer, haBearerTracer = nullTracer, haHandshakeCodec = Handshake.codecHandshake forwardingVersionCodec, - haVersionDataCodec = Handshake.cborTermVersionDataCodec forwardingCodecCBORTerm, + haVersionDataCodec = Handshake.mkVersionedCodecCBORTerm forwardingCodecCBORTerm, haAcceptVersion = Handshake.acceptableVersion, haQueryVersion = Handshake.queryVersion, haTimeLimits = timeLimits @@ -162,7 +162,7 @@ doListenToForwarderSocket snocket address netMagic timeLimits app = do haHandshakeTracer = nullTracer, haBearerTracer = nullTracer, haHandshakeCodec = Handshake.codecHandshake forwardingVersionCodec, - haVersionDataCodec = Handshake.cborTermVersionDataCodec forwardingCodecCBORTerm, + haVersionDataCodec = Handshake.mkVersionedCodecCBORTerm forwardingCodecCBORTerm, haAcceptVersion = Handshake.acceptableVersion, haQueryVersion = Handshake.queryVersion, haTimeLimits = timeLimits diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Forwarder.hs b/cardano-tracer/test/Cardano/Tracer/Test/Forwarder.hs index 90a277c8683..be93092b328 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/Forwarder.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/Forwarder.hs @@ -30,7 +30,7 @@ import Ouroboros.Network.Mux (MiniProtocol (..), MiniProtocolLimits (. miniProtocolLimits, miniProtocolNum, miniProtocolRun) import Ouroboros.Network.Protocol.Handshake (Handshake, HandshakeArguments (..)) import qualified Ouroboros.Network.Protocol.Handshake as Handshake -import Ouroboros.Network.Protocol.Handshake.Codec (cborTermVersionDataCodec, +import Ouroboros.Network.Protocol.Handshake.Codec (mkVersionedCodecCBORTerm, codecHandshake, noTimeLimitsHandshake) import qualified Ouroboros.Network.Server.Simple as Server import Ouroboros.Network.Snocket (MakeBearer, Snocket, localAddressFromPath, localSnocket, @@ -232,7 +232,7 @@ doConnectToAcceptor TestSetup{..} snocket muxBearer address timeLimits (ekgConfi args = ConnectToArgs { ctaHandshakeCodec = codecHandshake forwardingVersionCodec, ctaHandshakeTimeLimits = timeLimits, - ctaVersionDataCodec = cborTermVersionDataCodec forwardingCodecCBORTerm, + ctaVersionDataCodec = mkVersionedCodecCBORTerm forwardingCodecCBORTerm, ctaConnectTracers = nullNetworkConnectTracers, ctaHandshakeCallbacks = HandshakeCallbacks Handshake.acceptableVersion Handshake.queryVersion } @@ -281,7 +281,7 @@ doListenToAcceptor TestSetup{..} haHandshakeTracer = nullTracer, haBearerTracer = nullTracer, haHandshakeCodec = codecHandshake forwardingVersionCodec, - haVersionDataCodec = cborTermVersionDataCodec forwardingCodecCBORTerm, + haVersionDataCodec = mkVersionedCodecCBORTerm forwardingCodecCBORTerm, haAcceptVersion = Handshake.acceptableVersion, haQueryVersion = Handshake.queryVersion, haTimeLimits = timeLimits diff --git a/flake.lock b/flake.lock index 3be1a1293b2..2f38f596a20 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "CHaP": { "flake": false, "locked": { - "lastModified": 1779876270, - "narHash": "sha256-FA9E1EaQvPITpO/8weQyi7p3KHgyNb9GiwM6F96Aoeo=", + "lastModified": 1781102605, + "narHash": "sha256-lipiKNOJ/NIGO/pcwQT030+sSgbTkP3N0w/ck5jBJlw=", "owner": "intersectmbo", "repo": "cardano-haskell-packages", - "rev": "cb63b6483a5d6ce36fb07815736315bd4408162e", + "rev": "0aa7afd943dbcf2dc51fe652c982da281f8cb621", "type": "github" }, "original": { @@ -273,11 +273,11 @@ "hackageNix_2": { "flake": false, "locked": { - "lastModified": 1778061448, - "narHash": "sha256-cPUF8+l1ej7x4UZcuuf6IDsxU1WWmGWC0vFBH+6jXZk=", + "lastModified": 1781588317, + "narHash": "sha256-bLPzRuyhb9kLvcZd3jmLqEjxYXoF7kRSY36k1bpXSpQ=", "owner": "input-output-hk", "repo": "hackage.nix", - "rev": "ba6ab6f3b781c8f308cba4fa384eafa48033f3cc", + "rev": "0304ab7fcb57c73b725bfdc0ebecea4476700366", "type": "github" }, "original": { diff --git a/trace-forward/src/Trace/Forward/Forwarding.hs b/trace-forward/src/Trace/Forward/Forwarding.hs index 2960db6c85d..cb078f62470 100644 --- a/trace-forward/src/Trace/Forward/Forwarding.hs +++ b/trace-forward/src/Trace/Forward/Forwarding.hs @@ -24,7 +24,7 @@ import Ouroboros.Network.Mux (MiniProtocol (..), MiniProtocolLimits (. MiniProtocolNum (..), OuroborosApplication (..), RunMiniProtocol (..), miniProtocolLimits, miniProtocolNum, miniProtocolRun) import Ouroboros.Network.Protocol.Handshake (HandshakeArguments (..)) -import Ouroboros.Network.Protocol.Handshake.Codec (cborTermVersionDataCodec, +import Ouroboros.Network.Protocol.Handshake.Codec (mkVersionedCodecCBORTerm, codecHandshake, noTimeLimitsHandshake, timeLimitsHandshake) import Ouroboros.Network.Protocol.Handshake.Type (Handshake) import Ouroboros.Network.Protocol.Handshake.Version (acceptableVersion, queryVersion, @@ -288,7 +288,7 @@ doConnectToAcceptor magic snocket makeBearer configureSocket address timeLimits args = ConnectToArgs { ctaHandshakeCodec = codecHandshake forwardingVersionCodec, ctaHandshakeTimeLimits = timeLimits, - ctaVersionDataCodec = cborTermVersionDataCodec forwardingCodecCBORTerm, + ctaVersionDataCodec = mkVersionedCodecCBORTerm forwardingCodecCBORTerm, ctaConnectTracers = nullNetworkConnectTracers, ctaHandshakeCallbacks = HandshakeCallbacks acceptableVersion queryVersion } forwarderApp @@ -337,7 +337,7 @@ doListenToAcceptor magic snocket makeBearer configureSocket address timeLimits haBearerTracer = nullTracer, haHandshakeTracer = nullTracer, haHandshakeCodec = codecHandshake forwardingVersionCodec, - haVersionDataCodec = cborTermVersionDataCodec forwardingCodecCBORTerm, + haVersionDataCodec = mkVersionedCodecCBORTerm forwardingCodecCBORTerm, haAcceptVersion = acceptableVersion, haQueryVersion = queryVersion, haTimeLimits = timeLimits From b6d61b1dd10e6f41b8f5b4c4691fd8f8814c44fb Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Thu, 18 Jun 2026 19:09:00 +0300 Subject: [PATCH 14/40] Bump srps --- cabal.project | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cabal.project b/cabal.project index ae1112f8238..a97f680c2c9 100644 --- a/cabal.project +++ b/cabal.project @@ -91,8 +91,8 @@ allow-newer: source-repository-package type: git location: https://github.com/IntersectMBO/cardano-api.git - tag: 500e283a93fcb4c82b03febf243cb976431a613f - --sha256: sha256-aHpjdMQhjRpe0w5Fb9a5LeVGcO44VAl928CS7HoYB0Q= + tag: 9dd555a313edb9c3a1203506d69702036d3c170f + --sha256: sha256-qQlEg7VwDMo5xEuCgnzx3UPchZhDbMmizx0c3kMy578= subdir: cardano-api cardano-rpc @@ -108,8 +108,8 @@ source-repository-package source-repository-package type: git location: https://github.com/IntersectMBO/cardano-ledger.git - tag: 3f879bb37df4738ed8211e500c7d180443cfcbe4 - --sha256: sha256-uLjiIHiU1SzAmoKs+rynQphc3FUYXKeJLlOnp87uNdg= + tag: 8dc1c431e06db2c8b5d44fb4b3cca6419197d763 + --sha256: sha256-xKtgNFxjbJE6UWGvTioGX81NgvlKH3mHEaYMWDm8/UA= subdir: eras/allegra/impl eras/alonzo/impl @@ -136,8 +136,8 @@ source-repository-package source-repository-package type: git location: https://github.com/f-f/kes-agent.git - tag: fdb4f4db05e3744ed413f83477020fdf43cf32a2 - --sha256: sha256-eyQc8Dk7+upSRQvH5eXZuj6asYhOLsH59ABJZDyvQ6I= + tag: 32c1ed675d22a30735d9f22f7afa436a3ef3e64a + --sha256: sha256-o7hFX1JnraS6Xq0WoXQwd9Z8GsPPv0Ls2DWvZ08o0ZU= subdir: kes-agent kes-agent-crypto @@ -145,16 +145,16 @@ source-repository-package source-repository-package type: git location: https://github.com/IntersectMBO/ouroboros-consensus.git - tag: 0411b4d50dc62cab07bbbf75805cf585a7a1f8e7 - --sha256: sha256-DNnGHdo+oQDBbHzAl6UZ/VcPoS4TxGrC9eCOiMxgc8A= + tag: e6fad063078894cefece2a535a2e2b4d9a092e73 + --sha256: sha256-SPtbQ1+zN4uYiVx3rej2kMasbpfhSAyjRD5bdrnK3Rs= subdir: . source-repository-package type: git location: https://github.com/IntersectMBO/ouroboros-network.git - tag: 1881340e26ca98b3ee2b89ea536348406b79e051 - --sha256: sha256-O1Th+YI6LZqCLM6n75tVV+3InA4W8sC0yHEi09u8Fyw= + tag: 8b4dcb898b64615f574a7fbdcf119f77c7fbc598 + --sha256: sha256-t1tGV5uZwyKnSSCbaJJAteMYKnQYeO39hUUycMRuC2M= subdir: ./cardano-diffusion ./monoidal-synchronisation @@ -172,8 +172,8 @@ source-repository-package source-repository-package type: git location: https://github.com/IntersectMBO/plutus.git - tag: 5b401b3ba95d2b8ecf9454aca7bf704bfd69f13c - --sha256: sha256-1kIRfqJGitFLdg77kYrmA8P8gDDmnldRLqI1iBDSVRU= + tag: b1db04cc425fab303ac3b79d7140dc2a17f29e6d + --sha256: sha256-XtYzjNVx4+IGWpgm+p/YTf56DrJtK0I98umZCTE3U80= subdir: plutus-core plutus-ledger-api From eae2fc14cf7f891ba5a70f7976956bf2abcad34f Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Thu, 18 Jun 2026 21:04:37 +0300 Subject: [PATCH 15/40] Bump api srp --- cabal.project | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cabal.project b/cabal.project index a97f680c2c9..03ebe5305a2 100644 --- a/cabal.project +++ b/cabal.project @@ -91,8 +91,8 @@ allow-newer: source-repository-package type: git location: https://github.com/IntersectMBO/cardano-api.git - tag: 9dd555a313edb9c3a1203506d69702036d3c170f - --sha256: sha256-qQlEg7VwDMo5xEuCgnzx3UPchZhDbMmizx0c3kMy578= + tag: bb64ad40f904040a9b4d99306cf13eef4650a6c9 + --sha256: sha256-302P+qj+UkY/uSWHt8YMT4VzNhRS9tWMz0G6rjr4eVk= subdir: cardano-api cardano-rpc From 37507cef3279de4eb12c53b69d3ec6304c2dbf5b Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Fri, 19 Jun 2026 01:58:55 +0300 Subject: [PATCH 16/40] Update ledger --- .../src/Cardano/Node/Tracing/Era/Shelley.hs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cardano-node/src/Cardano/Node/Tracing/Era/Shelley.hs b/cardano-node/src/Cardano/Node/Tracing/Era/Shelley.hs index f9d0efe92f9..ae7c328db33 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Era/Shelley.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Era/Shelley.hs @@ -222,8 +222,8 @@ instance instance ( Consensus.ShelleyBasedEra era - , LogFormatting (PredicateFailure (ShelleyUTXO era)) - , LogFormatting (PredicateFailure (ShelleyUTXOW era)) + , LogFormatting (PredicateFailure (UTXO era)) + , LogFormatting (PredicateFailure (UTXOW era)) , LogFormatting (PredicateFailure (Ledger.EraRule "LEDGER" era)) , ToJSON (ApplyTxError era) ) => LogFormatting (ApplyTxError era) where @@ -251,8 +251,8 @@ instance instance ( Consensus.ShelleyBasedEra era - , LogFormatting (PredicateFailure (ShelleyUTXO era)) - , LogFormatting (PredicateFailure (ShelleyUTXOW era)) + , LogFormatting (PredicateFailure (UTXO era)) + , LogFormatting (PredicateFailure (UTXOW era)) , LogFormatting (PredicateFailure (Ledger.EraRule "BBODY" era)) , NFData (PredicateFailure (Ledger.EraRule "BBODY" era)) ) => LogFormatting (BlockTransitionError era) where @@ -323,8 +323,8 @@ instance LogFormatting PrtlSeqFailure where instance ( Consensus.ShelleyBasedEra era - , LogFormatting (PredicateFailure (ShelleyUTXO era)) - , LogFormatting (PredicateFailure (ShelleyUTXOW era)) + , LogFormatting (PredicateFailure (UTXO era)) + , LogFormatting (PredicateFailure (UTXOW era)) , LogFormatting (PredicateFailure (Ledger.EraRule "LEDGER" era)) , LogFormatting (PredicateFailure (Ledger.EraRule "LEDGERS" era)) ) => LogFormatting (ShelleyBbodyPredFailure era) where @@ -345,8 +345,8 @@ instance instance ( Consensus.ShelleyBasedEra era - , LogFormatting (PredicateFailure (ShelleyUTXO era)) - , LogFormatting (PredicateFailure (ShelleyUTXOW era)) + , LogFormatting (PredicateFailure (UTXO era)) + , LogFormatting (PredicateFailure (UTXOW era)) , LogFormatting (PredicateFailure (Ledger.EraRule "LEDGER" era)) ) => LogFormatting (ShelleyLedgersPredFailure era) where forMachine dtal (LedgerFailure f) = forMachine dtal f @@ -363,8 +363,8 @@ instance LogFormatting Withdrawals where instance ( Consensus.ShelleyBasedEra era - , LogFormatting (PredicateFailure (ShelleyUTXO era)) - , LogFormatting (PredicateFailure (ShelleyUTXOW era)) + , LogFormatting (PredicateFailure (UTXO era)) + , LogFormatting (PredicateFailure (UTXOW era)) , LogFormatting (PredicateFailure (Ledger.EraRule "DELEGS" era)) , LogFormatting (PredicateFailure (Ledger.EraRule "UTXOW" era)) ) => LogFormatting (ShelleyLedgerPredFailure era) where @@ -433,7 +433,7 @@ formatAsHex (Just bs) = show bs instance ( Consensus.ShelleyBasedEra era - , LogFormatting (PredicateFailure (ShelleyUTXO era)) + , LogFormatting (PredicateFailure (UTXO era)) , LogFormatting (PredicateFailure (Ledger.EraRule "UTXO" era)) ) => LogFormatting (ShelleyUtxowPredFailure era) where forMachine _dtal (InvalidWitnessesUTXOW wits') = From 21a12f0ac69bc288544cedd9dbcec97646b5897a Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Mon, 22 Jun 2026 12:41:44 +0300 Subject: [PATCH 17/40] Allow packages from hackage --- cabal.project | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cabal.project b/cabal.project index 03ebe5305a2..72751dd7d92 100644 --- a/cabal.project +++ b/cabal.project @@ -16,10 +16,6 @@ index-state: , hackage.haskell.org 2026-06-10T14:43:25Z , cardano-haskell-packages 2026-06-10T14:16:00Z -active-repositories: - , :rest - , cardano-haskell-packages:override - constraints: -- haskell.nix patch does not work for 1.6.8 , any.crypton-x509-system < 1.6.8 From 6d538096bd5aef381009dda09311deee5e1faa6d Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Mon, 22 Jun 2026 18:31:14 +0300 Subject: [PATCH 18/40] Apply remaining changes from predictable-snapshots branch --- cardano-node/cardano-node.cabal | 2 +- .../src/Cardano/Node/Configuration/POM.hs | 65 ++- .../Cardano/Node/Tracing/Tracers/ChainDB.hs | 82 ++-- .../test/Test/Cardano/Config/Mainnet.hs | 6 +- .../test/Test/Cardano/Node/FilePermissions.hs | 29 +- cardano-node/test/Test/Cardano/Node/POM.hs | 59 ++- .../cardano/mainnet-config-legacy.json | 6 +- configuration/cardano/mainnet-config.json | 6 +- configuration/cardano/mainnet-config.yaml | 19 +- .../testnet-template-config-legacy.json | 6 +- .../cardano/testnet-template-config.json | 6 +- nix/workbench/service/nodes.nix | 431 ++++++++++-------- 12 files changed, 432 insertions(+), 285 deletions(-) diff --git a/cardano-node/cardano-node.cabal b/cardano-node/cardano-node.cabal index a4c2c092439..7203af08ef3 100644 --- a/cardano-node/cardano-node.cabal +++ b/cardano-node/cardano-node.cabal @@ -232,6 +232,7 @@ test-suite cardano-node-test , contra-tracer , directory , filepath + , fs-api , hedgehog , hedgehog-corpus , hedgehog-extras ^>= 0.10 @@ -240,7 +241,6 @@ test-suite cardano-node-test , ouroboros-consensus:{ouroboros-consensus, diffusion} , ouroboros-network:{api, framework, ouroboros-network} , text - , trace-dispatcher , transformers , vector , yaml diff --git a/cardano-node/src/Cardano/Node/Configuration/POM.hs b/cardano-node/src/Cardano/Node/Configuration/POM.hs index 09590a9b929..1b648370811 100644 --- a/cardano-node/src/Cardano/Node/Configuration/POM.hs +++ b/cardano-node/src/Cardano/Node/Configuration/POM.hs @@ -28,6 +28,7 @@ module Cardano.Node.Configuration.POM where import Cardano.Crypto (RequiresNetworkMagic (..)) +import Cardano.Ledger.BaseTypes.NonZero (nonZero) import Cardano.Logging.Types import Cardano.Network.ConsensusMode (ConsensusMode (..), defaultConsensusMode) import qualified Cardano.Network.Diffusion.Configuration as Cardano @@ -45,10 +46,10 @@ import Ouroboros.Consensus.Node (NodeDatabasePaths (..)) import Ouroboros.Consensus.Node.Genesis (GenesisConfig, GenesisConfigFlags, defaultGenesisConfigFlags, mkGenesisConfig) import Ouroboros.Consensus.Storage.LedgerDB.Args (QueryBatchSize (..)) +import Ouroboros.Consensus.Storage.LedgerDB.Snapshots (NumOfDiskSnapshots (..), + SnapshotDelayRange (..), SnapshotFrequency (..), SnapshotFrequencyArgs (..), + SnapshotPolicyArgs (..), defaultSnapshotPolicyArgs, mithrilSnapshotPolicyArgs) import Ouroboros.Consensus.Util.Args (OverrideOrDefault (..)) -import Ouroboros.Consensus.Storage.LedgerDB.Snapshots ( - SnapshotFrequency (..), SnapshotFrequencyArgs (..), SnapshotPolicyArgs (..), - defaultSnapshotPolicyArgs, NumOfDiskSnapshots (..)) import Ouroboros.Network.Diffusion.Configuration as Configuration import qualified Ouroboros.Network.Diffusion.Configuration as Ouroboros import qualified Ouroboros.Network.Mux as Mux @@ -65,6 +66,7 @@ import Data.Hashable (Hashable) import Data.Maybe import Data.Monoid (Last (..)) import Data.Text (Text) +import qualified Data.Text as Text import Data.Time.Clock (DiffTime, secondsToDiffTime) import Data.Yaml (decodeFileThrow) import GHC.Generics (Generic) @@ -75,8 +77,6 @@ import System.Random (randomIO) import Generic.Data (gmappend) import Generic.Data.Orphans () -import Cardano.Ledger.BaseTypes.NonZero (nonZero) - -- | Isomorphic to a `Maybe DiffTime`, but expresses what `Nothing` means, in -- this case that we want to /NOT/ override the default timeout. data TimeoutOverride = NoTimeoutOverride | TimeoutOverride DiffTime @@ -517,25 +517,52 @@ instance FromJSON PartialNodeConfiguration where return $ Just $ LedgerDbConfiguration spArgs DefaultQueryBatchSize V2InMemory deprecatedOpts Just ledgerDB -> flip (withObject "LedgerDB") ledgerDB $ \o -> do - ldbSnapInterval <- (getLast . (Last mTopLevelSnapInterval <>) . Last <$> snapInterval o) .!= UseDefault - ldbSnapNum <- (getLast . (Last mTopLevelSnapNum <>) . Last <$> snapNum o) .!= UseDefault - ldbSnapOffset <- (fmap Override <$> o .:? "SlotOffset") .!= UseDefault - ldbSnapRateLimit<- (fmap (Override . secondsToDiffTime) <$> o .:? "RateLimit") .!= UseDefault - qsize <- (fmap RequestedQueryBatchSize <$> o .:? "QueryBatchSize") .!= DefaultQueryBatchSize - backend <- o .:? "Backend" .!= "V2InMemory" - selector <- case backend of + -- Parse snapshot options from an object, honouring any top-level + -- (deprecated) SnapshotInterval / NumOfDiskSnapshots overrides. + let parseSnapshotOpts s = do + sInterval <- (getLast . (Last mTopLevelSnapInterval <>) . Last <$> snapInterval s) .!= UseDefault + sNum <- (getLast . (Last mTopLevelSnapNum <>) . Last <$> snapNum s) .!= UseDefault + sOffset <- (fmap Override <$> s .:? "SlotOffset") .!= UseDefault + sRateLimit <- (fmap (Override . secondsToDiffTime) <$> s .:? "RateLimit") .!= UseDefault + sMinDelay <- s .:? "MinDelay" + sMaxDelay <- s .:? "MaxDelay" + sDelayRange <- + case (sMinDelay, sMaxDelay) of + (Just minDelay, Just maxDelay) -> + if minDelay <= maxDelay then + pure (Override (SnapshotDelayRange (secondsToDiffTime minDelay) (secondsToDiffTime maxDelay))) + else fail $ "Invalid ledger snapshot delay range, MinDelay > MaxDelay: " + <> show minDelay <> " > " <> show maxDelay + _ -> pure UseDefault + let sf = SnapshotFrequencyArgs { + sfaInterval = sInterval + , sfaOffset = sOffset + , sfaRateLimit = sRateLimit + , sfaDelaySnapshotRange = sDelayRange + } + pure $ SnapshotPolicyArgs (SnapshotFrequency sf) sNum + + qsize <- (fmap RequestedQueryBatchSize <$> o .:? "QueryBatchSize") .!= DefaultQueryBatchSize + backend <- o .:? "Backend" .!= "V2InMemory" + selector <- case backend of "V2InMemory" -> return V2InMemory "V2LSM" -> do lsmPath :: Maybe FilePath <- o .:? "LSMDatabasePath" pure $ V2LSM lsmPath _ -> fail $ "Malformed LedgerDB Backend: " <> backend - let sf = SnapshotFrequencyArgs { - sfaInterval = ldbSnapInterval - , sfaOffset = ldbSnapOffset - , sfaRateLimit = ldbSnapRateLimit - , sfaDelaySnapshotRange = UseDefault - } - spArgs = SnapshotPolicyArgs (SnapshotFrequency sf) ldbSnapNum + + -- A named policy (e.g. `Snapshots: Mithril`) selects a whole predefined + -- set of args; an object is parsed field-by-field; absence falls back to + -- the legacy top-level options for backward compatibility. + mSnapshotsVal <- o .:? "Snapshots" + spArgs <- case mSnapshotsVal of + Just (String name) -> case name of + "Mithril" -> pure mithrilSnapshotPolicyArgs + _ -> fail $ "Unknown named ledger snapshot policy: " <> Text.unpack name + <> ". Expected \"Mithril\" or an object with snapshot options." + Just sv -> withObject "Snapshots" parseSnapshotOpts sv + Nothing -> parseSnapshotOpts o + pure $ Just $ LedgerDbConfiguration spArgs qsize selector deprecatedOpts parseByronProtocol v = do diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs index 5954059c499..4380a8e9414 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs @@ -46,18 +46,19 @@ import qualified Ouroboros.Consensus.Storage.LedgerDB.Snapshots as LedgerDB import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.Backend as V2 import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.InMemory as InMemory import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.LSM as LSM +import qualified Ouroboros.Consensus.Storage.PerasCertDB as PerasCertDB +import qualified Ouroboros.Consensus.Storage.PerasVoteDB as PerasVoteDB import qualified Ouroboros.Consensus.Storage.VolatileDB as VolDB import Ouroboros.Consensus.TypeFamilyWrappers import Ouroboros.Consensus.Util.Condense (condense) import Ouroboros.Consensus.Util.Enclose import qualified Ouroboros.Network.AnchoredFragment as AF import Ouroboros.Network.Block (MaxSlotNo (..)) -import qualified Ouroboros.Consensus.Storage.PerasVoteDB as PerasVoteDB -import qualified Ouroboros.Consensus.Storage.PerasCertDB as PerasCertDB import Data.Aeson (Object, ToJSON, Value (Object, String), object, toJSON, (.=)) import qualified Data.ByteString.Base16 as B16 import Data.Int (Int64) +import qualified Data.List.NonEmpty as NonEmpty import Data.SOP (All, K (..), hcmap, hcollapse) import Data.Text (Text) import qualified Data.Text as Text @@ -1667,20 +1668,21 @@ instance MetaTrace (PerasVoteDB.TraceEvent blk) where documentFor (Namespace _ ["GarbageCollected"]) = Just "GarbageCollected" documentFor _ = Nothing -instance LogFormatting (PerasVoteDB.TraceEvent blk) where - forHuman (PerasVoteDB.AddVote {}) = "PerasVoteDB.AddVote" - forHuman (PerasVoteDB.GarbageCollected {}) = "PerasVoteDB.GarbageCollected" +instance StandardHash blk => LogFormatting (PerasVoteDB.TraceEvent blk) where + forHuman (PerasVoteDB.AddVote voteId _vote result) = + "Peras vote " <> Text.pack (show voteId) <> ": " <> Text.pack (show result) + forHuman (PerasVoteDB.GarbageCollected slotNo) = + "Peras vote DB garbage collected at slot " <> Text.pack (show slotNo) - forMachine _dtal (PerasVoteDB.AddVote cert _ _) = - mconcat - [ "kind" .= String "AddVote" - , "cert" .= String (Text.pack $ show cert) - ] + forMachine _dtal (PerasVoteDB.AddVote voteId _vote result) = + mconcat [ "kind" .= String "AddVote" + , "voteId" .= String (Text.pack $ show voteId) + , "result" .= String (Text.pack $ show result) + ] forMachine _dtal (PerasVoteDB.GarbageCollected slotNo) = - mconcat - [ "kind" .= String "GarbageCollected" - , "slotNo" .= String (Text.pack $ show slotNo) - ] + mconcat [ "kind" .= String "GarbageCollected" + , "slot" .= String (Text.pack $ show slotNo) + ] asMetrics _ = [] @@ -1701,19 +1703,20 @@ instance MetaTrace (PerasCertDB.TraceEvent blk) where documentFor _ = Nothing instance LogFormatting (PerasCertDB.TraceEvent blk) where - forHuman (PerasCertDB.AddCert _ _ _) = "PerasCertDB.AddCert" - forHuman (PerasCertDB.GarbageCollected _slotNo) = "PerasCertDB.GarbageCollected" - - forMachine _dtal (PerasCertDB.AddCert cert _ _) = - mconcat - [ "kind" .= String "AddCert" - , "cert" .= String (Text.pack $ show cert) - ] + forHuman (PerasCertDB.AddCert roundNo _cert result) = + "Peras certificate for round " <> Text.pack (show roundNo) <> ": " <> Text.pack (show result) + forHuman (PerasCertDB.GarbageCollected slotNo) = + "Peras certificate DB garbage collected at slot " <> Text.pack (show slotNo) + + forMachine _dtal (PerasCertDB.AddCert roundNo _cert result) = + mconcat [ "kind" .= String "AddCert" + , "round" .= String (Text.pack $ show roundNo) + , "result" .= String (Text.pack $ show result) + ] forMachine _dtal (PerasCertDB.GarbageCollected slotNo) = - mconcat - [ "kind" .= String "GarbageCollected" - , "slotNo" .= String (Text.pack $ show slotNo) - ] + mconcat [ "kind" .= String "GarbageCollected" + , "slot" .= String (Text.pack $ show slotNo) + ] asMetrics _ = [] @@ -1788,8 +1791,13 @@ instance MetaTrace (LedgerDB.TraceEvent blk) where instance ( StandardHash blk , ConvertRawHash blk) => LogFormatting (LedgerDB.TraceSnapshotEvent blk) where - forHuman (LedgerDB.SnapshotRequestDelayed {}) = "LedgerDB.SnapshotRequestDelayed" - forHuman (LedgerDB.SnapshotRequestCompleted) = "LedgerDB.SnapshotRequestCompleted" + forHuman (LedgerDB.SnapshotRequestDelayed _snapshotRequestTime delayBeforeSnapshotting slots) = + Text.unwords [ "Scheduling to take ledger state snapshots at slots " + , showT (NonEmpty.toList slots) + , ", with a randomised delay of" + , showT delayBeforeSnapshotting + ] + forHuman LedgerDB.SnapshotRequestCompleted = "Completed taking a ledger state snapshot" forHuman (LedgerDB.TookSnapshot snap pt RisingEdge) = Text.unwords [ "Taking ledger snapshot" , showT snap @@ -1828,11 +1836,13 @@ instance ( StandardHash blk " Snapshot was created for a different backend. Convert it with `snapshot-converter`." _ -> "" - -- TODO: Create a proper log for SnapshotRequestDelayed - forMachine _ (LedgerDB.SnapshotRequestDelayed _ _ _) = + forMachine _dtals (LedgerDB.SnapshotRequestDelayed snapshotRequestTime delayBeforeSnapshotting slots) = mconcat [ "kind" .= String "SnapshotRequestDelayed" + , "requestTime" .= show snapshotRequestTime + , "delayBeforeSnapshotting" .= show delayBeforeSnapshotting + , "slots" .= toJSON (NonEmpty.toList slots) ] - forMachine _ (LedgerDB.SnapshotRequestCompleted) = + forMachine _dtals LedgerDB.SnapshotRequestCompleted = mconcat [ "kind" .= String "SnapshotRequestCompleted" ] forMachine dtals (LedgerDB.TookSnapshot snap pt enclosedTiming) = @@ -1856,8 +1866,8 @@ instance MetaTrace (LedgerDB.TraceSnapshotEvent blk) where namespaceFor LedgerDB.DeletedSnapshot {} = Namespace [] ["DeletedSnapshot"] namespaceFor LedgerDB.InvalidSnapshot {} = Namespace [] ["InvalidSnapshot"] - severityFor (Namespace _ ["SnapshotRequestDelayed"]) _ = Just Info - severityFor (Namespace _ ["SnapshotRequestCompleted"]) _ = Just Info + severityFor (Namespace _ ["SnapshotRequestDelayed"]) _ = Just Debug + severityFor (Namespace _ ["SnapshotRequestCompleted"]) _ = Just Debug severityFor (Namespace _ ["TookSnapshot"]) _ = Just Info severityFor (Namespace _ ["DeletedSnapshot"]) _ = Just Debug severityFor (Namespace _ ["InvalidSnapshot"]) _ = Just Error @@ -1875,12 +1885,18 @@ instance MetaTrace (LedgerDB.TraceSnapshotEvent blk) where , " seems to be from an old node or different backend, it will" , " be deleted" ] + documentFor (Namespace _ ["SnapshotRequestDelayed"]) = Just + "A delayed snapshot request was issued. The snapshot will be initiated at the specified timestamp, with the specified delay and for the specified slots" + documentFor (Namespace _ ["SnapshotRequestCompleted"]) = Just + "The delayed snapshot request was completed" documentFor _ = Nothing allNamespaces = [ Namespace [] ["TookSnapshot"] , Namespace [] ["DeletedSnapshot"] , Namespace [] ["InvalidSnapshot"] + , Namespace [] ["SnapshotRequestDelayed"] + , Namespace [] ["SnapshotRequestCompleted"] ] -------------------------------------------------------------------------------- diff --git a/cardano-node/test/Test/Cardano/Config/Mainnet.hs b/cardano-node/test/Test/Cardano/Config/Mainnet.hs index 154f710d8cf..e9a02a56ac0 100644 --- a/cardano-node/test/Test/Cardano/Config/Mainnet.hs +++ b/cardano-node/test/Test/Cardano/Config/Mainnet.hs @@ -15,6 +15,9 @@ import qualified Data.Yaml as Y import qualified GHC.Stack as GHC import qualified System.Directory as IO import System.FilePath (()) +import System.FS.API (SomeHasFS (..)) +import System.FS.API.Types (MountPoint (MountPoint)) +import System.FS.IO (ioHasFS) import Hedgehog (Property, (===)) import qualified Hedgehog as H @@ -24,7 +27,8 @@ import qualified Hedgehog.Extras.Test.Process as H hprop_configMainnetHash :: Property hprop_configMainnetHash = H.propertyOnce $ do base <- H.note =<< H.evalIO . IO.canonicalizePath =<< H.getProjectBase - result <- H.evalIO $ runExceptT $ initialLedgerState $ File $ base "configuration/cardano/mainnet-config.json" + let fs = SomeHasFS (ioHasFS (MountPoint (base "configuration/cardano"))) + result <- H.evalIO $ runExceptT $ initialLedgerState fs $ File $ base "configuration/cardano/mainnet-config.json" case result of Right (_, _) -> return () Left e -> H.failWithCustom GHC.callStack Nothing (displayError e) diff --git a/cardano-node/test/Test/Cardano/Node/FilePermissions.hs b/cardano-node/test/Test/Cardano/Node/FilePermissions.hs index 0aa16e86453..faa50a0f01c 100644 --- a/cardano-node/test/Test/Cardano/Node/FilePermissions.hs +++ b/cardano-node/test/Test/Cardano/Node/FilePermissions.hs @@ -14,42 +14,39 @@ module Test.Cardano.Node.FilePermissions ( tests ) where -import Control.Monad.Except -import "contra-tracer" Control.Tracer -import Control.Tracer.Arrow -import Data.Foldable -import Data.IORef -import System.Directory (removeFile) - import Cardano.Api + import Cardano.Node.Run (checkVRFFilePermissions) +import Cardano.Node.Types (VRFPrivateKeyFilePermissionError (..)) + +import Control.Exception (bracket) import Control.Monad (Monad (..)) +import Control.Monad.Except import Control.Monad.Except (runExceptT) import Control.Monad.IO.Class (MonadIO (liftIO)) +import "contra-tracer" Control.Tracer +import Control.Tracer.Arrow import Data.Bool (Bool, not) import Data.Either (Either (..)) import Data.Eq ((==)) +import Data.Foldable import Data.Foldable (foldl', length) import Data.Function (const, ($), (.)) +import Data.IORef import qualified Data.List as L import Data.Maybe (Maybe (..)) import Data.Semigroup (Semigroup (..)) -import Hedgehog -import Hedgehog.Internal.Property (Group (..), failWith) +import System.Directory (removeFile) import System.IO (FilePath, IO) -import Text.Show (Show (..)) -import Cardano.Node.Types (VRFPrivateKeyFilePermissionError (..)) -import Control.Exception (bracket) - -#ifdef UNIX - import System.Posix.Files import System.Posix.IO (closeFd, createFile) import System.Posix.Types (FileMode) +import Text.Show (Show (..)) import Hedgehog import qualified Hedgehog.Extras as H import qualified Hedgehog.Gen as Gen +import Hedgehog.Internal.Property (Group (..), failWith) #endif @@ -166,7 +163,7 @@ mkCapturingTracer = do messages <- liftIO $ newIORef [] let registerMessage :: String -> IO () registerMessage msg = atomicModifyIORef messages (\msgs -> (msgs <> [msg], ())) - pure (Tracer registerMessage, messages) + pure (Tracer (emit registerMessage), messages) #endif -- ----------------------------------------------------------------------------- diff --git a/cardano-node/test/Test/Cardano/Node/POM.hs b/cardano-node/test/Test/Cardano/Node/POM.hs index 2d131f83235..ef6d2def806 100644 --- a/cardano-node/test/Test/Cardano/Node/POM.hs +++ b/cardano-node/test/Test/Cardano/Node/POM.hs @@ -1,6 +1,7 @@ {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE PatternSynonyms #-} +{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TemplateHaskell #-} module Test.Cardano.Node.POM @@ -22,13 +23,15 @@ import Cardano.Rpc.Server.Config (makeRpcConfig) import Ouroboros.Consensus.Node (NodeDatabasePaths (..)) import Ouroboros.Consensus.Node.Genesis (disableGenesisConfig) import Ouroboros.Consensus.Storage.LedgerDB.Args -import Ouroboros.Consensus.Storage.LedgerDB.Snapshots (NumOfDiskSnapshots (..), - SnapshotInterval (..)) +import Ouroboros.Consensus.Storage.LedgerDB.Snapshots (defaultSnapshotPolicyArgs, + mithrilSnapshotPolicyArgs) import Ouroboros.Network.Block (SlotNo (..)) import Ouroboros.Network.PeerSelection.PeerSharing (PeerSharing (..)) import Ouroboros.Network.TxSubmission.Inbound.V2.Types +import Data.Aeson (eitherDecode) import Data.Bifunctor (first) +import qualified Data.ByteString.Lazy as LBS import Data.Monoid (Last (..)) import Data.String import Data.Text (Text) @@ -284,12 +287,62 @@ eExpectedConfig = do , ncConsensusMode = PraosMode , ncGenesisConfig = disableGenesisConfig , ncResponderCoreAffinityPolicy = NoResponderCoreAffinity - , ncLedgerDbConfig = LedgerDbConfiguration DefaultNumOfDiskSnapshots DefaultSnapshotInterval DefaultQueryBatchSize V2InMemory noDeprecatedOptions + , ncLedgerDbConfig = LedgerDbConfiguration defaultSnapshotPolicyArgs DefaultQueryBatchSize V2InMemory noDeprecatedOptions , ncRpcConfig , ncTxSubmissionLogicVersion = TxSubmissionLogicV1 , ncTxSubmissionInitDelay = defaultTxSubmissionInitDelay } +-- | Test that the legacy flat LedgerDB snapshot config format (options directly +-- under LedgerDB) parses identically to the new nested Snapshots format. +-- +-- TODO: this test could be removed once the old format is deprecated. +prop_legacySnapshotFormat_POM :: Property +prop_legacySnapshotFormat_POM = + withTests 1 . Hedgehog.property $ do + let legacyJson = "{ " <> dummyRequiredValues <> ", " + <> "\"LedgerDB\": {" + <> " \"Backend\": \"V2InMemory\"," + <> " \"SnapshotInterval\": 4320," + <> " \"NumOfDiskSnapshots\": 2" + <> "} }" + newJson = "{ " <> dummyRequiredValues <> ", " + <> "\"LedgerDB\": {" + <> " \"Backend\": \"V2InMemory\"," + <> " \"Snapshots\": {" + <> " \"SnapshotInterval\": 4320," + <> " \"NumOfDiskSnapshots\": 2" + <> " }" + <> "} }" + legacyConfig :: PartialNodeConfiguration <- evalEither $ eitherDecode legacyJson + newConfig :: PartialNodeConfiguration <- evalEither $ eitherDecode newJson + pncLedgerDbConfig legacyConfig === pncLedgerDbConfig newConfig + +-- | Test that the named \"Mithril\" snapshot policy selects +-- 'mithrilSnapshotPolicyArgs' as a whole. +prop_mithrilSnapshotPolicy_POM :: Property +prop_mithrilSnapshotPolicy_POM = + withTests 1 . Hedgehog.property $ do + let json = "{ " <> dummyRequiredValues <> ", " + <> "\"LedgerDB\": {" + <> " \"Backend\": \"V2InMemory\"," + <> " \"Snapshots\": \"Mithril\"" + <> "} }" + config :: PartialNodeConfiguration <- evalEither $ eitherDecode json + getLast (pncLedgerDbConfig config) === + Just (LedgerDbConfiguration mithrilSnapshotPolicyArgs DefaultQueryBatchSize V2InMemory noDeprecatedOptions) + +dummyRequiredValues :: LBS.ByteString +dummyRequiredValues = mconcat + [ "\"ByronGenesisFile\": \"x\"" + , ", \"ShelleyGenesisFile\": \"x\"" + , ", \"AlonzoGenesisFile\": \"x\"" + , ", \"ConwayGenesisFile\": \"x\"" + , ", \"LastKnownBlockVersion-Major\": 0" + , ", \"LastKnownBlockVersion-Minor\": 0" + , ", \"LastKnownBlockVersion-Alt\": 0" + ] + -- ----------------------------------------------------------------------------- tests :: IO Bool diff --git a/configuration/cardano/mainnet-config-legacy.json b/configuration/cardano/mainnet-config-legacy.json index f4bc557037e..a83cf83f481 100644 --- a/configuration/cardano/mainnet-config-legacy.json +++ b/configuration/cardano/mainnet-config-legacy.json @@ -13,9 +13,11 @@ "LastKnownBlockVersion-Minor": 0, "LedgerDB": { "Backend": "V2InMemory", - "NumOfDiskSnapshots": 2, "QueryBatchSize": 100000, - "SnapshotInterval": 4320 + "Snapshots": { + "NumOfDiskSnapshots": 2, + "SnapshotInterval": 4320 + } }, "MaxKnownMajorProtocolVersion": 2, "MinNodeVersion": "10.7.0", diff --git a/configuration/cardano/mainnet-config.json b/configuration/cardano/mainnet-config.json index b587e72e99d..a63aa6f45c9 100644 --- a/configuration/cardano/mainnet-config.json +++ b/configuration/cardano/mainnet-config.json @@ -13,9 +13,11 @@ "LastKnownBlockVersion-Minor": 0, "LedgerDB": { "Backend": "V2InMemory", - "NumOfDiskSnapshots": 2, "QueryBatchSize": 100000, - "SnapshotInterval": 4320 + "Snapshots": { + "NumOfDiskSnapshots": 2, + "SnapshotInterval": 4320 + } }, "MaxKnownMajorProtocolVersion": 2, "MinNodeVersion": "10.7.0", diff --git a/configuration/cardano/mainnet-config.yaml b/configuration/cardano/mainnet-config.yaml index 86f805d4e8e..b0ebdd0892e 100644 --- a/configuration/cardano/mainnet-config.yaml +++ b/configuration/cardano/mainnet-config.yaml @@ -80,19 +80,22 @@ ConsensusMode: PraosMode # Additional configuration options can be found at: # https://ouroboros-consensus.cardano.intersectmbo.org/docs/for-developers/utxo-hd/migrating LedgerDB: - # The time interval between snapshots, in seconds. - SnapshotInterval: 4320 - - # The number of disk snapshots to keep. - NumOfDiskSnapshots: 2 + # The backend can either be in memory with `V2InMemory` or on disk with + # `V2LSM`. + Backend: V2InMemory # When querying the store for a big range of UTxOs (such as with # QueryUTxOByAddress), the store will be read in batches of this size. QueryBatchSize: 100000 - # The backend can either be in memory with `V2InMemory` or on disk with - # `V1LMDB`. - Backend: V2InMemory + # Instead of an object with individual options, a predefined snapshot + # policy can be selected by name, e.g. `Snapshots: Mithril`. + Snapshots: + # The time interval between snapshots, in slots. + SnapshotInterval: 4320 + + # The number of disk snapshots to keep. + NumOfDiskSnapshots: 2 ##### Version Information ##### diff --git a/configuration/cardano/testnet-template-config-legacy.json b/configuration/cardano/testnet-template-config-legacy.json index 1d58efae3f3..42a402569f7 100644 --- a/configuration/cardano/testnet-template-config-legacy.json +++ b/configuration/cardano/testnet-template-config-legacy.json @@ -12,9 +12,11 @@ "LastKnownBlockVersion-Minor": 1, "LedgerDB": { "Backend": "V2InMemory", - "NumOfDiskSnapshots": 2, "QueryBatchSize": 100000, - "SnapshotInterval": 216 + "Snapshots": { + "NumOfDiskSnapshots": 2, + "SnapshotInterval": 216 + } }, "MaxConcurrencyDeadline": 4, "MaxKnownMajorProtocolVersion": 2, diff --git a/configuration/cardano/testnet-template-config.json b/configuration/cardano/testnet-template-config.json index 0ab850d4bad..ad1471cb456 100644 --- a/configuration/cardano/testnet-template-config.json +++ b/configuration/cardano/testnet-template-config.json @@ -13,9 +13,11 @@ "LastKnownBlockVersion-Minor": 1, "LedgerDB": { "Backend": "V2InMemory", - "NumOfDiskSnapshots": 2, "QueryBatchSize": 100000, - "SnapshotInterval": 216 + "Snapshots": { + "NumOfDiskSnapshots": 2, + "SnapshotInterval": 216 + } }, "MaxConcurrencyDeadline": 4, "MaxKnownMajorProtocolVersion": 2, diff --git a/nix/workbench/service/nodes.nix b/nix/workbench/service/nodes.nix index 22e5a5f85cd..8766d5949ed 100644 --- a/nix/workbench/service/nodes.nix +++ b/nix/workbench/service/nodes.nix @@ -1,30 +1,28 @@ -{ pkgs -, workbenchNix - -## The cardano-node config used as baseline: -, baseNodeConfig - -, backend -, profile -, profiling -, nodeSpecs - -# Derivations used to generate the topology projections -, profileJsonPath, topologyJsonPath +{ + pkgs, + workbenchNix, + ## The cardano-node config used as baseline: + baseNodeConfig, + backend, + profile, + profiling, + nodeSpecs, + # Derivations used to generate the topology projections + profileJsonPath, + topologyJsonPath, }: - -with pkgs.lib; - -let - readJSONMay = fp: - let fv = __tryEval (__readFile fp); - in if fv.success - then __fromJSON fv.value - else {}; +with pkgs.lib; let + readJSONMay = fp: let + fv = __tryEval (__readFile fp); + in + if fv.success + then __fromJSON fv.value + else {}; profileName = profile.name; - eras = [ ## This defines the order of eras -- which is important. + eras = [ + ## This defines the order of eras -- which is important. "byron" "shelley" "allegra" @@ -34,24 +32,26 @@ let "conway" ]; - configHardforksIntoEra = era: - let go = acc: curEra: rest: - let ret = acc // eraSetupHardforks.${curEra}; - in if curEra == era - then ret - else go ret (__head rest) (__tail rest); - eraSetupHardforks = { - byron = {}; - shelley = { TestShelleyHardForkAtEpoch = 0; }; - allegra = { TestAllegraHardForkAtEpoch = 0; }; - mary = { TestMaryHardForkAtEpoch = 0; }; - alonzo = { TestAlonzoHardForkAtEpoch = 0; }; - babbage = { TestBabbageHardForkAtEpoch = 0; }; - conway = { TestConwayHardForkAtEpoch = 0; }; - }; - in if __hasAttr era eraSetupHardforks - then go {} (__head eras) (__tail eras) - else throw "configHardforksIntoEra: unknown era '${era}'"; + configHardforksIntoEra = era: let + go = acc: curEra: rest: let + ret = acc // eraSetupHardforks.${curEra}; + in + if curEra == era + then ret + else go ret (__head rest) (__tail rest); + eraSetupHardforks = { + byron = {}; + shelley = {TestShelleyHardForkAtEpoch = 0;}; + allegra = {TestAllegraHardForkAtEpoch = 0;}; + mary = {TestMaryHardForkAtEpoch = 0;}; + alonzo = {TestAlonzoHardForkAtEpoch = 0;}; + babbage = {TestBabbageHardForkAtEpoch = 0;}; + conway = {TestConwayHardForkAtEpoch = 0;}; + }; + in + if __hasAttr era eraSetupHardforks + then go {} (__head eras) (__tail eras) + else throw "configHardforksIntoEra: unknown era '${era}'"; liveTablesPath = i: if (profile.node ? "ssd_directory" && profile.node.ssd_directory != null) @@ -61,24 +61,30 @@ let ## ## nodeServiceConfig :: NodeSpec -> ServiceConfig ## - nodeServiceConfig = - { name, i, kind, port, isProducer, ... }@nodeSpec: valency: + nodeServiceConfig = { + name, + i, + kind, + port, + isProducer, + ... + } @ nodeSpec: valency: { inherit isProducer port; inherit (profile.node) rts_flags_override; - nodeId = i; - databasePath = "db"; - socketPath = "node.socket"; - topology = "topology.json"; + nodeId = i; + databasePath = "db"; + socketPath = "node.socket"; + topology = "topology.json"; nodeConfigFile = "config.json"; # Allow for local clusters to have multiple LMDB directories in the same physical ssd_directory; # non-block producers (like the explorer node) keep using the in-memory backend - withUtxoHdLmdb = profile.node.utxo_lmdb && isProducer; - withUtxoHdLsmt = profile.node.utxo_lsmt && isProducer; + withUtxoHdLmdb = profile.node.utxo_lmdb && isProducer; + withUtxoHdLsmt = profile.node.utxo_lsmt && isProducer; lmdbDatabasePath = liveTablesPath i; - lsmDatabasePath = liveTablesPath i; + lsmDatabasePath = liveTablesPath i; ## Combine: ## 0. baseNodeConfig (coming cardanoLib's testnet environ) @@ -88,123 +94,146 @@ let ## 4. tracing backend's config nodeConfig = import ./tracing.nix - { - inherit (pkgs) lib; - inherit nodeSpec; - inherit (profile.node) tracing_backend tracer; - } + { + inherit (pkgs) lib; + inherit nodeSpec; + inherit (profile.node) tracing_backend tracer; + } + (recursiveUpdate (recursiveUpdate - (recursiveUpdate + ( + recursiveUpdate (removeAttrs baseNodeConfig - [ ## Let the genesis hashes be auto-computed by the node: + [ + ## Let the genesis hashes be auto-computed by the node: "ByronGenesisHash" "ShelleyGenesisHash" "AlonzoGenesisHash" "ConwayGenesisHash" "DijkstraGenesisHash" - ] // - { + ] + // { ExperimentalHardForksEnabled = true; ExperimentalProtocolsEnabled = true; - TurnOnLogMetrics = true; - SnapshotInterval = 4230; - ChainSyncIdleTimeout = 0; - PeerSharing = false; + TurnOnLogMetrics = true; + ChainSyncIdleTimeout = 0; + PeerSharing = false; ## defaults taken from: ouroboros-network/src/Ouroboros/Network/Diffusion/Configuration.hs ## NB. the following inequality must hold: known >= established >= active >= 0 - SyncTargetNumberOfActivePeers = max 15 valency; # set to same value as TargetNumberOfActivePeers + SyncTargetNumberOfActivePeers = max 15 valency; # set to same value as TargetNumberOfActivePeers SyncTargetNumberOfEstablishedPeers = max 40 valency; - TargetNumberOfActivePeers = max 15 valency; - TargetNumberOfEstablishedPeers = max 40 valency; + TargetNumberOfActivePeers = max 15 valency; + TargetNumberOfEstablishedPeers = max 40 valency; - ByronGenesisFile = "../genesis/byron/genesis.json"; - ShelleyGenesisFile = "../genesis/genesis-shelley.json"; - AlonzoGenesisFile = "../genesis/genesis.alonzo.json"; - ConwayGenesisFile = "../genesis/genesis.conway.json"; - DijkstraGenesisFile = "../genesis/genesis.dijkstra.json"; - } // optionalAttrs (profile.node.utxo_lsmt && isProducer) + ByronGenesisFile = "../genesis/byron/genesis.json"; + ShelleyGenesisFile = "../genesis/genesis-shelley.json"; + AlonzoGenesisFile = "../genesis/genesis.alonzo.json"; + ConwayGenesisFile = "../genesis/genesis.conway.json"; + DijkstraGenesisFile = "../genesis/genesis.dijkstra.json"; + } + // optionalAttrs (profile.node.utxo_lsmt && isProducer) { LedgerDB = { Backend = "V2LSM"; LSMDatabasePath = liveTablesPath i; }; - } // optionalAttrs (profile.node.utxo_lmdb && isProducer) + } + // optionalAttrs (profile.node.utxo_lmdb && isProducer) { LedgerDB = { Backend = "V1LMDB"; LiveTablesPath = liveTablesPath i; }; }) - (if __hasAttr "preset" profile && profile.preset != null - ## It's either an undisturbed preset, - ## or a hardforked setup. - then readJSONMay (../profile/presets + "/${profile.preset}/config.json") - else configHardforksIntoEra profile.era)) - profile.node.verbatim); + { + ## This LedgerDB attrset is defined regardless of backend choice. + ## It assumes the Backend default to be "V2InMemory"; it must not overwrite any of the above, more specfic choices. + LedgerDB = { + Snapshots = { + SnapshotInterval = 4230; + # Disable the randomised delay for kicking off snapshots: + # For benchmarks, exact timing needs to be reproducible, and identical for all nodes. + MinDelay = 0; + MaxDelay = 0; + }; + }; + } + ) + ( + if __hasAttr "preset" profile && profile.preset != null + ## It's either an undisturbed preset, + ## or a hardforked setup. + then readJSONMay (../profile/presets + "/${profile.preset}/config.json") + else configHardforksIntoEra profile.era + )) + profile.node.verbatim); extraArgs = - [ "+RTS" "-scardano-node.gcstats" "-RTS" ] - ++ - optionals (nodeSpec.shutdown_on_block_synced != null) [ + ["+RTS" "-scardano-node.gcstats" "-RTS"] + ++ optionals (nodeSpec.shutdown_on_block_synced != null) [ "--shutdown-on-block-synced" (toString nodeSpec.shutdown_on_block_synced) - ] ++ - optionals (nodeSpec.shutdown_on_slot_synced != null) [ + ] + ++ optionals (nodeSpec.shutdown_on_slot_synced != null) [ "--shutdown-on-slot-synced" (toString nodeSpec.shutdown_on_slot_synced) ]; - } // optionalAttrs ((profiling.profilingTypeParam or "none") != "none") { + } + // optionalAttrs ((profiling.profilingTypeParam or "none") != "none") { # Add the profiling `-h*` RTS option. profiling = profiling.profilingTypeParam; - } // optionalAttrs (profiling.eventlog or false) { + } + // optionalAttrs (profiling.eventlog or false) { # Add the `-l` RTS param with profiling. eventlog = true; - # Decide where the executable comes from: - ######################################### - } // optionalAttrs (!backend.useCabalRun) { - package = workbenchNix.haskellProject.exes.cardano-node; - } // optionalAttrs backend.useCabalRun { + # Decide where the executable comes from: + ######################################### + } + // optionalAttrs (!backend.useCabalRun) { + package = workbenchNix.haskellProject.exes.cardano-node; + } + // optionalAttrs backend.useCabalRun { # Allow the shell function to take precedence. executable = "cardano-node"; - ######################################### - } // optionalAttrs isProducer { + ######################################### + } + // optionalAttrs isProducer { operationalCertificate = "../genesis/node-keys/node${toString i}.opcert"; - kesKey = "../genesis/node-keys/node-kes${toString i}.skey"; - vrfKey = "../genesis/node-keys/node-vrf${toString i}.skey"; - } // optionalAttrs profile.node.tracer { + kesKey = "../genesis/node-keys/node-kes${toString i}.skey"; + vrfKey = "../genesis/node-keys/node-vrf${toString i}.skey"; + } + // optionalAttrs profile.node.tracer { tracerSocketPathConnect = mkDefault "../tracer/tracer.socket"; }; - time_fmtstr = - "{ " + escape [''"''] (concatStringsSep ''\n, '' time_entries) + " }"; - time_entries = [ - ''"wall_clock_s": %e'' - ''"user_cpu_s": %U'' - ''"sys_cpu_s": %S'' - ''"avg_cpu_pct": "%P"'' - ''"rss_peak_kb": %M'' - ''"signals_received": %k'' - ''"ctxsw_involuntary": %c'' - ''"ctxsw_volunt_waits": %w'' - ''"pageflt_major": %F'' - ''"pageflt_minor": %R'' - ''"swaps": %W'' - ''"io_fs_reads": %I'' - ''"io_fs_writes": %O'' - ''"cmdline": "%C"'' - ''"exit_code": %x'' - ]; + time_fmtstr = + "{ " + escape [''"''] (concatStringsSep ''\n, '' time_entries) + " }"; + time_entries = [ + ''"wall_clock_s": %e'' + ''"user_cpu_s": %U'' + ''"sys_cpu_s": %S'' + ''"avg_cpu_pct": "%P"'' + ''"rss_peak_kb": %M'' + ''"signals_received": %k'' + ''"ctxsw_involuntary": %c'' + ''"ctxsw_volunt_waits": %w'' + ''"pageflt_major": %F'' + ''"pageflt_minor": %R'' + ''"swaps": %W'' + ''"io_fs_reads": %I'' + ''"io_fs_writes": %O'' + ''"cmdline": "%C"'' + ''"exit_code": %x'' + ]; ## Given an env config, evaluate it and produce the node service. ## Call the given function on this service. ## ## evalServiceConfigToService :: NodeServiceConfig -> NodeService ## - evalServiceConfigToService = - serviceConfig: - let + evalServiceConfigToService = serviceConfig: let systemdCompat.options = { systemd.services = mkOption {}; systemd.sockets = mkOption {}; @@ -215,98 +244,108 @@ let }; eval = let extra = { - services.cardano-node = { - enable = true; - } // serviceConfig; + services.cardano-node = + { + enable = true; + } + // serviceConfig; }; - in evalModules { - prefix = []; - modules = import ../../nixos/module-list.nix - ++ [ systemdCompat extra - { config._module.args = { inherit pkgs; }; } - ] - ++ [ backend.service-modules.node or {} ]; - }; in - eval.config.services.cardano-node; - - topologiesJsonPaths = - let projections = - pkgs.runCommand "workbench-profile-files-${profileName}-topologies" - { nativeBuildInputs = with pkgs; - # A workbench with only the dependencies needed for this command. - [ workbenchNix.workbench - jq - workbenchNix.haskellProject.exes.cardano-topology - ]; - } - '' - mkdir "$out" - ${builtins.concatStringsSep - "\n" - (pkgs.lib.mapAttrsToList - (name: nodeSpec: '' - wb topology projection-for \ - "local-${nodeSpec.kind}" \ - "${toString nodeSpec.i}" \ - "${profileJsonPath}" \ - "${topologyJsonPath}" \ - "${toString backend.basePort}" \ - > "$out"/"topology-${name}.json" - '') - nodeSpecs - ) - } - '' - ; - in pkgs.lib.mapAttrs - (name: _: "${projections}/topology-${name}.json") - nodeSpecs - ; + evalModules { + prefix = []; + modules = + import ../../nixos/module-list.nix + ++ [ + systemdCompat + extra + {config._module.args = {inherit pkgs;};} + ] + ++ [backend.service-modules.node or {}]; + }; + in + eval.config.services.cardano-node; - nodeService = - { name, i, mode ? null, ... }@nodeSpec: - let - modeIdSuffix = if mode == null then "" else "." + mode; - serviceConfig = nodeServiceConfig nodeSpec valency; - service = evalServiceConfigToService serviceConfig; + topologiesJsonPaths = let + projections = + pkgs.runCommand "workbench-profile-files-${profileName}-topologies" + { + nativeBuildInputs = with pkgs; + # A workbench with only the dependencies needed for this command. + [ + workbenchNix.workbench + jq + workbenchNix.haskellProject.exes.cardano-topology + ]; + } + '' + mkdir "$out" + ${ + builtins.concatStringsSep + "\n" + ( + pkgs.lib.mapAttrsToList + (name: nodeSpec: '' + wb topology projection-for \ + "local-${nodeSpec.kind}" \ + "${toString nodeSpec.i}" \ + "${profileJsonPath}" \ + "${topologyJsonPath}" \ + "${toString backend.basePort}" \ + > "$out"/"topology-${name}.json" + '') + nodeSpecs + ) + } + ''; + in + pkgs.lib.mapAttrs + (name: _: "${projections}/topology-${name}.json") + nodeSpecs; - topology = - rec { - JSON = topologiesJsonPaths.${name}; - value = __fromJSON (__readFile JSON); - } - ; + nodeService = { + name, + i, + mode ? null, + ... + } @ nodeSpec: let + modeIdSuffix = + if mode == null + then "" + else "." + mode; + serviceConfig = nodeServiceConfig nodeSpec valency; + service = evalServiceConfigToService serviceConfig; - valency = - let - topo = topology.value; - val = if hasAttr "localRoots" topo && __length topo.localRoots > 0 - then let lr = head topo.localRoots; in lr.valency - else length (topo.Producers or []); - in val; + topology = rec { + JSON = topologiesJsonPaths.${name}; + value = __fromJSON (__readFile JSON); + }; - in { - start = - '' - #!${pkgs.stdenv.shell} + valency = let + topo = topology.value; + val = + if hasAttr "localRoots" topo && __length topo.localRoots > 0 + then let lr = head topo.localRoots; in lr.valency + else length (topo.Producers or []); + in + val; + in { + start = '' + #!${pkgs.stdenv.shell} - export TRACE_DISPATCHER_LOGGING_HOSTNAME=${name} + export TRACE_DISPATCHER_LOGGING_HOSTNAME=${name} - ${service.script} - '' - ; + ${service.script} + ''; - config = service.nodeConfig; + config = service.nodeConfig; - inherit topology; - }; + inherit topology; + }; ## ## node-services :: Map NodeName (NodeSpec, ServiceConfig, Service, NodeConfig, Script) ## node-services = mapAttrs (_: nodeService) nodeSpecs; -in -{ +in { inherit node-services; } From 74c263a375be0775161d3fc9a99acc426521951f Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Mon, 11 May 2026 17:20:48 -0500 Subject: [PATCH 19/40] bump: iohkNix for node 11.1 cfg changes --- flake.lock | 7 ++++--- flake.nix | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 2f38f596a20..209ea32951a 100644 --- a/flake.lock +++ b/flake.lock @@ -604,15 +604,16 @@ "sodium": "sodium" }, "locked": { - "lastModified": 1777941182, - "narHash": "sha256-FX3+8GIrB2z4akmcYTStELDKVJWgqy9yFt0mxwpU3Qc=", + "lastModified": 1781758324, + "narHash": "sha256-DxBUgw7medqNE9zBhlaf7yez8iKDzILnofByNJjn1D8=", "owner": "input-output-hk", "repo": "iohk-nix", - "rev": "9de00113c11ba8cac908a63acf34b193cda7475b", + "rev": "4d76d0289b39f9e9ffac2dbe925bbe863b76f988", "type": "github" }, "original": { "owner": "input-output-hk", + "ref": "node-11.1", "repo": "iohk-nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 9d2e94b87f8..555edf34766 100644 --- a/flake.nix +++ b/flake.nix @@ -48,7 +48,7 @@ incl.url = "github:divnix/incl"; iohkNix = { - url = "github:input-output-hk/iohk-nix"; + url = "github:input-output-hk/iohk-nix/node-11.1"; inputs.nixpkgs.follows = "nixpkgs"; }; From 00c384aa67cd6d24b5fa3ab92375bcb8234e971d Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Mon, 11 May 2026 17:25:42 -0500 Subject: [PATCH 20/40] tracing: rm old tracing system cfg --- .../cardano/mainnet-config-legacy.json | 115 ----------------- configuration/cardano/mainnet-config.json | 8 +- configuration/cardano/mainnet-config.yaml | 11 -- .../testnet-template-config-legacy.json | 116 ------------------ configuration/cardano/update-config-files.sh | 2 - nix/binary-release.nix | 6 - nix/docker/README.md | 20 --- nix/docker/default.nix | 2 +- nix/haskell.nix | 1 - scripts/lite/mainnet-legacy-tracing.sh | 33 ----- 10 files changed, 2 insertions(+), 312 deletions(-) delete mode 100644 configuration/cardano/mainnet-config-legacy.json delete mode 100644 configuration/cardano/testnet-template-config-legacy.json delete mode 100755 scripts/lite/mainnet-legacy-tracing.sh diff --git a/configuration/cardano/mainnet-config-legacy.json b/configuration/cardano/mainnet-config-legacy.json deleted file mode 100644 index a83cf83f481..00000000000 --- a/configuration/cardano/mainnet-config-legacy.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "AlonzoGenesisFile": "mainnet-alonzo-genesis.json", - "AlonzoGenesisHash": "7e94a15f55d1e82d10f09203fa1d40f8eede58fd8066542cf6566008068ed874", - "ByronGenesisFile": "mainnet-byron-genesis.json", - "ByronGenesisHash": "5f20df933584822601f9e3f8c024eb5eb252fe8cefb24d1317dc3d432e940ebb", - "CheckpointsFile": "mainnet-checkpoints.json", - "CheckpointsFileHash": "3e6dee5bae7acc6d870187e72674b37c929be8c66e62a552cf6a876b1af31ade", - "ConsensusMode": "PraosMode", - "ConwayGenesisFile": "mainnet-conway-genesis.json", - "ConwayGenesisHash": "15a199f895e461ec0ffc6dd4e4028af28a492ab4e806d39cb674c88f7643ef62", - "LastKnownBlockVersion-Alt": 0, - "LastKnownBlockVersion-Major": 3, - "LastKnownBlockVersion-Minor": 0, - "LedgerDB": { - "Backend": "V2InMemory", - "QueryBatchSize": 100000, - "Snapshots": { - "NumOfDiskSnapshots": 2, - "SnapshotInterval": 4320 - } - }, - "MaxKnownMajorProtocolVersion": 2, - "MinNodeVersion": "10.7.0", - "Protocol": "Cardano", - "RequiresNetworkMagic": "RequiresNoMagic", - "ShelleyGenesisFile": "mainnet-shelley-genesis.json", - "ShelleyGenesisHash": "1a3be38bcbb7911969283716ad7aa550250226b76a61fc51cc9a9a35d9276d81", - "TraceAcceptPolicy": true, - "TraceBlockFetchClient": false, - "TraceBlockFetchDecisions": false, - "TraceBlockFetchProtocol": false, - "TraceBlockFetchProtocolSerialised": false, - "TraceBlockFetchServer": false, - "TraceChainDb": true, - "TraceChainSyncBlockServer": false, - "TraceChainSyncClient": false, - "TraceChainSyncHeaderServer": false, - "TraceChainSyncProtocol": false, - "TraceConnectionManager": true, - "TraceDNSResolver": true, - "TraceDNSSubscription": true, - "TraceDiffusionInitialization": true, - "TraceErrorPolicy": true, - "TraceForge": true, - "TraceHandshake": true, - "TraceInboundGovernor": true, - "TraceIpSubscription": true, - "TraceLedgerPeers": true, - "TraceLocalChainSyncProtocol": false, - "TraceLocalConnectionManager": true, - "TraceLocalErrorPolicy": true, - "TraceLocalHandshake": true, - "TraceLocalRootPeers": true, - "TraceLocalTxSubmissionProtocol": false, - "TraceLocalTxSubmissionServer": false, - "TraceMempool": false, - "TraceMux": false, - "TracePeerSelection": true, - "TracePeerSelectionActions": true, - "TracePublicRootPeers": true, - "TraceServer": true, - "TraceTxInbound": false, - "TraceTxOutbound": false, - "TraceTxSubmissionProtocol": false, - "TracingVerbosity": "NormalVerbosity", - "TurnOnLogMetrics": true, - "TurnOnLogging": true, - "UseTraceDispatcher": false, - "defaultBackends": [ - "KatipBK" - ], - "defaultScribes": [ - [ - "StdoutSK", - "stdout" - ] - ], - "hasEKG": 12788, - "hasPrometheus": [ - "127.0.0.1", - 12798 - ], - "minSeverity": "Info", - "options": { - "mapBackends": { - "cardano.node.metrics": [ - "EKGViewBK" - ], - "cardano.node.resources": [ - "EKGViewBK" - ] - }, - "mapSubtrace": { - "cardano.node.metrics": { - "subtrace": "Neutral" - } - } - }, - "rotation": { - "rpKeepFilesNum": 10, - "rpLogLimitBytes": 5000000, - "rpMaxAgeHours": 24 - }, - "setupBackends": [ - "KatipBK" - ], - "setupScribes": [ - { - "scFormat": "ScText", - "scKind": "StdoutSK", - "scName": "stdout", - "scRotation": null - } - ] -} diff --git a/configuration/cardano/mainnet-config.json b/configuration/cardano/mainnet-config.json index a63aa6f45c9..e1472171fcf 100644 --- a/configuration/cardano/mainnet-config.json +++ b/configuration/cardano/mainnet-config.json @@ -112,11 +112,5 @@ }, "TurnOnLogMetrics": true, "TurnOnLogging": true, - "UseTraceDispatcher": true, - "defaultBackends": [], - "defaultScribes": [], - "minSeverity": "Critical", - "options": {}, - "setupBackends": [], - "setupScribes": [] + "UseTraceDispatcher": true } diff --git a/configuration/cardano/mainnet-config.yaml b/configuration/cardano/mainnet-config.yaml index b0ebdd0892e..0fe7036107a 100644 --- a/configuration/cardano/mainnet-config.yaml +++ b/configuration/cardano/mainnet-config.yaml @@ -258,17 +258,6 @@ TraceOptions: Mempool.SyncNotNeeded: severity: Silence -# Required by the legacy tracing system, this key is still required for -# cardano-node to start. -minSeverity: Critical - -# Required by some legacy tests which may otherwise fail to start. -defaultBackends: [] -defaultScribes: [] -options: {} -setupBackends: [] -setupScribes: [] - # Set or unset the mempool capacity override in number of bytes. # # This is intended for testing, and for low-resource machines to run with a smaller mempool. diff --git a/configuration/cardano/testnet-template-config-legacy.json b/configuration/cardano/testnet-template-config-legacy.json deleted file mode 100644 index 42a402569f7..00000000000 --- a/configuration/cardano/testnet-template-config-legacy.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "AlonzoGenesisFile": "alonzo-genesis.json", - "ApplicationName": "cardano-sl", - "ApplicationVersion": 0, - "ByronGenesisFile": "byron-genesis.json", - "ConwayGenesisFile": "conway-genesis.json", - "DijkstraGenesisFile": "dijkstra-genesis.json", - "ExperimentalHardForksEnabled": false, - "ExperimentalProtocolsEnabled": true, - "LastKnownBlockVersion-Alt": 0, - "LastKnownBlockVersion-Major": 3, - "LastKnownBlockVersion-Minor": 1, - "LedgerDB": { - "Backend": "V2InMemory", - "QueryBatchSize": 100000, - "Snapshots": { - "NumOfDiskSnapshots": 2, - "SnapshotInterval": 216 - } - }, - "MaxConcurrencyDeadline": 4, - "MaxKnownMajorProtocolVersion": 2, - "PBftSignatureThreshold": 1.1, - "Protocol": "Cardano", - "RequiresNetworkMagic": "RequiresMagic", - "ShelleyGenesisFile": "shelley-genesis.json", - "TestAllegraHardForkAtEpoch": 0, - "TestAlonzoHardForkAtEpoch": 0, - "TestMaryHardForkAtEpoch": 0, - "TestShelleyHardForkAtEpoch": 0, - "TraceAcceptPolicy": true, - "TraceBlockFetchClient": false, - "TraceBlockFetchDecisions": false, - "TraceBlockFetchProtocol": false, - "TraceBlockFetchProtocolSerialised": false, - "TraceBlockFetchServer": false, - "TraceChainDb": true, - "TraceChainSyncBlockServer": false, - "TraceChainSyncClient": false, - "TraceChainSyncHeaderServer": false, - "TraceChainSyncProtocol": false, - "TraceConnectionManager": true, - "TraceDNSResolver": true, - "TraceDNSSubscription": true, - "TraceDiffusionInitialization": true, - "TraceErrorPolicy": true, - "TraceForge": true, - "TraceHandshake": false, - "TraceInboundGovernor": true, - "TraceIpSubscription": true, - "TraceLedgerPeers": true, - "TraceLocalChainSyncProtocol": false, - "TraceLocalErrorPolicy": true, - "TraceLocalHandshake": false, - "TraceLocalRootPeers": true, - "TraceLocalTxSubmissionProtocol": false, - "TraceLocalTxSubmissionServer": false, - "TraceMempool": false, - "TraceMux": false, - "TracePeerSelection": true, - "TracePeerSelectionActions": true, - "TracePublicRootPeers": true, - "TraceServer": true, - "TraceTxInbound": false, - "TraceTxOutbound": false, - "TraceTxSubmissionProtocol": false, - "TracingVerbosity": "NormalVerbosity", - "TurnOnLogMetrics": true, - "TurnOnLogging": true, - "UseTraceDispatcher": false, - "defaultBackends": [ - "KatipBK" - ], - "defaultScribes": [ - [ - "StdoutSK", - "cardano" - ] - ], - "hasEKG": 12788, - "hasPrometheus": [ - "127.0.0.1", - 12798 - ], - "minSeverity": "Debug", - "options": { - "mapBackends": { - "cardano.node.metrics": [ - "EKGViewBK" - ], - "cardano.node.resources": [ - "EKGViewBK" - ] - }, - "mapSubtrace": { - "cardano.node.metrics": { - "subtrace": "Neutral" - } - } - }, - "rotation": { - "rpKeepFilesNum": 10, - "rpLogLimitBytes": 5000000, - "rpMaxAgeHours": 24 - }, - "setupBackends": [ - "KatipBK" - ], - "setupScribes": [ - { - "scFormat": "ScText", - "scKind": "StdoutSK", - "scName": "cardano" - } - ] -} diff --git a/configuration/cardano/update-config-files.sh b/configuration/cardano/update-config-files.sh index 28ca99ed46b..f7693918c8b 100755 --- a/configuration/cardano/update-config-files.sh +++ b/configuration/cardano/update-config-files.sh @@ -30,7 +30,6 @@ copyCfg "mainnet-alonzo-genesis.json" copyCfg "mainnet-byron-genesis.json" copyCfg "mainnet-checkpoints.json" copyCfg "mainnet-config.json" -copyCfg "mainnet-config-legacy.json" copyCfg "mainnet-conway-genesis.json" copyCfg "mainnet-peer-snapshot.json" copyCfg "mainnet-shelley-genesis.json" @@ -40,7 +39,6 @@ copyCfg "mainnet-topology.json" copyTmplCfg "alonzo.json" copyTmplCfg "byron.json" copyTmplCfg "config.json" -copyTmplCfg "config-legacy.json" copyTmplCfg "conway.json" copyTmplCfg "dijkstra.json" copyTmplCfg "shelley.json" diff --git a/nix/binary-release.nix b/nix/binary-release.nix index 8a5e68bb5cc..5f4a7bbb79b 100644 --- a/nix/binary-release.nix +++ b/nix/binary-release.nix @@ -41,11 +41,6 @@ let (builtins.toJSON (env.nodeConfig // genesisAttrs)); - nodeConfigLegacy= pkgs.writeText - "config-legacy.json" - (builtins.toJSON - (env.nodeConfigLegacy // genesisAttrs)); - submitApiConfig = pkgs.writeText "submit-api-config.json" (builtins.toJSON env.submitApiConfig); @@ -68,7 +63,6 @@ let '' mkdir -p "share/${name}" jq . < "${nodeConfig}" > share/${name}/config.json - jq . < "${nodeConfigLegacy}" > share/${name}/config-legacy.json jq . < "${submitApiConfig}" > share/${name}/submit-api-config.json jq . < "${tracerConfig}" > share/${name}/tracer-config.json jq . < "${peerSnapshot}" > share/${name}/peer-snapshot.json diff --git a/nix/docker/README.md b/nix/docker/README.md index 902fe87ac80..d6d51575f1e 100644 --- a/nix/docker/README.md +++ b/nix/docker/README.md @@ -232,26 +232,6 @@ otherwise ledger replay from genesis will re-occur. For more info, see the [UTxO Migration Guide](https://ouroboros-consensus.cardano.intersectmbo.org/docs/references/miscellaneous/utxo-hd/migrating/). -## Legacy Tracing System -Cardano-node now defaults to using the new tracing system. The legacy tracing -system is deprecated and will be removed in a future node version. While still -available, the legacy tracing system can be used by following the example -above in "custom" mode whereby config is passed, and in this case, the config -passed is the legacy style configuration. - -Legacy default configuration files are also available within the image at paths: -`/opt/cardano/config/$NETWORK/config-legacy.json` - -An example of legacy tracing system usage is: -``` -docker run \ - -v preprod-data:/data \ - -e CARDANO_CONFIG="/opt/cardano/config/preprod/config-legacy.json" \ - -e CARDANO_TOPOLOGY="/opt/cardano/config/preprod/topology.json" \ - ghcr.io/intersectmbo/cardano-node:dev \ - run -``` - # Cardano Submit API Image Operation ## Scripts Mode diff --git a/nix/docker/default.nix b/nix/docker/default.nix index 3805b4f85ce..cde340be66e 100644 --- a/nix/docker/default.nix +++ b/nix/docker/default.nix @@ -118,7 +118,7 @@ let done # Adjust genesis file, config refs - for i in config config-legacy db-sync-config; do + for i in config db-sync-config; do if [ -f "$out/config/$ENV/$i.json" ]; then sed -i "s|\"$ENV-|\"|g" "$out/config/$ENV/$i.json" fi diff --git a/nix/haskell.nix b/nix/haskell.nix index d086e60638f..d3da3f0c03d 100644 --- a/nix/haskell.nix +++ b/nix/haskell.nix @@ -216,7 +216,6 @@ let mainnetConfigFiles = [ "configuration/cardano/mainnet-config.yaml" "configuration/cardano/mainnet-config.json" - "configuration/cardano/mainnet-config-legacy.json" "configuration/cardano/mainnet-byron-genesis.json" "configuration/cardano/mainnet-shelley-genesis.json" "configuration/cardano/mainnet-alonzo-genesis.json" diff --git a/scripts/lite/mainnet-legacy-tracing.sh b/scripts/lite/mainnet-legacy-tracing.sh deleted file mode 100755 index 3fbe4b3d2ae..00000000000 --- a/scripts/lite/mainnet-legacy-tracing.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# This script connects a node to mainnet - -ROOT="$(realpath "$(dirname "$0")/../..")" -configuration="${ROOT}/configuration/cardano" - -data_dir=mainnetsingle -mkdir -p "${data_dir}" -db_dir="${data_dir}/db/node" -mkdir -p "${db_dir}" -socket_dir="${data_dir}/socket" -mkdir -p "${socket_dir}" - -# Launch a node -cabal run exe:cardano-node -- run \ - --config "${configuration}/mainnet-config-legacy.json" \ - --topology "${configuration}/mainnet-topology.json" \ - --database-path "${db_dir}" \ - --socket-path "${socket_dir}/node-1-socket" \ - --host-addr "0.0.0.0" \ - --port "3001" - - - -function cleanup() -{ - for child in $(jobs -p); do - echo kill "$child" && kill "$child" - done -} - -trap cleanup EXIT From cdfc8295a3477f32e01075a4572827d62f67253e Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Tue, 23 Jun 2026 21:24:30 -0500 Subject: [PATCH 21/40] cfg/ci: update cfg to iohkNix and match ci --- .github/workflows/check-mainnet-config.yml | 1 - configuration/cardano/mainnet-config.json | 7 +--- configuration/cardano/mainnet-config.yaml | 40 ++++++++++--------- .../cardano/testnet-template-config.json | 11 +---- 4 files changed, 24 insertions(+), 35 deletions(-) diff --git a/.github/workflows/check-mainnet-config.yml b/.github/workflows/check-mainnet-config.yml index 9b3c7ebf98a..4f4eb657b9b 100644 --- a/.github/workflows/check-mainnet-config.yml +++ b/.github/workflows/check-mainnet-config.yml @@ -43,7 +43,6 @@ jobs: 'mainnet-byron-genesis.json' 'mainnet-checkpoints.json' 'mainnet-config.json' - 'mainnet-config-legacy.json' 'mainnet-peer-snapshot.json' 'mainnet-shelley-genesis.json' 'mainnet-topology.json' diff --git a/configuration/cardano/mainnet-config.json b/configuration/cardano/mainnet-config.json index e1472171fcf..5027e9dd27a 100644 --- a/configuration/cardano/mainnet-config.json +++ b/configuration/cardano/mainnet-config.json @@ -20,7 +20,7 @@ } }, "MaxKnownMajorProtocolVersion": 2, - "MinNodeVersion": "10.7.0", + "MinNodeVersion": "11.1.0", "Protocol": "Cardano", "RequiresNetworkMagic": "RequiresNoMagic", "ShelleyGenesisFile": "mainnet-shelley-genesis.json", @@ -109,8 +109,5 @@ "Startup.DiffusionInit": { "severity": "Info" } - }, - "TurnOnLogMetrics": true, - "TurnOnLogging": true, - "UseTraceDispatcher": true + } } diff --git a/configuration/cardano/mainnet-config.yaml b/configuration/cardano/mainnet-config.yaml index 0fe7036107a..51c99d46458 100644 --- a/configuration/cardano/mainnet-config.yaml +++ b/configuration/cardano/mainnet-config.yaml @@ -79,43 +79,45 @@ ConsensusMode: PraosMode # Additional configuration options can be found at: # https://ouroboros-consensus.cardano.intersectmbo.org/docs/for-developers/utxo-hd/migrating -LedgerDB: - # The backend can either be in memory with `V2InMemory` or on disk with - # `V2LSM`. - Backend: V2InMemory +LedgerDB: # When querying the store for a big range of UTxOs (such as with # QueryUTxOByAddress), the store will be read in batches of this size. QueryBatchSize: 100000 + # The backend can either be in memory with `V2InMemory` or on disk with + # `V2LSM`. + Backend: V2InMemory + # Instead of an object with individual options, a predefined snapshot # policy can be selected by name, e.g. `Snapshots: Mithril`. Snapshots: - # The time interval between snapshots, in slots. + # The snapshot interval in slots. SnapshotInterval: 4320 + # Start taking the snaphots at a slot offset. + # SlotOffset = 172800; + + # A minimum duration between snapshots, in seconds (used to avoid excessive snapshots while syncing). + # Default is 10 minutes. + # RateLimit = 600; + + # Randomised snapshot delay range, in seconds. + # Both Min and Max need to be specified, otherwise the default delay of (5min, 10min) will be used. + # MinDelay = 300; + # MaxDelay = 600; + # The number of disk snapshots to keep. NumOfDiskSnapshots: 2 ##### Version Information ##### -# Min is currently 10.7.0 due to change of config bundled peer-snapshot -# version. -MinNodeVersion: "10.7.0" +# Min is currently 11.1.0 due to removal of legacy tracing system and +# introduction of deterministic snapshots. +MinNodeVersion: "11.1.0" ##### Logging configuration ##### -# Enable or disable logging overall -TurnOnLogging: True - -# Enable the collection of various OS metrics such as memory and CPU use. -# These metrics are traced in the context name: 'cardano.node.metrics' and can -# be directed to the logs or monitoring backends. -TurnOnLogMetrics: True - -# Use the modern tracing system instead of the legacy tracing system. -UseTraceDispatcher: True - # Match the metrics prefix of the legacy tracing system to minimize breaking # changes. TraceOptionMetricsPrefix: "cardano.node.metrics." diff --git a/configuration/cardano/testnet-template-config.json b/configuration/cardano/testnet-template-config.json index ad1471cb456..720f362803d 100644 --- a/configuration/cardano/testnet-template-config.json +++ b/configuration/cardano/testnet-template-config.json @@ -113,14 +113,5 @@ "Startup.DiffusionInit": { "severity": "Info" } - }, - "TurnOnLogMetrics": true, - "TurnOnLogging": true, - "UseTraceDispatcher": true, - "defaultBackends": [], - "defaultScribes": [], - "minSeverity": "Critical", - "options": {}, - "setupBackends": [], - "setupScribes": [] + } } From df779aa20c17eacbfa0993ff4e271c76c902254d Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Mon, 18 May 2026 16:12:14 -0500 Subject: [PATCH 22/40] nodeNixosSvc: add lib.types.path to avoid eval fails on path type checks --- nix/nixos/cardano-node-service.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix index 33860e28d54..0c29c6f4f97 100644 --- a/nix/nixos/cardano-node-service.nix +++ b/nix/nixos/cardano-node-service.nix @@ -5,7 +5,7 @@ with lib; with builtins; let - inherit (types) attrs attrsOf bool either enum functionTo int listOf package nullOr str; + inherit (types) attrs attrsOf bool either enum functionTo int listOf package path nullOr str; cfg = config.services.cardano-node; envConfig = cfg.environments.${cfg.environment}; From c13b829c014e1c8aad3ed640c84a3c89a2dd58b9 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Mon, 11 May 2026 19:16:11 -0500 Subject: [PATCH 23/40] releaseBins: add mithril flake input and extract mithril-signer for x86_64-linux release artifact --- flake.lock | 130 +++++++++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 15 +++++-- 2 files changed, 141 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 209ea32951a..087cd2031d4 100644 --- a/flake.lock +++ b/flake.lock @@ -142,6 +142,21 @@ "type": "github" } }, + "crane": { + "locked": { + "lastModified": 1776635034, + "narHash": "sha256-OEOJrT3ZfwbChzODfIH4GzlNTtOFuZFWPtW7jIeR8xU=", + "owner": "ipetkov", + "repo": "crane", + "rev": "dc7496d8ea6e526b1254b55d09b966e94673750f", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, "customConfig": { "locked": { "lastModified": 1630400035, @@ -206,6 +221,24 @@ "type": "github" } }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1775087534, + "narHash": "sha256-91qqW8lhL7TLwgQWijoGBbiD4t7/q75KTi8NxjVmSmA=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "3107b77cd68437b9a76194f0f7f9c55f2329ca5b", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "flake-utils": { "locked": { "lastModified": 1667395993, @@ -635,6 +668,29 @@ "type": "github" } }, + "mithril": { + "inputs": { + "crane": "crane", + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay", + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1776926918, + "narHash": "sha256-muV2LpheC4OZsU1ipL9j6TmjGufMpGrXzvon+eWahgg=", + "owner": "input-output-hk", + "repo": "mithril", + "rev": "2478748ea9771baed8181ef0938c78f79ed60760", + "type": "github" + }, + "original": { + "owner": "input-output-hk", + "ref": "refs/tags/2617.0", + "repo": "mithril", + "type": "github" + } + }, "nixlib": { "locked": { "lastModified": 1667696192, @@ -650,6 +706,22 @@ "type": "github" } }, + "nixpkgs": { + "locked": { + "lastModified": 1776329215, + "narHash": "sha256-a8BYi3mzoJ/AcJP8UldOx8emoPRLeWqALZWu4ZvjPXw=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "b86751bc4085f48661017fa226dee99fab6c651b", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs-2305": { "locked": { "lastModified": 1705033721, @@ -730,6 +802,21 @@ "type": "github" } }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1774748309, + "narHash": "sha256-+U7gF3qxzwD5TZuANzZPeJTZRHS29OFQgkQ2kiTJBIQ=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "333c4e0545a6da976206c74db8773a1645b5870a", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, "nixpkgs-unstable": { "locked": { "lastModified": 1759070547, @@ -774,6 +861,7 @@ "haskellNix": "haskellNix", "incl": "incl", "iohkNix": "iohkNix", + "mithril": "mithril", "nixpkgs": [ "haskellNix", "nixpkgs-unstable" @@ -781,6 +869,27 @@ "utils": "utils" } }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "mithril", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1776654897, + "narHash": "sha256-Vqi4AiJVCcBGn/RmBtRCgyH5rCxqm/w0xV9diJWF1Ic=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "25d75be8139815a53560745fa060909777495105", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, "secp256k1": { "flake": false, "locked": { @@ -846,6 +955,27 @@ "type": "github" } }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "mithril", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1775636079, + "narHash": "sha256-pc20NRoMdiar8oPQceQT47UUZMBTiMdUuWrYu2obUP0=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "790751ff7fd3801feeaf96d7dc416a8d581265ba", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + }, "utils": { "inputs": { "systems": "systems" diff --git a/flake.nix b/flake.nix index 555edf34766..a8959cce99c 100644 --- a/flake.nix +++ b/flake.nix @@ -55,6 +55,10 @@ nixpkgs.follows = "haskellNix/nixpkgs-unstable"; utils.url = "github:numtide/flake-utils"; + + # Mithril signer is required as a release artifact constitutent. + # Use explicit ref tag path to ensure we get exactly what we expect. + mithril.url = "github:input-output-hk/mithril?ref=refs/tags/2617.0"; }; outputs = { @@ -63,6 +67,7 @@ haskellNix, incl, iohkNix, + mithril, nixpkgs, self, utils, @@ -70,7 +75,7 @@ } @ input: let inherit (builtins) elem match; inherit (nixpkgs) lib; - inherit (lib) collect getAttr genAttrs filterAttrs hasPrefix head isDerivation mapAttrs optionalAttrs optionals recursiveUpdate; + inherit (lib) collect getAttr genAttrs filterAttrs hasPrefix head isDerivation mapAttrs optionalAttrs optional optionals recursiveUpdate; inherit (utils.lib) eachSystem flattenTree; inherit (iohkNix.lib) prefixNamesWith; removeRecurse = lib.filterAttrsRecursive (n: _: n != "recurseForDerivations"); @@ -364,9 +369,11 @@ inherit pkgs; inherit (exes.cardano-node.identifier) version; platform = "linux"; - exes = collect isDerivation ( - filterAttrs (n: _: elem n releaseBins) projectExes - ); + exes = + collect isDerivation ( + filterAttrs (n: _: elem n releaseBins) projectExes + ) + ++ optional (system == "x86_64-linux") mithril.packages.${system}.mithril-signer; }; internal.roots.project = muslProject.roots; variants = mapAttrs (_: v: removeAttrs v.musl ["variants"]) ciJobsVariants; From 8b4a6a19e3093870384b086b55f890aa62916930 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Tue, 16 Jun 2026 21:45:27 -0500 Subject: [PATCH 24/40] nodeNixosSvc: rm useNewTopology w/ p2p as the only networking mode --- nix/nixos/cardano-node-service.nix | 93 ++++++------------------------ 1 file changed, 19 insertions(+), 74 deletions(-) diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix index 0c29c6f4f97..c2c128afe86 100644 --- a/nix/nixos/cardano-node-service.nix +++ b/nix/nixos/cardano-node-service.nix @@ -32,16 +32,6 @@ let peerSnapshotFile = cfg.peerSnapshotFile i; }; - oldTopology = i: { - Producers = concatMap (g: map (a: { - addr = a.address; - inherit (a) port; - valency = a.valency or 1; - }) g.accessPoints) ( - cfg.producers ++ (cfg.instanceProducers i) ++ cfg.publicProducers ++ (cfg.instancePublicProducers i) - ); - }; - assertNewTopology = i: let checkEval = tryEval ( @@ -58,7 +48,7 @@ let selectTopology = i: if cfg.topology != null then cfg.topology - else toFile "topology.json" (toJSON (if (cfg.useNewTopology != false) then assertNewTopology i else oldTopology i)); + else toFile "topology.json" (toJSON (assertNewTopology i)); topology = i: if cfg.useSystemdReload @@ -72,27 +62,17 @@ let // (mapAttrs' (era: epoch: nameValuePair "Test${era}HardForkAtEpoch" epoch ) cfg.forceHardForks) - // (optionalAttrs (cfg.useNewTopology != false) ( - { - MaxConcurrencyBulkSync = 2; - } // optionalAttrs (cfg.useNewTopology == true) { - # Starting with node 10.6.0, p2p is the only network - # operating mode and EnableP2P becomes a no-op and is not - # declared by default. - # - # Older node versions which still require an explicit - # declaration can set useNewTopology true. - EnableP2P = true; - } // optionalAttrs (cfg.targetNumberOfRootPeers != null) { - TargetNumberOfRootPeers = cfg.targetNumberOfRootPeers; - } // optionalAttrs (cfg.targetNumberOfKnownPeers != null) { - TargetNumberOfKnownPeers = cfg.targetNumberOfKnownPeers; - } // optionalAttrs (cfg.targetNumberOfEstablishedPeers != null) { - TargetNumberOfEstablishedPeers = cfg.targetNumberOfEstablishedPeers; - } // optionalAttrs (cfg.targetNumberOfActivePeers != null) { - TargetNumberOfActivePeers = cfg.targetNumberOfActivePeers; - }) - ) + // { + MaxConcurrencyBulkSync = 2; + } // optionalAttrs (cfg.targetNumberOfRootPeers != null) { + TargetNumberOfRootPeers = cfg.targetNumberOfRootPeers; + } // optionalAttrs (cfg.targetNumberOfKnownPeers != null) { + TargetNumberOfKnownPeers = cfg.targetNumberOfKnownPeers; + } // optionalAttrs (cfg.targetNumberOfEstablishedPeers != null) { + TargetNumberOfEstablishedPeers = cfg.targetNumberOfEstablishedPeers; + } // optionalAttrs (cfg.targetNumberOfActivePeers != null) { + TargetNumberOfActivePeers = cfg.targetNumberOfActivePeers; + } ) cfg.extraNodeConfig; baseInstanceConfig = i: @@ -635,31 +615,6 @@ in { ''; }; - useNewTopology = mkOption { - type = nullOr bool; - default = cfg.nodeConfig.EnableP2P or null; - description = '' - Use new, p2p and ledger peers compatible topology. - - The useNewTopology option is deprecated and will be removed in the - future. As of cardano-node 10.6.0, this option should remain null. - For older node versions, a bool value can be set, but this will only - be supported until the Dijkstra hard fork at which point all - cardano-node versions will be compelled to upgrade and the - useNewTopology option will be removed. - - For node version < 10.6.0, useNewTopology will need to be explicitly - declared true or false to behave accordingly. If left null while - also using the auto-generated p2p topology, node will fail to start. - - For node version >= 10.6.0, useNewTopology should be left as null - until the option is removed after the Dijkstra hard fork. If - explicitly declared true, node will continue to work, but if declared - false while using the auto-generated legacy topology, node will fail to - start. - ''; - }; - useLegacyTracing = mkOption { type = bool; default = false; @@ -877,12 +832,9 @@ in { # # Mainnet does not yet require it, but declaring it will also # facilitate testing. - if (cfg.useNewTopology != false) - then - if cfg.useSystemdReload - then "peer-snapshot-${toString i}.json" - else toFile "peer-snapshot.json" (toJSON (envConfig.peerSnapshot)) - else null; + if cfg.useSystemdReload + then "peer-snapshot-${toString i}.json" + else toFile "peer-snapshot.json" (toJSON (envConfig.peerSnapshot)); example = i: "/etc/cardano-node/peer-snapshot-${toString i}.json"; apply = x: if lib.isFunction x then x else _: x; description = '' @@ -926,7 +878,7 @@ in { (acc: i: recursiveUpdate acc {"cardano-node/topology-${toString i}.json".source = selectTopology i;}) {} (range 0 (cfg.instances - 1))) ) - (mkIf ((cfg.useNewTopology != false) && cfg.useSystemdReload) + (mkIf cfg.useSystemdReload (foldl' (acc: i: recursiveUpdate acc ( optionalAttrs (cfg.peerSnapshotFile i != null) { @@ -950,12 +902,12 @@ in { wants = [ "network-online.target" ]; wantedBy = [ "multi-user.target" ]; partOf = mkIf (cfg.instances > 1) ["cardano-node.service"]; - reloadTriggers = mkIf (cfg.useSystemdReload && (cfg.useNewTopology != false)) [ (selectTopology i) ]; + reloadTriggers = mkIf cfg.useSystemdReload [ (selectTopology i) ]; script = mkScript cfg i; serviceConfig = { User = "cardano-node"; Group = "cardano-node"; - ExecReload = mkIf (cfg.useSystemdReload && (cfg.useNewTopology != false)) "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; + ExecReload = mkIf cfg.useSystemdReload "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; Restart = "always"; RuntimeDirectory = mkIf (!cfg.systemdSocketActivation) (removePrefix cfg.runDirBase (runtimeDir i)); @@ -1033,7 +985,7 @@ in { ''; } { - assertion = !(cfg.systemdSocketActivation && (cfg.useNewTopology != false)); + assertion = !cfg.systemdSocketActivation; message = "Systemd socket activation cannot be used with p2p topology due to a systemd socket re-use issue."; } { @@ -1058,13 +1010,6 @@ in { message = "Only one option of services.cardano-node.tracerSocket(PathAccept|PathConnect|NetworkAccept|NetworkConnect) can be declared."; } ]; - - warnings = optional (cfg.useNewTopology != null) '' - The useNewTopology option is deprecated and will be removed in the future. As of cardano-node 10.6.0, this option should remain null. - For older node versions, a bool value can be set, but this will only be supported until the Dijkstra hard fork at which point all - cardano-node versions will be compelled to upgrade and the useNewTopology option will be removed. See the services.cardano-node.useNewTopology - option description for further details. - ''; } ]); } From c638c20a8c3bce7cbfa68e32e7f147a4aa54d3a0 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Thu, 18 Jun 2026 00:22:59 -0500 Subject: [PATCH 25/40] nix: remove lmdb --- nix/docker/README.md | 6 ++--- nix/haskell.nix | 1 - nix/nixos/cardano-node-service.nix | 35 ---------------------------- nix/workbench/backend/nomad/cloud.sh | 4 ++-- nix/workbench/service/nodes.nix | 15 +++--------- 5 files changed, 8 insertions(+), 53 deletions(-) diff --git a/nix/docker/README.md b/nix/docker/README.md index d6d51575f1e..78f0a6a7a89 100644 --- a/nix/docker/README.md +++ b/nix/docker/README.md @@ -216,13 +216,13 @@ state types as needed, without relying on host level tooling or full chain ledger replays. An example follows to convert preprod ledger state in a named docker volume -from a memory based ledger snapshot to an LMDB snapshot when node is not +from a memory based ledger snapshot to an LSM snapshot when node is not already running: ``` docker run -v preprod-data:/data --rm -it --entrypoint=bash ghcr.io/intersectmbo/cardano-node:dev -c ' mv /data/db/ledger /data/db/ledger-old \ - && mkdir -p /data/db/ledger \ - && snapshot-converter --mem-in /data/db/ledger-old/20807240 --lmdb-out /data/db/ledger/20807240 --config /opt/cardano/config/preprod/config.json + && mkdir -p /data/db/ledger /data/db/lsm \ + && snapshot-converter --input-mem /data/db/ledger-old/20807240 --output-lsm-snapshot /data/db/ledger/20807240 --output-lsm-database /data/db/lsm --config /opt/cardano/config/preprod/config.json ' ``` diff --git a/nix/haskell.nix b/nix/haskell.nix index d3da3f0c03d..13ba8ff8e2f 100644 --- a/nix/haskell.nix +++ b/nix/haskell.nix @@ -54,7 +54,6 @@ let # These programs will be available inside the nix-shell. nativeBuildInputs = with pkgs.pkgsBuildBuild; [ alejandra - lmdb nix-prefetch-git pkg-config git diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix index c2c128afe86..259a37a25ab 100644 --- a/nix/nixos/cardano-node-service.nix +++ b/nix/nixos/cardano-node-service.nix @@ -82,12 +82,6 @@ let Backend = "V2LSM"; LSMDatabasePath = cfg.lsmDatabasePath i; }; - } - // optionalAttrs (cfg.withUtxoHdLmdb i){ - LedgerDB = { - Backend = "V1LMDB"; - LiveTablesPath = cfg.lmdbDatabasePath i; - }; }; in i: let instanceConfig = recursiveUpdate (baseInstanceConfig i) (cfg.extraNodeInstanceConfig i); @@ -393,16 +387,6 @@ in { description = ''The node database path, for each instance.''; }; - lmdbDatabasePath = mkOption { - type = funcToOr nullOrStr; - default = null; - apply = x : if lib.isFunction x then x else if x == null then _: null else _: x; - description = '' - A node UTxO-HD on-disk LMDB path for performant disk I/O, for each instance. - This could point to a direct-access SSD, with a specifically created journal-less file system and optimized mount options. - ''; - }; - lsmDatabasePath = mkOption { type = funcToOr nullOrStr; default = null; @@ -757,16 +741,6 @@ in { ''; }; - withUtxoHdLmdb = mkOption { - type = funcToOr bool; - default = false; - apply = x: if lib.isFunction x then x else _: x; - description = '' - On a UTxO-HD enabled node, the in-memory backend is the default. - This activates the on-disk backend (LMDB) instead. - ''; - }; - withUtxoHdLsmt = mkOption { type = funcToOr bool; default = false; @@ -858,7 +832,6 @@ in { }; config = mkIf cfg.enable ( let - lmdbPaths = filter (x: x != null) (map (e: cfg.lmdbDatabasePath e) (genList trivial.id cfg.instances)); lsmPaths = filter (x: x != null) (map (e: cfg.lsmDatabasePath e) (genList trivial.id cfg.instances)); genInstanceConf = f: listToAttrs (if cfg.instances > 1 then genList (i: let n = "cardano-node-${toString i}"; in nameValuePair n (f n i)) cfg.instances @@ -988,18 +961,10 @@ in { assertion = !cfg.systemdSocketActivation; message = "Systemd socket activation cannot be used with p2p topology due to a systemd socket re-use issue."; } - { - assertion = (length lmdbPaths) == (length (lists.unique lmdbPaths)); - message = "When configuring multiple LMDB enabled nodes on one instance, lmdbDatabasePath must be unique."; - } { assertion = (length lsmPaths) == (length (lists.unique lsmPaths)); message = "When configuring multiple LSM enabled nodes on one instance, lsmDatabasePath must be unique."; } - { - assertion = all (i: !(cfg.withUtxoHdLmdb i && cfg.withUtxoHdLsmt i)) (genList trivial.id cfg.instances); - message = "Each instance can only declare either withUtxoHdLmdb or withUtxoHdLsmt"; - } { assertion = count (o: o != null) (with cfg; [ (tracerSocketPathAccept i) diff --git a/nix/workbench/backend/nomad/cloud.sh b/nix/workbench/backend/nomad/cloud.sh index f99cadf919c..d46f619a044 100644 --- a/nix/workbench/backend/nomad/cloud.sh +++ b/nix/workbench/backend/nomad/cloud.sh @@ -701,13 +701,13 @@ allocate-run-nomadcloud() { read -p "Hit enter to continue ..." fi fi - # Clean, only producers, the "host_volumes" if being used for LMDB/LSMT. + # Clean, only producers, the "host_volumes" if being used for LSMT. # We do this for each producer instead of for all producer at once because # even if modules have the same name from a Nomad perspective, in each # client the real path is defined in Nomad's config file and may differ! # It's "slow" (fetches individual client configs), done only if necessary. if test "${node_name}" != "explorer" \ - && jqtest '.node.utxo_lmdb or .node.utxo_lsmt' "${dir}"/profile.json \ + && jqtest '.node.utxo_lsmt' "${dir}"/profile.json \ && jqtest '(.cluster.nomad.host_volumes.producer | length) > 0' "${dir}"/profile.json then # Iterate over the profile's Nomad "host_volumes" array by key/index. diff --git a/nix/workbench/service/nodes.nix b/nix/workbench/service/nodes.nix index 8766d5949ed..0d277eb4e8e 100644 --- a/nix/workbench/service/nodes.nix +++ b/nix/workbench/service/nodes.nix @@ -79,12 +79,10 @@ with pkgs.lib; let topology = "topology.json"; nodeConfigFile = "config.json"; - # Allow for local clusters to have multiple LMDB directories in the same physical ssd_directory; + # Allow for local clusters to have multiple on-disk (LSM-tree) directories in the same physical ssd_directory; # non-block producers (like the explorer node) keep using the in-memory backend - withUtxoHdLmdb = profile.node.utxo_lmdb && isProducer; - withUtxoHdLsmt = profile.node.utxo_lsmt && isProducer; - lmdbDatabasePath = liveTablesPath i; - lsmDatabasePath = liveTablesPath i; + withUtxoHdLsmt = profile.node.utxo_lsmt && isProducer; + lsmDatabasePath = liveTablesPath i; ## Combine: ## 0. baseNodeConfig (coming cardanoLib's testnet environ) @@ -139,13 +137,6 @@ with pkgs.lib; let Backend = "V2LSM"; LSMDatabasePath = liveTablesPath i; }; - } - // optionalAttrs (profile.node.utxo_lmdb && isProducer) - { - LedgerDB = { - Backend = "V1LMDB"; - LiveTablesPath = liveTablesPath i; - }; }) { ## This LedgerDB attrset is defined regardless of backend choice. From 0e5c80c4637cdbd0116a220f7920d34c72162307 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Sun, 21 Jun 2026 11:22:58 -0500 Subject: [PATCH 26/40] nideNixosSvc: preserve new LedgerDB.Snapshots attrset on lsmt attrs merge --- nix/nixos/cardano-node-service.nix | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix index 259a37a25ab..9bcc7d5d47a 100644 --- a/nix/nixos/cardano-node-service.nix +++ b/nix/nixos/cardano-node-service.nix @@ -76,13 +76,13 @@ let ) cfg.extraNodeConfig; baseInstanceConfig = i: - baseConfig - // optionalAttrs (cfg.withUtxoHdLsmt i){ - LedgerDB = { - Backend = "V2LSM"; - LSMDatabasePath = cfg.lsmDatabasePath i; - }; + recursiveUpdate + baseConfig (optionalAttrs (cfg.withUtxoHdLsmt i) { + LedgerDB = { + Backend = "V2LSM"; + LSMDatabasePath = cfg.lsmDatabasePath i; }; + }); in i: let instanceConfig = recursiveUpdate (baseInstanceConfig i) (cfg.extraNodeInstanceConfig i); nodeConfigFile = if (cfg.nodeConfigFile != null) then cfg.nodeConfigFile From e909b56b31fc8dd16abe1c83972f37d602c6274c Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Sun, 21 Jun 2026 18:15:44 -0500 Subject: [PATCH 27/40] bump: iohkNix for updated submit-api cfg --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 087cd2031d4..412d023b1fc 100644 --- a/flake.lock +++ b/flake.lock @@ -637,11 +637,11 @@ "sodium": "sodium" }, "locked": { - "lastModified": 1781758324, - "narHash": "sha256-DxBUgw7medqNE9zBhlaf7yez8iKDzILnofByNJjn1D8=", + "lastModified": 1782080462, + "narHash": "sha256-jHlPlNk/3PKEx+A++IiWGX3jQtTYzsIM0iTub/VOn2g=", "owner": "input-output-hk", "repo": "iohk-nix", - "rev": "4d76d0289b39f9e9ffac2dbe925bbe863b76f988", + "rev": "d7af9d6d6cd3efc91a23772e52fc24a01117f798", "type": "github" }, "original": { From 28ba75fb8ed8a4bd073674c69045cd2557093403 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Sun, 21 Jun 2026 18:16:39 -0500 Subject: [PATCH 28/40] submitApiNixosSvc: update the submit-api service defn for new tracing cfg, group socket use --- nix/nixos/cardano-submit-api-service.nix | 64 ++++++++++++++++++------ 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/nix/nixos/cardano-submit-api-service.nix b/nix/nixos/cardano-submit-api-service.nix index fcebf68ca94..55e9e97c1a4 100644 --- a/nix/nixos/cardano-submit-api-service.nix +++ b/nix/nixos/cardano-submit-api-service.nix @@ -1,68 +1,91 @@ +# This service exposes an http port, and connects to a cardano-node over a UNIX socket { config, lib, pkgs, ... }: - -# notes: -# this service exposes an http port, and connects to a cardano-node over a UNIX socket let - cfg = config.services.cardano-submit-api; + inherit (builtins) fromJSON readFile; inherit (cfg.cardanoNodePackages) cardanoLib; - envConfig = cfg.environment; + + cfg = config.services.cardano-submit-api; in { options = { services.cardano-submit-api = { - enable = lib.mkEnableOption "enable the cardano-submit-api api"; + enable = lib.mkEnableOption "Enable the cardano-submit-api api"; + script = lib.mkOption { internal = true; type = lib.types.package; + description = "Generated cardano-submit-api launch script (internal)."; }; + package = lib.mkOption { type = lib.types.package; default = cfg.cardanoNodePackages.cardano-submit-api; + description = "The cardano-submit-api package to run."; }; + port = lib.mkOption { type = lib.types.port; default = 8090; + description = "HTTP port submit-api listens on."; }; + listenAddress = lib.mkOption { type = lib.types.str; default = "127.0.0.1"; + description = "Host address submit-api binds to."; }; + socketPath = lib.mkOption { type = lib.types.nullOr lib.types.path; default = null; description = '' - cardano node socket path. If set, the entrypoint - takes this value over CARDANO_NODE_SOCKET_PATH env - variable. + The cardano node socket path. If set, the entrypoint takes this value + over CARDANO_NODE_SOCKET_PATH env variable. + ''; + }; + + group = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + Optional supplementary group added to the service's dynamic user, + typically the cardano-node socket group, so cardano-submit-api can + access the node socket at CARDANO_NODE_SOCKET_PATH. ''; }; + config = lib.mkOption { type = lib.types.nullOr lib.types.attrs; - default = cardanoLib.defaultExplorerLogConfig; + default = cardanoLib.defaultSubmitApiConfig; + description = "Tracing configuration passed to submit-api."; }; + network = lib.mkOption { type = lib.types.nullOr lib.types.str; - description = "network name"; + description = "Network name."; default = null; }; + environment = lib.mkOption { type = lib.types.nullOr lib.types.attrs; default = cfg.cardanoNodePackages.cardanoLib.environments.${cfg.network}; + description = "Cardano environment attrset for the selected network."; }; + cardanoNodePackages = lib.mkOption { type = lib.types.attrs; default = pkgs.cardanoNodePackages or (import ../. {}).cardanoNodePackages; defaultText = "cardano-node packages"; description = '' The cardano-node packages and library that should be used. - Main usage is sharing optimization: - reduce eval time when service is instantiated multiple times. + Main usage is sharing optimization to reduce eval time when services + are instantiated multiple times. ''; }; }; }; config = let envNodeCfg = cfg.environment.nodeConfig; - shelleyGenesisParams = __fromJSON (__readFile envNodeCfg.ShelleyGenesisFile); + shelleyGenesisParams = fromJSON (readFile envNodeCfg.ShelleyGenesisFile); envFlag = if cfg.network == "mainnet" then "--mainnet" else "--testnet-magic ${toString shelleyGenesisParams.networkMagic}"; in lib.mkIf cfg.enable { services.cardano-submit-api.script = pkgs.writeShellScript "cardano-submit-api" '' @@ -80,6 +103,19 @@ in { serviceConfig = { ExecStart = config.services.cardano-submit-api.script; DynamicUser = true; + + # The api connects to the node over a UNIX socket that only becomes + # available once the node has started; `after` orders startup but does + # not wait for the socket. Default to restarting until it is reachable + # rather than failing permanently on a fresh boot. These are mkDefault + # so a consumer can impose a bounded restart + start-limit policy that + # lets persistent failures surface as a failed unit for alerting. + Restart = lib.mkDefault "always"; + RestartSec = lib.mkDefault 1; + } // lib.optionalAttrs (cfg.group != null) { + # A DynamicUser is not otherwise a member of the node socket group, so + # without this it cannot open CARDANO_NODE_SOCKET_PATH. + SupplementaryGroups = [ cfg.group ]; }; wantedBy = [ "multi-user.target" ]; after = [ "cardano-node.service" ]; From 60a272a034063b99f2399fc44d7a03e13aa23fa5 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Sun, 21 Jun 2026 18:19:22 -0500 Subject: [PATCH 29/40] submitApiNixosSvc: alejandra fmt and option alpha sort --- nix/nixos/cardano-submit-api-service.nix | 166 ++++++++++++----------- 1 file changed, 90 insertions(+), 76 deletions(-) diff --git a/nix/nixos/cardano-submit-api-service.nix b/nix/nixos/cardano-submit-api-service.nix index 55e9e97c1a4..36c3bb83456 100644 --- a/nix/nixos/cardano-submit-api-service.nix +++ b/nix/nixos/cardano-submit-api-service.nix @@ -1,6 +1,10 @@ # This service exposes an http port, and connects to a cardano-node over a UNIX socket -{ config, lib, pkgs, ... }: -let +{ + config, + lib, + pkgs, + ... +}: let inherit (builtins) fromJSON readFile; inherit (cfg.cardanoNodePackages) cardanoLib; @@ -10,37 +14,27 @@ in { services.cardano-submit-api = { enable = lib.mkEnableOption "Enable the cardano-submit-api api"; - script = lib.mkOption { - internal = true; - type = lib.types.package; - description = "Generated cardano-submit-api launch script (internal)."; - }; - - package = lib.mkOption { - type = lib.types.package; - default = cfg.cardanoNodePackages.cardano-submit-api; - description = "The cardano-submit-api package to run."; - }; - - port = lib.mkOption { - type = lib.types.port; - default = 8090; - description = "HTTP port submit-api listens on."; + cardanoNodePackages = lib.mkOption { + type = lib.types.attrs; + default = pkgs.cardanoNodePackages or (import ../. {}).cardanoNodePackages; + defaultText = "cardano-node packages"; + description = '' + The cardano-node packages and library that should be used. + Main usage is sharing optimization to reduce eval time when services + are instantiated multiple times. + ''; }; - listenAddress = lib.mkOption { - type = lib.types.str; - default = "127.0.0.1"; - description = "Host address submit-api binds to."; + config = lib.mkOption { + type = lib.types.nullOr lib.types.attrs; + default = cardanoLib.defaultSubmitApiConfig; + description = "Tracing configuration passed to submit-api."; }; - socketPath = lib.mkOption { - type = lib.types.nullOr lib.types.path; - default = null; - description = '' - The cardano node socket path. If set, the entrypoint takes this value - over CARDANO_NODE_SOCKET_PATH env variable. - ''; + environment = lib.mkOption { + type = lib.types.nullOr lib.types.attrs; + default = cfg.cardanoNodePackages.cardanoLib.environments.${cfg.network}; + description = "Cardano environment attrset for the selected network."; }; group = lib.mkOption { @@ -53,10 +47,10 @@ in { ''; }; - config = lib.mkOption { - type = lib.types.nullOr lib.types.attrs; - default = cardanoLib.defaultSubmitApiConfig; - description = "Tracing configuration passed to submit-api."; + listenAddress = lib.mkOption { + type = lib.types.str; + default = "127.0.0.1"; + description = "Host address submit-api binds to."; }; network = lib.mkOption { @@ -65,20 +59,30 @@ in { default = null; }; - environment = lib.mkOption { - type = lib.types.nullOr lib.types.attrs; - default = cfg.cardanoNodePackages.cardanoLib.environments.${cfg.network}; - description = "Cardano environment attrset for the selected network."; + package = lib.mkOption { + type = lib.types.package; + default = cfg.cardanoNodePackages.cardano-submit-api; + description = "The cardano-submit-api package to run."; }; - cardanoNodePackages = lib.mkOption { - type = lib.types.attrs; - default = pkgs.cardanoNodePackages or (import ../. {}).cardanoNodePackages; - defaultText = "cardano-node packages"; + port = lib.mkOption { + type = lib.types.port; + default = 8090; + description = "HTTP port submit-api listens on."; + }; + + script = lib.mkOption { + internal = true; + type = lib.types.package; + description = "Generated cardano-submit-api launch script (internal)."; + }; + + socketPath = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; description = '' - The cardano-node packages and library that should be used. - Main usage is sharing optimization to reduce eval time when services - are instantiated multiple times. + The cardano node socket path. If set, the entrypoint takes this value + over CARDANO_NODE_SOCKET_PATH env variable. ''; }; }; @@ -86,39 +90,49 @@ in { config = let envNodeCfg = cfg.environment.nodeConfig; shelleyGenesisParams = fromJSON (readFile envNodeCfg.ShelleyGenesisFile); - envFlag = if cfg.network == "mainnet" then "--mainnet" else "--testnet-magic ${toString shelleyGenesisParams.networkMagic}"; - in lib.mkIf cfg.enable { - services.cardano-submit-api.script = pkgs.writeShellScript "cardano-submit-api" '' - ${if (cfg.socketPath == null) then ''if [ -z "$CARDANO_NODE_SOCKET_PATH" ] - then - echo "You must set \$CARDANO_NODE_SOCKET_PATH" - exit 1 - fi'' else "export \"CARDANO_NODE_SOCKET_PATH=${cfg.socketPath}\""} - exec ${cfg.package}/bin/cardano-submit-api --socket-path "$CARDANO_NODE_SOCKET_PATH" ${envFlag} \ - --port ${toString cfg.port} \ - --listen-address ${cfg.listenAddress} \ - --config ${builtins.toFile "submit-api.json" (builtins.toJSON cfg.config)} - ''; - systemd.services.cardano-submit-api = { - serviceConfig = { - ExecStart = config.services.cardano-submit-api.script; - DynamicUser = true; + envFlag = + if cfg.network == "mainnet" + then "--mainnet" + else "--testnet-magic ${toString shelleyGenesisParams.networkMagic}"; + in + lib.mkIf cfg.enable { + services.cardano-submit-api.script = pkgs.writeShellScript "cardano-submit-api" '' + ${ + if (cfg.socketPath == null) + then '' if [ -z "$CARDANO_NODE_SOCKET_PATH" ] + then + echo "You must set \$CARDANO_NODE_SOCKET_PATH" + exit 1 + fi'' + else "export \"CARDANO_NODE_SOCKET_PATH=${cfg.socketPath}\"" + } + exec ${cfg.package}/bin/cardano-submit-api --socket-path "$CARDANO_NODE_SOCKET_PATH" ${envFlag} \ + --port ${toString cfg.port} \ + --listen-address ${cfg.listenAddress} \ + --config ${builtins.toFile "submit-api.json" (builtins.toJSON cfg.config)} + ''; + systemd.services.cardano-submit-api = { + serviceConfig = + { + ExecStart = config.services.cardano-submit-api.script; + DynamicUser = true; - # The api connects to the node over a UNIX socket that only becomes - # available once the node has started; `after` orders startup but does - # not wait for the socket. Default to restarting until it is reachable - # rather than failing permanently on a fresh boot. These are mkDefault - # so a consumer can impose a bounded restart + start-limit policy that - # lets persistent failures surface as a failed unit for alerting. - Restart = lib.mkDefault "always"; - RestartSec = lib.mkDefault 1; - } // lib.optionalAttrs (cfg.group != null) { - # A DynamicUser is not otherwise a member of the node socket group, so - # without this it cannot open CARDANO_NODE_SOCKET_PATH. - SupplementaryGroups = [ cfg.group ]; + # The api connects to the node over a UNIX socket that only becomes + # available once the node has started; `after` orders startup but does + # not wait for the socket. Default to restarting until it is reachable + # rather than failing permanently on a fresh boot. These are mkDefault + # so a consumer can impose a bounded restart + start-limit policy that + # lets persistent failures surface as a failed unit for alerting. + Restart = lib.mkDefault "always"; + RestartSec = lib.mkDefault 1; + } + // lib.optionalAttrs (cfg.group != null) { + # A DynamicUser is not otherwise a member of the node socket group, so + # without this it cannot open CARDANO_NODE_SOCKET_PATH. + SupplementaryGroups = [cfg.group]; + }; + wantedBy = ["multi-user.target"]; + after = ["cardano-node.service"]; }; - wantedBy = [ "multi-user.target" ]; - after = [ "cardano-node.service" ]; }; - }; } From 3725f97c7f8964dd4bc039a9ffc44e9373505121 Mon Sep 17 00:00:00 2001 From: Robin Stumm Date: Tue, 9 Jun 2026 19:46:12 +0200 Subject: [PATCH 30/40] nixos: gate RTS stats/profiling flags + add `profilingoutputdir` The cardano-node and cardano-tracer service modules unconditionally emitted `--machine-readable -tcardano-node.stats -pocardano-node` in the default `profilingArgs`, regardless of `cfg.profiling`. GHC's `-tFILE` always writes the stats file on shutdown, which broke startup on read-only filesystems even though no profiling was configured. Gate the three flags on `cfg.profiling != "none" || cfg.eventlog` so the default RTS command line is empty when nothing is requested. When profiling is enabled, the same flags now consult a new option, `services.cardano-node.profilingOutputDir` (and the tracer equivalent), to prefix the output file paths. The option defaults to null, preserving today's relative-path behavior on NixOS where systemd's `WorkingDirectory` equals `cfg.stateDir`. `scripts.nix` sets it to `/logs` for the OCI script wrappers, so profile output under the read-only OCI image lands on the writable `/logs` mount. In response to #6470. --- nix/nixos/cardano-node-service.nix | 23 +++++++++++++++++++---- nix/nixos/cardano-tracer-service.nix | 23 +++++++++++++++++++---- nix/scripts.nix | 4 ++++ 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix index 9bcc7d5d47a..9bdad69d15d 100644 --- a/nix/nixos/cardano-node-service.nix +++ b/nix/nixos/cardano-node-service.nix @@ -772,12 +772,27 @@ in { description = ''Extra CLI args for cardano-node, to be surrounded by "+RTS"/"-RTS"''; }; + profilingOutputDir = mkOption { + type = nullOrStr; + default = null; + description = '' + Optional directory prefix for GHC RTS profiling output files + (cardano-node.stats, cardano-node.prof, cardano-node.hp, etc.). + When null, files are written relative to the working directory + (the systemd unit's WorkingDirectory for NixOS deployments, which + is cfg.stateDir). + ''; + }; + profilingArgs = mkOption { type = listOf str; - default = - [ "--machine-readable" - "-tcardano-node.stats" - "-pocardano-node" + default = let + prefix = if cfg.profilingOutputDir == null then "" else "${cfg.profilingOutputDir}/"; + in + optionals (cfg.profiling != "none" || cfg.eventlog) [ + "--machine-readable" + "-t${prefix}cardano-node.stats" + "-po${prefix}cardano-node" ] ++ optional (cfg.eventlog) "-l" ++ ( diff --git a/nix/nixos/cardano-tracer-service.nix b/nix/nixos/cardano-tracer-service.nix index 581dd90e342..4cfa71be2f9 100644 --- a/nix/nixos/cardano-tracer-service.nix +++ b/nix/nixos/cardano-tracer-service.nix @@ -501,12 +501,27 @@ in { ''; }; + profilingOutputDir = mkOption { + type = nullOr str; + default = null; + description = '' + Optional directory prefix for GHC RTS profiling output files + (cardano-node.stats, cardano-node.prof, cardano-node.hp, etc.). + When null, files are written relative to the working directory + (the systemd unit's WorkingDirectory for NixOS deployments, which + is cfg.stateDir). + ''; + }; + profilingArgs = mkOption { type = listOf str; - default = - [ "--machine-readable" - "-tcardano-node.stats" - "-pocardano-node" + default = let + prefix = if cfg.profilingOutputDir == null then "" else "${cfg.profilingOutputDir}/"; + in + optionals (cfg.profiling != "none" || cfg.eventlog) [ + "--machine-readable" + "-t${prefix}cardano-node.stats" + "-po${prefix}cardano-node" ] ++ optional (cfg.eventlog) "-l" ++ ( diff --git a/nix/scripts.nix b/nix/scripts.nix index 421d09f6530..f6443da273d 100644 --- a/nix/scripts.nix +++ b/nix/scripts.nix @@ -16,6 +16,10 @@ let nodeConfig = cfg.environments.${cfg.environment}.nodeConfig; stateDir = mkDefault "state-node-${cfg.environment}"; runtimeDir = mkDefault null; + # When profiling is enabled, direct GHC RTS output + # (stats, prof, hp, ...) to /logs so the OCI image works + # with a read-only root filesystem. No effect when profiling = "none". + profilingOutputDir = mkDefault "/logs"; } // optionalAttrs (envConfig ? topology) { topology = mkDefault envConfig.topology; }; From 541a78971d2fc435fccb47186f58171a1dcb4e78 Mon Sep 17 00:00:00 2001 From: Robin Stumm Date: Tue, 9 Jun 2026 19:52:26 +0200 Subject: [PATCH 31/40] docker: move env snapshot to `/tmp`, drop dead topologyUpdater mapping `run-node` and `run-tracer` wrote a sourceable env file to `/usr/local/bin/env` at every startup, which fails under `--read-only` / `readOnlyRootFilesystem`. The file was introduced in d9c8317e5 (#2801) to feed the topologyUpdate script added the same day in c652f1055. topologyUpdate was removed in 56266c061 and never reintroduced; the writer was preserved by accident when 34a4796ed re-created the docker context, along with a stale `# Mapping for topologyUpdater` comment. Across all branches, there is noconsumer besides the deleted topologyUpdater. The `CARDANO_*` snapshot remains useful for operators that exec into the container and want the resolved (post-defaults) config in a shell, so the writer is kept but redirected to `/tmp/cardano-env` (writable when `/tmp` is mounted as tmpfs/emptyDir). A build-time symlink at `/usr/local/bin/env -> /tmp/cardano-env` keeps the legacy path resolving in case any out-of-tree consumer depends on it. The variables that existed only for topologyUpdater are dropped from the snapshot. The README's new "Read-Only Root Filesystem" section documents the required writable mounts, the new env-snapshot path, and how custom-mode operators should direct any profile output to a writable mount. In response to #6470. --- nix/docker/README.md | 24 ++++++++++++++++++++ nix/docker/context/node/bin/run-node | 28 ++++++------------------ nix/docker/context/tracer/bin/run-tracer | 4 ++-- nix/docker/default.nix | 5 +++++ nix/docker/tracer.nix | 5 +++++ 5 files changed, 43 insertions(+), 23 deletions(-) diff --git a/nix/docker/README.md b/nix/docker/README.md index 78f0a6a7a89..32ad00d5580 100644 --- a/nix/docker/README.md +++ b/nix/docker/README.md @@ -149,6 +149,30 @@ respectively. This makes bind mounting easier when switching between default state directory locations, `/{data,ipc,logs}`, will work for both modes. +## Read-Only Root Filesystem +The image is compatible with `--read-only` (Docker/Podman) and +`securityContext.readOnlyRootFilesystem: true` (Kubernetes), provided the +runtime supplies writable storage for the state directories described above +(`/data`, `/ipc`, `/logs`) and a writable `/tmp` (tmpfs or `emptyDir`). + +A resolved-configuration snapshot is written at runtime to `/tmp/cardano-env` +and can be `source`d for an interactive debug shell inside the container. +The legacy path `/usr/local/bin/env` is preserved as a symlink to +`/tmp/cardano-env` for backwards compatibility. + +In "scripts" mode, GHC RTS profiling output (`cardano-node.stats`, +`cardano-node.prof`, `cardano-node.hp`, etc.) is directed to `/logs/` so the +image keeps working when profiling is enabled under a read-only root. +In "custom" mode the operator chooses the RTS flags, so any profiling output +must similarly be directed to a writable mount, for example: +``` +... run \ + --config /opt/cardano/config/mainnet/config.json \ + ... \ + +RTS --machine-readable -t/logs/cardano-node.stats -po/logs/cardano-node -p -RTS +``` + + ## Cardano-node Socket Sharing To share a cardano-node socket with a different container, a volume can be made for establishing cross-container communication: diff --git a/nix/docker/context/node/bin/run-node b/nix/docker/context/node/bin/run-node index 1229631f801..607dc670e61 100755 --- a/nix/docker/context/node/bin/run-node +++ b/nix/docker/context/node/bin/run-node @@ -98,7 +98,7 @@ printRunEnv () { # writeRootEnv () { -cat << EOF > /usr/local/bin/env +cat << EOF > /tmp/cardano-env #!/usr/bin/env bash # Docker run ENV vars @@ -106,30 +106,30 @@ EOF if [[ -n ${CARDANO_SHELLEY_KES_AGENT_SOCKET:-} ]]; then echo "CARDANO_SHELLEY_KES_AGENT_SOCKET=\"$CARDANO_SHELLEY_KES_AGENT_SOCKET\"" \ - >> /usr/local/bin/env + >> /tmp/cardano-env fi if [[ -n ${CARDANO_TRACER_SOCKET_NETWORK_ACCEPT:-} ]]; then echo "CARDANO_TRACER_SOCKET_NETWORK_ACCEPT=\"$CARDANO_TRACER_SOCKET_NETWORK_ACCEPT\"" \ - >> /usr/local/bin/env + >> /tmp/cardano-env fi if [[ -n ${CARDANO_TRACER_SOCKET_NETWORK_CONNECT:-} ]]; then echo "CARDANO_TRACER_SOCKET_NETWORK_CONNECT=\"$CARDANO_TRACER_SOCKET_NETWORK_CONNECT\"" \ - >> /usr/local/bin/env + >> /tmp/cardano-env fi if [[ -n ${CARDANO_TRACER_SOCKET_PATH_ACCEPT:-} ]]; then echo "CARDANO_TRACER_SOCKET_PATH_ACCEPT=\"$CARDANO_TRACER_SOCKET_PATH_ACCEPT\"" \ - >> /usr/local/bin/env + >> /tmp/cardano-env fi if [[ -n ${CARDANO_TRACER_SOCKET_PATH_CONNECT:-} ]]; then echo "CARDANO_TRACER_SOCKET_PATH_CONNECT=\"$CARDANO_TRACER_SOCKET_PATH_CONNECT\"" \ - >> /usr/local/bin/env + >> /tmp/cardano-env fi -cat << EOF >> /usr/local/bin/env +cat << EOF >> /tmp/cardano-env CARDANO_BIND_ADDR="$CARDANO_BIND_ADDR" CARDANO_BLOCK_PRODUCER=$CARDANO_BLOCK_PRODUCER CARDANO_CONFIG="$CARDANO_CONFIG" @@ -138,20 +138,6 @@ CARDANO_LOG_DIR="$CARDANO_LOG_DIR" CARDANO_PORT=$CARDANO_PORT CARDANO_SOCKET_PATH="$CARDANO_SOCKET_PATH" CARDANO_TOPOLOGY="$CARDANO_TOPOLOGY" - -CARDANO_PUBLIC_IP="${CARDANO_PUBLIC_IP:-}" -CARDANO_CUSTOM_PEERS="${CARDANO_CUSTOM_PEERS:-}" - -# Mapping for topologyUpdater -CNODE_HOSTNAME="${CARDANO_PUBLIC_IP:-}" -CNODE_PORT=$CARDANO_PORT -CUSTOM_PEERS="${CARDANO_CUSTOM_PEERS:-}" - -# Derived from CARDANO_CONFIG to support non-mainnet deployments -GENESIS_JSON="$(dirname "$CARDANO_CONFIG")/shelley-genesis.json" - -TOPOLOGY="$CARDANO_TOPOLOGY" -LOG_DIR="$CARDANO_LOG_DIR" EOF } diff --git a/nix/docker/context/tracer/bin/run-tracer b/nix/docker/context/tracer/bin/run-tracer index 3c0bf3f48f7..b8fc44018e4 100755 --- a/nix/docker/context/tracer/bin/run-tracer +++ b/nix/docker/context/tracer/bin/run-tracer @@ -34,7 +34,7 @@ printRunEnv () { # writeRootEnv () { -cat << EOF > /usr/local/bin/env +cat << EOF > /tmp/cardano-env #!/usr/bin/env bash # Docker run ENV vars @@ -44,7 +44,7 @@ EOF if [[ -n ${CARDANO_MIN_LOG_SEVERITY:-} ]]; then echo "CARDANO_MIN_LOG_SEVERITY=\"$CARDANO_MIN_LOG_SEVERITY\"" \ - >> /usr/local/bin/env + >> /tmp/cardano-env fi } diff --git a/nix/docker/default.nix b/nix/docker/default.nix index cde340be66e..ed3675ece47 100644 --- a/nix/docker/default.nix +++ b/nix/docker/default.nix @@ -176,6 +176,11 @@ in ln -sv ${snapshot-converter}/bin/snapshot-converter usr/local/bin/snapshot-converter ln -sv ${jq}/bin/jq usr/local/bin/jq + # Backwards-compatible alias for the resolved-config env snapshot + # written by run-node. The runtime writer targets /tmp/cardano-env so + # the image remains compatible with a read-only root filesystem. + ln -sv /tmp/cardano-env usr/local/bin/env + # Create iohk-nix network configs, organized by network directory. SRC="${genCfgs}" DST="opt/cardano" diff --git a/nix/docker/tracer.nix b/nix/docker/tracer.nix index db18fb0731f..15c5fd08c62 100644 --- a/nix/docker/tracer.nix +++ b/nix/docker/tracer.nix @@ -153,6 +153,11 @@ in ln -sv ${cardano-tracer}/bin/cardano-tracer usr/local/bin/cardano-tracer ln -sv ${jq}/bin/jq usr/local/bin/jq + # Backwards-compatible alias for the resolved-config env snapshot + # written by run-tracer. The runtime writer targets /tmp/cardano-env so + # the image remains compatible with a read-only root filesystem. + ln -sv /tmp/cardano-env usr/local/bin/env + # Create iohk-nix network configs, organized by network directory. SRC="${genCfgs}" DST="opt/cardano" From e9085e6cb85f6622b9f63c35323d5f22d2369a01 Mon Sep 17 00:00:00 2001 From: Robin Stumm Date: Fri, 12 Jun 2026 14:31:21 +0200 Subject: [PATCH 32/40] oci: redirect merge-mode writes to `/tmp` The entrypoints wrote `{{,tracer-}config,topology}-merged.json` to `/opt/cardano/config/$NETWORK/`, which is image content owned by root. Non-root containers and read-only-root mounts both fail on this write. Redirect the writes to `/tmp/cardano-{{,tracer-}config,topology}-merged.json`. The upstream network config keeps relative refs to genesis / peer-snapshot files, all resolved relative to the config file's own directory. Naively moving the merged output to `/tmp` would break those refs. The jq merge now rewrites them to absolute paths anchored at `/opt/cardano/config/$NETWORK/`. The key match uses a regex `(test("GenesisFile$|^CheckpointsFile$"))` so future protocol-era keys pick up the rewrite without code changes. Both entrypoints also bail out early with a clear error message if `/tmp` is not writable, instead of letting the failure cascade into a less-obvious jq write error. In response to #6484. --- nix/docker/README.md | 19 +++++++++++++++++++ nix/docker/default.nix | 9 +++++++++ nix/docker/tracer.nix | 11 +++++++++++ 3 files changed, 39 insertions(+) diff --git a/nix/docker/README.md b/nix/docker/README.md index 32ad00d5580..36232991417 100644 --- a/nix/docker/README.md +++ b/nix/docker/README.md @@ -173,6 +173,25 @@ must similarly be directed to a writable mount, for example: ``` +## Non-Root User +The image can run as any non-root user (`docker run --user ` / +Kubernetes `securityContext.runAsUser`). None of the entrypoint or +`run-node` startup logic touches image-content directories at runtime; +all generated artifacts live under `/tmp`. + +The mount-point directories (`/data`, `/ipc`, `/logs`) are owned by +GID 0 and group-writable in the image, so non-root containers can write +to freshly-created Docker or Kubernetes volumes mounted at those paths +without an init container or pre-chown. To inherit the group-writable +perm, the non-root user needs to run with primary group 0 (the Kubernetes +default for `runAsUser`) or with supplementary group 0. In Kubernetes +you can also set `securityContext.fsGroup: 0` to have the kubelet chown +the volume on mount. For Docker, `--user :0` is the equivalent. + +The image defaults to running as root; specify a UID explicitly +to opt into a non-root run. + + ## Cardano-node Socket Sharing To share a cardano-node socket with a different container, a volume can be made for establishing cross-container communication: diff --git a/nix/docker/default.nix b/nix/docker/default.nix index ed3675ece47..0945c8f8998 100644 --- a/nix/docker/default.nix +++ b/nix/docker/default.nix @@ -152,6 +152,15 @@ in # Similarly, make a root level dir for logs: mkdir -p logs + # Make the mount-point directories group-writable. Group is already + # 0 (the build env writes files as 0:0). When a fresh Docker volume + # is first mounted at one of these paths, the perms propagate from + # the image, so non-root containers (running as a UID in group 0 — + # the K8s default for runAsUser — or with explicit fsGroup) can + # write to a freshly-created volume without an init container or + # pre-chown. + chmod g+w data ipc logs + # The "custom" operation mode of this image, when the NETWORK env is # unset and "run" is provided as an entrypoint arg, will use the # following default directories. To reduce confusion caused by default diff --git a/nix/docker/tracer.nix b/nix/docker/tracer.nix index 15c5fd08c62..fb767efeede 100644 --- a/nix/docker/tracer.nix +++ b/nix/docker/tracer.nix @@ -130,11 +130,21 @@ in # The "scripts" operation mode of this image, when the NETWORK env var is # set to a valid network, will use the following default directories # mounted at /: + mkdir -p data mkdir -p ipc # Similarly, make a root level dir for logs: mkdir -p logs + # Make the mount-point directories group-writable. Group is already + # 0 (the build env writes files as 0:0). When a fresh Docker volume + # is first mounted at one of these paths, the perms propagate from + # the image, so non-root containers (running as a UID in group 0 — + # the K8s default for runAsUser — or with explicit fsGroup) can + # write to a freshly-created volume without an init container or + # pre-chown. + chmod g+w data ipc logs + # The "custom" operation mode of this image, when the NETWORK env is # unset and "run" is provided as an entrypoint arg, will use the # following default directories. To reduce confusion caused by default @@ -143,6 +153,7 @@ in # permit use of volume mounts at the root directory location regardless # of which mode the image is operating in. mkdir -p opt/cardano + ln -sv /data opt/cardano/data ln -sv /ipc opt/cardano/ipc ln -sv /logs opt/cardano/logs From 65d8fee0a69df64ad503da4d9ef5a3a8be70f881 Mon Sep 17 00:00:00 2001 From: Robin Stumm Date: Fri, 12 Jun 2026 14:42:55 +0200 Subject: [PATCH 33/40] oci: make mount-point directories group-writable for non-root operation The mount-point directories /data, /ipc, /logs were created owned by root. When a volume was first mounted there, Docker propagated the permissions from the image, so a container running as non-root could not write. Apply `chmod g+w` to open up the permissions enough for: - Kubernetes `runAsUser` (assigns primary group 0 by default) - Docker `--user ` (assigns primary group 0 by default) - OpenShift's arbitrary-UID assignment (random UID, GID 0) For the tracer image, also add `mkdir -p data` and the matching symlink at `/opt/cardano/data` for parity with the node image's convention. `run-tracer` defaults `CARDANO_STATE_DIR` to `/data/tracer` and assumed that this symlink existed. This finally makes that true. The default state dir now works under non-root without an explicit `/data` mount. In response to #6484. --- nix/docker/README.md | 8 +++++ nix/docker/context/node/bin/entrypoint | 41 ++++++++++++++++++++---- nix/docker/context/tracer/bin/entrypoint | 19 +++++++++-- 3 files changed, 60 insertions(+), 8 deletions(-) diff --git a/nix/docker/README.md b/nix/docker/README.md index 36232991417..55450c5f442 100644 --- a/nix/docker/README.md +++ b/nix/docker/README.md @@ -120,6 +120,14 @@ docker run \ ghcr.io/intersectmbo/cardano-node:dev ``` +The resulting merged config and topology are written to +`/tmp/cardano-{{,tracer-}config,topology}-merged.json` +and used as the runtime configuration. +Relative file references are rewritten to absolute paths +anchored at `/opt/cardano/config/$NETWORK/` +so they resolve from the new location. + + ## CLI Mode To run cardano-cli, leave the `NETWORK` env variable unset and provide entrypoint args starting with `cli` followed by cardano-cli command args. diff --git a/nix/docker/context/node/bin/entrypoint b/nix/docker/context/node/bin/entrypoint index a160f641cb0..88062d4b5e6 100755 --- a/nix/docker/context/node/bin/entrypoint +++ b/nix/docker/context/node/bin/entrypoint @@ -3,6 +3,15 @@ set -euo pipefail [[ -n ${DEBUG:-} ]] && set -x +# The image writes a resolved-config env snapshot and any merge-mode +# artifacts to /tmp. Catch the common operator mistake of running +# with a read-only filesystem without mounting a writable /tmp. +if ! [[ -w /tmp ]]; then + echo "ERROR: /tmp is not writable." >&2 + echo "With a read-only filesystem, mount a tmpfs or emptyDir at /tmp." >&2 + exit 1 +fi + # If the NETWORK env var is set to a valid cardano network, pre-defined # configuration will be used. if [[ -n ${NETWORK:-} ]]; then @@ -30,24 +39,44 @@ if [[ -n ${NETWORK:-} ]]; then # full replacement and null values persist. # # jq -S sorts output keys alphabetically for deterministic diffs. + # + # Merged files are written to /tmp so that the image can run as a + # non-root user ($CFG is image content and only writable by root) + # and under read-only root filesystems. + # Relative file references are rewritten to absolute paths + # anchored at $CFG/$NETWORK/ so they resolve from the new location. if [[ -n ${CARDANO_CONFIG_JSON_MERGE:-} ]]; then jq -S \ + --arg cfgDir "$CFG/$NETWORK" \ --argjson deepMerge "$CARDANO_CONFIG_JSON_MERGE" \ - '. * $deepMerge' \ + '. * $deepMerge + | with_entries( + if ((.key | test("GenesisFile$|^CheckpointsFile$")) + and (.value | type == "string") + and (.value | startswith("/") | not)) + then .value = "\($cfgDir)/\(.value)" + else . + end + )' \ < "$CFG/$NETWORK/config.json" \ - > "$CFG/$NETWORK/config-merged.json" - export CARDANO_CONFIG="$CFG/$NETWORK/config-merged.json" + > /tmp/cardano-config-merged.json + export CARDANO_CONFIG=/tmp/cardano-config-merged.json else export CARDANO_CONFIG="$CFG/$NETWORK/config.json" fi if [[ -n ${CARDANO_TOPOLOGY_JSON_MERGE:-} ]]; then jq -S \ + --arg cfgDir "$CFG/$NETWORK" \ --argjson deepMerge "$CARDANO_TOPOLOGY_JSON_MERGE" \ - '. * $deepMerge' \ + '. * $deepMerge + | if (.peerSnapshotFile? | type) == "string" and (.peerSnapshotFile | startswith("/") | not) + then .peerSnapshotFile = "\($cfgDir)/\(.peerSnapshotFile)" + else . + end' \ < "$CFG/$NETWORK/topology.json" \ - > "$CFG/$NETWORK/topology-merged.json" - export CARDANO_TOPOLOGY="$CFG/$NETWORK/topology-merged.json" + > /tmp/cardano-topology-merged.json + export CARDANO_TOPOLOGY=/tmp/cardano-topology-merged.json else export CARDANO_TOPOLOGY="$CFG/$NETWORK/topology.json" fi diff --git a/nix/docker/context/tracer/bin/entrypoint b/nix/docker/context/tracer/bin/entrypoint index d6f9ff1cfc8..193fbe61a2f 100755 --- a/nix/docker/context/tracer/bin/entrypoint +++ b/nix/docker/context/tracer/bin/entrypoint @@ -3,6 +3,15 @@ set -euo pipefail [[ -n ${DEBUG:-} ]] && set -x +# The image writes a resolved-config env snapshot and any merge-mode +# artifacts to /tmp. Catch the common operator mistake of running +# with a read-only filesystem without mounting a writable /tmp. +if ! [[ -w /tmp ]]; then + echo "ERROR: /tmp is not writable." >&2 + echo "With a read-only filesystem, mount a tmpfs or emptyDir at /tmp." >&2 + exit 1 +fi + # If the NETWORK env var is set to a valid cardano network, pre-defined # configuration will be used. if [[ -n ${NETWORK:-} ]]; then @@ -30,13 +39,19 @@ if [[ -n ${NETWORK:-} ]]; then # full replacement and null values persist. # # jq -S sorts output keys alphabetically for deterministic diffs. + # + # Merged files are written to /tmp so that the image can run as a + # non-root user ($CFG is image content and only writable by root) + # and under read-only root filesystems. + # The base tracer config has no relative file + # references, so no path rewriting is needed. if [[ -n ${CARDANO_CONFIG_JSON_MERGE:-} ]]; then jq -S \ --argjson deepMerge "$CARDANO_CONFIG_JSON_MERGE" \ '. * $deepMerge' \ < "$CFG/$NETWORK/tracer-config.json" \ - > "$CFG/$NETWORK/tracer-config-merged.json" - export CARDANO_CONFIG="$CFG/$NETWORK/tracer-config-merged.json" + > /tmp/cardano-tracer-config-merged.json + export CARDANO_CONFIG=/tmp/cardano-tracer-config-merged.json else export CARDANO_CONFIG="$CFG/$NETWORK/tracer-config.json" fi From 79f9df39ca4107f7c57c459f4c8ab80c9c66c1b6 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Thu, 25 Jun 2026 19:39:08 -0500 Subject: [PATCH 34/40] nixosSvc: add profilingOutputDir; keep RTS stats, gate profiling/eventlog for node/tracer --- nix/nixos/cardano-node-service.nix | 18 +++++++++++++----- nix/nixos/cardano-tracer-service.nix | 20 ++++++++++++++------ 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix index 9bdad69d15d..48f0e5a8307 100644 --- a/nix/nixos/cardano-node-service.nix +++ b/nix/nixos/cardano-node-service.nix @@ -202,6 +202,12 @@ in { description = '' Haskell profiling types which are available and will be applied to the cardano-node binary if declared. + + Note: the default `profilingArgs` always include the lightweight + `--machine-readable -t...cardano-node.stats` RTS summary (written at + exit, useful in any build). The cost-centre/heap profiling flags and + the `-po` output stem are only added when this is not "none" or + `eventlog` is enabled. ''; }; @@ -789,11 +795,13 @@ in { default = let prefix = if cfg.profilingOutputDir == null then "" else "${cfg.profilingOutputDir}/"; in - optionals (cfg.profiling != "none" || cfg.eventlog) [ - "--machine-readable" - "-t${prefix}cardano-node.stats" - "-po${prefix}cardano-node" - ] + # Always emit the lightweight machine-readable RTS/GC summary at + # exit. It works in any build, costs nothing, and is useful + # telemetry. The OCI images use the profilingOutputDir option to + # ensure it lands on a writable mount under a read-only-root OCI + # image. + [ "--machine-readable" "-t${prefix}cardano-node.stats" ] + ++ optionals (cfg.profiling != "none" || cfg.eventlog) [ "-po${prefix}cardano-node" ] ++ optional (cfg.eventlog) "-l" ++ ( if cfg.profiling == "time" then ["-p"] diff --git a/nix/nixos/cardano-tracer-service.nix b/nix/nixos/cardano-tracer-service.nix index 4cfa71be2f9..45707949f1b 100644 --- a/nix/nixos/cardano-tracer-service.nix +++ b/nix/nixos/cardano-tracer-service.nix @@ -498,6 +498,12 @@ in { description = '' Haskell profiling types which are available and will be applied to the cardano-tracer binary if declared. + + Note: the default `profilingArgs` always include the lightweight + `--machine-readable -t...cardano-tracer.stats` RTS summary (written + at exit, useful in any build). The cost-centre/heap profiling flags + and the `-po` output stem are only added when this is not "none" or + `eventlog` is enabled. ''; }; @@ -506,7 +512,7 @@ in { default = null; description = '' Optional directory prefix for GHC RTS profiling output files - (cardano-node.stats, cardano-node.prof, cardano-node.hp, etc.). + (cardano-tracer.stats, cardano-tracer.prof, cardano-tracer.hp, etc.). When null, files are written relative to the working directory (the systemd unit's WorkingDirectory for NixOS deployments, which is cfg.stateDir). @@ -518,11 +524,13 @@ in { default = let prefix = if cfg.profilingOutputDir == null then "" else "${cfg.profilingOutputDir}/"; in - optionals (cfg.profiling != "none" || cfg.eventlog) [ - "--machine-readable" - "-t${prefix}cardano-node.stats" - "-po${prefix}cardano-node" - ] + # Always emit the lightweight machine-readable RTS/GC summary at + # exit. It works in any build, costs nothing, and is useful + # telemetry. The OCI images use the profilingOutputDir option to + # ensure it lands on a writable mount under a read-only-root OCI + # image. + [ "--machine-readable" "-t${prefix}cardano-tracer.stats" ] + ++ optionals (cfg.profiling != "none" || cfg.eventlog) [ "-po${prefix}cardano-tracer" ] ++ optional (cfg.eventlog) "-l" ++ ( if cfg.profiling == "time" then ["-p"] From 6fc770f02c54d0b5f56bdbd2a816a12604fe120f Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Thu, 25 Jun 2026 19:49:34 -0500 Subject: [PATCH 35/40] oci: direct RTS profiling output to /logs for the images only, not scripts --- nix/pkgs.nix | 16 ++++++++++++++++ nix/scripts.nix | 4 ---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/nix/pkgs.nix b/nix/pkgs.nix index dbcc28fb13a..73e47e27c63 100644 --- a/nix/pkgs.nix +++ b/nix/pkgs.nix @@ -106,6 +106,15 @@ in with final; stateDir = "/data"; dbPrefix = "db"; socketPath = "/ipc/node.socket"; + # Direct GHC RTS output to /logs (a writable mount) so it stays off + # the container's read-only root. This matters even with + # profiling = "none": the lightweight `-t...cardano-node.stats` + # summary is emitted on every run, and without a prefix it would be + # written to the container's cwd (/), which fails under --read-only. + # Profiling/heap/eventlog output (when enabled) lands here too. + # Scoped to the image here rather than scripts.nix so bare + # `nix run .#/node` is unaffected. + profilingOutputDir = "/logs"; }; in callPackage ./docker { @@ -145,6 +154,13 @@ in with final; logFormat = "ForHuman"; } ]; + # As with the node image: direct GHC RTS output to /logs (a writable + # mount) so it stays off the read-only root. This matters even with + # profiling = "none" -- the always-on `-t...cardano-tracer.stats` + # summary would otherwise be written to the container cwd (/) and + # fail under --read-only. Profiling/heap/eventlog output lands here + # too when enabled. + profilingOutputDir = "/logs"; }; in callPackage ./docker/tracer.nix { diff --git a/nix/scripts.nix b/nix/scripts.nix index f6443da273d..421d09f6530 100644 --- a/nix/scripts.nix +++ b/nix/scripts.nix @@ -16,10 +16,6 @@ let nodeConfig = cfg.environments.${cfg.environment}.nodeConfig; stateDir = mkDefault "state-node-${cfg.environment}"; runtimeDir = mkDefault null; - # When profiling is enabled, direct GHC RTS output - # (stats, prof, hp, ...) to /logs so the OCI image works - # with a read-only root filesystem. No effect when profiling = "none". - profilingOutputDir = mkDefault "/logs"; } // optionalAttrs (envConfig ? topology) { topology = mkDefault envConfig.topology; }; From 9b3ed4ff979c03566efa7c66618f4305ddf3e7c6 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Thu, 25 Jun 2026 19:53:44 -0500 Subject: [PATCH 36/40] oci: support ro root/non-root via symlink-following/TOCTOU protected private /tmp runtime dir --- nix/docker/README.md | 95 +++++++++++++++++++----- nix/docker/context/node/bin/entrypoint | 64 ++++++++++++---- nix/docker/context/node/bin/run-node | 14 ++-- nix/docker/context/tracer/bin/entrypoint | 32 +++++++- nix/docker/context/tracer/bin/run-tracer | 4 +- nix/docker/default.nix | 7 +- nix/docker/tracer.nix | 7 +- 7 files changed, 173 insertions(+), 50 deletions(-) diff --git a/nix/docker/README.md b/nix/docker/README.md index 55450c5f442..131a6b8e34c 100644 --- a/nix/docker/README.md +++ b/nix/docker/README.md @@ -120,12 +120,14 @@ docker run \ ghcr.io/intersectmbo/cardano-node:dev ``` -The resulting merged config and topology are written to -`/tmp/cardano-{{,tracer-}config,topology}-merged.json` -and used as the runtime configuration. -Relative file references are rewritten to absolute paths -anchored at `/opt/cardano/config/$NETWORK/` -so they resolve from the new location. +The resulting merged config and topology are written to a private, +per-container runtime directory under `/tmp` (see +[Read-Only Root Filesystem](#read-only-root-filesystem) for the exact path) +as `config-merged.json` / `topology-merged.json` (node) or +`tracer-config-merged.json` (tracer), and used as the runtime configuration. +Relative file references (the config's `*File` keys and the topology's +`peerSnapshotFile`) are rewritten to absolute paths anchored at +`/opt/cardano/config/$NETWORK/` so they resolve from the new location. ## CLI Mode @@ -158,21 +160,62 @@ default state directory locations, `/{data,ipc,logs}`, will work for both modes. ## Read-Only Root Filesystem -The image is compatible with `--read-only` (Docker/Podman) and +Under a normal writable root filesystem no `/tmp` mount is needed. This holds +for every mode, run as root or as a non-root user, so existing deployments need +no change. + +A writable `/tmp` must be supplied explicitly only when the root filesystem is +made read-only. The image is compatible with `--read-only` (Docker/Podman) and `securityContext.readOnlyRootFilesystem: true` (Kubernetes), provided the runtime supplies writable storage for the state directories described above -(`/data`, `/ipc`, `/logs`) and a writable `/tmp` (tmpfs or `emptyDir`). +(`/data`, `/ipc`, `/logs`) and a writable `/tmp` (tmpfs or `emptyDir`). Under +`--read-only` this applies to every run mode except `cli`, which does not use +`/tmp`. + +For example, scripts mode with a read-only root filesystem, a per-container +tmpfs at `/tmp`, and named volumes for the state directories: +``` +docker run \ + --read-only \ + --tmpfs /tmp \ + -v mainnet-data:/data \ + -v mainnet-ipc:/ipc \ + -v mainnet-logs:/logs \ + -e NETWORK=mainnet \ + ghcr.io/intersectmbo/cardano-node:dev +``` +All runtime-generated artifacts (the merged config/topology and the env +snapshot below) are written under a private, `0700` runtime directory that +the entrypoint creates in `/tmp`, at a fixed, predictable per-role path: + +``` +/tmp/cardano-node/ # node image: config-merged.json, topology-merged.json, env +/tmp/cardano-tracer/ # tracer image: tracer-config-merged.json, env +``` + +The path is fixed so operators and tooling can refer to the effective +config/topology dependably. The entrypoint creates the directory atomically +with `mkdir -m 700` and refuses to start if the path already exists and is +not a private directory it owns, so it never follows an attacker-planted +symlink. A consequence of the fixed name is that **a `/tmp` mount must not be +shared across containers of the same role** — a second node (or tracer) +container sharing one `/tmp` would refuse to start rather than collide. Use a +per-container `/tmp`. -A resolved-configuration snapshot is written at runtime to `/tmp/cardano-env` -and can be `source`d for an interactive debug shell inside the container. -The legacy path `/usr/local/bin/env` is preserved as a symlink to -`/tmp/cardano-env` for backwards compatibility. +A resolved-configuration snapshot is written at runtime to `env` inside that +directory and can be `source`d for an interactive debug shell inside the +container. The path `/usr/local/bin/env` is preserved as a symlink to it for +backwards compatibility, so `source /usr/local/bin/env` keeps working. This +is the supported way to locate the effective configuration: sourcing it +exports `CARDANO_CONFIG`, `CARDANO_TOPOLOGY`, the socket path, etc. as the +node was actually launched with. -In "scripts" mode, GHC RTS profiling output (`cardano-node.stats`, -`cardano-node.prof`, `cardano-node.hp`, etc.) is directed to `/logs/` so the -image keeps working when profiling is enabled under a read-only root. -In "custom" mode the operator chooses the RTS flags, so any profiling output -must similarly be directed to a writable mount, for example: +In "scripts" mode GHC RTS output is directed to `/logs/` so the image keeps +working under a read-only root. The lightweight machine-readable RTS summary +(`/logs/cardano-node.stats`, written at process exit) is always produced; the +heavier profiling/eventlog outputs are produced only when profiling or eventlog +is enabled. In "custom" mode the operator chooses the RTS flags, so any such +output must similarly be directed to a writable mount, for example: ``` ... run \ --config /opt/cardano/config/mainnet/config.json \ @@ -180,6 +223,10 @@ must similarly be directed to a writable mount, for example: +RTS --machine-readable -t/logs/cardano-node.stats -po/logs/cardano-node -p -RTS ``` +The read-only, non-root and private-`/tmp` behaviors are exercised by the +`nixosTests/cardanoNodeOciReadonly` NixOS test +(`nix build .#checks..nixosTests/cardanoNodeOciReadonly`). + ## Non-Root User The image can run as any non-root user (`docker run --user ` / @@ -196,6 +243,20 @@ default for `runAsUser`) or with supplementary group 0. In Kubernetes you can also set `securityContext.fsGroup: 0` to have the kubelet chown the volume on mount. For Docker, `--user :0` is the equivalent. +For example, the read-only invocation above, run as a non-root UID in +group 0: +``` +docker run \ + --read-only \ + --tmpfs /tmp \ + --user 1000:0 \ + -v mainnet-data:/data \ + -v mainnet-ipc:/ipc \ + -v mainnet-logs:/logs \ + -e NETWORK=mainnet \ + ghcr.io/intersectmbo/cardano-node:dev +``` + The image defaults to running as root; specify a UID explicitly to opt into a non-root run. diff --git a/nix/docker/context/node/bin/entrypoint b/nix/docker/context/node/bin/entrypoint index 88062d4b5e6..7acf6077ce8 100755 --- a/nix/docker/context/node/bin/entrypoint +++ b/nix/docker/context/node/bin/entrypoint @@ -3,13 +3,38 @@ set -euo pipefail [[ -n ${DEBUG:-} ]] && set -x -# The image writes a resolved-config env snapshot and any merge-mode -# artifacts to /tmp. Catch the common operator mistake of running -# with a read-only filesystem without mounting a writable /tmp. -if ! [[ -w /tmp ]]; then - echo "ERROR: /tmp is not writable." >&2 - echo "With a read-only filesystem, mount a tmpfs or emptyDir at /tmp." >&2 - exit 1 +# Every mode except "cli" (which only execs cardano-cli) writes a +# resolved-config env snapshot and any merge-mode artifacts under a private +# runtime directory in /tmp. +if [[ ${1:-} != "cli" ]]; then + # Catch the common operator mistake of running with a read-only + # filesystem without mounting a writable /tmp. + if ! [[ -w /tmp ]]; then + echo "ERROR: /tmp is not writable." >&2 + echo "With a read-only filesystem, mount a tmpfs or emptyDir at /tmp." >&2 + exit 1 + fi + + # Generated artifacts go under a private, user-owned 0700 directory at a + # fixed, predictable per-role path, so operators and tooling can refer to + # the effective config/topology dependably. The effective file paths are + # also recorded in the env snapshot below; `source /usr/local/bin/env` + # exposes them as $CARDANO_CONFIG / $CARDANO_TOPOLOGY. + # + # This approach avoids a symlink/TOCTOU vector. Consequently a second node + # container sharing one /tmp mount refuses to start with a clear error rather + # than colliding; sharing a /tmp across same-role containers is unsupported. + # CARDANO_RUNTIME_DIR is exported for the run-* scripts. + export CARDANO_RUNTIME_DIR=/tmp/cardano-node + if ! mkdir -m 700 "$CARDANO_RUNTIME_DIR" 2>/dev/null; then + if [[ -L $CARDANO_RUNTIME_DIR || ! -d $CARDANO_RUNTIME_DIR || ! -O $CARDANO_RUNTIME_DIR ]]; then + echo "ERROR: $CARDANO_RUNTIME_DIR exists and is not a private directory owned by this user." >&2 + echo "Refusing to use it to avoid following an attacker-planted path." >&2 + exit 1 + fi + # Re-assert restrictive perms in case an earlier run left them looser. + chmod 700 "$CARDANO_RUNTIME_DIR" + fi fi # If the NETWORK env var is set to a valid cardano network, pre-defined @@ -43,15 +68,24 @@ if [[ -n ${NETWORK:-} ]]; then # Merged files are written to /tmp so that the image can run as a # non-root user ($CFG is image content and only writable by root) # and under read-only root filesystems. - # Relative file references are rewritten to absolute paths - # anchored at $CFG/$NETWORK/ so they resolve from the new location. + # + # cardano-node resolves relative file references in the config relative + # to the config file's own directory. Since the merged config now lives + # in /tmp instead of $CFG/$NETWORK, any relative reference would resolve + # against /tmp and fail. Rewrite each relative "*File" value to an + # absolute path anchored at the original config dir. + # + # The node config schema's file references are the "*File" keys + # (ByronGenesisFile, ShelleyGenesisFile, AlonzoGenesisFile, + # ConwayGenesisFile, CheckpointsFile, ...); matching the "File" suffix + # keeps this working if more are added. if [[ -n ${CARDANO_CONFIG_JSON_MERGE:-} ]]; then jq -S \ --arg cfgDir "$CFG/$NETWORK" \ --argjson deepMerge "$CARDANO_CONFIG_JSON_MERGE" \ '. * $deepMerge | with_entries( - if ((.key | test("GenesisFile$|^CheckpointsFile$")) + if ((.key | endswith("File")) and (.value | type == "string") and (.value | startswith("/") | not)) then .value = "\($cfgDir)/\(.value)" @@ -59,12 +93,14 @@ if [[ -n ${NETWORK:-} ]]; then end )' \ < "$CFG/$NETWORK/config.json" \ - > /tmp/cardano-config-merged.json - export CARDANO_CONFIG=/tmp/cardano-config-merged.json + > "$CARDANO_RUNTIME_DIR/config-merged.json" + export CARDANO_CONFIG="$CARDANO_RUNTIME_DIR/config-merged.json" else export CARDANO_CONFIG="$CFG/$NETWORK/config.json" fi + # peerSnapshotFile is the only relative file reference in the topology + # schema; rewrite it to absolute for the same reason as the config above. if [[ -n ${CARDANO_TOPOLOGY_JSON_MERGE:-} ]]; then jq -S \ --arg cfgDir "$CFG/$NETWORK" \ @@ -75,8 +111,8 @@ if [[ -n ${NETWORK:-} ]]; then else . end' \ < "$CFG/$NETWORK/topology.json" \ - > /tmp/cardano-topology-merged.json - export CARDANO_TOPOLOGY=/tmp/cardano-topology-merged.json + > "$CARDANO_RUNTIME_DIR/topology-merged.json" + export CARDANO_TOPOLOGY="$CARDANO_RUNTIME_DIR/topology-merged.json" else export CARDANO_TOPOLOGY="$CFG/$NETWORK/topology.json" fi diff --git a/nix/docker/context/node/bin/run-node b/nix/docker/context/node/bin/run-node index 607dc670e61..a59ccad7721 100755 --- a/nix/docker/context/node/bin/run-node +++ b/nix/docker/context/node/bin/run-node @@ -98,7 +98,7 @@ printRunEnv () { # writeRootEnv () { -cat << EOF > /tmp/cardano-env +cat << EOF > "$CARDANO_RUNTIME_DIR/env" #!/usr/bin/env bash # Docker run ENV vars @@ -106,30 +106,30 @@ EOF if [[ -n ${CARDANO_SHELLEY_KES_AGENT_SOCKET:-} ]]; then echo "CARDANO_SHELLEY_KES_AGENT_SOCKET=\"$CARDANO_SHELLEY_KES_AGENT_SOCKET\"" \ - >> /tmp/cardano-env + >> "$CARDANO_RUNTIME_DIR/env" fi if [[ -n ${CARDANO_TRACER_SOCKET_NETWORK_ACCEPT:-} ]]; then echo "CARDANO_TRACER_SOCKET_NETWORK_ACCEPT=\"$CARDANO_TRACER_SOCKET_NETWORK_ACCEPT\"" \ - >> /tmp/cardano-env + >> "$CARDANO_RUNTIME_DIR/env" fi if [[ -n ${CARDANO_TRACER_SOCKET_NETWORK_CONNECT:-} ]]; then echo "CARDANO_TRACER_SOCKET_NETWORK_CONNECT=\"$CARDANO_TRACER_SOCKET_NETWORK_CONNECT\"" \ - >> /tmp/cardano-env + >> "$CARDANO_RUNTIME_DIR/env" fi if [[ -n ${CARDANO_TRACER_SOCKET_PATH_ACCEPT:-} ]]; then echo "CARDANO_TRACER_SOCKET_PATH_ACCEPT=\"$CARDANO_TRACER_SOCKET_PATH_ACCEPT\"" \ - >> /tmp/cardano-env + >> "$CARDANO_RUNTIME_DIR/env" fi if [[ -n ${CARDANO_TRACER_SOCKET_PATH_CONNECT:-} ]]; then echo "CARDANO_TRACER_SOCKET_PATH_CONNECT=\"$CARDANO_TRACER_SOCKET_PATH_CONNECT\"" \ - >> /tmp/cardano-env + >> "$CARDANO_RUNTIME_DIR/env" fi -cat << EOF >> /tmp/cardano-env +cat << EOF >> "$CARDANO_RUNTIME_DIR/env" CARDANO_BIND_ADDR="$CARDANO_BIND_ADDR" CARDANO_BLOCK_PRODUCER=$CARDANO_BLOCK_PRODUCER CARDANO_CONFIG="$CARDANO_CONFIG" diff --git a/nix/docker/context/tracer/bin/entrypoint b/nix/docker/context/tracer/bin/entrypoint index 193fbe61a2f..b06e80d8407 100755 --- a/nix/docker/context/tracer/bin/entrypoint +++ b/nix/docker/context/tracer/bin/entrypoint @@ -4,14 +4,38 @@ set -euo pipefail [[ -n ${DEBUG:-} ]] && set -x # The image writes a resolved-config env snapshot and any merge-mode -# artifacts to /tmp. Catch the common operator mistake of running -# with a read-only filesystem without mounting a writable /tmp. +# artifacts under a private runtime directory in /tmp. + +# Catch the common operator mistake of running with a read-only filesystem +# without mounting a writable /tmp. if ! [[ -w /tmp ]]; then echo "ERROR: /tmp is not writable." >&2 echo "With a read-only filesystem, mount a tmpfs or emptyDir at /tmp." >&2 exit 1 fi +# Generated artifacts go under a private, user-owned 0700 directory at a +# fixed, predictable per-role path, so operators and tooling can refer to the +# effective config dependably. The "-tracer" suffix keeps it distinct from +# the node image's dir when both share a pod's /tmp. The effective config +# path is also recorded in the env snapshot below; `source /usr/local/bin/env` +# exposes it as $CARDANO_CONFIG. +# +# This approach avoids a symlink/TOCTOU vector. Consequently a second tracer +# container sharing one /tmp mount refuses to start with a clear error rather +# than colliding; sharing a /tmp across same-role containers is unsupported. +# CARDANO_RUNTIME_DIR is exported for the run-* scripts. +export CARDANO_RUNTIME_DIR=/tmp/cardano-tracer +if ! mkdir -m 700 "$CARDANO_RUNTIME_DIR" 2>/dev/null; then + if [[ -L $CARDANO_RUNTIME_DIR || ! -d $CARDANO_RUNTIME_DIR || ! -O $CARDANO_RUNTIME_DIR ]]; then + echo "ERROR: $CARDANO_RUNTIME_DIR exists and is not a private directory owned by this user." >&2 + echo "Refusing to use it to avoid following an attacker-planted path." >&2 + exit 1 + fi + # Re-assert restrictive perms in case an earlier run left them looser. + chmod 700 "$CARDANO_RUNTIME_DIR" +fi + # If the NETWORK env var is set to a valid cardano network, pre-defined # configuration will be used. if [[ -n ${NETWORK:-} ]]; then @@ -50,8 +74,8 @@ if [[ -n ${NETWORK:-} ]]; then --argjson deepMerge "$CARDANO_CONFIG_JSON_MERGE" \ '. * $deepMerge' \ < "$CFG/$NETWORK/tracer-config.json" \ - > /tmp/cardano-tracer-config-merged.json - export CARDANO_CONFIG=/tmp/cardano-tracer-config-merged.json + > "$CARDANO_RUNTIME_DIR/tracer-config-merged.json" + export CARDANO_CONFIG="$CARDANO_RUNTIME_DIR/tracer-config-merged.json" else export CARDANO_CONFIG="$CFG/$NETWORK/tracer-config.json" fi diff --git a/nix/docker/context/tracer/bin/run-tracer b/nix/docker/context/tracer/bin/run-tracer index b8fc44018e4..858ba01b166 100755 --- a/nix/docker/context/tracer/bin/run-tracer +++ b/nix/docker/context/tracer/bin/run-tracer @@ -34,7 +34,7 @@ printRunEnv () { # writeRootEnv () { -cat << EOF > /tmp/cardano-env +cat << EOF > "$CARDANO_RUNTIME_DIR/env" #!/usr/bin/env bash # Docker run ENV vars @@ -44,7 +44,7 @@ EOF if [[ -n ${CARDANO_MIN_LOG_SEVERITY:-} ]]; then echo "CARDANO_MIN_LOG_SEVERITY=\"$CARDANO_MIN_LOG_SEVERITY\"" \ - >> /tmp/cardano-env + >> "$CARDANO_RUNTIME_DIR/env" fi } diff --git a/nix/docker/default.nix b/nix/docker/default.nix index 0945c8f8998..f95f4e2b1cf 100644 --- a/nix/docker/default.nix +++ b/nix/docker/default.nix @@ -186,9 +186,10 @@ in ln -sv ${jq}/bin/jq usr/local/bin/jq # Backwards-compatible alias for the resolved-config env snapshot - # written by run-node. The runtime writer targets /tmp/cardano-env so - # the image remains compatible with a read-only root filesystem. - ln -sv /tmp/cardano-env usr/local/bin/env + # written by run-node at the fixed per-role path /tmp/cardano-node/env, + # so `source /usr/local/bin/env` keeps working while the image stays + # compatible with a read-only root filesystem. + ln -sv /tmp/cardano-node/env usr/local/bin/env # Create iohk-nix network configs, organized by network directory. SRC="${genCfgs}" diff --git a/nix/docker/tracer.nix b/nix/docker/tracer.nix index fb767efeede..42ce15c40b6 100644 --- a/nix/docker/tracer.nix +++ b/nix/docker/tracer.nix @@ -165,9 +165,10 @@ in ln -sv ${jq}/bin/jq usr/local/bin/jq # Backwards-compatible alias for the resolved-config env snapshot - # written by run-tracer. The runtime writer targets /tmp/cardano-env so - # the image remains compatible with a read-only root filesystem. - ln -sv /tmp/cardano-env usr/local/bin/env + # written by run-tracer at the fixed per-role path /tmp/cardano-tracer/env, + # so `source /usr/local/bin/env` keeps working while the image stays + # compatible with a read-only root filesystem. + ln -sv /tmp/cardano-tracer/env usr/local/bin/env # Create iohk-nix network configs, organized by network directory. SRC="${genCfgs}" From ae3429dcd4aa8f8521f27e45fe6090de18961459 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Thu, 25 Jun 2026 20:13:00 -0500 Subject: [PATCH 37/40] nixosTests: add cardano-node OCI read-only/non-root test --- nix/nixos/tests/cardano-node-oci-readonly.nix | 107 ++++++++++++++++++ nix/nixos/tests/default.nix | 6 + 2 files changed, 113 insertions(+) create mode 100644 nix/nixos/tests/cardano-node-oci-readonly.nix diff --git a/nix/nixos/tests/cardano-node-oci-readonly.nix b/nix/nixos/tests/cardano-node-oci-readonly.nix new file mode 100644 index 00000000000..4f5c1f5d420 --- /dev/null +++ b/nix/nixos/tests/cardano-node-oci-readonly.nix @@ -0,0 +1,107 @@ +{ + pkgs, + dockerImage, + ... +}: let + inherit (pkgs) lib; + + # The image ref as `docker load` will tag it (repoName:gitrev). + imageRef = "${dockerImage.imageName}:${dockerImage.imageTag}"; + + network = "preview"; + + # Harmless config/topology merge keys, present only to exercise merge mode + # which is what triggers the /tmp runtime dir + relative-path rewriting. + # The double quotes are backslash-escaped because these are interpolated + # into double-quoted Python string literals in the testScript below. + configMerge = ''{\"MaxConcurrencyBulkSync\":2}''; + topologyMerge = ''{\"useLedgerAfterSlot\":1}''; + + rt = "/tmp/cardano-node"; +in { + name = "cardano-node-oci-readonly-test"; + + nodes = { + machine = {...}: { + nixpkgs.pkgs = pkgs; + + # Room for the image load + overlay + a starting node db. + virtualisation.diskSize = 8192; + virtualisation.memorySize = 3072; + virtualisation.docker.enable = true; + }; + }; + + # Like the other tests here, this is sandboxed: the node cannot sync, but it + # starts and resolves its configuration, which is all these assertions need. + testScript = '' + start_all() + machine.wait_for_unit("multi-user.target") + machine.wait_until_succeeds("docker info", timeout=120) + machine.succeed("docker load -i ${dockerImage}") + + # Read-only root + non-root UID in group 0 + merge mode. Asserts the + # container starts as read-only and group-0 volume writes work, creates its + # fixed 0700 runtime dir, writes the merged config with absolute genesis + # paths, writes the env snapshot, and the /usr/local/bin/env alias sources + # it. + machine.succeed( + "docker run -d --name n1 --read-only --tmpfs /tmp --user 1000:0 " + "-v n1data:/data -v n1ipc:/ipc -v n1logs:/logs " + "-e NETWORK=${network} " + "-e CARDANO_CONFIG_JSON_MERGE='${configMerge}' " + "-e CARDANO_TOPOLOGY_JSON_MERGE='${topologyMerge}' " + "${imageRef}" + ) + + # The runtime artifacts are written by the entrypoint/run-node before the + # node execs, so they appear shortly after start. Waiting on the env + # snapshot also confirms the container did not crash under --read-only and + # --user since `docker exec` fails against an exited container. + machine.wait_until_succeeds("docker exec n1 test -f ${rt}/env", timeout=60) + machine.succeed("[ \"$(docker inspect -f '{{.State.Running}}' n1)\" = true ]") + + machine.succeed("docker exec n1 test -d ${rt}") + machine.succeed("docker exec n1 sh -c '[ \"$(stat -c %a ${rt})\" = 700 ]'") + machine.succeed("docker exec n1 test -f ${rt}/config-merged.json") + machine.succeed("docker exec n1 test -f ${rt}/topology-merged.json") + + # The merged config's relative "*File" references (ByronGenesisFile, + # ShelleyGenesisFile, ..., CheckpointsFile) must be rewritten to absolute + # paths so they resolve from /tmp. Mirror the entrypoint's predicate -- any + # top-level key ending in "File" with a string value -- and assert all are + # absolute. Use jq from the container. + machine.succeed( + "docker exec n1 jq -e " + "'[to_entries[]|select(.key|endswith(\"File\"))|.value|select(type==\"string\")] as $v " + "| ($v|length>0) and ($v|all(startswith(\"/\")))' " + "${rt}/config-merged.json" + ) + + # The merged topology's relative peerSnapshotFile must likewise be rewritten + # to an absolute path. + machine.succeed( + "docker exec n1 jq -e " + "'(.peerSnapshotFile|type==\"string\") and (.peerSnapshotFile|startswith(\"/\"))' " + "${rt}/topology-merged.json" + ) + + # The env snapshot is POSIX-sourceable; use sh and `.` to load it and + # confirm it populated the env. + machine.succeed( + "docker exec --user 1000:0 n1 sh -c " + "'. /usr/local/bin/env && [ -n \"$CARDANO_CONFIG\" ]'" + ) + machine.succeed("docker rm -f n1") + + # Read-only root WITHOUT a writable /tmp must fail fast with guidance. + status, out = machine.execute( + "docker run --rm --read-only -e NETWORK=${network} ${imageRef} 2>&1" + ) + assert status != 0, "expected non-zero exit when /tmp is not writable" + assert "/tmp is not writable" in out, f"missing actionable /tmp error; got: {out}" + + # Cli mode must NOT require a writable /tmp (read-only, no tmpfs). + machine.succeed("docker run --rm --read-only ${imageRef} cli version") + ''; +} diff --git a/nix/nixos/tests/default.nix b/nix/nixos/tests/default.nix index c058a7d00e3..208c7ecadb9 100644 --- a/nix/nixos/tests/default.nix +++ b/nix/nixos/tests/default.nix @@ -30,4 +30,10 @@ in { # Tests a mainnet edge node with submit-api using nixos service config. cardanoNodeEdge = callTest ./cardano-node-edge.nix {}; + + # Tests the OCI image under a read-only root, a non-root UID in group 0, and + # a private /tmp: container startup, the fixed 0700 runtime dir, merged-config + # genesis path rewriting, the env snapshot/alias, the fail-fast on a missing + # writable /tmp, and that cli mode does not require /tmp. + cardanoNodeOciReadonly = callTest ./cardano-node-oci-readonly.nix {dockerImage = pkgs.dockerImage;}; } From c33cbd2fe105c74a741b9af61bb14a01a2e62d89 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Thu, 25 Jun 2026 22:33:37 -0500 Subject: [PATCH 38/40] nixosTests: expand to covering tracer and submit-api also --- nix/docker/README.md | 6 +- ...-readonly.nix => cardano-oci-readonly.nix} | 57 ++++++++++++++++--- nix/nixos/tests/default.nix | 14 +++-- 3 files changed, 60 insertions(+), 17 deletions(-) rename nix/nixos/tests/{cardano-node-oci-readonly.nix => cardano-oci-readonly.nix} (59%) diff --git a/nix/docker/README.md b/nix/docker/README.md index 131a6b8e34c..ee55d9617b2 100644 --- a/nix/docker/README.md +++ b/nix/docker/README.md @@ -223,9 +223,9 @@ output must similarly be directed to a writable mount, for example: +RTS --machine-readable -t/logs/cardano-node.stats -po/logs/cardano-node -p -RTS ``` -The read-only, non-root and private-`/tmp` behaviors are exercised by the -`nixosTests/cardanoNodeOciReadonly` NixOS test -(`nix build .#checks..nixosTests/cardanoNodeOciReadonly`). +The read-only, non-root and private-`/tmp` behaviors of all three images are +exercised by the `nixosTests/cardanoOciReadonly` NixOS test +(`nix build .#checks..nixosTests/cardanoOciReadonly`). ## Non-Root User diff --git a/nix/nixos/tests/cardano-node-oci-readonly.nix b/nix/nixos/tests/cardano-oci-readonly.nix similarity index 59% rename from nix/nixos/tests/cardano-node-oci-readonly.nix rename to nix/nixos/tests/cardano-oci-readonly.nix index 4f5c1f5d420..e01f495012c 100644 --- a/nix/nixos/tests/cardano-node-oci-readonly.nix +++ b/nix/nixos/tests/cardano-oci-readonly.nix @@ -1,33 +1,40 @@ { pkgs, dockerImage, + tracerDockerImage, + submitApiDockerImage, ... }: let inherit (pkgs) lib; - # The image ref as `docker load` will tag it (repoName:gitrev). + # Image refs as `docker load` will tag them (repoName:gitrev). imageRef = "${dockerImage.imageName}:${dockerImage.imageTag}"; + tracerImageRef = "${tracerDockerImage.imageName}:${tracerDockerImage.imageTag}"; + submitApiImageRef = "${submitApiDockerImage.imageName}:${submitApiDockerImage.imageTag}"; network = "preview"; - # Harmless config/topology merge keys, present only to exercise merge mode - # which is what triggers the /tmp runtime dir + relative-path rewriting. - # The double quotes are backslash-escaped because these are interpolated - # into double-quoted Python string literals in the testScript below. + # Harmless merge keys, present only to exercise merge mode which is what + # triggers the /tmp runtime dir + relative-path rewriting. The double quotes + # are backslash-escaped because these are interpolated into double-quoted + # Python string literals in the testScript below. configMerge = ''{\"MaxConcurrencyBulkSync\":2}''; topologyMerge = ''{\"useLedgerAfterSlot\":1}''; + tracerConfigMerge = ''{\"verbosity\":\"Minimum\"}''; rt = "/tmp/cardano-node"; + tracerRt = "/tmp/cardano-tracer"; in { - name = "cardano-node-oci-readonly-test"; + name = "cardano-oci-readonly-test"; nodes = { machine = {...}: { nixpkgs.pkgs = pkgs; - # Room for the image load + overlay + a starting node db. - virtualisation.diskSize = 8192; - virtualisation.memorySize = 3072; + # Room for loading three images (node, tracer, submit-api) + overlay + + # a starting node/tracer. + virtualisation.diskSize = 10240; + virtualisation.memorySize = 4096; virtualisation.docker.enable = true; }; }; @@ -103,5 +110,37 @@ in { # Cli mode must NOT require a writable /tmp (read-only, no tmpfs). machine.succeed("docker run --rm --read-only ${imageRef} cli version") + + # Tracer image: same read-only + non-root + merge-mode behavior as the + # node, minus the genesis/topology rewrites as tracer config has no + # relative file references. + machine.succeed("docker load -i ${tracerDockerImage}") + machine.succeed( + "docker run -d --name t1 --read-only --tmpfs /tmp --user 1000:0 " + "-v t1data:/data -v t1ipc:/ipc -v t1logs:/logs " + "-e NETWORK=${network} " + "-e CARDANO_CONFIG_JSON_MERGE='${tracerConfigMerge}' " + "${tracerImageRef}" + ) + # Give the entrypoint time to write artifacts and exec the tracer. If the + # container exits early, surface its logs rather than failing later with an + # opaque `docker exec` timeout against a dead container. + machine.sleep(10) + if machine.succeed("docker inspect -f '{{.State.Running}}' t1").strip() != "true": + _, logs = machine.execute("docker logs t1 2>&1") + raise Exception("tracer container exited early; docker logs:\n" + logs) + machine.succeed("docker exec t1 sh -c '[ \"$(stat -c %a ${tracerRt})\" = 700 ]'") + machine.succeed("docker exec t1 test -f ${tracerRt}/tracer-config-merged.json") + machine.succeed("docker exec t1 test -f ${tracerRt}/env") + machine.succeed( + "docker exec --user 1000:0 t1 sh -c " + "'. /usr/local/bin/env && [ -n \"$CARDANO_CONFIG\" ]'" + ) + machine.succeed("docker rm -f t1") + + # Submit-api image: stateless (no env snapshot / merge / state writes), + # so it only needs to execute under --read-only and as non-root. + machine.succeed("docker load -i ${submitApiDockerImage}") + machine.succeed("docker run --rm --read-only --user 1000:0 ${submitApiImageRef} --help >/dev/null") ''; } diff --git a/nix/nixos/tests/default.nix b/nix/nixos/tests/default.nix index 208c7ecadb9..1ffa7813506 100644 --- a/nix/nixos/tests/default.nix +++ b/nix/nixos/tests/default.nix @@ -31,9 +31,13 @@ in { # Tests a mainnet edge node with submit-api using nixos service config. cardanoNodeEdge = callTest ./cardano-node-edge.nix {}; - # Tests the OCI image under a read-only root, a non-root UID in group 0, and - # a private /tmp: container startup, the fixed 0700 runtime dir, merged-config - # genesis path rewriting, the env snapshot/alias, the fail-fast on a missing - # writable /tmp, and that cli mode does not require /tmp. - cardanoNodeOciReadonly = callTest ./cardano-node-oci-readonly.nix {dockerImage = pkgs.dockerImage;}; + # Tests the OCI images (node, tracer, submit-api) under a read-only root, a + # non-root UID in group 0, and a private /tmp: container startup, the fixed + # 0700 runtime dir, merged-config path rewriting, the env snapshot/alias, the + # fail-fast on a missing writable /tmp, and that cli mode does not require /tmp. + cardanoOciReadonly = callTest ./cardano-oci-readonly.nix { + dockerImage = pkgs.dockerImage; + tracerDockerImage = pkgs.tracerDockerImage; + submitApiDockerImage = pkgs.submitApiDockerImage; + }; } From 0ce8ad4f8acd3adc85e02df56689d6f0b5e0d429 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Thu, 25 Jun 2026 22:36:20 -0500 Subject: [PATCH 39/40] tracerOci: fix tracer aborting under set -e when CARDANO_MIN_LOG_SEVERITY is unset --- nix/docker/context/tracer/bin/run-tracer | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nix/docker/context/tracer/bin/run-tracer b/nix/docker/context/tracer/bin/run-tracer index 858ba01b166..f62c3cbdb31 100755 --- a/nix/docker/context/tracer/bin/run-tracer +++ b/nix/docker/context/tracer/bin/run-tracer @@ -25,7 +25,10 @@ printRunEnv () { echo "CARDANO_CONFIG=$CARDANO_CONFIG" echo "CARDANO_STATE_DIR=$CARDANO_STATE_DIR" - [[ -n ${CARDANO_MIN_LOG_SEVERITY:-} ]] && echo "CARDANO_MIN_LOG_SEVERITY=$CARDANO_MIN_LOG_SEVERITY" + + if [[ -n ${CARDANO_MIN_LOG_SEVERITY:-} ]]; then + echo "CARDANO_MIN_LOG_SEVERITY=$CARDANO_MIN_LOG_SEVERITY" + fi } ##################################################################### From 333c6753efb5b126b8ae0dda693c76d0a94fc924 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Wed, 1 Jul 2026 15:17:01 +0300 Subject: [PATCH 40/40] Update srps --- cabal.project | 26 +++++++++++---------- cardano-submit-api/cardano-submit-api.cabal | 4 ++-- cardano-testnet/cardano-testnet.cabal | 4 ++-- flake.lock | 12 +++++----- 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/cabal.project b/cabal.project index 72751dd7d92..1cedda9fb24 100644 --- a/cabal.project +++ b/cabal.project @@ -13,8 +13,8 @@ repository cardano-haskell-packages -- See CONTRIBUTING for information about these, including some Nix commands -- you need to run if you change them index-state: - , hackage.haskell.org 2026-06-10T14:43:25Z - , cardano-haskell-packages 2026-06-10T14:16:00Z + , hackage.haskell.org 2026-06-29T22:49:53Z + , cardano-haskell-packages 2026-06-29T12:05:19Z constraints: -- haskell.nix patch does not work for 1.6.8 @@ -78,6 +78,8 @@ package plutus-scripts-bench allow-newer: , io-sim:time , io-classes:time + -- TODO: waiting for upstream to remove the bound + , *:aeson -- IMPORTANT -- Do NOT add more source-repository-package stanzas here unless they are strictly @@ -87,8 +89,8 @@ allow-newer: source-repository-package type: git location: https://github.com/IntersectMBO/cardano-api.git - tag: bb64ad40f904040a9b4d99306cf13eef4650a6c9 - --sha256: sha256-302P+qj+UkY/uSWHt8YMT4VzNhRS9tWMz0G6rjr4eVk= + tag: a775a6a9d581e2216ebc31f27572f4ae1c499e48 + --sha256: sha256-U0UDIuhsDfJ4PDV1BqDTY7/2gWpf8c/Z2IVqUGMNhKQ= subdir: cardano-api cardano-rpc @@ -96,16 +98,16 @@ source-repository-package source-repository-package type: git location: https://github.com/IntersectMBO/cardano-cli.git - tag: 9678a844fd2b53bc3777906d61b53fe0ff93e8ed - --sha256: sha256-0Eo3txVuqkmvZdsyalKbS+bnmFzevG4O99CavSjo7SM= + tag: b698d1f0f5681da0ae43dc9db10f82b2de35c0c6 + --sha256: sha256-f0Twa/vH7R98xN03XFPlfoIX3mZlkNTGJ+gHMjNaIkU= subdir: cardano-cli source-repository-package type: git location: https://github.com/IntersectMBO/cardano-ledger.git - tag: 8dc1c431e06db2c8b5d44fb4b3cca6419197d763 - --sha256: sha256-xKtgNFxjbJE6UWGvTioGX81NgvlKH3mHEaYMWDm8/UA= + tag: 245e63dcd335e3ca29823609e4aaa8f5f096a425 + --sha256: sha256-chsapfNnY/V7y1snBzjP0LSVnYl/Axt6GjahTPNOKj0= subdir: eras/allegra/impl eras/alonzo/impl @@ -141,16 +143,16 @@ source-repository-package source-repository-package type: git location: https://github.com/IntersectMBO/ouroboros-consensus.git - tag: e6fad063078894cefece2a535a2e2b4d9a092e73 - --sha256: sha256-SPtbQ1+zN4uYiVx3rej2kMasbpfhSAyjRD5bdrnK3Rs= + tag: cb5b3c1ab46b562810d058b51fbce0ee407ce3ae + --sha256: sha256-42StkJb5M3tEwNtJ9Y4CL7FCLnsz7AmNyYKMAF2amDY= subdir: . source-repository-package type: git location: https://github.com/IntersectMBO/ouroboros-network.git - tag: 8b4dcb898b64615f574a7fbdcf119f77c7fbc598 - --sha256: sha256-t1tGV5uZwyKnSSCbaJJAteMYKnQYeO39hUUycMRuC2M= + tag: b47dbc2c29108e593cc47524cffb75008b88fe90 + --sha256: sha256-7TwcI1/m6aEBmiGYPXKHbIWlxzRja4qFyqr+apcR9p4= subdir: ./cardano-diffusion ./monoidal-synchronisation diff --git a/cardano-submit-api/cardano-submit-api.cabal b/cardano-submit-api/cardano-submit-api.cabal index 9d8f950df0d..7a1be8a0ac1 100644 --- a/cardano-submit-api/cardano-submit-api.cabal +++ b/cardano-submit-api/cardano-submit-api.cabal @@ -48,7 +48,7 @@ library , http-media , mtl , network - , optparse-applicative + , optparse-applicative-fork , ouroboros-consensus:cardano , ouroboros-network:{protocols} , prometheus >= 2.2.4 @@ -84,7 +84,7 @@ executable cardano-submit-api hs-source-dirs: app ghc-options: -threaded -rtsopts "-with-rtsopts=-T -I0" build-depends: base - , optparse-applicative + , optparse-applicative-fork , cardano-cli , cardano-crypto-class , cardano-git-rev ^>=0.2.2 diff --git a/cardano-testnet/cardano-testnet.cabal b/cardano-testnet/cardano-testnet.cabal index 0f4d1f2976e..9eb689f5972 100644 --- a/cardano-testnet/cardano-testnet.cabal +++ b/cardano-testnet/cardano-testnet.cabal @@ -82,7 +82,7 @@ library , mono-traversable , mtl , network - , optparse-applicative + , optparse-applicative-fork , parsec , ouroboros-network:{api, framework, ouroboros-network} ^>= 1.1 , cardano-diffusion:{api, cardano-diffusion} ^>= 1.0 @@ -156,7 +156,7 @@ executable cardano-testnet build-depends: cardano-crypto-class ^>=2.5 , cardano-cli , cardano-testnet - , optparse-applicative + , optparse-applicative-fork ghc-options: -threaded -rtsopts "-with-rtsopts=-N -T" diff --git a/flake.lock b/flake.lock index 412d023b1fc..95f4374f6f2 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "CHaP": { "flake": false, "locked": { - "lastModified": 1781102605, - "narHash": "sha256-lipiKNOJ/NIGO/pcwQT030+sSgbTkP3N0w/ck5jBJlw=", + "lastModified": 1782769557, + "narHash": "sha256-tCYMTwH15mNajHybvC3KGVPBPMlLlqIOYE0ZgcqRkJU=", "owner": "intersectmbo", "repo": "cardano-haskell-packages", - "rev": "0aa7afd943dbcf2dc51fe652c982da281f8cb621", + "rev": "3ee6b1ecce230d62f0083590f38c4ae3457da226", "type": "github" }, "original": { @@ -306,11 +306,11 @@ "hackageNix_2": { "flake": false, "locked": { - "lastModified": 1781588317, - "narHash": "sha256-bLPzRuyhb9kLvcZd3jmLqEjxYXoF7kRSY36k1bpXSpQ=", + "lastModified": 1782826502, + "narHash": "sha256-G6bt7DeWkDXJWI/fHME467V/SOaUdfG6hk7bTRuWyKg=", "owner": "input-output-hk", "repo": "hackage.nix", - "rev": "0304ab7fcb57c73b725bfdc0ebecea4476700366", + "rev": "6a87e2657c145eb81d5f0f53702d4f26c08f9cbf", "type": "github" }, "original": {