diff --git a/.github/workflows/cre-system-tests.yaml b/.github/workflows/cre-system-tests.yaml index 2ba0a2a6df9..dcd3d36a053 100644 --- a/.github/workflows/cre-system-tests.yaml +++ b/.github/workflows/cre-system-tests.yaml @@ -122,6 +122,9 @@ jobs: ], "Test_CRE_V2_HTTP_Action_Multi_Gateway": [ {"topology":"workflow-gateway-capabilities-multi-gateway","configs":"configs/workflow-gateway-capabilities-multi-gateway-don.toml"} + ], + "Test_CRE_V2_ConfidentialWorkflows_Relay": [ + {"topology":"workflow-gateway-capabilities-confidential-workflows","configs":"configs/workflow-gateway-capabilities-don-confidential-workflows.toml"} ] }' @@ -326,6 +329,49 @@ jobs: done exit 1 + # The confidential compute relay leg needs a chainlink-confidential-compute + # checkout: the CRE nodes run its confidential-http capability binary, and + # the test drives its enclave harness (tests/testhelpers) to start local + # enclaves. These runners are not Nitro-capable, so the harness falls back + # to fake enclaves (local processes over loopback vsock emulation). + # The confidential-workflows *capability* binary ships in the node image via + # plugins/plugins.public.yaml. The *enclave* side is not a plugin: the test + # drives chainlink-confidential-compute's enclave harness, which shells out + # to that repo's fake-enclave runner. Check it out at the same revision the + # plugin is built from so the capability and the enclave app always match. + - name: Resolve confidential compute revision + if: ${{ contains(matrix.tests.test_name, 'ConfidentialWorkflows') }} + id: cc-rev + shell: bash + run: | + set -euo pipefail + ref=$(yq -r '.plugins."confidential-workflows"[0].gitRef' plugins/plugins.public.yaml) + if [ -z "$ref" ] || [ "$ref" = "null" ]; then + echo "could not resolve confidential-workflows gitRef from plugins/plugins.public.yaml" >&2 + exit 1 + fi + echo "Resolved confidential compute revision: $ref" + echo "ref=$ref" >> "${GITHUB_OUTPUT}" + + - name: Checkout chainlink-confidential-compute + if: ${{ contains(matrix.tests.test_name, 'ConfidentialWorkflows') }} + uses: actions/checkout@v7 + with: + repository: smartcontractkit/chainlink-confidential-compute + ref: ${{ steps.cc-rev.outputs.ref }} + path: chainlink-confidential-compute + persist-credentials: false + + # Prebuild the enclave app and host binaries the fake-enclave runner + # executes, so the cost lands here rather than in the test's own timeout. + # confidential-workflows is its own Go module, so this has to build from + # inside it rather than from the repository root. + - name: Prebuild fake enclave binaries + if: ${{ contains(matrix.tests.test_name, 'ConfidentialWorkflows') }} + shell: bash + working-directory: chainlink-confidential-compute/enclave/apps/confidential-workflows + run: go build ./environments/fake/... + - name: Start local CRE${{ matrix.tests.cre_version }} id: start-local-cre uses: ./.github/actions/start-local-cre-environment @@ -337,6 +383,9 @@ jobs: chip-router-image: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1" ctf-configs: ${{ matrix.tests.configs }} + # The confidential workflows test starts enclaves on the harness's + # default ports; the gateway needs them allowlisted to reach them. + env-start-extra-args: ${{ contains(matrix.tests.test_name, 'ConfidentialWorkflows') && '-e 8080,8081,8082,8083' || '' }} retry-count: "3" retry-delay-seconds: "15" cleanup-on-error: "false" @@ -362,6 +411,7 @@ jobs: TEST_TIMEOUT: 7m # let's leave 3 minutes for other steps (the whole job times out after 10 minutes) RUN_QUARANTINED_TESTS: "true" # always run quarantined tests in CI TOPOLOGY_NAME: ${{ matrix.tests.topology }} + CONFIDENTIAL_COMPUTE_ROOT: ${{ github.workspace }}/chainlink-confidential-compute GITHUB_TOKEN: ${{ steps.github-token.outputs.access-token || '' }} # to avoid rate limiting when downloading protobuf files from GitHub PARALLEL_COUNT: "10" CRE_TEST_PARALLEL_ENABLED: "true" diff --git a/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don-confidential-workflows.toml b/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don-confidential-workflows.toml new file mode 100644 index 00000000000..fc904e526de --- /dev/null +++ b/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don-confidential-workflows.toml @@ -0,0 +1,154 @@ +# Topology for the confidential workflows engine E2E test. +# +# Differences from workflow-gateway-capabilities-don.toml: +# - the workflow DON also hosts "confidential-workflows" (the capability that +# routes execution into the enclaves) and "confidential-relay" (the gateway +# handler the enclaves call back through) +# - PerWorkflow.ConfidentialWorkflows is enabled so the engine will honour a +# workflow registered with {"confidential":true} attributes +# - the workflow DON exposes remote capabilities and enables the DKG recipient +# so the pre-enclave secret fetch can reach the vault DON +# +# The confidential-workflows capability binary is not part of this repository; it +# is built from a chainlink-confidential-compute checkout and mounted at +# ./binaries/confidential-workflows (see .github/workflows/cre-system-tests.yaml). + +# Fake enclaves emit a sentinel attestation document rather than real PCRs, so +# attestation validation is relaxed. INSECURE; for tests only. +[capability_configs.confidential-relay] + [capability_configs.confidential-relay.values] + trustEnclaves = true + requireBFTQuorum = true + +# The "enclaves" value is a JSON array of enclaves the capability routes to. +# Whoever starts the enclaves supplies it, since their addresses are only known +# once they are running; absent it, the capability registers an empty list. +[capability_configs.confidential-workflows] + binary_name = "confidential-workflows" + [capability_configs.confidential-workflows.values] + version = "1.0.0-alpha" + +[chip_router] + image = "local-cre-chip-router:v1.0.1" + +[[blockchains]] + type = "anvil" + chain_id = "1337" + container_name = "anvil-1337" + docker_cmd_params = ["-b", "0.5", "--mixed-mining"] + +[[blockchains]] + type = "anvil" + chain_id = "2337" + container_name = "anvil-2337" + port = "8546" + docker_cmd_params = ["-b", "0.5", "--mixed-mining"] + +[jd] + csa_encryption_key = "d1093c0060d50a3c89c189b2e485da5a3ce57f3dcb38ab7e2c0d5f0bb2314a44" # any random 32 byte hex string + image = "job-distributor:0.28.0" + +[fake] + port = 8171 + +[fake_http] + port = 8666 + +[infra] + # either "docker" or "kubernetes" + type = "docker" + +[[nodesets]] + nodes = 4 + name = "workflow" + don_family = "test-don-family" + don_types = ["workflow"] + override_mode = "all" + http_port_range_start = 10100 + + supported_evm_chains = [1337, 2337] + + env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"VaultOrgIdAsSecretOwnerEnabled":false}}', CL_CRE_SETTINGS_DEFAULT = '{"RemoteExecutableWorkflowDONBindingEnabled":"true","PerWorkflow":{"ConfidentialWorkflows":{"Enabled":"true"}}}' } + capabilities = ["consensus", "confidential-workflows", "confidential-relay", "cron", "http-action", "http-trigger", "don-time", "evm-1337"] + exposes_remote_capabilities = true + registry_based_launch_allowlist = ["cron-trigger@1.0.0"] + + [nodesets.db] + image = "postgres:12.0" + port = 13000 + + [[nodesets.node_specs]] + roles = ["plugin"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + user_config_overrides = """ + [P2P] + EnableExperimentalRageP2P = true + + [CRE] + EnableDKGRecipient = true + + # The test copies the workflow binary and config into the containers, so the + # syncer reads them from disk rather than fetching them remotely. + [CRE.WorkflowFetcher] + URL = "file:///home/chainlink/workflows" + """ + +[[nodesets]] + nodes = 4 + name = "capabilities" + don_family = "test-don-family" + don_types = ["capabilities"] + exposes_remote_capabilities = true + override_mode = "all" + http_port_range_start = 10200 + + supported_evm_chains = [1337, 2337] + + env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"VaultOrgIdAsSecretOwnerEnabled":false}}', CL_CRE_SETTINGS_DEFAULT = '{"RemoteExecutableWorkflowDONBindingEnabled":"true"}' } + capabilities = ["vault", "evm-2337"] + + [nodesets.db] + image = "postgres:12.0" + port = 13100 + + [[nodesets.node_specs]] + roles = ["plugin"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + user_config_overrides = """ + [P2P] + EnableExperimentalRageP2P = true + + [CRE] + EnableDKGRecipient = true + """ + +[[nodesets]] + nodes = 1 + name = "bootstrap-gateway" + don_family = "test-don-family" + don_types = ["bootstrap", "gateway"] + override_mode = "each" + http_port_range_start = 10300 + + env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}' } + supported_evm_chains = [1337, 2337] + + [nodesets.db] + image = "postgres:12.0" + port = 13200 + + [[nodesets.node_specs]] + roles = ["bootstrap", "gateway"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + # 5002 is the web API capabilities port for incoming requests + # 15002 is the vault port for incoming requests + custom_ports = ["5002:5002","15002:15002"] + user_config_overrides = "" diff --git a/core/scripts/cre/environment/environment/environment.go b/core/scripts/cre/environment/environment/environment.go index 810ec3a9329..5b36b7a3938 100644 --- a/core/scripts/cre/environment/environment/environment.go +++ b/core/scripts/cre/environment/environment/environment.go @@ -363,18 +363,7 @@ func startCmd() *cobra.Command { } features := feature_set.New() - extraAllowedPorts := append([]int(nil), extraAllowedGatewayPorts...) - if in.Fake != nil { - extraAllowedPorts = append(extraAllowedPorts, in.Fake.Port) - } - if in.FakeHTTP != nil { - extraAllowedPorts = append(extraAllowedPorts, in.FakeHTTP.Port) - } - - gatewayWhitelistConfig := gateway.WhitelistConfig{ - ExtraAllowedPorts: extraAllowedPorts, - ExtraAllowedIPsCIDR: []string{"0.0.0.0/0"}, - } + gatewayWhitelistConfig := defaultGatewayWhitelistConfig(in, extraAllowedGatewayPorts) output, startErr := StartCLIEnvironment(cmdContext, relativePathToRepoRoot, in, nil, features, nil, envDependencies, gatewayWhitelistConfig) if startErr != nil { fmt.Fprintf(os.Stderr, "Error: %s\n", startErr) @@ -855,6 +844,23 @@ func statusCmd() *cobra.Command { return cmd } +// defaultGatewayWhitelistConfig builds the Gateway Connector's outbound allowlist: +// the caller's extra ports plus the fake service ports the config declares. +func defaultGatewayWhitelistConfig(in *envconfig.Config, extraAllowedPorts []int) gateway.WhitelistConfig { + ports := append([]int(nil), extraAllowedPorts...) + if in.Fake != nil { + ports = append(ports, in.Fake.Port) + } + if in.FakeHTTP != nil { + ports = append(ports, in.FakeHTTP.Port) + } + + return gateway.WhitelistConfig{ + ExtraAllowedPorts: ports, + ExtraAllowedIPsCIDR: []string{"0.0.0.0/0"}, + } +} + func StartCLIEnvironment( cmdContext context.Context, relativePathToRepoRoot string, diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 4eab4cdba38..dde35f35ff2 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -487,6 +487,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb // indirect github.com/smartcontractkit/chainlink-ccv v0.1.1-0.20260716164331-d938b371c5d6 // indirect github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 // indirect + github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 // indirect github.com/smartcontractkit/chainlink-data-streams v1.1.0 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index 59fbc9d487c..8888312092f 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1584,6 +1584,8 @@ github.com/smartcontractkit/chainlink-common/keystore v1.3.0 h1:V05Rp9/dTc4Wyips github.com/smartcontractkit/chainlink-common/keystore v1.3.0/go.mod h1:vHV8BGm6TN7jBbMsWxq1Hqm3HbCtYFwzvKS0CCczxG8= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 h1:uWEwl7i2ryuRVoV4DmIKm6mqYevf1lH/8cQYhw/JXko= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72/go.mod h1:UYcRMb4dZcoaIPgZJ3hckCySTqtJc9K4Q+tOKErwTq0= +github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61E4EYRzQDtRwqunDBlKpsZc1SvFEUlSLE= +github.com/smartcontractkit/chainlink-confidential-compute v1.3.0/go.mod h1:fq9n85XoREIUxgdXIX3RyDIRCRyxZKZ5NCBbFDfdySo= github.com/smartcontractkit/chainlink-data-streams v1.1.0 h1:O5ngSwpwey7kQ3t4YgFiXzu3L3EvYiPS4c93gMWNRbk= github.com/smartcontractkit/chainlink-data-streams v1.1.0/go.mod h1:dF5JiHWueHjYguUUUrFeb03MkcDqha/tssEkqTkgzp4= github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 h1:mzbvXxdbE/96Pdj1zyPKzf25ZlDR48+iTTDTbaITvmk= diff --git a/go.md b/go.md index 78d4792fe11..0e6294d64a0 100644 --- a/go.md +++ b/go.md @@ -346,6 +346,10 @@ flowchart LR click chainlink-common/pkg/values href "https://github.com/smartcontractkit/chainlink-common" chainlink-common/pkg/workflows/sdk/v2/pb --> chainlink-common/pkg/values click chainlink-common/pkg/workflows/sdk/v2/pb href "https://github.com/smartcontractkit/chainlink-common" + chainlink-confidential-compute --> tdh2/go/tdh2 + click chainlink-confidential-compute href "https://github.com/smartcontractkit/chainlink-confidential-compute" + chainlink-confidential-compute/tests/testhelpers --> chainlink-confidential-compute + click chainlink-confidential-compute/tests/testhelpers href "https://github.com/smartcontractkit/chainlink-confidential-compute" chainlink-data-streams --> chainlink-common/keystore chainlink-data-streams --> chainlink-evm/gethwrappers click chainlink-data-streams href "https://github.com/smartcontractkit/chainlink-data-streams" @@ -503,10 +507,12 @@ flowchart LR chainlink/load-tests --> chainlink-testing-framework/havoc chainlink/load-tests --> chainlink/integration-tests click chainlink/load-tests href "https://github.com/smartcontractkit/chainlink" + chainlink/system-tests/lib --> chainlink-confidential-compute chainlink/system-tests/lib --> chainlink-testing-framework/framework/components/chiprouter chainlink/system-tests/lib --> chainlink-testing-framework/framework/components/dockercompose chainlink/system-tests/lib --> chainlink-testing-framework/framework/components/fake click chainlink/system-tests/lib href "https://github.com/smartcontractkit/chainlink" + chainlink/system-tests/tests --> chainlink-confidential-compute/tests/testhelpers chainlink/system-tests/tests --> chainlink/core/scripts/cre/environment/examples/workflows/cron chainlink/system-tests/tests --> chainlink/core/scripts/cre/environment/examples/workflows/proof-of-reserve/cron-based chainlink/system-tests/tests --> chainlink/system-tests/lib @@ -710,6 +716,12 @@ flowchart LR end click chainlink-common-repo href "https://github.com/smartcontractkit/chainlink-common" + subgraph chainlink-confidential-compute-repo[chainlink-confidential-compute] + chainlink-confidential-compute + chainlink-confidential-compute/tests/testhelpers + end + click chainlink-confidential-compute-repo href "https://github.com/smartcontractkit/chainlink-confidential-compute" + subgraph chainlink-evm-repo[chainlink-evm] chainlink-evm chainlink-evm/contracts/cre/gobindings @@ -806,5 +818,5 @@ flowchart LR click testrig-repo href "https://github.com/smartcontractkit/testrig" classDef outline stroke-dasharray:6,fill:none; - class chainlink-repo,chainlink-aptos-repo,chainlink-ccip-repo,chainlink-ccv-repo,chainlink-common-repo,chainlink-evm-repo,chainlink-framework-repo,chainlink-protos-repo,chainlink-solana-repo,chainlink-stellar-repo,chainlink-sui-repo,chainlink-testing-framework-repo,chainlink-ton-repo,cre-sdk-go-repo,testrig-repo outline + class chainlink-repo,chainlink-aptos-repo,chainlink-ccip-repo,chainlink-ccv-repo,chainlink-common-repo,chainlink-confidential-compute-repo,chainlink-evm-repo,chainlink-framework-repo,chainlink-protos-repo,chainlink-solana-repo,chainlink-stellar-repo,chainlink-sui-repo,chainlink-testing-framework-repo,chainlink-ton-repo,cre-sdk-go-repo,testrig-repo outline ``` diff --git a/system-tests/lib/cre/capabilities/confidentialcompute/confidentialcompute.go b/system-tests/lib/cre/capabilities/confidentialcompute/confidentialcompute.go new file mode 100644 index 00000000000..27c7c1aa830 --- /dev/null +++ b/system-tests/lib/cre/capabilities/confidentialcompute/confidentialcompute.go @@ -0,0 +1,225 @@ +// Package confidentialcompute holds the pieces shared by confidential compute +// capabilities (e.g. confidential-workflows, confidential-http): reading the +// enclave list from configuration, sealing the capability's API key to each +// node, and building the on-chain registry entry that publishes the enclaves. +// +// The confidential relay handler reads that registry entry to discover which +// enclaves it may route to, so the list must be supplied before the environment +// starts. Features under cre/features consume these. +package confidentialcompute + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/pkg/errors" + "golang.org/x/crypto/nacl/box" + "google.golang.org/protobuf/proto" + + capabilitiespb "github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb" + cctypes "github.com/smartcontractkit/chainlink-confidential-compute/types" + kcr "github.com/smartcontractkit/chainlink-evm/gethwrappers/keystone/generated/capabilities_registry_1_1_0" + "github.com/smartcontractkit/chainlink-protos/cre/go/values" + keystone_changeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre" +) + +// apiKey is the API key the capability presents to the enclaves. Enclaves in +// tests are started with a matching key; a real key is not needed, but using a +// non-empty one keeps the encrypt/decrypt path exercised. +const apiKey = "foobar" + +// workflowEncryptionKey reads a node's workflow public encryption key, which is +// used to seal the capability's API key so it is not stored in plaintext. +func workflowEncryptionKey(workerNode *cre.Node) ([32]byte, error) { + var publicKey [32]byte + + apiClient := workerNode.Clients.RestClient.APIClient + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, apiClient.BaseURL+"/v2/keys/workflow", nil) + if err != nil { + return publicKey, errors.Wrap(err, "failed to create request to get workflow keys") + } + if len(apiClient.Cookies) == 0 { + return publicKey, errors.New("no session cookie available for get workflow keys request") + } + req.AddCookie(apiClient.Cookies[0]) + + resp, err := apiClient.GetClient().Do(req) + if err != nil { + return publicKey, errors.Wrap(err, "failed to send request to get workflow keys") + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return publicKey, fmt.Errorf("expected 200 OK from get workflow keys request, got %d", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return publicKey, errors.Wrap(err, "failed to read response body from get workflow keys request") + } + + var workflowKeysResp struct { + Data []struct { + Attributes struct { + PublicKey string `json:"publicKey"` + } `json:"attributes"` + } `json:"data"` + } + if err = json.Unmarshal(body, &workflowKeysResp); err != nil { + return publicKey, errors.Wrap(err, "failed to unmarshal workflow keys response") + } + if len(workflowKeysResp.Data) == 0 { + return publicKey, errors.New("no workflow keys found in response") + } + + publicKeyBytes, err := hex.DecodeString(workflowKeysResp.Data[0].Attributes.PublicKey) + if err != nil { + return publicKey, errors.Wrap(err, "failed to decode public key hex") + } + if len(publicKeyBytes) != len(publicKey) { + return publicKey, fmt.Errorf("expected public key to be %d bytes, got %d", len(publicKey), len(publicKeyBytes)) + } + copy(publicKey[:], publicKeyBytes) + + return publicKey, nil +} + +// EnclavesConfigKey is the capability config value holding a JSON array of +// enclaves, letting a topology declare them instead of passing Go values. +const EnclavesConfigKey = "enclaves" + +// EncryptedAPIKeys seals the capability's API key to each worker node's workflow +// public key, so it is never stored in plaintext. The capability is configured +// with every node's sealed copy; each node decrypts only its own. +func EncryptedAPIKeys(workerNodes []*cre.Node) ([]string, error) { + encrypted := make([]string, 0, len(workerNodes)) + for _, workerNode := range workerNodes { + publicKey, kErr := workflowEncryptionKey(workerNode) + if kErr != nil { + return nil, kErr + } + + ctxt, sErr := box.SealAnonymous(nil, []byte(apiKey), &publicKey, rand.Reader) + if sErr != nil { + return nil, errors.Wrap(sErr, "failed to seal API key") + } + encrypted = append(encrypted, hex.EncodeToString(ctxt)) + } + + return encrypted, nil +} + +// JobConfigJSON builds the capability job's config. Liveness detection is kept +// aggressive so failover traffic starts only once each node has had a chance to +// observe a dead enclave. +func JobConfigJSON(encryptedAPIKeys []string) (string, error) { + config := map[string]any{ + "InsecureSkipTLSVerify": true, + "EncryptedAPIKeys": strings.Join(encryptedAPIKeys, ","), + "EnableCache": true, + "EnableProactiveRefresh": true, + "MaxRetries": 3, + "RetryBackoffSeconds": 5, + } + + configBytes, err := json.Marshal(config) + if err != nil { + return "", errors.Wrap(err, "failed to marshal capability config") + } + + return string(configBytes), nil +} + +// RegistryCapabilityConfig builds the on-chain registry entry that publishes the +// enclave list, which is how the confidential relay handler discovers where it +// may route requests. +func RegistryCapabilityConfig(name, version string, enclaves []cctypes.Enclave) (keystone_changeset.DONCapabilityWithConfig, error) { + wrappedConfig, err := values.WrapMap(cctypes.EnclavesList{Enclaves: enclaves}) + if err != nil { + return keystone_changeset.DONCapabilityWithConfig{}, errors.Wrap(err, "failed to wrap enclave list config") + } + + return keystone_changeset.DONCapabilityWithConfig{ + Capability: kcr.CapabilitiesRegistryCapability{ + LabelledName: name, + Version: version, + CapabilityType: 1, // ACTION + }, + Config: &capabilitiespb.CapabilityConfig{ + DefaultConfig: values.Proto(wrappedConfig).GetMapValue(), + LocalOnly: true, + }, + }, nil +} + +// MarshalRegistryConfig encodes an enclave list as the capability's on-chain +// registry config, for publishing enclaves to a DON that is already running. +// +// A fresh CapabilityConfig is built rather than decoding and re-encoding what is +// already on-chain, so config left by a broken earlier run cannot corrupt this one. +func MarshalRegistryConfig(enclaves []cctypes.Enclave) ([]byte, error) { + wrappedConfig, err := values.WrapMap(cctypes.EnclavesList{Enclaves: enclaves}) + if err != nil { + return nil, errors.Wrap(err, "failed to wrap enclave list config") + } + + // LocalOnly must match what RegistryCapabilityConfig writes at startup, or + // updating the enclave list would silently change the capability's scope. + encoded, err := proto.Marshal(&capabilitiespb.CapabilityConfig{ + DefaultConfig: values.Proto(wrappedConfig).GetMapValue(), + LocalOnly: true, + }) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal capability config") + } + + return encoded, nil +} + +// MarshalEnclaves encodes an enclave list for EnclavesConfigKey, for callers +// that discover their enclaves at runtime and hand them to the environment as +// configuration. +func MarshalEnclaves(enclaves []cctypes.Enclave) (string, error) { + encoded, err := json.Marshal(enclaves) + if err != nil { + return "", errors.Wrap(err, "failed to marshal enclave list") + } + + return string(encoded), nil +} + +// EnclavesFromConfig reads the enclave list a topology declared for the named +// capability. Returns nil when the capability has no config or no enclaves key. +func EnclavesFromConfig(nodeSet *cre.NodeSet, name string) ([]cctypes.Enclave, error) { + if nodeSet == nil { + return nil, nil + } + + capConfig, ok := nodeSet.CapabilityConfigs[name] + if !ok || capConfig.Values == nil { + return nil, nil + } + + raw, ok := capConfig.Values[EnclavesConfigKey] + if !ok { + return nil, nil + } + + encoded, ok := raw.(string) + if !ok { + return nil, errors.Errorf("capability %q: %q must be a JSON string, got %T", name, EnclavesConfigKey, raw) + } + + var enclaves []cctypes.Enclave + if err := json.Unmarshal([]byte(encoded), &enclaves); err != nil { + return nil, errors.Wrapf(err, "capability %q: failed to parse %q", name, EnclavesConfigKey) + } + + return enclaves, nil +} diff --git a/system-tests/lib/cre/capabilities/confidentialcompute/confidentialcompute_test.go b/system-tests/lib/cre/capabilities/confidentialcompute/confidentialcompute_test.go new file mode 100644 index 00000000000..6101f62b915 --- /dev/null +++ b/system-tests/lib/cre/capabilities/confidentialcompute/confidentialcompute_test.go @@ -0,0 +1,94 @@ +package confidentialcompute + +import ( + "testing" + + "github.com/stretchr/testify/require" + + cctypes "github.com/smartcontractkit/chainlink-confidential-compute/types" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre" +) + +func nodeSetWithValues(name string, values map[string]any) *cre.NodeSet { + return &cre.NodeSet{ + CapabilityConfigs: map[cre.CapabilityFlag]cre.CapabilityConfig{ + name: {Values: values}, + }, + } +} + +func TestEnclavesFromConfig(t *testing.T) { + t.Parallel() + + const name = "confidential-workflows" + + t.Run("parses declared enclaves", func(t *testing.T) { + t.Parallel() + + ns := nodeSetWithValues(name, map[string]any{ + EnclavesConfigKey: `[{"enclaveURL":"http://10.0.0.1:8080","enclaveAuthHeader":"key-a"},` + + `{"enclaveURL":"http://10.0.0.1:8081"}]`, + }) + + got, err := EnclavesFromConfig(ns, name) + require.NoError(t, err) + require.Len(t, got, 2) + require.Equal(t, "http://10.0.0.1:8080", got[0].EnclaveURL) + require.Equal(t, "key-a", got[0].EnclaveAuthHeader) + require.Equal(t, "http://10.0.0.1:8081", got[1].EnclaveURL) + }) + + t.Run("round-trips a marshalled enclave list", func(t *testing.T) { + t.Parallel() + + encoded, err := MarshalEnclaves([]cctypes.Enclave{{ + EnclaveURL: "http://10.0.0.2:8080", + TrustedValues: [][]byte{[]byte("fake-measurements")}, + Region: "us-west-2", + }}) + require.NoError(t, err) + + got, err := EnclavesFromConfig(nodeSetWithValues(name, map[string]any{EnclavesConfigKey: encoded}), name) + require.NoError(t, err) + require.Len(t, got, 1) + require.Equal(t, "http://10.0.0.2:8080", got[0].EnclaveURL) + require.Equal(t, "us-west-2", got[0].Region) + require.Equal(t, [][]byte{[]byte("fake-measurements")}, got[0].TrustedValues) + }) + + t.Run("returns nil when unconfigured", func(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + desc string + nodeSet *cre.NodeSet + }{ + {"nil node set", nil}, + {"no values", nodeSetWithValues(name, nil)}, + {"no enclaves key", nodeSetWithValues(name, map[string]any{"other": "x"})}, + {"different capability", nodeSetWithValues("confidential-http", map[string]any{EnclavesConfigKey: "[]"})}, + } { + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + got, err := EnclavesFromConfig(tc.nodeSet, name) + require.NoError(t, err) + require.Nil(t, got) + }) + } + }) + + t.Run("errors on a non-string value", func(t *testing.T) { + t.Parallel() + + _, err := EnclavesFromConfig(nodeSetWithValues(name, map[string]any{EnclavesConfigKey: 42}), name) + require.ErrorContains(t, err, "must be a JSON string") + }) + + t.Run("errors on malformed JSON", func(t *testing.T) { + t.Parallel() + + _, err := EnclavesFromConfig(nodeSetWithValues(name, map[string]any{EnclavesConfigKey: "{not json"}), name) + require.ErrorContains(t, err, "failed to parse") + }) +} diff --git a/system-tests/lib/cre/features/confidentialrelay/confidentialrelay.go b/system-tests/lib/cre/features/confidentialrelay/confidentialrelay.go index 8d50f800552..8783ce6fb6b 100644 --- a/system-tests/lib/cre/features/confidentialrelay/confidentialrelay.go +++ b/system-tests/lib/cre/features/confidentialrelay/confidentialrelay.go @@ -29,6 +29,19 @@ func (o *ConfidentialRelay) Flag() cre.CapabilityFlag { return flag } +// boolFromValues reads a bool from a capability config's values, falling back to +// def when the key is absent or not a bool. Lets a topology set the relay's knobs +// in TOML rather than requiring a Go-constructed feature. +func boolFromValues(values map[string]any, key string, def bool) bool { + if v, ok := values[key]; ok { + if b, isBool := v.(bool); isBool { + return b + } + } + + return def +} + func (o *ConfidentialRelay) PreEnvStartup( ctx context.Context, testLogger zerolog.Logger, @@ -66,8 +79,8 @@ func (o *ConfidentialRelay) PreEnvStartup( } enabled := true - trustEnclaves := o.TrustEnclaves - requireBFTQuorum := o.RequireBFTQuorum + trustEnclaves := boolFromValues(capConfig.Values, "trustEnclaves", o.TrustEnclaves) + requireBFTQuorum := boolFromValues(capConfig.Values, "requireBFTQuorum", o.RequireBFTQuorum) typedConfig.CRE.ConfidentialRelay = &coretoml.ConfidentialRelayConfig{ Enabled: &enabled, TrustEnclaves: &trustEnclaves, diff --git a/system-tests/lib/cre/features/confidentialworkflows/confidentialworkflows.go b/system-tests/lib/cre/features/confidentialworkflows/confidentialworkflows.go new file mode 100644 index 00000000000..ea14cf2a0ea --- /dev/null +++ b/system-tests/lib/cre/features/confidentialworkflows/confidentialworkflows.go @@ -0,0 +1,175 @@ +// Package confidentialworkflows installs the confidential workflows capability +// as a standard CRE feature: it publishes the enclave list to the on-chain +// registry and proposes the capability job on each worker node. +// +// The enclave list comes from the DON's capability config rather than Go values, +// so a topology can declare it and `cre env start` can stand the capability up +// without a test driving the environment in-process. +package confidentialworkflows + +import ( + "context" + "fmt" + + "dario.cat/mergo" + "github.com/google/uuid" + "github.com/pkg/errors" + "github.com/rs/zerolog" + + cre_jobs "github.com/smartcontractkit/chainlink/deployment/cre/jobs" + cre_jobs_ops "github.com/smartcontractkit/chainlink/deployment/cre/jobs/operations" + job_types "github.com/smartcontractkit/chainlink/deployment/cre/jobs/types" + "github.com/smartcontractkit/chainlink/deployment/cre/pkg/offchain" + keystone_changeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre/capabilities/confidentialcompute" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/jobs" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/jobs/standardcapability" +) + +const flag = cre.ConfidentialWorkflowsCapability + +// VersionConfigKey is the capability config value holding the version the +// capability registers under. +const VersionConfigKey = "version" + +// defaultVersion is used when the topology does not declare one. +const defaultVersion = "1.0.0" + +type ConfidentialWorkflows struct{} + +func (o *ConfidentialWorkflows) Flag() cre.CapabilityFlag { + return flag +} + +// PreEnvStartup publishes the enclave list the topology declared, which the +// confidential relay handler reads to discover where it may route requests. +func (o *ConfidentialWorkflows) PreEnvStartup( + _ context.Context, + _ zerolog.Logger, + don *cre.DonMetadata, + _ *cre.Topology, + _ *cre.Environment, +) (*cre.PreEnvStartupOutput, error) { + if !don.HasFlag(flag) { + return &cre.PreEnvStartupOutput{}, nil + } + + nodeSet := don.MustNodeSet() + enclaves, eErr := confidentialcompute.EnclavesFromConfig(nodeSet, flag) + if eErr != nil { + return nil, eErr + } + + capabilityConfig, cErr := confidentialcompute.RegistryCapabilityConfig(flag, version(don), enclaves) + if cErr != nil { + return nil, cErr + } + + return &cre.PreEnvStartupOutput{ + DONCapabilityWithConfig: []keystone_changeset.DONCapabilityWithConfig{capabilityConfig}, + }, nil +} + +// PostEnvStartup proposes the capability job on each worker node, sealing the +// capability's API key to every node's workflow key. +func (o *ConfidentialWorkflows) PostEnvStartup( + ctx context.Context, + _ zerolog.Logger, + don *cre.Don, + dons *cre.Dons, + creEnv *cre.Environment, +) error { + if !don.HasFlag(flag) { + return nil + } + + capabilityConfig, ok := don.GetCapabilityConfig(flag) + if !ok { + return fmt.Errorf("config for '%s' capability not found for %s DON", flag, don.GetName()) + } + + command, cErr := standardcapability.GetCommand(capabilityConfig.BinaryName) + if cErr != nil { + return errors.Wrapf(cErr, "failed to get command for %s capability", flag) + } + + workerNodes, wErr := don.Workers() + if wErr != nil { + return errors.Wrap(wErr, "failed to find worker nodes") + } + + encryptedAPIKeys, eErr := confidentialcompute.EncryptedAPIKeys(workerNodes) + if eErr != nil { + return eErr + } + + configJSON, jErr := confidentialcompute.JobConfigJSON(encryptedAPIKeys) + if jErr != nil { + return jErr + } + + input := cre_jobs.ProposeJobSpecInput{ + Domain: offchain.ProductLabel, + Environment: creEnv.CldfEnvironment.Name, + DONName: don.Name, + JobName: flag + "-worker", + ExtraLabels: map[string]string{cre.CapabilityLabelKey: flag}, + DONFilters: []offchain.TargetDONFilter{ + {Key: offchain.FilterKeyDONName, Value: don.Name}, + }, + Template: job_types.Cron, + Inputs: job_types.JobSpecInput{ + "command": command, + "config": configJSON, + }, + } + if creEnv.FreshExternalJobIDs { + input.Inputs["externalJobID"] = uuid.NewString() + } + + if err := (cre_jobs.ProposeJobSpec{}).VerifyPreconditions(*creEnv.CldfEnvironment, input); err != nil { + return errors.Wrapf(err, "precondition verification failed for %s worker job", flag) + } + + report, err := (cre_jobs.ProposeJobSpec{}).Apply(*creEnv.CldfEnvironment, input) + if err != nil { + return errors.Wrapf(err, "failed to propose %s worker job spec", flag) + } + + // Proposing only queues the spec in Job Distributor; without approval the + // nodes never run it and the capability never registers locally. + specs := make(map[string][]string) + for _, r := range report.Reports { + out, ok := r.Output.(cre_jobs_ops.ProposeStandardCapabilityJobOutput) + if !ok { + return fmt.Errorf("unable to cast to ProposeStandardCapabilityJobOutput, actual type: %T", r.Output) + } + if mErr := mergo.Merge(&specs, out.Specs, mergo.WithAppendSlice); mErr != nil { + return errors.Wrapf(mErr, "failed to merge %s worker job specs", flag) + } + } + + if aErr := jobs.Approve(ctx, creEnv.CldfEnvironment.Offchain, dons, specs); aErr != nil { + return errors.Wrapf(aErr, "failed to approve %s worker jobs", flag) + } + + return nil +} + +// version returns the version the capability registers under, from the DON's +// capability config when declared. +func version(don *cre.DonMetadata) string { + cfg, ok := don.CapabilityConfigs[flag] + if !ok || cfg.Values == nil { + return defaultVersion + } + + if v, found := cfg.Values[VersionConfigKey]; found { + if s, isString := v.(string); isString && s != "" { + return s + } + } + + return defaultVersion +} diff --git a/system-tests/lib/cre/features/sets/sets.go b/system-tests/lib/cre/features/sets/sets.go index d0037f4d16c..8db4cb51b28 100644 --- a/system-tests/lib/cre/features/sets/sets.go +++ b/system-tests/lib/cre/features/sets/sets.go @@ -3,6 +3,8 @@ package sets import ( "github.com/smartcontractkit/chainlink/system-tests/lib/cre" aptos_feature "github.com/smartcontractkit/chainlink/system-tests/lib/cre/features/aptos" + confidential_relay_feature "github.com/smartcontractkit/chainlink/system-tests/lib/cre/features/confidentialrelay" + confidential_workflows_feature "github.com/smartcontractkit/chainlink/system-tests/lib/cre/features/confidentialworkflows" consensus_feature "github.com/smartcontractkit/chainlink/system-tests/lib/cre/features/consensus/v2" cron_feature "github.com/smartcontractkit/chainlink/system-tests/lib/cre/features/cron" don_time_feature "github.com/smartcontractkit/chainlink/system-tests/lib/cre/features/don_time" @@ -26,5 +28,7 @@ func New() cre.Features { &solana_feature.Solana{}, &stellar_feature.Stellar{}, &vault_feature.Vault{}, + &confidential_relay_feature.ConfidentialRelay{}, + &confidential_workflows_feature.ConfidentialWorkflows{}, ) } diff --git a/system-tests/lib/cre/flags/provider.go b/system-tests/lib/cre/flags/provider.go index 54fc0839c47..e045aef025d 100644 --- a/system-tests/lib/cre/flags/provider.go +++ b/system-tests/lib/cre/flags/provider.go @@ -19,6 +19,8 @@ func NewDefaultCapabilityFlagsProvider() *DefaultCapbilityFlagsProvider { cre.EVMCapability, cre.AptosCapability, cre.StellarCapability, + cre.ConfidentialRelayCapability, + cre.ConfidentialWorkflowsCapability, }, } } @@ -40,6 +42,8 @@ func NewExtensibleCapabilityFlagsProvider(extraGlobalFlags []string) *Extensible cre.VaultCapability, cre.HTTPTriggerCapability, cre.HTTPActionCapability, + cre.ConfidentialRelayCapability, + cre.ConfidentialWorkflowsCapability, }, extraGlobalFlags...), chainSpecificCapabilities: []cre.CapabilityFlag{ cre.EVMCapability, diff --git a/system-tests/lib/cre/registry_update.go b/system-tests/lib/cre/registry_update.go new file mode 100644 index 00000000000..029e2c0a9a2 --- /dev/null +++ b/system-tests/lib/cre/registry_update.go @@ -0,0 +1,72 @@ +package cre + +import ( + "context" + "strings" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/pkg/errors" + + capabilities_registry_v2 "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/capabilities_registry_wrapper_v2" + "github.com/smartcontractkit/chainlink-testing-framework/seth" +) + +// UpdateDONCapabilityConfig rewrites one capability's config on a DON, leaving +// the DON's other capabilities, nodes, and settings as they are. +// +// This exists for configuration only knowable once the environment is running. +// Capabilities that poll the registry pick the new config up on their next +// refresh, so publishing it does not require restarting the environment. +// +// capabilityName is matched without a version, since the registry keys +// capabilities as "name@version". +func UpdateDONCapabilityConfig( + ctx context.Context, + sethClient *seth.Client, + capabilitiesRegistryAddr string, + donName string, + capabilityName string, + config []byte, +) error { + capReg, err := capabilities_registry_v2.NewCapabilitiesRegistry( + common.HexToAddress(capabilitiesRegistryAddr), sethClient.Client, + ) + if err != nil { + return errors.Wrap(err, "failed to create capabilities registry wrapper") + } + + don, err := capReg.GetDONByName(&bind.CallOpts{Context: ctx}, donName) + if err != nil { + return errors.Wrapf(err, "failed to fetch DON %q from capabilities registry", donName) + } + + // updateDON replaces the whole set, so carry every capability the DON has. + updated := make([]capabilities_registry_v2.CapabilitiesRegistryCapabilityConfiguration, len(don.CapabilityConfigurations)) + copy(updated, don.CapabilityConfigurations) + + var found bool + for i := range updated { + if strings.HasPrefix(updated[i].CapabilityId, capabilityName+"@") { + updated[i].Config = config + found = true + } + } + if !found { + return errors.Errorf("capability %q is not configured on DON %q", capabilityName, donName) + } + + _, err = sethClient.Decode(capReg.UpdateDON(sethClient.NewTXOpts(), don.Id, capabilities_registry_v2.CapabilitiesRegistryUpdateDONParams{ + Name: don.Name, + Config: don.Config, + CapabilityConfigurations: updated, + Nodes: don.NodeP2PIds, + F: don.F, + IsPublic: don.IsPublic, + })) + if err != nil { + return errors.Wrapf(err, "failed to submit updateDON for DON %q", donName) + } + + return nil +} diff --git a/system-tests/lib/cre/types.go b/system-tests/lib/cre/types.go index 2ce0687d527..436c41d741d 100644 --- a/system-tests/lib/cre/types.go +++ b/system-tests/lib/cre/types.go @@ -66,8 +66,10 @@ const ( HTTPActionCapability CapabilityFlag = "http-action" SolanaCapability CapabilityFlag = "solana" ConfidentialRelayCapability CapabilityFlag = "confidential-relay" - AptosCapability CapabilityFlag = "aptos" - StellarCapability CapabilityFlag = "stellar" + // ConfidentialWorkflowsCapability doubles as the enclave application name. + ConfidentialWorkflowsCapability CapabilityFlag = "confidential-workflows" + AptosCapability CapabilityFlag = "aptos" + StellarCapability CapabilityFlag = "stellar" // Add more capabilities as needed ) diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index 7c609269d0e..6c51f4e2a17 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -39,6 +39,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-common v0.11.2-0.20260811140401-3fb1738abb75 github.com/smartcontractkit/chainlink-common/keystore v1.3.0 + github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260810110946-8174b6bb7fc9 github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b @@ -60,6 +61,7 @@ require ( github.com/stretchr/testify v1.11.1 go.uber.org/ratelimit v0.3.1 go.uber.org/zap v1.28.0 + golang.org/x/crypto v0.54.0 golang.org/x/sync v0.22.0 google.golang.org/grpc v1.82.1 google.golang.org/protobuf v1.36.11 @@ -571,7 +573,6 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect go.yaml.in/yaml/v4 v4.0.0-rc.4 // indirect golang.org/x/arch v0.22.0 // indirect - golang.org/x/crypto v0.54.0 // indirect golang.org/x/exp v0.0.0-20260508232706-74f9aab9d74a // indirect golang.org/x/mod v0.37.0 // indirect golang.org/x/net v0.57.0 // indirect diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index be25be3ae0e..745418d7c27 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -1555,6 +1555,8 @@ github.com/smartcontractkit/chainlink-common/keystore v1.3.0 h1:V05Rp9/dTc4Wyips github.com/smartcontractkit/chainlink-common/keystore v1.3.0/go.mod h1:vHV8BGm6TN7jBbMsWxq1Hqm3HbCtYFwzvKS0CCczxG8= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 h1:uWEwl7i2ryuRVoV4DmIKm6mqYevf1lH/8cQYhw/JXko= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72/go.mod h1:UYcRMb4dZcoaIPgZJ3hckCySTqtJc9K4Q+tOKErwTq0= +github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61E4EYRzQDtRwqunDBlKpsZc1SvFEUlSLE= +github.com/smartcontractkit/chainlink-confidential-compute v1.3.0/go.mod h1:fq9n85XoREIUxgdXIX3RyDIRCRyxZKZ5NCBbFDfdySo= github.com/smartcontractkit/chainlink-data-streams v1.1.0 h1:O5ngSwpwey7kQ3t4YgFiXzu3L3EvYiPS4c93gMWNRbk= github.com/smartcontractkit/chainlink-data-streams v1.1.0/go.mod h1:dF5JiHWueHjYguUUUrFeb03MkcDqha/tssEkqTkgzp4= github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 h1:mzbvXxdbE/96Pdj1zyPKzf25ZlDR48+iTTDTbaITvmk= diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index f15b464adea..7c0da76bdc5 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -66,6 +66,8 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb github.com/smartcontractkit/chainlink-common v0.11.2-0.20260811140401-3fb1738abb75 github.com/smartcontractkit/chainlink-common/keystore v1.3.0 + github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 + github.com/smartcontractkit/chainlink-confidential-compute/tests/testhelpers v0.0.0-20260812145307-d77342c53d7d github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b @@ -94,7 +96,7 @@ require ( github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/solana/sollogtrigger v0.0.0-00010101000000-000000000000 github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/solana/solread v0.0.0-20260609191154-1ecc282df958 github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/solana/solwrite v0.0.0-00010101000000-000000000000 - github.com/smartcontractkit/chainlink/v2 v2.29.0 + github.com/smartcontractkit/chainlink/v2 v2.32.0 github.com/smartcontractkit/cld-changesets v0.5.0 github.com/stellar/go-stellar-sdk v0.6.0 github.com/stretchr/testify v1.11.1 @@ -179,6 +181,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/memberlist v0.5.4 // indirect github.com/hashicorp/serf v0.10.2 // indirect + github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9 // indirect github.com/huandu/xstrings v1.5.0 // indirect github.com/in-toto/attestation v1.2.0 // indirect github.com/influxdata/tdigest v0.0.2-0.20210216194612-fc98d27c9e8b // indirect @@ -324,7 +327,7 @@ require ( github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect github.com/alitto/pond/v2 v2.5.0 // indirect - github.com/andybalholm/brotli v1.2.1 // indirect + github.com/andybalholm/brotli v1.2.1 github.com/apache/arrow-go/v18 v18.6.0 // indirect github.com/aptos-labs/aptos-go-sdk v1.13.0 github.com/atombender/go-jsonschema v0.16.1-0.20240916205339-a74cd4e2851c // indirect @@ -644,7 +647,7 @@ require ( github.com/smartcontractkit/chainlink-protos/billing/go v0.0.0-20251024234028-0988426d98f4 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect - github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 // indirect + github.com/smartcontractkit/chainlink-protos/storage-service v0.3.0 github.com/smartcontractkit/chainlink-protos/svr v1.3.0 // indirect github.com/smartcontractkit/chainlink-stellar v0.0.3 // indirect github.com/smartcontractkit/chainlink-stellar/bindings v0.0.0-20260727172856-734bee1b2489 // indirect diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index 050bf93ca30..b9de3ad87b1 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -653,6 +653,7 @@ github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx5 github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQmYw= github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= +github.com/fxamacker/cbor/v2 v2.2.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/fxamacker/cbor/v2 v2.9.2 h1:X4Ksno9+x3cz0TZv69ec1hxP/+tymuR8PXQJyDwfh78= github.com/fxamacker/cbor/v2 v2.9.2/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= @@ -1085,6 +1086,8 @@ github.com/hdevalence/ed25519consensus v0.2.0/go.mod h1:w3BHWjwJbFU29IRHL1Iqkw3s github.com/heetch/avro v0.3.1/go.mod h1:4xn38Oz/+hiEUTpbVfGVLfvOg0yKLlRP7Q9+gJJILgA= github.com/hetznercloud/hcloud-go/v2 v2.36.0 h1:HlLL/aaVXUulqe+rsjoJmrxKhPi1MflL5O9iq5QEtvo= github.com/hetznercloud/hcloud-go/v2 v2.36.0/go.mod h1:MnN/QJEa/RYNQiiVoJjNHPntM7Z1wlYPgJ2HA40/cDE= +github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9 h1:pU32bJGmZwF4WXb9Yaz0T8vHDtIPVxqDOdmYdwTQPqw= +github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9/go.mod h1:MJsac5D0fKcNWfriUERtln6segcGfD6Nu0V5uGBbPf8= github.com/holiman/billy v0.0.0-20250707135307-f2f9b9aae7db h1:IZUYC/xb3giYwBLMnr8d0TGTzPKFGNTCGgGLoyeX330= github.com/holiman/billy v0.0.0-20250707135307-f2f9b9aae7db/go.mod h1:xTEYN9KCHxuYHs+NmrmzFcnvHMzLLNiGFafCb1n3Mfg= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= @@ -1762,6 +1765,10 @@ github.com/smartcontractkit/chainlink-common/keystore v1.3.0 h1:V05Rp9/dTc4Wyips github.com/smartcontractkit/chainlink-common/keystore v1.3.0/go.mod h1:vHV8BGm6TN7jBbMsWxq1Hqm3HbCtYFwzvKS0CCczxG8= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 h1:uWEwl7i2ryuRVoV4DmIKm6mqYevf1lH/8cQYhw/JXko= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72/go.mod h1:UYcRMb4dZcoaIPgZJ3hckCySTqtJc9K4Q+tOKErwTq0= +github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61E4EYRzQDtRwqunDBlKpsZc1SvFEUlSLE= +github.com/smartcontractkit/chainlink-confidential-compute v1.3.0/go.mod h1:fq9n85XoREIUxgdXIX3RyDIRCRyxZKZ5NCBbFDfdySo= +github.com/smartcontractkit/chainlink-confidential-compute/tests/testhelpers v0.0.0-20260812145307-d77342c53d7d h1:CZ0Om7lANyhpFJcmZE8RwQpEP4PyvoNLhOJckjm+zao= +github.com/smartcontractkit/chainlink-confidential-compute/tests/testhelpers v0.0.0-20260812145307-d77342c53d7d/go.mod h1:Q5q/ohoAF5N7GXZS6QxgPWSaSLFtS6/53QHJbBUkCmU= github.com/smartcontractkit/chainlink-data-streams v1.1.0 h1:O5ngSwpwey7kQ3t4YgFiXzu3L3EvYiPS4c93gMWNRbk= github.com/smartcontractkit/chainlink-data-streams v1.1.0/go.mod h1:dF5JiHWueHjYguUUUrFeb03MkcDqha/tssEkqTkgzp4= github.com/smartcontractkit/chainlink-deployments-framework v0.111.1-0.20260612191326-e31c0ae4cd54 h1:mzbvXxdbE/96Pdj1zyPKzf25ZlDR48+iTTDTbaITvmk= @@ -2628,6 +2635,7 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105210202-9ed45478a130/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= diff --git a/system-tests/tests/smoke/cre/confidential_workflows_test.go b/system-tests/tests/smoke/cre/confidential_workflows_test.go new file mode 100644 index 00000000000..024ebd6ecda --- /dev/null +++ b/system-tests/tests/smoke/cre/confidential_workflows_test.go @@ -0,0 +1,533 @@ +package cre + +import ( + "bytes" + "context" + "fmt" + "math/big" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-confidential-compute/tests/testhelpers" + cctypes "github.com/smartcontractkit/chainlink-confidential-compute/types" + capabilities_registry_v2 "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/capabilities_registry_wrapper_v2" + "github.com/smartcontractkit/chainlink-testing-framework/framework" + ns "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set" + "github.com/smartcontractkit/chainlink-testing-framework/seth" + keystone_changeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset" + crelib "github.com/smartcontractkit/chainlink/system-tests/lib/cre" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre/capabilities/confidentialcompute" + crecontracts "github.com/smartcontractkit/chainlink/system-tests/lib/cre/contracts" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains/evm" + creworkflow "github.com/smartcontractkit/chainlink/system-tests/lib/cre/workflow" + t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers" + ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration" +) + +const ( + // confidentialWorkflowsConfigPath is the topology this test runs against, + // relative to the CRE environment directory. + confidentialWorkflowsConfigPath = "/configs/workflow-gateway-capabilities-don-confidential-workflows.toml" + + // confidentialWorkflowName is the on-chain workflow name. + confidentialWorkflowName = "confidential-workflows-e2e" + + // confidentialEchoURL is the outbound target the workflow fetches from inside + // the enclave. The enclave's default egress policy allows public HTTPS. + confidentialEchoURL = "https://postman-echo.com/post" + + // confidentialSecretName is the vault secret the workflow reads via GetSecret. + confidentialSecretName = "MOCK_SECRET" + + // confidentialVaultThreshold matches the 4-node F=1 vault DON. + confidentialVaultThreshold = 1 + + // confidentialDONPageLimit bounds the getDONs page read when locating a + // capability's DON. The topology has a handful of DONs, so one page covers it. + confidentialDONPageLimit = 100 +) + +// Test_CRE_V2_ConfidentialWorkflows_Relay exercises the confidential workflows +// engine path end to end: +// +// syncer -> ConfidentialModule -> confidential-workflows capability -> enclave +// -> WASM (cron trigger) -> GetSecret (remote dispatch to the vault DON via the +// confidential relay) + http.SendRequest (intercepted and executed in-enclave) +// +// The enclaves run locally. On a Nitro-capable host they are real Nitro +// enclaves; elsewhere (including the CRE CI runners) the harness falls back to +// fake enclaves, which run the same binaries as local processes over emulated +// vsock. Attestation validation is relaxed only in the fake case. +// +// The chain-write leg of the workflow (ReportFromDon + evm.WriteReport) is left +// disabled here: it needs a deployed report receiver, and the secret and HTTP +// legs are what prove the relay and enclave routing work. +func Test_CRE_V2_ConfidentialWorkflows_Relay(t *testing.T) { + testLogger := framework.L + + t.Run("Confidential Workflows Relay - "+topology, func(t *testing.T) { + // Resolved first so the test skips before standing up anything when the + // confidential-compute checkout is not available. + ccRoot := confidentialComputeRoot(t) + + fake := testhelpers.UseFakeEnclave() + testLogger.Info().Bool("fakeEnclaves", fake).Msg("Starting confidential workflows relay test") + + tconf := t_helpers.GetTestConfig(t, confidentialWorkflowsConfigPath) + t.Setenv("CTF_CONFIGS", tconf.EnvironmentConfigPath) + + // 1. Stand up the host-side services the enclaves need to reach. Their + // addresses have to be known before the enclaves start, because they are + // baked into the settings the enclave receives at startup. The gateway + // URL is not known until the CRE environment is up, hence the proxy. + gwProxy := newDeferredGatewayProxy(t, confidentialGatewayProxyPort) + enclaveHost := confidentialEnclaveHostAddr(fake) + storageAddr, storageSvc := startFakeStorageService(t, enclaveHost) + // Both env vars below configure the enclave *host servers*, not this repo: + // the harness launches them with this process's environment inherited + // (testhelpers.MustSetupEnclaveWithEnv appends to os.Environ()) + // REQUIRE_BFT_QUORUM makes each host demand a 2f+1 BFT supermajority of node + // signatures instead of f+1 (enclave/nitro/host), matching the relay's + // requireBFTQuorum = true in the topology TOML. + t.Setenv("REQUIRE_BFT_QUORUM", "true") + t.Setenv("ENCLAVE_SETTINGS", fmt.Sprintf( + `{"storageKey":%q,"storageServiceUrl":%q,"storageServiceTls":false,"gatewayUrl":%q}`, + confidentialStorageKeyHex, + storageAddr, + fmt.Sprintf("http://%s:%d", enclaveHost, confidentialGatewayProxyPort), + )) + + // 2. Start the enclaves. This is the whole point of depending on + // chainlink-confidential-compute's harness from this repository. + enclaveCfg := testhelpers.DefaultLocalEnclaveSetupConfig(ccRoot, confidentialWorkflowsApp) + enclaveCfg.Region = confidentialEnclaveRegion + enclaves := testhelpers.SetupLocalEnclaves(t, enclaveCfg) + t.Cleanup(enclaves.CleanupAll) + testLogger.Info(). + Str("hostIP", enclaves.HostIP). + Int("count", len(enclaves.Enclaves)). + Msg("Local enclaves ready") + + // 3. Build the environment the standard way. The capability and the relay + // come from the standard feature set, configured by the topology TOML, + // so nothing here has to be injected as Go values. + testEnv := t_helpers.SetupTestEnvironmentWithConfig(t, tconf) + + // 4. Publish the enclave list to the capability's on-chain registry config, + // which the relay handler reads to decide where to route. The capability + // registers with an empty list and refreshes from the registry on a timer, + // so this can land after the environment is already running. + publishEnclaves(t, testEnv, testLogger, enclaves.Enclaves) + + // 5. Point the proxy at the real gateway now that it exists. + gatewayURL := confidentialGatewayURL(t, testEnv) + require.NoError(t, gwProxy.SetTarget(gatewayURL), "failed to set gateway proxy target") + testLogger.Info().Str("gatewayURL", gatewayURL).Msg("Gateway proxy target set") + + // 6. The vault DON only serves its public key once DKG has produced a + // result package on every worker. Fetching before then times out, and + // how long DKG takes tracks how loaded the runner is. + ensureVaultDKGResultPackages(t, testEnv) + + // 6a. The engine's pre-enclave secret fetch reads VaultPublicKey and + // Threshold from the vault capability's registry config, which is + // registered empty. Without this, GetSecret fails inside the workflow. + vaultPublicKey := injectVaultPublicKey(t, testEnv, testLogger, gatewayURL) + + // 6b. The enclaves boot with no signer set and no master public key, so they + // reject every compute request until this lands. + configureEnclaves(t, testEnv, testLogger, enclaves.ConfigURLs, vaultPublicKey) + + // 6c. Store the secret the workflow reads via GetSecret. Without it the + // request really reaches the vault DON and really comes back empty. + storeConfidentialWorkflowSecret(t, testEnv, testLogger, gatewayURL, vaultPublicKey, + confidentialSecretName, "s3cret-from-vault") + + // 7. Compile and serve the workflow from the confidential-compute checkout. + // ConsumerAddress is left empty, which the workflow treats as "skip the + // chain-write leg". + configJSON := fmt.Sprintf(`{"echo_url":%q}`, confidentialEchoURL) + artifacts := buildAndServeConfidentialWorkflow(t, ccRoot, configJSON, testhelpers.DetectHostIP()) + testLogger.Info(). + Str("binaryURL", artifacts.BinaryURL). + Str("configURL", artifacts.ConfigURL). + Msg("Workflow artifacts served") + + // The artifact server binds 0.0.0.0, but the enclave reaches it at a + // different host than Docker does, so swap only the host portion. + parsed, pErr := url.Parse(artifacts.BinaryURL) + require.NoError(t, pErr, "parsing workflow binary URL") + storageSvc.setURL(fmt.Sprintf("http://%s:%s%s", enclaveHost, parsed.Port(), parsed.Path)) + + // 8. The syncer reads the binary and config from disk (see the topology's + // CRE.WorkflowFetcher override), so copy them into the containers. + copyWorkflowArtifactsToContainers(t, testEnv, artifacts) + + // 9. Register the workflow as confidential and wait for a successful + // execution. The workflow returns an error if either GetSecret or the + // in-enclave HTTP fetch fails, so a successful execution implies the + // whole relay + enclave path worked. + workflowID := registerConfidentialWorkflow(t, testEnv, testLogger, artifacts) + waitForConfidentialWorkflowExecution(t, testEnv, testLogger, workflowID, 5*time.Minute) + + testLogger.Info().Msg("Confidential workflows relay E2E passed") + }) +} + +// confidentialEnclaveHostAddr is the address the enclave reaches host-local test +// servers at: loopback for fake enclaves (ordinary local processes), the wg0 host +// IP for real Nitro enclaves. +func confidentialEnclaveHostAddr(fake bool) string { + if fake { + return "localhost" + } + return "100.64.0.3" +} + +// confidentialComputeRoot resolves the chainlink-confidential-compute checkout the +// enclave harness builds and runs the enclave from. The test is skipped when it is +// not set, so a local `go test ./...` does not fail on a missing checkout. +func confidentialComputeRoot(t *testing.T) string { + t.Helper() + + root := os.Getenv("CONFIDENTIAL_COMPUTE_ROOT") + if root == "" { + t.Skip("CONFIDENTIAL_COMPUTE_ROOT must point at a chainlink-confidential-compute checkout; " + + "CI sets this from the confidential-workflows gitRef in plugins/plugins.public.yaml") + } + + abs, err := filepath.Abs(root) + require.NoError(t, err, "resolving CONFIDENTIAL_COMPUTE_ROOT") + return abs +} + +// confidentialGatewayURL builds the externally reachable gateway URL. +func confidentialGatewayURL(t *testing.T, testEnv *ttypes.TestEnvironment) string { + t.Helper() + + require.NotEmpty(t, testEnv.Dons.GatewayConnectors.Configurations, "no gateway connector configurations") + incoming := testEnv.Dons.GatewayConnectors.Configurations[0].Incoming + host := incoming.Host + if host == "" { + host = testhelpers.DetectHostIP() + } + return fmt.Sprintf("%s://%s:%d%s", incoming.Protocol, host, incoming.ExternalPort, incoming.Path) +} + +// publishEnclaves writes the enclave list into the capability's on-chain +// registry config and waits for the capability to pick it up. +// +// The capability refreshes from the registry on a ticker +// (DefaultEnclaveRefreshIntervalSeconds, 10s), so this waits two intervals +// rather than one: a single interval races a refresh that started just before +// the transaction landed and therefore read the old config. +func publishEnclaves( + t *testing.T, + testEnv *ttypes.TestEnvironment, + testLogger zerolog.Logger, + enclaves []cctypes.Enclave, +) { + t.Helper() + + ctx := t.Context() + + require.IsType(t, &evm.Blockchain{}, testEnv.CreEnvironment.Blockchains[0], "expected EVM blockchain") + sethClient := testEnv.CreEnvironment.Blockchains[0].(*evm.Blockchain).SethClient + + capRegAddr := crecontracts.MustGetAddressFromDataStore( + testEnv.CreEnvironment.CldfEnvironment.DataStore, + testEnv.CreEnvironment.Blockchains[0].ChainSelector(), + keystone_changeset.CapabilitiesRegistry.String(), + testEnv.CreEnvironment.ContractVersions[keystone_changeset.CapabilitiesRegistry.String()], + "", + ) + + config, err := confidentialcompute.MarshalRegistryConfig(enclaves) + require.NoError(t, err, "failed to encode enclave list for the registry") + + donName := donNameForCapability(t, sethClient, capRegAddr, confidentialWorkflowsApp) + + require.NoError(t, + crelib.UpdateDONCapabilityConfig(ctx, sethClient, capRegAddr, donName, confidentialWorkflowsApp, config), + "failed to publish enclave list to the capabilities registry", + ) + + testLogger.Info(). + Str("don", donName). + Int("count", len(enclaves)). + Dur("wait", confidentialEnclaveRefreshWait). + Msg("Published enclave list; waiting for the capability to refresh from the registry") + time.Sleep(confidentialEnclaveRefreshWait) +} + +// donNameForCapability returns the registry name of the DON providing the named +// capability. The registry derives DON names from the topology, so the name is +// resolved rather than assumed: a wrong name makes getDONByName revert with an +// opaque custom error, whereas this reports which DONs actually exist. +func donNameForCapability( + t *testing.T, + sethClient *seth.Client, + capabilitiesRegistryAddr string, + capabilityName string, +) string { + t.Helper() + + capReg, err := capabilities_registry_v2.NewCapabilitiesRegistry( + common.HexToAddress(capabilitiesRegistryAddr), sethClient.Client, + ) + require.NoError(t, err, "failed to create capabilities registry wrapper") + + allDONs, err := capReg.GetDONs(&bind.CallOpts{Context: t.Context()}, big.NewInt(0), big.NewInt(confidentialDONPageLimit)) + require.NoError(t, err, "failed to list DONs from the capabilities registry") + + names := make([]string, 0, len(allDONs)) + for i := range allDONs { + names = append(names, allDONs[i].Name) + // The registry keys capabilities as "name@version". + for _, capabilityConfig := range allDONs[i].CapabilityConfigurations { + if strings.HasPrefix(capabilityConfig.CapabilityId, capabilityName+"@") { + return allDONs[i].Name + } + } + } + + require.FailNowf(t, "capability is not registered on any DON", + "no DON provides capability %q; DONs present: %s", capabilityName, strings.Join(names, ", ")) + + return "" +} + +// injectVaultPublicKey writes the vault DON's DKG public key and threshold into +// the vault capability's registry config. +func injectVaultPublicKey(t *testing.T, testEnv *ttypes.TestEnvironment, testLogger zerolog.Logger, gatewayURL string) string { + t.Helper() + + ctx := t.Context() + vaultPublicKey, err := creworkflow.FetchVaultPublicKey(ctx, gatewayURL) + require.NoError(t, err, "failed to fetch vault public key from gateway") + + require.IsType(t, &evm.Blockchain{}, testEnv.CreEnvironment.Blockchains[0], "expected EVM blockchain") + sethClient := testEnv.CreEnvironment.Blockchains[0].(*evm.Blockchain).SethClient + + capRegAddr := crecontracts.MustGetAddressFromDataStore( + testEnv.CreEnvironment.CldfEnvironment.DataStore, + testEnv.CreEnvironment.Blockchains[0].ChainSelector(), + keystone_changeset.CapabilitiesRegistry.String(), + testEnv.CreEnvironment.ContractVersions[keystone_changeset.CapabilitiesRegistry.String()], + "", + ) + + vaultDON, _, err := crelib.GetVaultCapabilityDON(ctx, sethClient, capRegAddr) + require.NoError(t, err, "failed to locate vault capability DON in registry") + + require.NoError(t, + creworkflow.UpdateVaultCapabilityConfig(ctx, sethClient, capRegAddr, vaultDON, vaultPublicKey, confidentialVaultThreshold), + "failed to inject VaultPublicKey/Threshold into the vault capability config") + testLogger.Info().Msg("Injected VaultPublicKey + Threshold into the vault capability config") + + return vaultPublicKey +} + +// copyWorkflowArtifactsToContainers copies the compiled binary and its config into +// every workflow DON container so the syncer's file fetcher can read them. +func copyWorkflowArtifactsToContainers(t *testing.T, testEnv *ttypes.TestEnvironment, artifacts confidentialWorkflowArtifacts) { + t.Helper() + + for _, don := range testEnv.Dons.List() { + if !don.HasFlag(crelib.WorkflowDON) { + continue + } + for _, filename := range []string{confidentialWorkflowBinaryFilename, confidentialWorkflowConfigFilename} { + require.NoError(t, + creworkflow.CopyArtifactsToDockerContainers( + creworkflow.DefaultWorkflowTargetDir, + ns.NodeNamePrefix(don.Name), + filepath.Join(artifacts.ArtifactDir, filename), + ), + "failed to copy %s to the %s DON containers", filename, don.Name) + } + } +} + +// registerConfidentialWorkflow registers the workflow on-chain with confidential +// attributes and returns its workflow ID. +func registerConfidentialWorkflow( + t *testing.T, + testEnv *ttypes.TestEnvironment, + testLogger zerolog.Logger, + artifacts confidentialWorkflowArtifacts, +) string { + t.Helper() + + require.IsType(t, &evm.Blockchain{}, testEnv.CreEnvironment.Blockchains[0], "expected EVM blockchain") + sethClient := testEnv.CreEnvironment.Blockchains[0].(*evm.Blockchain).SethClient + + wfRegistryRef := crecontracts.MustGetAddressRefFromDataStore( + testEnv.CreEnvironment.CldfEnvironment.DataStore, + testEnv.CreEnvironment.Blockchains[0].ChainSelector(), + keystone_changeset.WorkflowRegistry.String(), + testEnv.CreEnvironment.ContractVersions[keystone_changeset.WorkflowRegistry.String()], + "", + ) + + // The confidential attribute is what routes execution into the enclave rather + // than running the WASM on the workflow DON. + attributes := []byte(`{"confidential":true}`) + configURL := artifacts.ConfigURL + + workflowID, err := creworkflow.RegisterWithContract( + context.Background(), + sethClient, + common.HexToAddress(wfRegistryRef.Address), + wfRegistryRef.Version, + 0, // donID unused for v2 + testEnv.Dons.MustWorkflowDON().DonFamily, + confidentialWorkflowName, + workflowTag, + artifacts.BinaryURL, + &configURL, + nil, // no secrets URL + attributes, + nil, // keep the HTTP URL on-chain; the enclave fetches the binary itself + ) + require.NoError(t, err, "failed to register confidential workflow") + testLogger.Info().Str("workflowID", workflowID).Msg("Confidential workflow registered") + + t.Cleanup(func() { + _ = creworkflow.DeleteWithContract( + context.Background(), + sethClient, + common.HexToAddress(wfRegistryRef.Address), + wfRegistryRef.Version, + confidentialWorkflowName, + ) + }) + + return workflowID +} + +// waitForConfidentialWorkflowExecution waits for the engine to log a successful +// execution for this workflow. The engine emits that line once per successful +// trigger execution, not for the Subscribe-phase call at engine startup, so +// finding it means the cron trigger fired and the whole enclave path succeeded. +func waitForConfidentialWorkflowExecution( + t *testing.T, + testEnv *ttypes.TestEnvironment, + testLogger zerolog.Logger, + workflowID string, + timeout time.Duration, +) { + t.Helper() + + containers := confidentialWorkflowDONContainers(testEnv) + require.NotEmpty(t, containers, "no workflow DON containers found to scrape") + + needleMsg := []byte(`"msg":"Workflow execution finished successfully"`) + needleID := []byte(workflowID) + testLogger.Info(). + Str("workflowID", workflowID). + Strs("containers", containers). + Msg("Waiting for a successful workflow execution") + + deadline := time.Now().Add(timeout) + for { + for _, name := range containers { + out, _ := exec.CommandContext(t.Context(), "docker", "logs", "--tail", "10000", name).CombinedOutput() + for line := range bytes.SplitSeq(out, []byte{'\n'}) { + if bytes.Contains(line, needleMsg) && bytes.Contains(line, needleID) { + testLogger.Info().Str("container", name).Msg("Found successful execution log") + return + } + } + } + if time.Now().After(deadline) { + t.Fatalf("timed out after %s waiting for a successful execution of workflow %s\n%s", + timeout, workflowID, confidentialExecutionDiagnostics(t, containers)) + } + time.Sleep(5 * time.Second) + } +} + +// confidentialExecutionDiagnostics summarises why no execution succeeded, so a +// timeout reports the node-side cause instead of only the absence of a success +// log. Returns one deduplicated line per distinct error. +func confidentialExecutionDiagnostics(t *testing.T, containers []string) string { + t.Helper() + + needles := [][]byte{ + []byte("Workflow Engine initialization failed"), + []byte("Workflow execution failed"), + []byte("failed to get regions from"), + []byte("no compatible capability found"), + } + + seen := map[string]bool{} + var found []string + for _, name := range containers { + out, _ := exec.CommandContext(t.Context(), "docker", "logs", "--tail", "10000", name).CombinedOutput() + for line := range bytes.SplitSeq(out, []byte{'\n'}) { + for _, needle := range needles { + if !bytes.Contains(line, needle) { + continue + } + // Key on the message alone; every node logs the same failure. + key := string(needle) + if !seen[key] { + seen[key] = true + found = append(found, fmt.Sprintf(" [%s] %s", name, truncateForLog(line, 400))) + } + break + } + } + } + + if len(found) == 0 { + return "no workflow engine or execution errors found in the node logs" + } + + return "node-side errors:\n" + strings.Join(found, "\n") +} + +// truncateForLog shortens a log line so a failure message stays readable. +func truncateForLog(line []byte, maxLen int) string { + if len(line) <= maxLen { + return string(line) + } + + return string(line[:maxLen]) + "... (truncated)" +} + +// confidentialWorkflowDONContainers returns the chainlink container names for +// every nodeset whose DON carries the workflow DON flag. +func confidentialWorkflowDONContainers(testEnv *ttypes.TestEnvironment) []string { + workflowDONNames := map[string]bool{} + for _, don := range testEnv.Dons.List() { + if don.HasFlag(crelib.WorkflowDON) { + workflowDONNames[don.Name] = true + } + } + + var names []string + for _, nodeSet := range testEnv.Config.NodeSets { + if !workflowDONNames[nodeSet.Name] || nodeSet.Out == nil { + continue + } + for _, cl := range nodeSet.Out.CLNodes { + if cl == nil || cl.Node == nil || cl.Node.ContainerName == "" { + continue + } + names = append(names, cl.Node.ContainerName) + } + } + return names +} diff --git a/system-tests/tests/smoke/cre/confidential_workflows_test_helpers.go b/system-tests/tests/smoke/cre/confidential_workflows_test_helpers.go new file mode 100644 index 00000000000..1bb8092ae55 --- /dev/null +++ b/system-tests/tests/smoke/cre/confidential_workflows_test_helpers.go @@ -0,0 +1,402 @@ +package cre + +import ( + "bytes" + "context" + "crypto/sha256" + "crypto/tls" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/http/httputil" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/andybalholm/brotli" + "github.com/ethereum/go-ethereum/common" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/smartcontractkit/chainlink-confidential-compute/tests/testhelpers" + cctypes "github.com/smartcontractkit/chainlink-confidential-compute/types" + "github.com/smartcontractkit/chainlink-confidential-compute/util" + workflow_registry_v2_wrapper "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2" + storage_service "github.com/smartcontractkit/chainlink-protos/storage-service/go" + keystone_changeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset" + crelib "github.com/smartcontractkit/chainlink/system-tests/lib/cre" + crecontracts "github.com/smartcontractkit/chainlink/system-tests/lib/cre/contracts" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains/evm" + ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaultutils" +) + +const ( + // confidentialWorkflowsApp is the enclave application, the DON capability flag + // and the capability binary name; all three share this value. + confidentialWorkflowsApp = crelib.ConfidentialWorkflowsCapability + + // confidentialServerReadHeaderTimeout bounds header reads on the test's local + // HTTP servers. + confidentialServerReadHeaderTimeout = 10 * time.Second + + // confidentialEnclaveRefreshWait covers two of the capability's registry + // refresh intervals, so a refresh already in flight when the enclave list + // lands cannot be mistaken for the one that picks it up. + confidentialEnclaveRefreshWait = 25 * time.Second + + // confidentialGatewayProxyPort is the fixed port the enclaves are told to reach + // the CRE gateway on. It must be known before the enclaves start, which is why + // the proxy in front of it resolves its target lazily. + confidentialGatewayProxyPort = 9999 + + // confidentialStorageKeyHex is a deterministic ed25519 seed the enclave uses to + // authenticate to the fake storage service. The fake does not verify the JWT. + confidentialStorageKeyHex = "0000000000000000000000000000000000000000000000000000000000000001" + + confidentialWorkflowBinaryFilename = "workflow-test-confidential.br.b64" + confidentialWorkflowConfigFilename = "workflow-test-config.json" + + // confidentialWorkflowSrcRelDir is the WASM workflow this test compiles, + // relative to the chainlink-confidential-compute checkout. The source is not + // vendored here on purpose: it depends on cre-sdk-go versions that predate the + // removal of the in-TEE HTTP API, which this repository's dependency + // validation rejects, and the compiled artifact is covered by .gitignore. + confidentialWorkflowSrcRelDir = "tests/e2e/testdata/workflow" + + // confidentialEnclaveRegion is recorded on each enclave descriptor. Descriptor + // hashes cover it, so it must match what the enclave itself reports. + confidentialEnclaveRegion = "us-west-2" +) + +// --------------------------------------------------------------------------- +// Deferred gateway proxy +// --------------------------------------------------------------------------- + +// deferredGatewayProxy is a reverse proxy on a fixed port that returns 502 until +// SetTarget is called with the real gateway URL. This resolves a chicken-and-egg +// problem: the enclaves are told their gateway URL at startup, but the real URL +// is only known once the CRE environment is up. +type deferredGatewayProxy struct { + mu sync.RWMutex + target *url.URL + server *http.Server +} + +func newDeferredGatewayProxy(t *testing.T, port int) *deferredGatewayProxy { + t.Helper() + + p := &deferredGatewayProxy{} + rp := &httputil.ReverseProxy{ + Director: func(req *http.Request) { + p.mu.RLock() + defer p.mu.RUnlock() + if p.target != nil { + req.URL.Scheme = p.target.Scheme + req.URL.Host = p.target.Host + req.Host = p.target.Host + } + }, + } + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + p.mu.RLock() + hasTarget := p.target != nil + p.mu.RUnlock() + if !hasTarget { + http.Error(w, "gateway not ready", http.StatusBadGateway) + return + } + rp.ServeHTTP(w, r) + }) + + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", fmt.Sprintf("0.0.0.0:%d", port)) + require.NoError(t, err, "failed to listen on port %d for gateway proxy", port) + + p.server = &http.Server{Handler: handler, ReadHeaderTimeout: confidentialServerReadHeaderTimeout} + go func() { _ = p.server.Serve(listener) }() + t.Cleanup(func() { _ = p.server.Close() }) + + return p +} + +func (p *deferredGatewayProxy) SetTarget(rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return err + } + p.mu.Lock() + defer p.mu.Unlock() + p.target = u + return nil +} + +// --------------------------------------------------------------------------- +// Fake CRE storage service +// --------------------------------------------------------------------------- + +// fakeStorageService is a minimal in-process CRE storage NodeService. The enclave +// fetches the workflow binary itself: it calls DownloadArtifact over JWT-authed +// gRPC, gets a pre-signed URL, downloads it and verifies the hash. This fake +// returns the URL of the base64 WASM server the test stands up. +type fakeStorageService struct { + storage_service.UnimplementedNodeServiceServer + mu sync.Mutex + url string +} + +func (f *fakeStorageService) setURL(u string) { + f.mu.Lock() + f.url = u + f.mu.Unlock() +} + +func (f *fakeStorageService) DownloadArtifact(_ context.Context, req *storage_service.DownloadArtifactRequest) (*storage_service.DownloadArtifactResponse, error) { + f.mu.Lock() + u := f.url + f.mu.Unlock() + + // Mirror real storage-service semantics: the id must be a bare artifact id, + // not a full URL. Rejecting the URL shape here keeps a regression failing in + // this test rather than only in a live environment. + if strings.Contains(req.GetId(), "://") { + return nil, status.Errorf(codes.NotFound, "fake storage: artifact with id %q not found (expected a bare id, not a URL)", req.GetId()) + } + if u == "" { + return nil, errors.New("fake storage: artifact url not set yet") + } + return &storage_service.DownloadArtifactResponse{Url: u}, nil +} + +// startFakeStorageService starts a gRPC NodeService bound to 0.0.0.0 and returns +// the address the enclave dials it at, plus the service so the test can set the +// artifact URL once the WASM server is up. +func startFakeStorageService(t *testing.T, enclaveHost string) (string, *fakeStorageService) { + t.Helper() + + // Binds every interface because the enclave dials it from outside this process. + lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "0.0.0.0:0") + require.NoError(t, err, "fake storage listener") + + svc := &fakeStorageService{} + grpcSrv := grpc.NewServer() + storage_service.RegisterNodeServiceServer(grpcSrv, svc) + go func() { _ = grpcSrv.Serve(lis) }() + t.Cleanup(grpcSrv.Stop) + + return fmt.Sprintf("%s:%d", enclaveHost, lis.Addr().(*net.TCPAddr).Port), svc +} + +// --------------------------------------------------------------------------- +// Workflow artifacts +// --------------------------------------------------------------------------- + +// confidentialWorkflowArtifacts holds everything derived from compiling the test +// workflow: the URLs the syncer and enclave fetch it from, and the on-disk +// directory the test copies into the DON containers. +type confidentialWorkflowArtifacts struct { + BinaryURL string + ConfigURL string + ArtifactDir string + BinaryHash []byte +} + +// buildAndServeConfidentialWorkflow compiles the test workflow to wasip1/wasm +// from the chainlink-confidential-compute checkout, brotli-compresses and +// base64-encodes it (the format the syncer and the enclave both expect), and +// serves the binary and its config over HTTP bound to 0.0.0.0 so the host, the +// Docker containers and the enclaves can all fetch them. +// +// Compiling from the checkout rather than vendoring the source keeps the +// workflow single-sourced and keeps its cre-sdk-go pins out of this +// repository's module graph. +func buildAndServeConfidentialWorkflow(t *testing.T, ccRoot string, configJSON string, hostIP string) confidentialWorkflowArtifacts { + t.Helper() + + srcDir := filepath.Join(ccRoot, confidentialWorkflowSrcRelDir) + require.DirExists(t, srcDir, "confidential workflow source not found in the chainlink-confidential-compute checkout") + + tmpDir := t.TempDir() + outFile := filepath.Join(tmpDir, "workflow-test.wasm") + + cmd := exec.CommandContext(t.Context(), "go", "build", "-o", outFile, ".") + cmd.Dir = srcDir + cmd.Env = append(os.Environ(), "GOOS=wasip1", "GOARCH=wasm", "CGO_ENABLED=0") + output, err := cmd.CombinedOutput() + require.NoError(t, err, "compiling confidential workflow WASM: %s", string(output)) + + raw, err := os.ReadFile(outFile) + require.NoError(t, err, "reading compiled WASM") + + var compressed bytes.Buffer + w := brotli.NewWriter(&compressed) + _, err = w.Write(raw) + require.NoError(t, err, "brotli compressing WASM") + require.NoError(t, w.Close(), "closing brotli writer") + + binary := compressed.Bytes() + hash := sha256.Sum256(binary) + encoded := base64.StdEncoding.EncodeToString(binary) + + // The syncer's file fetcher reads both files from disk inside the container, + // so they have to exist as real files the test can copy in. + require.NoError(t, + os.WriteFile(filepath.Join(tmpDir, confidentialWorkflowBinaryFilename), []byte(encoded), 0o600), + "staging workflow binary artifact") + require.NoError(t, + os.WriteFile(filepath.Join(tmpDir, confidentialWorkflowConfigFilename), []byte(configJSON), 0o600), + "staging workflow config artifact") + + mux := http.NewServeMux() + mux.HandleFunc("/"+confidentialWorkflowBinaryFilename, func(rw http.ResponseWriter, _ *http.Request) { + _, _ = rw.Write([]byte(encoded)) + }) + mux.HandleFunc("/"+confidentialWorkflowConfigFilename, func(rw http.ResponseWriter, _ *http.Request) { + _, _ = rw.Write([]byte(configJSON)) + }) + + // Binds every interface because the enclave fetches artifacts over the host network. + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "0.0.0.0:0") + require.NoError(t, err, "workflow artifact listener") + srv := &http.Server{Handler: mux, ReadHeaderTimeout: confidentialServerReadHeaderTimeout} + go func() { _ = srv.Serve(listener) }() + t.Cleanup(func() { _ = srv.Close() }) + + port := listener.Addr().(*net.TCPAddr).Port + base := fmt.Sprintf("http://%s:%d/", hostIP, port) + + return confidentialWorkflowArtifacts{ + BinaryURL: base + confidentialWorkflowBinaryFilename, + ConfigURL: base + confidentialWorkflowConfigFilename, + ArtifactDir: tmpDir, + BinaryHash: hash[:], + } +} + +// --------------------------------------------------------------------------- +// Enclave configuration +// --------------------------------------------------------------------------- + +// configureEnclaves POSTs the enclave config to every enclave's config endpoint. +// Until this lands, an enclave has no signer set and no master public key, so it +// rejects every incoming compute request. +// +// The signer set is the workflow DON's worker P2P IDs, and F is derived as +// 2*don.F + 1 to match the relay DON quorum the enclave expects. +func configureEnclaves( + t *testing.T, + testEnv *ttypes.TestEnvironment, + testLogger zerolog.Logger, + configURLs []string, + vaultPublicKey string, +) { + t.Helper() + + don := testEnv.Dons.MustWorkflowDON() + workers, err := don.Workers() + require.NoError(t, err, "failed to get worker nodes from topology") + require.NotEmpty(t, workers, "workflow DON has no worker nodes") + + signers := make([][]byte, 0, len(workers)) + for _, node := range workers { + signers = append(signers, node.Keys.P2PKey.PeerID[:]) + } + + masterPublicKey, err := hex.DecodeString(vaultPublicKey) + require.NoError(t, err, "failed to hex-decode vault public key") + + // Quorum tracks the DON's registered fault tolerance (Don.F, computed as + // (workers-1)/3 in NewDON), not a re-derivation from the worker count: the + // two diverge for e.g. 6-node DONs, and the enclave would then demand more + // signatures than the DON can produce. + quorum := 2*uint32(don.F) + 1 + + config := cctypes.EnclaveConfig{ + Signers: signers, + MasterPublicKey: masterPublicKey, + T: quorum, + F: quorum, + } + configBytes, err := json.Marshal(config) + require.NoError(t, err, "failed to marshal enclave config") + + enclaveType := cctypes.EnclaveTypeNitro + if testhelpers.UseFakeEnclave() { + enclaveType = cctypes.EnclaveTypeFake + } + + client := http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // local test enclaves use self-signed certs + }, + } + + for i, configURL := range configURLs { + _, err := util.SetNodeConfig( + t.Context(), + cctypes.Enclave{ + EnclaveURL: configURL, + EnclaveType: enclaveType, + TrustedValues: [][]byte{}, + Region: confidentialEnclaveRegion, + }, + cctypes.ConfigRequest{Config: configBytes}, + &client, + ) + require.NoError(t, err, "failed to set config on enclave %d (%s)", i, configURL) + testLogger.Info().Int("enclave", i).Str("configURL", configURL).Msg("Enclave configured") + } +} + +// storeConfidentialWorkflowSecret encrypts a secret to the vault's public key and +// stores it in the vault DON through the gateway, so the workflow's GetSecret call +// resolves. Reuses the vault request helpers already in this package. +func storeConfidentialWorkflowSecret( + t *testing.T, + testEnv *ttypes.TestEnvironment, + testLogger zerolog.Logger, + gatewayURL string, + vaultPublicKey string, + secretKey string, + secretValue string, +) { + t.Helper() + + require.IsType(t, &evm.Blockchain{}, testEnv.CreEnvironment.Blockchains[0], "expected EVM blockchain") + sethClient := testEnv.CreEnvironment.Blockchains[0].(*evm.Blockchain).SethClient + owner := sethClient.MustGetRootKeyAddress().Hex() + + wfRegAddr := crecontracts.MustGetAddressFromDataStore( + testEnv.CreEnvironment.CldfEnvironment.DataStore, + testEnv.CreEnvironment.Blockchains[0].ChainSelector(), + keystone_changeset.WorkflowRegistry.String(), + testEnv.CreEnvironment.ContractVersions[keystone_changeset.WorkflowRegistry.String()], + "", + ) + wfReg, err := workflow_registry_v2_wrapper.NewWorkflowRegistry(common.HexToAddress(wfRegAddr), sethClient.Client) + require.NoError(t, err, "failed to build workflow registry wrapper") + + // The vault DON only accepts secrets from an owner linked in the registry. + requireVaultLinkOwner(t, sethClient, common.HexToAddress(wfRegAddr), + testEnv.CreEnvironment.ContractVersions[keystone_changeset.WorkflowRegistry.String()]) + + parsedKey := mustVaultPublicKey(t, vaultPublicKey) + encryptedSecret, err := vaultutils.EncryptSecretWithWorkflowOwner(secretValue, parsedKey, sethClient.MustGetRootKeyAddress()) + require.NoError(t, err, "failed to encrypt secret for the vault DON") + + auth := newAllowlistVaultRequestAuth(owner, sethClient, wfReg) + executeVaultSecretsCreateWithAuth(t, auth, encryptedSecret, secretKey, owner, gatewayURL, []string{"main"}) + + testLogger.Info().Str("key", secretKey).Str("owner", owner).Msg("Stored workflow secret in the vault DON") +}