diff --git a/ansible/inventories/devnet-0/templates/tysm/tysm-hooks.yaml.j2 b/ansible/inventories/devnet-0/templates/tysm/tysm-hooks.yaml.j2 new file mode 100644 index 0000000..4644260 --- /dev/null +++ b/ansible/inventories/devnet-0/templates/tysm/tysm-hooks.yaml.j2 @@ -0,0 +1,88 @@ +# tysm hook-control config for {{ inventory_hostname }}. ${TYSM_TOKEN} resolves from the +# beacon container env at load; it must equal the token bad-tysm validates with. +api: + enabled: true + listen: "0.0.0.0:8675" + auth_token: "${TYSM_TOKEN}" + max_activation_duration: "4h" + instance_id: "{{ inventory_hostname }}" + +discovery: + enabled: true + bad_tysm_url: "https://bad-tysm.analytics.production.platform.ethpandaops.io" + auth_token: "${TYSM_TOKEN}" + # Address bad-tysm dials back to reach this node's control API. Terraform opens + # 8675 for tysm-group members; the beacon container must still publish the port. + self_url: "http://{{ ansible_host }}:8675" + name: "{{ inventory_hostname }}" + namespace: "{{ ethereum_network_name }}" + pod: "{{ inventory_hostname }}" + node_name: "{{ inventory_hostname }}" + heartbeat_interval: "30s" + labels: + network: "{{ ethereum_network_name }}" + cl: "{{ ethereum_node_cl }}" + el: "{{ ethereum_node_el }}" + +hook_logging: + enabled: true + classes: ["validate", "mutate"] + +hooks: + - name: epbs-mutator + enabled: false + config: + mutationDelayMs: 0 + logMutationDetails: true + topics: + execution_payload_bid: + enabledStrategies: [] + mutationProbability: 0.0 + slotPattern: "*" + strategies: + drop: + applyProbability: 1.0 + value_inflate: + mode: "uint64_max" + absolute: 0 + multiplier: 10.0 + execution_payload_envelope: + enabledStrategies: [] + mutationProbability: 0.0 + slotPattern: "*" + strategies: + withhold: + probability: 1.0 + slotPattern: "*" + delay: + delayMs: 9500 + jitterMs: 0 + applyProbability: 1.0 + selective_drop: + targetPeerRole: "" + dropRate: 0.4 + seed: 42 + free_option: + triggerSource: "unix_socket" + socketPath: "/tmp/tysm-mev-trigger.sock" + httpURL: "" + defaultState: false + pollIntervalMs: 100 + payload_attestation_message: + enabledStrategies: [] + mutationProbability: 0.0 + slotPattern: "*" + strategies: + flip_bit: + field: "payload_present" + targetValue: "false" + corrupt_field: + field: "beacon_block_root" + mode: "random_32" + drop: + applyProbability: 1.0 + targetPeerRole: "" + targetValidators: [] + delay: + delayMs: 12000 + applyProbability: 1.0 diff --git a/ansible/inventories/devnet-0/templates/tysm/tysm-xatu-config.yaml.j2 b/ansible/inventories/devnet-0/templates/tysm/tysm-xatu-config.yaml.j2 new file mode 100644 index 0000000..de7099c --- /dev/null +++ b/ansible/inventories/devnet-0/templates/tysm/tysm-xatu-config.yaml.j2 @@ -0,0 +1,16 @@ +# Minimal Xatu config: every tysm beacon requires --xatu-config-file to start. +# The xatu hook is disabled and this output points at a local no-op address that +# is never required to connect; devnet events still ship via the xatu-sentry sidecar. +name: "{{ inventory_hostname }}" + +ethereum: + network: "{{ ethereum_network_name }}" + +outputs: + - name: noop + type: xatu + config: + address: localhost:19999 + tls: false + maxQueueSize: 1000 + batchTimeout: 5s diff --git a/ansible/playbook.yaml b/ansible/playbook.yaml index 305a39b..dadcdeb 100644 --- a/ansible/playbook.yaml +++ b/ansible/playbook.yaml @@ -85,6 +85,38 @@ - role: ethpandaops.general.generate_basic_auth_nginx tags: [docker_nginx_proxy] +# Hosts in the tysm group (tysm = true on the node entry in terraform nodes.tf) +# run the tysm beacon canary and need the hook + xatu config files in the prysm +# datadir before the container starts. The group is empty by default. +- hosts: tysm + become: true + tasks: + - name: Deploy tysm config files + tags: [ethereum, ethereum_node, tysm] + block: + - name: Add prysm user + ansible.builtin.user: + name: "{{ prysm_user | default('prysm') }}" + + - name: Create prysm data dir + ansible.builtin.file: + path: "{{ prysm_datadir | default('/data/prysm') }}" + state: directory + mode: "0750" + owner: "{{ prysm_user | default('prysm') }}" + group: "{{ prysm_user | default('prysm') }}" + + - name: Template tysm config files + ansible.builtin.template: + src: "{{ inventory_dir }}/templates/tysm/{{ item }}.j2" + dest: "{{ prysm_datadir | default('/data/prysm') }}/{{ item }}" + mode: "0640" + owner: "{{ prysm_user | default('prysm') }}" + group: "{{ prysm_user | default('prysm') }}" + loop: + - tysm-hooks.yaml + - tysm-xatu-config.yaml + - hosts: ethereum_node become: true roles: diff --git a/terraform/devnet-0/ansible_inventory.tmpl b/terraform/devnet-0/ansible_inventory.tmpl index 7a2829d..142d2e8 100644 --- a/terraform/devnet-0/ansible_inventory.tmpl +++ b/terraform/devnet-0/ansible_inventory.tmpl @@ -84,6 +84,16 @@ ${replace(gid, "-", "_")} [buildoor:vars] ansible_group_priority=100 +# tysm canary hosts (tysm = true on the node entry in nodes.tf); the group always +# exists so the tysm play in the playbook resolves, it is just empty by default. + +[tysm] +%{ for key, host in hosts ~} +%{ if host.tysm != "" ~} +${host.hostname} +%{ endif ~} +%{ endfor ~} + # Global groups [consensus_node:children] diff --git a/terraform/devnet-0/digitalocean.tf b/terraform/devnet-0/digitalocean.tf index fddba71..a5155b0 100644 --- a/terraform/devnet-0/digitalocean.tf +++ b/terraform/devnet-0/digitalocean.tf @@ -68,6 +68,9 @@ locals { # Builder index for buildoor nodes (start index + instance offset); null = no tag builder_index = node.builder_start != null ? node.builder_start + i : null + # tysm canary membership; true = tysm tag, [tysm] inventory group, port 8675 + tysm = node.tysm + # Supernode: explicit > bootnode/mev > validator_count >= 128 supernode = ( node.supernode != null ? node.supernode : @@ -100,6 +103,7 @@ locals { id = group.id group_key = group.group_name vm_key = vm_key + tysm = vm.tysm name = group.id ssh_keys = [data.digitalocean_ssh_key.main.fingerprint] @@ -121,7 +125,8 @@ locals { ], compact([ can(regex("bootnode", group.group_name)) ? "bootnode:${var.ethereum_network}" : null, can(regex("mev-relay", group.group_name)) ? "mev-relay:${var.ethereum_network}" : null, - vm.builder_index != null ? "builder_index:${vm.builder_index}" : null + vm.builder_index != null ? "builder_index:${vm.builder_index}" : null, + vm.tysm ? "tysm:${var.ethereum_network}" : null ])) } ] diff --git a/terraform/devnet-0/firewall.tf b/terraform/devnet-0/firewall.tf index bfb643d..465147c 100644 --- a/terraform/devnet-0/firewall.tf +++ b/terraform/devnet-0/firewall.tf @@ -133,6 +133,20 @@ resource "digitalocean_firewall" "bootnode" { depends_on = [digitalocean_project_resources.droplets] } +resource "digitalocean_firewall" "tysm" { + count = length([for vm in local.digitalocean_vms : vm.id if vm.tysm]) > 0 ? 1 : 0 + name = "${var.ethereum_network}-nodes-tysm" + tags = ["tysm:${var.ethereum_network}"] + + // TYSM control API (bad-tysm reaches back on this port) + inbound_rule { + protocol = "tcp" + port_range = "8675" + source_addresses = ["0.0.0.0/0", "::/0"] + } + depends_on = [digitalocean_project_resources.droplets] +} + resource "digitalocean_firewall" "mev_relay" { count = contains(keys(digitalocean_droplet.main), "mev-relay-1") ? 1 : 0 name = "${var.ethereum_network}-nodes-mev-relay" @@ -326,6 +340,24 @@ resource "hcloud_firewall" "bootnode_firewall" { } } +resource "hcloud_firewall" "tysm_firewall" { + count = length([for vm in local.hcloud_vms : vm.id if vm.tysm]) > 0 ? 1 : 0 + name = "${var.ethereum_network}-tysm-firewall" + + apply_to { + label_selector = "tysm=${var.ethereum_network}" + } + + // TYSM control API (bad-tysm reaches back on this port) + rule { + description = "Allow TYSM control API port TCP" + direction = "in" + protocol = "tcp" + port = "8675" + source_ips = ["0.0.0.0/0", "::/0"] + } +} + resource "hcloud_firewall" "mev_relay_firewall" { count = contains(keys(hcloud_server.main), "mev-relay-1") ? 1 : 0 name = "${var.ethereum_network}-mev-relay-firewall" diff --git a/terraform/devnet-0/hetzner.tf b/terraform/devnet-0/hetzner.tf index 476bafa..7f840a6 100644 --- a/terraform/devnet-0/hetzner.tf +++ b/terraform/devnet-0/hetzner.tf @@ -96,6 +96,9 @@ locals { # Builder index for buildoor nodes (start index + instance offset); null = no tag builder_index = node.builder_start != null ? node.builder_start + i : null + # tysm canary membership; true = tysm label, [tysm] inventory group, port 8675 + tysm = node.tysm + # Supernode: explicit > bootnode/mev > validator_count >= 128 supernode = ( node.supernode != null ? node.supernode : @@ -137,6 +140,7 @@ locals { id = group.id group_key = group.group_name vm_key = vm_key + tysm = vm.tysm name = group.id ipv4_enabled = vm.ipv4_enabled @@ -159,7 +163,8 @@ locals { ], compact([ can(regex("bootnode", group.group_name)) ? "bootnode:${var.ethereum_network}" : null, can(regex("mev-relay", group.group_name)) ? "mev:${var.ethereum_network}" : null, - vm.builder_index != null ? "builder_index:${vm.builder_index}" : null + vm.builder_index != null ? "builder_index:${vm.builder_index}" : null, + vm.tysm ? "tysm:${var.ethereum_network}" : null ])) } ] diff --git a/terraform/devnet-0/main.tf b/terraform/devnet-0/main.tf index 0c5cf9f..5777b94 100644 --- a/terraform/devnet-0/main.tf +++ b/terraform/devnet-0/main.tf @@ -81,6 +81,7 @@ locals { validator_start = try(node.validator_start, 0) validator_end = try(node.validator_end, 0) builder_start = try(node.builder_start, null) + tysm = try(node.tysm, false) size = try(node.size, null) region = try(node.region, null) location = try(node.location, try(node.region, null)) diff --git a/terraform/devnet-0/nodes.tf b/terraform/devnet-0/nodes.tf index 340c889..6780408 100644 --- a/terraform/devnet-0/nodes.tf +++ b/terraform/devnet-0/nodes.tf @@ -16,11 +16,15 @@ # - supernode : Force supernode=true/false (auto-detected from name) # - builder_start : First builder index (buildoor nodes only). Exposes a # builder_index=N server tag and inventory var per instance. +# - tysm : true puts the instances in the [tysm] inventory group: the +# playbook deploys the tysm canary configs there and a +# group-scoped firewall opens the control port 8675. # # Examples: # { name = "bootnode", count = 1, cloud = "digitalocean" } # { name = "lighthouse-geth-super", count = 2, cloud = "hetzner", validator_start = 0, validator_end = 200 } # { name = "mev-relay", count = 1, cloud = "hetzner", size = "ccx53" } +# { name = "prysm-geth", count = 1, cloud = "hetzner", tysm = true } # ######################################################################################## diff --git a/terraform/devnet-0/outputs.tf b/terraform/devnet-0/outputs.tf index d252f29..dccb196 100644 --- a/terraform/devnet-0/outputs.tf +++ b/terraform/devnet-0/outputs.tf @@ -21,6 +21,7 @@ resource "local_file" "ansible_inventory" { supernode = try(title([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^supernode:", tag))][0]), "undefined") arch = try([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^arch:", tag))][0], "amd64") builder_index = try([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^builder_index:", tag))][0], "") + tysm = try([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^tysm:", tag))][0], "") tags = "${server.tags}" hostname = "${split(".", key)[0]}" cloud = "digitalocean" @@ -37,6 +38,7 @@ resource "local_file" "ansible_inventory" { supernode = server.labels.supernode arch = server.labels.arch builder_index = try(server.labels.builder_index, "") + tysm = try(server.labels.tysm, "") tags = server.labels hostname = split(".", key)[0] cloud = "hetzner"