playbook: deploy tysm canary configs from templates

ansible/inventories/devnet-0/templates/tysm/tysm-hooks.yaml.j2

@@ -0,0 +1,88 @@
+# tysm hook-control config for {{ inventory_hostname }}. ${TYSM_TOKEN} resolves from the
+# beacon container env at load; it must equal the token bad-tysm validates with.
+api:
+  enabled: true
+  listen: "0.0.0.0:8675"
+  auth_token: "${TYSM_TOKEN}"
+  max_activation_duration: "4h"
+  instance_id: "{{ inventory_hostname }}"
+
+discovery:
+  enabled: true
+  bad_tysm_url: "https://bad-tysm.analytics.production.platform.ethpandaops.io"
+  auth_token: "${TYSM_TOKEN}"
+  # Address bad-tysm dials back to reach this node's control API. Terraform opens
+  # 8675 for tysm-group members; the beacon container must still publish the port.
+  self_url: "http://{{ ansible_host }}:8675"
+  name: "{{ inventory_hostname }}"
+  namespace: "{{ ethereum_network_name }}"
+  pod: "{{ inventory_hostname }}"
+  node_name: "{{ inventory_hostname }}"
+  heartbeat_interval: "30s"
+  labels:
+    network: "{{ ethereum_network_name }}"
+    cl: "{{ ethereum_node_cl }}"
+    el: "{{ ethereum_node_el }}"
+
+hook_logging:
+  enabled: true
+  classes: ["validate", "mutate"]
+
+hooks:
+  - name: epbs-mutator
+    enabled: false
+    config:
+      mutationDelayMs: 0
+      logMutationDetails: true
+      topics:
+        execution_payload_bid:
+          enabledStrategies: []
+          mutationProbability: 0.0
+          slotPattern: "*"
+          strategies:
+            drop:
+              applyProbability: 1.0
+            value_inflate:
+              mode: "uint64_max"
+              absolute: 0
+              multiplier: 10.0
+        execution_payload_envelope:
+          enabledStrategies: []
+          mutationProbability: 0.0
+          slotPattern: "*"
+          strategies:
+            withhold:
+              probability: 1.0
+              slotPattern: "*"
+            delay:
+              delayMs: 9500
+              jitterMs: 0
+              applyProbability: 1.0
+            selective_drop:
+              targetPeerRole: ""
+              dropRate: 0.4
+              seed: 42
+            free_option:
+              triggerSource: "unix_socket"
+              socketPath: "/tmp/tysm-mev-trigger.sock"
+              httpURL: ""
+              defaultState: false
+              pollIntervalMs: 100
+        payload_attestation_message:
+          enabledStrategies: []
+          mutationProbability: 0.0
+          slotPattern: "*"
+          strategies:
+            flip_bit:
+              field: "payload_present"
+              targetValue: "false"
+            corrupt_field:
+              field: "beacon_block_root"
+              mode: "random_32"
+            drop:
+              applyProbability: 1.0
+              targetPeerRole: ""
+              targetValidators: []
+            delay:
+              delayMs: 12000
+              applyProbability: 1.0

ansible/inventories/devnet-0/templates/tysm/tysm-xatu-config.yaml.j2

@@ -0,0 +1,16 @@
+# Minimal Xatu config: every tysm beacon requires --xatu-config-file to start.
+# The xatu hook is disabled and this output points at a local no-op address that
+# is never required to connect; devnet events still ship via the xatu-sentry sidecar.
+name: "{{ inventory_hostname }}"
+
+ethereum:
+  network: "{{ ethereum_network_name }}"
+
+outputs:
+  - name: noop
+    type: xatu
+    config:
+      address: localhost:19999
+      tls: false
+      maxQueueSize: 1000
+      batchTimeout: 5s

ansible/playbook.yaml

@@ -85,6 +85,38 @@
     - role: ethpandaops.general.generate_basic_auth_nginx
       tags: [docker_nginx_proxy]
 
+# Hosts in the tysm group (tysm = true on the node entry in terraform nodes.tf)
+# run the tysm beacon canary and need the hook + xatu config files in the prysm
+# datadir before the container starts. The group is empty by default.
+- hosts: tysm
+  become: true
+  tasks:
+    - name: Deploy tysm config files
+      tags: [ethereum, ethereum_node, tysm]
+      block:
+        - name: Add prysm user
+          ansible.builtin.user:
+            name: "{{ prysm_user | default('prysm') }}"
+
+        - name: Create prysm data dir
+          ansible.builtin.file:
+            path: "{{ prysm_datadir | default('/data/prysm') }}"
+            state: directory
+            mode: "0750"
+            owner: "{{ prysm_user | default('prysm') }}"
+            group: "{{ prysm_user | default('prysm') }}"
+
+        - name: Template tysm config files
+          ansible.builtin.template:
+            src: "{{ inventory_dir }}/templates/tysm/{{ item }}.j2"
+            dest: "{{ prysm_datadir | default('/data/prysm') }}/{{ item }}"
+            mode: "0640"
+            owner: "{{ prysm_user | default('prysm') }}"
+            group: "{{ prysm_user | default('prysm') }}"
+          loop:
+            - tysm-hooks.yaml
+            - tysm-xatu-config.yaml
+
 - hosts: ethereum_node
   become: true
   roles:

terraform/devnet-0/ansible_inventory.tmpl

@@ -84,6 +84,16 @@ ${replace(gid, "-", "_")}
 [buildoor:vars]
 ansible_group_priority=100
 
+# tysm canary hosts (tysm = true on the node entry in nodes.tf); the group always
+# exists so the tysm play in the playbook resolves, it is just empty by default.
+
+[tysm]
+%{ for key, host in hosts ~}
+%{ if host.tysm != "" ~}
+${host.hostname}
+%{ endif ~}
+%{ endfor ~}
+
 # Global groups
 
 [consensus_node:children]

terraform/devnet-0/digitalocean.tf

@@ -68,6 +68,9 @@ locals {
             # Builder index for buildoor nodes (start index + instance offset); null = no tag
             builder_index = node.builder_start != null ? node.builder_start + i : null
 
+            # tysm canary membership; true = tysm tag, [tysm] inventory group, port 8675
+            tysm = node.tysm
+
             # Supernode: explicit > bootnode/mev > validator_count >= 128
             supernode = (
               node.supernode != null ? node.supernode :
@@ -100,6 +103,7 @@ locals {
         id        = group.id
         group_key = group.group_name
         vm_key    = vm_key
+        tysm      = vm.tysm
 
         name        = group.id
         ssh_keys    = [data.digitalocean_ssh_key.main.fingerprint]
@@ -121,7 +125,8 @@ locals {
           ], compact([
             can(regex("bootnode", group.group_name)) ? "bootnode:${var.ethereum_network}" : null,
             can(regex("mev-relay", group.group_name)) ? "mev-relay:${var.ethereum_network}" : null,
-            vm.builder_index != null ? "builder_index:${vm.builder_index}" : null
+            vm.builder_index != null ? "builder_index:${vm.builder_index}" : null,
+            vm.tysm ? "tysm:${var.ethereum_network}" : null
         ]))
       }
     ]

terraform/devnet-0/firewall.tf

@@ -133,6 +133,20 @@ resource "digitalocean_firewall" "bootnode" {
   depends_on = [digitalocean_project_resources.droplets]
 }
 
+resource "digitalocean_firewall" "tysm" {
+  count = length([for vm in local.digitalocean_vms : vm.id if vm.tysm]) > 0 ? 1 : 0
+  name  = "${var.ethereum_network}-nodes-tysm"
+  tags  = ["tysm:${var.ethereum_network}"]
+
+  // TYSM control API (bad-tysm reaches back on this port)
+  inbound_rule {
+    protocol         = "tcp"
+    port_range       = "8675"
+    source_addresses = ["0.0.0.0/0", "::/0"]
+  }
+  depends_on = [digitalocean_project_resources.droplets]
+}
+
 resource "digitalocean_firewall" "mev_relay" {
   count = contains(keys(digitalocean_droplet.main), "mev-relay-1") ? 1 : 0
   name  = "${var.ethereum_network}-nodes-mev-relay"
@@ -326,6 +340,24 @@ resource "hcloud_firewall" "bootnode_firewall" {
   }
 }
 
+resource "hcloud_firewall" "tysm_firewall" {
+  count = length([for vm in local.hcloud_vms : vm.id if vm.tysm]) > 0 ? 1 : 0
+  name  = "${var.ethereum_network}-tysm-firewall"
+
+  apply_to {
+    label_selector = "tysm=${var.ethereum_network}"
+  }
+
+  // TYSM control API (bad-tysm reaches back on this port)
+  rule {
+    description = "Allow TYSM control API port TCP"
+    direction   = "in"
+    protocol    = "tcp"
+    port        = "8675"
+    source_ips  = ["0.0.0.0/0", "::/0"]
+  }
+}
+
 resource "hcloud_firewall" "mev_relay_firewall" {
   count = contains(keys(hcloud_server.main), "mev-relay-1") ? 1 : 0
   name  = "${var.ethereum_network}-mev-relay-firewall"

terraform/devnet-0/hetzner.tf

@@ -96,6 +96,9 @@ locals {
             # Builder index for buildoor nodes (start index + instance offset); null = no tag
             builder_index = node.builder_start != null ? node.builder_start + i : null
 
+            # tysm canary membership; true = tysm label, [tysm] inventory group, port 8675
+            tysm = node.tysm
+
             # Supernode: explicit > bootnode/mev > validator_count >= 128
             supernode = (
               node.supernode != null ? node.supernode :
@@ -137,6 +140,7 @@ locals {
         id        = group.id
         group_key = group.group_name
         vm_key    = vm_key
+        tysm      = vm.tysm
 
         name         = group.id
         ipv4_enabled = vm.ipv4_enabled
@@ -159,7 +163,8 @@ locals {
           ], compact([
             can(regex("bootnode", group.group_name)) ? "bootnode:${var.ethereum_network}" : null,
             can(regex("mev-relay", group.group_name)) ? "mev:${var.ethereum_network}" : null,
-            vm.builder_index != null ? "builder_index:${vm.builder_index}" : null
+            vm.builder_index != null ? "builder_index:${vm.builder_index}" : null,
+            vm.tysm ? "tysm:${var.ethereum_network}" : null
         ]))
       }
     ]

terraform/devnet-0/main.tf

@@ -81,6 +81,7 @@ locals {
       validator_start = try(node.validator_start, 0)
       validator_end   = try(node.validator_end, 0)
       builder_start   = try(node.builder_start, null)
+      tysm            = try(node.tysm, false)
       size            = try(node.size, null)
       region          = try(node.region, null)
       location        = try(node.location, try(node.region, null))

terraform/devnet-0/nodes.tf

@@ -16,11 +16,15 @@
 #     - supernode       : Force supernode=true/false (auto-detected from name)
 #     - builder_start   : First builder index (buildoor nodes only). Exposes a
 #                         builder_index=N server tag and inventory var per instance.
+#     - tysm            : true puts the instances in the [tysm] inventory group: the
+#                         playbook deploys the tysm canary configs there and a
+#                         group-scoped firewall opens the control port 8675.
 #
 # Examples:
 #   { name = "bootnode", count = 1, cloud = "digitalocean" }
 #   { name = "lighthouse-geth-super", count = 2, cloud = "hetzner", validator_start = 0, validator_end = 200 }
 #   { name = "mev-relay", count = 1, cloud = "hetzner", size = "ccx53" }
+#   { name = "prysm-geth", count = 1, cloud = "hetzner", tysm = true }
 #
 ########################################################################################
 

terraform/devnet-0/outputs.tf

@@ -21,6 +21,7 @@ resource "local_file" "ansible_inventory" {
             supernode       = try(title([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^supernode:", tag))][0]), "undefined")
             arch            = try([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^arch:", tag))][0], "amd64")
             builder_index   = try([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^builder_index:", tag))][0], "")
+            tysm            = try([for tag in tolist(server.tags) : split(":", tag)[1] if can(regex("^tysm:", tag))][0], "")
             tags            = "${server.tags}"
             hostname        = "${split(".", key)[0]}"
             cloud           = "digitalocean"
@@ -37,6 +38,7 @@ resource "local_file" "ansible_inventory" {
             supernode       = server.labels.supernode
             arch            = server.labels.arch
             builder_index   = try(server.labels.builder_index, "")
+            tysm            = try(server.labels.tysm, "")
             tags            = server.labels
             hostname        = split(".", key)[0]
             cloud           = "hetzner"