(Draft) Bob L2 Image

DEVELOPMENT.md

@@ -27,6 +27,7 @@ flashboxes/
 │   └── debloat*.sh         # System cleanup scripts
 ├── bob-common/             # TEE Searcher common image
 ├── bob-l1/                 # L1 TEE Searcher sandbox image
+├── bob-l2/                 # L2 TEE Searcher sandbox image
 ├── buildernet/             # BuilderNet
 ├── tdx-dummy/              # TDX test environment
 ├── kernel/                 # Kernel configuration

Makefile

@@ -35,11 +35,11 @@ setup: ## Install dependencies (Linux only)
 
 # Build module
 build: setup ## Build the specified module
-	$(WRAPPER) mkosi --force --image-id $(IMAGE) -I $(IMAGE).conf
+	$(WRAPPER) mkosi --force --image-id $(IMAGE) --include=$(IMAGE).conf
 
 # Build module with devtools profile
 build-dev: setup ## Build module with development tools
-	$(WRAPPER) mkosi --force --image-id $(IMAGE)-dev --profile=devtools -I $(IMAGE).conf
+	$(WRAPPER) mkosi --force --image-id $(IMAGE)-dev --profile=devtools --include=$(IMAGE).conf
 
 ##@ Utilities
 

base/debloat-systemd.sh

@@ -16,6 +16,7 @@ systemd_svc_whitelist=(
     "systemd-journald-dev-log.socket"
     "systemd-remount-fs.service"
     "systemd-sysctl.service"
+    "chrony.service"
 )
 
 # Keep only essential systemd binaries
@@ -43,3 +44,8 @@ done
 
 # Set default target
 ln -sf minimal.target "$SYSTEMD_DIR/default.target"
+
+# Enable chrony and link to minimal.target
+mkdir -p "$BUILDROOT/etc/systemd/system/minimal.target.wants"
+mkosi-chroot systemctl enable chrony.service
+ln -sf /lib/systemd/system/chrony.service "$BUILDROOT/etc/systemd/system/minimal.target.wants/"

base/mkosi.conf

@@ -39,6 +39,7 @@ Packages=kmod
          iproute2
          udhcpc
          e2fsprogs
+         chrony
 BuildPackages=build-essential
               git
               curl

base/mkosi.skeleton/etc/chrony/chrony.conf

@@ -0,0 +1,18 @@
+# GCP metadata server - preferred when available
+server metadata.google.internal iburst prefer
+
+# Public NTP pools as fallback
+pool pool.ntp.org iburst
+
+# Store drift information
+driftfile /var/lib/chrony/chrony.drift
+
+# Step the system clock instead of slewing if adjustment is larger than
+# one second, but only in the first three clock updates
+makestep 1 3
+
+# Enable kernel synchronization of the real-time clock
+rtcsync
+
+# Stop bad estimates upsetting machine clock
+maxupdateskew 100.0

bob-l1/kernel.config → bob-common/kernel.config

@@ -1,3 +1,4 @@
+# Netfilter / nftables for iptables interface
 CONFIG_IPV6=n
 CONFIG_NETFILTER_NETLINK=y
 CONFIG_NETFILTER_NETLINK_LOG=y
@@ -34,6 +35,8 @@ CONFIG_IP_NF_TARGET_REDIRECT=y
 CONFIG_IP_NF_MANGLE=y
 CONFIG_IP_NF_RAW=y
 CONFIG_NET_SCHED=y
+
+# For tdx-init
 CONFIG_CRYPTO_USER_API_HASH=y
 CONFIG_CRYPTO_USER_API_SKCIPHER=y
 CONFIG_CRYPTO_USER_API_RNG=y

bob-common/mkosi.build

@@ -27,7 +27,7 @@ build_rust_package \
 # Build tdx-init
 make_git_package \
     "tdx-init" \
-    "v0.3.0" \
+    "v0.3.1" \
     "https://github.com/flashbots/tdx-init" \
     'go build -trimpath -ldflags "-s -w -buildid=" -o ./build/tdx-init' \
     "build/tdx-init:/usr/bin/tdx-init"

bob-common/mkosi.conf

@@ -1,4 +1,5 @@
 [Build]
+Environment=KERNEL_CONFIG_SNIPPETS=kernel/snippets/ubuntu.config KERNEL_CONFIG_SNIPPETS_BOB=bob-common/kernel.config
 WithNetwork=true
 
 [Content]

bob-common/mkosi.extra/usr/bin/init-container.sh

@@ -5,7 +5,6 @@ NAME=searcher-container
 
 # PORT FORWARDS
 SEARCHER_SSH_PORT=10022
-EL_P2P_PORT=30303
 SEARCHER_INPUT_CHANNEL=27017
 
 # Run extra commands which are customized per image,
@@ -21,8 +20,6 @@ su -s /bin/sh searcher -c "cd ~ && podman run -d \
     --name $NAME --replace \
     --init \
     -p ${SEARCHER_SSH_PORT}:22 \
-    -p ${EL_P2P_PORT}:${EL_P2P_PORT} \
-    -p ${EL_P2P_PORT}:${EL_P2P_PORT}/udp \
     -p ${SEARCHER_INPUT_CHANNEL}:${SEARCHER_INPUT_CHANNEL}/udp \
     -v /persistent/searcher:/persistent:rw \
     -v /etc/searcher/ssh_hostkey:/etc/searcher/ssh_hostkey:rw \

bob-common/mkosi.extra/usr/bin/init-firewall.sh

@@ -124,6 +124,18 @@ accept_dst_ip_port() {
                          -m comment --comment "$comment"
 }
 
+accept_src_ip_dst_port() {
+    chain="$1"
+    protocol="$2"
+    ip="$3"
+    port="$4"
+    comment="$5"
+
+    iptables -A "$chain" -p "$protocol" -s "$ip" --dport "$port" \
+                         -m conntrack --ctstate NEW -j ACCEPT \
+                         -m comment --comment "$comment"
+}
+
 drop_dst_ip() {
     chain="$1"
     ip="$2"

bob-common/mkosi.extra/usr/bin/toggle

@@ -45,21 +45,16 @@ CHAIN_MAINTENANCE_OUT="MAINTENANCE_OUT"
 CHAIN_PRODUCTION_IN="PRODUCTION_IN"
 CHAIN_PRODUCTION_OUT="PRODUCTION_OUT"
 
-# Titan builder IP
-TITAN_BUILDER_IP="52.207.17.217"
-
-# Flashbots IPs
-FLASHBOTS_TX_IP1="3.136.107.142"
-FLASHBOTS_TX_IP2="3.149.14.12"
-FLASHBOTS_BUNDLE_IP1="18.221.59.61"
-FLASHBOTS_BUNDLE_IP2="3.15.88.156"
-
-# Maintenance ports
-SSH_DATA_PORT=10022
-DNS_PORT=53
-HTTP_PORT=80
-HTTPS_PORT=443
-EL_P2P_PORT=30303
+# Source image-specific configuration if available
+TOGGLE_CONFIG="/etc/bob/toggle-config"
+if [ -f "$TOGGLE_CONFIG" ]; then
+    . "$TOGGLE_CONFIG"
+else
+    echo "Warning: No toggle-config found at $TOGGLE_CONFIG, using defaults"
+    IMAGE_TYPE="unknown"
+    PRODUCTION_ENDPOINTS=()
+    MAINTENANCE_ENDPOINTS=()
+fi
 
 ###############################################################################
 # Delay between stopping and starting maintenance
@@ -110,31 +105,47 @@ check_delay() {
 ###############################################################################
 # Kill established connections via conntrack
 ###############################################################################
+kill_production_connections() {
+    if [ ${#PRODUCTION_ENDPOINTS[@]} -eq 0 ]; then
+        echo "Killing leftover production flows:"
+        echo "     - No production endpoints configured"
+        return
+    fi
+
+    echo "Killing leftover production flows:"
+    for endpoint in "${PRODUCTION_ENDPOINTS[@]}"; do
+        local ip="${endpoint%%:*}"
+        local name="${endpoint#*:}"
+        echo "     - $name ($ip)"
+        $CONNTRACK -D -d "$ip" 2>/dev/null || true
+    done
+}
+
+kill_maintenance_connections() {
+    echo "Killing leftover maintenance flows:"
+    local displayed_ports=""
+    for endpoint in "${MAINTENANCE_ENDPOINTS[@]}"; do
+        local protocol="${endpoint%%:*}"
+        local rest="${endpoint#*:}"
+        local port="${rest%%:*}"
+        local name="${rest#*:}"
+
+        # Only display unique port descriptions
+        if [[ ! "$displayed_ports" =~ ":$port:" ]]; then
+            echo "     - $name ($port)"
+            displayed_ports="$displayed_ports:$port:"
+        fi
+
+        $CONNTRACK -D -p "$protocol" --dport "$port" 2>/dev/null || true
+    done
+}
+
 kill_established_connections() {
     local old_mode="$1"
     if [ "$old_mode" = "production" ]; then
-        echo "Killing leftover production flows:"
-        echo " - Titan state diff ($TITAN_BUILDER_IP)"
-        echo " - Flashbots protect tx ($FLASHBOTS_TX_IP1, $FLASHBOTS_TX_IP2)"
-        $CONNTRACK -D -d $TITAN_BUILDER_IP 2>/dev/null
-        $CONNTRACK -D -d $FLASHBOTS_TX_IP1 2>/dev/null
-        $CONNTRACK -D -d $FLASHBOTS_TX_IP2 2>/dev/null
-
+        kill_production_connections
     elif [ "$old_mode" = "maintenance" ]; then
-        echo "Killing leftover maintenance flows:"
-        echo " - SSH data port $SSH_DATA_PORT"
-        echo " - DNS $DNS_PORT, HTTP $HTTP_PORT, HTTPS $HTTPS_PORT, EL P2P $EL_P2P_PORT"
-        # SSH data plane
-        $CONNTRACK -D -p tcp --dport $SSH_DATA_PORT 2>/dev/null
-        # DNS
-        $CONNTRACK -D -p tcp --dport $DNS_PORT 2>/dev/null
-        $CONNTRACK -D -p udp --dport $DNS_PORT 2>/dev/null
-        # HTTP/HTTPS
-        $CONNTRACK -D -p tcp --dport $HTTP_PORT 2>/dev/null
-        $CONNTRACK -D -p tcp --dport $HTTPS_PORT 2>/dev/null
-        # EL P2P
-        $CONNTRACK -D -p tcp --dport $EL_P2P_PORT 2>/dev/null
-        $CONNTRACK -D -p udp --dport $EL_P2P_PORT 2>/dev/null
+        kill_maintenance_connections
     fi
     # if old_mode="stopped", nothing to kill
 }

bob-l1/mkosi.conf

@@ -1,5 +1,5 @@
 [Build]
-Environment=LIGHTHOUSE_BINARY KERNEL_CONFIG_SNIPPETS=kernel/snippets/ubuntu.config,bob-l1/kernel.config KERNEL_VERSION=6.13.12
+Environment=LIGHTHOUSE_BINARY
 WithNetwork=true
 
 [Content]

bob-l1/mkosi.extra/etc/bob/searcher-container-before-init

@@ -2,9 +2,12 @@
 # See also: bob-common/mkosi.extra/usr/bin/init-container.sh
 
 ENGINE_API_PORT=8551
+EL_P2P_PORT=30303
 
 BOB_SEARCHER_EXTRA_PODMAN_FLAGS="\
     -p ${ENGINE_API_PORT}:${ENGINE_API_PORT} \
+    -p ${EL_P2P_PORT}:${EL_P2P_PORT} \
+    -p ${EL_P2P_PORT}:${EL_P2P_PORT}/udp \
     -v /persistent/lighthouse_logs:/var/log/lighthouse:ro \
     -v /tmp/jwt.hex:/secrets/jwt.hex:ro \
 "

bob-l1/mkosi.extra/etc/bob/toggle-config

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Bob-L1 Toggle Configuration
+# This file is sourced by the common toggle script to provide image-specific behavior
+
+# Image identifier for logging
+IMAGE_TYPE="bob-l1"
+
+# Production endpoints (format: "IP:Description")
+PRODUCTION_ENDPOINTS=(
+    "52.207.17.217:Titan state diff"
+    "3.136.107.142:Flashbots protect tx"
+    "3.149.14.12:Flashbots protect tx"
+)
+
+# Maintenance ports (format: "protocol:port:description")
+MAINTENANCE_ENDPOINTS=(
+    "tcp:10022:SSH data port"
+    "tcp:53:DNS"
+    "udp:53:DNS"
+    "tcp:80:HTTP"
+    "tcp:443:HTTPS"
+    "tcp:30303:EL P2P"
+    "udp:30303:EL P2P"
+)

bob-l2.conf

@@ -0,0 +1,13 @@
+[Include]
+Include=base/mkosi.conf
+Include=bob-common/mkosi.conf
+Include=bob-l2/mkosi.conf
+
+[Config]
+Profiles=gcp
+
+[Distribution]
+Mirror=https://snapshot.debian.org/archive/debian/20251113T083151Z/
+
+[Build]
+ToolsTreeMirror=https://snapshot.debian.org/archive/debian/20251113T083151Z/

bob-l2/mkosi.conf

@@ -0,0 +1,7 @@
+[Build]
+WithNetwork=true
+
+[Content]
+ExtraTrees=bob-l2/mkosi.extra
+
+Packages=dmidecode

bob-l2/mkosi.extra/etc/bob/firewall-config

@@ -0,0 +1,87 @@
+# This script is sourced from firewall script and contains image-specific rules
+# See also: bob-common/mkosi.extra/usr/bin/init-firewall.sh
+
+# Image-specific ports
+SSH_CONTROL_PORT=22
+SSH_DATA_PORT=10022
+SSH_REGISTER_PORT=8080
+CVM_REVERSE_PROXY_PORT=8745
+SEARCHER_INPUT_PORT=27017
+
+# Well-known ports
+DNS_PORT=53
+HTTP_PORT=80
+HTTPS_PORT=443
+NTP_PORT=123
+OP_NODE_P2P_PORT=9222
+OP_GETH_P2P_PORT=40404
+ENGINE_API_PORT=8651
+
+# Simulator ports (production only)
+SIMULATOR_RPC_PORT=8645
+SIMULATOR_WS_PORT=8646
+
+###########################################################################
+# (1) ALWAYS_IN: Inbound rules that are always applied
+###########################################################################
+
+accept_dst_port $CHAIN_ALWAYS_IN tcp $SSH_CONTROL_PORT "SSH control port"
+accept_dst_port $CHAIN_ALWAYS_IN udp $SEARCHER_INPUT_PORT "Searcher input channel"
+
+# We drive op-geth in the searcher container from external op-node
+# We assume here that static peers in config are only syn nodes
+accept_src_ip_dst_port $CHAIN_ALWAYS_IN tcp "$CONFIG_EL_PEERS_IPS" $ENGINE_API_PORT "Engine API"
+
+# CVM reverse-proxy serves server attestation
+# Also forwards request to ssh pubkey server on localhost:5001,
+#    which serves searcher-container openssh server pubkey
+accept_dst_port $CHAIN_ALWAYS_IN tcp $CVM_REVERSE_PROXY_PORT "CVM reverse-proxy"
+
+###########################################################################
+# (2) ALWAYS_OUT: Outbound rules that are always applied
+###########################################################################
+
+# Note: this is accessible only from host, searcher netns has DROP on those
+# See also init-container.sh
+accept_dst_port $CHAIN_ALWAYS_OUT udp $NTP_PORT "NTP"
+
+###########################################################################
+# (3) MAINTENANCE_IN: Inbound rules for Maintenance Mode
+###########################################################################
+
+accept_dst_port $CHAIN_MAINTENANCE_IN tcp $SSH_DATA_PORT "SSH data plane"
+accept_dst_port $CHAIN_MAINTENANCE_IN tcp $SSH_REGISTER_PORT "SSH register service"
+
+accept_dst_port $CHAIN_MAINTENANCE_IN tcp $OP_GETH_P2P_PORT "op-geth P2P (TCP)"
+accept_dst_port $CHAIN_MAINTENANCE_IN udp $OP_GETH_P2P_PORT "op-geth P2P (UDP)"
+
+###########################################################################
+# (4) MAINTENANCE_OUT: Outbound rules for Maintenance Mode
+###########################################################################
+
+# Block ALL traffic to simulator during maintenance
+# Simulator endpoints (8645, 8646) are production-only
+drop_dst_ip $CHAIN_MAINTENANCE_OUT "$CONFIG_SIMULATOR_IP" "Simulator (blocked in maintenance)"
+
+accept_dst_port $CHAIN_MAINTENANCE_OUT udp $DNS_PORT "DNS (UDP)"
+accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $DNS_PORT "DNS (TCP)"
+
+accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $HTTP_PORT "HTTP"
+accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $HTTPS_PORT "HTTPS"
+
+accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $OP_GETH_P2P_PORT "op-geth P2P (TCP)"
+accept_dst_port $CHAIN_MAINTENANCE_OUT udp $OP_GETH_P2P_PORT "op-geth P2P (UDP)"
+
+###########################################################################
+# (5) PRODUCTION_IN: Inbound rules for Production Mode
+###########################################################################
+
+# None at the moment
+
+###########################################################################
+# (6) PRODUCTION_OUT: Outbound rules for Production Mode
+###########################################################################
+
+# Allow simulator endpoints in production mode
+accept_dst_ip_port $CHAIN_PRODUCTION_OUT tcp "$CONFIG_SIMULATOR_IP" $SIMULATOR_RPC_PORT "Simulator RPC (sendBackrun)"
+accept_dst_ip_port $CHAIN_PRODUCTION_OUT tcp "$CONFIG_SIMULATOR_IP" $SIMULATOR_WS_PORT "Simulator WebSocket (tx stream)"

bob-l2/mkosi.extra/etc/bob/searcher-container-after-init

@@ -0,0 +1,7 @@
+# This script is sourced from init-container.sh and contains image-specific stuff
+# See also: bob-common/mkosi.extra/usr/bin/init-container.sh
+
+exec_in_container "
+    cat <<EOF >> /etc/hosts
+$CONFIG_SIMULATOR_IP simulator.internal
+EOF"