feat: add metrics, health endpoint, and structured logging (#441)

DOCKER_QUICKSTART.md

@@ -146,6 +146,33 @@ docker logs -f lean-spec-node
 docker stop lean-spec-node
 ```
 
+### 8. Metrics & Observability
+
+Enable Prometheus metrics and health endpoint:
+
+```bash
+docker run --rm \
+  -v /path/to/genesis:/app/data:ro \
+  -p 9000:9000 \
+  -p 8008:8008 \
+  lean-spec:node \
+  --genesis /app/data/config.yaml \
+  --metrics-port 8008 \
+  --log-format json
+```
+
+You can now access:
+- Metrics: `curl http://localhost:8008/metrics`
+- Health: `curl http://localhost:8008/health`
+
+#### Full Observability Stack
+
+A complete setup with Prometheus and Grafana is available:
+
+```bash
+docker-compose -f docker-compose.observability.yml up
+```
+
 ## Using with lean-quickstart Genesis
 
 If you have the lean-quickstart repo with generated genesis:
@@ -180,6 +207,9 @@ docker run --rm \
 | `--checkpoint-sync-url URL` | URL for checkpoint sync (e.g., `http://host:5052`) | No |
 | `--validator-keys PATH` | Path to validator keys directory | No |
 | `--node-id ID` | Node identifier for validator assignment (default: `lean_spec_0`) | No |
+| `--metrics-port PORT` | Port to expose Prometheus metrics and health endpoint | No |
+| `--log-format {text,json}` | Log output format (default: `text`) | No |
+| `--log-level LEVEL` | Log level (default: `INFO`) | No |
 | `-v, --verbose` | Enable debug logging | No |
 
 Run `docker run lean-spec:node --help` to see all available options.

Dockerfile

@@ -118,6 +118,9 @@ RUN mkdir -p /app/data
 # Expose p2p port
 EXPOSE 9000
 
+# Expose metrics port
+EXPOSE 8008
+
 # Set entrypoint to lean_spec directly
 # Users can pass CLI arguments directly: docker run lean_spec --genesis /data/config.yaml --bootnode ...
 ENTRYPOINT ["uv", "run", "python", "-m", "lean_spec"]

docker-compose.observability.yml

@@ -0,0 +1,36 @@
+version: '3.8'
+
+services:
+  leanspec-node:
+    build:
+      context: .
+      target: node
+    ports:
+      - "9000:9000"
+      - "8008:8008"
+    command: >
+      --genesis /app/data/config.yaml
+      --metrics-port 8008
+      --log-format json
+    volumes:
+      - ./data:/app/data
+
+  prometheus:
+    image: prom/prometheus:latest
+    volumes:
+      - ./observability/prometheus.yml:/etc/prometheus/prometheus.yml
+    ports:
+      - "9090:9090"
+
+  grafana:
+    image: grafana/grafana:latest
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+    volumes:
+      - grafana-storage:/var/lib/grafana
+
+volumes:
+  grafana-storage:

observability/grafana/dashboard.json

@@ -0,0 +1,194 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "leanspec_slot_current",
+          "format": "time_series",
+          "instant": false,
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Current Slot",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "leanspec_peers_connected",
+          "format": "time_series",
+          "instant": false,
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Connected Peers",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "y": 8,
+        "x": 0
+      },
+      "id": 3,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "expr": "rate(leanspec_blocks_imported_total[5m])",
+          "legendFormat": "Blocks/sec",
+          "refId": "A"
+        }
+      ],
+      "title": "Block Import Rate",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "Prometheus",
+          "value": "prometheus"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrl": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "leanSpec Node Dashboard",
+  "uid": "leanspec-node",
+  "version": 1,
+  "weekStart": ""
+}

observability/prometheus.yml

@@ -0,0 +1,7 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: 'leanspec-node'
+    static_configs:
+      - targets: ['leanspec-node:8008']

pyproject.toml

@@ -36,6 +36,8 @@ dependencies = [
     "numpy>=2.0.0,<3",
     "aioquic>=1.2.0,<2",
     "pyyaml>=6.0.0,<7",
+    "prometheus-client>=0.21.0,<1",
+    "python-json-logger>=3.2.0,<4",
 ]
 
 [project.license]