diff --git a/.github/workflows/vm-tests.yaml b/.github/workflows/vm-tests.yaml new file mode 100644 index 0000000000..faf3bdf813 --- /dev/null +++ b/.github/workflows/vm-tests.yaml @@ -0,0 +1,30 @@ +name: vm tests +on: + pull_request: + paths: + - 'vm/**' + - 'qemu.pkr.hcl' + - 'hack/test-vm.sh' + - '.github/workflows/vm-tests.yaml' + workflow_dispatch: + +permissions: + contents: read + +# The Packer image build only runs from main after a release, so a broken collector config would otherwise be +# found an hour into a build. This validates the config with the exact collector version the image pins. +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Install shellcheck + run: sudo apt-get update -q && sudo apt-get install -y -q shellcheck + - name: Download the pinned otelcol-contrib + run: | + v=$(sed -n 's/^VERSION_OTELCOL_CONTRIB=//p' vm/observability.sh) + curl -fsSLo otelcol.tgz "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${v}/otelcol-contrib_${v}_linux_amd64.tar.gz" + tar -xzf otelcol.tgz otelcol-contrib && ./otelcol-contrib --version + - name: hack/test-vm.sh + run: OTELCOL_BIN=./otelcol-contrib ./hack/test-vm.sh diff --git a/README.md b/README.md index e215919f94..48cd9d1fbe 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,40 @@ The command lives at `.devcontainer/libexec/captain_utils/crds`, installed to `/ and deliberately off the PATH; its file header carries the full usage and the contract it is held to. Tests: `hack/test.sh` (offline, seconds) and `CRDS_TEST_KIND=1 hack/test.sh` for the cluster test. +## VM metrics: node exporter → OpenTelemetry Collector + +Every CDE VM built from this image carries [`prometheus-node-exporter`](https://packages.debian.org/trixie/prometheus-node-exporter) +(Debian package, loopback-only on `127.0.0.1:9100`) and the [OpenTelemetry Collector](https://github.com/open-telemetry/opentelemetry-collector-releases) +(`otelcol-contrib`), which scrapes it, reads per-container stats from the Docker socket, and ships everything as +OTLP/HTTP. Steady-state cost is ~22 MB for the exporter and ~190 MB / well under 1% of a core for the collector. + +**The image ships it inert.** `otelcol-contrib.service` has `ConditionPathExists=/etc/glueops/otel.env` and starts +only when that file exists; a VM without it — anything created before the Slack bot wrote one, or a hand-made VM — +leaves the unit inactive, not failed. Nothing per-VM is baked in: the endpoint and the VM's identity both come from +the file, which is written by [slackbot-developer-workspaces](https://github.com/GlueOps/slackbot-developer-workspaces) +through cloud-init, the same way it writes `cde_token` and `tunnel_endpoint`. + +`/etc/glueops/otel.env` is a systemd `EnvironmentFile` (root-only is fine — PID 1 reads it) and must contain: + +``` +OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-http-cde.observability.glueopshosted.com +OTEL_RESOURCE_ATTRIBUTES=cloud.region=,host.type=,host.image.name=,deployment.environment.name=,glueops.cde.owner= +``` + +The endpoint is the base URL (`/v1/metrics` is appended by the exporter). `host.name` is not needed — the collector +takes it from the hostname, which cloud-init sets to the server name — and it doubles as `service.instance.id`, so +node-exporter dashboards that key on `instance` see the VM name. Values in `OTEL_RESOURCE_ATTRIBUTES` may not +contain `,` or `=`. To turn reporting off on one VM, delete the file and `systemctl stop otelcol-contrib`; to point +one elsewhere, edit it and `systemctl restart otelcol-contrib`. + +Files live in `vm/`: `observability.sh` (the Packer step), `otelcol/config.yaml`, `otelcol/glueops.conf` (the +drop-in), `node-exporter/prometheus-node-exporter.default`. `hack/test-vm.sh` runs the offline checks and validates +the config against the pinned collector when one is available (CI downloads it); the installer's own smoke test in +the Packer build VM proves the units start, both receivers produce data, and the gate holds without the file. +Debugging on a VM: `systemctl status otelcol-contrib`, `journalctl -u otelcol-contrib`, and the collector's own +metrics at `curl 127.0.0.1:8888/metrics` (`otelcol_exporter_send_failed_metric_points` is the one to look at +when nothing arrives). + # Releasing: - Please stick to semver standards when dropping a new tag. - Once you publish a release a new image will be built and uploaded to GHCR.io: https://github.com/GlueOps/codespaces/pkgs/container/codespaces diff --git a/hack/test-vm.sh b/hack/test-vm.sh new file mode 100755 index 0000000000..c3896ee94f --- /dev/null +++ b/hack/test-vm.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Offline checks for the VM metrics pieces under vm/: shell syntax, shellcheck (when installed), the structural +# contract between the files, and — when a collector binary is available — `otelcol-contrib validate` of the config. +# +# The collector binary is not required: locally the validate step is skipped unless otelcol-contrib is on the PATH +# or OTELCOL_BIN points at one. CI downloads the pinned release so validate always runs there. +# The real end-to-end proof (units start, receivers produce data, the gate holds) is the smoke test inside +# vm/observability.sh, which runs in the Packer build VM. +set -e -u -o pipefail +cd "$(dirname "$0")/.." +fail=0 +ok() { echo " ok $*"; } +bad() { echo " FAIL $*"; fail=1; } +check(){ if eval "$2"; then ok "$1"; else bad "$1"; fi; } + +echo "== bash -n" +bash -n vm/observability.sh hack/test-vm.sh +if command -v shellcheck >/dev/null; then + echo "== shellcheck $(shellcheck --version | awk '/^version:/{print $2}')" + shellcheck -S warning vm/observability.sh hack/test-vm.sh +else + echo "== shellcheck: not installed, skipped" +fi + +echo "== contract" +# The version the installer downloads is the one CI validates against, and renovate can find it. +check "installer pins one renovate-annotated collector version" \ + '[ "$(grep -c "^VERSION_OTELCOL_CONTRIB=" vm/observability.sh)" = 1 ] && grep -B1 "^VERSION_OTELCOL_CONTRIB=" vm/observability.sh | grep -q "renovate: datasource=github-releases depName=open-telemetry/opentelemetry-collector-releases"' +# Inert-without-the-file is the back-compat contract with VMs the bot never configured. +check "drop-in gates the unit on /etc/glueops/otel.env" \ + 'grep -q "^ConditionPathExists=/etc/glueops/otel.env$" vm/otelcol/glueops.conf && grep -q "^EnvironmentFile=/etc/glueops/otel.env$" vm/otelcol/glueops.conf' +check "drop-in orders after docker and cloud-init config stage" \ + 'grep -q "^After=.*docker.service" vm/otelcol/glueops.conf && grep -q "^After=.*cloud-config.service" vm/otelcol/glueops.conf' +# Nothing per-VM may be baked into the image: the endpoint has to come from otel.env. +check "config takes the endpoint from the environment, not a literal" \ + 'grep -q "endpoint: \${env:OTEL_EXPORTER_OTLP_ENDPOINT}" vm/otelcol/config.yaml && ! grep -q "glueopshosted" vm/otelcol/config.yaml' +check "config has the memory_limiter backstop in the pipeline" \ + 'grep -q "processors: \[memory_limiter," vm/otelcol/config.yaml' +check "node exporter binds loopback only" \ + 'grep -q -- "--web.listen-address=127.0.0.1:9100" vm/node-exporter/prometheus-node-exporter.default' +check "installer stops the postinst instance and re-checks the gate" \ + 'grep -q "systemctl stop otelcol-contrib" vm/observability.sh && grep -q "ConditionPathExists gate is broken" vm/observability.sh' +check "packer uploads vm/ and runs the installer after developer-setup.sh" \ + 'grep -q "source *= *\"vm\"" qemu.pkr.hcl && grep -A3 "\"developer-setup.sh\"" qemu.pkr.hcl | grep -q "\"vm/observability.sh\""' + +echo "== otelcol-contrib validate" +bin=${OTELCOL_BIN:-$(command -v otelcol-contrib || true)} +if [ -n "$bin" ]; then + want=$(sed -n 's/^VERSION_OTELCOL_CONTRIB=//p' vm/observability.sh) + have=$("$bin" --version | awk '{print $NF}') + [ "$have" = "$want" ] && ok "binary is the pinned version $want" || echo " note validating with $have, image pins $want" + if OTEL_EXPORTER_OTLP_ENDPOINT=https://example.invalid "$bin" validate --config vm/otelcol/config.yaml; then + ok "config validates" + else + bad "config does not validate" + fi + # A config that validates without the endpoint would silently ship an image whose collector runs with none. + if "$bin" validate --config vm/otelcol/config.yaml >/dev/null 2>&1; then + bad "config validates without OTEL_EXPORTER_OTLP_ENDPOINT set" + else + ok "config refuses to start without OTEL_EXPORTER_OTLP_ENDPOINT" + fi +else + echo " skipped: no otelcol-contrib on PATH and OTELCOL_BIN unset" +fi + +[ "$fail" = 0 ] && echo "ALL PASS" || { echo "FAILED"; exit 1; } diff --git a/qemu.pkr.hcl b/qemu.pkr.hcl index 83f836052f..cd2248daaf 100644 --- a/qemu.pkr.hcl +++ b/qemu.pkr.hcl @@ -33,10 +33,18 @@ source "qemu" "qemu-amd64" { build { sources = ["source.qemu.qemu-amd64"] + # vm/ holds the files vm/observability.sh installs (collector config, systemd drop-in, node exporter defaults). + # Uploaded as a directory into /tmp, so the scripts below find them at /tmp/vm. + provisioner "file" { + source = "vm" + destination = "/tmp" + } + provisioner "shell" { scripts = [ "os-setup-start.sh", "developer-setup.sh", + "vm/observability.sh", "os-setup-finish.sh", ] environment_vars = [ diff --git a/vm/node-exporter/prometheus-node-exporter.default b/vm/node-exporter/prometheus-node-exporter.default new file mode 100644 index 0000000000..298f0db35d --- /dev/null +++ b/vm/node-exporter/prometheus-node-exporter.default @@ -0,0 +1,6 @@ +# Installed to /etc/default/prometheus-node-exporter by vm/observability.sh (read by Debian's unit as $ARGS). +# +# Loopback only: these VMs carry a public cloud address and a Tailscale one, and the only consumer is the +# collector on the same host. (The codespace container runs with --net=host, so it can still reach it.) +# The textfile directory is where a later change can drop platform gauges — e.g. which image tag the container is on. +ARGS="--web.listen-address=127.0.0.1:9100 --collector.textfile.directory=/var/lib/prometheus/node-exporter" diff --git a/vm/observability.sh b/vm/observability.sh new file mode 100755 index 0000000000..0fe4524e75 --- /dev/null +++ b/vm/observability.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# VM metrics for the CDE image: prometheus-node-exporter + the OpenTelemetry Collector that ships it. +# +# Runs as a Packer shell provisioner (see qemu.pkr.hcl) after developer-setup.sh, which it depends on for Docker +# and the `docker` group. Files under vm/ are uploaded to $VM_FILES_DIR (default /tmp/vm) by the file provisioner. +# +# What lands on the VM: +# prometheus-node-exporter Debian package, bound to 127.0.0.1:9100 (vm/node-exporter/*.default) +# otelcol-contrib upstream .deb, pinned + checksum-verified below (vm/otelcol/config.yaml) +# systemd drop-in makes the collector INERT until /etc/glueops/otel.env exists (vm/otelcol/glueops.conf) +# +# The Slack bot writes /etc/glueops/otel.env through cloud-init; nothing about the endpoint or the VM's identity is +# baked into the image. README, "VM metrics", documents that file's contract. +set -e -u -o pipefail + +VM_FILES_DIR=${VM_FILES_DIR:-/tmp/vm} +# renovate: datasource=github-releases depName=open-telemetry/opentelemetry-collector-releases +VERSION_OTELCOL_CONTRIB=0.159.0 + +echo "--> node exporter (Debian package)" +# --no-install-recommends: the package recommends prometheus-node-exporter-collectors, a set of apt/smartmon cron +# jobs that these VMs have no use for. +sudo apt-get install -y --no-install-recommends prometheus-node-exporter +sudo install -m 0644 "$VM_FILES_DIR/node-exporter/prometheus-node-exporter.default" /etc/default/prometheus-node-exporter +sudo systemctl enable prometheus-node-exporter +sudo systemctl restart prometheus-node-exporter + +echo "--> otelcol-contrib ${VERSION_OTELCOL_CONTRIB}" +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +base="https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${VERSION_OTELCOL_CONTRIB}" +deb="otelcol-contrib_${VERSION_OTELCOL_CONTRIB}_linux_amd64.deb" +curl -fsSL -o "$tmp/$deb" "$base/$deb" +curl -fsSL -o "$tmp/$deb.sha256" "$base/$deb.sha256" +# The upstream .sha256 asset is the bare digest; sha256sum -c wants "digest filename". +echo "$(cat "$tmp/$deb.sha256") $deb" | (cd "$tmp" && sha256sum -c -) +# postinst enables the unit and starts it with the stock config. The stock config is replaced right below and the +# unit re-evaluated, so that first start is a few seconds of a default pipeline that exports nothing anywhere. +sudo dpkg -i "$tmp/$deb" + +sudo install -m 0644 "$VM_FILES_DIR/otelcol/config.yaml" /etc/otelcol-contrib/config.yaml +sudo install -d -m 0755 /etc/systemd/system/otelcol-contrib.service.d +sudo install -m 0644 "$VM_FILES_DIR/otelcol/glueops.conf" /etc/systemd/system/otelcol-contrib.service.d/glueops.conf +# docker_stats reads /var/run/docker.sock. Membership in `docker` is root-equivalent on the host — acceptable on a +# single-tenant dev VM and the one privilege this setup grants; it is the same membership developer-setup.sh gives +# the vscode user. +sudo usermod -aG docker otelcol-contrib +sudo systemctl daemon-reload +sudo systemctl enable otelcol-contrib +# Stock-config instance from postinst: stop it. The drop-in's ConditionPathExists keeps it down from here on. +sudo systemctl stop otelcol-contrib + +echo "--> smoke test" +# The build VM is the only place before release with a real systemd, Docker and node exporter, so prove the +# pieces fit here rather than on a developer's first boot. The endpoint override points at a closed local port: +# nothing leaves the build VM, and the collector's own startup does not depend on the endpoint answering. +sudo env OTEL_EXPORTER_OTLP_ENDPOINT=https://example.invalid /usr/bin/otelcol-contrib validate --config /etc/otelcol-contrib/config.yaml +sudo install -d -m 0755 /etc/glueops +printf 'OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:1\nOTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=packer-build\n' \ + | sudo tee /etc/glueops/otel.env >/dev/null +sudo chmod 0600 /etc/glueops/otel.env +sudo systemctl start otelcol-contrib +sleep 8 +systemctl is-active prometheus-node-exporter otelcol-contrib +curl -fsS 127.0.0.1:9100/metrics | grep -q '^node_cpu_seconds_total' && echo "node exporter: serving node_* metrics" +# Both receivers have to have produced something: a docker_stats that failed to start would have killed the +# collector (is-active above), but a receiver that starts and silently yields nothing would not. +self=$(curl -fsS 127.0.0.1:8888/metrics) +grep -q 'otelcol_receiver_accepted_metric_points.*receiver="prometheus"' <<<"$self" && echo "collector: scraped node exporter" +grep -q 'otelcol_receiver_accepted_metric_points.*receiver="docker_stats"' <<<"$self" && echo "collector: read docker stats" +sudo systemctl stop otelcol-contrib +sudo rm -f /etc/glueops/otel.env +# Without the file the unit must stay down — that is the contract older VMs and hand-made VMs rely on. +if sudo systemctl start otelcol-contrib 2>/dev/null && systemctl is-active --quiet otelcol-contrib; then + echo "otelcol-contrib started without /etc/glueops/otel.env; the ConditionPathExists gate is broken" >&2 + exit 1 +fi +echo "collector: inert without /etc/glueops/otel.env" +echo "✅ VM metrics installed: node exporter ($(prometheus-node-exporter --version 2>&1 | head -1)), otelcol-contrib ${VERSION_OTELCOL_CONTRIB}" diff --git a/vm/otelcol/config.yaml b/vm/otelcol/config.yaml new file mode 100644 index 0000000000..8dc27df409 --- /dev/null +++ b/vm/otelcol/config.yaml @@ -0,0 +1,86 @@ +# OpenTelemetry Collector config for a CDE VM. Installed to /etc/otelcol-contrib/config.yaml by vm/observability.sh. +# +# Scrapes the local node exporter and the Docker daemon, stamps the VM's identity on the data, and ships it as +# OTLP/HTTP to the GlueOps observability endpoint. Nothing here is per-VM: every value that differs between VMs +# (the endpoint and the OTEL_RESOURCE_ATTRIBUTES identity) arrives through /etc/glueops/otel.env — see the +# "VM metrics" section of the README for that file's contract. The service does not start without it. +# +# Validate offline: OTEL_EXPORTER_OTLP_ENDPOINT=https://example.invalid otelcol-contrib validate --config vm/otelcol/config.yaml + +receivers: + prometheus: + config: + scrape_configs: + - job_name: node + scrape_interval: 30s + static_configs: + - targets: ["127.0.0.1:9100"] + + # Per-container CPU / memory / block I/O / network for everything on the host daemon: the codespace container and + # whatever the developer runs beside it (kind/k3d nodes show up here too). + docker_stats: + endpoint: unix:///var/run/docker.sock + # The receiver's default API version is rejected outright by current Docker ("client version 1.25 is too old"). + # A version below the daemon's minimum makes the receiver fail to start, which takes the whole collector down + # with it; the build-time smoke test in vm/observability.sh catches that before an image ships. + api_version: "1.44" + collection_interval: 30s + timeout: 20s + +processors: + # Backstop for an unreachable endpoint: without it the retry queue grows without bound. Sized for a fleet agent, + # not a gateway — steady state measured at ~190 MB RSS with both receivers, so 256 MiB is a ceiling, not a target. + memory_limiter: + check_interval: 5s + limit_mib: 256 + spike_limit_mib: 64 + + # env: reads OTEL_RESOURCE_ATTRIBUTES (cloud.region, host.type, owner, ... — written by the Slack bot). + # system: host.name from the VM hostname, which cloud-init sets to the server name. + resourcedetection: + detectors: [env, system] + system: + hostname_sources: [os] + + # Backends that ingest OTLP into a Prometheus-style store map service.name -> job and service.instance.id -> + # instance. The prometheus receiver would otherwise set instance to "127.0.0.1:9100" on every VM, which breaks + # every node-exporter dashboard that keys on it. Point it at the hostname instead, and give the docker_stats + # resources (which carry no service.name) a job of their own. + resource: + attributes: + - key: service.instance.id + from_attribute: host.name + action: upsert + - key: service.name + value: docker_stats + action: insert + + batch: + timeout: 10s + +exporters: + otlphttp: + endpoint: ${env:OTEL_EXPORTER_OTLP_ENDPOINT} + compression: gzip + retry_on_failure: + enabled: true + # 50 batches at one scrape per 30s is ~25 minutes of buffering; older data is dropped rather than kept forever. + sending_queue: + enabled: true + queue_size: 50 + +service: + telemetry: + metrics: + level: basic + readers: + - pull: + exporter: + prometheus: + host: 127.0.0.1 + port: 8888 + pipelines: + metrics: + receivers: [prometheus, docker_stats] + processors: [memory_limiter, resourcedetection, resource, batch] + exporters: [otlphttp] diff --git a/vm/otelcol/glueops.conf b/vm/otelcol/glueops.conf new file mode 100644 index 0000000000..a85325b3e3 --- /dev/null +++ b/vm/otelcol/glueops.conf @@ -0,0 +1,21 @@ +# systemd drop-in for otelcol-contrib.service on a CDE VM. +# Installed to /etc/systemd/system/otelcol-contrib.service.d/glueops.conf by vm/observability.sh. +# +# The unit ships INERT: it only starts when /etc/glueops/otel.env exists. The Slack bot writes that file through +# cloud-init on VMs it wants reporting (see the README's "VM metrics" section for the contract). A VM without the +# file — every VM created before the bot shipped it, or a hand-made one — leaves the unit inactive, not failed. + +[Unit] +ConditionPathExists=/etc/glueops/otel.env +# cloud-config.service: cloud-init's write_files (which lands otel.env) has run by then, so the condition above is +# evaluated against the file's real state on first boot rather than racing it. +# docker.service: the docker_stats receiver refuses to start without a reachable socket, and a receiver that fails +# to start takes the collector down with it. +After=network-online.target cloud-config.service docker.service +Wants=network-online.target + +[Service] +# EnvironmentFile is read by PID 1, so the file can stay root-only even though the collector runs unprivileged. +EnvironmentFile=/etc/glueops/otel.env +Restart=always +RestartSec=10s