Start one icons server per environment, and register it only once it answers

mise-tasks/services/realm-server

@@ -15,6 +15,10 @@ SCRIPTS_DIR="./scripts"
 # loki.source.file. Alloy's Docker discovery doesn't see native processes.
 LOG_TEE="../observability/scripts/dev-log-tee.sh"
 
+# Started here so that dev-all, which does not list services:icons, still has
+# an icons server. The stacks that do list it start one as well; start-icons.sh
+# steps aside when icons already answer for this environment, so either order
+# leaves one server registered.
 sh "$SCRIPTS_DIR/start-icons.sh" &
 ICONS_PID=$!
 cleanup_icons_server() {

packages/realm-server/scripts/start-icons.sh

@@ -4,37 +4,90 @@ SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
 . "$SCRIPTS_DIR/../../../scripts/env-slug.sh"
 
 if [ -n "$BOXEL_ENVIRONMENT" ]; then
-  # In environment mode, use port 0 (dynamic) and register with Traefik.
-  # http-server doesn't support port 0, so we pick a free port ourselves.
+  ENV_SLUG=$(resolve_env_slug)
+  # A file from the dist rather than `/`: a 200 for it means an icons server
+  # is answering, not merely something on the port.
+  PROBE_PATH="/@cardstack/boxel-icons/v1/icons/folder-pen.js"
+  answers() {
+    curl --fail --silent --max-time 2 "http://127.0.0.1:$1${PROBE_PATH}" >/dev/null 2>&1
+  }
+
+  # The route file lives in the directory the running Traefik container
+  # watches, which in a worktree can differ from this checkout's own
+  # traefik/dynamic (see scripts/start-traefik.sh). Resolved once, here, and
+  # handed to the registration below so the check and the write agree.
+  DYNAMIC_DIR=$(docker inspect boxel-traefik --format '{{range .Mounts}}{{if eq .Destination "/etc/traefik/dynamic"}}{{.Source}}{{end}}{{end}}' 2>/dev/null)
+  [ -n "$DYNAMIC_DIR" ] || DYNAMIC_DIR="$SCRIPTS_DIR/../../../traefik/dynamic"
+  CONFIG_PATH="$DYNAMIC_DIR/${ENV_SLUG}-icons.yml"
+
+  # More than one task starts this script for the same environment:
+  # services:icons on its own, and services:realm-server, which spawns it so
+  # that dev-all — which does not list services:icons — still gets an icons
+  # server. Each start registers its own port under this one route file, last
+  # writer winning, so a second instance that failed to come up would take the
+  # route away from a first that was serving. An instance that finds the
+  # registered server answering stands by instead. The check goes straight to
+  # the port the route names, the way Traefik reaches it, rather than through
+  # the hostname: that depends on nothing but the server being up — not on
+  # DNS, the mkcert trust, or Traefik having finished a reload.
+  #
+  # Standing by rather than exiting: the server just probed may belong to a
+  # stack that is on its way down — a restart overlaps the previous stack's
+  # shutdown — and a single probe cannot tell. This instance keeps probing and
+  # starts a server of its own the moment the registered one stops answering,
+  # so the environment never ends up with a route and nothing behind it.
+  if [ -f "$CONFIG_PATH" ]; then
+    REGISTERED_PORT=$(sed -n 's/.*host\.docker\.internal:\([0-9]*\).*/\1/p' "$CONFIG_PATH" | head -1)
+    if [ -n "$REGISTERED_PORT" ] && answers "$REGISTERED_PORT"; then
+      echo "icons already served for icons.${ENV_SLUG}.localhost on port ${REGISTERED_PORT}; standing by to replace it"
+      while answers "$REGISTERED_PORT"; do
+        sleep 5
+      done
+      echo "icons server on port ${REGISTERED_PORT} stopped answering; starting a replacement"
+    fi
+  fi
+
+  # http-server doesn't support port 0, so pick a free port here. The port is
+  # found by binding it and letting it go, so http-server's own bind can still
+  # fail; the wait below is what makes that safe.
   ICONS_PORT=$(node -e 'const s=require("net").createServer();s.listen(0,()=>{console.log(s.address().port);s.close();})')
   echo "Starting icons server on dynamic port ${ICONS_PORT}"
   cd "$(dirname "$0")/../../boxel-icons" && npx http-server --cors=Origin,X-Requested-With,Content-Type,Accept,Range,Authorization,X-Boxel-Assume-User --port "${ICONS_PORT}" dist &
   ICONS_PID=$!
 
+  # Register only a server that answers. http-server binds after this script
+  # has moved on, and a route written before it does would point Traefik at
+  # the port whether or not anything ever listens there; every icon request
+  # would then fail with a 502 that no service log records, and the host app
+  # cannot render a card without its icons.
+  tries=0
+  until answers "$ICONS_PORT"; do
+    if ! kill -0 "$ICONS_PID" 2>/dev/null; then
+      echo "icons server exited before it answered on port ${ICONS_PORT}; not registering it" >&2
+      exit 1
+    fi
+    tries=$((tries + 1))
+    if [ "$tries" -ge 60 ]; then
+      echo "icons server did not answer on port ${ICONS_PORT} within 30s; not registering it" >&2
+      kill "$ICONS_PID" 2>/dev/null
+      exit 1
+    fi
+    sleep 0.5
+  done
+
   # Register icons service with Traefik via a small node script.
   # Mirrors dev-service-registry.ts: a `websecure` router terminates TLS
   # at Traefik (mkcert leaf) and a sibling `-http` router on :80
   # 308-redirects to https. The host bundle is loaded over https, so an
   # `http://icons.<slug>.localhost/...` upstream would be mixed-content
   # blocked AND fail the CORS preflight on the redirect.
-  ENV_SLUG=$(resolve_env_slug)
-  node -e "
+  CONFIG_PATH="$CONFIG_PATH" node -e "
     const fs = require('fs');
-    const path = require('path');
-    const { execSync, spawn } = require('child_process');
-    let dir;
-    try {
-      const mounted = execSync(
-        \"docker inspect boxel-traefik --format '{{range .Mounts}}{{if eq .Destination \\\"/etc/traefik/dynamic\\\"}}{{.Source}}{{end}}{{end}}'\",
-        { encoding: 'utf-8' },
-      ).trim();
-      if (mounted) dir = mounted;
-    } catch {}
-    if (!dir) dir = path.resolve(__dirname, '..', '..', 'traefik', 'dynamic');
+    const { spawn } = require('child_process');
     const slug = '${ENV_SLUG}';
     const routerKey = 'icons-' + slug;
     const redirectMiddleware = routerKey + '-https-redirect';
-    const configPath = path.join(dir, slug + '-icons.yml');
+    const configPath = process.env.CONFIG_PATH;
     const entry = [
       'http:',
       '  routers:',
@@ -66,7 +119,6 @@ if [ -n "$BOXEL_ENVIRONMENT" ]; then
     const tmp = configPath + '.tmp';
     fs.writeFileSync(tmp, entry, 'utf-8');
     fs.renameSync(tmp, configPath);
-    console.log('Registered icons at icons.${ENV_SLUG}.localhost -> localhost:${ICONS_PORT}');
     // Bounce Traefik on macOS — Docker Desktop's bind mounts don't
     // propagate inotify, and Traefik v3 file provider has no polling
     // option. See dev-service-registry.ts for the full rationale.
@@ -78,8 +130,20 @@ if [ -n "$BOXEL_ENVIRONMENT" ]; then
       child.unref();
     }
   "
+  echo "Registered icons at icons.${ENV_SLUG}.localhost -> localhost:${ICONS_PORT}"
 
+  # The route is right only while this server is up. When it exits, take the
+  # route with it — if it still names this port — so Traefik answers 404 for a
+  # missing route rather than 502 for a dead upstream, and the next start of
+  # any icons task registers afresh instead of stepping aside for a route that
+  # only looks live.
   wait $ICONS_PID
+  ICONS_STATUS=$?
+  if grep -qF "host.docker.internal:${ICONS_PORT}\"" "$CONFIG_PATH" 2>/dev/null; then
+    rm -f "$CONFIG_PATH"
+    echo "icons server on port ${ICONS_PORT} exited (status ${ICONS_STATUS}); removed its route" >&2
+  fi
+  exit $ICONS_STATUS
 else
   if curl --fail --silent --show-error http://localhost:4206 >/dev/null 2>&1; then
     echo "icons server already running on http://localhost:4206, skipping startup"