Service last-access accounting

agent/.fpm

@@ -9,6 +9,9 @@
 --depends nodejs
 --depends nginx
 --depends libnginx-mod-stream
+--depends libnginx-mod-http-js
+--depends libnginx-mod-stream-js
+--depends ca-certificates
 --depends ssl-cert
 --depends dnsmasq
 --deb-no-default-config-files

agent/Makefile

@@ -21,7 +21,7 @@ help:
 	@echo "Targets:"
 	@echo "  deps     install dependencies (npm ci)"
 	@echo "  build    compile the TypeScript sources"
-	@echo "  test     run tests (none yet)"
+	@echo "  test     run tests"
 	@echo "  install  stage files into DESTDIR (default /)"
 	@echo "  deb      build the .deb package"
 	@echo "  rpm      build the .rpm package"
@@ -42,13 +42,13 @@ build: deps
 
 dev: build
 
-# No test suite yet; kept as a no-op so the repo-wide `make test` passes.
-test:
+test: deps
+	npm test
 
 install: build
 	$(INSTALL) -d $(DESTBIN)
 	$(INSTALL_DATA) package.json $(DESTBIN)/
-	cp -a dist templates node_modules $(DESTBIN)/
+	cp -a dist templates njs node_modules $(DESTBIN)/
 	$(INSTALL) -d $(UNIT_DIR)
 	$(INSTALL_DATA) contrib/systemd/opensource-agent.service $(UNIT_DIR)/
 	$(INSTALL_DATA) contrib/systemd/opensource-agent.timer $(UNIT_DIR)/

agent/njs/accounting.js

@@ -0,0 +1,111 @@
+/**
+ * Service last-access accounting for the opensource-server proxy.
+ *
+ * Loaded by nginx's js module (njs) in both the http and stream contexts.
+ * Each proxied request/connection checks a shared dict whose entries expire
+ * after 10 minutes (the zone's timeout=): a successful add() means no
+ * report was sent in the current window, so this handler claims it —
+ * atomically, across all workers — and POSTs to the manager, which stamps
+ * the service's lastAccessedAt with its own clock. Everything is fail-open:
+ * accounting can never affect the proxied request.
+ *
+ * Injected via nginx variables (see templates/nginx.conf.ejs):
+ *   $osaas_service_id   Services.id for this vhost / stream server
+ *   $osaas_manager_url  http-context: manager base URL, no trailing slash
+ *   $osaas_api_key      http-context: admin API key; empty for the manager's
+ *                       own agent, which reports over localhost without
+ *                       credentials
+ *   $osaas_relay_url    stream-context: localhost relay base URL — a unix
+ *                       socket, so it carries the njs form
+ *                       "http://unix:/path/to.sock:" (trailing colon
+ *                       separates the socket path from the request URI)
+ */
+
+// `host` is only needed for unix-socket targets: njs would otherwise derive a
+// Host header from the socket path ("host: /run/....sock:0"), which nginx
+// rejects with 400. TCP targets pass it undefined and njs sets Host itself.
+function report(base, apiKey, serviceId, host) {
+  const headers = {};
+  if (apiKey) {
+    headers.Authorization = 'Bearer ' + apiKey;
+  }
+  if (host) {
+    headers.Host = host;
+  }
+  return ngx.fetch(base + '/api/v1/services/' + serviceId + '/last-access', {
+    method: 'POST',
+    headers,
+  });
+}
+
+/**
+ * http handler (js_content in the internal mirror location). The mirror
+ * subrequest runs in parallel with proxy_pass; awaiting the fetch keeps the
+ * subrequest — never the client response — alive until the report lands.
+ */
+async function http_record(r) {
+  try {
+    const id = r.variables.osaas_service_id;
+    if (id && ngx.shared.osaas_http.add(id, '1')) {
+      const reply = await report(r.variables.osaas_manager_url, r.variables.osaas_api_key, id);
+      if (reply.status !== 204) {
+        r.warn('osaas accounting: manager returned ' + reply.status + ' for service ' + id);
+      }
+    }
+  } catch (e) {
+    r.warn('osaas accounting: ' + e.message);
+  }
+  r.return(204);
+}
+
+/**
+ * stream handler (js_access). Reports go via the localhost http relay (a unix
+ * socket) — Debian's njs stream module has no fetch-TLS — and the fetch is
+ * deliberately not awaited: the connection proceeds immediately. The explicit
+ * Host is required for the socket fetch (see report()).
+ */
+function stream_record(s) {
+  try {
+    const id = s.variables.osaas_service_id;
+    if (id && ngx.shared.osaas_stream.add(id, '1')) {
+      report(s.variables.osaas_relay_url, '', id, 'localhost')
+        .then((reply) => {
+          if (reply.status !== 204) {
+            s.warn('osaas accounting: relay returned ' + reply.status + ' for service ' + id);
+          }
+        })
+        .catch((e) => {
+          s.warn('osaas accounting: ' + e.message);
+        });
+    }
+  } catch (e) {
+    s.warn('osaas accounting: ' + e.message);
+  }
+  s.allow();
+}
+
+/**
+ * http relay for stream-context reports (js_content on the localhost-only
+ * relay server). Debian's njs stream module is built without NGX_STREAM_SSL,
+ * so stream_record cannot fetch an https manager directly; it POSTs to this
+ * relay over plain local http and the http js VM — full fetch-TLS — forwards
+ * to the manager. The path shape is validated so the relay can never be used
+ * to reach any other manager endpoint with the embedded credential.
+ */
+async function relay(r) {
+  try {
+    const m = r.uri.match(/^\/api\/v1\/services\/(\d+)\/last-access$/);
+    if (!m) {
+      r.return(404);
+      return;
+    }
+    const reply = await report(r.variables.osaas_manager_url, r.variables.osaas_api_key, m[1]);
+    r.return(reply.status);
+    return;
+  } catch (e) {
+    r.warn('osaas accounting relay: ' + e.message);
+  }
+  r.return(502);
+}
+
+export default { http_record, stream_record, relay };

agent/package.json

@@ -6,7 +6,8 @@
   "license": "Apache-2.0",
   "scripts": {
     "build": "tsc",
-    "start": "node dist/index.js"
+    "start": "node dist/index.js",
+    "test": "node --test \"test/**/*.test.js\""
   },
   "dependencies": {
     "@particle/dbus-next": "^0.11.4",

agent/src/apply.ts

@@ -12,21 +12,24 @@ import { execFileSync } from 'child_process';
 import ejs from 'ejs';
 import { reloadOrRestartService, restartService, sighupService } from './system';
 import { log, commandOutput } from './log';
+import type { AgentConfig } from './config';
 import type { ApplyResult, SiteConfig } from './types';
 
 const TEMPLATES_DIR = path.join(__dirname, '..', 'templates');
 
 interface RenderedFile {
   dest: string;
   content: string;
+  /** File mode for atomic writes (e.g. 0o600 for configs holding secrets). */
+  mode?: number;
 }
 
 export interface ManagedService {
   /** systemd unit name (also the key reported at check-in). */
   unit: string;
   /** Render all managed files. Returns null when there is nothing to manage
    * yet (e.g. dnsmasq before the site exists). */
-  render(config: SiteConfig): Promise<RenderedFile[] | null>;
+  render(config: SiteConfig, agent: AgentConfig): Promise<RenderedFile[] | null>;
   /** Command that validates the staged config before it is kept. */
   test?: string[];
   /** Reload/restart after a successful apply. */
@@ -48,10 +51,19 @@ function run(cmd: string[]): string {
 export const services: ManagedService[] = [
   {
     unit: 'nginx',
-    async render(config) {
+    async render(config, agent) {
       return [{
         dest: '/etc/nginx/nginx.conf',
-        content: await renderTemplate('nginx.conf.ejs', config.nginx),
+        content: await renderTemplate('nginx.conf.ejs', {
+          ...config.nginx,
+          accounting: {
+            managerUrl: agent.managerUrl,
+            apiKey: agent.apiKey ?? '',
+          },
+        }),
+        // The rendered config embeds the manager API key: root-only. nginx's
+        // master process reads the config as root before forking workers.
+        mode: 0o600,
       }];
     },
     test: ['nginx', '-t'],
@@ -100,15 +112,19 @@ function readIfExists(file: string): string | null {
 
 // Write via temp file + rename so a crash mid-write can never leave a
 // truncated config on disk.
-function writeFileAtomic(dest: string, content: string): void {
+function writeFileAtomic(dest: string, content: string, mode?: number): void {
   const tmp = `${dest}.tmp-${process.pid}`;
-  fs.writeFileSync(tmp, content);
+  fs.writeFileSync(tmp, content, mode !== undefined ? { mode } : {});
   fs.renameSync(tmp, dest);
 }
 
-export async function applyService(svc: ManagedService, config: SiteConfig): Promise<ApplyResult> {
+export async function applyService(
+  svc: ManagedService,
+  config: SiteConfig,
+  agent: AgentConfig,
+): Promise<ApplyResult> {
   log.debug(`${svc.unit}: rendering config`);
-  const files = await svc.render(config);
+  const files = await svc.render(config, agent);
   if (!files) {
     log.debug(`${svc.unit}: nothing to manage yet, skipping`);
     return 'success';
@@ -123,17 +139,18 @@ export async function applyService(svc: ManagedService, config: SiteConfig): Pro
 
   log.info(`${svc.unit}: ${changed.length} file(s) changed, applying: ${changed.join(', ')}`);
 
+  const modeByDest = new Map(files.map((f) => [f.dest, f.mode]));
   // Stage the new files (previous contents kept in memory for rollback).
   for (const f of files) {
     fs.mkdirSync(path.dirname(f.dest), { recursive: true });
-    writeFileAtomic(f.dest, f.content);
+    writeFileAtomic(f.dest, f.content, f.mode);
     log.debug(`${svc.unit}: wrote ${f.dest}`);
   }
 
   const rollback = () => {
     for (const [dest, prev] of current) {
       if (prev === null) fs.rmSync(dest, { force: true });
-      else writeFileAtomic(dest, prev);
+      else writeFileAtomic(dest, prev, modeByDest.get(dest));
     }
     log.debug(`${svc.unit}: rolled back to previous config`);
   };

agent/src/index.ts

@@ -56,7 +56,7 @@ async function main(): Promise<void> {
 
     log.info(`check-in: new config received (etag=${result.etag ?? '(none)'}), applying`);
     for (const svc of services) {
-      state.lastApply[svc.unit] = await applyService(svc, result.config);
+      state.lastApply[svc.unit] = await applyService(svc, result.config, cfg);
     }
 
     // The ETag is saved even after a failed apply: a rejected config won't

agent/src/types.ts

@@ -48,6 +48,8 @@ export interface SiteInfo {
 }
 
 export interface HttpService {
+  /** Manager Services.id — reported back by the accounting module. */
+  id: number;
   internalPort: number;
   container: { ipv4Address: string };
   externalHostname: string;
@@ -57,6 +59,8 @@ export interface HttpService {
 }
 
 export interface StreamService {
+  /** Manager Services.id — reported back by the accounting module. */
+  id: number;
   internalPort: number;
   container: { ipv4Address: string };
   externalPort: number;

agent/templates/dnsmasq/conf.ejs

@@ -5,6 +5,13 @@ dhcp-authoritative
 domain-needed
 bogus-priv
 
+# Resolve localhost to loopback (RFC 6761). no-hosts above makes dnsmasq
+# ignore /etc/hosts, so without this "localhost" does not resolve — which
+# breaks nginx's resolver-based lookups (e.g. the njs accounting module's
+# ngx.fetch to the manager's own http://localhost:3000). Covers *.localhost.
+address=/localhost/127.0.0.1
+address=/localhost/::1
+
 # internal domain for the site
 domain=<%= site.internalDomain %>
 

agent/templates/nginx.conf.ejs

@@ -43,6 +43,26 @@ http {
   proxy_cache_path /var/cache/nginx/auth_cache levels=1:2 keys_zone=auth_cache:1m
                    max_size=10m inactive=5m;
 
+  <%_ /*
+    Service last-access accounting (docs/superpowers/specs/
+    2026-08-11-service-last-access-accounting-design.md). The static njs
+    module claims a 10-minute window per service in the shared dict (entries
+    expire via timeout=; the shm zone survives reloads) and reports the first
+    access of each window to the manager from a parallel mirror subrequest.
+    resolver / js_fetch_timeout / js_fetch_trusted_certificate are set here at
+    http{} level so they inherit into both the mirror location and the relay
+    server below — no per-location copies needed.
+  */ -%>
+  js_import accounting from /opt/opensource-server/agent/njs/accounting.js;
+  js_shared_dict_zone zone=osaas_http:256k timeout=10m evict;
+  js_var $osaas_manager_url "<%= accounting.managerUrl %>";
+  js_var $osaas_api_key "<%= accounting.apiKey %>";
+  <%_ /* ngx.fetch needs a resolver for the manager hostname (dnsmasq runs on
+        every agent host) and the CA bundle for the https manager URL. */ -%>
+  resolver 127.0.0.1;
+  js_fetch_timeout 5s;
+  js_fetch_trusted_certificate /etc/ssl/certs/ca-certificates.crt;
+
   <%_ /*
     Upload throughput over HTTP/2 and HTTP/3. The small defaults (64k body
     preread / h3 stream buffer, 256k per-worker h2 recv buffer) cap how much
@@ -93,6 +113,24 @@ http {
     location /504.html { }
     location /auth-unavailable.html { }
   }
+
+  <%_ /*
+    Localhost-only relay for stream-context accounting reports: Debian's njs
+    stream module lacks fetch-TLS (built without NGX_STREAM_SSL), so stream
+    servers POST plain-http to this unix socket and this http-context handler
+    forwards to the manager over https with the credential. A unix socket
+    (rather than a TCP port) keeps the relay off the network entirely.
+    resolver / js_fetch_timeout / js_fetch_trusted_certificate are inherited
+    from the http{} block above.
+  */ -%>
+  server {
+    listen unix:/run/nginx-osaas-relay.sock;
+    access_log off;
+
+    location / {
+      js_content accounting.relay;
+    }
+  }
 
   server {
     listen 80 default_server;
@@ -294,6 +332,13 @@ http {
     <%_ } else { _%>
     <%_ /* Proxy settings */ -%>
     location / {
+      <%_ /* Last-access accounting: runs in a parallel mirror subrequest,
+            after the auth gate. mirror_request_body off is REQUIRED to keep
+            uploads unbuffered (see #395). */ -%>
+      set $osaas_service_id "<%= service.id %>";
+      mirror /_osaas_accounting;
+      mirror_request_body off;
+
       <%_ if (authRequired && authServer) { _%>
       auth_request /oauth2/auth;
 
@@ -342,6 +387,13 @@ http {
       proxy_buffers 4 32k;
       proxy_busy_buffers_size 64k;
     }
+
+    location = /_osaas_accounting {
+      internal;
+      <%_ /* resolver / js_fetch_timeout / js_fetch_trusted_certificate are
+            inherited from the http{} block. */ -%>
+      js_content accounting.http_record;
+    }
     <%_ } _%>
   }
   <%_ }) _%>
@@ -446,9 +498,27 @@ stream {
 
   access_log  /var/log/nginx/stream-access.log  main;
 
+  <%_ /*
+    Service last-access accounting, stream flavor (dicts are per-context, so
+    the stream side gets its own zone). js_access claims the window and
+    fires the report without awaiting it — connection setup never waits.
+    Debian's libnginx-mod-stream-js is built without NGX_STREAM_SSL, so
+    stream-side ngx.fetch has no TLS: reports hop through the localhost-only
+    http relay unix socket (see the http context), whose handler forwards to
+    the manager over https with the credential. The relay URL is the njs
+    unix-socket form — the trailing ':' separates the socket path from the
+    request URI that report() appends.
+  */ -%>
+  js_import accounting from /opt/opensource-server/agent/njs/accounting.js;
+  js_shared_dict_zone zone=osaas_stream:256k timeout=10m evict;
+  js_var $osaas_relay_url "http://unix:/run/nginx-osaas-relay.sock:";
+  js_fetch_timeout 5s;
+
   <%_ streamServices.forEach((service, index) => { _%>
   server {
     listen <%= service.externalPort %><%= service.protocol === 'udp' ? ' udp' : '' %>;
+    js_var $osaas_service_id "<%= service.id %>";
+    js_access accounting.stream_record;
     proxy_pass <%= service.container.ipv4Address %>:<%= service.internalPort %>;
   }
   <%_ }) _%>