From 6e38cd3cec3576603e624f1243a270336d633965 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 11 Aug 2026 14:27:49 -0400 Subject: [PATCH 01/17] feat(manager): record service last-access from the site proxy --- ...0811000000-add-service-last-accessed-at.js | 15 +++ create-a-container/models/service.js | 6 ++ .../services/__tests__/services.api.test.js | 93 +++++++++++++++++++ .../resources/services/controller.js | 9 ++ .../resources/services/repository.js | 17 ++++ .../resources/services/router.js | 33 +++++++ .../resources/services/service.js | 9 ++ .../resources/services/validator.js | 8 ++ create-a-container/routers/api/v1/index.js | 1 + 9 files changed, 191 insertions(+) create mode 100644 create-a-container/migrations/20260811000000-add-service-last-accessed-at.js create mode 100644 create-a-container/resources/services/__tests__/services.api.test.js create mode 100644 create-a-container/resources/services/controller.js create mode 100644 create-a-container/resources/services/repository.js create mode 100644 create-a-container/resources/services/router.js create mode 100644 create-a-container/resources/services/service.js create mode 100644 create-a-container/resources/services/validator.js diff --git a/create-a-container/migrations/20260811000000-add-service-last-accessed-at.js b/create-a-container/migrations/20260811000000-add-service-last-accessed-at.js new file mode 100644 index 00000000..55f53db5 --- /dev/null +++ b/create-a-container/migrations/20260811000000-add-service-last-accessed-at.js @@ -0,0 +1,15 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.addColumn('Services', 'lastAccessedAt', { + type: Sequelize.DATE, + allowNull: true, + }); + }, + + async down(queryInterface) { + await queryInterface.removeColumn('Services', 'lastAccessedAt'); + } +}; diff --git a/create-a-container/models/service.js b/create-a-container/models/service.js index bc6a828b..7caa8c5a 100644 --- a/create-a-container/models/service.js +++ b/create-a-container/models/service.js @@ -21,6 +21,12 @@ module.exports = (sequelize, DataTypes) => { internalPort: { type: DataTypes.INTEGER.UNSIGNED, allowNull: false + }, + // Stamped by POST /api/v1/services/:id/last-access when the site proxy + // reports traffic (at most once per 10 minutes per service). + lastAccessedAt: { + type: DataTypes.DATE, + allowNull: true } }, { sequelize, diff --git a/create-a-container/resources/services/__tests__/services.api.test.js b/create-a-container/resources/services/__tests__/services.api.test.js new file mode 100644 index 00000000..974d0745 --- /dev/null +++ b/create-a-container/resources/services/__tests__/services.api.test.js @@ -0,0 +1,93 @@ +/** + * POST /api/v1/services/:id/last-access — proxy accounting endpoint. + * Trust model mirrors the agent check-in (routers/api/v1/agents.js): + * localhost posts without credentials (the manager's own proxy), remote + * proxies authenticate with an admin API key. supertest connects from + * 127.0.0.1; an X-Forwarded-For with a public IP marks a request as remote. + */ + +const request = require('supertest'); +const { buildApp, bearer } = require('../../../tests/helpers/app'); +const { resetDb, closeDb, createUser, createApiKey } = require('../../../tests/helpers/db'); +const { Site, Node, Container, Service } = require('../../../models'); + +describe('POST /api/v1/services/:id/last-access', () => { + let app; + let service; + + beforeEach(async () => { + await resetDb(); + app = buildApp(); + const site = await Site.create({ name: 'test-site' }); + const node = await Node.create({ siteId: site.id, name: 'node1', nodeType: 'dummy' }); + const container = await Container.create({ + hostname: 'testct', + username: 'tester', + nodeId: node.id, + siteId: site.id, + }); + service = await Service.create({ containerId: container.id, type: 'http', internalPort: 80 }); + }); + + afterAll(async () => { + await closeDb(); + }); + + test('localhost without credentials records access', async () => { + const before = Date.now(); + const res = await request(app).post(`/api/v1/services/${service.id}/last-access`); + expect(res.status).toBe(204); + + await service.reload(); + expect(service.lastAccessedAt).not.toBeNull(); + const stamped = new Date(service.lastAccessedAt).getTime(); + expect(stamped).toBeGreaterThanOrEqual(before - 1000); + expect(stamped).toBeLessThanOrEqual(Date.now() + 1000); + }); + + test('remote without credentials is rejected', async () => { + const res = await request(app) + .post(`/api/v1/services/${service.id}/last-access`) + .set('X-Forwarded-For', '203.0.113.7'); + expect([401, 403]).toContain(res.status); + + await service.reload(); + expect(service.lastAccessedAt).toBeNull(); + }); + + test('remote with admin API key records access', async () => { + const admin = await createUser({ admin: true }); + const { plainKey } = await createApiKey(admin); + const res = await request(app) + .post(`/api/v1/services/${service.id}/last-access`) + .set('X-Forwarded-For', '203.0.113.7') + .set(...bearer(plainKey)); + expect(res.status).toBe(204); + + await service.reload(); + expect(service.lastAccessedAt).not.toBeNull(); + }); + + test('remote with non-admin API key is 403', async () => { + // First user after resetDb is auto-promoted to sysadmins; burn one. + await createUser(); + const plain = await createUser(); + const { plainKey } = await createApiKey(plain); + const res = await request(app) + .post(`/api/v1/services/${service.id}/last-access`) + .set('X-Forwarded-For', '203.0.113.7') + .set(...bearer(plainKey)); + expect(res.status).toBe(403); + }); + + test('unknown service id is 404', async () => { + const res = await request(app).post('/api/v1/services/999999/last-access'); + expect(res.status).toBe(404); + expect(res.body.error.code).toBe('not_found'); + }); + + test('non-numeric id is 400', async () => { + const res = await request(app).post('/api/v1/services/abc/last-access'); + expect(res.status).toBe(400); + }); +}); diff --git a/create-a-container/resources/services/controller.js b/create-a-container/resources/services/controller.js new file mode 100644 index 00000000..60a4bc79 --- /dev/null +++ b/create-a-container/resources/services/controller.js @@ -0,0 +1,9 @@ +const svc = require('./service'); +const { asyncHandler, noContent } = require('../../middlewares/api'); + +const recordAccess = asyncHandler(async (req, res) => { + await svc.recordAccess(req.validated.params.id); + return noContent(res); +}); + +module.exports = { recordAccess }; diff --git a/create-a-container/resources/services/repository.js b/create-a-container/resources/services/repository.js new file mode 100644 index 00000000..b9ebc76a --- /dev/null +++ b/create-a-container/resources/services/repository.js @@ -0,0 +1,17 @@ +const { Service } = require('../../models'); + +/** + * Stamp lastAccessedAt = now for one service. A single UPDATE statement — + * no SELECT, no row instantiation (the endpoint is called by nginx on the + * hot-ish path and must stay cheap). Returns the affected row count + * (0 = unknown id). + */ +async function recordAccess(id) { + const [count] = await Service.update( + { lastAccessedAt: new Date() }, + { where: { id } }, + ); + return count; +} + +module.exports = { recordAccess }; diff --git a/create-a-container/resources/services/router.js b/create-a-container/resources/services/router.js new file mode 100644 index 00000000..86abe4c7 --- /dev/null +++ b/create-a-container/resources/services/router.js @@ -0,0 +1,33 @@ +/** + * /api/v1/services — machine-facing service accounting. + * + * POST /:id/last-access proxy accounting: the site's nginx reports the + * first access to a service after 10+ minutes of + * none; sets lastAccessedAt to the current server + * time (agent clocks are never trusted). + */ + +const express = require('express'); +const { isLocalhostRequest } = require('../../middlewares'); +const { apiAuth, apiAdmin } = require('../../middlewares/api'); +const { validate } = require('../../middlewares/validate'); +const { idParam } = require('./validator'); +const ctrl = require('./controller'); + +const router = express.Router(); + +// Same trust model as the agent check-in (routers/api/v1/agents.js): +// the manager's own proxy reports over localhost without credentials +// (bootstrap: no API key exists yet); remote proxies authenticate with an +// admin API key. +function accountingAuth(req, res, next) { + if (isLocalhostRequest(req)) return next(); + return apiAuth(req, res, (err) => { + if (err) return next(err); + return apiAdmin(req, res, next); + }); +} + +router.post('/:id/last-access', accountingAuth, validate({ params: idParam }), ctrl.recordAccess); + +module.exports = router; diff --git a/create-a-container/resources/services/service.js b/create-a-container/resources/services/service.js new file mode 100644 index 00000000..2838cf72 --- /dev/null +++ b/create-a-container/resources/services/service.js @@ -0,0 +1,9 @@ +const repo = require('./repository'); +const { ApiError } = require('../../middlewares/api'); + +async function recordAccess(id) { + const count = await repo.recordAccess(id); + if (count === 0) throw new ApiError(404, 'not_found', 'Service not found'); +} + +module.exports = { recordAccess }; diff --git a/create-a-container/resources/services/validator.js b/create-a-container/resources/services/validator.js new file mode 100644 index 00000000..9561f021 --- /dev/null +++ b/create-a-container/resources/services/validator.js @@ -0,0 +1,8 @@ +const { z } = require('zod'); + +// Service ids are integer PKs; coerce because path params arrive as strings. +const idParam = z.object({ + id: z.coerce.number().int().positive(), +}); + +module.exports = { idParam }; diff --git a/create-a-container/routers/api/v1/index.js b/create-a-container/routers/api/v1/index.js index eb83aa46..3e45093a 100644 --- a/create-a-container/routers/api/v1/index.js +++ b/create-a-container/routers/api/v1/index.js @@ -95,6 +95,7 @@ router.use('/settings', require('./settings')); router.use('/jobs', require('./jobs')); router.use('/resource-requests', require('./resource-requests')); router.use('/notifications', require('../../../resources/notifications/router')); +router.use('/services', require('../../../resources/services/router')); // Final error handler — must come after all routes router.use(jsonErrorHandler); From b2a3cd6b31b66dc623cbdb91c9e3ec134aad2f52 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 11 Aug 2026 14:35:43 -0400 Subject: [PATCH 02/17] feat(manager): include service ids in the agent config snapshot --- .../utils/__tests__/agent-config.test.js | 59 +++++++++++++++++++ create-a-container/utils/agent-config.js | 1 + 2 files changed, 60 insertions(+) create mode 100644 create-a-container/utils/__tests__/agent-config.test.js diff --git a/create-a-container/utils/__tests__/agent-config.test.js b/create-a-container/utils/__tests__/agent-config.test.js new file mode 100644 index 00000000..8cc2f3f7 --- /dev/null +++ b/create-a-container/utils/__tests__/agent-config.test.js @@ -0,0 +1,59 @@ +/** + * buildAgentConfig — the snapshot must carry Services.id on every http and + * stream entry (the nginx accounting module reports last-access by service + * id), and stay deterministic so the strong ETag is stable. + */ + +const { resetDb, closeDb } = require('../../tests/helpers/db'); +const { + Site, Node, Container, Service, HTTPService, TransportService, ExternalDomain, +} = require('../../models'); +const { buildAgentConfig, computeConfigEtag } = require('../agent-config'); + +describe('buildAgentConfig service ids', () => { + let site; + let httpSvc; + let streamSvc; + + beforeEach(async () => { + await resetDb(); + site = await Site.create({ name: 'test-site' }); + const node = await Node.create({ siteId: site.id, name: 'node1', nodeType: 'dummy' }); + const container = await Container.create({ + hostname: 'testct', + username: 'tester', + nodeId: node.id, + siteId: site.id, + ipv4Address: '10.254.1.5', + }); + const domain = await ExternalDomain.create({ name: 'example.com' }); + + httpSvc = await Service.create({ containerId: container.id, type: 'http', internalPort: 3000 }); + await HTTPService.create({ + serviceId: httpSvc.id, + externalHostname: 'myapp', + externalDomainId: domain.id, + }); + + streamSvc = await Service.create({ containerId: container.id, type: 'transport', internalPort: 22 }); + await TransportService.create({ serviceId: streamSvc.id, protocol: 'tcp', externalPort: 30022 }); + }); + + afterAll(async () => { + await closeDb(); + }); + + test('http and stream entries carry the Services.id', async () => { + const config = await buildAgentConfig(site.id); + expect(config.nginx.httpServices).toHaveLength(1); + expect(config.nginx.httpServices[0].id).toBe(httpSvc.id); + expect(config.nginx.streamServices).toHaveLength(1); + expect(config.nginx.streamServices[0].id).toBe(streamSvc.id); + }); + + test('snapshot stays deterministic (stable ETag)', async () => { + const etag1 = computeConfigEtag(await buildAgentConfig(site.id)); + const etag2 = computeConfigEtag(await buildAgentConfig(site.id)); + expect(etag1).toBe(etag2); + }); +}); diff --git a/create-a-container/utils/agent-config.js b/create-a-container/utils/agent-config.js index 781d16f5..7a09c041 100644 --- a/create-a-container/utils/agent-config.js +++ b/create-a-container/utils/agent-config.js @@ -69,6 +69,7 @@ async function buildAgentConfig(siteId) { for (const container of node.containers || []) { for (const service of container.services || []) { const base = { + id: service.id, internalPort: service.internalPort, container: { ipv4Address: container.ipv4Address }, }; From 066a67283ba97ab1411155f2209394727c75e4f3 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 11 Aug 2026 14:41:33 -0400 Subject: [PATCH 03/17] feat(manager): expose service lastAccessedAt with container rollup --- .../v1/__tests__/containers.serialize.test.js | 73 +++++++++++++++++++ .../routers/api/v1/containers.js | 10 +++ 2 files changed, 83 insertions(+) create mode 100644 create-a-container/routers/api/v1/__tests__/containers.serialize.test.js diff --git a/create-a-container/routers/api/v1/__tests__/containers.serialize.test.js b/create-a-container/routers/api/v1/__tests__/containers.serialize.test.js new file mode 100644 index 00000000..ccf44f5e --- /dev/null +++ b/create-a-container/routers/api/v1/__tests__/containers.serialize.test.js @@ -0,0 +1,73 @@ +/** + * serializeContainer — per-service lastAccessedAt plus the container-level + * rollup (max across services; null when never accessed). Exercised directly + * against stub objects so no Proxmox/status machinery is involved. + */ + +const { serializeContainer } = require('../containers'); +const { closeDb } = require('../../../../tests/helpers/db'); + +afterAll(async () => { + await closeDb(); +}); + +function stubService(overrides = {}) { + return { + id: 10, + type: 'http', + internalPort: 80, + httpService: null, + transportService: null, + dnsService: null, + lastAccessedAt: null, + ...overrides, + }; +} + +function stubContainer(services) { + return { + id: 1, + containerId: '101', + hostname: 'testct', + username: 'alice', + collaboratorNames: () => [], + ipv4Address: '10.254.1.5', + macAddress: null, + template: null, + creationJobId: null, + entrypoint: null, + environmentVars: null, + nvidiaRequested: false, + node: null, + createdAt: new Date('2026-08-01T00:00:00Z'), + services, + }; +} + +const site = { externalIp: null }; + +test('services carry lastAccessedAt and the container rolls up the max', () => { + const older = new Date('2026-08-11T09:00:00Z'); + const newer = new Date('2026-08-11T10:30:00Z'); + const out = serializeContainer( + stubContainer([ + stubService({ id: 10, lastAccessedAt: older }), + stubService({ id: 11, internalPort: 8080, lastAccessedAt: newer }), + stubService({ id: 12, internalPort: 9090, lastAccessedAt: null }), + ]), + site, + 'running', + ); + expect(out.services.map((s) => s.lastAccessedAt)).toEqual([older, newer, null]); + expect(out.lastAccessedAt).toEqual(newer); +}); + +test('container lastAccessedAt is null when no service was ever accessed', () => { + const out = serializeContainer(stubContainer([stubService()]), site, 'running'); + expect(out.lastAccessedAt).toBeNull(); +}); + +test('container lastAccessedAt is null with no services', () => { + const out = serializeContainer(stubContainer([]), site, 'running'); + expect(out.lastAccessedAt).toBeNull(); +}); diff --git a/create-a-container/routers/api/v1/containers.js b/create-a-container/routers/api/v1/containers.js index 8d1d676b..1f7cfb0f 100644 --- a/create-a-container/routers/api/v1/containers.js +++ b/create-a-container/routers/api/v1/containers.js @@ -115,6 +115,12 @@ function serializeContainer(c, site, status) { return { port: s.internalPort, externalUrl: host ? `https://${host}` : null }; }); const primaryHttp = httpEntries[0] || null; + // Most recent proxy-reported access across this container's services + // (see POST /api/v1/services/:id/last-access). Null = never accessed. + const lastAccessedAt = services.reduce( + (max, s) => (s.lastAccessedAt && (!max || s.lastAccessedAt > max) ? s.lastAccessedAt : max), + null, + ); return { id: c.id, containerId: c.containerId, @@ -139,12 +145,14 @@ function serializeContainer(c, site, status) { sshPort: ssh?.transportService?.externalPort || null, sshHost: primaryHttp?.externalUrl ? new URL(primaryHttp.externalUrl).hostname : site?.externalIp, httpEntries, + lastAccessedAt, nodeName: c.node ? c.node.name : null, nodeApiUrl: c.node ? c.node.apiUrl : null, services: services.map((s) => ({ id: s.id, type: s.type, internalPort: s.internalPort, + lastAccessedAt: s.lastAccessedAt ?? null, httpService: s.httpService ? { id: s.httpService.id, @@ -913,3 +921,5 @@ router.delete( ); module.exports = router; +// Exported for unit tests (containers.serialize.test.js). +module.exports.serializeContainer = serializeContainer; From e39e5c8a8bbfe16255215eb924142ad52afaad17 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 11 Aug 2026 14:48:35 -0400 Subject: [PATCH 04/17] feat(manager): document service last-access in OpenAPI and client types --- create-a-container/client/src/lib/types.ts | 4 +++ create-a-container/openapi.v1.yaml | 36 ++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/create-a-container/client/src/lib/types.ts b/create-a-container/client/src/lib/types.ts index 963dbae5..dd1da301 100644 --- a/create-a-container/client/src/lib/types.ts +++ b/create-a-container/client/src/lib/types.ts @@ -142,6 +142,8 @@ export interface ContainerService { id: number; type: 'http' | 'transport' | 'dns'; internalPort: number; + /** Proxy-reported last access (ISO datetime); null when never accessed. */ + lastAccessedAt: string | null; httpService: ServiceHttp | null; transportService: ServiceTransport | null; dnsService: ServiceDns | null; @@ -165,6 +167,8 @@ export interface Container { sshPort: number | null; sshHost: string | null; httpEntries: { port: number; externalUrl: string | null }[]; + /** Max lastAccessedAt across services; null when never accessed. */ + lastAccessedAt: string | null; nodeName: string | null; nodeApiUrl: string | null; services: ContainerService[]; diff --git a/create-a-container/openapi.v1.yaml b/create-a-container/openapi.v1.yaml index c88603a7..e5815231 100644 --- a/create-a-container/openapi.v1.yaml +++ b/create-a-container/openapi.v1.yaml @@ -169,6 +169,13 @@ components: properties: port: { type: integer } externalUrl: { type: string, nullable: true } + lastAccessedAt: + type: string + format: date-time + nullable: true + description: >- + Most recent proxy-reported access across the container's services + (max of the services' lastAccessedAt). Null when never accessed. nodeName: { type: string, nullable: true } nodeApiUrl: { type: string, nullable: true } services: @@ -181,6 +188,13 @@ components: id: { type: integer } type: { type: string, enum: [http, transport, dns] } internalPort: { type: integer } + lastAccessedAt: + type: string + format: date-time + nullable: true + description: >- + Set when the site proxy reports traffic to this service — at most + once per 10 minutes per service. Null when never accessed. httpService: type: object nullable: true @@ -1298,6 +1312,28 @@ paths: data: { $ref: '#/components/schemas/Notification' } '401': { description: Authentication required, content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } '404': { description: Not found or not owned by the caller, content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } + /services/{id}/last-access: + post: + operationId: record_service_access + tags: [Agents] + summary: Record service access (proxy accounting) + description: | + Called by the site proxy (nginx accounting module) on the first + request/connection to a service after 10+ minutes of none. Sets the + service's lastAccessedAt to the current server time. Allowed from + localhost without credentials (manager bootstrap) or with an admin + API key. Exempt from the CSRF guard for Bearer/localhost callers. + parameters: + - in: path + name: id + required: true + schema: { type: integer } + responses: + '204': { description: Access recorded } + '400': { description: 'Non-numeric id (code: invalid_request)', content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } + '401': { description: 'Missing/invalid credentials from a non-localhost caller', content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } /external-domains: get: From c73a914220ed7ab1eea5e18c6a024ab1b0a62cf8 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 11 Aug 2026 14:54:15 -0400 Subject: [PATCH 05/17] feat(client): show container last-access on the containers dashboard --- .../containers/ContainersDataGrid.tsx | 15 ++++++++++++++ .../client/src/lib/formatRelativeTime.ts | 20 +++++++++++++++++++ .../src/pages/agents/AgentsListPage.tsx | 12 ++--------- 3 files changed, 37 insertions(+), 10 deletions(-) create mode 100644 create-a-container/client/src/lib/formatRelativeTime.ts diff --git a/create-a-container/client/src/components/containers/ContainersDataGrid.tsx b/create-a-container/client/src/components/containers/ContainersDataGrid.tsx index e91b316d..481d8df8 100644 --- a/create-a-container/client/src/components/containers/ContainersDataGrid.tsx +++ b/create-a-container/client/src/components/containers/ContainersDataGrid.tsx @@ -2,6 +2,7 @@ import { useCallback, useEffect, useMemo } from 'react'; import { DataVisNitroGrid, DataVisNitroSource } from '@mieweb/ui/datavis'; import type { ColumnFilterConfig, TableColumn, TableRendererProps } from '@mieweb/datavis'; import { User } from 'lucide-react'; +import { formatRelativeTime } from '@/lib/formatRelativeTime'; import type { Container } from '@/lib/types'; import { HttpLinks } from './HttpLinks'; import { NodeLink } from './NodeLink'; @@ -29,6 +30,7 @@ const TYPE_INFO: { field: string; type: string }[] = [ { field: 'nodeName', type: 'string' }, { field: 'owner', type: 'string' }, { field: 'template', type: 'string' }, + { field: 'lastAccessedAt', type: 'string' }, ]; const asContainer = (row: Record) => row as unknown as Container; @@ -100,6 +102,13 @@ export function ContainersDataGrid({ return c.sshHost && c.sshPort ? `${c.sshHost}:${c.sshPort}` : ''; }, }, + { + field: 'lastAccessedAt', + header: 'Last Access', + sortable: true, + filterable: false, + getSearchText: (_v, row) => formatRelativeTime(asContainer(row).lastAccessedAt), + }, { field: 'actions', header: '', @@ -172,6 +181,12 @@ export function ContainersDataGrid({ /> ); + case 'lastAccessedAt': + return ( + + {formatRelativeTime(c.lastAccessedAt)} + + ); default: return value as React.ReactNode; } diff --git a/create-a-container/client/src/lib/formatRelativeTime.ts b/create-a-container/client/src/lib/formatRelativeTime.ts new file mode 100644 index 00000000..65564c80 --- /dev/null +++ b/create-a-container/client/src/lib/formatRelativeTime.ts @@ -0,0 +1,20 @@ +/** + * "42s ago" / "5m ago" / "3h ago" / locale datetime / "never". + * + * Pass `secondsSince` when the server computed the age (e.g. the agents + * endpoint's secondsSinceCheckin) so the judgment doesn't depend on the + * client clock; otherwise the age is derived from Date.now(), which is fine + * for display-only relative times. + */ +export function formatRelativeTime( + iso: string | null | undefined, + secondsSince?: number | null, +): string { + if (!iso) return 'never'; + const seconds = + secondsSince ?? Math.max(0, Math.round((Date.now() - new Date(iso).getTime()) / 1000)); + if (seconds < 60) return `${seconds}s ago`; + if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago`; + if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago`; + return new Date(iso).toLocaleString(); +} diff --git a/create-a-container/client/src/pages/agents/AgentsListPage.tsx b/create-a-container/client/src/pages/agents/AgentsListPage.tsx index 96bbfbb1..b7a04c01 100644 --- a/create-a-container/client/src/pages/agents/AgentsListPage.tsx +++ b/create-a-container/client/src/pages/agents/AgentsListPage.tsx @@ -13,21 +13,13 @@ import { } from '@mieweb/ui'; import { Radio } from 'lucide-react'; import { ApiError } from '@/lib/api'; +import { formatRelativeTime } from '@/lib/formatRelativeTime'; import { keys, queries } from '@/lib/queries'; import type { Agent } from '@/lib/types'; import { useDocumentTitle } from '@/lib/useDocumentTitle'; import { AgentServiceBadges } from './AgentServiceBadges'; import { OnlineBadge } from './OnlineBadge'; -function formatLastCheckin(agent: Agent): string { - const seconds = agent.secondsSinceCheckin; - if (seconds === null || !agent.lastCheckinAt) return 'never'; - if (seconds < 60) return `${seconds}s ago`; - if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago`; - if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago`; - return new Date(agent.lastCheckinAt).toLocaleString(); -} - export function AgentsListPage() { useDocumentTitle('Agents'); const { data, isLoading, error } = useQuery({ @@ -88,7 +80,7 @@ export function AgentsListPage() { - {formatLastCheckin(agent)} + {formatRelativeTime(agent.lastCheckinAt, agent.secondsSinceCheckin)} ))} From 2e80674b0dc09bd92c7c9e22c71816ac2d5af0d6 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 11 Aug 2026 14:59:23 -0400 Subject: [PATCH 06/17] feat(agent): add nginx last-access accounting module and packaging --- agent/.fpm | 2 ++ agent/Makefile | 2 +- agent/njs/accounting.js | 75 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 agent/njs/accounting.js diff --git a/agent/.fpm b/agent/.fpm index f7646757..c74bca2a 100644 --- a/agent/.fpm +++ b/agent/.fpm @@ -9,6 +9,8 @@ --depends nodejs --depends nginx --depends libnginx-mod-stream +--depends libnginx-mod-http-js +--depends libnginx-mod-stream-js --depends ssl-cert --depends dnsmasq --deb-no-default-config-files diff --git a/agent/Makefile b/agent/Makefile index 20a4abdd..a2f9aadd 100644 --- a/agent/Makefile +++ b/agent/Makefile @@ -48,7 +48,7 @@ test: install: build $(INSTALL) -d $(DESTBIN) $(INSTALL_DATA) package.json $(DESTBIN)/ - cp -a dist templates node_modules $(DESTBIN)/ + cp -a dist templates njs node_modules $(DESTBIN)/ $(INSTALL) -d $(UNIT_DIR) $(INSTALL_DATA) contrib/systemd/opensource-agent.service $(UNIT_DIR)/ $(INSTALL_DATA) contrib/systemd/opensource-agent.timer $(UNIT_DIR)/ diff --git a/agent/njs/accounting.js b/agent/njs/accounting.js new file mode 100644 index 00000000..15c68b99 --- /dev/null +++ b/agent/njs/accounting.js @@ -0,0 +1,75 @@ +/** + * Service last-access accounting for the opensource-server proxy. + * + * Loaded by nginx's js module (njs) in both the http and stream contexts. + * Each proxied request/connection checks a shared dict whose entries expire + * after 10 minutes (the zone's timeout=): a successful add() means no + * report was sent in the current window, so this handler claims it — + * atomically, across all workers — and POSTs to the manager, which stamps + * the service's lastAccessedAt with its own clock. Everything is fail-open: + * accounting can never affect the proxied request. + * + * Injected via nginx variables (see templates/nginx.conf.ejs): + * $osaas_service_id Services.id for this vhost / stream server + * $osaas_manager_url manager base URL, no trailing slash + * $osaas_api_key admin API key; empty for the manager's own agent, + * which reports over localhost without credentials + */ + +function report(managerUrl, apiKey, serviceId) { + const headers = {}; + if (apiKey) { + headers.Authorization = 'Bearer ' + apiKey; + } + return ngx.fetch(managerUrl + '/api/v1/services/' + serviceId + '/last-access', { + method: 'POST', + headers, + }); +} + +/** + * http handler (js_content in the internal mirror location). The mirror + * subrequest runs in parallel with proxy_pass; awaiting the fetch keeps the + * subrequest — never the client response — alive until the report lands. + */ +async function http_record(r) { + try { + const id = r.variables.osaas_service_id; + if (id && ngx.shared.osaas_http.add(id, '1')) { + const reply = await report(r.variables.osaas_manager_url, r.variables.osaas_api_key, id); + if (reply.status !== 204) { + r.log('osaas accounting: manager returned ' + reply.status + ' for service ' + id); + } + } + } catch (e) { + r.log('osaas accounting: ' + e.message); + } + r.return(204); +} + +/** + * stream handler (js_access). The fetch is deliberately not awaited: the + * connection proceeds immediately and the report completes in the + * background while the session lives. + */ +function stream_record(s) { + try { + const id = s.variables.osaas_service_id; + if (id && ngx.shared.osaas_stream.add(id, '1')) { + report(s.variables.osaas_manager_url, s.variables.osaas_api_key, id) + .then((reply) => { + if (reply.status !== 204) { + s.log('osaas accounting: manager returned ' + reply.status + ' for service ' + id); + } + }) + .catch((e) => { + s.log('osaas accounting: ' + e.message); + }); + } + } catch (e) { + s.log('osaas accounting: ' + e.message); + } + s.allow(); +} + +export default { http_record, stream_record }; From 6bad6a10372a728c56fc729014c7785ad87745a2 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 11 Aug 2026 15:09:32 -0400 Subject: [PATCH 07/17] feat(agent): record http service last-access via mirror subrequests --- agent/Makefile | 6 +-- agent/package.json | 3 +- agent/src/apply.ts | 35 +++++++++++----- agent/src/index.ts | 2 +- agent/src/types.ts | 4 ++ agent/templates/nginx.conf.ejs | 29 +++++++++++++ agent/test/nginx-template.test.js | 67 +++++++++++++++++++++++++++++++ 7 files changed, 132 insertions(+), 14 deletions(-) create mode 100644 agent/test/nginx-template.test.js diff --git a/agent/Makefile b/agent/Makefile index a2f9aadd..fb051803 100644 --- a/agent/Makefile +++ b/agent/Makefile @@ -21,7 +21,7 @@ help: @echo "Targets:" @echo " deps install dependencies (npm ci)" @echo " build compile the TypeScript sources" - @echo " test run tests (none yet)" + @echo " test run tests" @echo " install stage files into DESTDIR (default /)" @echo " deb build the .deb package" @echo " rpm build the .rpm package" @@ -42,8 +42,8 @@ build: deps dev: build -# No test suite yet; kept as a no-op so the repo-wide `make test` passes. -test: +test: deps + npm test install: build $(INSTALL) -d $(DESTBIN) diff --git a/agent/package.json b/agent/package.json index 22ed975d..dbfc21b2 100644 --- a/agent/package.json +++ b/agent/package.json @@ -6,7 +6,8 @@ "license": "Apache-2.0", "scripts": { "build": "tsc", - "start": "node dist/index.js" + "start": "node dist/index.js", + "test": "node --test \"test/**/*.test.js\"" }, "dependencies": { "@particle/dbus-next": "^0.11.4", diff --git a/agent/src/apply.ts b/agent/src/apply.ts index a83c67ef..0efe413b 100644 --- a/agent/src/apply.ts +++ b/agent/src/apply.ts @@ -12,6 +12,7 @@ import { execFileSync } from 'child_process'; import ejs from 'ejs'; import { reloadOrRestartService, restartService, sighupService } from './system'; import { log, commandOutput } from './log'; +import type { AgentConfig } from './config'; import type { ApplyResult, SiteConfig } from './types'; const TEMPLATES_DIR = path.join(__dirname, '..', 'templates'); @@ -19,6 +20,8 @@ const TEMPLATES_DIR = path.join(__dirname, '..', 'templates'); interface RenderedFile { dest: string; content: string; + /** File mode for atomic writes (e.g. 0o600 for configs holding secrets). */ + mode?: number; } export interface ManagedService { @@ -26,7 +29,7 @@ export interface ManagedService { unit: string; /** Render all managed files. Returns null when there is nothing to manage * yet (e.g. dnsmasq before the site exists). */ - render(config: SiteConfig): Promise; + render(config: SiteConfig, agent: AgentConfig): Promise; /** Command that validates the staged config before it is kept. */ test?: string[]; /** Reload/restart after a successful apply. */ @@ -48,10 +51,19 @@ function run(cmd: string[]): string { export const services: ManagedService[] = [ { unit: 'nginx', - async render(config) { + async render(config, agent) { return [{ dest: '/etc/nginx/nginx.conf', - content: await renderTemplate('nginx.conf.ejs', config.nginx), + content: await renderTemplate('nginx.conf.ejs', { + ...config.nginx, + accounting: { + managerUrl: agent.managerUrl, + apiKey: agent.apiKey ?? '', + }, + }), + // The rendered config embeds the manager API key: root-only. nginx's + // master process reads the config as root before forking workers. + mode: 0o600, }]; }, test: ['nginx', '-t'], @@ -100,15 +112,19 @@ function readIfExists(file: string): string | null { // Write via temp file + rename so a crash mid-write can never leave a // truncated config on disk. -function writeFileAtomic(dest: string, content: string): void { +function writeFileAtomic(dest: string, content: string, mode?: number): void { const tmp = `${dest}.tmp-${process.pid}`; - fs.writeFileSync(tmp, content); + fs.writeFileSync(tmp, content, mode !== undefined ? { mode } : {}); fs.renameSync(tmp, dest); } -export async function applyService(svc: ManagedService, config: SiteConfig): Promise { +export async function applyService( + svc: ManagedService, + config: SiteConfig, + agent: AgentConfig, +): Promise { log.debug(`${svc.unit}: rendering config`); - const files = await svc.render(config); + const files = await svc.render(config, agent); if (!files) { log.debug(`${svc.unit}: nothing to manage yet, skipping`); return 'success'; @@ -123,17 +139,18 @@ export async function applyService(svc: ManagedService, config: SiteConfig): Pro log.info(`${svc.unit}: ${changed.length} file(s) changed, applying: ${changed.join(', ')}`); + const modeByDest = new Map(files.map((f) => [f.dest, f.mode])); // Stage the new files (previous contents kept in memory for rollback). for (const f of files) { fs.mkdirSync(path.dirname(f.dest), { recursive: true }); - writeFileAtomic(f.dest, f.content); + writeFileAtomic(f.dest, f.content, f.mode); log.debug(`${svc.unit}: wrote ${f.dest}`); } const rollback = () => { for (const [dest, prev] of current) { if (prev === null) fs.rmSync(dest, { force: true }); - else writeFileAtomic(dest, prev); + else writeFileAtomic(dest, prev, modeByDest.get(dest)); } log.debug(`${svc.unit}: rolled back to previous config`); }; diff --git a/agent/src/index.ts b/agent/src/index.ts index 071e6faa..d8340758 100644 --- a/agent/src/index.ts +++ b/agent/src/index.ts @@ -56,7 +56,7 @@ async function main(): Promise { log.info(`check-in: new config received (etag=${result.etag ?? '(none)'}), applying`); for (const svc of services) { - state.lastApply[svc.unit] = await applyService(svc, result.config); + state.lastApply[svc.unit] = await applyService(svc, result.config, cfg); } // The ETag is saved even after a failed apply: a rejected config won't diff --git a/agent/src/types.ts b/agent/src/types.ts index 4918e55d..b6775d3f 100644 --- a/agent/src/types.ts +++ b/agent/src/types.ts @@ -48,6 +48,8 @@ export interface SiteInfo { } export interface HttpService { + /** Manager Services.id — reported back by the accounting module. */ + id: number; internalPort: number; container: { ipv4Address: string }; externalHostname: string; @@ -57,6 +59,8 @@ export interface HttpService { } export interface StreamService { + /** Manager Services.id — reported back by the accounting module. */ + id: number; internalPort: number; container: { ipv4Address: string }; externalPort: number; diff --git a/agent/templates/nginx.conf.ejs b/agent/templates/nginx.conf.ejs index c57395a5..94a52807 100644 --- a/agent/templates/nginx.conf.ejs +++ b/agent/templates/nginx.conf.ejs @@ -43,6 +43,18 @@ http { proxy_cache_path /var/cache/nginx/auth_cache levels=1:2 keys_zone=auth_cache:1m max_size=10m inactive=5m; + <%_ /* + Service last-access accounting (docs/superpowers/specs/ + 2026-08-11-service-last-access-accounting-design.md). The static njs + module claims a 10-minute window per service in the shared dict (entries + expire via timeout=; the shm zone survives reloads) and reports the first + access of each window to the manager from a parallel mirror subrequest. + */ -%> + js_import accounting from /opt/opensource-server/agent/njs/accounting.js; + js_shared_dict_zone zone=osaas_http:256k timeout=10m evict; + js_var $osaas_manager_url "<%= accounting.managerUrl %>"; + js_var $osaas_api_key "<%= accounting.apiKey %>"; + <%_ /* Upload throughput over HTTP/2 and HTTP/3. The small defaults (64k body preread / h3 stream buffer, 256k per-worker h2 recv buffer) cap how much @@ -294,6 +306,13 @@ http { <%_ } else { _%> <%_ /* Proxy settings */ -%> location / { + <%_ /* Last-access accounting: runs in a parallel mirror subrequest, + after the auth gate. mirror_request_body off is REQUIRED to keep + uploads unbuffered (see #395). */ -%> + set $osaas_service_id "<%= service.id %>"; + mirror /_osaas_accounting; + mirror_request_body off; + <%_ if (authRequired && authServer) { _%> auth_request /oauth2/auth; @@ -342,6 +361,16 @@ http { proxy_buffers 4 32k; proxy_busy_buffers_size 64k; } + + location = /_osaas_accounting { + internal; + <%_ /* ngx.fetch needs a resolver for the manager hostname (dnsmasq + runs on every agent host) and the system CA bundle for https. */ -%> + resolver 127.0.0.1; + js_fetch_trusted_certificate /etc/ssl/certs/ca-certificates.crt; + js_fetch_timeout 5s; + js_content accounting.http_record; + } <%_ } _%> } <%_ }) _%> diff --git a/agent/test/nginx-template.test.js b/agent/test/nginx-template.test.js new file mode 100644 index 00000000..17bb560c --- /dev/null +++ b/agent/test/nginx-template.test.js @@ -0,0 +1,67 @@ +/** + * nginx.conf.ejs render tests (node --test, no extra deps). Pins the + * last-access accounting hooks: module/zone/vars in the http context, + * per-service mirror + internal location, and their absence from the + * default/landing/wildcard servers. + */ + +const test = require('node:test'); +const assert = require('node:assert'); +const path = require('path'); +const ejs = require('ejs'); + +const TEMPLATE = path.join(__dirname, '..', 'templates', 'nginx.conf.ejs'); + +const accounting = { managerUrl: 'https://manager.example.com', apiKey: 'test-key-123' }; + +function render(data) { + return ejs.renderFile(TEMPLATE, { + httpServices: [], + streamServices: [], + externalDomains: [], + accounting, + ...data, + }); +} + +const httpService = { + id: 42, + internalPort: 3000, + container: { ipv4Address: '10.254.1.5' }, + externalHostname: 'myapp', + backendProtocol: 'http', + authRequired: false, + externalDomain: { name: 'example.com', authServer: null }, +}; + +test('http context loads the accounting module, dict, and vars', async () => { + const conf = await render({}); + assert.match(conf, /js_import accounting from \/opt\/opensource-server\/agent\/njs\/accounting\.js;/); + assert.match(conf, /js_shared_dict_zone zone=osaas_http:256k timeout=10m evict;/); + assert.match(conf, /js_var \$osaas_manager_url "https:\/\/manager\.example\.com";/); + assert.match(conf, /js_var \$osaas_api_key "test-key-123";/); +}); + +test('http service location records last-access via a parallel mirror', async () => { + const conf = await render({ httpServices: [httpService] }); + assert.match(conf, /set \$osaas_service_id "42";/); + assert.match(conf, /mirror \/_osaas_accounting;/); + assert.match(conf, /mirror_request_body off;/); + assert.match(conf, /location = \/_osaas_accounting/); + assert.match(conf, /js_content accounting\.http_record;/); +}); + +test('auth-required service without an auth server gets no accounting hooks', async () => { + const conf = await render({ + httpServices: [{ ...httpService, authRequired: true }], + }); + // location / returns 503 — nothing is proxied, nothing is recorded. + assert.doesNotMatch(conf, /osaas_service_id/); + assert.doesNotMatch(conf, /mirror \//); +}); + +test('default, wildcard, and landing servers get no accounting hooks', async () => { + const conf = await render({ externalDomains: [{ name: 'example.com' }] }); + assert.doesNotMatch(conf, /osaas_service_id/); + assert.doesNotMatch(conf, /mirror \//); +}); From 765c9aa9abf37a3727991d95d32cdc21a60d83ee Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 11 Aug 2026 15:16:39 -0400 Subject: [PATCH 08/17] feat(agent): record stream service last-access at the access phase --- agent/templates/nginx.conf.ejs | 15 +++++++++++++++ agent/test/nginx-template.test.js | 24 ++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/agent/templates/nginx.conf.ejs b/agent/templates/nginx.conf.ejs index 94a52807..4cc81291 100644 --- a/agent/templates/nginx.conf.ejs +++ b/agent/templates/nginx.conf.ejs @@ -475,9 +475,24 @@ stream { access_log /var/log/nginx/stream-access.log main; + <%_ /* + Service last-access accounting, stream flavor (dicts are per-context, so + the stream side gets its own zone). js_access claims the window and + fires the report without awaiting it — connection setup never waits. + */ -%> + js_import accounting from /opt/opensource-server/agent/njs/accounting.js; + js_shared_dict_zone zone=osaas_stream:256k timeout=10m evict; + js_var $osaas_manager_url "<%= accounting.managerUrl %>"; + js_var $osaas_api_key "<%= accounting.apiKey %>"; + resolver 127.0.0.1; + js_fetch_trusted_certificate /etc/ssl/certs/ca-certificates.crt; + js_fetch_timeout 5s; + <%_ streamServices.forEach((service, index) => { _%> server { listen <%= service.externalPort %><%= service.protocol === 'udp' ? ' udp' : '' %>; + js_var $osaas_service_id "<%= service.id %>"; + js_access accounting.stream_record; proxy_pass <%= service.container.ipv4Address %>:<%= service.internalPort %>; } <%_ }) _%> diff --git a/agent/test/nginx-template.test.js b/agent/test/nginx-template.test.js index 17bb560c..c78cec86 100644 --- a/agent/test/nginx-template.test.js +++ b/agent/test/nginx-template.test.js @@ -65,3 +65,27 @@ test('default, wildcard, and landing servers get no accounting hooks', async () assert.doesNotMatch(conf, /osaas_service_id/); assert.doesNotMatch(conf, /mirror \//); }); + +const streamService = { + id: 77, + internalPort: 22, + container: { ipv4Address: '10.254.1.6' }, + externalPort: 30022, + protocol: 'tcp', +}; + +test('stream context loads the accounting module, dict, and vars', async () => { + const conf = await render({ streamServices: [streamService] }); + assert.match(conf, /js_shared_dict_zone zone=osaas_stream:256k timeout=10m evict;/); + // js_import appears twice: once in http, once in stream. + const imports = conf.match(/js_import accounting from \/opt\/opensource-server\/agent\/njs\/accounting\.js;/g); + assert.strictEqual(imports.length, 2); +}); + +test('stream server records last-access at the access phase', async () => { + const conf = await render({ streamServices: [streamService] }); + assert.match(conf, /js_var \$osaas_service_id "77";/); + assert.match(conf, /js_access accounting\.stream_record;/); + assert.match(conf, /listen 30022;/); + assert.match(conf, /proxy_pass 10\.254\.1\.6:22;/); +}); From 3f0e66c275b79043d9ba4c80701c353d2956073c Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 11 Aug 2026 15:45:06 -0400 Subject: [PATCH 09/17] fix(agent): nginx -t corrections for accounting template --- agent/templates/nginx.conf.ejs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/agent/templates/nginx.conf.ejs b/agent/templates/nginx.conf.ejs index 4cc81291..afbe7810 100644 --- a/agent/templates/nginx.conf.ejs +++ b/agent/templates/nginx.conf.ejs @@ -479,13 +479,15 @@ stream { Service last-access accounting, stream flavor (dicts are per-context, so the stream side gets its own zone). js_access claims the window and fires the report without awaiting it — connection setup never waits. + No js_fetch_trusted_certificate here: Debian's libnginx-mod-stream-js is + built without stream TLS fetch support, so the directive is rejected by + nginx -t (and https reports fail-open at runtime; see accounting.js). */ -%> js_import accounting from /opt/opensource-server/agent/njs/accounting.js; js_shared_dict_zone zone=osaas_stream:256k timeout=10m evict; js_var $osaas_manager_url "<%= accounting.managerUrl %>"; js_var $osaas_api_key "<%= accounting.apiKey %>"; resolver 127.0.0.1; - js_fetch_trusted_certificate /etc/ssl/certs/ca-certificates.crt; js_fetch_timeout 5s; <%_ streamServices.forEach((service, index) => { _%> From 59031712fee6afd74911124427d97ecdb0788132 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 11 Aug 2026 17:01:23 -0400 Subject: [PATCH 10/17] feat(agent): relay stream last-access reports through the http context --- agent/njs/accounting.js | 44 ++++++++++++++++++++++++------- agent/templates/nginx.conf.ejs | 33 ++++++++++++++++++----- agent/test/nginx-template.test.js | 13 ++++++++- 3 files changed, 74 insertions(+), 16 deletions(-) diff --git a/agent/njs/accounting.js b/agent/njs/accounting.js index 15c68b99..ae28b66a 100644 --- a/agent/njs/accounting.js +++ b/agent/njs/accounting.js @@ -11,9 +11,11 @@ * * Injected via nginx variables (see templates/nginx.conf.ejs): * $osaas_service_id Services.id for this vhost / stream server - * $osaas_manager_url manager base URL, no trailing slash - * $osaas_api_key admin API key; empty for the manager's own agent, - * which reports over localhost without credentials + * $osaas_manager_url http-context: manager base URL, no trailing slash + * $osaas_api_key http-context: admin API key; empty for the manager's + * own agent, which reports over localhost without + * credentials + * $osaas_relay_url stream-context: localhost relay base URL */ function report(managerUrl, apiKey, serviceId) { @@ -48,18 +50,18 @@ async function http_record(r) { } /** - * stream handler (js_access). The fetch is deliberately not awaited: the - * connection proceeds immediately and the report completes in the - * background while the session lives. + * stream handler (js_access). Reports go via the localhost http relay — + * Debian's njs stream module has no fetch-TLS — and the fetch is + * deliberately not awaited: the connection proceeds immediately. */ function stream_record(s) { try { const id = s.variables.osaas_service_id; if (id && ngx.shared.osaas_stream.add(id, '1')) { - report(s.variables.osaas_manager_url, s.variables.osaas_api_key, id) + report(s.variables.osaas_relay_url, '', id) .then((reply) => { if (reply.status !== 204) { - s.log('osaas accounting: manager returned ' + reply.status + ' for service ' + id); + s.log('osaas accounting: relay returned ' + reply.status + ' for service ' + id); } }) .catch((e) => { @@ -72,4 +74,28 @@ function stream_record(s) { s.allow(); } -export default { http_record, stream_record }; +/** + * http relay for stream-context reports (js_content on the localhost-only + * relay server). Debian's njs stream module is built without NGX_STREAM_SSL, + * so stream_record cannot fetch an https manager directly; it POSTs to this + * relay over plain local http and the http js VM — full fetch-TLS — forwards + * to the manager. The path shape is validated so the relay can never be used + * to reach any other manager endpoint with the embedded credential. + */ +async function relay(r) { + try { + const m = r.uri.match(/^\/api\/v1\/services\/(\d+)\/last-access$/); + if (!m) { + r.return(404); + return; + } + const reply = await report(r.variables.osaas_manager_url, r.variables.osaas_api_key, m[1]); + r.return(reply.status); + return; + } catch (e) { + r.log('osaas accounting relay: ' + e.message); + } + r.return(502); +} + +export default { http_record, stream_record, relay }; diff --git a/agent/templates/nginx.conf.ejs b/agent/templates/nginx.conf.ejs index afbe7810..4eddbf50 100644 --- a/agent/templates/nginx.conf.ejs +++ b/agent/templates/nginx.conf.ejs @@ -105,6 +105,26 @@ http { location /504.html { } location /auth-unavailable.html { } } + + <%_ /* + Localhost-only relay for stream-context accounting reports: Debian's njs + stream module lacks fetch-TLS (built without NGX_STREAM_SSL), so stream + servers POST plain-http here and this http-context handler forwards to + the manager over https with the credential. Port 1985 cannot collide + with transport services — the manager auto-allocates their external + ports from 2000 up and users never choose them. + */ -%> + server { + listen 127.0.0.1:1985; + access_log off; + + location / { + resolver 127.0.0.1; + js_fetch_trusted_certificate /etc/ssl/certs/ca-certificates.crt; + js_fetch_timeout 5s; + js_content accounting.relay; + } + } server { listen 80 default_server; @@ -479,15 +499,16 @@ stream { Service last-access accounting, stream flavor (dicts are per-context, so the stream side gets its own zone). js_access claims the window and fires the report without awaiting it — connection setup never waits. - No js_fetch_trusted_certificate here: Debian's libnginx-mod-stream-js is - built without stream TLS fetch support, so the directive is rejected by - nginx -t (and https reports fail-open at runtime; see accounting.js). + Debian's libnginx-mod-stream-js is built without NGX_STREAM_SSL, so + stream-side ngx.fetch has no TLS (and no js_fetch_trusted_certificate + directive): reports hop through the localhost-only http relay server + (127.0.0.1:1985, see the http context), whose handler forwards to the + manager over https with the credential. The relay target is an IP + literal, so no resolver is needed here. */ -%> js_import accounting from /opt/opensource-server/agent/njs/accounting.js; js_shared_dict_zone zone=osaas_stream:256k timeout=10m evict; - js_var $osaas_manager_url "<%= accounting.managerUrl %>"; - js_var $osaas_api_key "<%= accounting.apiKey %>"; - resolver 127.0.0.1; + js_var $osaas_relay_url "http://127.0.0.1:1985"; js_fetch_timeout 5s; <%_ streamServices.forEach((service, index) => { _%> diff --git a/agent/test/nginx-template.test.js b/agent/test/nginx-template.test.js index c78cec86..62940c07 100644 --- a/agent/test/nginx-template.test.js +++ b/agent/test/nginx-template.test.js @@ -74,12 +74,23 @@ const streamService = { protocol: 'tcp', }; -test('stream context loads the accounting module, dict, and vars', async () => { +test('stream context loads the accounting module, dict, and relay url', async () => { const conf = await render({ streamServices: [streamService] }); assert.match(conf, /js_shared_dict_zone zone=osaas_stream:256k timeout=10m evict;/); // js_import appears twice: once in http, once in stream. const imports = conf.match(/js_import accounting from \/opt\/opensource-server\/agent\/njs\/accounting\.js;/g); assert.strictEqual(imports.length, 2); + // Streams report via the localhost relay (stream js has no fetch-TLS on + // Debian njs 0.8.9), so the manager URL and API key must NOT appear in the + // stream context: the key appears exactly once, as the http-context js_var. + assert.match(conf, /js_var \$osaas_relay_url "http:\/\/127\.0\.0\.1:1985";/); + assert.strictEqual(conf.match(/test-key-123/g).length, 1); +}); + +test('http context exposes the localhost relay for stream reports', async () => { + const conf = await render({}); + assert.match(conf, /listen 127\.0\.0\.1:1985;/); + assert.match(conf, /js_content accounting\.relay;/); }); test('stream server records last-access at the access phase', async () => { From 9d7f66eb363e4d06eba87b81de37f0ff4a23ab84 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Wed, 12 Aug 2026 10:14:15 -0400 Subject: [PATCH 11/17] build(agent): declare ca-certificates dependency for nginx js fetch TLS --- agent/.fpm | 1 + 1 file changed, 1 insertion(+) diff --git a/agent/.fpm b/agent/.fpm index c74bca2a..3c06e600 100644 --- a/agent/.fpm +++ b/agent/.fpm @@ -11,6 +11,7 @@ --depends libnginx-mod-stream --depends libnginx-mod-http-js --depends libnginx-mod-stream-js +--depends ca-certificates --depends ssl-cert --depends dnsmasq --deb-no-default-config-files From fe1dae3f3e06194cb437ed1a865b2e3560d24ebb Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Wed, 12 Aug 2026 11:00:56 -0400 Subject: [PATCH 12/17] refactor(agent): relay over a unix socket; hoist njs fetch directives to http{} Move the stream-report relay off TCP 127.0.0.1:1985 to a unix socket (/run/nginx-osaas-relay.sock), keeping it off the network entirely. The stream fetch uses the njs unix-socket URL form (trailing ':' before the URI) and sends an explicit Host header, which njs otherwise derives from the socket path and nginx rejects with 400. Hoist resolver, js_fetch_timeout, and js_fetch_trusted_certificate to the http{} block so the mirror location and the relay server inherit them instead of each carrying a copy. Verified on debian:trixie (nginx 1.26.3 / njs 0.8.9): nginx -t passes on the 0600 full render, and a stream connection delivers exactly one correctly-shaped POST through the socket relay to the manager. --- agent/njs/accounting.js | 24 ++++++++++++------ agent/templates/nginx.conf.ejs | 41 +++++++++++++++++-------------- agent/test/nginx-template.test.js | 8 ++++-- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/agent/njs/accounting.js b/agent/njs/accounting.js index ae28b66a..0ef29ee5 100644 --- a/agent/njs/accounting.js +++ b/agent/njs/accounting.js @@ -15,15 +15,24 @@ * $osaas_api_key http-context: admin API key; empty for the manager's * own agent, which reports over localhost without * credentials - * $osaas_relay_url stream-context: localhost relay base URL + * $osaas_relay_url stream-context: localhost relay base URL — a unix + * socket, so it carries the njs form + * "http://unix:/path/to.sock:" (trailing colon + * separates the socket path from the request URI) */ -function report(managerUrl, apiKey, serviceId) { +// `host` is only needed for unix-socket targets: njs would otherwise derive a +// Host header from the socket path ("host: /run/....sock:0"), which nginx +// rejects with 400. TCP targets pass it undefined and njs sets Host itself. +function report(base, apiKey, serviceId, host) { const headers = {}; if (apiKey) { headers.Authorization = 'Bearer ' + apiKey; } - return ngx.fetch(managerUrl + '/api/v1/services/' + serviceId + '/last-access', { + if (host) { + headers.Host = host; + } + return ngx.fetch(base + '/api/v1/services/' + serviceId + '/last-access', { method: 'POST', headers, }); @@ -50,15 +59,16 @@ async function http_record(r) { } /** - * stream handler (js_access). Reports go via the localhost http relay — - * Debian's njs stream module has no fetch-TLS — and the fetch is - * deliberately not awaited: the connection proceeds immediately. + * stream handler (js_access). Reports go via the localhost http relay (a unix + * socket) — Debian's njs stream module has no fetch-TLS — and the fetch is + * deliberately not awaited: the connection proceeds immediately. The explicit + * Host is required for the socket fetch (see report()). */ function stream_record(s) { try { const id = s.variables.osaas_service_id; if (id && ngx.shared.osaas_stream.add(id, '1')) { - report(s.variables.osaas_relay_url, '', id) + report(s.variables.osaas_relay_url, '', id, 'localhost') .then((reply) => { if (reply.status !== 204) { s.log('osaas accounting: relay returned ' + reply.status + ' for service ' + id); diff --git a/agent/templates/nginx.conf.ejs b/agent/templates/nginx.conf.ejs index 4eddbf50..16fae25e 100644 --- a/agent/templates/nginx.conf.ejs +++ b/agent/templates/nginx.conf.ejs @@ -49,11 +49,19 @@ http { module claims a 10-minute window per service in the shared dict (entries expire via timeout=; the shm zone survives reloads) and reports the first access of each window to the manager from a parallel mirror subrequest. + resolver / js_fetch_timeout / js_fetch_trusted_certificate are set here at + http{} level so they inherit into both the mirror location and the relay + server below — no per-location copies needed. */ -%> js_import accounting from /opt/opensource-server/agent/njs/accounting.js; js_shared_dict_zone zone=osaas_http:256k timeout=10m evict; js_var $osaas_manager_url "<%= accounting.managerUrl %>"; js_var $osaas_api_key "<%= accounting.apiKey %>"; + <%_ /* ngx.fetch needs a resolver for the manager hostname (dnsmasq runs on + every agent host) and the CA bundle for the https manager URL. */ -%> + resolver 127.0.0.1; + js_fetch_timeout 5s; + js_fetch_trusted_certificate /etc/ssl/certs/ca-certificates.crt; <%_ /* Upload throughput over HTTP/2 and HTTP/3. The small defaults (64k body @@ -109,19 +117,17 @@ http { <%_ /* Localhost-only relay for stream-context accounting reports: Debian's njs stream module lacks fetch-TLS (built without NGX_STREAM_SSL), so stream - servers POST plain-http here and this http-context handler forwards to - the manager over https with the credential. Port 1985 cannot collide - with transport services — the manager auto-allocates their external - ports from 2000 up and users never choose them. + servers POST plain-http to this unix socket and this http-context handler + forwards to the manager over https with the credential. A unix socket + (rather than a TCP port) keeps the relay off the network entirely. + resolver / js_fetch_timeout / js_fetch_trusted_certificate are inherited + from the http{} block above. */ -%> server { - listen 127.0.0.1:1985; + listen unix:/run/nginx-osaas-relay.sock; access_log off; location / { - resolver 127.0.0.1; - js_fetch_trusted_certificate /etc/ssl/certs/ca-certificates.crt; - js_fetch_timeout 5s; js_content accounting.relay; } } @@ -384,11 +390,8 @@ http { location = /_osaas_accounting { internal; - <%_ /* ngx.fetch needs a resolver for the manager hostname (dnsmasq - runs on every agent host) and the system CA bundle for https. */ -%> - resolver 127.0.0.1; - js_fetch_trusted_certificate /etc/ssl/certs/ca-certificates.crt; - js_fetch_timeout 5s; + <%_ /* resolver / js_fetch_timeout / js_fetch_trusted_certificate are + inherited from the http{} block. */ -%> js_content accounting.http_record; } <%_ } _%> @@ -500,15 +503,15 @@ stream { the stream side gets its own zone). js_access claims the window and fires the report without awaiting it — connection setup never waits. Debian's libnginx-mod-stream-js is built without NGX_STREAM_SSL, so - stream-side ngx.fetch has no TLS (and no js_fetch_trusted_certificate - directive): reports hop through the localhost-only http relay server - (127.0.0.1:1985, see the http context), whose handler forwards to the - manager over https with the credential. The relay target is an IP - literal, so no resolver is needed here. + stream-side ngx.fetch has no TLS: reports hop through the localhost-only + http relay unix socket (see the http context), whose handler forwards to + the manager over https with the credential. The relay URL is the njs + unix-socket form — the trailing ':' separates the socket path from the + request URI that report() appends. */ -%> js_import accounting from /opt/opensource-server/agent/njs/accounting.js; js_shared_dict_zone zone=osaas_stream:256k timeout=10m evict; - js_var $osaas_relay_url "http://127.0.0.1:1985"; + js_var $osaas_relay_url "http://unix:/run/nginx-osaas-relay.sock:"; js_fetch_timeout 5s; <%_ streamServices.forEach((service, index) => { _%> diff --git a/agent/test/nginx-template.test.js b/agent/test/nginx-template.test.js index 62940c07..36f6a31a 100644 --- a/agent/test/nginx-template.test.js +++ b/agent/test/nginx-template.test.js @@ -40,6 +40,10 @@ test('http context loads the accounting module, dict, and vars', async () => { assert.match(conf, /js_shared_dict_zone zone=osaas_http:256k timeout=10m evict;/); assert.match(conf, /js_var \$osaas_manager_url "https:\/\/manager\.example\.com";/); assert.match(conf, /js_var \$osaas_api_key "test-key-123";/); + // fetch settings are hoisted to http{} level so the mirror location and the + // relay server both inherit them — one copy each, not per-location. + assert.strictEqual(conf.match(/js_fetch_trusted_certificate/g).length, 1); + assert.strictEqual((conf.match(/js_content accounting\.relay;/g) || []).length, 1); }); test('http service location records last-access via a parallel mirror', async () => { @@ -83,13 +87,13 @@ test('stream context loads the accounting module, dict, and relay url', async () // Streams report via the localhost relay (stream js has no fetch-TLS on // Debian njs 0.8.9), so the manager URL and API key must NOT appear in the // stream context: the key appears exactly once, as the http-context js_var. - assert.match(conf, /js_var \$osaas_relay_url "http:\/\/127\.0\.0\.1:1985";/); + assert.match(conf, /js_var \$osaas_relay_url "http:\/\/unix:\/run\/nginx-osaas-relay\.sock:";/); assert.strictEqual(conf.match(/test-key-123/g).length, 1); }); test('http context exposes the localhost relay for stream reports', async () => { const conf = await render({}); - assert.match(conf, /listen 127\.0\.0\.1:1985;/); + assert.match(conf, /listen unix:\/run\/nginx-osaas-relay\.sock;/); assert.match(conf, /js_content accounting\.relay;/); }); From a0f773e914e4ba6fd7dcb14b832c053161f66d74 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Wed, 12 Aug 2026 12:07:11 -0400 Subject: [PATCH 13/17] fix(create-a-container): rename Notifications ctid column (Postgres reserved name) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The notifications migration (from the notification-queue feature) creates a "ctid" column, which Postgres reserves as a system column on every table. CREATE TABLE therefore fails with 42701 ("column name \"ctid\" conflicts with a system column name"), and since migrations run at startup the manager cannot boot on Postgres at all. It only slipped through because the test suite runs on SQLite, where "ctid" is not reserved. Rename the physical column to "containerId" and map the model's `ctid` attribute onto it via Sequelize's `field:` option, so the API/JSON/query surface (webhook payload, serializer, repository lookups, validator, tests) is completely unchanged — only the column name differs. The migration has never applied successfully on Postgres (it fails on CREATE TABLE, so no table and no SequelizeMeta row exist), so editing the migration in place is safe — no follow-up migration is needed. Verified against real Postgres 16: CREATE TABLE now succeeds with a containerId column, and Notification.create({ ctid }) / findOne({ where: { ctid } }) round-trip correctly onto that column. --- .../migrations/20260731120000-create-notifications.js | 8 ++++++-- create-a-container/models/notification.js | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/create-a-container/migrations/20260731120000-create-notifications.js b/create-a-container/migrations/20260731120000-create-notifications.js index c79d09bc..3034a4e2 100644 --- a/create-a-container/migrations/20260731120000-create-notifications.js +++ b/create-a-container/migrations/20260731120000-create-notifications.js @@ -13,8 +13,12 @@ module.exports = { // Hypervisor node name the event originated on. node: { type: Sequelize.STRING(255), allowNull: true }, // Container id on the hypervisor (CTID/VMID). STRING to match - // Containers.containerId, which was widened to a string. - ctid: { type: Sequelize.STRING(255), allowNull: true }, + // Containers.containerId, which was widened to a string. The physical + // column is "containerId" (not "ctid"): Postgres reserves "ctid" as a + // system column on every table, so CREATE TABLE ... "ctid" fails with + // 42701. The model maps its `ctid` attribute onto this column via + // Sequelize's `field:` option, so the API/JSON field stays `ctid`. + containerId: { type: Sequelize.STRING(255), allowNull: true }, // Owning user (Users.uid). Drives per-user UI visibility. Resolved from // node+ctid at ingest time when the payload omits it. owner: { type: Sequelize.STRING(255), allowNull: true }, diff --git a/create-a-container/models/notification.js b/create-a-container/models/notification.js index f5f7d1be..8389fd67 100644 --- a/create-a-container/models/notification.js +++ b/create-a-container/models/notification.js @@ -29,7 +29,11 @@ module.exports = (sequelize, DataTypes) => { validate: { isIn: [SEVERITIES] }, }, node: { type: DataTypes.STRING(255), allowNull: true }, - ctid: { type: DataTypes.STRING(255), allowNull: true }, + // Physical column is "containerId": Postgres reserves "ctid" as a system + // column name, so the table cannot have a column literally named "ctid". + // The attribute stays `ctid` (API/JSON/query surface unchanged) and maps + // onto the containerId column via `field`. + ctid: { type: DataTypes.STRING(255), allowNull: true, field: 'containerId' }, owner: { type: DataTypes.STRING(255), allowNull: true }, // Free-form (node-side tools may emit new actions); bounded length only. action: { type: DataTypes.STRING(255), allowNull: true }, From e6b943aa7fb705f99f0031b19555c4774554c2e0 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Wed, 12 Aug 2026 12:22:33 -0400 Subject: [PATCH 14/17] fix(agent): resolve localhost in dnsmasq; log accounting failures at warn level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes to the nginx last-access accounting on the manager's own embedded agent: 1. dnsmasq did not resolve "localhost". The rendered dnsmasq config uses `no-hosts`, so /etc/hosts (where 127.0.0.1 localhost lives) is ignored, and there was no other source for the name. nginx's `resolver 127.0.0.1` (this dnsmasq) therefore failed the njs accounting module's ngx.fetch to the manager's own http://localhost:3000, logged as: js: osaas accounting: "localhost" could not be resolved (3: Host not found) Add `address=/localhost/127.0.0.1` and `.../::1` so localhost (and *.localhost) resolve to loopback, per RFC 6761. Verified against real dnsmasq on trixie: localhost A -> 127.0.0.1, AAAA -> ::1, foo.localhost -> 127.0.0.1, with no wildcarding of other names. 2. The accounting module logged via r.log/s.log, which write at njs's `info` level. The rendered error_log threshold is `notice`, one level above info, so every fail-open diagnostic was silently dropped. These lines only fire on actual failures (non-204 responses, fetch exceptions), so raise them to r.warn/s.warn (warning level) — visible under the `notice` threshold without turning on info-level noise. Adds a dnsmasq conf.ejs render test pinning the localhost override. --- agent/njs/accounting.js | 12 ++++----- agent/templates/dnsmasq/conf.ejs | 7 +++++ agent/test/dnsmasq-template.test.js | 41 +++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 agent/test/dnsmasq-template.test.js diff --git a/agent/njs/accounting.js b/agent/njs/accounting.js index 0ef29ee5..9fc343c2 100644 --- a/agent/njs/accounting.js +++ b/agent/njs/accounting.js @@ -49,11 +49,11 @@ async function http_record(r) { if (id && ngx.shared.osaas_http.add(id, '1')) { const reply = await report(r.variables.osaas_manager_url, r.variables.osaas_api_key, id); if (reply.status !== 204) { - r.log('osaas accounting: manager returned ' + reply.status + ' for service ' + id); + r.warn('osaas accounting: manager returned ' + reply.status + ' for service ' + id); } } } catch (e) { - r.log('osaas accounting: ' + e.message); + r.warn('osaas accounting: ' + e.message); } r.return(204); } @@ -71,15 +71,15 @@ function stream_record(s) { report(s.variables.osaas_relay_url, '', id, 'localhost') .then((reply) => { if (reply.status !== 204) { - s.log('osaas accounting: relay returned ' + reply.status + ' for service ' + id); + s.warn('osaas accounting: relay returned ' + reply.status + ' for service ' + id); } }) .catch((e) => { - s.log('osaas accounting: ' + e.message); + s.warn('osaas accounting: ' + e.message); }); } } catch (e) { - s.log('osaas accounting: ' + e.message); + s.warn('osaas accounting: ' + e.message); } s.allow(); } @@ -103,7 +103,7 @@ async function relay(r) { r.return(reply.status); return; } catch (e) { - r.log('osaas accounting relay: ' + e.message); + r.warn('osaas accounting relay: ' + e.message); } r.return(502); } diff --git a/agent/templates/dnsmasq/conf.ejs b/agent/templates/dnsmasq/conf.ejs index 1c8f90a2..1893a8a3 100644 --- a/agent/templates/dnsmasq/conf.ejs +++ b/agent/templates/dnsmasq/conf.ejs @@ -5,6 +5,13 @@ dhcp-authoritative domain-needed bogus-priv +# Resolve localhost to loopback (RFC 6761). no-hosts above makes dnsmasq +# ignore /etc/hosts, so without this "localhost" does not resolve — which +# breaks nginx's resolver-based lookups (e.g. the njs accounting module's +# ngx.fetch to the manager's own http://localhost:3000). Covers *.localhost. +address=/localhost/127.0.0.1 +address=/localhost/::1 + # internal domain for the site domain=<%= site.internalDomain %> diff --git a/agent/test/dnsmasq-template.test.js b/agent/test/dnsmasq-template.test.js new file mode 100644 index 00000000..cd21b7f7 --- /dev/null +++ b/agent/test/dnsmasq-template.test.js @@ -0,0 +1,41 @@ +/** + * dnsmasq conf.ejs render test (node --test, no extra deps). + * + * Pins that dnsmasq resolves `localhost` to loopback. The config uses + * `no-hosts`, so dnsmasq ignores /etc/hosts and would otherwise fail to + * resolve `localhost` — which breaks nginx's resolver-based lookups, e.g. + * the njs accounting module's ngx.fetch to the manager's own + * http://localhost:3000 (observed as: `"localhost" could not be resolved`). + */ + +const test = require('node:test'); +const assert = require('node:assert'); +const path = require('path'); +const ejs = require('ejs'); + +const TEMPLATE = path.join(__dirname, '..', 'templates', 'dnsmasq', 'conf.ejs'); + +const site = { + internalDomain: 'site.example', + dhcpRange: '10.254.1.100,10.254.1.200', + subnetMask: '255.255.255.0', + gateway: '10.254.1.1', + dnsForwarders: '1.1.1.1', + nodes: [], +}; + +function render() { + return ejs.renderFile(TEMPLATE, { site }); +} + +test('dnsmasq resolves localhost to loopback (v4 and v6)', async () => { + const conf = await render(); + assert.match(conf, /^address=\/localhost\/127\.0\.0\.1$/m); + assert.match(conf, /^address=\/localhost\/::1$/m); +}); + +test('the localhost override coexists with no-hosts', async () => { + const conf = await render(); + // no-hosts is why the explicit address= is needed; both must be present. + assert.match(conf, /^no-hosts$/m); +}); From 580f1b04deb61a4b94b2246ad46cda668c727715 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Wed, 12 Aug 2026 13:04:33 -0400 Subject: [PATCH 15/17] refactor(manager): share localhost-or-admin auth between check-in and accounting The service-accounting router (POST /api/v1/services/:id/last-access) had a verbatim copy of the agent check-in's auth logic (localhost bypass, else apiAuth + apiAdmin). Extract it once as localhostOrAdmin in middlewares/api (next to apiAuth/apiAdmin, same lazy isLocalhostRequest require the csrfGuard already uses) and have both routers use it. Behavior is unchanged; the duplicated checkinAuth/accountingAuth functions are removed. --- create-a-container/middlewares/api.js | 29 +++++++++++++++---- .../resources/services/router.js | 20 ++++--------- .../api/v1/__tests__/agents.checkin.test.js | 2 +- create-a-container/routers/api/v1/agents.js | 17 ++--------- 4 files changed, 31 insertions(+), 37 deletions(-) diff --git a/create-a-container/middlewares/api.js b/create-a-container/middlewares/api.js index ee4e6205..fa841ba1 100644 --- a/create-a-container/middlewares/api.js +++ b/create-a-container/middlewares/api.js @@ -38,12 +38,12 @@ function csrfGuard(req, res, next) { } // The manager's own agent checks in over localhost without credentials // during bootstrap (no site or API key exists yet), so it can carry neither - // a Bearer token nor a CSRF token. The agents router applies the same - // localhost bypass at the route level (checkinAuth); mirror it here so this - // app-level guard doesn't reject the credential-less check-in first. Remote - // clients are never localhost (isLocalhostRequest also rejects proxied - // requests via X-Real-IP / X-Forwarded-For). Required lazily to avoid a - // load-order cycle with middlewares/index. + // a Bearer token nor a CSRF token. The check-in and accounting routes apply + // the same localhost bypass at the route level (localhostOrAdmin); mirror it + // here so this app-level guard doesn't reject the credential-less check-in + // first. Remote clients are never localhost (isLocalhostRequest also rejects + // proxied requests via X-Real-IP / X-Forwarded-For). Required lazily to avoid + // a load-order cycle with middlewares/index. const { isLocalhostRequest } = require('./index'); if (isLocalhostRequest(req)) return next(); const auth = req.get('Authorization') || ''; @@ -93,6 +93,22 @@ function apiAdmin(req, res, next) { return res.status(403).json({ error: { code: 'forbidden', message: 'Admin access required' } }); } +// Trust model for machine-facing endpoints the site proxy/agent calls: the +// manager's own agent reaches them over localhost without credentials +// (bootstrap — no site or API key exists yet), while remote agents must +// present an admin API key (apiAuth + apiAdmin, so error responses follow the +// v1 JSON envelope). Shared by the agent check-in and the service-accounting +// routes. isLocalhostRequest is required lazily to avoid a load-order cycle +// with middlewares/index (same pattern as csrfGuard above). +function localhostOrAdmin(req, res, next) { + const { isLocalhostRequest } = require('./index'); + if (isLocalhostRequest(req)) return next(); + return apiAuth(req, res, (err) => { + if (err) return next(err); + return apiAdmin(req, res, next); + }); +} + // --- Helpers ------------------------------------------------------------------------------ function asyncHandler(fn) { return (req, res, next) => Promise.resolve(fn(req, res, next)).catch(next); @@ -153,6 +169,7 @@ class ApiError extends Error { module.exports = { apiAuth, apiAdmin, + localhostOrAdmin, csrfGuard, generateCsrfToken, asyncHandler, diff --git a/create-a-container/resources/services/router.js b/create-a-container/resources/services/router.js index 86abe4c7..044ce4b2 100644 --- a/create-a-container/resources/services/router.js +++ b/create-a-container/resources/services/router.js @@ -8,26 +8,16 @@ */ const express = require('express'); -const { isLocalhostRequest } = require('../../middlewares'); -const { apiAuth, apiAdmin } = require('../../middlewares/api'); +const { localhostOrAdmin } = require('../../middlewares/api'); const { validate } = require('../../middlewares/validate'); const { idParam } = require('./validator'); const ctrl = require('./controller'); const router = express.Router(); -// Same trust model as the agent check-in (routers/api/v1/agents.js): -// the manager's own proxy reports over localhost without credentials -// (bootstrap: no API key exists yet); remote proxies authenticate with an -// admin API key. -function accountingAuth(req, res, next) { - if (isLocalhostRequest(req)) return next(); - return apiAuth(req, res, (err) => { - if (err) return next(err); - return apiAdmin(req, res, next); - }); -} - -router.post('/:id/last-access', accountingAuth, validate({ params: idParam }), ctrl.recordAccess); +// Trust model: the manager's own proxy reports over localhost without +// credentials (bootstrap); remote proxies authenticate with an admin API key. +// Shared with the agent check-in route (see middlewares/api localhostOrAdmin). +router.post('/:id/last-access', localhostOrAdmin, validate({ params: idParam }), ctrl.recordAccess); module.exports = router; diff --git a/create-a-container/routers/api/v1/__tests__/agents.checkin.test.js b/create-a-container/routers/api/v1/__tests__/agents.checkin.test.js index 8c95751e..ba2b98df 100644 --- a/create-a-container/routers/api/v1/__tests__/agents.checkin.test.js +++ b/create-a-container/routers/api/v1/__tests__/agents.checkin.test.js @@ -2,7 +2,7 @@ * Agent check-in auth — the manager's own agent bootstraps over localhost with * no site row and no API key, so its POST /api/v1/agents carries neither a * Bearer token nor a CSRF token. Two guards must let it through: the app-level - * csrfGuard (app.js) and the route-level checkinAuth (agents.js). This pins + * csrfGuard (app.js) and the route-level localhostOrAdmin (agents.js). This pins * that the credential-less localhost check-in is NOT rejected with 403, while * a non-localhost (proxied) credential-less check-in still is. * diff --git a/create-a-container/routers/api/v1/agents.js b/create-a-container/routers/api/v1/agents.js index 84b6b45a..d9a014ee 100644 --- a/create-a-container/routers/api/v1/agents.js +++ b/create-a-container/routers/api/v1/agents.js @@ -9,25 +9,12 @@ const express = require('express'); const { Agent, Site } = require('../../../models'); -const { isLocalhostRequest } = require('../../../middlewares'); -const { apiAuth, apiAdmin, asyncHandler, ok, fail } = require('../../../middlewares/api'); +const { apiAuth, apiAdmin, localhostOrAdmin, asyncHandler, ok, fail } = require('../../../middlewares/api'); const { buildAgentConfig, computeConfigEtag } = require('../../../utils/agent-config'); const router = express.Router(); -// Check-in auth: the manager's own agent checks in over localhost without -// credentials (bootstrap: no site, no API key exist yet); remote agents -// authenticate with an admin API key via apiAuth/apiAdmin so error -// responses follow the v1 JSON envelope. -function checkinAuth(req, res, next) { - if (isLocalhostRequest(req)) return next(); - return apiAuth(req, res, (err) => { - if (err) return next(err); - return apiAdmin(req, res, next); - }); -} - -router.post('/', checkinAuth, asyncHandler(async (req, res) => { +router.post('/', localhostOrAdmin, asyncHandler(async (req, res) => { const { siteId, hostname, ipv4Address, services } = req.body || {}; const parsedSiteId = typeof siteId === 'number' ? siteId : Number(siteId); if (!Number.isInteger(parsedSiteId) || !hostname || typeof hostname !== 'string') { From 1fd4063255b7cc9f2787edac894407011d3b56e6 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Wed, 12 Aug 2026 13:11:28 -0400 Subject: [PATCH 16/17] refactor(manager): move isLocalhostRequest into middlewares/api isLocalhostRequest was defined in middlewares/index but its only consumers are in middlewares/api (csrfGuard and localhostOrAdmin), which imported it twice via lazy require('./index') to sidestep a load-order cycle. Move the function to api.js next to its callers: both lazy requires and the cycle guard disappear, and the duplicate destructure is gone. Removed from the index barrel's exports (nothing imports it from there); api.js exports it for discoverability alongside the other guards. --- create-a-container/middlewares/api.js | 28 +++++++++++++++++++------ create-a-container/middlewares/index.js | 25 ---------------------- 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/create-a-container/middlewares/api.js b/create-a-container/middlewares/api.js index fa841ba1..76ceccd8 100644 --- a/create-a-container/middlewares/api.js +++ b/create-a-container/middlewares/api.js @@ -26,6 +26,25 @@ const { req.headers['x-csrf-token'] || (req.body && req.body._csrf), }); +// True when the request comes directly from localhost (and was not proxied +// on behalf of a remote client, per X-Real-IP / X-Forwarded-For). Used by the +// CSRF guard and the localhost-or-admin auth below to let the manager's own +// agent through without credentials during bootstrap. +function isLocalhostRequest(req) { + const isLocalhost = (ip) => + ip === '127.0.0.1' || ip === '::1' || ip === '::ffff:127.0.0.1' || ip === 'localhost'; + + const directIp = req.connection?.remoteAddress || req.socket?.remoteAddress || req.ip; + const realIp = req.get('X-Real-IP'); + // First hop of X-Forwarded-For is the original client (covers proxies that + // set it without also setting X-Real-IP). + const forwardedFor = (req.get('X-Forwarded-For') || '').split(',')[0].trim(); + + return isLocalhost(directIp) + && (!realIp || isLocalhost(realIp)) + && (!forwardedFor || isLocalhost(forwardedFor)); +} + // CSRF guard: enforce on state-changing methods. Only exempt requests that // are purely Bearer-authenticated (i.e., do NOT also carry a session cookie). // A session cookie is always sent by browsers, so a Bearer header alone @@ -42,9 +61,7 @@ function csrfGuard(req, res, next) { // the same localhost bypass at the route level (localhostOrAdmin); mirror it // here so this app-level guard doesn't reject the credential-less check-in // first. Remote clients are never localhost (isLocalhostRequest also rejects - // proxied requests via X-Real-IP / X-Forwarded-For). Required lazily to avoid - // a load-order cycle with middlewares/index. - const { isLocalhostRequest } = require('./index'); + // proxied requests via X-Real-IP / X-Forwarded-For). if (isLocalhostRequest(req)) return next(); const auth = req.get('Authorization') || ''; const hasBearer = auth.startsWith('Bearer '); @@ -98,10 +115,8 @@ function apiAdmin(req, res, next) { // (bootstrap — no site or API key exists yet), while remote agents must // present an admin API key (apiAuth + apiAdmin, so error responses follow the // v1 JSON envelope). Shared by the agent check-in and the service-accounting -// routes. isLocalhostRequest is required lazily to avoid a load-order cycle -// with middlewares/index (same pattern as csrfGuard above). +// routes. function localhostOrAdmin(req, res, next) { - const { isLocalhostRequest } = require('./index'); if (isLocalhostRequest(req)) return next(); return apiAuth(req, res, (err) => { if (err) return next(err); @@ -170,6 +185,7 @@ module.exports = { apiAuth, apiAdmin, localhostOrAdmin, + isLocalhostRequest, csrfGuard, generateCsrfToken, asyncHandler, diff --git a/create-a-container/middlewares/index.js b/create-a-container/middlewares/index.js index b2f8654a..1bfe4540 100644 --- a/create-a-container/middlewares/index.js +++ b/create-a-container/middlewares/index.js @@ -73,35 +73,10 @@ function requireAdmin(req, res, next) { return res.status(403).send('Forbidden: Admin access required'); } -// True when the request comes directly from localhost (and was not proxied -// on behalf of a remote client, per X-Real-IP / X-Forwarded-For). -function isLocalhostRequest(req) { - const isLocalhost = (ip) => { - return ip === '127.0.0.1' || - ip === '::1' || - ip === '::ffff:127.0.0.1' || - ip === 'localhost'; - }; - - const directIp = req.connection?.remoteAddress || - req.socket?.remoteAddress || - req.ip; - - const realIp = req.get('X-Real-IP'); - // First hop of X-Forwarded-For is the original client (covers proxies that - // set it without also setting X-Real-IP). - const forwardedFor = (req.get('X-Forwarded-For') || '').split(',')[0].trim(); - - return isLocalhost(directIp) - && (!realIp || isLocalhost(realIp)) - && (!forwardedFor || isLocalhost(forwardedFor)); -} - const { setCurrentSite, loadSites } = require('./currentSite'); module.exports = { isApiRequest, - isLocalhostRequest, requireAuth, requireAdmin, setCurrentSite, From 829b6c6c2a36e72f860e85cc2c639f8824c587bb Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Wed, 12 Aug 2026 13:26:31 -0400 Subject: [PATCH 17/17] test(manager): pin CSRF behavior on the service last-access endpoint Add session-authenticated coverage for POST /api/v1/services/:id/last-access: a remote (X-Forwarded-For) session-cookie request is 403 without an X-CSRF-Token and 204 with a valid one. Guards against the Bearer/localhost CSRF exemptions silently regressing into a hole for cookie-authenticated callers. Verified against real Postgres 16 (8/8 in the suite). Addresses the reviewer note on services.api.test.js. --- .../services/__tests__/services.api.test.js | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/create-a-container/resources/services/__tests__/services.api.test.js b/create-a-container/resources/services/__tests__/services.api.test.js index 974d0745..8c51e756 100644 --- a/create-a-container/resources/services/__tests__/services.api.test.js +++ b/create-a-container/resources/services/__tests__/services.api.test.js @@ -90,4 +90,52 @@ describe('POST /api/v1/services/:id/last-access', () => { const res = await request(app).post('/api/v1/services/abc/last-access'); expect(res.status).toBe(400); }); + + // The route is machine-facing (localhost or Bearer), but /api/v1 is behind + // csrfGuard, which enforces CSRF only for session-cookie requests that carry + // no Bearer. Pin that a remote session-authenticated request is rejected + // without a CSRF token and succeeds with one, so the Bearer/localhost + // exemptions can't silently regress into a CSRF hole. + describe('session-authenticated (CSRF-guarded) access', () => { + // A supertest agent persists cookies across requests, so the session + // established by /csrf-token and /auth/login carries into the accounting + // POST. X-Forwarded-For marks the request as remote so the localhost + // bypass does not apply. + async function loginAgent() { + const admin = await createUser({ admin: true }); + const agent = request.agent(app); + const tokenRes = await agent.get('/api/v1/csrf-token'); + const csrfToken = tokenRes.body.data.csrfToken; + const loginRes = await agent + .post('/api/v1/auth/login') + .set('X-CSRF-Token', csrfToken) + .send({ username: admin.uid, password: 'correct horse battery staple' }); + expect(loginRes.status).toBe(200); + return { agent, csrfToken }; + } + + test('remote session request without a CSRF token is 403', async () => { + const { agent } = await loginAgent(); + const res = await agent + .post(`/api/v1/services/${service.id}/last-access`) + .set('X-Forwarded-For', '203.0.113.7'); + expect(res.status).toBe(403); + expect(res.body.error.code).toBe('csrf_invalid'); + + await service.reload(); + expect(service.lastAccessedAt).toBeNull(); + }); + + test('remote session request with a valid CSRF token records access', async () => { + const { agent, csrfToken } = await loginAgent(); + const res = await agent + .post(`/api/v1/services/${service.id}/last-access`) + .set('X-Forwarded-For', '203.0.113.7') + .set('X-CSRF-Token', csrfToken); + expect(res.status).toBe(204); + + await service.reload(); + expect(service.lastAccessedAt).not.toBeNull(); + }); + }); });