From 1c6078678d3ddb5389f6a6469d896e789550ba0b Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Tue, 11 Aug 2026 09:31:32 -0700 Subject: [PATCH 1/3] Add per-container usage metrics collector with OTLP export (issue #440) usage-collector.js is a third long-running service (beside server.js and job-runner.js) that polls one clusterResources('lxc') call per Proxmox cluster and exports per-container resource metrics via OTLP http/protobuf using the standard OTEL_* environment variables. It is a no-op (exits 0) unless OTEL_EXPORTER_OTLP_ENDPOINT is configured, so the observability backend stays out of this repo. Metrics (all attributed with container.id, site.id, proxmox.node, owner, container.name, container.status): - container.cpu.usage / container.cpu.limit ({cpu}) - container.memory.usage / container.memory.limit (By) - container.disk.usage / container.disk.limit (By) - container.disk.io (By, disk.io.direction=read|write) - container.network.io (By, network.io.direction=receive|transmit) - container.uptime (s) Owner attribution uses the Proxmox owner tag (written at container creation) as the source of truth, cross-checked against the manager DB; drift and unattributed containers are logged. Pure mapping logic lives in utils/usage-sample.js with unit tests. DummyApi now returns simulated usage numbers and owner tags so the collector works in dev. Ships with a systemd unit (usage-collector.service), Makefile/postinstall wiring, an npm script, and a 'make dev' process. --- create-a-container/Makefile | 10 +- create-a-container/README.md | 18 +- create-a-container/contrib/postinstall.sh | 2 +- create-a-container/contrib/preremove.sh | 2 +- .../contrib/systemd/usage-collector.service | 18 ++ create-a-container/example.env | 11 + create-a-container/package-lock.json | 171 +++++++++++ create-a-container/package.json | 5 + create-a-container/usage-collector.js | 272 ++++++++++++++++++ .../utils/__tests__/usage-sample.test.js | 133 +++++++++ create-a-container/utils/dummy-api.js | 17 +- create-a-container/utils/usage-sample.js | 110 +++++++ .../docs/developers/development-workflow.md | 3 +- .../docs/developers/release-pipeline.md | 2 +- 14 files changed, 759 insertions(+), 15 deletions(-) create mode 100644 create-a-container/contrib/systemd/usage-collector.service create mode 100644 create-a-container/usage-collector.js create mode 100644 create-a-container/utils/__tests__/usage-sample.test.js create mode 100644 create-a-container/utils/usage-sample.js diff --git a/create-a-container/Makefile b/create-a-container/Makefile index 76334078..27312f0d 100644 --- a/create-a-container/Makefile +++ b/create-a-container/Makefile @@ -13,7 +13,7 @@ INSTALL_DATA := $(INSTALL) -m 0644 # Server runtime files and directories. Client sources, dev tooling and repo # metadata are intentionally excluded; only the built client/dist ships. -APP_FILES := server.js app.js job-runner.js package.json package-lock.json openapi.v1.yaml +APP_FILES := server.js app.js job-runner.js usage-collector.js package.json package-lock.json openapi.v1.yaml APP_DIRS := bin config data middlewares migrations models node_modules \ public resources routers seeders utils @@ -56,11 +56,13 @@ build: deps # rewrites package-lock.json (so `make dev` leaves the lockfile untouched even # when the local npm would otherwise renormalize it). Then run migrations (which # also runs the dev-only seeders, e.g. a localhost site + dummy node, via -# `db:seed:all`), build the client once, and run three processes together via +# `db:seed:all`), build the client once, and run four processes together via # concurrently: # - server (nodemon, restarts on change) # - job-runner (so container creation exercises the real POST /containers -> # Job -> job-runner -> bin/create-container.js -> DummyApi path) +# - usage-collector (per-container OTLP metrics; exits immediately unless an +# OTLP endpoint is configured in .env) # - client build in watch mode (rebuilds client/dist, which the server serves) # # Override behavior on the command line, e.g.: @@ -76,9 +78,10 @@ dev: npm run db:migrate npm --prefix client run build @echo "Starting Manager (server + job-runner + client watch) at http://localhost:3000 ..." - $(LOG_LEVEL_PREFIX)npx concurrently -n server,jobs,client -c blue,magenta,green \ + $(LOG_LEVEL_PREFIX)npx concurrently -n server,jobs,usage,client -c blue,magenta,yellow,green \ "npx nodemon server.js" \ "node job-runner.js" \ + "node usage-collector.js" \ "npm --prefix client run build:watch" # Run the server test suite (jest + supertest against a throwaway SQLite DB in @@ -99,6 +102,7 @@ install: build $(INSTALL) -d $(UNIT_DIR) $(INSTALL_DATA) contrib/systemd/container-creator.service $(UNIT_DIR)/ $(INSTALL_DATA) contrib/systemd/job-runner.service $(UNIT_DIR)/ + $(INSTALL_DATA) contrib/systemd/usage-collector.service $(UNIT_DIR)/ $(INSTALL) -d $(DESTDIR)/etc/logrotate.d $(INSTALL_DATA) contrib/opensource-server.logrotate $(DESTDIR)/etc/logrotate.d/opensource-server diff --git a/create-a-container/README.md b/create-a-container/README.md index c20ae675..01d27bd8 100644 --- a/create-a-container/README.md +++ b/create-a-container/README.md @@ -23,10 +23,11 @@ make dev ``` That's all you need. `make dev` installs dependencies, runs database migrations -and dev seeders, builds the client, and starts the server, the job-runner, and -the client build watcher together. It uses SQLite and a dummy (mock) hypervisor, -so **no `.env`, PostgreSQL, or Proxmox cluster is required** — the Manager comes -up at and can "create" containers locally (simulated). +and dev seeders, builds the client, and starts the server, the job-runner, the +usage-collector, and the client build watcher together. It uses SQLite and a +dummy (mock) hypervisor, so **no `.env`, PostgreSQL, or Proxmox cluster is +required** — the Manager comes up at and can "create" +containers locally (simulated). Pass `LOG_LEVEL=trace` to additionally log every SQL query: @@ -46,13 +47,16 @@ The Manager is not installed by hand in production. It ships as: - distribution **packages** built from this directory with `make deb`, `make rpm`, or `make apk` (via [fpm](https://fpm.readthedocs.io/)), which install the app under `/opt/opensource-server/create-a-container` and register the - `container-creator` and `job-runner` systemd services. The package depends + `container-creator`, `job-runner`, and `usage-collector` systemd services. + The package depends on `opensource-mcp` — the [MCP server](../manager-control-program/) as its own package — and the Manager reverse-proxies `/mcp` to its service (`MCP_SERVER_URL`). -In both cases the app runs `server.js` (HTTP API + UI) and `job-runner.js` -(background worker). Database connection settings come from the environment (see +In both cases the app runs `server.js` (HTTP API + UI), `job-runner.js` +(background worker), and `usage-collector.js` (per-container resource metrics +exported as OTLP; a no-op unless `OTEL_EXPORTER_OTLP_ENDPOINT` is configured). +Database connection settings come from the environment (see [Configuration](#configuration)); the manager image provisions PostgreSQL and writes these to `/etc/default/container-creator` on first boot. diff --git a/create-a-container/contrib/postinstall.sh b/create-a-container/contrib/postinstall.sh index e92c7acf..4ec3a434 100755 --- a/create-a-container/contrib/postinstall.sh +++ b/create-a-container/contrib/postinstall.sh @@ -1,7 +1,7 @@ #!/bin/sh set -e -UNITS="container-creator.service job-runner.service" +UNITS="container-creator.service job-runner.service usage-collector.service" # Nothing to do without systemctl (non-systemd container/chroot). command -v systemctl >/dev/null 2>&1 || exit 0 diff --git a/create-a-container/contrib/preremove.sh b/create-a-container/contrib/preremove.sh index a3ebf879..7d4f9400 100755 --- a/create-a-container/contrib/preremove.sh +++ b/create-a-container/contrib/preremove.sh @@ -1,7 +1,7 @@ #!/bin/sh set -e -UNITS="container-creator.service job-runner.service" +UNITS="container-creator.service job-runner.service usage-collector.service" # $1 is "remove"/"purge" (deb) or the remaining-version count (rpm: 0 on final # removal). Only act on a real removal, not an upgrade. diff --git a/create-a-container/contrib/systemd/usage-collector.service b/create-a-container/contrib/systemd/usage-collector.service new file mode 100644 index 00000000..daea5894 --- /dev/null +++ b/create-a-container/contrib/systemd/usage-collector.service @@ -0,0 +1,18 @@ +[Unit] +Description=Per-container usage metrics (OTLP) for create-a-container +After=network.target container-creator-init.service +Wants=container-creator-init.service + +[Service] +Type=simple +User=root +WorkingDirectory=/opt/opensource-server/create-a-container +ExecStart=/usr/bin/node /opt/opensource-server/create-a-container/usage-collector.js +# Exits 0 (and stays stopped) unless OTEL_EXPORTER_OTLP_ENDPOINT is set in the +# EnvironmentFile — the collector is a no-op without an OTel backend. +Restart=on-failure +Environment=NODE_ENV=production +EnvironmentFile=/etc/default/container-creator + +[Install] +WantedBy=multi-user.target diff --git a/create-a-container/example.env b/create-a-container/example.env index 9c6e8ab5..efa05e50 100644 --- a/create-a-container/example.env +++ b/create-a-container/example.env @@ -35,6 +35,17 @@ ACCESS_LOG= # leave unset to disable the proxy. MCP_SERVER_URL= +# --- Per-container usage metrics (optional) --- +# usage-collector.js emits per-container resource metrics (CPU/memory/disk/ +# network, attributed to owners via Proxmox tags) as OTLP over http/protobuf. +# It is a no-op unless an OTLP endpoint is configured — the standard OTel +# environment variables apply (e.g. OTEL_EXPORTER_OTLP_HEADERS for auth). +# Example: http://otel-collector:4318 +OTEL_EXPORTER_OTLP_ENDPOINT= +# Export/poll interval in milliseconds (default 300000 = 5 minutes). Each +# cycle issues one Proxmox /cluster/resources call per cluster. +USAGE_COLLECTOR_INTERVAL_MS= + # --- OIDC / single sign-on (optional) --- # SSO is enabled only when OIDC_ISSUER_URL, OIDC_CLIENT_ID, and # OIDC_CLIENT_SECRET are all set. When enabled, the login page redirects to the diff --git a/create-a-container/package-lock.json b/create-a-container/package-lock.json index 97fe17e6..b52c7816 100644 --- a/create-a-container/package-lock.json +++ b/create-a-container/package-lock.json @@ -6,6 +6,10 @@ "": { "hasInstallScript": true, "dependencies": { + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/exporter-metrics-otlp-http": "^0.221.0", + "@opentelemetry/resources": "^2.10.0", + "@opentelemetry/sdk-metrics": "^2.10.0", "argon2": "^0.44.0", "axios": "^1.18.0", "cookie-parser": "^1.4.7", @@ -1058,6 +1062,173 @@ "dev": true, "license": "MIT" }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.221.0.tgz", + "integrity": "sha512-OlanaW1vv7ufTqQ3/fPLI4arGt5ZoM+P8abOMki6uEYnpRazepSWDwDnnw+la7kE26SHVC18//SMccrDvLKOXQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.10.0.tgz", + "integrity": "sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-http": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-http/-/exporter-metrics-otlp-http-0.221.0.tgz", + "integrity": "sha512-sRfCKbOzgy8xZQV2as0RzIZlnCmCseCKZGLfRcrpo2CBngJDr+rPtX0zkG0+oUCV5kfQPUoW3W3C96Ag3Y/Clg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/otlp-exporter-base": "0.221.0", + "@opentelemetry/otlp-transformer": "0.221.0", + "@opentelemetry/resources": "2.10.0", + "@opentelemetry/sdk-metrics": "2.10.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.221.0.tgz", + "integrity": "sha512-UFPIq80OH3Ns/oPFHRj14d4DTOxUo+MUFU8hUiCq5jTqFhdeJnfVSANHT+xp92409cA+oxzvlZCe6NM1wvCuBA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/otlp-transformer": "0.221.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.221.0.tgz", + "integrity": "sha512-lg6lkOU08Az23jVcn/0Els9HP+V8PnR4Km6p0KgpTggS0n/WuhnmY64rSh83Of9iR9nD+dpWr6adlcX8KzAwjg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.221.0", + "@opentelemetry/core": "2.10.0", + "@opentelemetry/resources": "2.10.0", + "@opentelemetry/sdk-logs": "0.221.0", + "@opentelemetry/sdk-metrics": "2.10.0", + "@opentelemetry/sdk-trace": "2.10.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.10.0.tgz", + "integrity": "sha512-q6MMm2zhggzsHVNbabYwut+a6nbuQQe3URUoxaojM/8K1IBfwwPzvxIjNi2/lI1TFe+fMHMW9MWhrtDLEXEnkA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.221.0.tgz", + "integrity": "sha512-FaDcazjyMp7TZZZAsqbo4IkovP0UegoCu0EBkiNt+qCqvUf7FPAsfcrZ3+ZEkKgXZ/jHafop+JoGPDk3A0SmLg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.221.0", + "@opentelemetry/core": "2.10.0", + "@opentelemetry/resources": "2.10.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.10.0.tgz", + "integrity": "sha512-t6r1VSvXNtSDnPXU1FbZeetJb7yyovHmgu0wRSoftxtE0g2rSNhQZQUy69sRUCL+iioJpX8SN/S6wq6ZtvLySQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/resources": "2.10.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace/-/sdk-trace-2.10.0.tgz", + "integrity": "sha512-MfQGq3GRmTh5fM/y+OjaO0vj6+luCB1XO2gfXCalKCfgKw0eHL++sm75DNweC6ohlp+aFvACqeE0fYayqdRaoQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/resources": "2.10.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.43.0.tgz", + "integrity": "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/@paralleldrive/cuid2": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/@paralleldrive/cuid2/-/cuid2-2.3.1.tgz", diff --git a/create-a-container/package.json b/create-a-container/package.json index 6a856a5e..40cc12af 100644 --- a/create-a-container/package.json +++ b/create-a-container/package.json @@ -12,6 +12,7 @@ "dev:server": "nodemon server.js", "db:migrate": "sequelize-cli db:migrate && sequelize-cli db:seed:all", "job-runner": "node job-runner.js", + "usage-collector": "node usage-collector.js", "client:install": "npm --prefix client install", "client:dev": "npm --prefix client run dev", "client:build": "npm --prefix client run build", @@ -20,6 +21,10 @@ "client:type-check": "npm --prefix client run type-check" }, "dependencies": { + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/exporter-metrics-otlp-http": "^0.221.0", + "@opentelemetry/resources": "^2.10.0", + "@opentelemetry/sdk-metrics": "^2.10.0", "argon2": "^0.44.0", "axios": "^1.18.0", "cookie-parser": "^1.4.7", diff --git a/create-a-container/usage-collector.js b/create-a-container/usage-collector.js new file mode 100644 index 00000000..df53bc46 --- /dev/null +++ b/create-a-container/usage-collector.js @@ -0,0 +1,272 @@ +#!/usr/bin/env node +/** + * usage-collector.js — per-container resource metrics via OpenTelemetry + * (issue #440). + * + * On every export cycle (5 minutes by default) it issues ONE Proxmox + * `/cluster/resources` call per cluster (nodes sharing a cluster are + * deduplicated by the node names present in each response — never one call + * per container), attributes each LXC to an owner via its Proxmox tag + * (cross-checked against Container.username; divergence is logged as + * attribution drift), and emits OTLP metrics with `owner`, `container.id`, + * `container.name`, `proxmox.node`, and `site.id` attributes — so any OTel + * backend can group per-owner (e.g. PromQL `sum by (owner)`). + * + * Metrics (OTel semantic conventions where they exist): + * container.cpu.usage / container.cpu.limit gauge {cpu} cores + * container.memory.usage / container.memory.limit gauge By + * container.disk.usage / container.disk.limit gauge By + * container.disk.io counter By (attr disk.io.direction: read|write) + * container.network.io counter By (attr network.io.direction: receive|transmit) + * container.uptime gauge s + * + * The counters are cumulative since container boot as reported by Proxmox; a + * decrease (container reboot) is a normal counter reset for the backend. + * + * Export target comes from the standard OTLP environment variables + * (OTEL_EXPORTER_OTLP_ENDPOINT et al., http/protobuf). Without an endpoint + * configured the collector logs that it is disabled and exits — the service + * is a no-op until an OTel collector exists to receive the data. + */ + +const db = require('./models'); +const { buildUsageSample } = require('./utils/usage-sample'); + +const EXPORT_INTERVAL_MS = parseInt(process.env.USAGE_COLLECTOR_INTERVAL_MS || '300000', 10); + +/** + * Load every DB container that exists in Proxmox, keyed by `${nodeId}:${vmid}` + * for O(1) attribution lookups against cluster-resources entries. + * @returns {Promise>} + */ +async function loadContainerIndex() { + const containers = await db.Container.findAll({ + where: { containerId: { [db.Sequelize.Op.ne]: null } }, + attributes: ['id', 'containerId', 'nodeId', 'username'], + }); + const index = new Map(); + for (const c of containers) { + index.set(`${c.nodeId}:${c.containerId}`, c); + } + return index; +} + +/** + * Gather one cycle of normalized usage samples. One API call per cluster: + * after each successful `clusterResources('lxc')` call, every node name + * appearing in the response is marked covered (within the same site), so + * cluster peers are not re-polled. Per-node failures are logged and skipped; + * the cycle always completes. + * @returns {Promise>} Normalized samples (see utils/usage-sample.js) + */ +async function collectSamples() { + const nodes = await db.Node.findAll({ where: db.Node.provisionableWhere() }); + if (nodes.length === 0) return []; + + const containerIndex = await loadContainerIndex(); + + // Node lookup by `${siteId}:${name}` — cluster-resources rows carry the + // Proxmox node name, and node names are only meaningful within a site. + const nodesByName = new Map(nodes.map((n) => [`${n.siteId}:${n.name}`, n])); + const covered = new Set(); + + const samples = []; + const findings = []; + let unknownNodeRows = 0; + + for (const node of nodes) { + const nodeKey = `${node.siteId}:${node.name}`; + if (covered.has(nodeKey)) continue; + covered.add(nodeKey); + + let resources; + try { + const api = await node.api(); + resources = await api.clusterResources('lxc'); + } catch (err) { + console.error(`UsageCollector: node ${node.name} (site ${node.siteId}) unreachable: ${err.message}`); + continue; + } + if (!Array.isArray(resources)) continue; + + for (const resource of resources) { + if (resource.vmid == null) continue; + const resourceNodeKey = `${node.siteId}:${resource.node}`; + covered.add(resourceNodeKey); + + const dbNode = nodesByName.get(resourceNodeKey); + if (!dbNode) { + // Cluster member not registered in the DB — no site/node to attribute + // the sample to; count it so the drift is visible in the summary. + unknownNodeRows++; + continue; + } + + const container = containerIndex.get(`${dbNode.id}:${resource.vmid}`) || null; + const { sample, finding } = buildUsageSample({ resource, node: dbNode, container }); + samples.push(sample); + if (finding) findings.push(finding); + } + } + + for (const finding of findings) { + if (finding.kind === 'drift') { + console.warn( + `UsageCollector: attribution drift on CT ${finding.vmid}: ` + + `Proxmox tag '${finding.tagOwner}' != Container.username '${finding.dbOwner}'` + ); + } else { + console.warn(`UsageCollector: CT ${finding.vmid} has no owner (no tag, not in DB)`); + } + } + + const driftCount = findings.filter((f) => f.kind === 'drift').length; + console.log( + `UsageCollector: observed ${samples.length} containers ` + + `(${driftCount} drift, ${findings.length - driftCount} unattributed` + + `${unknownNodeRows ? `, ${unknownNodeRows} on unregistered nodes` : ''})` + ); + + return samples; +} + +/** + * OTel attribute set identifying one container series. + * @param {object} sample - Normalized sample from utils/usage-sample.js + * @returns {object} + */ +function sampleAttributes(sample) { + const attributes = { + 'container.id': sample.vmid, + 'site.id': String(sample.siteId), + 'proxmox.node': sample.node, + }; + if (sample.owner) attributes.owner = sample.owner; + if (sample.name) attributes['container.name'] = sample.name; + if (sample.status) attributes['container.status'] = sample.status; + return attributes; +} + +/** + * Register the meter provider, instruments, and the batch observable callback + * that drives collection once per export interval. + * @returns {import('@opentelemetry/sdk-metrics').MeterProvider} + */ +function setupMetrics() { + const { diag, DiagConsoleLogger, DiagLogLevel } = require('@opentelemetry/api'); + const { MeterProvider, PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics'); + const { OTLPMetricExporter } = require('@opentelemetry/exporter-metrics-otlp-http'); + const { resourceFromAttributes } = require('@opentelemetry/resources'); + + // Surface export failures (endpoint down, auth, ...) on the console. The + // standard OTEL_LOG_LEVEL variable raises verbosity for debugging. + const diagLevel = DiagLogLevel[(process.env.OTEL_LOG_LEVEL || 'WARN').toUpperCase()] ?? DiagLogLevel.WARN; + diag.setLogger(new DiagConsoleLogger(), diagLevel); + + const meterProvider = new MeterProvider({ + resource: resourceFromAttributes({ + 'service.name': process.env.OTEL_SERVICE_NAME || 'usage-collector', + }), + readers: [ + new PeriodicExportingMetricReader({ + // Endpoint/headers/protocol come from the standard OTEL_EXPORTER_OTLP_* + // environment variables. + exporter: new OTLPMetricExporter(), + exportIntervalMillis: EXPORT_INTERVAL_MS, + }), + ], + }); + + const meter = meterProvider.getMeter('usage-collector'); + + const cpuUsage = meter.createObservableGauge('container.cpu.usage', { unit: '{cpu}', description: 'CPU cores in use' }); + const cpuLimit = meter.createObservableGauge('container.cpu.limit', { unit: '{cpu}', description: 'CPU cores allocated' }); + const memUsage = meter.createObservableGauge('container.memory.usage', { unit: 'By', description: 'Memory in use' }); + const memLimit = meter.createObservableGauge('container.memory.limit', { unit: 'By', description: 'Memory allocated' }); + const diskUsage = meter.createObservableGauge('container.disk.usage', { unit: 'By', description: 'Root filesystem in use' }); + const diskLimit = meter.createObservableGauge('container.disk.limit', { unit: 'By', description: 'Root filesystem allocated' }); + const diskIo = meter.createObservableCounter('container.disk.io', { unit: 'By', description: 'Disk bytes transferred since container boot' }); + const networkIo = meter.createObservableCounter('container.network.io', { unit: 'By', description: 'Network bytes transferred since container boot' }); + const uptime = meter.createObservableGauge('container.uptime', { unit: 's', description: 'Seconds since container boot' }); + + const instruments = [cpuUsage, cpuLimit, memUsage, memLimit, diskUsage, diskLimit, diskIo, networkIo, uptime]; + + meter.addBatchObservableCallback(async (result) => { + let samples; + try { + samples = await collectSamples(); + } catch (err) { + console.error('UsageCollector cycle error:', err); + return; + } + + for (const sample of samples) { + const attributes = sampleAttributes(sample); + const gauges = [ + [cpuUsage, sample.cpuUsed], + [cpuLimit, sample.cpuAlloc], + [memUsage, sample.memUsed], + [memLimit, sample.memAlloc], + [diskUsage, sample.diskUsed], + [diskLimit, sample.diskAlloc], + [uptime, sample.uptime], + ]; + for (const [instrument, value] of gauges) { + if (value != null) result.observe(instrument, value, attributes); + } + if (sample.diskReadBytes != null) { + result.observe(diskIo, sample.diskReadBytes, { ...attributes, 'disk.io.direction': 'read' }); + } + if (sample.diskWriteBytes != null) { + result.observe(diskIo, sample.diskWriteBytes, { ...attributes, 'disk.io.direction': 'write' }); + } + if (sample.netInBytes != null) { + result.observe(networkIo, sample.netInBytes, { ...attributes, 'network.io.direction': 'receive' }); + } + if (sample.netOutBytes != null) { + result.observe(networkIo, sample.netOutBytes, { ...attributes, 'network.io.direction': 'transmit' }); + } + } + }, instruments); + + return meterProvider; +} + +function otlpConfigured() { + return !!(process.env.OTEL_EXPORTER_OTLP_ENDPOINT || process.env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT); +} + +async function start() { + if (!otlpConfigured()) { + console.log( + 'UsageCollector: no OTLP endpoint configured (set OTEL_EXPORTER_OTLP_ENDPOINT ' + + 'or OTEL_EXPORTER_OTLP_METRICS_ENDPOINT); exiting.' + ); + process.exit(0); + } + + console.log(`UsageCollector starting, export interval ${EXPORT_INTERVAL_MS}ms`); + const meterProvider = setupMetrics(); + + // The SDK's periodic reader unrefs its timer, so it alone does not keep the + // process alive; hold the event loop open until a signal arrives. + const keepAlive = setInterval(() => {}, 60 * 1000); + + const shutdown = async (signal) => { + console.log(`UsageCollector shutting down (${signal})`); + clearInterval(keepAlive); + try { + await meterProvider.shutdown(); // flushes the final export + } catch (err) { + console.error('UsageCollector shutdown error:', err); + } + process.exit(0); + }; + process.on('SIGINT', () => shutdown('SIGINT')); + process.on('SIGTERM', () => shutdown('SIGTERM')); +} + +start().catch((err) => { + console.error('UsageCollector failed to start:', err); + process.exit(1); +}); diff --git a/create-a-container/utils/__tests__/usage-sample.test.js b/create-a-container/utils/__tests__/usage-sample.test.js new file mode 100644 index 00000000..7818fc66 --- /dev/null +++ b/create-a-container/utils/__tests__/usage-sample.test.js @@ -0,0 +1,133 @@ +'use strict'; + +const { parseOwnerTag, cpuCoresUsed, buildUsageSample } = require('../usage-sample'); + +describe('parseOwnerTag', () => { + test('returns the single tag as owner', () => { + expect(parseOwnerTag('horner')).toBe('horner'); + }); + + test('returns the first tag of a semicolon-separated list', () => { + expect(parseOwnerTag('cmyers;gpu;prod')).toBe('cmyers'); + }); + + test('skips empty leading segments and trims whitespace', () => { + expect(parseOwnerTag('; rgara ;x')).toBe('rgara'); + }); + + test('returns null for missing or non-string tags', () => { + expect(parseOwnerTag(undefined)).toBeNull(); + expect(parseOwnerTag(null)).toBeNull(); + expect(parseOwnerTag('')).toBeNull(); + expect(parseOwnerTag(';;')).toBeNull(); + expect(parseOwnerTag(42)).toBeNull(); + }); +}); + +describe('cpuCoresUsed', () => { + test('converts the utilization fraction to cores', () => { + expect(cpuCoresUsed(0.5, 4)).toBe(2); + expect(cpuCoresUsed(0, 8)).toBe(0); + }); + + test('returns null when either input is missing', () => { + expect(cpuCoresUsed(undefined, 4)).toBeNull(); + expect(cpuCoresUsed(0.5, undefined)).toBeNull(); + }); +}); + +describe('buildUsageSample', () => { + const node = { name: 'pve2', siteId: 3 }; + const resource = { + vmid: 284, + name: 'ozwell-studio-92e3d441', + node: 'pve2', + status: 'running', + tags: 'zbarrell', + cpu: 0.25, + maxcpu: 4, + mem: 4262461440, + maxmem: 4294967296, + disk: 10737418240, + maxdisk: 53687091200, + diskread: 1000, + diskwrite: 2000, + netin: 3000, + netout: 4000, + uptime: 86400, + }; + + test('maps all fields from a cluster-resources entry', () => { + const container = { id: 11, username: 'zbarrell' }; + const { sample, finding } = buildUsageSample({ resource, node, container }); + + expect(finding).toBeNull(); + expect(sample).toEqual({ + vmid: '284', + name: 'ozwell-studio-92e3d441', + owner: 'zbarrell', + node: 'pve2', + siteId: 3, + status: 'running', + cpuUsed: 1, + cpuAlloc: 4, + memUsed: 4262461440, + memAlloc: 4294967296, + diskUsed: 10737418240, + diskAlloc: 53687091200, + diskReadBytes: 1000, + diskWriteBytes: 2000, + netInBytes: 3000, + netOutBytes: 4000, + uptime: 86400, + }); + }); + + test('reports drift when tag and DB owner disagree; tag wins', () => { + const container = { id: 11, username: 'someoneelse' }; + const { sample, finding } = buildUsageSample({ resource, node, container }); + + expect(sample.owner).toBe('zbarrell'); + expect(finding).toEqual({ + kind: 'drift', + vmid: '284', + tagOwner: 'zbarrell', + dbOwner: 'someoneelse', + }); + }); + + test('falls back to the DB owner when the container is untagged', () => { + const untagged = { ...resource, tags: undefined }; + const container = { id: 11, username: 'cmyers' }; + const { sample, finding } = buildUsageSample({ resource: untagged, node, container }); + + expect(sample.owner).toBe('cmyers'); + expect(finding).toBeNull(); + }); + + test('flags unattributed containers (no tag, not in DB)', () => { + const untagged = { ...resource, tags: undefined }; + const { sample, finding } = buildUsageSample({ resource: untagged, node, container: null }); + + expect(sample.owner).toBeNull(); + expect(finding).toEqual({ + kind: 'unattributed', + vmid: '284', + tagOwner: null, + dbOwner: null, + }); + }); + + test('maps absent metrics to null (sparse sources like DummyApi)', () => { + const sparse = { vmid: '5', node: 'dummy', status: 'running', tags: 'cmyers' }; + const { sample } = buildUsageSample({ resource: sparse, node, container: null }); + + expect(sample.cpuUsed).toBeNull(); + expect(sample.cpuAlloc).toBeNull(); + expect(sample.memUsed).toBeNull(); + expect(sample.diskReadBytes).toBeNull(); + expect(sample.uptime).toBeNull(); + expect(sample.name).toBeNull(); + expect(sample.status).toBe('running'); + }); +}); diff --git a/create-a-container/utils/dummy-api.js b/create-a-container/utils/dummy-api.js index 0dc0de0c..f5a11745 100644 --- a/create-a-container/utils/dummy-api.js +++ b/create-a-container/utils/dummy-api.js @@ -315,7 +315,7 @@ class DummyApi { const { Container } = require('../models'); const containers = await Container.findAll({ where: { nodeId: this.node.id, containerId: { [require('sequelize').Op.ne]: null } }, - attributes: ['containerId', 'hostname'], + attributes: ['containerId', 'hostname', 'username'], }); return containers.map((c) => ({ vmid: c.containerId, @@ -323,6 +323,21 @@ class DummyApi { type: 'lxc', status: 'running', node: this.node.name || 'dummy', + // Real nodes tag containers with their owner (see bin/create-container.js); + // mirror that so owner attribution (usage-collector.js) works in dev. + tags: c.username, + // Simulated resource usage so the usage collector has data in dev. + cpu: 0.05, + maxcpu: 2, + mem: 256 * 1024 * 1024, + maxmem: 1024 * 1024 * 1024, + disk: 2 * 1024 * 1024 * 1024, + maxdisk: 8 * 1024 * 1024 * 1024, + diskread: 100 * 1024 * 1024, + diskwrite: 50 * 1024 * 1024, + netin: 10 * 1024 * 1024, + netout: 5 * 1024 * 1024, + uptime: 3600, })); } } diff --git a/create-a-container/utils/usage-sample.js b/create-a-container/utils/usage-sample.js new file mode 100644 index 00000000..2bc9c67f --- /dev/null +++ b/create-a-container/utils/usage-sample.js @@ -0,0 +1,110 @@ +'use strict'; + +/** + * usage-sample.js — pure helpers for the usage collector (issue #440). + * + * Turns one Proxmox `/cluster/resources` LXC entry into a normalized usage + * sample and performs owner attribution: the Proxmox tag is the primary source + * (it lives on the container itself and survives DB loss), cross-checked + * against Container.username. Divergence between the two is reported as + * attribution drift. + * + * No I/O here — everything is a pure function so it is trivially unit-tested; + * usage-collector.js owns the polling, DB lookups, and OTLP emission. + */ + +/** + * Extract the owner from a Proxmox `tags` field. create-container.js and the + * owner-change path in routers/api/v1/containers.js write the owner username + * as the container's tag; Proxmox stores tags as a `;`-separated list, so the + * first tag is the owner. + * @param {string|null|undefined} tags - Raw `tags` value from /cluster/resources + * @returns {string|null} Owner username, or null when untagged + */ +function parseOwnerTag(tags) { + if (!tags || typeof tags !== 'string') return null; + const first = tags.split(';').map((t) => t.trim()).find((t) => t.length > 0); + return first || null; +} + +/** + * Convert Proxmox's `cpu` field (utilization as a fraction of the container's + * allocated cores) into cores in use. + * @param {number|undefined} cpu - Fraction 0..1 of maxcpu + * @param {number|undefined} maxcpu - Allocated cores + * @returns {number|null} Cores in use + */ +function cpuCoresUsed(cpu, maxcpu) { + if (typeof cpu !== 'number' || typeof maxcpu !== 'number') return null; + return cpu * maxcpu; +} + +/** + * Coerce an optional numeric API field, mapping absent/invalid to null so + * sparse sources (e.g. DummyApi snapshots) yield omitted metrics, not NaN. + * @param {*} value + * @returns {number|null} + */ +function numberOrNull(value) { + return typeof value === 'number' && Number.isFinite(value) ? value : null; +} + +/** + * Build one normalized usage sample (plus any attribution finding) from a + * cluster-resources LXC entry. + * + * Owner resolution: Proxmox tag if present, else the DB container's username. + * When both exist and disagree, the tag wins and a `drift` finding is + * returned. When neither exists an `unattributed` finding is returned. + * + * Metric semantics: `cpuUsed`/`memUsed`/`diskUsed` and their `*Alloc` + * counterparts are point-in-time gauges; `diskReadBytes`/`diskWriteBytes`/ + * `netInBytes`/`netOutBytes` are cumulative counters since container boot + * (monotonic sums — a decrease means the container rebooted). + * + * @param {object} params + * @param {object} params.resource - One `/cluster/resources` entry (type lxc) + * @param {object} params.node - DB Node the entry belongs to (name, siteId) + * @param {object|null} params.container - Matching DB Container row, if any + * @returns {{ sample: object, finding: { kind: 'drift'|'unattributed', vmid: string, tagOwner: string|null, dbOwner: string|null }|null }} + */ +function buildUsageSample({ resource, node, container }) { + const tagOwner = parseOwnerTag(resource.tags); + const dbOwner = container ? container.username : null; + const owner = tagOwner || dbOwner; + + let finding = null; + if (tagOwner && dbOwner && tagOwner !== dbOwner) { + finding = { kind: 'drift', vmid: String(resource.vmid), tagOwner, dbOwner }; + } else if (!owner) { + finding = { kind: 'unattributed', vmid: String(resource.vmid), tagOwner, dbOwner }; + } + + const sample = { + vmid: String(resource.vmid), + name: resource.name || null, + owner, + node: node.name, + siteId: node.siteId, + status: resource.status || null, + cpuUsed: cpuCoresUsed(resource.cpu, resource.maxcpu), + cpuAlloc: numberOrNull(resource.maxcpu), + memUsed: numberOrNull(resource.mem), + memAlloc: numberOrNull(resource.maxmem), + diskUsed: numberOrNull(resource.disk), + diskAlloc: numberOrNull(resource.maxdisk), + diskReadBytes: numberOrNull(resource.diskread), + diskWriteBytes: numberOrNull(resource.diskwrite), + netInBytes: numberOrNull(resource.netin), + netOutBytes: numberOrNull(resource.netout), + uptime: numberOrNull(resource.uptime), + }; + + return { sample, finding }; +} + +module.exports = { + parseOwnerTag, + cpuCoresUsed, + buildUsageSample, +}; diff --git a/mie-opensource-landing/docs/developers/development-workflow.md b/mie-opensource-landing/docs/developers/development-workflow.md index 3ace4b25..2280209a 100644 --- a/mie-opensource-landing/docs/developers/development-workflow.md +++ b/mie-opensource-landing/docs/developers/development-workflow.md @@ -31,7 +31,8 @@ This will: `localhost` external domain) — but only when the database is empty, so it never interferes with the Docker stack's bootstrap. 4. Build the React client. -5. Start the **server** and the **job-runner** together, serving at +5. Start the **server**, the **job-runner**, and the **usage-collector** + together, serving at [http://localhost:3000](http://localhost:3000). No configuration is required — the server's defaults are sufficient for diff --git a/mie-opensource-landing/docs/developers/release-pipeline.md b/mie-opensource-landing/docs/developers/release-pipeline.md index a1b68747..b91f5edc 100644 --- a/mie-opensource-landing/docs/developers/release-pipeline.md +++ b/mie-opensource-landing/docs/developers/release-pipeline.md @@ -79,7 +79,7 @@ make -C mie-opensource-landing dev # docs live server Each component has a `.fpm` options file holding the static package metadata (name, architecture, dependencies, description, scripts, config files). The Makefile's `package` target stages the component into a `.pkg/buildroot` and runs [fpm](https://fpm.readthedocs.io/) with the dynamic options on the command line — output type, version (composed per format by `./package-version`), and the staging dir. fpm's `dir` input copies the staged tree verbatim from `-C .pkg/buildroot`, preserving symlinks (e.g. `node_modules/.bin/sequelize`) and the directory layout. The same definition produces deb, rpm, and apk, so `make rpm` and `make apk` also work. -- `opensource-server` ships the `container-creator` and `job-runner` systemd units and enables them via an `after-install` script (`before-remove` disables them on real removal). The log directory is created on demand by the unit's `LogsDirectory`, not shipped in the package. The logrotate drop-in is the only config file. The `container-creator-init` unit (which provisions a *local* PostgreSQL) is **not** in the package — it is part of the manager image, since the package only suggests postgresql and works with a remote database too. +- `opensource-server` ships the `container-creator`, `job-runner` and `usage-collector` systemd units and enables them via an `after-install` script (`before-remove` disables them on real removal). The log directory is created on demand by the unit's `LogsDirectory`, not shipped in the package. The logrotate drop-in is the only config file. The `container-creator-init` unit (which provisions a *local* PostgreSQL) is **not** in the package — it is part of the manager image, since the package only suggests postgresql and works with a remote database too. - `opensource-agent` ships the compiled check-in agent, its config templates, the systemd service + 30s timer (enabled via an `after-install` script) and the static error pages. These are program code, not configuration (runtime config comes from `/etc/environment`), so nothing is marked as a config file. - `opensource-docs` ships content only. From 54c883b690cd8fcabfae2aded16bea7f7b60e609 Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Tue, 11 Aug 2026 14:52:16 -0700 Subject: [PATCH 2/3] Add live per-owner usage report (issue #440) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /api/v1/sites/:siteId/usage computes the per-owner report from the same one-call-per-cluster sampling cycle the OTLP collector uses, extracted into utils/usage-collection.js so both consumers see identical data. Aggregation (utils/usage-report.js) is pure and unit-tested; every metric shows allocated alongside used, never used alone. Visibility mirrors the containers list: admins see every owner on the site (plus attribution findings — tag/DB drift, unattributed containers, unregistered cluster nodes); other users see their own containers plus ones shared with them. Client: /sites/:siteId/usage page (sidebar link for all users) renders a DataVis grid of owners — each row expands to per-container detail — with an admin-only attribution warning banner. Byte formatting moved to lib/format.ts, shared with ResourceBar. --- create-a-container/README.md | 2 + create-a-container/client/src/app/Sidebar.tsx | 13 ++ create-a-container/client/src/app/router.tsx | 3 + .../src/components/nodes/ResourceBar.tsx | 15 +-- .../components/usage/AttributionWarnings.tsx | 50 ++++++++ .../components/usage/OwnerContainersTable.tsx | 68 +++++++++++ .../src/components/usage/UsageDataGrid.tsx | 113 ++++++++++++++++++ create-a-container/client/src/lib/format.ts | 17 +++ create-a-container/client/src/lib/queries.ts | 6 + create-a-container/client/src/lib/types.ts | 58 +++++++++ .../client/src/pages/usage/UsagePage.tsx | 62 ++++++++++ create-a-container/openapi.v1.yaml | 35 ++++++ create-a-container/routers/api/v1/sites.js | 1 + create-a-container/routers/api/v1/usage.js | 55 +++++++++ create-a-container/usage-collector.js | 78 +----------- .../utils/__tests__/usage-report.test.js | 76 ++++++++++++ .../utils/__tests__/usage-sample.test.js | 2 + create-a-container/utils/usage-collection.js | 104 ++++++++++++++++ create-a-container/utils/usage-report.js | 52 ++++++++ create-a-container/utils/usage-sample.js | 3 + 20 files changed, 727 insertions(+), 86 deletions(-) create mode 100644 create-a-container/client/src/components/usage/AttributionWarnings.tsx create mode 100644 create-a-container/client/src/components/usage/OwnerContainersTable.tsx create mode 100644 create-a-container/client/src/components/usage/UsageDataGrid.tsx create mode 100644 create-a-container/client/src/lib/format.ts create mode 100644 create-a-container/client/src/pages/usage/UsagePage.tsx create mode 100644 create-a-container/routers/api/v1/usage.js create mode 100644 create-a-container/utils/__tests__/usage-report.test.js create mode 100644 create-a-container/utils/usage-collection.js create mode 100644 create-a-container/utils/usage-report.js diff --git a/create-a-container/README.md b/create-a-container/README.md index 01d27bd8..d25f8808 100644 --- a/create-a-container/README.md +++ b/create-a-container/README.md @@ -56,6 +56,8 @@ The Manager is not installed by hand in production. It ships as: In both cases the app runs `server.js` (HTTP API + UI), `job-runner.js` (background worker), and `usage-collector.js` (per-container resource metrics exported as OTLP; a no-op unless `OTEL_EXPORTER_OTLP_ENDPOINT` is configured). +The same per-owner data is available live in the UI at `/sites/:siteId/usage` +(API: `GET /api/v1/sites/:siteId/usage`) with no OTel backend required. Database connection settings come from the environment (see [Configuration](#configuration)); the manager image provisions PostgreSQL and writes these to `/etc/default/container-creator` on first boot. diff --git a/create-a-container/client/src/app/Sidebar.tsx b/create-a-container/client/src/app/Sidebar.tsx index 33fe8dce..a2b83bb4 100644 --- a/create-a-container/client/src/app/Sidebar.tsx +++ b/create-a-container/client/src/app/Sidebar.tsx @@ -13,6 +13,7 @@ import { useSidebar, } from '@mieweb/ui'; import { + Activity, Box, Building2, ClipboardList, @@ -166,6 +167,12 @@ export function AppSidebar() { icon: , match: `/sites/${currentSiteId}/containers`, })} + {renderLink({ + to: `/sites/${currentSiteId}/usage`, + label: 'Usage', + icon: , + match: `/sites/${currentSiteId}/usage`, + })} {isAdmin && renderLink({ to: `/sites/${currentSiteId}/nodes`, label: 'Nodes', @@ -185,6 +192,12 @@ export function AppSidebar() { icon: , match: `/sites/${currentSiteId}/containers`, })} + {renderLink({ + to: `/sites/${currentSiteId}/usage`, + label: 'Usage', + icon: , + match: `/sites/${currentSiteId}/usage`, + })} {isAdmin && renderLink({ to: `/sites/${currentSiteId}/nodes`, label: 'Nodes', diff --git a/create-a-container/client/src/app/router.tsx b/create-a-container/client/src/app/router.tsx index 50e2d7cc..a0450d92 100644 --- a/create-a-container/client/src/app/router.tsx +++ b/create-a-container/client/src/app/router.tsx @@ -14,6 +14,7 @@ import { ContainerFormPage } from '@/pages/containers/ContainerFormPage'; import { NodesListPage } from '@/pages/nodes/NodesListPage'; import { NodeFormPage } from '@/pages/nodes/NodeFormPage'; import { NodeImportPage } from '@/pages/nodes/NodeImportPage'; +import { UsagePage } from '@/pages/usage/UsagePage'; import { ExternalDomainsListPage } from '@/pages/external-domains/ExternalDomainsListPage'; import { ExternalDomainFormPage } from '@/pages/external-domains/ExternalDomainFormPage'; import { AgentsListPage } from '@/pages/agents/AgentsListPage'; @@ -62,6 +63,8 @@ export const router = createBrowserRouter([ { path: '/sites/:siteId/nodes/import', element: }, { path: '/sites/:siteId/nodes/:id/edit', element: }, + { path: '/sites/:siteId/usage', element: }, + { path: '/external-domains', element: }, { path: '/external-domains/new', element: }, { path: '/external-domains/:id/edit', element: }, diff --git a/create-a-container/client/src/components/nodes/ResourceBar.tsx b/create-a-container/client/src/components/nodes/ResourceBar.tsx index fa081612..a4c9d11b 100644 --- a/create-a-container/client/src/components/nodes/ResourceBar.tsx +++ b/create-a-container/client/src/components/nodes/ResourceBar.tsx @@ -1,18 +1,5 @@ import { Progress } from '@mieweb/ui'; - -const KiB = 1024; -const MiB = KiB * 1024; -const GiB = MiB * 1024; -const TiB = GiB * 1024; - -/** Human-readable byte size using binary units labelled with familiar suffixes. */ -function formatBytes(bytes: number): string { - if (bytes >= TiB) return `${(bytes / TiB).toFixed(1)} TB`; - if (bytes >= GiB) return `${(bytes / GiB).toFixed(1)} GB`; - if (bytes >= MiB) return `${Math.round(bytes / MiB)} MB`; - if (bytes >= KiB) return `${Math.round(bytes / KiB)} KB`; - return `${bytes} B`; -} +import { formatBytes } from '@/lib/format'; function variantFor(pct: number): 'success' | 'warning' | 'danger' { if (pct >= 90) return 'danger'; diff --git a/create-a-container/client/src/components/usage/AttributionWarnings.tsx b/create-a-container/client/src/components/usage/AttributionWarnings.tsx new file mode 100644 index 00000000..93c94beb --- /dev/null +++ b/create-a-container/client/src/components/usage/AttributionWarnings.tsx @@ -0,0 +1,50 @@ +import { Alert, AlertDescription } from '@mieweb/ui'; +import type { UsageFinding } from '@/lib/types'; + +export interface AttributionWarningsProps { + findings: UsageFinding[]; + /** Cluster members seen in Proxmox but not registered in the manager DB. */ + unknownNodeRows?: number; +} + +/** + * Admin-only warning banner for owner-attribution problems: drift between the + * Proxmox owner tag and the manager DB, containers with no owner at all, and + * cluster nodes the manager does not know about. + */ +export function AttributionWarnings({ findings, unknownNodeRows = 0 }: AttributionWarningsProps) { + if (findings.length === 0 && unknownNodeRows === 0) return null; + + const drift = findings.filter((f) => f.kind === 'drift'); + const unattributed = findings.filter((f) => f.kind === 'unattributed'); + + return ( + + +
+ {drift.length > 0 && ( + + Attribution drift on {drift.length} container{drift.length === 1 ? '' : 's'}:{' '} + {drift + .map((f) => `CT ${f.vmid} (tag '${f.tagOwner}' ≠ DB '${f.dbOwner}')`) + .join(', ')} + + )} + {unattributed.length > 0 && ( + + {unattributed.length} container{unattributed.length === 1 ? '' : 's'} with no owner + (no Proxmox tag, not in the manager DB):{' '} + {unattributed.map((f) => `CT ${f.vmid}`).join(', ')} + + )} + {unknownNodeRows > 0 && ( + + {unknownNodeRows} container{unknownNodeRows === 1 ? '' : 's'} on cluster nodes not + registered in the manager. + + )} +
+ + + ); +} diff --git a/create-a-container/client/src/components/usage/OwnerContainersTable.tsx b/create-a-container/client/src/components/usage/OwnerContainersTable.tsx new file mode 100644 index 00000000..b5a7cce3 --- /dev/null +++ b/create-a-container/client/src/components/usage/OwnerContainersTable.tsx @@ -0,0 +1,68 @@ +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@mieweb/ui'; +import type { UsageContainer } from '@/lib/types'; +import { formatBytes } from '@/lib/format'; + +/** "left / right" pair where either side may be missing, e.g. used/alloc or read/write. */ +function pair( + left: number | null, + right: number | null, + format: (v: number) => string, +): string { + const l = left != null ? format(left) : '—'; + const r = right != null ? format(right) : '—'; + return `${l} / ${r}`; +} + +const cores = (v: number) => v.toFixed(2); + +export interface OwnerContainersTableProps { + containers: UsageContainer[]; +} + +/** + * Per-container usage detail shown when an owner row in the usage grid is + * expanded. I/O and network figures are cumulative since container boot. + */ +export function OwnerContainersTable({ containers }: OwnerContainersTableProps) { + return ( +
+ + + + CT + Name + Node + Status + CPU (cores) + Memory + Disk + Disk I/O (r / w) + Network (in / out) + + + + {containers.map((c) => ( + + {c.vmid} + {c.name || '—'} + {c.node} + {c.status || '—'} + {pair(c.cpuUsed, c.cpuAlloc, cores)} + {pair(c.memUsed, c.memAlloc, formatBytes)} + {pair(c.diskUsed, c.diskAlloc, formatBytes)} + {pair(c.diskReadBytes, c.diskWriteBytes, formatBytes)} + {pair(c.netInBytes, c.netOutBytes, formatBytes)} + + ))} + +
+
+ ); +} diff --git a/create-a-container/client/src/components/usage/UsageDataGrid.tsx b/create-a-container/client/src/components/usage/UsageDataGrid.tsx new file mode 100644 index 00000000..92028c99 --- /dev/null +++ b/create-a-container/client/src/components/usage/UsageDataGrid.tsx @@ -0,0 +1,113 @@ +import { useCallback, useEffect, useMemo } from 'react'; +import { DataVisNitroGrid, DataVisNitroSource } from '@mieweb/ui/datavis'; +import type { TableColumn, TableRendererProps } from '@mieweb/datavis'; +import type { UsageOwner } from '@/lib/types'; +import { formatBytes } from '@/lib/format'; +import { OwnerContainersTable } from './OwnerContainersTable'; + +// Numeric `*Used` fields drive sorting; formatCell renders "used / allocated". +const TYPE_INFO: { field: string; type: string }[] = [ + { field: 'owner', type: 'string' }, + { field: 'containerCount', type: 'number' }, + { field: 'cpuUsed', type: 'number' }, + { field: 'memUsed', type: 'number' }, + { field: 'diskUsed', type: 'number' }, + { field: 'diskReadBytes', type: 'number' }, + { field: 'netInBytes', type: 'number' }, +]; + +const asOwner = (row: Record) => row as unknown as UsageOwner; + +type FormatCell = NonNullable; +type DetailRow = NonNullable; + +export interface UsageDataGridProps { + owners: UsageOwner[]; +} + +/** + * Per-owner usage rows (allocated alongside used, always) rendered with + * DataVis NITRO. Each row expands to the owner's per-container detail. + */ +export function UsageDataGrid({ owners }: UsageDataGridProps) { + const url = useMemo(() => { + const payload = { typeInfo: TYPE_INFO, data: owners }; + const blob = new Blob([JSON.stringify(payload)], { type: 'application/json' }); + return URL.createObjectURL(blob); + }, [owners]); + useEffect(() => () => URL.revokeObjectURL(url), [url]); + + const columns = useMemo( + () => [ + { + field: 'owner', + header: 'Owner', + sortable: true, + filterable: true, + getSearchText: (_v, row) => asOwner(row).owner ?? 'unattributed', + }, + { + field: 'containerCount', + header: 'Containers (running)', + sortable: true, + filterable: false, + getSearchText: () => '', + }, + { field: 'cpuUsed', header: 'CPU cores (used / alloc)', sortable: true, filterable: false, getSearchText: () => '' }, + { field: 'memUsed', header: 'Memory (used / alloc)', sortable: true, filterable: false, getSearchText: () => '' }, + { field: 'diskUsed', header: 'Disk (used / alloc)', sortable: true, filterable: false, getSearchText: () => '' }, + { field: 'diskReadBytes', header: 'Disk I/O (r / w)', sortable: true, filterable: false, getSearchText: () => '' }, + { field: 'netInBytes', header: 'Network (in / out)', sortable: true, filterable: false, getSearchText: () => '' }, + ], + [], + ); + + const formatCell = useCallback((value, row, column) => { + const o = asOwner(row); + switch (column.field) { + case 'owner': + return o.owner ? ( + {o.owner} + ) : ( + unattributed + ); + case 'containerCount': + return `${o.containerCount} (${o.runningCount})`; + case 'cpuUsed': + return `${o.cpuUsed.toFixed(2)} / ${o.cpuAlloc}`; + case 'memUsed': + return `${formatBytes(o.memUsed)} / ${formatBytes(o.memAlloc)}`; + case 'diskUsed': + return `${formatBytes(o.diskUsed)} / ${formatBytes(o.diskAlloc)}`; + case 'diskReadBytes': + return `${formatBytes(o.diskReadBytes)} / ${formatBytes(o.diskWriteBytes)}`; + case 'netInBytes': + return `${formatBytes(o.netInBytes)} / ${formatBytes(o.netOutBytes)}`; + default: + return value as React.ReactNode; + } + }, []); + + const renderDetailRow = useCallback( + (row) => , + [], + ); + + return ( + + + + ); +} diff --git a/create-a-container/client/src/lib/format.ts b/create-a-container/client/src/lib/format.ts new file mode 100644 index 00000000..8ad32bc7 --- /dev/null +++ b/create-a-container/client/src/lib/format.ts @@ -0,0 +1,17 @@ +/** + * Shared display formatters for byte sizes and counts. + */ + +const KiB = 1024; +const MiB = KiB * 1024; +const GiB = MiB * 1024; +const TiB = GiB * 1024; + +/** Human-readable byte size using binary units labelled with familiar suffixes. */ +export function formatBytes(bytes: number): string { + if (bytes >= TiB) return `${(bytes / TiB).toFixed(1)} TB`; + if (bytes >= GiB) return `${(bytes / GiB).toFixed(1)} GB`; + if (bytes >= MiB) return `${Math.round(bytes / MiB)} MB`; + if (bytes >= KiB) return `${Math.round(bytes / KiB)} KB`; + return `${bytes} B`; +} diff --git a/create-a-container/client/src/lib/queries.ts b/create-a-container/client/src/lib/queries.ts index 54ec9821..40fec213 100644 --- a/create-a-container/client/src/lib/queries.ts +++ b/create-a-container/client/src/lib/queries.ts @@ -19,6 +19,7 @@ import type { AppSettings, ResourceRequest, Site, + UsageReport, User, } from './types'; @@ -30,6 +31,7 @@ export const keys = { ['sites', String(siteId), 'nodes', String(id)] as const, nodeStats: (siteId: number | string, id: number | string) => ['sites', String(siteId), 'nodes', String(id), 'stats'] as const, + usage: (siteId: number | string) => ['sites', String(siteId), 'usage'] as const, containers: (siteId: number | string, params?: Record) => ['sites', String(siteId), 'containers', params ?? {}] as const, container: (siteId: number | string, id: number | string) => @@ -67,6 +69,10 @@ export const queries = { getNodeStats: (siteId: number | string, id: number | string) => api.get(`/api/v1/sites/${siteId}/nodes/${id}/stats`), + // Usage + getUsage: (siteId: number | string) => + api.get(`/api/v1/sites/${siteId}/usage`), + // Agents listAgents: () => api.get('/api/v1/agents'), diff --git a/create-a-container/client/src/lib/types.ts b/create-a-container/client/src/lib/types.ts index d24f3b0b..0c67c6bc 100644 --- a/create-a-container/client/src/lib/types.ts +++ b/create-a-container/client/src/lib/types.ts @@ -73,6 +73,64 @@ export interface AgentServiceStatus { lastApply: 'success' | 'failure' | 'unknown'; } +/** One container's live usage sample in the per-owner usage report. */ +export interface UsageContainer { + vmid: string; + name: string | null; + owner: string | null; + containerDbId: number | null; + node: string; + siteId: number; + status: string | null; + cpuUsed: number | null; + cpuAlloc: number | null; + memUsed: number | null; + memAlloc: number | null; + diskUsed: number | null; + diskAlloc: number | null; + diskReadBytes: number | null; + diskWriteBytes: number | null; + netInBytes: number | null; + netOutBytes: number | null; + /** Seconds since container boot. */ + uptime: number | null; +} + +/** Per-owner aggregate row (owner null = unattributed, admin-visible only). */ +export interface UsageOwner { + owner: string | null; + containerCount: number; + runningCount: number; + cpuUsed: number; + cpuAlloc: number; + memUsed: number; + memAlloc: number; + diskUsed: number; + diskAlloc: number; + diskReadBytes: number; + diskWriteBytes: number; + netInBytes: number; + netOutBytes: number; + containers: UsageContainer[]; +} + +/** Owner-attribution problem detected during collection (admin-only). */ +export interface UsageFinding { + kind: 'drift' | 'unattributed'; + vmid: string; + tagOwner: string | null; + dbOwner: string | null; +} + +export interface UsageReport { + generatedAt: string; + owners: UsageOwner[]; + /** Admin-only. */ + findings?: UsageFinding[]; + /** Admin-only: cluster members not registered in the manager DB. */ + unknownNodeRows?: number; +} + export interface Agent { id: number; siteId: number; diff --git a/create-a-container/client/src/pages/usage/UsagePage.tsx b/create-a-container/client/src/pages/usage/UsagePage.tsx new file mode 100644 index 00000000..bb549172 --- /dev/null +++ b/create-a-container/client/src/pages/usage/UsagePage.tsx @@ -0,0 +1,62 @@ +import { useParams } from 'react-router'; +import { useQuery } from '@tanstack/react-query'; +import { Alert, AlertDescription, PageHeader, Spinner } from '@mieweb/ui'; +import { Activity } from 'lucide-react'; +import { AttributionWarnings } from '@/components/usage/AttributionWarnings'; +import { UsageDataGrid } from '@/components/usage/UsageDataGrid'; +import type { ApiError } from '@/lib/api'; +import { keys, queries } from '@/lib/queries'; + +/** + * Live per-owner resource usage for the current site (allocated vs used), + * computed on demand from the hypervisor. Admins see every owner plus + * attribution warnings; other users see their own and shared containers. + */ +export function UsagePage() { + const { siteId } = useParams<{ siteId: string }>(); + const { data: site } = useQuery({ + queryKey: keys.site(siteId!), + queryFn: () => queries.getSite(siteId!), + enabled: !!siteId, + }); + const { data, isLoading, error } = useQuery({ + queryKey: keys.usage(siteId!), + queryFn: () => queries.getUsage(siteId!), + enabled: !!siteId, + // The report is a live hypervisor query; keep it reasonably fresh while + // the page is open without hammering the Proxmox API. + refetchInterval: 60000, + }); + + return ( +
+ } + /> + {error && ( + + {(error as ApiError).message} + + )} + {isLoading && ( +
+ +
+ )} + {data && ( + + )} + {data && data.owners.length === 0 && ( + + No running containers to report on. + + )} + {data && data.owners.length > 0 && } +
+ ); +} diff --git a/create-a-container/openapi.v1.yaml b/create-a-container/openapi.v1.yaml index a9261dd3..aad0ca54 100644 --- a/create-a-container/openapi.v1.yaml +++ b/create-a-container/openapi.v1.yaml @@ -44,6 +44,7 @@ tags: - name: Sites - name: Containers - name: Nodes + - name: Usage - name: Agents - name: Jobs - name: External Domains @@ -1057,6 +1058,40 @@ paths: application/json: schema: { type: object, properties: { data: { type: object } } } '404': { description: 'Node not found (code: not_found)', content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } + /sites/{siteId}/usage: + get: + operationId: get_site_usage + tags: [Usage] + summary: Live per-owner resource usage report (allocated vs used), computed from one Proxmox cluster-resources call per cluster + description: >- + Admins see every owner on the site; other users see their own containers + plus containers shared with them. `findings` (owner-tag vs DB attribution + drift, unattributed containers) and `unknownNodeRows` are admin-only. + parameters: + - { in: path, name: siteId, required: true, schema: { type: integer } } + responses: + '200': + description: Per-owner usage rows with per-container detail + content: + application/json: + schema: + type: object + properties: + data: + type: object + properties: + generatedAt: { type: string, format: date-time } + owners: + type: array + items: + type: object + description: One row per owner (`owner` null = unattributed), with summed metrics and per-container `containers` detail + findings: + type: array + description: Admin-only attribution findings (kind drift|unattributed) + items: { type: object } + unknownNodeRows: { type: integer, description: 'Admin-only count of cluster members not registered in the manager DB' } + '404': { description: 'Site not found (code: site_not_found)', content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } /jobs/{id}: parameters: [{ in: path, name: id, required: true, schema: { type: integer } }] diff --git a/create-a-container/routers/api/v1/sites.js b/create-a-container/routers/api/v1/sites.js index 2ee1b59e..b21d2a93 100644 --- a/create-a-container/routers/api/v1/sites.js +++ b/create-a-container/routers/api/v1/sites.js @@ -15,6 +15,7 @@ router.use(apiAuth); // Nested mounts router.use('/:siteId/containers', require('./containers')); router.use('/:siteId/nodes', require('./nodes')); +router.use('/:siteId/usage', require('./usage')); function serialize(site) { return { diff --git a/create-a-container/routers/api/v1/usage.js b/create-a-container/routers/api/v1/usage.js new file mode 100644 index 00000000..e6cf98be --- /dev/null +++ b/create-a-container/routers/api/v1/usage.js @@ -0,0 +1,55 @@ +/** + * /api/v1/sites/:siteId/usage — live per-owner resource usage report + * (issue #440). Computed on demand from one Proxmox `/cluster/resources` + * call per cluster (utils/usage-collection.js — the same cycle the OTLP + * usage collector exports). + * + * Visibility mirrors the containers list: admins see every owner on the + * site; other users see their own containers plus containers shared with + * them. Attribution findings (tag/DB drift, unattributed containers) are + * admin-only. + */ + +const express = require('express'); +const { Site, ContainerCollaborator } = require('../../../models'); +const { apiAuth, asyncHandler, ok, ApiError } = require('../../../middlewares/api'); +const { collectUsage } = require('../../../utils/usage-collection'); +const { aggregateByOwner } = require('../../../utils/usage-report'); + +const router = express.Router({ mergeParams: true }); + +router.use(apiAuth); + +router.get( + '/', + asyncHandler(async (req, res) => { + const site = await Site.findByPk(parseInt(req.params.siteId, 10)); + if (!site) throw new ApiError(404, 'site_not_found', 'Site not found'); + + const { samples, findings, unknownNodeRows } = await collectUsage({ siteId: site.id }); + + let visible = samples; + if (!req.session.isAdmin) { + const shared = await ContainerCollaborator.findAll({ + where: { username: req.session.user }, + attributes: ['containerId'], + }); + const sharedIds = new Set(shared.map((row) => row.containerId)); + visible = samples.filter( + (s) => s.owner === req.session.user || (s.containerDbId && sharedIds.has(s.containerDbId)), + ); + } + + const payload = { + generatedAt: new Date().toISOString(), + owners: aggregateByOwner(visible), + }; + if (req.session.isAdmin) { + payload.findings = findings; + payload.unknownNodeRows = unknownNodeRows; + } + return ok(res, payload); + }), +); + +module.exports = router; diff --git a/create-a-container/usage-collector.js b/create-a-container/usage-collector.js index df53bc46..d3273dbf 100644 --- a/create-a-container/usage-collector.js +++ b/create-a-container/usage-collector.js @@ -27,87 +27,21 @@ * (OTEL_EXPORTER_OTLP_ENDPOINT et al., http/protobuf). Without an endpoint * configured the collector logs that it is disabled and exits — the service * is a no-op until an OTel collector exists to receive the data. + * + * The same sampling cycle backs the live report endpoint + * (/api/v1/sites/:siteId/usage) via the shared utils/usage-collection.js. */ -const db = require('./models'); -const { buildUsageSample } = require('./utils/usage-sample'); +const { collectUsage } = require('./utils/usage-collection'); const EXPORT_INTERVAL_MS = parseInt(process.env.USAGE_COLLECTOR_INTERVAL_MS || '300000', 10); /** - * Load every DB container that exists in Proxmox, keyed by `${nodeId}:${vmid}` - * for O(1) attribution lookups against cluster-resources entries. - * @returns {Promise>} - */ -async function loadContainerIndex() { - const containers = await db.Container.findAll({ - where: { containerId: { [db.Sequelize.Op.ne]: null } }, - attributes: ['id', 'containerId', 'nodeId', 'username'], - }); - const index = new Map(); - for (const c of containers) { - index.set(`${c.nodeId}:${c.containerId}`, c); - } - return index; -} - -/** - * Gather one cycle of normalized usage samples. One API call per cluster: - * after each successful `clusterResources('lxc')` call, every node name - * appearing in the response is marked covered (within the same site), so - * cluster peers are not re-polled. Per-node failures are logged and skipped; - * the cycle always completes. + * Run one collection cycle, logging attribution findings and the summary. * @returns {Promise>} Normalized samples (see utils/usage-sample.js) */ async function collectSamples() { - const nodes = await db.Node.findAll({ where: db.Node.provisionableWhere() }); - if (nodes.length === 0) return []; - - const containerIndex = await loadContainerIndex(); - - // Node lookup by `${siteId}:${name}` — cluster-resources rows carry the - // Proxmox node name, and node names are only meaningful within a site. - const nodesByName = new Map(nodes.map((n) => [`${n.siteId}:${n.name}`, n])); - const covered = new Set(); - - const samples = []; - const findings = []; - let unknownNodeRows = 0; - - for (const node of nodes) { - const nodeKey = `${node.siteId}:${node.name}`; - if (covered.has(nodeKey)) continue; - covered.add(nodeKey); - - let resources; - try { - const api = await node.api(); - resources = await api.clusterResources('lxc'); - } catch (err) { - console.error(`UsageCollector: node ${node.name} (site ${node.siteId}) unreachable: ${err.message}`); - continue; - } - if (!Array.isArray(resources)) continue; - - for (const resource of resources) { - if (resource.vmid == null) continue; - const resourceNodeKey = `${node.siteId}:${resource.node}`; - covered.add(resourceNodeKey); - - const dbNode = nodesByName.get(resourceNodeKey); - if (!dbNode) { - // Cluster member not registered in the DB — no site/node to attribute - // the sample to; count it so the drift is visible in the summary. - unknownNodeRows++; - continue; - } - - const container = containerIndex.get(`${dbNode.id}:${resource.vmid}`) || null; - const { sample, finding } = buildUsageSample({ resource, node: dbNode, container }); - samples.push(sample); - if (finding) findings.push(finding); - } - } + const { samples, findings, unknownNodeRows } = await collectUsage(); for (const finding of findings) { if (finding.kind === 'drift') { diff --git a/create-a-container/utils/__tests__/usage-report.test.js b/create-a-container/utils/__tests__/usage-report.test.js new file mode 100644 index 00000000..7a4de098 --- /dev/null +++ b/create-a-container/utils/__tests__/usage-report.test.js @@ -0,0 +1,76 @@ +'use strict'; + +const { aggregateByOwner } = require('../usage-report'); + +function sample(overrides = {}) { + return { + vmid: '100', + name: 'ct-100', + owner: 'cmyers', + containerDbId: 1, + node: 'pve1', + siteId: 1, + status: 'running', + cpuUsed: 0.5, + cpuAlloc: 4, + memUsed: 1024, + memAlloc: 4096, + diskUsed: 10, + diskAlloc: 50, + diskReadBytes: 100, + diskWriteBytes: 200, + netInBytes: 300, + netOutBytes: 400, + uptime: 3600, + ...overrides, + }; +} + +describe('aggregateByOwner', () => { + test('groups samples by owner and sums metrics', () => { + const rows = aggregateByOwner([ + sample({ vmid: '100', cpuUsed: 0.5, cpuAlloc: 4, memUsed: 1000 }), + sample({ vmid: '101', cpuUsed: 1.5, cpuAlloc: 2, memUsed: 2000, status: 'stopped' }), + sample({ vmid: '102', owner: 'horner', cpuUsed: 3, cpuAlloc: 8 }), + ]); + + expect(rows.map((r) => r.owner)).toEqual(['cmyers', 'horner']); + + const cmyers = rows[0]; + expect(cmyers.containerCount).toBe(2); + expect(cmyers.runningCount).toBe(1); + expect(cmyers.cpuUsed).toBe(2); + expect(cmyers.cpuAlloc).toBe(6); + expect(cmyers.memUsed).toBe(3000); + expect(cmyers.containers.map((c) => c.vmid)).toEqual(['100', '101']); + + expect(rows[1].containerCount).toBe(1); + expect(rows[1].cpuAlloc).toBe(8); + }); + + test('ignores null metrics instead of treating them as zero', () => { + const rows = aggregateByOwner([ + sample({ cpuUsed: null, memUsed: null }), + sample({ vmid: '101', cpuUsed: 1, memUsed: 500 }), + ]); + + expect(rows[0].cpuUsed).toBe(1); + expect(rows[0].memUsed).toBe(500); + expect(rows[0].containerCount).toBe(2); + }); + + test('collapses unattributed samples into one null-owner row sorted last', () => { + const rows = aggregateByOwner([ + sample({ owner: null, vmid: '200' }), + sample({ owner: 'zbarrell', vmid: '201' }), + sample({ owner: null, vmid: '202' }), + ]); + + expect(rows.map((r) => r.owner)).toEqual(['zbarrell', null]); + expect(rows[1].containerCount).toBe(2); + }); + + test('returns an empty array for no samples', () => { + expect(aggregateByOwner([])).toEqual([]); + }); +}); diff --git a/create-a-container/utils/__tests__/usage-sample.test.js b/create-a-container/utils/__tests__/usage-sample.test.js index 7818fc66..1a964f94 100644 --- a/create-a-container/utils/__tests__/usage-sample.test.js +++ b/create-a-container/utils/__tests__/usage-sample.test.js @@ -66,6 +66,7 @@ describe('buildUsageSample', () => { vmid: '284', name: 'ozwell-studio-92e3d441', owner: 'zbarrell', + containerDbId: 11, node: 'pve2', siteId: 3, status: 'running', @@ -110,6 +111,7 @@ describe('buildUsageSample', () => { const { sample, finding } = buildUsageSample({ resource: untagged, node, container: null }); expect(sample.owner).toBeNull(); + expect(sample.containerDbId).toBeNull(); expect(finding).toEqual({ kind: 'unattributed', vmid: '284', diff --git a/create-a-container/utils/usage-collection.js b/create-a-container/utils/usage-collection.js new file mode 100644 index 00000000..bb202beb --- /dev/null +++ b/create-a-container/utils/usage-collection.js @@ -0,0 +1,104 @@ +'use strict'; + +/** + * usage-collection.js — one polling cycle of per-container usage samples + * (issue #440). Shared by usage-collector.js (OTLP export) and the + * /sites/:siteId/usage report endpoint, so both see identical data. + * + * One Proxmox `/cluster/resources` call per cluster: after each successful + * call, every node name appearing in the response is marked covered (within + * the same site), so cluster peers are not re-polled. Per-node failures are + * logged and skipped; a cycle always completes. + * + * Pure per-entry mapping/attribution lives in utils/usage-sample.js. + */ + +const db = require('../models'); +const { buildUsageSample } = require('./usage-sample'); + +/** + * Load every DB container that exists in Proxmox, keyed by `${nodeId}:${vmid}` + * for O(1) attribution lookups against cluster-resources entries. + * @param {number|null} siteId - Restrict to one site, or null for all + * @returns {Promise>} + */ +async function loadContainerIndex(siteId) { + const where = { containerId: { [db.Sequelize.Op.ne]: null } }; + if (siteId != null) where.siteId = siteId; + const containers = await db.Container.findAll({ + where, + attributes: ['id', 'containerId', 'nodeId', 'username'], + }); + const index = new Map(); + for (const c of containers) { + index.set(`${c.nodeId}:${c.containerId}`, c); + } + return index; +} + +/** + * Gather one cycle of normalized usage samples plus attribution findings. + * @param {object} [options] + * @param {number|null} [options.siteId] - Restrict to one site, or null for all + * @returns {Promise<{ + * samples: Array, + * findings: Array<{ kind: 'drift'|'unattributed', vmid: string, tagOwner: string|null, dbOwner: string|null }>, + * unknownNodeRows: number, + * }>} + */ +async function collectUsage({ siteId = null } = {}) { + const nodeWhere = db.Node.provisionableWhere(); + if (siteId != null) nodeWhere.siteId = siteId; + const nodes = await db.Node.findAll({ where: nodeWhere }); + if (nodes.length === 0) return { samples: [], findings: [], unknownNodeRows: 0 }; + + const containerIndex = await loadContainerIndex(siteId); + + // Node lookup by `${siteId}:${name}` — cluster-resources rows carry the + // Proxmox node name, and node names are only meaningful within a site. + const nodesByName = new Map(nodes.map((n) => [`${n.siteId}:${n.name}`, n])); + const covered = new Set(); + + const samples = []; + const findings = []; + let unknownNodeRows = 0; + + for (const node of nodes) { + const nodeKey = `${node.siteId}:${node.name}`; + if (covered.has(nodeKey)) continue; + covered.add(nodeKey); + + let resources; + try { + const api = await node.api(); + resources = await api.clusterResources('lxc'); + } catch (err) { + console.error(`UsageCollection: node ${node.name} (site ${node.siteId}) unreachable: ${err.message}`); + continue; + } + if (!Array.isArray(resources)) continue; + + for (const resource of resources) { + if (resource.vmid == null) continue; + const resourceNodeKey = `${node.siteId}:${resource.node}`; + covered.add(resourceNodeKey); + + const dbNode = nodesByName.get(resourceNodeKey); + if (!dbNode) { + // Cluster member not registered in the DB — no site/node to attribute + // the sample to; count it so the drift is visible to callers. + unknownNodeRows++; + continue; + } + + const container = containerIndex.get(`${dbNode.id}:${resource.vmid}`) || null; + const { sample, finding } = buildUsageSample({ resource, node: dbNode, container }); + samples.push(sample); + if (finding) findings.push(finding); + } + } + + return { samples, findings, unknownNodeRows }; +} + +module.exports = { collectUsage }; diff --git a/create-a-container/utils/usage-report.js b/create-a-container/utils/usage-report.js new file mode 100644 index 00000000..462cf2a3 --- /dev/null +++ b/create-a-container/utils/usage-report.js @@ -0,0 +1,52 @@ +'use strict'; + +/** + * usage-report.js — pure aggregation for the live per-owner usage report + * (issue #440). Groups normalized samples (utils/usage-sample.js) by owner + * with allocated-vs-used totals; the /sites/:siteId/usage endpoint owns + * visibility filtering and serialization. + */ + +const SUM_FIELDS = [ + 'cpuUsed', 'cpuAlloc', + 'memUsed', 'memAlloc', + 'diskUsed', 'diskAlloc', + 'diskReadBytes', 'diskWriteBytes', + 'netInBytes', 'netOutBytes', +]; + +/** + * Group usage samples by owner. Null owners collapse into a single + * `owner: null` row (rendered as "unattributed" by callers). Absent metrics + * (null) are excluded from sums rather than treated as zero. + * @param {Array} samples - Normalized samples from utils/usage-sample.js + * @returns {Array} One row per owner, sorted by owner name (null last): + * { owner, containerCount, runningCount, , containers } + */ +function aggregateByOwner(samples) { + const byOwner = new Map(); + + for (const sample of samples) { + const key = sample.owner ?? null; + let row = byOwner.get(key); + if (!row) { + row = { owner: key, containerCount: 0, runningCount: 0, containers: [] }; + for (const field of SUM_FIELDS) row[field] = 0; + byOwner.set(key, row); + } + row.containerCount++; + if (sample.status === 'running') row.runningCount++; + for (const field of SUM_FIELDS) { + if (sample[field] != null) row[field] += sample[field]; + } + row.containers.push(sample); + } + + return [...byOwner.values()].sort((a, b) => { + if (a.owner === null) return 1; + if (b.owner === null) return -1; + return a.owner.localeCompare(b.owner); + }); +} + +module.exports = { aggregateByOwner }; diff --git a/create-a-container/utils/usage-sample.js b/create-a-container/utils/usage-sample.js index 2bc9c67f..5a58be69 100644 --- a/create-a-container/utils/usage-sample.js +++ b/create-a-container/utils/usage-sample.js @@ -84,6 +84,9 @@ function buildUsageSample({ resource, node, container }) { vmid: String(resource.vmid), name: resource.name || null, owner, + // DB primary key when the container is registered in the manager — used + // by the report endpoint to honour per-container sharing visibility. + containerDbId: container ? container.id : null, node: node.name, siteId: node.siteId, status: resource.status || null, From 1b530d377e225150b3881c6fdb2e1eb6f8b48b9a Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Wed, 12 Aug 2026 09:38:36 -0700 Subject: [PATCH 3/3] Add owner-stacked capacity bars and PSI pressure tracking (issue #440) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Usage page now leads with cluster-wide stacked bars — memory and CPU used vs physical capacity, one colored segment per owner (stable colors across bars, top consumers named, remainder bucketed as 'others', red tick marks capacity on over-commit). Capacity comes from the node rows of the same /cluster/resources call family (2 bulk calls per cluster). PSI (pressure stall information) is now collected: each cycle probes rrddata for the highest-utilization running containers, capped by USAGE_PSI_PROBE_LIMIT (default 16) so the fleet is never swept — raw stats miss exactly what PSI catches (a container at 99% memory can be healthy; one thrashing shows psiMemFull > 40 while raw memory looks identical). Candidate selection and RRD parsing are pure and unit-tested (utils/usage-psi.js). Surfaced as: - container.cpu/memory/io.pressure OTLP gauges (attr pressure.kind: some|full) on the collector - a Pressure column on the owner grid (worst PSI across their containers) and per-container badges with full readings in the tooltip; green/amber/red at 10/40 per the #440 incident evidence DummyApi simulates node capacity rows and PSI series for dev parity; ProxmoxApi gains rrdData(). --- .../components/usage/OwnerContainersTable.tsx | 20 ++++ .../src/components/usage/PressureBadge.tsx | 25 ++++ .../src/components/usage/StackedBar.tsx | 60 ++++++++++ .../src/components/usage/UsageDataGrid.tsx | 5 + .../src/components/usage/UsageStackedBars.tsx | 113 ++++++++++++++++++ create-a-container/client/src/lib/types.ts | 14 +++ .../client/src/pages/usage/UsagePage.tsx | 8 +- create-a-container/example.env | 7 +- create-a-container/openapi.v1.yaml | 9 +- create-a-container/routers/api/v1/usage.js | 4 +- create-a-container/usage-collector.js | 20 +++- .../utils/__tests__/usage-psi.test.js | 58 +++++++++ .../utils/__tests__/usage-report.test.js | 11 ++ .../utils/__tests__/usage-sample.test.js | 6 + create-a-container/utils/dummy-api.js | 33 ++++- create-a-container/utils/proxmox-api.js | 17 +++ create-a-container/utils/usage-collection.js | 69 +++++++++-- create-a-container/utils/usage-psi.js | 84 +++++++++++++ create-a-container/utils/usage-report.js | 19 ++- create-a-container/utils/usage-sample.js | 9 ++ 20 files changed, 573 insertions(+), 18 deletions(-) create mode 100644 create-a-container/client/src/components/usage/PressureBadge.tsx create mode 100644 create-a-container/client/src/components/usage/StackedBar.tsx create mode 100644 create-a-container/client/src/components/usage/UsageStackedBars.tsx create mode 100644 create-a-container/utils/__tests__/usage-psi.test.js create mode 100644 create-a-container/utils/usage-psi.js diff --git a/create-a-container/client/src/components/usage/OwnerContainersTable.tsx b/create-a-container/client/src/components/usage/OwnerContainersTable.tsx index b5a7cce3..2d0975e0 100644 --- a/create-a-container/client/src/components/usage/OwnerContainersTable.tsx +++ b/create-a-container/client/src/components/usage/OwnerContainersTable.tsx @@ -8,6 +8,7 @@ import { } from '@mieweb/ui'; import type { UsageContainer } from '@/lib/types'; import { formatBytes } from '@/lib/format'; +import { PressureBadge } from './PressureBadge'; /** "left / right" pair where either side may be missing, e.g. used/alloc or read/write. */ function pair( @@ -22,6 +23,13 @@ function pair( const cores = (v: number) => v.toFixed(2); +/** Worst of the six PSI readings for a container, or null when unprobed. */ +function worstPsi(c: UsageContainer): number | null { + const values = [c.psiCpuSome, c.psiCpuFull, c.psiMemSome, c.psiMemFull, c.psiIoSome, c.psiIoFull] + .filter((v): v is number => v != null); + return values.length > 0 ? Math.max(...values) : null; +} + export interface OwnerContainersTableProps { containers: UsageContainer[]; } @@ -45,6 +53,7 @@ export function OwnerContainersTable({ containers }: OwnerContainersTableProps) Disk Disk I/O (r / w) Network (in / out) + Pressure @@ -59,6 +68,17 @@ export function OwnerContainersTable({ containers }: OwnerContainersTableProps) {pair(c.diskUsed, c.diskAlloc, formatBytes)} {pair(c.diskReadBytes, c.diskWriteBytes, formatBytes)} {pair(c.netInBytes, c.netOutBytes, formatBytes)} + + v.toFixed(1))} · Mem ${pair(c.psiMemSome, c.psiMemFull, (v) => v.toFixed(1))} · I/O ${pair(c.psiIoSome, c.psiIoFull, (v) => v.toFixed(1))} (some / full, avg10 %)` + } + > + + + ))} diff --git a/create-a-container/client/src/components/usage/PressureBadge.tsx b/create-a-container/client/src/components/usage/PressureBadge.tsx new file mode 100644 index 00000000..2748a644 --- /dev/null +++ b/create-a-container/client/src/components/usage/PressureBadge.tsx @@ -0,0 +1,25 @@ +export interface PressureBadgeProps { + /** Worst PSI stall percentage (avg10), or null when not probed. */ + value: number | null; +} + +/** + * Colored PSI readout: the issue #440 evidence puts sustained full-stall + * above 40 firmly in "thrashing" territory; 10+ is worth watching. Null + * means the container was not probed this cycle (PSI probes are budget-capped), + * not that it is healthy. + */ +export function PressureBadge({ value }: PressureBadgeProps) { + if (value == null) return —; + const cls = + value >= 40 + ? 'bg-red-100 text-red-800 dark:bg-red-900/40 dark:text-red-300' + : value >= 10 + ? 'bg-amber-100 text-amber-800 dark:bg-amber-900/40 dark:text-amber-300' + : 'bg-green-100 text-green-800 dark:bg-green-900/40 dark:text-green-300'; + return ( + + {value.toFixed(1)}% + + ); +} diff --git a/create-a-container/client/src/components/usage/StackedBar.tsx b/create-a-container/client/src/components/usage/StackedBar.tsx new file mode 100644 index 00000000..e41f5ed2 --- /dev/null +++ b/create-a-container/client/src/components/usage/StackedBar.tsx @@ -0,0 +1,60 @@ +export interface StackedBarSegment { + label: string; + value: number; + color: string; +} + +export interface StackedBarProps { + /** e.g. "Memory" */ + title: string; + segments: StackedBarSegment[]; + /** Physical capacity the bar is drawn against. */ + capacity: number; + format: (value: number) => string; +} + +/** + * Horizontal stacked bar: one colored segment per owner, drawn against + * cluster capacity so the empty remainder is visible headroom. When the + * segments exceed capacity (over-commit) the scale grows to fit and a + * capacity tick marks 100%. + */ +export function StackedBar({ title, segments, capacity, format }: StackedBarProps) { + const total = segments.reduce((sum, s) => sum + s.value, 0); + const scale = Math.max(capacity, total); + if (scale <= 0) return null; + const capacityPct = (capacity / scale) * 100; + + return ( +
+
+ {title} + + {format(total)} / {format(capacity)} + {capacity > 0 && ` (${Math.round((total / capacity) * 100)}%)`} + +
+
+ {segments.map((s) => ( +
+ ))} + {total > capacity && capacity > 0 && ( +
+ )} +
+
+ ); +} diff --git a/create-a-container/client/src/components/usage/UsageDataGrid.tsx b/create-a-container/client/src/components/usage/UsageDataGrid.tsx index 92028c99..3f27698e 100644 --- a/create-a-container/client/src/components/usage/UsageDataGrid.tsx +++ b/create-a-container/client/src/components/usage/UsageDataGrid.tsx @@ -4,6 +4,7 @@ import type { TableColumn, TableRendererProps } from '@mieweb/datavis'; import type { UsageOwner } from '@/lib/types'; import { formatBytes } from '@/lib/format'; import { OwnerContainersTable } from './OwnerContainersTable'; +import { PressureBadge } from './PressureBadge'; // Numeric `*Used` fields drive sorting; formatCell renders "used / allocated". const TYPE_INFO: { field: string; type: string }[] = [ @@ -14,6 +15,7 @@ const TYPE_INFO: { field: string; type: string }[] = [ { field: 'diskUsed', type: 'number' }, { field: 'diskReadBytes', type: 'number' }, { field: 'netInBytes', type: 'number' }, + { field: 'pressureMax', type: 'number' }, ]; const asOwner = (row: Record) => row as unknown as UsageOwner; @@ -58,6 +60,7 @@ export function UsageDataGrid({ owners }: UsageDataGridProps) { { field: 'diskUsed', header: 'Disk (used / alloc)', sortable: true, filterable: false, getSearchText: () => '' }, { field: 'diskReadBytes', header: 'Disk I/O (r / w)', sortable: true, filterable: false, getSearchText: () => '' }, { field: 'netInBytes', header: 'Network (in / out)', sortable: true, filterable: false, getSearchText: () => '' }, + { field: 'pressureMax', header: 'Pressure', sortable: true, filterable: false, getSearchText: () => '' }, ], [], ); @@ -83,6 +86,8 @@ export function UsageDataGrid({ owners }: UsageDataGridProps) { return `${formatBytes(o.diskReadBytes)} / ${formatBytes(o.diskWriteBytes)}`; case 'netInBytes': return `${formatBytes(o.netInBytes)} / ${formatBytes(o.netOutBytes)}`; + case 'pressureMax': + return ; default: return value as React.ReactNode; } diff --git a/create-a-container/client/src/components/usage/UsageStackedBars.tsx b/create-a-container/client/src/components/usage/UsageStackedBars.tsx new file mode 100644 index 00000000..a8cf5dc9 --- /dev/null +++ b/create-a-container/client/src/components/usage/UsageStackedBars.tsx @@ -0,0 +1,113 @@ +import { useMemo } from 'react'; +import type { UsageOwner, UsageReport } from '@/lib/types'; +import { formatBytes } from '@/lib/format'; +import { StackedBar, type StackedBarSegment } from './StackedBar'; + +/** + * Deterministic owner colors: distinct hues for the top consumers, gray for + * the aggregated remainder. + */ +const PALETTE = [ + '#2563eb', '#dc2626', '#16a34a', '#d97706', '#9333ea', + '#0891b2', '#db2777', '#65a30d', '#7c3aed', '#ea580c', +]; +const OTHERS_COLOR = '#9ca3af'; +const OTHERS_LABEL = 'others'; +const TOP_OWNERS = PALETTE.length; + +/** + * One color per owner across every bar (ranked by memory use — the metric + * people get yelled at for); owners beyond the palette share a gray + * "others" bucket. + */ +function buildColorMap(owners: UsageOwner[]): Map { + const ranked = [...owners].sort((a, b) => b.memUsed - a.memUsed); + const map = new Map(); + ranked.forEach((o, i) => { + if (i < TOP_OWNERS) map.set(o.owner ?? 'unattributed', PALETTE[i]); + }); + return map; +} + +function segmentsFor( + owners: UsageOwner[], + metric: (o: UsageOwner) => number, + colorMap: Map, +): StackedBarSegment[] { + const named: StackedBarSegment[] = []; + let othersValue = 0; + let othersCount = 0; + + for (const o of owners) { + const value = metric(o); + if (value <= 0) continue; + const label = o.owner ?? 'unattributed'; + const color = colorMap.get(label); + if (color) { + named.push({ label, value, color }); + } else { + othersValue += value; + othersCount++; + } + } + + named.sort((a, b) => b.value - a.value); + if (othersValue > 0) { + named.push({ label: `${othersCount} ${OTHERS_LABEL}`, value: othersValue, color: OTHERS_COLOR }); + } + return named; +} + +const formatCores = (v: number) => `${v.toFixed(1)} cores`; + +export interface UsageStackedBarsProps { + report: UsageReport; +} + +/** + * Cluster-wide used-vs-capacity stacked bars for CPU and memory, one colored + * segment per owner — the "who is using the cluster" view. A shared legend + * covers both bars. + */ +export function UsageStackedBars({ report }: UsageStackedBarsProps) { + const { owners, capacity } = report; + + const colorMap = useMemo(() => buildColorMap(owners), [owners]); + const cpuSegments = useMemo(() => segmentsFor(owners, (o) => o.cpuUsed, colorMap), [owners, colorMap]); + const memSegments = useMemo(() => segmentsFor(owners, (o) => o.memUsed, colorMap), [owners, colorMap]); + + const legend = useMemo(() => { + const seen = new Map(); + for (const s of [...memSegments, ...cpuSegments]) { + if (!seen.has(s.label)) seen.set(s.label, s.color); + } + return [...seen.entries()]; + }, [cpuSegments, memSegments]); + + if (capacity.cpuCores <= 0 && capacity.memBytes <= 0) return null; + + return ( +
+ + +
    + {legend.map(([label, color]) => ( +
  • + + {label} +
    • + ))} +
+
+ ); +} diff --git a/create-a-container/client/src/lib/types.ts b/create-a-container/client/src/lib/types.ts index 0c67c6bc..e6e98552 100644 --- a/create-a-container/client/src/lib/types.ts +++ b/create-a-container/client/src/lib/types.ts @@ -94,6 +94,16 @@ export interface UsageContainer { netOutBytes: number | null; /** Seconds since container boot. */ uptime: number | null; + /** + * PSI pressure stall percentages (avg10), probed for the highest-utilization + * containers only; null means "not probed this cycle", not "no pressure". + */ + psiCpuSome: number | null; + psiCpuFull: number | null; + psiMemSome: number | null; + psiMemFull: number | null; + psiIoSome: number | null; + psiIoFull: number | null; } /** Per-owner aggregate row (owner null = unattributed, admin-visible only). */ @@ -111,6 +121,8 @@ export interface UsageOwner { diskWriteBytes: number; netInBytes: number; netOutBytes: number; + /** Worst PSI reading across this owner's probed containers (null = unprobed). */ + pressureMax: number | null; containers: UsageContainer[]; } @@ -125,6 +137,8 @@ export interface UsageFinding { export interface UsageReport { generatedAt: string; owners: UsageOwner[]; + /** Physical cluster capacity summed from the hypervisor node rows. */ + capacity: { cpuCores: number; memBytes: number; diskBytes: number }; /** Admin-only. */ findings?: UsageFinding[]; /** Admin-only: cluster members not registered in the manager DB. */ diff --git a/create-a-container/client/src/pages/usage/UsagePage.tsx b/create-a-container/client/src/pages/usage/UsagePage.tsx index bb549172..39ddef9b 100644 --- a/create-a-container/client/src/pages/usage/UsagePage.tsx +++ b/create-a-container/client/src/pages/usage/UsagePage.tsx @@ -4,6 +4,7 @@ import { Alert, AlertDescription, PageHeader, Spinner } from '@mieweb/ui'; import { Activity } from 'lucide-react'; import { AttributionWarnings } from '@/components/usage/AttributionWarnings'; import { UsageDataGrid } from '@/components/usage/UsageDataGrid'; +import { UsageStackedBars } from '@/components/usage/UsageStackedBars'; import type { ApiError } from '@/lib/api'; import { keys, queries } from '@/lib/queries'; @@ -56,7 +57,12 @@ export function UsagePage() { No running containers to report on. )} - {data && data.owners.length > 0 && } + {data && data.owners.length > 0 && ( + <> + + + + )}
); } diff --git a/create-a-container/example.env b/create-a-container/example.env index efa05e50..2d1a482e 100644 --- a/create-a-container/example.env +++ b/create-a-container/example.env @@ -43,8 +43,13 @@ MCP_SERVER_URL= # Example: http://otel-collector:4318 OTEL_EXPORTER_OTLP_ENDPOINT= # Export/poll interval in milliseconds (default 300000 = 5 minutes). Each -# cycle issues one Proxmox /cluster/resources call per cluster. +# cycle issues one Proxmox /cluster/resources call per cluster (plus one for +# node capacity). USAGE_COLLECTOR_INTERVAL_MS= +# PSI (pressure stall) probes are one rrddata call per container, so each +# cycle only probes the highest-utilization running containers up to this +# budget (default 16; 0 disables PSI collection). +USAGE_PSI_PROBE_LIMIT= # --- OIDC / single sign-on (optional) --- # SSO is enabled only when OIDC_ISSUER_URL, OIDC_CLIENT_ID, and diff --git a/create-a-container/openapi.v1.yaml b/create-a-container/openapi.v1.yaml index aad0ca54..44e3935c 100644 --- a/create-a-container/openapi.v1.yaml +++ b/create-a-container/openapi.v1.yaml @@ -1085,7 +1085,14 @@ paths: type: array items: type: object - description: One row per owner (`owner` null = unattributed), with summed metrics and per-container `containers` detail + description: One row per owner (`owner` null = unattributed), with summed metrics, worst PSI pressure, and per-container `containers` detail + capacity: + type: object + description: Physical cluster capacity summed from the hypervisor node rows + properties: + cpuCores: { type: number } + memBytes: { type: number } + diskBytes: { type: number } findings: type: array description: Admin-only attribution findings (kind drift|unattributed) diff --git a/create-a-container/routers/api/v1/usage.js b/create-a-container/routers/api/v1/usage.js index e6cf98be..d6d2bb48 100644 --- a/create-a-container/routers/api/v1/usage.js +++ b/create-a-container/routers/api/v1/usage.js @@ -26,7 +26,7 @@ router.get( const site = await Site.findByPk(parseInt(req.params.siteId, 10)); if (!site) throw new ApiError(404, 'site_not_found', 'Site not found'); - const { samples, findings, unknownNodeRows } = await collectUsage({ siteId: site.id }); + const { samples, findings, unknownNodeRows, capacity } = await collectUsage({ siteId: site.id }); let visible = samples; if (!req.session.isAdmin) { @@ -43,6 +43,8 @@ router.get( const payload = { generatedAt: new Date().toISOString(), owners: aggregateByOwner(visible), + // Physical cluster capacity (node rows), for used-vs-capacity charts. + capacity, }; if (req.session.isAdmin) { payload.findings = findings; diff --git a/create-a-container/usage-collector.js b/create-a-container/usage-collector.js index d3273dbf..8e8592b1 100644 --- a/create-a-container/usage-collector.js +++ b/create-a-container/usage-collector.js @@ -19,6 +19,10 @@ * container.disk.io counter By (attr disk.io.direction: read|write) * container.network.io counter By (attr network.io.direction: receive|transmit) * container.uptime gauge s + * container.cpu.pressure / container.memory.pressure / container.io.pressure + * gauge % (attr pressure.kind: some|full; only for + * the highest-utilization containers each + * cycle — see utils/usage-psi.js) * * The counters are cumulative since container boot as reported by Proxmox; a * decrease (container reboot) is a normal counter reset for the backend. @@ -122,8 +126,11 @@ function setupMetrics() { const diskIo = meter.createObservableCounter('container.disk.io', { unit: 'By', description: 'Disk bytes transferred since container boot' }); const networkIo = meter.createObservableCounter('container.network.io', { unit: 'By', description: 'Network bytes transferred since container boot' }); const uptime = meter.createObservableGauge('container.uptime', { unit: 's', description: 'Seconds since container boot' }); + const cpuPressure = meter.createObservableGauge('container.cpu.pressure', { unit: '%', description: 'CPU PSI stall percentage (avg10)' }); + const memPressure = meter.createObservableGauge('container.memory.pressure', { unit: '%', description: 'Memory PSI stall percentage (avg10)' }); + const ioPressure = meter.createObservableGauge('container.io.pressure', { unit: '%', description: 'I/O PSI stall percentage (avg10)' }); - const instruments = [cpuUsage, cpuLimit, memUsage, memLimit, diskUsage, diskLimit, diskIo, networkIo, uptime]; + const instruments = [cpuUsage, cpuLimit, memUsage, memLimit, diskUsage, diskLimit, diskIo, networkIo, uptime, cpuPressure, memPressure, ioPressure]; meter.addBatchObservableCallback(async (result) => { let samples; @@ -160,6 +167,17 @@ function setupMetrics() { if (sample.netOutBytes != null) { result.observe(networkIo, sample.netOutBytes, { ...attributes, 'network.io.direction': 'transmit' }); } + const pressures = [ + [cpuPressure, sample.psiCpuSome, 'some'], + [cpuPressure, sample.psiCpuFull, 'full'], + [memPressure, sample.psiMemSome, 'some'], + [memPressure, sample.psiMemFull, 'full'], + [ioPressure, sample.psiIoSome, 'some'], + [ioPressure, sample.psiIoFull, 'full'], + ]; + for (const [instrument, value, kind] of pressures) { + if (value != null) result.observe(instrument, value, { ...attributes, 'pressure.kind': kind }); + } } }, instruments); diff --git a/create-a-container/utils/__tests__/usage-psi.test.js b/create-a-container/utils/__tests__/usage-psi.test.js new file mode 100644 index 00000000..19f1ff43 --- /dev/null +++ b/create-a-container/utils/__tests__/usage-psi.test.js @@ -0,0 +1,58 @@ +'use strict'; + +const { selectPsiCandidates, latestPsi } = require('../usage-psi'); + +function sample(overrides = {}) { + return { + vmid: '100', + status: 'running', + cpuUsed: 0.1, + cpuAlloc: 4, + memUsed: 100, + memAlloc: 1000, + ...overrides, + }; +} + +describe('selectPsiCandidates', () => { + test('orders by worst utilization (memory or CPU fraction) and caps at the limit', () => { + const low = sample({ vmid: 'low', memUsed: 100 }); + const hot = sample({ vmid: 'hot', memUsed: 950 }); + const cpuHot = sample({ vmid: 'cpuhot', cpuUsed: 3.6 }); + const mid = sample({ vmid: 'mid', memUsed: 500 }); + + const picked = selectPsiCandidates([low, hot, cpuHot, mid], 2); + expect(picked.map((s) => s.vmid)).toEqual(['hot', 'cpuhot']); + }); + + test('only probes running containers', () => { + const stopped = sample({ vmid: 'stopped', status: 'stopped', memUsed: 999 }); + const running = sample({ vmid: 'running' }); + expect(selectPsiCandidates([stopped, running], 5).map((s) => s.vmid)).toEqual(['running']); + }); + + test('handles null metrics and a zero limit', () => { + const sparse = sample({ vmid: 'sparse', cpuUsed: null, memUsed: null, memAlloc: null, cpuAlloc: null }); + expect(selectPsiCandidates([sparse], 5)).toEqual([sparse]); + expect(selectPsiCandidates([sparse], 0)).toEqual([]); + }); +}); + +describe('latestPsi', () => { + test('takes the newest non-null value per field', () => { + const rows = [ + { time: 1, pressurememoryfull: 5, pressurecpusome: 1 }, + { time: 2, pressurememoryfull: 42, pressurecpusome: null }, + ]; + const psi = latestPsi(rows); + expect(psi.psiMemFull).toBe(42); + expect(psi.psiCpuSome).toBe(1); // newest row was null; falls back one row + expect(psi.psiIoFull).toBeNull(); // never present + }); + + test('returns null when the series is empty or has no PSI fields', () => { + expect(latestPsi([])).toBeNull(); + expect(latestPsi(null)).toBeNull(); + expect(latestPsi([{ time: 1, cpu: 0.5 }])).toBeNull(); + }); +}); diff --git a/create-a-container/utils/__tests__/usage-report.test.js b/create-a-container/utils/__tests__/usage-report.test.js index 7a4de098..53acc8d4 100644 --- a/create-a-container/utils/__tests__/usage-report.test.js +++ b/create-a-container/utils/__tests__/usage-report.test.js @@ -70,6 +70,17 @@ describe('aggregateByOwner', () => { expect(rows[1].containerCount).toBe(2); }); + test('pressureMax is the worst PSI across the owner containers, null when unprobed', () => { + const rows = aggregateByOwner([ + sample({ vmid: '100', psiMemFull: 42, psiIoSome: 5 }), + sample({ vmid: '101', psiCpuSome: 7 }), + sample({ vmid: '102', owner: 'horner' }), + ]); + + expect(rows[0].pressureMax).toBe(42); + expect(rows[1].pressureMax).toBeNull(); + }); + test('returns an empty array for no samples', () => { expect(aggregateByOwner([])).toEqual([]); }); diff --git a/create-a-container/utils/__tests__/usage-sample.test.js b/create-a-container/utils/__tests__/usage-sample.test.js index 1a964f94..5c0c934b 100644 --- a/create-a-container/utils/__tests__/usage-sample.test.js +++ b/create-a-container/utils/__tests__/usage-sample.test.js @@ -81,6 +81,12 @@ describe('buildUsageSample', () => { netInBytes: 3000, netOutBytes: 4000, uptime: 86400, + psiCpuSome: null, + psiCpuFull: null, + psiMemSome: null, + psiMemFull: null, + psiIoSome: null, + psiIoFull: null, }); }); diff --git a/create-a-container/utils/dummy-api.js b/create-a-container/utils/dummy-api.js index f5a11745..a7d45270 100644 --- a/create-a-container/utils/dummy-api.js +++ b/create-a-container/utils/dummy-api.js @@ -309,8 +309,21 @@ class DummyApi { * @returns {Promise>} */ async clusterResources(type = null) { - if (type && type !== 'lxc') return []; + if (type && type !== 'lxc' && type !== 'node') return []; if (this.node.id == null) return []; + // A single node row supplies simulated cluster capacity (matches the + // nodeStatus() figures) for the usage report's stacked bars. + if (type === 'node') { + const GiB = 1024 * 1024 * 1024; + return [{ + type: 'node', + node: this.node.name || 'dummy', + status: 'online', + maxcpu: 8, + maxmem: 32 * GiB, + maxdisk: 500 * GiB, + }]; + } // Lazy require to avoid a load-time cycle (models/node.js -> dummy-api.js). const { Container } = require('../models'); const containers = await Container.findAll({ @@ -340,6 +353,24 @@ class DummyApi { uptime: 3600, })); } + + /** + * Simulated RRD series with PSI pressure fields so the pressure path of the + * usage report/collector renders in dev. One recent point is enough — the + * consumers only read the latest values. + * @returns {Promise>} + */ + async rrdData() { + return [{ + time: Math.floor(Date.now() / 1000) - 60, + pressurecpusome: Math.random() * 5, + pressurecpufull: Math.random() * 1, + pressurememorysome: Math.random() * 10, + pressurememoryfull: Math.random() * 2, + pressureiosome: Math.random() * 8, + pressureiofull: Math.random() * 2, + }]; + } } module.exports = DummyApi; diff --git a/create-a-container/utils/proxmox-api.js b/create-a-container/utils/proxmox-api.js index e24fd86f..3e0b5189 100644 --- a/create-a-container/utils/proxmox-api.js +++ b/create-a-container/utils/proxmox-api.js @@ -164,6 +164,23 @@ class ProxmoxApi { return response.data.data.filter(r => r.type === type); } + /** + * Get RRD time-series data for a container. At 60 s resolution this includes + * the PSI pressure fields (pressurecpusome/full, pressurememorysome/full, + * pressureiosome/full) used for health detection. + * @param {string} node - The node name + * @param {number|string} vmid + * @param {'hour'|'day'|'week'|'month'|'year'} [timeframe] + * @returns {Promise>} - The API response data + */ + async rrdData(node, vmid, timeframe = 'hour') { + const response = await axios.get( + `${this.baseUrl}/api2/json/nodes/${node}/lxc/${vmid}/rrddata?timeframe=${timeframe}`, + this.options, + ); + return response.data.data; + } + /** * Get container configuration * @param {string} node diff --git a/create-a-container/utils/usage-collection.js b/create-a-container/utils/usage-collection.js index bb202beb..3e408e77 100644 --- a/create-a-container/utils/usage-collection.js +++ b/create-a-container/utils/usage-collection.js @@ -5,16 +5,23 @@ * (issue #440). Shared by usage-collector.js (OTLP export) and the * /sites/:siteId/usage report endpoint, so both see identical data. * - * One Proxmox `/cluster/resources` call per cluster: after each successful - * call, every node name appearing in the response is marked covered (within - * the same site), so cluster peers are not re-polled. Per-node failures are - * logged and skipped; a cycle always completes. + * One Proxmox `/cluster/resources` call per cluster for LXCs plus one for + * node capacity: after each successful call, every node name appearing in the + * response is marked covered (within the same site), so cluster peers are not + * re-polled. Per-node failures are logged and skipped; a cycle always + * completes. A second, budget-capped pass probes `rrddata` PSI for the + * highest-utilization containers (utils/usage-psi.js) — never the whole + * fleet. * * Pure per-entry mapping/attribution lives in utils/usage-sample.js. */ const db = require('../models'); const { buildUsageSample } = require('./usage-sample'); +const { selectPsiCandidates, latestPsi } = require('./usage-psi'); + +// PSI probes are one API call per container; cap the per-cycle fan-out. +const PSI_PROBE_LIMIT = parseInt(process.env.USAGE_PSI_PROBE_LIMIT || '16', 10); /** * Load every DB container that exists in Proxmox, keyed by `${nodeId}:${vmid}` @@ -37,20 +44,24 @@ async function loadContainerIndex(siteId) { } /** - * Gather one cycle of normalized usage samples plus attribution findings. + * Gather one cycle of normalized usage samples, attribution findings, and + * cluster capacity, then enrich the highest-utilization containers with PSI. * @param {object} [options] * @param {number|null} [options.siteId] - Restrict to one site, or null for all + * @param {number} [options.psiProbeLimit] - Max rrddata calls this cycle (0 disables) * @returns {Promise<{ * samples: Array, * findings: Array<{ kind: 'drift'|'unattributed', vmid: string, tagOwner: string|null, dbOwner: string|null }>, * unknownNodeRows: number, + * capacity: { cpuCores: number, memBytes: number, diskBytes: number }, * }>} */ -async function collectUsage({ siteId = null } = {}) { +async function collectUsage({ siteId = null, psiProbeLimit = PSI_PROBE_LIMIT } = {}) { const nodeWhere = db.Node.provisionableWhere(); if (siteId != null) nodeWhere.siteId = siteId; const nodes = await db.Node.findAll({ where: nodeWhere }); - if (nodes.length === 0) return { samples: [], findings: [], unknownNodeRows: 0 }; + const capacity = { cpuCores: 0, memBytes: 0, diskBytes: 0 }; + if (nodes.length === 0) return { samples: [], findings: [], unknownNodeRows: 0, capacity }; const containerIndex = await loadContainerIndex(siteId); @@ -61,6 +72,8 @@ async function collectUsage({ siteId = null } = {}) { const samples = []; const findings = []; + // API clients per sample, for the PSI probe pass below. + const apiBySample = new Map(); let unknownNodeRows = 0; for (const node of nodes) { @@ -68,14 +81,26 @@ async function collectUsage({ siteId = null } = {}) { if (covered.has(nodeKey)) continue; covered.add(nodeKey); + let api; let resources; + let nodeRows; try { - const api = await node.api(); + api = await node.api(); resources = await api.clusterResources('lxc'); + nodeRows = await api.clusterResources('node'); } catch (err) { console.error(`UsageCollection: node ${node.name} (site ${node.siteId}) unreachable: ${err.message}`); continue; } + + // Cluster capacity from the node rows (each cluster contributes once). + for (const row of Array.isArray(nodeRows) ? nodeRows : []) { + covered.add(`${node.siteId}:${row.node}`); + capacity.cpuCores += row.maxcpu || 0; + capacity.memBytes += row.maxmem || 0; + capacity.diskBytes += row.maxdisk || 0; + } + if (!Array.isArray(resources)) continue; for (const resource of resources) { @@ -94,11 +119,37 @@ async function collectUsage({ siteId = null } = {}) { const container = containerIndex.get(`${dbNode.id}:${resource.vmid}`) || null; const { sample, finding } = buildUsageSample({ resource, node: dbNode, container }); samples.push(sample); + apiBySample.set(sample, api); if (finding) findings.push(finding); } } - return { samples, findings, unknownNodeRows }; + await probePsi(samples, apiBySample, psiProbeLimit); + + return { samples, findings, unknownNodeRows, capacity }; +} + +/** + * Tier-2 PSI pass: probe `rrddata` for the highest-utilization running + * containers (budget-capped) and fill in their psi* sample fields in place. + * Probe failures are logged and leave the sample's PSI null. + * @param {Array} samples + * @param {Map} apiBySample - Sample -> API client that reported it + * @param {number} limit + */ +async function probePsi(samples, apiBySample, limit) { + const candidates = selectPsiCandidates(samples, limit); + await Promise.all(candidates.map(async (sample) => { + const api = apiBySample.get(sample); + if (!api || typeof api.rrdData !== 'function') return; + try { + const rows = await api.rrdData(sample.node, sample.vmid, 'hour'); + const psi = latestPsi(rows); + if (psi) Object.assign(sample, psi); + } catch (err) { + console.error(`UsageCollection: PSI probe failed for CT ${sample.vmid} on ${sample.node}: ${err.message}`); + } + })); } module.exports = { collectUsage }; diff --git a/create-a-container/utils/usage-psi.js b/create-a-container/utils/usage-psi.js new file mode 100644 index 00000000..d98bcaec --- /dev/null +++ b/create-a-container/utils/usage-psi.js @@ -0,0 +1,84 @@ +'use strict'; + +/** + * usage-psi.js — pure helpers for PSI (pressure stall information) collection + * (issue #440, tier 2). + * + * Proxmox exports per-container PSI through `rrddata` — but that is one API + * call per container, so the fleet is never swept. Each cycle probes only the + * containers most likely to be under pressure (highest memory/CPU utilization + * relative to their allocation), capped at a fixed budget. Raw stats miss + * exactly the incidents PSI catches (a container at 99% memory can be healthy; + * one at 99% with psiMemFull > 40 is thrashing), which is why the report + * carries both. + * + * No I/O here — candidate selection and RRD parsing are pure so they are + * trivially unit-tested; utils/usage-collection.js owns the API calls. + */ + +/** RRD field -> sample field for the six PSI series. */ +const PSI_FIELDS = { + pressurecpusome: 'psiCpuSome', + pressurecpufull: 'psiCpuFull', + pressurememorysome: 'psiMemSome', + pressurememoryfull: 'psiMemFull', + pressureiosome: 'psiIoSome', + pressureiofull: 'psiIoFull', +}; + +/** + * Utilization score used to prioritize PSI probes: the worst of memory and + * CPU usage as a fraction of allocation. Containers without usable ratios + * score 0 (still probed when the budget allows). + * @param {object} sample - Normalized sample (utils/usage-sample.js) + * @returns {number} + */ +function utilizationScore(sample) { + const mem = sample.memAlloc > 0 && sample.memUsed != null ? sample.memUsed / sample.memAlloc : 0; + const cpu = sample.cpuAlloc > 0 && sample.cpuUsed != null ? sample.cpuUsed / sample.cpuAlloc : 0; + return Math.max(mem, cpu); +} + +/** + * Pick which running containers to probe for PSI this cycle, ordered by + * utilization score (highest first) and capped at `limit`. + * @param {Array} samples - Normalized samples + * @param {number} limit - Probe budget for the cycle + * @returns {Array} Subset of samples to probe + */ +function selectPsiCandidates(samples, limit) { + if (limit <= 0) return []; + return samples + .filter((s) => s.status === 'running') + .sort((a, b) => utilizationScore(b) - utilizationScore(a)) + .slice(0, limit); +} + +/** + * Extract the most recent PSI readings from an rrddata series. RRD rows may + * trail off with nulls, so the scan walks backwards and takes the newest + * non-null value per field. + * @param {Array|null|undefined} rows - rrddata response (oldest first) + * @returns {object|null} `{ psiCpuSome, ..., psiIoFull }` (each number|null), + * or null when the series has no PSI data at all + */ +function latestPsi(rows) { + if (!Array.isArray(rows) || rows.length === 0) return null; + + const psi = {}; + let found = false; + for (const [rrdField, sampleField] of Object.entries(PSI_FIELDS)) { + psi[sampleField] = null; + for (let i = rows.length - 1; i >= 0; i--) { + const value = rows[i]?.[rrdField]; + if (typeof value === 'number' && Number.isFinite(value)) { + psi[sampleField] = value; + found = true; + break; + } + } + } + return found ? psi : null; +} + +module.exports = { selectPsiCandidates, latestPsi, PSI_FIELDS }; diff --git a/create-a-container/utils/usage-report.js b/create-a-container/utils/usage-report.js index 462cf2a3..5ead19c9 100644 --- a/create-a-container/utils/usage-report.js +++ b/create-a-container/utils/usage-report.js @@ -15,13 +15,21 @@ const SUM_FIELDS = [ 'netInBytes', 'netOutBytes', ]; +const PSI_SAMPLE_FIELDS = [ + 'psiCpuSome', 'psiCpuFull', + 'psiMemSome', 'psiMemFull', + 'psiIoSome', 'psiIoFull', +]; + /** * Group usage samples by owner. Null owners collapse into a single * `owner: null` row (rendered as "unattributed" by callers). Absent metrics - * (null) are excluded from sums rather than treated as zero. + * (null) are excluded from sums rather than treated as zero. `pressureMax` is + * the worst PSI reading across the owner's probed containers (null when none + * were probed this cycle). * @param {Array} samples - Normalized samples from utils/usage-sample.js * @returns {Array} One row per owner, sorted by owner name (null last): - * { owner, containerCount, runningCount, , containers } + * { owner, containerCount, runningCount, , pressureMax, containers } */ function aggregateByOwner(samples) { const byOwner = new Map(); @@ -30,7 +38,7 @@ function aggregateByOwner(samples) { const key = sample.owner ?? null; let row = byOwner.get(key); if (!row) { - row = { owner: key, containerCount: 0, runningCount: 0, containers: [] }; + row = { owner: key, containerCount: 0, runningCount: 0, pressureMax: null, containers: [] }; for (const field of SUM_FIELDS) row[field] = 0; byOwner.set(key, row); } @@ -39,6 +47,11 @@ function aggregateByOwner(samples) { for (const field of SUM_FIELDS) { if (sample[field] != null) row[field] += sample[field]; } + for (const field of PSI_SAMPLE_FIELDS) { + if (sample[field] != null && (row.pressureMax === null || sample[field] > row.pressureMax)) { + row.pressureMax = sample[field]; + } + } row.containers.push(sample); } diff --git a/create-a-container/utils/usage-sample.js b/create-a-container/utils/usage-sample.js index 5a58be69..d9ddb931 100644 --- a/create-a-container/utils/usage-sample.js +++ b/create-a-container/utils/usage-sample.js @@ -101,6 +101,15 @@ function buildUsageSample({ resource, node, container }) { netInBytes: numberOrNull(resource.netin), netOutBytes: numberOrNull(resource.netout), uptime: numberOrNull(resource.uptime), + // PSI pressure readings are filled in by the tier-2 rrddata probe + // (utils/usage-collection.js) for high-utilization containers only; + // null means "not probed this cycle", not "no pressure". + psiCpuSome: null, + psiCpuFull: null, + psiMemSome: null, + psiMemFull: null, + psiIoSome: null, + psiIoFull: null, }; return { sample, finding };