diff --git a/apps/public/content/docs/self-hosting/deploy-coolify.mdx b/apps/public/content/docs/self-hosting/deploy-coolify.mdx index 6ef7dbbdf..b35ceae05 100644 --- a/apps/public/content/docs/self-hosting/deploy-coolify.mdx +++ b/apps/public/content/docs/self-hosting/deploy-coolify.mdx @@ -89,7 +89,7 @@ Coolify deploys OpenPanel with the following services: - **opapi**: OpenPanel API server (handles `/api` routes) - **opdashboard**: OpenPanel dashboard (frontend) -- **opworker**: Background worker for processing events +- **opworker**: Background worker for processing events (no public domain, see below) - **opdb**: PostgreSQL database - **opkv**: Redis cache - **opch**: ClickHouse analytics database @@ -114,6 +114,12 @@ Coolify automatically handles these variables: You can configure optional variables like `ALLOW_REGISTRATION`, `RESEND_API_KEY`, `OPENAI_API_KEY`, etc. through Coolify's environment variable interface. +#### The worker service + +`opworker` has no public domain and ships with `DISABLE_BULLBOARD=1`. Nothing on it is meant to be reached from the internet; the API and dashboard talk to it over the internal network, and Coolify's healthcheck uses localhost. + +If you want the queue dashboard, set `BULLBOARD_USERNAME` and `BULLBOARD_PASSWORD`, remove `DISABLE_BULLBOARD`, and reach it through your own proxy or an SSH tunnel. Basic auth is a thin lock, so do not put it on a public hostname on its own. The dashboard is read-only unless you also set `BULLBOARD_READONLY=0`. + ### Updating OpenPanel To update OpenPanel in Coolify: diff --git a/apps/public/content/docs/self-hosting/environment-variables.mdx b/apps/public/content/docs/self-hosting/environment-variables.mdx index bb4442ebf..1d9d1bd21 100644 --- a/apps/public/content/docs/self-hosting/environment-variables.mdx +++ b/apps/public/content/docs/self-hosting/environment-variables.mdx @@ -603,6 +603,45 @@ Disable BullMQ board UI. Set to `true` or `1` to disable the queue monitoring da DISABLE_BULLBOARD=true ``` +### BULLBOARD_USERNAME + +**Type**: `string` +**Required**: No +**Default**: none + +Username for the queue dashboard. The dashboard is not mounted unless both `BULLBOARD_USERNAME` and `BULLBOARD_PASSWORD` are set; without them the worker returns 404 for `/` and `/api/queues`. Requests must then carry HTTP basic credentials. `/metrics`, `/healthcheck`, `/healthz/live` and `/healthz/ready` stay open either way. + +**Example**: +```bash +BULLBOARD_USERNAME=admin +``` + +### BULLBOARD_PASSWORD + +**Type**: `string` +**Required**: No +**Default**: none + +Password for the queue dashboard. See `BULLBOARD_USERNAME`. + +**Example**: +```bash +BULLBOARD_PASSWORD=a-long-random-string +``` + +### BULLBOARD_READONLY + +**Type**: `boolean` +**Required**: No +**Default**: `true` + +The dashboard is read-only by default: it shows queues and jobs but will not pause, retry, empty or add anything. Set to `0` or `false` to allow those actions. + +**Example**: +```bash +BULLBOARD_READONLY=0 +``` + ### DISABLE_WORKERS **Type**: `boolean` diff --git a/apps/worker/src/app.test.ts b/apps/worker/src/app.test.ts new file mode 100644 index 000000000..285288576 --- /dev/null +++ b/apps/worker/src/app.test.ts @@ -0,0 +1,237 @@ +/** + * The queue dashboard is mounted at `/` and therefore catches every path that + * nothing before it claimed. These tests pin down two things: it is not + * reachable without credentials, and it never shadows the metrics or health + * routes that container orchestration and Prometheus call with no credentials. + * + * The queue, db and redis modules are replaced wholesale — the routes are + * exercised over a real socket, but nothing here talks to a datastore. + */ + +import type { Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const { makeQueue } = vi.hoisted(() => { + const makeQueue = (name: string) => ({ + name, + // BullMQAdapter refuses anything that does not look like a BullMQ queue. + metaValues: { version: 'bullmq5.0.0' }, + getJobCounts: async () => ({ + active: 0, + waiting: 0, + 'waiting-children': 0, + prioritized: 0, + completed: 0, + failed: 0, + delayed: 0, + paused: 0, + }), + isPaused: async () => false, + getJobs: async () => [], + }); + return { makeQueue }; +}); + +vi.mock('@openpanel/queue', () => ({ + eventsGroupQueues: [], + sessionsQueue: makeQueue('sessions'), + cronQueue: makeQueue('cron'), + notificationQueue: makeQueue('notification'), + importQueue: makeQueue('import'), + insightsQueue: makeQueue('insights'), + gscQueue: makeQueue('gsc'), + cohortComputeQueue: makeQueue('cohortCompute'), +})); + +vi.mock('@openpanel/db', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + db: { $executeRaw: async () => 1 }, + chQuery: async () => [{ 1: 1 }], + }; +}); + +vi.mock('@openpanel/redis', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, getRedisCache: () => ({ ping: async () => 'PONG' }) }; +}); + +// The local-only cron trigger routes drag in every job module; they are not +// what is under test. +vi.mock('./boot-debug', () => ({ bootDebugRoutes: vi.fn() })); + +// An empty registry — the real one has collectors that scrape Redis and +// ClickHouse. What matters here is that /metrics answers, not what it says. +vi.mock('./metrics', async () => { + const client = (await import('prom-client')).default; + return { register: new client.Registry() }; +}); + +import { createApp } from './app'; + +const USERNAME = 'queues'; +const PASSWORD = 'correct-horse'; + +const basic = (username: string, password: string) => + `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`; + +let server: Server | undefined; + +/** Boots the app on an ephemeral port and returns its origin. */ +async function boot() { + const app = createApp(); + server = await new Promise((resolve) => { + const listening = app.listen(0, () => resolve(listening)); + }); + const { port } = server.address() as AddressInfo; + return `http://127.0.0.1:${port}`; +} + +const OPEN_ROUTES = [ + '/healthcheck', + '/healthz/live', + '/healthz/ready', + '/metrics', +]; + +beforeEach(() => { + for (const key of [ + 'BULLBOARD_USERNAME', + 'BULLBOARD_PASSWORD', + 'BULLBOARD_READONLY', + 'DISABLE_BULLBOARD', + ]) { + delete process.env[key]; + } +}); + +afterEach(async () => { + if (server) { + await new Promise((resolve) => server?.close(resolve)); + server = undefined; + } +}); + +describe('worker http app', () => { + describe('with credentials configured', () => { + beforeEach(() => { + process.env.BULLBOARD_USERNAME = USERNAME; + process.env.BULLBOARD_PASSWORD = PASSWORD; + }); + + it('rejects the dashboard and its api without an Authorization header', async () => { + const origin = await boot(); + + for (const path of ['/', '/api/queues']) { + const res = await fetch(`${origin}${path}`); + expect(res.status, path).toBe(401); + expect(res.headers.get('www-authenticate')).toMatch(/^Basic/); + } + }); + + it('rejects a mutating route without an Authorization header', async () => { + const origin = await boot(); + + const res = await fetch(`${origin}/api/queues/cron/pause`, { + method: 'PUT', + }); + + expect(res.status).toBe(401); + }); + + it('rejects a wrong password of the same length as the right one', async () => { + const origin = await boot(); + const wrong = `${'x'.repeat(PASSWORD.length - 1)}y`; + expect(wrong).toHaveLength(PASSWORD.length); + + const res = await fetch(`${origin}/api/queues`, { + headers: { authorization: basic(USERNAME, wrong) }, + }); + + expect(res.status).toBe(401); + }); + + it('serves the queue list, read-only, with the right credentials', async () => { + const origin = await boot(); + + const res = await fetch(`${origin}/api/queues`, { + headers: { authorization: basic(USERNAME, PASSWORD) }, + }); + + expect(res.status).toBe(200); + const body = (await res.json()) as { + queues: { + name: string; + readOnlyMode: boolean; + allowRetries: boolean; + }[]; + }; + expect(body.queues.map((queue) => queue.name)).toContain('cron'); + expect(body.queues.every((queue) => queue.readOnlyMode)).toBe(true); + expect(body.queues.some((queue) => queue.allowRetries)).toBe(false); + }); + + it('allows writes when BULLBOARD_READONLY is turned off', async () => { + process.env.BULLBOARD_READONLY = '0'; + const origin = await boot(); + + const res = await fetch(`${origin}/api/queues`, { + headers: { authorization: basic(USERNAME, PASSWORD) }, + }); + + const body = (await res.json()) as { + queues: { readOnlyMode: boolean; allowRetries: boolean }[]; + }; + expect(body.queues.every((queue) => queue.readOnlyMode)).toBe(false); + expect(body.queues.every((queue) => queue.allowRetries)).toBe(true); + }); + }); + + it('does not mount the dashboard when no credentials are configured', async () => { + const origin = await boot(); + + const res = await fetch(`${origin}/api/queues`); + + expect(res.status).toBe(404); + }); + + it('does not mount the dashboard when DISABLE_BULLBOARD is set', async () => { + process.env.DISABLE_BULLBOARD = '1'; + process.env.BULLBOARD_USERNAME = USERNAME; + process.env.BULLBOARD_PASSWORD = PASSWORD; + const origin = await boot(); + + const res = await fetch(`${origin}/api/queues`); + + expect(res.status).toBe(404); + }); + + describe.each([ + ['no credentials', {}], + [ + 'credentials set', + { BULLBOARD_USERNAME: USERNAME, BULLBOARD_PASSWORD: PASSWORD }, + ], + [ + 'dashboard disabled', + { + DISABLE_BULLBOARD: '1', + BULLBOARD_USERNAME: USERNAME, + BULLBOARD_PASSWORD: PASSWORD, + }, + ], + ])('metrics and health with %s', (_label, env) => { + it('answer without credentials', async () => { + Object.assign(process.env, env); + const origin = await boot(); + + for (const path of OPEN_ROUTES) { + const res = await fetch(`${origin}${path}`); + expect(res.status, path).not.toBe(401); + expect(res.status, path).toBe(200); + } + }); + }); +}); diff --git a/apps/worker/src/app.ts b/apps/worker/src/app.ts new file mode 100644 index 000000000..ac2c846a5 --- /dev/null +++ b/apps/worker/src/app.ts @@ -0,0 +1,230 @@ +import { timingSafeEqual } from 'node:crypto'; +import { createBullBoard } from '@bull-board/api'; +import { BullMQAdapter } from '@bull-board/api/bullMQAdapter'; +import { ExpressAdapter } from '@bull-board/express'; +import { tryCatch } from '@openpanel/common'; +import { chQuery, db } from '@openpanel/db'; +import { + cohortComputeQueue, + cronQueue, + eventsGroupQueues, + gscQueue, + importQueue, + insightsQueue, + notificationQueue, + sessionsQueue, +} from '@openpanel/queue'; +import { getRedisCache } from '@openpanel/redis'; +import express, { type Express, type RequestHandler } from 'express'; +import { BullBoardGroupMQAdapter } from 'groupmq'; +import { bootDebugRoutes } from './boot-debug'; +import { register } from './metrics'; +import { isShuttingDown } from './utils/graceful-shutdown'; +import { logger } from './utils/logger'; +import { getEventsHeartbeat } from './utils/worker-heartbeat'; + +const EVENTS_HEARTBEAT_STALE_MS = 60_000; + +const isOff = (value: string | undefined) => value === '1' || value === 'true'; + +/** + * Compare two strings without letting the time taken depend on how much of + * them matches. `timingSafeEqual` throws on differing lengths, so a mismatch + * still runs a same-length comparison before returning false. + */ +function safeEqual(a: string, b: string) { + const left = Buffer.from(a, 'utf8'); + const right = Buffer.from(b, 'utf8'); + if (left.length !== right.length) { + timingSafeEqual(left, left); + return false; + } + return timingSafeEqual(left, right); +} + +function basicAuth(username: string, password: string): RequestHandler { + return (req, res, next) => { + const [scheme, encoded] = (req.headers.authorization ?? '').split(' '); + + if (scheme?.toLowerCase() !== 'basic' || !encoded) { + res + .set('WWW-Authenticate', 'Basic realm="Queues"') + .status(401) + .send('Unauthorized'); + return; + } + + const decoded = Buffer.from(encoded, 'base64').toString('utf8'); + const separator = decoded.indexOf(':'); + const givenUsername = + separator === -1 ? decoded : decoded.slice(0, separator); + const givenPassword = separator === -1 ? '' : decoded.slice(separator + 1); + + // Both comparisons always run so a wrong username costs the same as a + // wrong password. + const usernameOk = safeEqual(givenUsername, username); + const passwordOk = safeEqual(givenPassword, password); + + if (!(usernameOk && passwordOk)) { + res + .set('WWW-Authenticate', 'Basic realm="Queues"') + .status(401) + .send('Unauthorized'); + return; + } + + next(); + }; +} + +/** + * The dashboard is mounted at `/` and swallows every unmatched path, so it can + * only go on after the routes that must stay open. It is mounted at all only + * when a username and a password are configured — without them there is + * nothing to mount it behind, and requests fall through to Express' 404. + */ +function mountBullBoard(app: Express) { + const username = process.env.BULLBOARD_USERNAME; + const password = process.env.BULLBOARD_PASSWORD; + + if (isOff(process.env.DISABLE_BULLBOARD)) { + return; + } + + if (!(username && password)) { + logger.warn( + 'Queue dashboard not mounted: set BULLBOARD_USERNAME and BULLBOARD_PASSWORD to enable it' + ); + return; + } + + const readOnly = + process.env.BULLBOARD_READONLY !== '0' && + process.env.BULLBOARD_READONLY !== 'false'; + const adapterOptions = { readOnlyMode: readOnly, allowRetries: !readOnly }; + + const serverAdapter = new ExpressAdapter(); + serverAdapter.setBasePath('/'); + createBullBoard({ + queues: [ + ...eventsGroupQueues.map( + (queue) => new BullBoardGroupMQAdapter(queue, adapterOptions) as any + ), + new BullMQAdapter(sessionsQueue, adapterOptions), + new BullMQAdapter(cronQueue, adapterOptions), + new BullMQAdapter(notificationQueue, adapterOptions), + new BullMQAdapter(importQueue, adapterOptions), + new BullMQAdapter(insightsQueue, adapterOptions), + new BullMQAdapter(gscQueue, adapterOptions), + new BullMQAdapter(cohortComputeQueue, adapterOptions), + ], + serverAdapter, + }); + + app.use('/', basicAuth(username, password), serverAdapter.getRouter()); +} + +export function createApp() { + const app = express(); + + // Local-only: trigger cron jobs on demand. Disabled in production. Mounted + // before bull-board so its routes take precedence. + if (process.env.NODE_ENV !== 'production') { + bootDebugRoutes(app); + } + + app.get('/metrics', (req, res) => { + res.set('Content-Type', register.contentType); + register + .metrics() + .then((metrics) => { + res.end(metrics); + }) + .catch((error) => { + res.status(500).end(error); + }); + }); + + app.get('/healthcheck', async (req, res) => { + const [redisResult, dbResult, chResult] = await Promise.all([ + tryCatch(async () => (await getRedisCache().ping()) === 'PONG'), + tryCatch(async () => !!(await db.$executeRaw`SELECT 1`)), + tryCatch(async () => (await chQuery('SELECT 1')).length > 0), + ]); + + const dependencies = { + redis: redisResult.ok && redisResult.data, + db: dbResult.ok && dbResult.data, + ch: chResult.ok && chResult.data, + }; + const dependencyErrors = { + redis: redisResult.error?.message, + db: dbResult.error?.message, + ch: chResult.error?.message, + }; + + const failedDependencies = Object.entries(dependencies) + .filter(([, ok]) => !ok) + .map(([name]) => name); + const workingDependencies = Object.entries(dependencies) + .filter(([, ok]) => ok) + .map(([name]) => name); + + const status = failedDependencies.length === 0 ? 200 : 503; + + if (status !== 200) { + logger.warn( + { + workingDependencies, + failedDependencies, + dependencies, + dependencyErrors, + }, + 'healthcheck failed' + ); + } + + res.status(status).json({ + ready: status === 200, + ...dependencies, + failedDependencies, + workingDependencies, + }); + }); + + // Kubernetes liveness — shallow, event loop only. + app.get('/healthz/live', (req, res) => { + res.status(200).json({ live: true }); + }); + + // Kubernetes readiness — shallow + shutdown-aware. When events workers run + // on this instance, also require the events consumer-loop heartbeat to be + // fresh (refreshed on each `completed`/`drained` event). If events are not + // enabled here, the heartbeat check is skipped. + app.get('/healthz/ready', (req, res) => { + if (isShuttingDown()) { + res.status(503).json({ ready: false, reason: 'shutting down' }); + return; + } + + const { enabled, lastActivityAt } = getEventsHeartbeat(); + if (enabled) { + const idleMs = Date.now() - lastActivityAt; + if (idleMs > EVENTS_HEARTBEAT_STALE_MS) { + res.status(503).json({ + ready: false, + reason: 'events consumer heartbeat stale', + idleMs, + thresholdMs: EVENTS_HEARTBEAT_STALE_MS, + }); + return; + } + } + + res.status(200).json({ ready: true }); + }); + + mountBullBoard(app); + + return app; +} diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index 0a38bffa2..795b566eb 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -1,34 +1,13 @@ import './utils/observability'; -import { createBullBoard } from '@bull-board/api'; -import { BullMQAdapter } from '@bull-board/api/bullMQAdapter'; -import { ExpressAdapter } from '@bull-board/express'; -import { tryCatch } from '@openpanel/common'; -import { chQuery, createInitialSalts, db } from '@openpanel/db'; -import { - cohortComputeQueue, - cronQueue, - eventsGroupQueues, - gscQueue, - importQueue, - insightsQueue, - notificationQueue, - sessionsQueue, -} from '@openpanel/queue'; -import { getRedisCache } from '@openpanel/redis'; -import express from 'express'; -import { BullBoardGroupMQAdapter } from 'groupmq'; +import { createInitialSalts } from '@openpanel/db'; import client from 'prom-client'; import sourceMapSupport from 'source-map-support'; +import { createApp } from './app'; import { bootCron } from './boot-cron'; -import { bootDebugRoutes } from './boot-debug'; import { bootWorkers } from './boot-workers'; import { register } from './metrics'; -import { isShuttingDown } from './utils/graceful-shutdown'; import { logger } from './utils/logger'; -import { getEventsHeartbeat } from './utils/worker-heartbeat'; - -const EVENTS_HEARTBEAT_STALE_MS = 60_000; sourceMapSupport.install(); @@ -37,129 +16,7 @@ async function start() { collectDefaultMetrics({ register }); const PORT = Number.parseInt(process.env.WORKER_PORT || '3000', 10); - const app = express(); - - // Local-only: trigger cron jobs on demand. Disabled in production. Mounted - // before bull-board so its routes take precedence. - if (process.env.NODE_ENV !== 'production') { - bootDebugRoutes(app); - } - - if ( - process.env.DISABLE_BULLBOARD !== '1' && - process.env.DISABLE_BULLBOARD !== 'true' - ) { - const serverAdapter = new ExpressAdapter(); - serverAdapter.setBasePath('/'); - createBullBoard({ - queues: [ - ...eventsGroupQueues.map( - (queue) => new BullBoardGroupMQAdapter(queue) as any - ), - new BullMQAdapter(sessionsQueue), - new BullMQAdapter(cronQueue), - new BullMQAdapter(notificationQueue), - new BullMQAdapter(importQueue), - new BullMQAdapter(insightsQueue), - new BullMQAdapter(gscQueue), - new BullMQAdapter(cohortComputeQueue), - ], - serverAdapter, - }); - - app.use('/', serverAdapter.getRouter()); - } - - app.get('/metrics', (req, res) => { - res.set('Content-Type', register.contentType); - register - .metrics() - .then((metrics) => { - res.end(metrics); - }) - .catch((error) => { - res.status(500).end(error); - }); - }); - - app.get('/healthcheck', async (req, res) => { - const [redisResult, dbResult, chResult] = await Promise.all([ - tryCatch(async () => (await getRedisCache().ping()) === 'PONG'), - tryCatch(async () => !!(await db.$executeRaw`SELECT 1`)), - tryCatch(async () => (await chQuery('SELECT 1')).length > 0), - ]); - - const dependencies = { - redis: redisResult.ok && redisResult.data, - db: dbResult.ok && dbResult.data, - ch: chResult.ok && chResult.data, - }; - const dependencyErrors = { - redis: redisResult.error?.message, - db: dbResult.error?.message, - ch: chResult.error?.message, - }; - - const failedDependencies = Object.entries(dependencies) - .filter(([, ok]) => !ok) - .map(([name]) => name); - const workingDependencies = Object.entries(dependencies) - .filter(([, ok]) => ok) - .map(([name]) => name); - - const status = failedDependencies.length === 0 ? 200 : 503; - - if (status !== 200) { - logger.warn( - { - workingDependencies, - failedDependencies, - dependencies, - dependencyErrors, - }, - 'healthcheck failed', - ); - } - - res.status(status).json({ - ready: status === 200, - ...dependencies, - failedDependencies, - workingDependencies, - }); - }); - - // Kubernetes liveness — shallow, event loop only. - app.get('/healthz/live', (req, res) => { - res.status(200).json({ live: true }); - }); - - // Kubernetes readiness — shallow + shutdown-aware. When events workers run - // on this instance, also require the events consumer-loop heartbeat to be - // fresh (refreshed on each `completed`/`drained` event). If events are not - // enabled here, the heartbeat check is skipped. - app.get('/healthz/ready', (req, res) => { - if (isShuttingDown()) { - res.status(503).json({ ready: false, reason: 'shutting down' }); - return; - } - - const { enabled, lastActivityAt } = getEventsHeartbeat(); - if (enabled) { - const idleMs = Date.now() - lastActivityAt; - if (idleMs > EVENTS_HEARTBEAT_STALE_MS) { - res.status(503).json({ - ready: false, - reason: 'events consumer heartbeat stale', - idleMs, - thresholdMs: EVENTS_HEARTBEAT_STALE_MS, - }); - return; - } - } - - res.status(200).json({ ready: true }); - }); + const app = createApp(); app.listen(PORT, () => { logger.info(`For the UI, open http://localhost:${PORT}/`); diff --git a/self-hosting/coolify.yml b/self-hosting/coolify.yml index 46aa5593f..d9d1b77f0 100644 --- a/self-hosting/coolify.yml +++ b/self-hosting/coolify.yml @@ -193,11 +193,13 @@ services: opapi: condition: service_healthy environment: - # FQDN - - SERVICE_FQDN_OPBULLBOARD # Common - NODE_ENV=production - SELF_HOSTED=true + # The queue dashboard is off by default. To use it, drop this line, set + # BULLBOARD_USERNAME and BULLBOARD_PASSWORD, and put it behind your own + # proxy rather than giving this service a public domain. + - DISABLE_BULLBOARD=1 # URLs - DATABASE_URL=postgres://${SERVICE_USER_POSTGRES}:${SERVICE_PASSWORD_POSTGRES}@opdb:5432/${OPENPANEL_POSTGRES_DB:-openpanel-db}?schema=public - DATABASE_URL_DIRECT=postgres://${SERVICE_USER_POSTGRES}:${SERVICE_PASSWORD_POSTGRES}@opdb:5432/${OPENPANEL_POSTGRES_DB:-openpanel-db}?schema=public