diff --git a/.gitignore b/.gitignore index a2f30c8d..c53977d7 100644 --- a/.gitignore +++ b/.gitignore @@ -14,11 +14,13 @@ build/ .env .env.local docker/.env +.instances/ .env.development.local .env.test.local .env.production.local .env.eng-104 .env.eng-104 +.shipsec-instance # Logs logs/ diff --git a/AGENTS.md b/AGENTS.md index b935db5e..fac41939 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,8 +3,9 @@ Security workflow orchestration platform. Visual builder + Temporal for reliability. ## Stack + - `frontend/` — React + Vite -- `backend/` — NestJS API +- `backend/` — NestJS API - `worker/` — Temporal activities + components - `packages/` — Shared code (component-sdk, backend-client) @@ -12,21 +13,69 @@ Security workflow orchestration platform. Visual builder + Temporal for reliabil ```bash just init # First time setup -just dev # Start everything -just dev stop # Stop -just dev logs # View logs +just dev # Start the active instance (default: 0) +just dev stop # Stop the active instance (does NOT stop shared infra) +just dev stop all # Stop all instances + shared infra +just dev logs # View logs for the active instance just help # All commands ``` -**URLs**: Frontend http://localhost:5173 | Backend http://localhost:3211 | Temporal http://localhost:8081 +**Active instance**: + +```bash +just instance show # Print active instance number +just instance use 5 # Set active instance for this workspace +``` + +**URLs**: + +- Frontend: `http://localhost:${5173 + instance*100}` +- Backend: `http://localhost:${3211 + instance*100}` +- Temporal UI (shared): http://localhost:8081 + +Full details: `docs/MULTI-INSTANCE-DEV.md` + +### Multi-Instance Local Dev (Important) + +Local development runs as **multiple app instances** (PM2) on top of **one shared Docker infra stack**. + +- Shared infra (Docker Compose project `shipsec-infra`): Postgres/Temporal/Redpanda/Redis/MinIO/Loki on fixed ports. +- Per-instance apps: `shipsec-{frontend,backend,worker}-N`. +- Isolation is via per-instance DB + Temporal namespace/task queue + Kafka topic suffixing (not per-instance infra containers). +- The workspace can have an **active instance** (stored in `.shipsec-instance`, gitignored). + +**Agent rule:** before running any dev commands, ensure you’re targeting the intended instance. + +- Always check: `just instance show` +- If the task is ambiguous (logs, curl, E2E, “run locally”, etc.), ask the user which instance to use. +- If the user says “use instance N”, prefer either: + - `just instance use N` then run `just dev` / `bun run test:e2e`, or + - explicit instance invocation (`just dev N ...`) for one-off commands. + +**Ports / URLs** + +- Frontend: `5173 + N*100` +- Backend: `3211 + N*100` +- Temporal UI (shared): http://localhost:8081 + +**E2E tests** + +- E2E targets the backend for `SHIPSEC_INSTANCE` (or the active instance). +- When asked to run E2E, confirm the instance and ensure that instance is running: `just dev N start`. + +**Keep docs in sync** + +If you change instance/infra behavior (justfile/scripts/pm2 config), update `docs/MULTI-INSTANCE-DEV.md` and this section accordingly in the same PR. ### After Backend Route Changes + ```bash bun --cwd backend run generate:openapi bun --cwd packages/backend-client run generate ``` ### Testing + ```bash bun run test # All tests bun run typecheck # Type check @@ -34,6 +83,7 @@ bun run lint # Lint ``` ### Database + ```bash just db-reset # Reset database bun --cwd backend run migration:push # Push schema @@ -41,6 +91,7 @@ bun --cwd backend run db:studio # View data ``` ## Rules + 1. TypeScript, 2-space indent 2. Conventional commits with DCO: `git commit -s -m "feat: ..."` 3. Tests alongside code in `__tests__/` folders @@ -64,11 +115,13 @@ Frontend ←→ Backend ←→ Temporal ←→ Worker ``` ### Component Runners + - **inline** — TypeScript code (HTTP calls, transforms, file ops) -- **docker** — Containers (security tools: Subfinder, DNSX, Nuclei) +- **docker** — Containers (security tools: Subfinder, DNSX, Nuclei) - **remote** — External executors (future: K8s, ECS) ### Real-time Streaming + - Terminal: Redis Streams → SSE → xterm.js - Events: Kafka → WebSocket - Logs: Loki + PostgreSQL @@ -83,9 +136,9 @@ When tasks match a skill, load it: `cat .claude/skills//SKILL.md` - component-development - Creating components (inline/docker). Dynamic ports, retry policies, PTY patterns, IsolatedContainerVolume. - project +component-development +Creating components (inline/docker). Dynamic ports, retry policies, PTY patterns, IsolatedContainerVolume. +project diff --git a/backend/.env.example b/backend/.env.example index b9fa520a..1964e7dc 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -55,4 +55,4 @@ OPENSEARCH_INDEX_PREFIX="logs-tenant" SECRET_STORE_MASTER_KEY="CHANGE_ME_32_CHAR_SECRET_KEY!!!!" # Kafka / Redpanda configuration for node I/O, log, and event ingestion -LOG_KAFKA_BROKERS="localhost:9092" +LOG_KAFKA_BROKERS="localhost:19092" diff --git a/backend/package.json b/backend/package.json index 6eaa3605..b4064185 100644 --- a/backend/package.json +++ b/backend/package.json @@ -26,6 +26,7 @@ "@nestjs/microservices": "^11.1.13", "@nestjs/platform-express": "^10.4.22", "@nestjs/swagger": "^11.2.5", + "@shipsec/backend-client": "workspace:*", "@shipsec/component-sdk": "workspace:*", "@shipsec/shared": "workspace:*", "@shipsec/studio-worker": "workspace:*", diff --git a/backend/src/agent-trace/agent-trace-ingest.service.ts b/backend/src/agent-trace/agent-trace-ingest.service.ts index 7756462f..61bfb322 100644 --- a/backend/src/agent-trace/agent-trace-ingest.service.ts +++ b/backend/src/agent-trace/agent-trace-ingest.service.ts @@ -1,5 +1,6 @@ import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; import { Consumer, Kafka } from 'kafkajs'; +import { getTopicResolver } from '../common/kafka-topic-resolver'; import { AgentTraceRepository, type AgentTraceEventInput } from './agent-trace.repository'; @@ -22,7 +23,10 @@ export class AgentTraceIngestService implements OnModuleInit, OnModuleDestroy { throw new Error('LOG_KAFKA_BROKERS must be configured for agent trace ingestion'); } - this.kafkaTopic = process.env.AGENT_TRACE_KAFKA_TOPIC ?? 'telemetry.agent-trace'; + // Use instance-aware topic name + const topicResolver = getTopicResolver(); + this.kafkaTopic = topicResolver.getAgentTraceTopic(); + this.kafkaGroupId = process.env.AGENT_TRACE_KAFKA_GROUP_ID ?? 'shipsec-agent-trace-ingestor'; this.kafkaClientId = process.env.AGENT_TRACE_KAFKA_CLIENT_ID ?? 'shipsec-backend-agent-trace'; } diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index 15f68ae1..81e3e26b 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -1,6 +1,7 @@ import { Module } from '@nestjs/common'; import { APP_GUARD } from '@nestjs/core'; import { ConfigModule } from '@nestjs/config'; +import { join } from 'node:path'; import { AppController } from './app.controller'; import { AppService } from './app.service'; @@ -47,11 +48,26 @@ const coreModules = [ const testingModules = process.env.NODE_ENV === 'production' ? [] : [TestingSupportModule]; +function getEnvFilePaths(): string[] { + // In multi-instance dev, each instance has its own env file under: + // .instances/instance-N/backend.env + // Backends run with cwd=backend/, so repo root is `..`. + const instance = process.env.SHIPSEC_INSTANCE; + if (instance) { + // Use only the instance env file. In multi-instance dev the workspace `.env` contains + // a default DATABASE_URL, and dotenv does not override already-set env vars; mixing + // would collapse isolation. + return [join(process.cwd(), '..', '.instances', `instance-${instance}`, 'backend.env')]; + } + + return ['.env', '../.env']; +} + @Module({ imports: [ ConfigModule.forRoot({ isGlobal: true, - envFilePath: ['.env', '../.env'], + envFilePath: getEnvFilePaths(), load: [authConfig], }), ...coreModules, diff --git a/backend/src/common/kafka-topic-resolver.ts b/backend/src/common/kafka-topic-resolver.ts new file mode 100644 index 00000000..203ecdf6 --- /dev/null +++ b/backend/src/common/kafka-topic-resolver.ts @@ -0,0 +1,107 @@ +/** + * Kafka Topic Resolver + * + * Provides instance-aware topic naming for multi-instance deployments. + * When SHIPSEC_INSTANCE is set, topics are namespaced with the instance number. + * + * Environment Variables: + * - SHIPSEC_INSTANCE: Instance number (0-9) for multi-instance isolation + * - LOG_KAFKA_TOPIC: Base topic for logs (default: telemetry.logs) + * - EVENT_KAFKA_TOPIC: Base topic for events (default: telemetry.events) + * - AGENT_TRACE_KAFKA_TOPIC: Base topic for agent traces (default: telemetry.agent-trace) + * - NODE_IO_KAFKA_TOPIC: Base topic for node I/O (default: telemetry.node-io) + */ + +export interface TopicResolverConfig { + instanceId?: string; + enableInstanceSuffix?: boolean; +} + +export class KafkaTopicResolver { + private instanceId: string | undefined; + private enableInstanceSuffix: boolean; + + constructor(config: TopicResolverConfig = {}) { + this.instanceId = config.instanceId ?? process.env.SHIPSEC_INSTANCE; + // Enable instance suffix only if SHIPSEC_INSTANCE is set + this.enableInstanceSuffix = config.enableInstanceSuffix ?? Boolean(this.instanceId); + } + + /** + * Resolve topic name with instance suffix if applicable + * @param baseTopic The base topic name + * @returns The topic name with instance suffix (if enabled) + */ + resolveTopic(baseTopic: string): string { + if (!this.enableInstanceSuffix || !this.instanceId) { + return baseTopic; + } + return `${baseTopic}.instance-${this.instanceId}`; + } + + /** + * Get logs topic + */ + getLogsTopic(): string { + const baseTopic = process.env.LOG_KAFKA_TOPIC ?? 'telemetry.logs'; + return this.resolveTopic(baseTopic); + } + + /** + * Get events topic + */ + getEventsTopic(): string { + const baseTopic = process.env.EVENT_KAFKA_TOPIC ?? 'telemetry.events'; + return this.resolveTopic(baseTopic); + } + + /** + * Get agent trace topic + */ + getAgentTraceTopic(): string { + const baseTopic = process.env.AGENT_TRACE_KAFKA_TOPIC ?? 'telemetry.agent-trace'; + return this.resolveTopic(baseTopic); + } + + /** + * Get node I/O topic + */ + getNodeIOTopic(): string { + const baseTopic = process.env.NODE_IO_KAFKA_TOPIC ?? 'telemetry.node-io'; + return this.resolveTopic(baseTopic); + } + + /** + * Check if instance isolation is enabled + */ + isInstanceIsolated(): boolean { + return this.enableInstanceSuffix; + } + + /** + * Get instance ID (if set) + */ + getInstanceId(): string | undefined { + return this.instanceId; + } +} + +// Singleton instance +let resolver: KafkaTopicResolver; + +/** + * Get or create the singleton topic resolver + */ +export function getTopicResolver(config?: TopicResolverConfig): KafkaTopicResolver { + if (!resolver) { + resolver = new KafkaTopicResolver(config); + } + return resolver; +} + +/** + * Reset the singleton (useful for testing) + */ +export function resetTopicResolver(): void { + resolver = undefined!; +} diff --git a/backend/src/events/event-ingest.service.ts b/backend/src/events/event-ingest.service.ts index d865a896..8132f826 100644 --- a/backend/src/events/event-ingest.service.ts +++ b/backend/src/events/event-ingest.service.ts @@ -1,5 +1,6 @@ import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; import { Consumer, Kafka } from 'kafkajs'; +import { getTopicResolver } from '../common/kafka-topic-resolver'; import { TraceRepository, type PersistedTraceEvent } from '../trace/trace.repository'; import type { TraceEventType } from '../trace/types'; @@ -38,7 +39,10 @@ export class EventIngestService implements OnModuleInit, OnModuleDestroy { throw new Error('LOG_KAFKA_BROKERS must be configured for event ingestion'); } - this.kafkaTopic = process.env.EVENT_KAFKA_TOPIC ?? 'telemetry.events'; + // Use instance-aware topic name + const topicResolver = getTopicResolver(); + this.kafkaTopic = topicResolver.getEventsTopic(); + this.kafkaGroupId = process.env.EVENT_KAFKA_GROUP_ID ?? 'shipsec-event-ingestor'; this.kafkaClientId = process.env.EVENT_KAFKA_CLIENT_ID ?? 'shipsec-backend-events'; } diff --git a/backend/src/logging/log-ingest.service.ts b/backend/src/logging/log-ingest.service.ts index 61706cc0..b4effe64 100644 --- a/backend/src/logging/log-ingest.service.ts +++ b/backend/src/logging/log-ingest.service.ts @@ -1,5 +1,6 @@ import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; import { Kafka, Consumer } from 'kafkajs'; +import { getTopicResolver } from '../common/kafka-topic-resolver'; import { LogStreamRepository } from '../trace/log-stream.repository'; import type { KafkaLogEntry } from './log-entry.types'; @@ -24,7 +25,11 @@ export class LogIngestService implements OnModuleInit, OnModuleDestroy { if (this.kafkaBrokers.length === 0) { throw new Error('LOG_KAFKA_BROKERS must be configured for Kafka log ingestion'); } - this.kafkaTopic = process.env.LOG_KAFKA_TOPIC ?? 'telemetry.logs'; + + // Use instance-aware topic name + const topicResolver = getTopicResolver(); + this.kafkaTopic = topicResolver.getLogsTopic(); + this.kafkaGroupId = process.env.LOG_KAFKA_GROUP_ID ?? 'shipsec-log-ingestor'; this.kafkaClientId = process.env.LOG_KAFKA_CLIENT_ID ?? 'shipsec-backend'; diff --git a/backend/src/main.ts b/backend/src/main.ts index 9471c461..9347d0a8 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -26,14 +26,23 @@ async function bootstrap() { } // Enable CORS for frontend + // Build dynamic origin list for multi-instance dev (instances 0-9) + const instanceOrigins: string[] = []; + for (let i = 0; i <= 9; i++) { + const frontendPort = 5173 + i * 100; + const backendPort = 3211 + i * 100; + instanceOrigins.push(`http://localhost:${frontendPort}`); + instanceOrigins.push(`http://127.0.0.1:${frontendPort}`); + instanceOrigins.push(`http://localhost:${backendPort}`); + instanceOrigins.push(`http://127.0.0.1:${backendPort}`); + } + app.enableCors({ origin: [ 'http://localhost', - 'http://localhost:5173', - 'http://localhost:5174', - 'http://localhost:3211', 'http://localhost:8090', 'https://studio.shipsec.ai', + ...instanceOrigins, ], credentials: true, methods: ['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], diff --git a/backend/src/node-io/node-io-ingest.service.ts b/backend/src/node-io/node-io-ingest.service.ts index ff493b8d..f7695cce 100644 --- a/backend/src/node-io/node-io-ingest.service.ts +++ b/backend/src/node-io/node-io-ingest.service.ts @@ -1,5 +1,6 @@ import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; import { Consumer, Kafka } from 'kafkajs'; +import { getTopicResolver } from '../common/kafka-topic-resolver'; import { NodeIORepository } from './node-io.repository'; @@ -42,7 +43,10 @@ export class NodeIOIngestService implements OnModuleInit, OnModuleDestroy { throw new Error('LOG_KAFKA_BROKERS must be configured for node I/O ingestion'); } - this.kafkaTopic = process.env.NODE_IO_KAFKA_TOPIC ?? 'telemetry.node-io'; + // Use instance-aware topic name + const topicResolver = getTopicResolver(); + this.kafkaTopic = topicResolver.getNodeIOTopic(); + this.kafkaGroupId = process.env.NODE_IO_KAFKA_GROUP_ID ?? 'shipsec-node-io-ingestor'; this.kafkaClientId = process.env.NODE_IO_KAFKA_CLIENT_ID ?? 'shipsec-backend-node-io'; } diff --git a/backend/tsconfig.json b/backend/tsconfig.json index f065601e..48e80ac8 100644 --- a/backend/tsconfig.json +++ b/backend/tsconfig.json @@ -2,13 +2,9 @@ "compilerOptions": { "target": "ES2022", "module": "ESNext", - "lib": [ - "ES2022" - ], + "lib": ["ES2022"], "moduleResolution": "bundler", - "types": [ - "bun-types" - ], + "types": ["bun-types"], "experimentalDecorators": true, "emitDecoratorMetadata": true, "esModuleInterop": true, @@ -22,13 +18,8 @@ "declaration": true, "emitDeclarationOnly": true }, - "include": [ - "src" - ], - "exclude": [ - "node_modules", - "build" - ], + "include": ["src"], + "exclude": ["node_modules", "build"], "references": [ { "path": "../packages/shared" @@ -36,8 +27,11 @@ { "path": "../packages/component-sdk" }, + { + "path": "../packages/backend-client" + }, { "path": "../worker" } ] -} \ No newline at end of file +} diff --git a/bun.lock b/bun.lock index 9cb8c615..4538ff20 100644 --- a/bun.lock +++ b/bun.lock @@ -39,6 +39,7 @@ "@nestjs/microservices": "^11.1.13", "@nestjs/platform-express": "^10.4.22", "@nestjs/swagger": "^11.2.5", + "@shipsec/backend-client": "workspace:*", "@shipsec/component-sdk": "workspace:*", "@shipsec/shared": "workspace:*", "@shipsec/studio-worker": "workspace:*", diff --git a/docker/docker-compose.infra.yml b/docker/docker-compose.infra.yml index b3e60115..1022273f 100644 --- a/docker/docker-compose.infra.yml +++ b/docker/docker-compose.infra.yml @@ -1,19 +1,18 @@ services: postgres: image: postgres:16-alpine - container_name: shipsec-postgres environment: POSTGRES_USER: shipsec POSTGRES_PASSWORD: shipsec POSTGRES_DB: shipsec POSTGRES_MULTIPLE_DATABASES: temporal ports: - - "5433:5432" + - '5433:5432' volumes: - postgres_data:/var/lib/postgresql/data - ./init-db:/docker-entrypoint-initdb.d healthcheck: - test: ["CMD-SHELL", "pg_isready -U shipsec"] + test: ['CMD-SHELL', 'pg_isready -U shipsec'] interval: 5s timeout: 3s retries: 10 @@ -21,7 +20,6 @@ services: temporal: image: temporalio/auto-setup:latest - container_name: shipsec-temporal depends_on: postgres: condition: service_healthy @@ -34,75 +32,71 @@ services: - POSTGRES_SEEDS=postgres - AUTO_SETUP=true ports: - - "7233:7233" + - '7233:7233' volumes: - temporal_data:/var/lib/temporal restart: unless-stopped temporal-ui: image: temporalio/ui:latest - container_name: shipsec-temporal-ui depends_on: - temporal environment: - TEMPORAL_ADDRESS=temporal:7233 - - TEMPORAL_CORS_ORIGINS=http://localhost:5173 + # Include several common dev frontend ports. + - TEMPORAL_CORS_ORIGINS=http://localhost:5173,http://localhost:5273,http://localhost:5373 ports: - - "8081:8080" + - '8081:8080' restart: unless-stopped minio: image: minio/minio:RELEASE.2024-10-02T17-50-41Z - container_name: shipsec-minio command: server /data --console-address ":9001" environment: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin ports: - - "9000:9000" - - "9001:9001" + - '9000:9000' + - '9001:9001' volumes: - minio_data:/data restart: unless-stopped healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + test: ['CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live'] interval: 30s timeout: 10s retries: 5 redis: image: redis:latest - container_name: shipsec-redis ports: - - "6379:6379" + - '6379:6379' volumes: - redis_data:/data restart: unless-stopped healthcheck: - test: ["CMD", "redis-cli", "ping"] + test: ['CMD', 'redis-cli', 'ping'] interval: 30s timeout: 10s retries: 5 loki: image: grafana/loki:3.2.1 - container_name: shipsec-loki command: -config.file=/etc/loki/local-config.yaml ports: - - "3100:3100" + - '3100:3100' volumes: - ./loki/loki-config.yaml:/etc/loki/local-config.yaml - loki_data:/loki restart: unless-stopped healthcheck: - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3100/ready"] + test: ['CMD', 'wget', '--no-verbose', '--tries=1', '--spider', 'http://localhost:3100/ready'] interval: 30s timeout: 10s retries: 5 redpanda: image: redpandadata/redpanda:v24.2.5 - container_name: shipsec-redpanda command: - redpanda - start @@ -112,28 +106,30 @@ services: - --overprovisioned - --node-id=0 - --check=false - - --advertise-kafka-addr=localhost:9092 + # Internal listener for Docker network containers (redpanda-console, etc.) + - --kafka-addr=internal://0.0.0.0:9092,external://0.0.0.0:19092 + - --advertise-kafka-addr=internal://redpanda:9092,external://localhost:19092 ports: - - "9092:9092" - - "9644:9644" + - '19092:19092' # External Kafka port for host apps + - '9092:9092' # Internal port (for Docker-to-Docker only, maps for debugging) + - '9644:9644' volumes: - redpanda_data:/var/lib/redpanda/data restart: unless-stopped healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9644/v1/status/ready"] + test: ['CMD', 'curl', '-f', 'http://localhost:9644/v1/status/ready'] interval: 30s timeout: 10s retries: 5 redpanda-console: image: redpandadata/console:v2.7.2 - container_name: shipsec-redpanda-console depends_on: - redpanda environment: CONFIG_FILEPATH: /etc/redpanda/console-config.yaml ports: - - "8082:8080" + - '8082:8080' volumes: - ./redpanda-console-config.yaml:/etc/redpanda/console-config.yaml:ro restart: unless-stopped @@ -145,7 +141,3 @@ volumes: temporal_data: redis_data: redpanda_data: - -networks: - default: - name: shipsec-network diff --git a/docker/init-db/01-create-instance-databases.sh b/docker/init-db/01-create-instance-databases.sh new file mode 100755 index 00000000..f31e3b5a --- /dev/null +++ b/docker/init-db/01-create-instance-databases.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Create instance-specific PostgreSQL databases +# This script is run automatically by PostgreSQL init-entrypoint + +set -e + +echo "🗄️ Creating instance-specific databases..." + +# Create databases for instances 0-9 +for i in {0..9}; do + DB_NAME="shipsec_instance_$i" + + # Check if database already exists + if psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d postgres -lqt | cut -d \| -f 1 | grep -qw "$DB_NAME"; then + echo " Database $DB_NAME already exists, skipping..." + else + echo " Creating $DB_NAME..." + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d postgres <<-EOSQL + CREATE DATABASE "$DB_NAME" OWNER "$POSTGRES_USER"; + GRANT ALL PRIVILEGES ON DATABASE "$DB_NAME" TO "$POSTGRES_USER"; +EOSQL + fi +done + +echo "✅ Instance-specific databases created successfully" +echo "" +echo "Available databases:" +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d postgres -c "\\l" | grep shipsec_instance diff --git a/docs/MULTI-INSTANCE-DEV.md b/docs/MULTI-INSTANCE-DEV.md new file mode 100644 index 00000000..71a994a7 --- /dev/null +++ b/docs/MULTI-INSTANCE-DEV.md @@ -0,0 +1,189 @@ +# Multi-Instance Development (Shared Infra) + +ShipSec Studio supports running multiple isolated dev instances (0-9) on one machine. + +The key design is: + +- **One shared Docker infra stack** (`shipsec-infra`): Postgres, Temporal, Redpanda, Redis, MinIO, Loki, etc. +- **Many app instances** (PM2): `shipsec-{backend,worker,frontend}-N` +- **Isolation comes from namespacing**, not per-instance infra containers: + - Postgres database: `shipsec_instance_N` + - Temporal namespace + task queue: `shipsec-dev-N` + - Kafka topics: `telemetry.*.instance-N` (via `SHIPSEC_INSTANCE`) + +## Quick Start + +```bash +# First-time setup +just init + +# Pick an "active" instance for this workspace (stored in .shipsec-instance) +just instance use 5 + +# Start the active instance (defaults to 0 if not set) +just dev + +# Start a specific instance explicitly +just dev 2 start + +# Stop just the active instance +just dev stop + +# Stop all instances + shared infra +just dev stop all +``` + +## Active Instance (Workspace Default) + +By default, `just dev` and related commands operate on an **active instance**. + +- Set it: `just instance use 5` +- Show it: `just instance show` +- Storage: `.shipsec-instance` (gitignored) +- Override per-shell: set `SHIPSEC_INSTANCE=N` in your environment +- Override per-command: pass an explicit instance number (`just dev 3 ...`) + +## Port Map + +Instance-scoped (offset by `N * 100`): + +| Service | Base | Instance 0 | Instance 1 | Instance 2 | Instance 5 | +| -------- | ---- | ---------- | ---------- | ---------- | ---------- | +| Frontend | 5173 | 5173 | 5273 | 5373 | 5673 | +| Backend | 3211 | 3211 | 3311 | 3411 | 3711 | + +Shared infra (fixed ports for all instances): + +| Service | Port | +| ---------------- | ----------- | +| Postgres | 5433 | +| Temporal | 7233 | +| Temporal UI | 8081 | +| Redis | 6379 | +| Redpanda (Kafka) | 19092 | +| Redpanda Console | 8082 | +| MinIO API/UI | 9000 / 9001 | +| Loki | 3100 | + +## Commands + +### Start / Stop + +```bash +# Start active instance +just dev + +# Start specific instance +just dev 1 start + +# Stop active instance (does NOT stop shared infra) +just dev stop + +# Stop a specific instance +just dev 1 stop + +# Stop all instances AND shared infra +just dev stop all +``` + +### Logs / Status + +```bash +# Logs/status for active instance +just dev logs +just dev status + +# Logs/status for a specific instance +just dev 2 logs +just dev 2 status + +# Infra + PM2 overview +just dev status all +``` + +### Clean (Reset Instance State) + +`clean` removes instance-local state and resets its “namespace”: + +- Drops/recreates `shipsec_instance_N` and reruns migrations +- Best-effort deletes Temporal namespace `shipsec-dev-N` +- Best-effort deletes Kafka topics `telemetry.*.instance-N` +- Deletes `.instances/instance-N/` + +```bash +just dev 0 clean +just dev 5 clean +``` + +## What Happens When You Run `just dev N start` + +1. Ensures `.instances/instance-N/{backend,worker,frontend}.env` exist (copied from root envs). +2. Brings up shared infra once (Docker Compose project `shipsec-infra`). +3. Bootstraps per-instance state: + - Ensures DB `shipsec_instance_N` exists + - Runs migrations against that DB + - Ensures Temporal namespace `shipsec-dev-N` exists + - Ensures per-instance Kafka topics exist (best-effort) +4. Starts 3 PM2 apps for that instance: + - `shipsec-backend-N` (port `3211 + N*100`) + - `shipsec-worker-N` (Temporal namespace/task queue `shipsec-dev-N`) + - `shipsec-frontend-N` (Vite port `5173 + N*100`, `VITE_API_URL` points at the instance backend) + +## Directory Structure + +Instance env overrides live in `.instances/` (auto-generated, safe to delete): + +``` +.instances/ + instance-0/ + backend.env + worker.env + frontend.env + instance-1/ + ... +``` + +## E2E Tests (Instance-Aware) + +E2E tests choose which backend to hit via instance selection: + +- `SHIPSEC_INSTANCE` (preferred) +- or `E2E_INSTANCE` +- or the workspace active instance (`.shipsec-instance`) + +Run E2E against the active instance: + +```bash +bun run test:e2e +``` + +Run E2E against a specific instance: + +```bash +SHIPSEC_INSTANCE=5 bun run test:e2e +``` + +## Troubleshooting + +### Port already in use (frontend/backend) + +```bash +lsof -i :3211 +lsof -i :5173 +``` + +### Instance is unhealthy but infra is fine + +```bash +just dev 5 logs +just dev 5 status +just dev 5 clean +just dev 5 start +``` + +### Infra conflicts / stuck containers + +```bash +just dev stop all +just infra clean +``` diff --git a/e2e-tests/alert-investigation.test.ts b/e2e-tests/alert-investigation.test.ts index 4c8d8ff0..a73d546b 100644 --- a/e2e-tests/alert-investigation.test.ts +++ b/e2e-tests/alert-investigation.test.ts @@ -3,7 +3,9 @@ import { spawnSync } from 'node:child_process'; import { readFileSync } from 'node:fs'; import { join } from 'node:path'; -const API_BASE = 'http://127.0.0.1:3211/api/v1'; +import { getApiBaseUrl } from './helpers/api-base'; + +const API_BASE = getApiBaseUrl(); const HEADERS = { 'Content-Type': 'application/json', 'x-internal-token': 'local-internal-token', diff --git a/e2e-tests/cleanup.ts b/e2e-tests/cleanup.ts index 8016163b..253fdcfc 100644 --- a/e2e-tests/cleanup.ts +++ b/e2e-tests/cleanup.ts @@ -5,7 +5,9 @@ * This keeps the workspace clean and prevents test artifact accumulation. */ -const API_BASE = 'http://localhost:3211/api/v1'; +import { getApiBaseUrl } from './helpers/api-base'; + +const API_BASE = getApiBaseUrl(); const HEADERS = { 'Content-Type': 'application/json', 'x-internal-token': 'local-internal-token', diff --git a/e2e-tests/error-handling.test.ts b/e2e-tests/error-handling.test.ts index d67cc4d4..11232a97 100644 --- a/e2e-tests/error-handling.test.ts +++ b/e2e-tests/error-handling.test.ts @@ -11,7 +11,9 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; -const API_BASE = 'http://localhost:3211/api/v1'; +import { getApiBaseUrl } from './helpers/api-base'; + +const API_BASE = getApiBaseUrl(); const HEADERS = { 'Content-Type': 'application/json', 'x-internal-token': 'local-internal-token', @@ -180,7 +182,7 @@ beforeAll(async () => { console.log(' 💡 To run E2E tests:'); console.log(' 1. Set RUN_E2E=true'); console.log(' 2. Start services: pm2 start pm2.config.cjs'); - console.log(' 3. Verify: curl http://localhost:3211/api/v1/health'); + console.log(` 3. Verify: curl ${API_BASE}/health`); return; } diff --git a/e2e-tests/helpers/api-base.ts b/e2e-tests/helpers/api-base.ts new file mode 100644 index 00000000..e92c548c --- /dev/null +++ b/e2e-tests/helpers/api-base.ts @@ -0,0 +1,26 @@ +const DEFAULT_INSTANCE = 0; +const BACKEND_BASE_PORT = 3211; + +function readInstance(): number { + const raw = process.env.E2E_INSTANCE ?? process.env.SHIPSEC_INSTANCE ?? String(DEFAULT_INSTANCE); + const parsed = Number.parseInt(raw, 10); + if (Number.isNaN(parsed) || parsed < 0) { + return DEFAULT_INSTANCE; + } + return parsed; +} + +export function getE2EInstance(): number { + return readInstance(); +} + +export function getBackendPortForInstance(instance: number): number { + return BACKEND_BASE_PORT + instance * 100; +} + +export function getApiBaseUrl(): string { + const instance = getE2EInstance(); + const port = getBackendPortForInstance(instance); + return `http://127.0.0.1:${port}/api/v1`; +} + diff --git a/e2e-tests/http-observability.test.ts b/e2e-tests/http-observability.test.ts index 4407ed70..d75d332a 100644 --- a/e2e-tests/http-observability.test.ts +++ b/e2e-tests/http-observability.test.ts @@ -11,7 +11,9 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; -const API_BASE = 'http://localhost:3211/api/v1'; +import { getApiBaseUrl } from './helpers/api-base'; + +const API_BASE = getApiBaseUrl(); const HEADERS = { 'Content-Type': 'application/json', 'x-internal-token': 'local-internal-token', @@ -122,7 +124,7 @@ beforeAll(async () => { console.log(' 💡 To run E2E tests:'); console.log(' 1. Set RUN_E2E=true'); console.log(' 2. Start services: pm2 start pm2.config.cjs'); - console.log(' 3. Verify: curl http://localhost:3211/api/v1/health'); + console.log(` 3. Verify: curl ${API_BASE}/health`); return; } diff --git a/e2e-tests/node-io-spilling.test.ts b/e2e-tests/node-io-spilling.test.ts index 38db43c9..f38eefb4 100644 --- a/e2e-tests/node-io-spilling.test.ts +++ b/e2e-tests/node-io-spilling.test.ts @@ -7,7 +7,9 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; -const API_BASE = 'http://localhost:3211/api/v1'; +import { getApiBaseUrl } from './helpers/api-base'; + +const API_BASE = getApiBaseUrl(); const HEADERS = { 'Content-Type': 'application/json', 'x-internal-token': 'local-internal-token', diff --git a/e2e-tests/secret-resolution.test.ts b/e2e-tests/secret-resolution.test.ts index 47021aea..25042dca 100644 --- a/e2e-tests/secret-resolution.test.ts +++ b/e2e-tests/secret-resolution.test.ts @@ -7,7 +7,9 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; -const API_BASE = 'http://localhost:3211/api/v1'; +import { getApiBaseUrl } from './helpers/api-base'; + +const API_BASE = getApiBaseUrl(); const HEADERS = { 'Content-Type': 'application/json', 'x-internal-token': 'local-internal-token', diff --git a/e2e-tests/subworkflow.test.ts b/e2e-tests/subworkflow.test.ts index 203f84ac..3b66793a 100644 --- a/e2e-tests/subworkflow.test.ts +++ b/e2e-tests/subworkflow.test.ts @@ -11,7 +11,9 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; -const API_BASE = 'http://localhost:3211/api/v1'; +import { getApiBaseUrl } from './helpers/api-base'; + +const API_BASE = getApiBaseUrl(); const HEADERS = { 'Content-Type': 'application/json', 'x-internal-token': 'local-internal-token', diff --git a/e2e-tests/webhooks.test.ts b/e2e-tests/webhooks.test.ts index cbe46be1..9a4a9230 100644 --- a/e2e-tests/webhooks.test.ts +++ b/e2e-tests/webhooks.test.ts @@ -6,7 +6,9 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; -const API_BASE = 'http://localhost:3211/api/v1'; +import { getApiBaseUrl } from './helpers/api-base'; + +const API_BASE = getApiBaseUrl(); const HEADERS = { 'Content-Type': 'application/json', 'x-internal-token': 'local-internal-token', diff --git a/justfile b/justfile index f5f801e8..145cf510 100644 --- a/justfile +++ b/justfile @@ -6,6 +6,24 @@ default: @just help +# Set/show the workspace "active" instance used when you run `just dev` without an explicit instance. +# This is stored in `.shipsec-instance` (gitignored). +instance action="show" value="": + #!/usr/bin/env bash + set -euo pipefail + case "{{action}}" in + show) + ./scripts/active-instance.sh get + ;; + use|set) + ./scripts/active-instance.sh set "{{value}}" + ;; + *) + echo "Usage: just instance [show|use] [0-9]" + exit 1 + ;; + esac + # === Development (recommended for contributors) === # Initialize environment files from examples @@ -34,15 +52,82 @@ init: echo " Then run: just dev" # Start development environment with hot-reload -dev action="start": +# Usage: just dev [instance] [action] +# Examples: just dev, just dev 1, just dev 2 start, just dev 1 logs, just dev stop all +dev *args: #!/usr/bin/env bash set -euo pipefail - case "{{action}}" in - start) - echo "🚀 Starting development environment..." + + # Parse arguments: instance can be 0-9, action is start/stop/logs/status/clean + INSTANCE="$(./scripts/active-instance.sh get)" + ACTION="start" + INFRA_PROJECT_NAME="shipsec-infra" + + # Process arguments + for arg in {{args}}; do + case "$arg" in + [0-9]) + INSTANCE="$arg" + ;; + all) + # Special instance selector for bulk operations (e.g. `just dev stop all`) + INSTANCE="all" + ;; + start|stop|logs|status|clean|all) + ACTION="$arg" + ;; + *) + echo "❌ Unknown argument: $arg" + echo "Usage: just dev [instance] [action]" + echo " instance: 0-9 (default: 0)" + echo " action: start|stop|logs|status|clean" + exit 1 + ;; + esac + done + + # Handle special case: dev stop all + if [ "$ACTION" = "all" ]; then + ACTION="stop" + fi + + # Handle "just dev stop" as "just dev 0 stop" + if [ "$ACTION" = "stop" ] && [ "$INSTANCE" = "0" ] && [ -z "{{args}}" ]; then + true # Keep defaults + fi + # Validate "all" usage + if [ "$INSTANCE" = "all" ] && [ "$ACTION" != "stop" ] && [ "$ACTION" != "status" ] && [ "$ACTION" != "logs" ] && [ "$ACTION" != "clean" ]; then + echo "❌ Instance 'all' is only supported for: stop|status|logs|clean" + exit 1 + fi + + # Get ports for this instance (skip for "all") + if [ "$INSTANCE" != "all" ]; then + eval "$(./scripts/dev-instance-manager.sh ports "$INSTANCE")" + INSTANCE_DIR=".instances/instance-$INSTANCE" + export FRONTEND BACKEND + fi + + case "$ACTION" in + start) + echo "🚀 Starting development environment (instance $INSTANCE)..." + + # Initialize instance if needed + if [ ! -d "$INSTANCE_DIR" ]; then + ./scripts/dev-instance-manager.sh init "$INSTANCE" + fi + # Check for required env files - if [ ! -f "backend/.env" ] || [ ! -f "worker/.env" ] || [ ! -f "frontend/.env" ]; then + if [ ! -f "$INSTANCE_DIR/backend.env" ] || [ ! -f "$INSTANCE_DIR/worker.env" ] || [ ! -f "$INSTANCE_DIR/frontend.env" ]; then + echo "❌ Environment files not found in $INSTANCE_DIR!" + echo "" + echo " Attempting to initialize instance $INSTANCE..." + ./scripts/dev-instance-manager.sh init "$INSTANCE" + fi + + # Check for original env files if instance is 0 + if [ "$INSTANCE" = "0" ] && { [ ! -f "backend/.env" ] || [ ! -f "worker/.env" ] || [ ! -f "frontend/.env" ]; }; then echo "❌ Environment files not found!" echo "" echo " Run this first: just init" @@ -50,52 +135,134 @@ dev action="start": echo " This will create .env files from the example templates." exit 1 fi - - # Start infrastructure - docker compose -f docker/docker-compose.infra.yml up -d - + + # Start shared infrastructure (one stack for all instances) + echo "⏳ Starting shared infrastructure..." + docker compose -f docker/docker-compose.infra.yml \ + --project-name="$INFRA_PROJECT_NAME" \ + up -d + # Wait for Postgres echo "⏳ Waiting for infrastructure..." - timeout 30s bash -c 'until docker exec shipsec-postgres pg_isready -U shipsec >/dev/null 2>&1; do sleep 1; done' || true + POSTGRES_CONTAINER="$(docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" ps -q postgres)" + if [ -n "$POSTGRES_CONTAINER" ]; then + timeout 30s bash -c "until docker exec $POSTGRES_CONTAINER pg_isready -U shipsec >/dev/null 2>&1; do sleep 1; done" || true + fi - # Update git SHA and start PM2 + # Ensure instance-specific DB/namespace exists and migrations are applied. + ./scripts/instance-bootstrap.sh "$INSTANCE" + + # Prepare PM2 environment variables + export SHIPSEC_INSTANCE="$INSTANCE" + export SHIPSEC_ENV=development + export NODE_ENV=development + export TERMINAL_REDIS_URL="redis://localhost:6379" + export LOG_KAFKA_BROKERS="localhost:19092" + export EVENT_KAFKA_BROKERS="localhost:19092" + + # Update git SHA and start PM2 with instance-specific config ./scripts/set-git-sha.sh || true - SHIPSEC_ENV=development NODE_ENV=development pm2 startOrReload pm2.config.cjs --only shipsec-frontend,shipsec-backend,shipsec-worker --update-env - + + pm2 startOrReload pm2.config.cjs \ + --only "shipsec-frontend-$INSTANCE,shipsec-backend-$INSTANCE,shipsec-worker-$INSTANCE" \ + --update-env + echo "" - echo "✅ Development environment ready" - echo " Frontend: http://localhost:5173" - echo " Backend: http://localhost:3211" - echo " Temporal UI: http://localhost:8081" + echo "✅ Development environment ready (instance $INSTANCE)" + ./scripts/dev-instance-manager.sh info "$INSTANCE" echo "" - echo "💡 just dev logs - View application logs" - echo "💡 just dev stop - Stop everything" + echo "💡 just dev $INSTANCE logs - View application logs" + echo "💡 just dev $INSTANCE stop - Stop this instance" echo "" - + # Version check bun backend/scripts/version-check-summary.ts 2>/dev/null || true ;; stop) - echo "🛑 Stopping development environment..." - pm2 delete shipsec-frontend shipsec-backend shipsec-worker shipsec-test-worker 2>/dev/null || true - docker compose -f docker/docker-compose.infra.yml down - echo "✅ Stopped" + if [ "$INSTANCE" = "all" ]; then + echo "🛑 Stopping all development environments..." + + # Stop all PM2 apps + pm2 delete shipsec-{frontend,backend,worker}-{0..9} 2>/dev/null || true + pm2 delete shipsec-test-worker 2>/dev/null || true + + # Stop shared infrastructure + just infra down + + echo "✅ All development environments stopped" + else + echo "🛑 Stopping development environment (instance $INSTANCE)..." + + # Stop PM2 apps for this instance + pm2 delete shipsec-{frontend,backend,worker}-"$INSTANCE" 2>/dev/null || true + + echo "✅ Instance $INSTANCE stopped" + fi ;; logs) - pm2 logs + if [ "$INSTANCE" = "all" ]; then + echo "📋 Viewing logs for all instances..." + pm2 logs + else + echo "📋 Viewing logs for instance $INSTANCE..." + pm2 logs "shipsec-frontend-$INSTANCE|shipsec-backend-$INSTANCE|shipsec-worker-$INSTANCE" + fi ;; status) - pm2 status - docker compose -f docker/docker-compose.infra.yml ps + if [ "$INSTANCE" = "all" ]; then + just status + else + echo "📊 Status of instance $INSTANCE:" + echo "" + pm2 status 2>/dev/null | grep -E "shipsec-(frontend|backend|worker)-$INSTANCE|error" || echo "(Instance $INSTANCE not running in PM2)" + echo "" + just status + fi ;; clean) - echo "🧹 Cleaning development environment..." - pm2 delete shipsec-frontend shipsec-backend shipsec-worker shipsec-test-worker 2>/dev/null || true - docker compose -f docker/docker-compose.infra.yml down -v - echo "✅ Development environment cleaned (PM2 stopped, infrastructure volumes removed)" + if [ "$INSTANCE" = "all" ]; then + echo "🧹 Cleaning all instances (0-9)..." + + # Stop all instance-specific PM2 apps + pm2 delete shipsec-{frontend,backend,worker}-{0..9} 2>/dev/null || true + + # Clean infra state for each instance + for i in {0..9}; do + if [ -d ".instances/instance-$i" ] || [ "$i" = "0" ]; then + echo " - Instance $i..." + ./scripts/instance-clean.sh "$i" >/dev/null 2>&1 || true + rm -rf ".instances/instance-$i" + fi + done + + # Cleanup root level instance marker if it exists + rm -f .shipsec-instance + + # Also clean global infra if requested? + # User usually runs `just infra clean` for that, but let's remind them. + echo "" + echo "💡 To also wipe all Docker volumes (PSQL, Kafka, etc.), run: just infra clean" + echo "✅ All instance-specific state cleaned" + else + echo "🧹 Cleaning instance $INSTANCE..." + + # Stop PM2 apps + pm2 delete shipsec-{frontend,backend,worker}-"$INSTANCE" 2>/dev/null || true + + # Remove instance-specific infra state (DB + Temporal namespace + topics, etc.) + ./scripts/instance-clean.sh "$INSTANCE" || true + + # Remove instance directory + rm -rf "$INSTANCE_DIR" + + echo "✅ Instance $INSTANCE cleaned" + fi ;; *) - echo "Usage: just dev [start|stop|logs|status|clean]" + echo "Usage: just dev [instance] [action]" + echo " instance: 0-9 (default: 0)" + echo " action: start|stop|logs|status|clean" + exit 1 ;; esac @@ -372,20 +539,21 @@ prod-images action="start": infra action="up": #!/usr/bin/env bash set -euo pipefail + INFRA_PROJECT_NAME="shipsec-infra" case "{{action}}" in up) - docker compose -f docker/docker-compose.infra.yml up -d + docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" up -d echo "✅ Infrastructure started (Postgres, Temporal, MinIO, Redis)" ;; down) - docker compose -f docker/docker-compose.infra.yml down + docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" down echo "✅ Infrastructure stopped" ;; logs) - docker compose -f docker/docker-compose.infra.yml logs -f + docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" logs -f ;; clean) - docker compose -f docker/docker-compose.infra.yml down -v + docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" down -v echo "✅ Infrastructure cleaned" ;; *) @@ -410,17 +578,21 @@ status: echo "=== Production Containers ===" docker compose -f docker/docker-compose.full.yml ps 2>/dev/null || echo " (Production not running)" -# Reset database (drops all data) -db-reset: +# Reset database for specific instance or all instances +# Usage: just db-reset [instance] +db-reset instance="0": #!/usr/bin/env bash set -euo pipefail - if ! docker ps --filter "name=shipsec-postgres" --format "{{{{.Names}}}}" | grep -q "shipsec-postgres"; then - echo "❌ PostgreSQL not running. Run: just dev" && exit 1 + + if [ "{{instance}}" = "all" ]; then + echo "🗑️ Resetting all instance databases..." + for i in {0..9}; do + ./scripts/db-reset-instance.sh "$i" 2>/dev/null || true + done + echo "✅ All instance databases reset" + else + ./scripts/db-reset-instance.sh "{{instance}}" fi - docker exec shipsec-postgres psql -U shipsec -d postgres -c "DROP DATABASE IF EXISTS shipsec;" - docker exec shipsec-postgres psql -U shipsec -d postgres -c "CREATE DATABASE shipsec;" - bun --cwd=backend run migration:push - echo "✅ Database reset" # Build production images without starting build: @@ -435,12 +607,22 @@ help: @echo "Getting Started:" @echo " just init Set up dependencies and environment files" @echo "" - @echo "Development (hot-reload):" - @echo " just dev Start development environment" - @echo " just dev stop Stop everything" - @echo " just dev logs View application logs" - @echo " just dev status Check service status" - @echo " just dev clean Stop and remove all data" + @echo "Development (hot-reload, multi-instance support):" + @echo " just dev Start the active instance (default: 0)" + @echo " just instance show Show active instance" + @echo " just instance use 5 Set active instance to 5 for this workspace" + @echo " just dev 1 Start instance 1" + @echo " just dev 2 start Explicitly start instance 2" + @echo " just dev 1 stop Stop instance 1" + @echo " just dev 2 logs View instance 2 logs" + @echo " just dev 0 status Check instance 0 status" + @echo " just dev 1 clean Stop and remove instance 1 data" + @echo " just dev stop all Stop all instances at once" + @echo " just dev status all Check status of all instances" + @echo "" + @echo " Note: Instances share one Docker infra stack (Postgres/Temporal/Redpanda/Redis/etc)" + @echo " Isolation comes from per-instance DB + Temporal namespace/task-queue + Kafka topic suffix" + @echo " Instance N uses base_port + N*100 (e.g., instance 0 uses 5173, instance 1 uses 5273)" @echo "" @echo "Production (Docker):" @echo " just prod-init Generate secrets in docker/.env (run once)" @@ -460,6 +642,8 @@ help: @echo " just infra clean Remove infrastructure data" @echo "" @echo "Utilities:" - @echo " just status Show status of all services" - @echo " just db-reset Reset database" - @echo " just build Build images only" + @echo " just status Show status of all services" + @echo " just db-reset Reset instance 0 database" + @echo " just db-reset 1 Reset instance 1 database" + @echo " just db-reset all Reset all instance databases" + @echo " just build Build images only" diff --git a/package.json b/package.json index 1b6e8dc3..73ef79cc 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "mcp:status": "pm2 status shipsec-mcp-server", "typecheck": "tsc --build", "test": "rm -rf worker/dist && bun test", - "test:e2e": "RUN_E2E=true bun test e2e-tests", + "test:e2e": "bash -lc 'SHIPSEC_INSTANCE=${SHIPSEC_INSTANCE:-$(./scripts/active-instance.sh get)} RUN_E2E=true bun test --force-exit e2e-tests'", "dev:docs": "cd docs && mint dev", "lint": "bun run lint:frontend && bun run lint:backend && bun run lint:worker", "lint:frontend": "bun --cwd=frontend run lint", diff --git a/packages/backend-client/package.json b/packages/backend-client/package.json index 2878df4a..dc0ab716 100644 --- a/packages/backend-client/package.json +++ b/packages/backend-client/package.json @@ -2,6 +2,7 @@ "name": "@shipsec/backend-client", "version": "0.1.0", "type": "module", + "private": false, "main": "./src/index.ts", "types": "./src/index.ts", "scripts": { @@ -18,4 +19,4 @@ "openapi-typescript": "^7.9.1", "typescript": "^5.9.3" } -} \ No newline at end of file +} diff --git a/pm2.config.cjs b/pm2.config.cjs index 4be50e8d..71452c39 100644 --- a/pm2.config.cjs +++ b/pm2.config.cjs @@ -200,6 +200,18 @@ const frontendEnv = loadFrontendEnv(); const environment = process.env.SHIPSEC_ENV || process.env.NODE_ENV || 'development'; const isProduction = environment === 'production'; +// Get instance number (0-9) for multi-instance support +const instanceNum = process.env.SHIPSEC_INSTANCE || '0'; +const instanceDatabaseUrl = `postgresql://shipsec:shipsec@localhost:5433/shipsec_instance_${instanceNum}`; +// Only set these defaults for local development. In production, credentials must +// come from the environment / deployment config. +const devInstanceEnv = isProduction + ? {} + : { + DATABASE_URL: instanceDatabaseUrl, + SECRET_STORE_MASTER_KEY: process.env.SECRET_STORE_MASTER_KEY || 'ShipSecLocalDevKey32Bytes!!!!!!!', + }; + // Environment-specific configuration const envConfig = { development: { @@ -216,64 +228,97 @@ const envConfig = { const currentEnvConfig = envConfig[isProduction ? 'production' : 'development']; +// Helper to get instance-specific env file path +function getInstanceEnvFile(appName, instance) { + return __dirname + `/.instances/instance-${instance}/${appName}.env`; +} + +// Helper to get instance-specific ports +function getInstancePort(basePort, instance) { + return basePort + parseInt(instance) * 100; +} + +// Get env file (use instance-specific if it exists, otherwise fall back to root) +function resolveEnvFile(appName, instance) { + const instancePath = getInstanceEnvFile(appName, instance); + const rootPath = __dirname + `/${appName}/.env`; + + if (fs.existsSync(instancePath)) { + return instancePath; + } + return rootPath; +} + module.exports = { apps: [ { - name: 'shipsec-backend', + name: `shipsec-backend-${instanceNum}`, cwd: __dirname + '/backend', script: 'bun', args: isProduction ? 'src/main.ts' : 'run dev', interpreter: 'none', - env_file: __dirname + '/backend/.env', + env_file: resolveEnvFile('backend', instanceNum), env: { ...currentEnvConfig, + PORT: getInstancePort(3211, instanceNum), + // Ensure instance DB isolation even if dotenv auto-loads a workspace/default `.env`. + ...devInstanceEnv, TERMINAL_REDIS_URL: process.env.TERMINAL_REDIS_URL || 'redis://localhost:6379', - LOG_KAFKA_BROKERS: process.env.LOG_KAFKA_BROKERS || 'localhost:9092', + LOG_KAFKA_BROKERS: process.env.LOG_KAFKA_BROKERS || 'localhost:19092', LOG_KAFKA_TOPIC: process.env.LOG_KAFKA_TOPIC || 'telemetry.logs', - LOG_KAFKA_CLIENT_ID: process.env.LOG_KAFKA_CLIENT_ID || 'shipsec-backend', - LOG_KAFKA_GROUP_ID: process.env.LOG_KAFKA_GROUP_ID || 'shipsec-backend-log-consumer', + LOG_KAFKA_CLIENT_ID: process.env.LOG_KAFKA_CLIENT_ID || `shipsec-backend-${instanceNum}`, + LOG_KAFKA_GROUP_ID: process.env.LOG_KAFKA_GROUP_ID || `shipsec-backend-log-consumer-${instanceNum}`, EVENT_KAFKA_TOPIC: process.env.EVENT_KAFKA_TOPIC || 'telemetry.events', - EVENT_KAFKA_CLIENT_ID: process.env.EVENT_KAFKA_CLIENT_ID || 'shipsec-backend-events', - EVENT_KAFKA_GROUP_ID: process.env.EVENT_KAFKA_GROUP_ID || 'shipsec-event-ingestor', + EVENT_KAFKA_CLIENT_ID: process.env.EVENT_KAFKA_CLIENT_ID || `shipsec-backend-events-${instanceNum}`, + EVENT_KAFKA_GROUP_ID: process.env.EVENT_KAFKA_GROUP_ID || `shipsec-event-ingestor-${instanceNum}`, ENABLE_INGEST_SERVICES: process.env.ENABLE_INGEST_SERVICES || 'true', INTERNAL_SERVICE_TOKEN: process.env.INTERNAL_SERVICE_TOKEN || 'local-internal-token', + TEMPORAL_ADDRESS: process.env.TEMPORAL_ADDRESS || 'localhost:7233', + TEMPORAL_NAMESPACE: `shipsec-dev-${instanceNum}`, + TEMPORAL_TASK_QUEUE: `shipsec-dev-${instanceNum}`, }, watch: !isProduction ? ['src'] : false, ignore_watch: ['node_modules', 'dist', '*.log'], max_memory_restart: '500M', }, { - name: 'shipsec-frontend', + name: `shipsec-frontend-${instanceNum}`, cwd: __dirname + '/frontend', script: 'bun', - args: 'run dev', - env_file: __dirname + '/frontend/.env', + // Ensure each instance binds to its own Vite port (default is 5173). + args: ['run', 'dev', '--', '--port', String(getInstancePort(5173, instanceNum)), '--strictPort'], + env_file: resolveEnvFile('frontend', instanceNum), env: { ...frontendEnv, ...currentEnvConfig, + VITE_API_URL: `http://localhost:${getInstancePort(3211, instanceNum)}`, }, watch: !isProduction ? ['src'] : false, ignore_watch: ['node_modules', 'dist', '*.log'], }, { - name: 'shipsec-worker', + name: `shipsec-worker-${instanceNum}`, cwd: __dirname + '/worker', // Run the worker with Node + tsx to avoid Bun's SWC binding issues script: __dirname + '/node_modules/.bin/tsx', args: 'src/temporal/workers/dev.worker.ts', - env_file: __dirname + '/worker/.env', + env_file: resolveEnvFile('worker', instanceNum), env: Object.assign( { ...currentEnvConfig, NAPI_RS_FORCE_WASI: '1', INTERNAL_SERVICE_TOKEN: process.env.INTERNAL_SERVICE_TOKEN || 'local-internal-token', - STUDIO_API_BASE_URL: process.env.STUDIO_API_BASE_URL || 'http://localhost:3211/api/v1', + STUDIO_API_BASE_URL: process.env.STUDIO_API_BASE_URL || `http://localhost:${getInstancePort(3211, instanceNum)}/api/v1`, + ...devInstanceEnv, TERMINAL_REDIS_URL: process.env.TERMINAL_REDIS_URL || 'redis://localhost:6379', - LOG_KAFKA_BROKERS: process.env.LOG_KAFKA_BROKERS || 'localhost:9092', + LOG_KAFKA_BROKERS: process.env.LOG_KAFKA_BROKERS || 'localhost:19092', LOG_KAFKA_TOPIC: process.env.LOG_KAFKA_TOPIC || 'telemetry.logs', - LOG_KAFKA_CLIENT_ID: process.env.LOG_KAFKA_CLIENT_ID || 'shipsec-worker', + LOG_KAFKA_CLIENT_ID: process.env.LOG_KAFKA_CLIENT_ID || `shipsec-worker-${instanceNum}`, EVENT_KAFKA_TOPIC: process.env.EVENT_KAFKA_TOPIC || 'telemetry.events', - EVENT_KAFKA_CLIENT_ID: process.env.EVENT_KAFKA_CLIENT_ID || 'shipsec-worker-events', + EVENT_KAFKA_CLIENT_ID: process.env.EVENT_KAFKA_CLIENT_ID || `shipsec-worker-events-${instanceNum}`, + TEMPORAL_ADDRESS: process.env.TEMPORAL_ADDRESS || 'localhost:7233', + TEMPORAL_NAMESPACE: `shipsec-dev-${instanceNum}`, + TEMPORAL_TASK_QUEUE: `shipsec-dev-${instanceNum}`, }, swcBinaryPath ? { SWC_BINARY_PATH: swcBinaryPath } : {}, ), diff --git a/scripts/active-instance.sh b/scripts/active-instance.sh new file mode 100755 index 00000000..ef63dbbf --- /dev/null +++ b/scripts/active-instance.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Workspace-scoped "active instance" selector for multi-instance dev & tests. +# +# Usage: +# ./scripts/active-instance.sh get +# ./scripts/active-instance.sh set 5 +# +# Behavior: +# - Uses .shipsec-instance in repo root +# - Instance must be an integer 0-9 + +set -euo pipefail + +FILE=".shipsec-instance" +CMD="${1:-get}" + +die() { + echo "❌ $*" 1>&2 + exit 1 +} + +is_digit() { + [[ "$1" =~ ^[0-9]+$ ]] +} + +case "$CMD" in + get) + if [ -n "${SHIPSEC_INSTANCE:-}" ]; then + echo "${SHIPSEC_INSTANCE}" + exit 0 + fi + if [ -f "$FILE" ]; then + val="$(tr -d '[:space:]' < "$FILE" || true)" + if [ -n "$val" ]; then + echo "$val" + exit 0 + fi + fi + echo "0" + ;; + set) + val="${2:-}" + [ -n "$val" ] || die "Missing instance number. Example: ./scripts/active-instance.sh set 5" + is_digit "$val" || die "Instance must be a number (0-9). Got: $val" + if [ "$val" -lt 0 ] || [ "$val" -gt 9 ]; then + die "Instance must be 0-9. Got: $val" + fi + echo "$val" > "$FILE" + echo "✅ Active instance set to $val" + ;; + *) + die "Unknown command: $CMD (expected: get|set)" + ;; +esac + diff --git a/scripts/db-reset-instance.sh b/scripts/db-reset-instance.sh new file mode 100755 index 00000000..a4fd64f9 --- /dev/null +++ b/scripts/db-reset-instance.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Reset database for a specific instance +# Usage: ./scripts/db-reset-instance.sh [instance_number] + +set -euo pipefail + +INSTANCE=${1:-0} +COMPOSE_PROJECT_NAME="shipsec-infra" +DB_NAME="shipsec_instance_$INSTANCE" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +log_info() { + echo -e "${BLUE}ℹ${NC} $*" +} + +log_success() { + echo -e "${GREEN}✅${NC} $*" +} + +log_error() { + echo -e "${RED}❌${NC} $*" +} + +log_info "Resetting database for instance $INSTANCE..." +echo "" + +# Find PostgreSQL container +POSTGRES_CONTAINER=$(docker compose -f docker/docker-compose.infra.yml \ + --project-name="$COMPOSE_PROJECT_NAME" \ + ps -q postgres 2>/dev/null || echo "") + +if [ -z "$POSTGRES_CONTAINER" ]; then + log_error "PostgreSQL container not found for instance $INSTANCE" + log_error "Is the instance running? Try: just dev $INSTANCE start" + exit 1 +fi + +log_info "Found PostgreSQL container: $POSTGRES_CONTAINER" + +# Drop and recreate database +log_info "Dropping database $DB_NAME..." +docker exec "$POSTGRES_CONTAINER" \ + psql -v ON_ERROR_STOP=1 -U shipsec -d postgres \ + -c "DROP DATABASE IF EXISTS \"$DB_NAME\";" || true + +log_info "Creating database $DB_NAME..." +docker exec "$POSTGRES_CONTAINER" \ + psql -v ON_ERROR_STOP=1 -U shipsec -d postgres \ + -c "CREATE DATABASE \"$DB_NAME\" OWNER shipsec;" + +docker exec "$POSTGRES_CONTAINER" \ + psql -v ON_ERROR_STOP=1 -U shipsec -d postgres \ + -c "GRANT ALL PRIVILEGES ON DATABASE \"$DB_NAME\" TO shipsec;" + +# Run migrations +log_info "Running migrations for instance $INSTANCE..." +export SHIPSEC_INSTANCE="$INSTANCE" +export DATABASE_URL="postgresql://shipsec:shipsec@localhost:5433/$DB_NAME" + +if bun --cwd backend run migration:push > /dev/null 2>&1; then + log_success "Migrations completed" +else + log_error "Migrations failed" + log_error "Check backend logs: just dev $INSTANCE logs" + exit 1 +fi + +echo "" +log_success "Database reset for instance $INSTANCE" +log_info "Database: $DB_NAME" +log_info "Connection: postgresql://shipsec:shipsec@localhost:5433/$DB_NAME" diff --git a/scripts/dev-instance-manager.sh b/scripts/dev-instance-manager.sh new file mode 100755 index 00000000..7e1cf001 --- /dev/null +++ b/scripts/dev-instance-manager.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash +# Multi-instance dev stack manager for ShipSec Studio +# Handles isolated Docker containers and PM2 processes per instance + +set -euo pipefail + +# Configuration +INSTANCES_DIR=".instances" + +# Base port mappings +declare -A BASE_PORTS=( + [FRONTEND]=5173 + [BACKEND]=3211 +) + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +# Helper functions +log_info() { + echo -e "${BLUE}ℹ${NC} $*" +} + +log_success() { + echo -e "${GREEN}✅${NC} $*" +} + +log_warn() { + echo -e "${YELLOW}⚠️${NC} $*" +} + +log_error() { + echo -e "${RED}❌${NC} $*" +} + +get_instance_dir() { + local instance=$1 + echo "$INSTANCES_DIR/instance-$instance" +} + +get_port() { + local port_name=$1 + local instance=$2 + local base_port="${BASE_PORTS[$port_name]}" + + if [[ -z "$base_port" ]]; then + log_error "Unknown port: $port_name" + return 1 + fi + + # Port offset: instance N uses base_port + N*100 + echo $((base_port + instance * 100)) +} + +ensure_instance_dir() { + local instance=$1 + local inst_dir=$(get_instance_dir "$instance") + + if [ ! -d "$inst_dir" ]; then + mkdir -p "$inst_dir" + log_info "Created instance directory: $inst_dir" + fi +} + +copy_env_files() { + local instance=$1 + local inst_dir=$(get_instance_dir "$instance") + local root_env=".env" + + # Read KEY from repo root .env (first match). Prints empty string if missing. + get_root_env_value() { + local key=$1 + if [ ! -f "$root_env" ]; then + echo "" + return 0 + fi + # Keep everything after the first '=' (values can contain '=') + rg -m1 "^${key}=" "$root_env" 2>/dev/null | sed -E "s/^${key}=//" || true + } + + # Append KEY=VALUE to dest if KEY is not already present. + ensure_env_key() { + local dest=$1 + local key=$2 + local value=$3 + if [ -z "$value" ]; then + return 0 + fi + if rg -q "^${key}=" "$dest" 2>/dev/null; then + return 0 + fi + echo "${key}=${value}" >> "$dest" + } + + # Copy and modify .env files for this instance + for app_dir in backend worker frontend; do + local src_file="$app_dir/.env" + if [ -f "$src_file" ]; then + local dest="$inst_dir/${app_dir}.env" + cp "$src_file" "$dest" + + # Ensure each instance points at its own Postgres database. + # Backend uses quoted DATABASE_URL, worker typically does not. + if [ "$app_dir" = "backend" ] || [ "$app_dir" = "worker" ]; then + sed -i.bak -E \ + -e "s|(DATABASE_URL=.*\\/)(shipsec)(\"?)$|\\1shipsec_instance_${instance}\\3|" \ + "$dest" + fi + + # Ensure secrets/internal auth keys exist for dev processes. These live in repo root `.env`. + # Keep instance env self-contained so backend/worker don't need to load root `.env` (which + # contains a default DATABASE_URL and would break isolation). + if [ "$app_dir" = "backend" ] || [ "$app_dir" = "worker" ]; then + ensure_env_key "$dest" "INTERNAL_SERVICE_TOKEN" "$(get_root_env_value INTERNAL_SERVICE_TOKEN)" + ensure_env_key "$dest" "SECRET_STORE_MASTER_KEY" "$(get_root_env_value SECRET_STORE_MASTER_KEY)" + fi + + rm -f "$dest.bak" + log_success "Created $dest" + fi + done +} + +get_docker_compose_project_name() { + local instance=$1 + echo "shipsec-dev-$instance" +} + +validate_instance_setup() { + local instance=$1 + local inst_dir=$(get_instance_dir "$instance") + + # Check that all required env files exist + for env_file in backend worker frontend; do + if [ ! -f "$inst_dir/${env_file}.env" ]; then + log_error "Missing $inst_dir/${env_file}.env" + return 1 + fi + done + + log_success "Instance $instance configuration validated" + return 0 +} + +show_instance_info() { + local instance=$1 + + echo "" + echo -e "${BLUE}=== Instance $instance ===${NC}" + echo "Directory: $(get_instance_dir "$instance")" + echo "" + echo "Ports:" + echo " Frontend: http://localhost:$(get_port FRONTEND $instance)" + echo " Backend: http://localhost:$(get_port BACKEND $instance)" + echo " Temporal UI: http://localhost:8081" + echo "" + echo "Database: postgresql://shipsec:shipsec@localhost:5433/shipsec_instance_$instance" + echo "MinIO API: http://localhost:9000" + echo "MinIO UI: http://localhost:9001" + echo "Redis: redis://localhost:6379" + echo "" +} + +initialize_instance() { + local instance=$1 + + log_info "Initializing instance $instance..." + ensure_instance_dir "$instance" + copy_env_files "$instance" + + if validate_instance_setup "$instance"; then + show_instance_info "$instance" + log_success "Instance $instance initialized successfully" + return 0 + else + log_error "Instance $instance initialization failed" + return 1 + fi +} + +# Main command handler +main() { + local command=${1:-help} + local instance=${2:-0} + + case "$command" in + init) + initialize_instance "$instance" + ;; + info) + show_instance_info "$instance" + ;; + ports) + echo "FRONTEND=$(get_port FRONTEND $instance)" + echo "BACKEND=$(get_port BACKEND $instance)" + ;; + project-name) + get_docker_compose_project_name "$instance" + ;; + *) + log_error "Unknown command: $command" + echo "Usage: $0 {init|info|ports|project-name} [instance]" + exit 1 + ;; + esac +} + +main "$@" diff --git a/scripts/instance-bootstrap.sh b/scripts/instance-bootstrap.sh new file mode 100755 index 00000000..845c88ce --- /dev/null +++ b/scripts/instance-bootstrap.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Bootstrap shared infra resources for a specific instance. +# - Ensure instance DB exists +# - Run migrations against that DB +# - Ensure Temporal namespace exists +# - Ensure Kafka topics exist (best-effort) +# +# Usage: ./scripts/instance-bootstrap.sh [instance_number] + +set -euo pipefail + +INSTANCE="${1:-0}" +INFRA_PROJECT_NAME="shipsec-infra" +DB_NAME="shipsec_instance_${INSTANCE}" +NAMESPACE="shipsec-dev-${INSTANCE}" +TEMPORAL_ADDRESS="127.0.0.1:7233" + +BLUE='\033[0;34m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' + +log_info() { echo -e "${BLUE}ℹ${NC} $*"; } +log_success() { echo -e "${GREEN}✅${NC} $*"; } +log_warn() { echo -e "${YELLOW}⚠️${NC} $*"; } +log_error() { echo -e "${RED}❌${NC} $*"; } + +POSTGRES_CONTAINER="$( + docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" ps -q postgres 2>/dev/null || true +)" + +if [ -z "$POSTGRES_CONTAINER" ]; then + log_error "Postgres container not found (infra project: $INFRA_PROJECT_NAME). Is infra running?" + exit 1 +fi + +log_info "Ensuring database exists: $DB_NAME" +docker exec "$POSTGRES_CONTAINER" \ + psql -v ON_ERROR_STOP=1 -U shipsec -d postgres </dev/null 2>&1; then + log_success "Migrations completed" +else + log_error "Migrations failed" + exit 1 +fi + +if ! command -v temporal >/dev/null 2>&1; then + log_info "temporal CLI not found; skipping Temporal namespace bootstrap" +else + log_info "Ensuring Temporal namespace exists: $NAMESPACE" + # Temporal can take a few seconds to accept CLI requests after the container is "Started". + for _ in {1..30}; do + if temporal operator namespace list --address "$TEMPORAL_ADDRESS" >/dev/null 2>&1; then + break + fi + sleep 1 + done + + if temporal operator namespace describe --address "$TEMPORAL_ADDRESS" --namespace "$NAMESPACE" >/dev/null 2>&1; then + log_success "Temporal namespace exists" + else + # Create is not idempotent; it errors if the namespace already exists. Treat that as success. + if temporal operator namespace create --address "$TEMPORAL_ADDRESS" --namespace "$NAMESPACE" --retention 72h >/dev/null 2>&1; then + log_success "Temporal namespace created" + elif temporal operator namespace describe --address "$TEMPORAL_ADDRESS" --namespace "$NAMESPACE" >/dev/null 2>&1; then + log_success "Temporal namespace exists" + else + log_warn "Unable to ensure Temporal namespace (will likely break worker); continuing anyway" + fi + fi +fi + +# Best-effort Kafka topic creation in shared Redpanda. +REDPANDA_CONTAINER="$( + docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" ps -q redpanda 2>/dev/null || true +)" +if [ -n "$REDPANDA_CONTAINER" ]; then + log_info "Ensuring Kafka topics exist for instance $INSTANCE (best-effort)..." + for base in telemetry.logs telemetry.events telemetry.agent-trace telemetry.node-io; do + topic="${base}.instance-${INSTANCE}" + docker exec "$REDPANDA_CONTAINER" rpk topic create "$topic" --brokers redpanda:9092 >/dev/null 2>&1 || true + done + log_success "Kafka topics ensured" +fi diff --git a/scripts/instance-clean.sh b/scripts/instance-clean.sh new file mode 100755 index 00000000..927353cd --- /dev/null +++ b/scripts/instance-clean.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Clean shared infra resources for a specific instance. +# - Drop/recreate instance DB and re-run migrations (reset) +# - Delete Temporal namespace (best-effort) +# - Delete instance-scoped Kafka topics (best-effort) +# +# Usage: ./scripts/instance-clean.sh [instance_number] + +set -euo pipefail + +INSTANCE="${1:-0}" +INFRA_PROJECT_NAME="shipsec-infra" +DB_NAME="shipsec_instance_${INSTANCE}" +NAMESPACE="shipsec-dev-${INSTANCE}" +TEMPORAL_ADDRESS="127.0.0.1:7233" + +BLUE='\033[0;34m' +GREEN='\033[0;32m' +RED='\033[0;31m' +NC='\033[0m' + +log_info() { echo -e "${BLUE}ℹ${NC} $*"; } +log_success() { echo -e "${GREEN}✅${NC} $*"; } +log_error() { echo -e "${RED}❌${NC} $*"; } + +POSTGRES_CONTAINER="$( + docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" ps -q postgres 2>/dev/null || true +)" + +if [ -z "$POSTGRES_CONTAINER" ]; then + log_error "Postgres container not found (infra project: $INFRA_PROJECT_NAME). Is infra running?" + exit 1 +fi + +log_info "Resetting database for instance $INSTANCE..." +if ! ./scripts/db-reset-instance.sh "$INSTANCE" >/dev/null; then + log_error "Failed to reset database for instance $INSTANCE" + exit 1 +fi +log_success "Database reset complete" + +if command -v temporal >/dev/null 2>&1; then + log_info "Deleting Temporal namespace (best-effort): $NAMESPACE" + temporal operator namespace delete --address "$TEMPORAL_ADDRESS" --namespace "$NAMESPACE" --yes >/dev/null 2>&1 || true +fi + +REDPANDA_CONTAINER="$( + docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" ps -q redpanda 2>/dev/null || true +)" +if [ -n "$REDPANDA_CONTAINER" ]; then + log_info "Deleting Kafka topics for instance $INSTANCE (best-effort)..." + for base in telemetry.logs telemetry.events telemetry.agent-trace telemetry.node-io; do + topic="${base}.instance-${INSTANCE}" + docker exec "$REDPANDA_CONTAINER" rpk topic delete "$topic" --brokers redpanda:9092 >/dev/null 2>&1 || true + done +fi + +log_success "Instance $INSTANCE infra state cleaned" diff --git a/worker/src/common/kafka-topic-resolver.ts b/worker/src/common/kafka-topic-resolver.ts new file mode 100644 index 00000000..203ecdf6 --- /dev/null +++ b/worker/src/common/kafka-topic-resolver.ts @@ -0,0 +1,107 @@ +/** + * Kafka Topic Resolver + * + * Provides instance-aware topic naming for multi-instance deployments. + * When SHIPSEC_INSTANCE is set, topics are namespaced with the instance number. + * + * Environment Variables: + * - SHIPSEC_INSTANCE: Instance number (0-9) for multi-instance isolation + * - LOG_KAFKA_TOPIC: Base topic for logs (default: telemetry.logs) + * - EVENT_KAFKA_TOPIC: Base topic for events (default: telemetry.events) + * - AGENT_TRACE_KAFKA_TOPIC: Base topic for agent traces (default: telemetry.agent-trace) + * - NODE_IO_KAFKA_TOPIC: Base topic for node I/O (default: telemetry.node-io) + */ + +export interface TopicResolverConfig { + instanceId?: string; + enableInstanceSuffix?: boolean; +} + +export class KafkaTopicResolver { + private instanceId: string | undefined; + private enableInstanceSuffix: boolean; + + constructor(config: TopicResolverConfig = {}) { + this.instanceId = config.instanceId ?? process.env.SHIPSEC_INSTANCE; + // Enable instance suffix only if SHIPSEC_INSTANCE is set + this.enableInstanceSuffix = config.enableInstanceSuffix ?? Boolean(this.instanceId); + } + + /** + * Resolve topic name with instance suffix if applicable + * @param baseTopic The base topic name + * @returns The topic name with instance suffix (if enabled) + */ + resolveTopic(baseTopic: string): string { + if (!this.enableInstanceSuffix || !this.instanceId) { + return baseTopic; + } + return `${baseTopic}.instance-${this.instanceId}`; + } + + /** + * Get logs topic + */ + getLogsTopic(): string { + const baseTopic = process.env.LOG_KAFKA_TOPIC ?? 'telemetry.logs'; + return this.resolveTopic(baseTopic); + } + + /** + * Get events topic + */ + getEventsTopic(): string { + const baseTopic = process.env.EVENT_KAFKA_TOPIC ?? 'telemetry.events'; + return this.resolveTopic(baseTopic); + } + + /** + * Get agent trace topic + */ + getAgentTraceTopic(): string { + const baseTopic = process.env.AGENT_TRACE_KAFKA_TOPIC ?? 'telemetry.agent-trace'; + return this.resolveTopic(baseTopic); + } + + /** + * Get node I/O topic + */ + getNodeIOTopic(): string { + const baseTopic = process.env.NODE_IO_KAFKA_TOPIC ?? 'telemetry.node-io'; + return this.resolveTopic(baseTopic); + } + + /** + * Check if instance isolation is enabled + */ + isInstanceIsolated(): boolean { + return this.enableInstanceSuffix; + } + + /** + * Get instance ID (if set) + */ + getInstanceId(): string | undefined { + return this.instanceId; + } +} + +// Singleton instance +let resolver: KafkaTopicResolver; + +/** + * Get or create the singleton topic resolver + */ +export function getTopicResolver(config?: TopicResolverConfig): KafkaTopicResolver { + if (!resolver) { + resolver = new KafkaTopicResolver(config); + } + return resolver; +} + +/** + * Reset the singleton (useful for testing) + */ +export function resetTopicResolver(): void { + resolver = undefined!; +} diff --git a/worker/src/temporal/workers/dev.worker.ts b/worker/src/temporal/workers/dev.worker.ts index 4bfd5571..7b75bbe5 100644 --- a/worker/src/temporal/workers/dev.worker.ts +++ b/worker/src/temporal/workers/dev.worker.ts @@ -53,11 +53,19 @@ import { KafkaNodeIOAdapter, } from '../../adapters'; import { ConfigurationError } from '@shipsec/component-sdk'; +import { getTopicResolver } from '../../common/kafka-topic-resolver'; import * as schema from '../../adapters/schema'; import { logHeartbeat } from '../../utils/debug-logger'; -// Load environment variables from .env file -config({ path: join(dirname(fileURLToPath(import.meta.url)), '../../..', '.env') }); +// Load environment variables from instance-specific env if set, otherwise fall back +// to the worker's default `.env`. +const workerRoot = join(dirname(fileURLToPath(import.meta.url)), '../../..'); +const instanceNum = process.env.SHIPSEC_INSTANCE; +const instanceEnvPath = instanceNum + ? join(workerRoot, '..', '.instances', `instance-${instanceNum}`, 'worker.env') + : undefined; + +config({ path: instanceEnvPath ?? join(workerRoot, '.env') }); if (typeof globalThis.crypto === 'undefined') { Object.defineProperty(globalThis, 'crypto', { @@ -140,22 +148,28 @@ async function main() { }); } + // Get instance-aware topic names + const topicResolver = getTopicResolver(); + const instanceMsg = topicResolver.isInstanceIsolated() + ? ` (instance ${topicResolver.getInstanceId()})` + : ''; + const traceAdapter = new KafkaTraceAdapter({ brokers: kafkaBrokers, - topic: process.env.EVENT_KAFKA_TOPIC ?? 'telemetry.events', + topic: topicResolver.getEventsTopic(), clientId: process.env.EVENT_KAFKA_CLIENT_ID ?? 'shipsec-worker-events', }); const agentTracePublisher = new KafkaAgentTracePublisher({ brokers: kafkaBrokers, - topic: process.env.AGENT_TRACE_KAFKA_TOPIC ?? 'telemetry.agent-trace', + topic: topicResolver.getAgentTraceTopic(), clientId: process.env.AGENT_TRACE_KAFKA_CLIENT_ID ?? 'shipsec-worker-agent-trace', }); const nodeIOAdapter = new KafkaNodeIOAdapter( { brokers: kafkaBrokers, - topic: process.env.NODE_IO_KAFKA_TOPIC ?? 'telemetry.node-io', + topic: topicResolver.getNodeIOTopic(), clientId: process.env.NODE_IO_KAFKA_CLIENT_ID ?? 'shipsec-worker-node-io', }, storageAdapter, @@ -165,10 +179,10 @@ async function main() { try { logAdapter = new KafkaLogAdapter({ brokers: kafkaBrokers, - topic: process.env.LOG_KAFKA_TOPIC ?? 'telemetry.logs', + topic: topicResolver.getLogsTopic(), clientId: process.env.LOG_KAFKA_CLIENT_ID ?? 'shipsec-worker', }); - console.log(`✅ Kafka logging enabled (${kafkaBrokers.join(', ')})`); + console.log(`✅ Kafka logging enabled (${kafkaBrokers.join(', ')})${instanceMsg}`); } catch (error) { console.error('❌ Failed to initialize Kafka logging', error); throw error; diff --git a/worker/tsconfig.json b/worker/tsconfig.json index 13535ca7..51090687 100644 --- a/worker/tsconfig.json +++ b/worker/tsconfig.json @@ -31,6 +31,9 @@ { "path": "../packages/component-sdk" }, + { + "path": "../packages/backend-client" + }, { "path": "../packages/contracts" }