feat: Multi-instance development stack with full isolation

.gitignore

@@ -14,11 +14,13 @@ build/
 .env
 .env.local
 docker/.env
+.instances/
 .env.development.local
 .env.test.local
 .env.production.local
 .env.eng-104
 .env.eng-104
+.shipsec-instance
 
 # Logs
 logs/

AGENTS.md

@@ -3,44 +3,95 @@
 Security workflow orchestration platform. Visual builder + Temporal for reliability.
 
 ## Stack
+
 - `frontend/` — React + Vite
-- `backend/` — NestJS API  
+- `backend/` — NestJS API
 - `worker/` — Temporal activities + components
 - `packages/` — Shared code (component-sdk, backend-client)
 
 ## Development
 
 ```bash
 just init              # First time setup
-just dev               # Start everything
-just dev stop          # Stop
-just dev logs          # View logs
+just dev               # Start the active instance (default: 0)
+just dev stop          # Stop the active instance (does NOT stop shared infra)
+just dev stop all      # Stop all instances + shared infra
+just dev logs          # View logs for the active instance
 just help              # All commands
 ```
 
-**URLs**: Frontend http://localhost:5173 | Backend http://localhost:3211 | Temporal http://localhost:8081
+**Active instance**:
+
+```bash
+just instance show     # Print active instance number
+just instance use 5    # Set active instance for this workspace
+```
+
+**URLs**:
+
+- Frontend: `http://localhost:${5173 + instance*100}`
+- Backend: `http://localhost:${3211 + instance*100}`
+- Temporal UI (shared): http://localhost:8081
+
+Full details: `docs/MULTI-INSTANCE-DEV.md`
+
+### Multi-Instance Local Dev (Important)
+
+Local development runs as **multiple app instances** (PM2) on top of **one shared Docker infra stack**.
+
+- Shared infra (Docker Compose project `shipsec-infra`): Postgres/Temporal/Redpanda/Redis/MinIO/Loki on fixed ports.
+- Per-instance apps: `shipsec-{frontend,backend,worker}-N`.
+- Isolation is via per-instance DB + Temporal namespace/task queue + Kafka topic suffixing (not per-instance infra containers).
+- The workspace can have an **active instance** (stored in `.shipsec-instance`, gitignored).
+
+**Agent rule:** before running any dev commands, ensure you’re targeting the intended instance.
+
+- Always check: `just instance show`
+- If the task is ambiguous (logs, curl, E2E, “run locally”, etc.), ask the user which instance to use.
+- If the user says “use instance N”, prefer either:
+  - `just instance use N` then run `just dev` / `bun run test:e2e`, or
+  - explicit instance invocation (`just dev N ...`) for one-off commands.
+
+**Ports / URLs**
+
+- Frontend: `5173 + N*100`
+- Backend: `3211 + N*100`
+- Temporal UI (shared): http://localhost:8081
+
+**E2E tests**
+
+- E2E targets the backend for `SHIPSEC_INSTANCE` (or the active instance).
+- When asked to run E2E, confirm the instance and ensure that instance is running: `just dev N start`.
+
+**Keep docs in sync**
+
+If you change instance/infra behavior (justfile/scripts/pm2 config), update `docs/MULTI-INSTANCE-DEV.md` and this section accordingly in the same PR.
 
 ### After Backend Route Changes
+
 ```bash
 bun --cwd backend run generate:openapi
 bun --cwd packages/backend-client run generate
 ```
 
 ### Testing
+
 ```bash
 bun run test           # All tests
 bun run typecheck      # Type check
 bun run lint           # Lint
 ```
 
 ### Database
+
 ```bash
 just db-reset                              # Reset database
 bun --cwd backend run migration:push       # Push schema
 bun --cwd backend run db:studio            # View data
 ```
 
 ## Rules
+
 1. TypeScript, 2-space indent
 2. Conventional commits with DCO: `git commit -s -m "feat: ..."`
 3. Tests alongside code in `__tests__/` folders
@@ -64,11 +115,13 @@ Frontend ←→ Backend ←→ Temporal ←→ Worker
 ```
 
 ### Component Runners
+
 - **inline** — TypeScript code (HTTP calls, transforms, file ops)
-- **docker** — Containers (security tools: Subfinder, DNSX, Nuclei)  
+- **docker** — Containers (security tools: Subfinder, DNSX, Nuclei)
 - **remote** — External executors (future: K8s, ECS)
 
 ### Real-time Streaming
+
 - Terminal: Redis Streams → SSE → xterm.js
 - Events: Kafka → WebSocket
 - Logs: Loki + PostgreSQL
@@ -83,9 +136,9 @@ When tasks match a skill, load it: `cat .claude/skills/<name>/SKILL.md`
 
 <available_skills>
 <skill>
-  <name>component-development</name>
-  <description>Creating components (inline/docker). Dynamic ports, retry policies, PTY patterns, IsolatedContainerVolume.</description>
-  <location>project</location>
+<name>component-development</name>
+<description>Creating components (inline/docker). Dynamic ports, retry policies, PTY patterns, IsolatedContainerVolume.</description>
+<location>project</location>
 </skill>
 </available_skills>
 

backend/.env.example

@@ -55,4 +55,4 @@ OPENSEARCH_INDEX_PREFIX="logs-tenant"
 SECRET_STORE_MASTER_KEY="CHANGE_ME_32_CHAR_SECRET_KEY!!!!"
 
 # Kafka / Redpanda configuration for node I/O, log, and event ingestion
-LOG_KAFKA_BROKERS="localhost:9092"
+LOG_KAFKA_BROKERS="localhost:19092"

backend/package.json

@@ -26,6 +26,7 @@
     "@nestjs/microservices": "^11.1.13",
     "@nestjs/platform-express": "^10.4.22",
     "@nestjs/swagger": "^11.2.5",
+    "@shipsec/backend-client": "workspace:*",
     "@shipsec/component-sdk": "workspace:*",
     "@shipsec/shared": "workspace:*",
     "@shipsec/studio-worker": "workspace:*",

backend/src/agent-trace/agent-trace-ingest.service.ts

@@ -1,5 +1,6 @@
 import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
 import { Consumer, Kafka } from 'kafkajs';
+import { getTopicResolver } from '../common/kafka-topic-resolver';
 
 import { AgentTraceRepository, type AgentTraceEventInput } from './agent-trace.repository';
 
@@ -22,7 +23,10 @@ export class AgentTraceIngestService implements OnModuleInit, OnModuleDestroy {
       throw new Error('LOG_KAFKA_BROKERS must be configured for agent trace ingestion');
     }
 
-    this.kafkaTopic = process.env.AGENT_TRACE_KAFKA_TOPIC ?? 'telemetry.agent-trace';
+    // Use instance-aware topic name
+    const topicResolver = getTopicResolver();
+    this.kafkaTopic = topicResolver.getAgentTraceTopic();
+
     this.kafkaGroupId = process.env.AGENT_TRACE_KAFKA_GROUP_ID ?? 'shipsec-agent-trace-ingestor';
     this.kafkaClientId = process.env.AGENT_TRACE_KAFKA_CLIENT_ID ?? 'shipsec-backend-agent-trace';
   }

backend/src/app.module.ts

@@ -1,6 +1,7 @@
 import { Module } from '@nestjs/common';
 import { APP_GUARD } from '@nestjs/core';
 import { ConfigModule } from '@nestjs/config';
+import { join } from 'node:path';
 
 import { AppController } from './app.controller';
 import { AppService } from './app.service';
@@ -47,11 +48,26 @@ const coreModules = [
 
 const testingModules = process.env.NODE_ENV === 'production' ? [] : [TestingSupportModule];
 
+function getEnvFilePaths(): string[] {
+  // In multi-instance dev, each instance has its own env file under:
+  //   .instances/instance-N/backend.env
+  // Backends run with cwd=backend/, so repo root is `..`.
+  const instance = process.env.SHIPSEC_INSTANCE;
+  if (instance) {
+    // Use only the instance env file. In multi-instance dev the workspace `.env` contains
+    // a default DATABASE_URL, and dotenv does not override already-set env vars; mixing
+    // would collapse isolation.
+    return [join(process.cwd(), '..', '.instances', `instance-${instance}`, 'backend.env')];
+  }
+
+  return ['.env', '../.env'];
+}
+
 @Module({
   imports: [
     ConfigModule.forRoot({
       isGlobal: true,
-      envFilePath: ['.env', '../.env'],
+      envFilePath: getEnvFilePaths(),
       load: [authConfig],
     }),
     ...coreModules,

backend/src/common/kafka-topic-resolver.ts

@@ -0,0 +1,107 @@
+/**
+ * Kafka Topic Resolver
+ *
+ * Provides instance-aware topic naming for multi-instance deployments.
+ * When SHIPSEC_INSTANCE is set, topics are namespaced with the instance number.
+ *
+ * Environment Variables:
+ * - SHIPSEC_INSTANCE: Instance number (0-9) for multi-instance isolation
+ * - LOG_KAFKA_TOPIC: Base topic for logs (default: telemetry.logs)
+ * - EVENT_KAFKA_TOPIC: Base topic for events (default: telemetry.events)
+ * - AGENT_TRACE_KAFKA_TOPIC: Base topic for agent traces (default: telemetry.agent-trace)
+ * - NODE_IO_KAFKA_TOPIC: Base topic for node I/O (default: telemetry.node-io)
+ */
+
+export interface TopicResolverConfig {
+  instanceId?: string;
+  enableInstanceSuffix?: boolean;
+}
+
+export class KafkaTopicResolver {
+  private instanceId: string | undefined;
+  private enableInstanceSuffix: boolean;
+
+  constructor(config: TopicResolverConfig = {}) {
+    this.instanceId = config.instanceId ?? process.env.SHIPSEC_INSTANCE;
+    // Enable instance suffix only if SHIPSEC_INSTANCE is set
+    this.enableInstanceSuffix = config.enableInstanceSuffix ?? Boolean(this.instanceId);
+  }
+
+  /**
+   * Resolve topic name with instance suffix if applicable
+   * @param baseTopic The base topic name
+   * @returns The topic name with instance suffix (if enabled)
+   */
+  resolveTopic(baseTopic: string): string {
+    if (!this.enableInstanceSuffix || !this.instanceId) {
+      return baseTopic;
+    }
+    return `${baseTopic}.instance-${this.instanceId}`;
+  }
+
+  /**
+   * Get logs topic
+   */
+  getLogsTopic(): string {
+    const baseTopic = process.env.LOG_KAFKA_TOPIC ?? 'telemetry.logs';
+    return this.resolveTopic(baseTopic);
+  }
+
+  /**
+   * Get events topic
+   */
+  getEventsTopic(): string {
+    const baseTopic = process.env.EVENT_KAFKA_TOPIC ?? 'telemetry.events';
+    return this.resolveTopic(baseTopic);
+  }
+
+  /**
+   * Get agent trace topic
+   */
+  getAgentTraceTopic(): string {
+    const baseTopic = process.env.AGENT_TRACE_KAFKA_TOPIC ?? 'telemetry.agent-trace';
+    return this.resolveTopic(baseTopic);
+  }
+
+  /**
+   * Get node I/O topic
+   */
+  getNodeIOTopic(): string {
+    const baseTopic = process.env.NODE_IO_KAFKA_TOPIC ?? 'telemetry.node-io';
+    return this.resolveTopic(baseTopic);
+  }
+
+  /**
+   * Check if instance isolation is enabled
+   */
+  isInstanceIsolated(): boolean {
+    return this.enableInstanceSuffix;
+  }
+
+  /**
+   * Get instance ID (if set)
+   */
+  getInstanceId(): string | undefined {
+    return this.instanceId;
+  }
+}
+
+// Singleton instance
+let resolver: KafkaTopicResolver;
+
+/**
+ * Get or create the singleton topic resolver
+ */
+export function getTopicResolver(config?: TopicResolverConfig): KafkaTopicResolver {
+  if (!resolver) {
+    resolver = new KafkaTopicResolver(config);
+  }
+  return resolver;
+}
+
+/**
+ * Reset the singleton (useful for testing)
+ */
+export function resetTopicResolver(): void {
+  resolver = undefined!;
+}

backend/src/events/event-ingest.service.ts

@@ -1,5 +1,6 @@
 import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
 import { Consumer, Kafka } from 'kafkajs';
+import { getTopicResolver } from '../common/kafka-topic-resolver';
 
 import { TraceRepository, type PersistedTraceEvent } from '../trace/trace.repository';
 import type { TraceEventType } from '../trace/types';
@@ -38,7 +39,10 @@ export class EventIngestService implements OnModuleInit, OnModuleDestroy {
       throw new Error('LOG_KAFKA_BROKERS must be configured for event ingestion');
     }
 
-    this.kafkaTopic = process.env.EVENT_KAFKA_TOPIC ?? 'telemetry.events';
+    // Use instance-aware topic name
+    const topicResolver = getTopicResolver();
+    this.kafkaTopic = topicResolver.getEventsTopic();
+
     this.kafkaGroupId = process.env.EVENT_KAFKA_GROUP_ID ?? 'shipsec-event-ingestor';
     this.kafkaClientId = process.env.EVENT_KAFKA_CLIENT_ID ?? 'shipsec-backend-events';
   }

backend/src/logging/log-ingest.service.ts

@@ -1,5 +1,6 @@
 import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
 import { Kafka, Consumer } from 'kafkajs';
+import { getTopicResolver } from '../common/kafka-topic-resolver';
 
 import { LogStreamRepository } from '../trace/log-stream.repository';
 import type { KafkaLogEntry } from './log-entry.types';
@@ -24,7 +25,11 @@ export class LogIngestService implements OnModuleInit, OnModuleDestroy {
     if (this.kafkaBrokers.length === 0) {
       throw new Error('LOG_KAFKA_BROKERS must be configured for Kafka log ingestion');
     }
-    this.kafkaTopic = process.env.LOG_KAFKA_TOPIC ?? 'telemetry.logs';
+
+    // Use instance-aware topic name
+    const topicResolver = getTopicResolver();
+    this.kafkaTopic = topicResolver.getLogsTopic();
+
     this.kafkaGroupId = process.env.LOG_KAFKA_GROUP_ID ?? 'shipsec-log-ingestor';
     this.kafkaClientId = process.env.LOG_KAFKA_CLIENT_ID ?? 'shipsec-backend';
 

backend/src/main.ts

@@ -26,14 +26,23 @@ async function bootstrap() {
   }
 
   // Enable CORS for frontend
+  // Build dynamic origin list for multi-instance dev (instances 0-9)
+  const instanceOrigins: string[] = [];
+  for (let i = 0; i <= 9; i++) {
+    const frontendPort = 5173 + i * 100;
+    const backendPort = 3211 + i * 100;
+    instanceOrigins.push(`http://localhost:${frontendPort}`);
+    instanceOrigins.push(`http://127.0.0.1:${frontendPort}`);
+    instanceOrigins.push(`http://localhost:${backendPort}`);
+    instanceOrigins.push(`http://127.0.0.1:${backendPort}`);
+  }
+
   app.enableCors({
     origin: [
       'http://localhost',
-      'http://localhost:5173',
-      'http://localhost:5174',
-      'http://localhost:3211',
       'http://localhost:8090',
       'https://studio.shipsec.ai',
+      ...instanceOrigins,
     ],
     credentials: true,
     methods: ['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'],