diff --git a/.ai/analytics-output-port-design.md b/.ai/analytics-output-port-design.md new file mode 100644 index 00000000..41ce63cc --- /dev/null +++ b/.ai/analytics-output-port-design.md @@ -0,0 +1,190 @@ +# Analytics Output Port Design + +## Status: Approved +## Date: 2025-01-21 + +## Problem Statement + +When connecting a component's `rawOutput` (which contains complex nested JSON) to the Analytics Sink, OpenSearch hits the default field limit of 1000 fields. This is because: + +1. **Dynamic mapping explosion**: Elasticsearch/OpenSearch creates a field for every unique JSON path +2. **Nested structures**: Arrays with objects like `issues[0].metadata.schema` create many paths +3. **Varying schemas**: Different scanner outputs accumulate unique field paths over time + +Example error: +``` +illegal_argument_exception: Limit of total fields [1000] has been exceeded +``` + +## Solution + +### Design Decisions + +1. **Each component owns its analytics schema** + - Components output structured `list` through dedicated ports (`findings`, `results`, `secrets`, `issues`) + - Component authors define the structure appropriate for their tool + - No generic "one schema fits all" approach + +2. **Analytics Sink accepts `list`** + - Input type: `z.array(z.record(z.string(), z.unknown()))` + - Each item in the array is indexed as a separate document + - Rejects arbitrary nested objects (must be an array) + +3. **Same timestamp for all findings in a batch** + - All findings from one component execution share the same `@timestamp` + - Captured once at the start of indexing, applied to all documents + +4. **Nested `shipsec` context** + - Workflow context stored under `shipsec.*` namespace + - Prevents field name collision with component data + - Clear separation: component fields at root, system fields under `shipsec` + +5. **Nested objects serialized before indexing** + - Any nested object or array within a finding is JSON-stringified + - Prevents field explosion from dynamic mapping + - Trade-off: Can't query inside serialized fields directly, but prevents index corruption + +6. **No `data` wrapper** + - Original PRD design wrapped component output in a `data` field + - New design: finding fields are at the top level for easier querying + +### Document Structure + +**Before (PRD design):** +```json +{ + "workflow_id": "...", + "workflow_name": "...", + "run_id": "...", + "node_ref": "...", + "component_id": "...", + "@timestamp": "...", + "asset_key": "...", + "data": { + "check_id": "DB_RLS_DISABLED", + "severity": "CRITICAL", + "metadata": { "schema": "public", "table": "users" } + } +} +``` + +**After (new design):** +```json +{ + "check_id": "DB_RLS_DISABLED", + "severity": "CRITICAL", + "title": "RLS Disabled on Table: users", + "resource": "public.users", + "metadata": "{\"schema\":\"public\",\"table\":\"users\"}", + "scanner": "supabase-scanner", + "asset_key": "abcdefghij1234567890", + "finding_hash": "a1b2c3d4e5f67890", + + "shipsec": { + "organization_id": "org_123", + "run_id": "shipsec-run-xxx", + "workflow_id": "d1d33161-929f-4af4-9a64-xxx", + "workflow_name": "Supabase Security Audit", + "component_id": "core.analytics.sink", + "node_ref": "analytics-sink-1" + }, + + "@timestamp": "2025-01-21T10:30:00.000Z" +} +``` + +### Component Output Ports + +Components should use their existing structured list outputs: + +| Component | Port | Type | Notes | +|-----------|------|------|-------| +| Nuclei | `results` | `z.array(z.record(z.string(), z.unknown()))` | Scanner + asset_key added | +| TruffleHog | `results` | `z.array(z.record(z.string(), z.unknown()))` | Scanner + asset_key added | +| Supabase Scanner | `results` | `z.array(z.record(z.string(), z.unknown()))` | Scanner + asset_key added | + +All `results` ports include: +- `scanner`: Scanner identifier (e.g., `'nuclei'`, `'trufflehog'`, `'supabase-scanner'`) +- `asset_key`: Primary asset identifier from the finding +- `finding_hash`: Stable hash for deduplication (16-char hex from SHA-256) + +### Finding Hash for Deduplication + +The `finding_hash` enables tracking findings across workflow runs: + +**Generation:** +```typescript +import { createHash } from 'crypto'; + +function generateFindingHash(...fields: (string | undefined | null)[]): string { + const normalized = fields.map((f) => (f ?? '').toLowerCase().trim()).join('|'); + return createHash('sha256').update(normalized).digest('hex').slice(0, 16); +} +``` + +**Key fields per scanner:** +| Scanner | Hash Fields | +|---------|-------------| +| Nuclei | `templateId + host + matchedAt` | +| TruffleHog | `DetectorType + Redacted + filePath` | +| Supabase Scanner | `check_id + projectRef + resource` | + +**Use cases:** +- **New vs recurring**: Is this finding appearing for the first time? +- **First-seen / last-seen**: When did we first detect this? Is it still present? +- **Resolution tracking**: Findings that stop appearing may be resolved +- **Deduplication**: Remove duplicates in dashboards across runs + +### `shipsec` Context Fields + +The indexer automatically adds these fields under `shipsec`: + +| Field | Description | +|-------|-------------| +| `organization_id` | Organization that owns the workflow | +| `run_id` | Unique identifier for this workflow execution | +| `workflow_id` | ID of the workflow definition | +| `workflow_name` | Human-readable workflow name | +| `component_id` | Component type (e.g., `core.analytics.sink`) | +| `node_ref` | Node reference in the workflow graph | +| `asset_key` | Auto-detected or specified asset identifier | + +### Querying in OpenSearch + +With this structure, users can: +- Filter by organization: `shipsec.organization_id: "org_123"` +- Filter by workflow: `shipsec.workflow_id: "xxx"` +- Filter by run: `shipsec.run_id: "xxx"` +- Filter by asset: `asset_key: "api.example.com"` +- Filter by scanner: `scanner: "nuclei"` +- Filter by component-specific fields: `severity: "CRITICAL"` +- Aggregate by severity: `terms` aggregation on `severity` field +- Track finding history: `finding_hash: "a1b2c3d4" | sort @timestamp` +- Find recurring findings: Group by `finding_hash`, count occurrences + +### Trade-offs + +| Decision | Pro | Con | +|----------|-----|-----| +| Serialize nested objects | Prevents field explosion | Can't query inside serialized fields | +| `shipsec` namespace | No field collision | Slightly more verbose queries | +| No generic schema | Better fit per component | Less consistency across components | +| Same timestamp per batch | Accurate (same scan time) | Can't distinguish individual finding times | + +### Implementation Files + +1. `/worker/src/utils/opensearch-indexer.ts` - Add `shipsec` context, serialize nested objects +2. `/worker/src/components/core/analytics-sink.ts` - Accept `list`, consistent timestamp +3. Component files - Ensure structured output, add `results` port where missing + +### Backward Compatibility + +- Existing workflows connecting `rawOutput` to Analytics Sink will still work +- Analytics Sink continues to accept any data type for backward compatibility +- New `list` processing only triggers when input is an array + +### Future Considerations + +1. **Index templates**: Create OpenSearch index template with explicit mappings for `shipsec.*` fields +2. **Field discovery**: Build UI to show available fields from indexed data +3. **Schema validation**: Optional strict mode to validate findings against expected schema diff --git a/Dockerfile b/Dockerfile index ecbca77a..c2f28972 100644 --- a/Dockerfile +++ b/Dockerfile @@ -89,6 +89,7 @@ ARG VITE_DEFAULT_ORG_ID=local-dev ARG VITE_GIT_SHA=unknown ARG VITE_PUBLIC_POSTHOG_KEY="" ARG VITE_PUBLIC_POSTHOG_HOST="" +ARG VITE_OPENSEARCH_DASHBOARDS_URL="" ENV VITE_AUTH_PROVIDER=${VITE_AUTH_PROVIDER} ENV VITE_CLERK_PUBLISHABLE_KEY=${VITE_CLERK_PUBLISHABLE_KEY} @@ -98,6 +99,7 @@ ENV VITE_DEFAULT_ORG_ID=${VITE_DEFAULT_ORG_ID} ENV VITE_GIT_SHA=${VITE_GIT_SHA} ENV VITE_PUBLIC_POSTHOG_KEY=${VITE_PUBLIC_POSTHOG_KEY} ENV VITE_PUBLIC_POSTHOG_HOST=${VITE_PUBLIC_POSTHOG_HOST} +ENV VITE_OPENSEARCH_DASHBOARDS_URL=${VITE_OPENSEARCH_DASHBOARDS_URL} # Set working directory for frontend USER shipsec @@ -129,6 +131,7 @@ ARG VITE_DEFAULT_ORG_ID=local-dev ARG VITE_GIT_SHA=unknown ARG VITE_PUBLIC_POSTHOG_KEY="" ARG VITE_PUBLIC_POSTHOG_HOST="" +ARG VITE_OPENSEARCH_DASHBOARDS_URL="" ENV VITE_AUTH_PROVIDER=${VITE_AUTH_PROVIDER} ENV VITE_CLERK_PUBLISHABLE_KEY=${VITE_CLERK_PUBLISHABLE_KEY} @@ -138,6 +141,7 @@ ENV VITE_DEFAULT_ORG_ID=${VITE_DEFAULT_ORG_ID} ENV VITE_GIT_SHA=${VITE_GIT_SHA} ENV VITE_PUBLIC_POSTHOG_KEY=${VITE_PUBLIC_POSTHOG_KEY} ENV VITE_PUBLIC_POSTHOG_HOST=${VITE_PUBLIC_POSTHOG_HOST} +ENV VITE_OPENSEARCH_DASHBOARDS_URL=${VITE_OPENSEARCH_DASHBOARDS_URL} # Set working directory for frontend USER shipsec diff --git a/backend/.env.example b/backend/.env.example index 1964e7dc..fc68355d 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -32,6 +32,8 @@ AUTH_PROVIDER="local" # If AUTH_LOCAL_ALLOW_UNAUTHENTICATED=false, clients must present AUTH_LOCAL_API_KEY in the Authorization header. AUTH_LOCAL_ALLOW_UNAUTHENTICATED="true" AUTH_LOCAL_API_KEY="" +# Required in production for session auth cookie signing +SESSION_SECRET="" # Clerk provider options # Required when AUTH_PROVIDER="clerk" @@ -44,15 +46,24 @@ PLATFORM_SERVICE_TOKEN="" # Optional: override request timeout in milliseconds (default 5000) PLATFORM_API_TIMEOUT_MS="" -# OpenSearch configuration -OPENSEARCH_URL="http://localhost:9200" -OPENSEARCH_INDEX_PREFIX="logs-tenant" -# OPENSEARCH_USERNAME="" -# OPENSEARCH_PASSWORD="" +# OpenSearch configuration for security analytics indexing +# Optional: if not set, security analytics indexing will be disabled +OPENSEARCH_URL="" +OPENSEARCH_USERNAME="" +OPENSEARCH_PASSWORD="" + +# OpenSearch Dashboards configuration for analytics visualization +# Optional: if not set, Dashboards link will not appear in frontend sidebar +# Example: "http://localhost:5601" or "https://dashboards.example.com" +OPENSEARCH_DASHBOARDS_URL="" # Secret encryption key (must be exactly 32 characters, NOT hex-encoded) # Generate with: openssl rand -base64 24 | head -c 32 SECRET_STORE_MASTER_KEY="CHANGE_ME_32_CHAR_SECRET_KEY!!!!" +# Redis configuration for rate limiting and caching +# Optional: if not set, rate limiting will use in-memory storage (not recommended for production) +REDIS_URL="" + # Kafka / Redpanda configuration for node I/O, log, and event ingestion LOG_KAFKA_BROKERS="localhost:19092" diff --git a/backend/package.json b/backend/package.json index b4064185..676958ce 100644 --- a/backend/package.json +++ b/backend/package.json @@ -14,18 +14,22 @@ "generate:openapi": "bun scripts/generate-openapi.ts", "migration:push": "bun x drizzle-kit push", "migration:smoke": "bun scripts/migration-smoke.ts", - "delete:runs": "bun scripts/delete-all-workflow-runs.ts" + "delete:runs": "bun scripts/delete-all-workflow-runs.ts", + "setup:opensearch": "bun scripts/setup-opensearch.ts" }, "dependencies": { "@clerk/backend": "^2.29.5", "@clerk/types": "^4.101.13", "@grpc/grpc-js": "^1.14.3", + "@nest-lab/throttler-storage-redis": "^1.1.0", "@nestjs/common": "^10.4.22", "@nestjs/config": "^3.3.0", "@nestjs/core": "^10.4.22", "@nestjs/microservices": "^11.1.13", "@nestjs/platform-express": "^10.4.22", "@nestjs/swagger": "^11.2.5", + "@nestjs/throttler": "^6.5.0", + "@opensearch-project/opensearch": "^3.5.1", "@shipsec/backend-client": "workspace:*", "@shipsec/component-sdk": "workspace:*", "@shipsec/shared": "workspace:*", @@ -62,6 +66,7 @@ "@eslint/js": "^9.39.2", "@nestjs/testing": "^10.4.22", "@types/bcryptjs": "^3.0.0", + "@types/cookie-parser": "^1.4.10", "@types/express-serve-static-core": "^4.19.8", "@types/har-format": "^1.2.16", "@types/multer": "^2.0.0", diff --git a/backend/scripts/setup-opensearch.ts b/backend/scripts/setup-opensearch.ts new file mode 100644 index 00000000..bb4646e0 --- /dev/null +++ b/backend/scripts/setup-opensearch.ts @@ -0,0 +1,93 @@ +import { Client } from '@opensearch-project/opensearch'; +import { config } from 'dotenv'; + +// Load environment variables +config(); + +async function main() { + const url = process.env.OPENSEARCH_URL; + const username = process.env.OPENSEARCH_USERNAME; + const password = process.env.OPENSEARCH_PASSWORD; + + if (!url) { + console.error('āŒ OPENSEARCH_URL environment variable is required'); + process.exit(1); + } + + console.log('šŸ” Connecting to OpenSearch...'); + + const client = new Client({ + node: url, + auth: username && password ? { username, password } : undefined, + ssl: { + rejectUnauthorized: process.env.NODE_ENV === 'production', + }, + }); + + try { + // Test connection + const healthCheck = await client.cluster.health(); + console.log(`āœ… Connected to OpenSearch cluster (status: ${healthCheck.body.status})`); + + // Create index template for security-findings-* + const templateName = 'security-findings-template'; + console.log(`\nšŸ“‹ Creating index template: ${templateName}`); + + await client.indices.putIndexTemplate({ + name: templateName, + body: { + index_patterns: ['security-findings-*'], + template: { + settings: { + number_of_shards: 1, + number_of_replicas: 1, + }, + mappings: { + properties: { + '@timestamp': { type: 'date' }, + // Root-level analytics fields + scanner: { type: 'keyword' }, + severity: { type: 'keyword' }, + finding_hash: { type: 'keyword' }, + asset_key: { type: 'keyword' }, + // Workflow context under shipsec namespace + shipsec: { + type: 'object', + dynamic: true, + properties: { + organization_id: { type: 'keyword' }, + run_id: { type: 'keyword' }, + workflow_id: { type: 'keyword' }, + workflow_name: { type: 'keyword' }, + component_id: { type: 'keyword' }, + node_ref: { type: 'keyword' }, + asset_key: { type: 'keyword' }, + }, + }, + }, + }, + }, + }, + }); + + console.log(`āœ… Index template '${templateName}' created successfully`); + console.log('\nšŸ“Š Template configuration:'); + console.log(' - Index pattern: security-findings-*'); + console.log(' - Shards: 1, Replicas: 1'); + console.log(' - Mappings: @timestamp (date)'); + console.log(' root: scanner, severity, finding_hash, asset_key (keyword)'); + console.log(' shipsec.*: organization_id, run_id, workflow_id, workflow_name,'); + console.log(' component_id, node_ref, asset_key (keyword)'); + console.log('\nšŸŽ‰ OpenSearch setup completed successfully!'); + } catch (error) { + console.error('āŒ OpenSearch setup failed'); + console.error(error); + process.exit(1); + } +} + +main().catch((error) => { + console.error('āŒ Unexpected error during OpenSearch setup'); + console.error(error); + process.exit(1); +}); diff --git a/backend/src/analytics/analytics.controller.ts b/backend/src/analytics/analytics.controller.ts new file mode 100644 index 00000000..a8cef78a --- /dev/null +++ b/backend/src/analytics/analytics.controller.ts @@ -0,0 +1,297 @@ +import { + BadRequestException, + Body, + Controller, + ForbiddenException, + Get, + Headers, + Post, + Put, + UnauthorizedException, +} from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { ApiOkResponse, ApiTags, ApiHeader } from '@nestjs/swagger'; +import { Throttle, SkipThrottle } from '@nestjs/throttler'; + +import { SecurityAnalyticsService } from './security-analytics.service'; +import { OrganizationSettingsService } from './organization-settings.service'; +import { OpenSearchTenantService } from './opensearch-tenant.service'; +import { AnalyticsQueryRequestDto, AnalyticsQueryResponseDto } from './dto/analytics-query.dto'; +import { + AnalyticsSettingsResponseDto, + UpdateAnalyticsSettingsDto, + TIER_LIMITS, +} from './dto/analytics-settings.dto'; +import { CurrentAuth } from '../auth/auth-context.decorator'; +import { Public } from '../auth/public.decorator'; +import type { AuthContext } from '../auth/types'; + +const MAX_QUERY_SIZE = 1000; +const MAX_QUERY_FROM = 10000; + +function isValidNonNegativeInt(value: unknown): value is number { + return typeof value === 'number' && Number.isInteger(value) && value >= 0; +} + +@ApiTags('analytics') +@Controller('analytics') +export class AnalyticsController { + private readonly internalServiceToken: string; + + constructor( + private readonly securityAnalyticsService: SecurityAnalyticsService, + private readonly organizationSettingsService: OrganizationSettingsService, + private readonly openSearchTenantService: OpenSearchTenantService, + private readonly configService: ConfigService, + ) { + this.internalServiceToken = this.configService.get('INTERNAL_SERVICE_TOKEN') || ''; + } + + @Post('query') + @Throttle({ default: { limit: 100, ttl: 60000 } }) // 100 requests per minute per user + @ApiOkResponse({ + description: 'Query analytics data for the authenticated organization', + type: AnalyticsQueryResponseDto, + }) + @ApiHeader({ + name: 'X-RateLimit-Limit', + description: 'Maximum number of requests allowed per minute', + schema: { type: 'integer', example: 100 }, + }) + @ApiHeader({ + name: 'X-RateLimit-Remaining', + description: 'Number of requests remaining in the current time window', + schema: { type: 'integer', example: 99 }, + }) + async queryAnalytics( + @CurrentAuth() auth: AuthContext | null, + @Body() queryDto: AnalyticsQueryRequestDto, + ): Promise { + // Require authentication + if (!auth || !auth.isAuthenticated) { + throw new UnauthorizedException('Authentication required'); + } + + // Require organization context + if (!auth.organizationId) { + throw new UnauthorizedException('Organization context required'); + } + + // Validate query syntax + if (queryDto.query && typeof queryDto.query !== 'object') { + throw new BadRequestException('Invalid query syntax: query must be an object'); + } + + if (queryDto.aggs && typeof queryDto.aggs !== 'object') { + throw new BadRequestException('Invalid query syntax: aggs must be an object'); + } + + // Set defaults + const size = queryDto.size ?? 10; + const from = queryDto.from ?? 0; + + if (!isValidNonNegativeInt(size)) { + throw new BadRequestException('Invalid size: must be a non-negative integer'); + } + + if (!isValidNonNegativeInt(from)) { + throw new BadRequestException('Invalid from: must be a non-negative integer'); + } + + if (size > MAX_QUERY_SIZE) { + throw new BadRequestException(`Invalid size: maximum is ${MAX_QUERY_SIZE}`); + } + + if (from > MAX_QUERY_FROM) { + throw new BadRequestException(`Invalid from: maximum is ${MAX_QUERY_FROM}`); + } + + // Call the service to execute the query + return this.securityAnalyticsService.query(auth.organizationId, { + query: queryDto.query, + size, + from, + aggs: queryDto.aggs, + }); + } + + @Get('settings') + @ApiOkResponse({ + description: 'Get analytics settings for the authenticated organization', + type: AnalyticsSettingsResponseDto, + }) + async getAnalyticsSettings( + @CurrentAuth() auth: AuthContext | null, + ): Promise { + // Require authentication + if (!auth || !auth.isAuthenticated) { + throw new UnauthorizedException('Authentication required'); + } + + // Require organization context + if (!auth.organizationId) { + throw new UnauthorizedException('Organization context required'); + } + + // Get or create organization settings + const settings = await this.organizationSettingsService.getOrganizationSettings( + auth.organizationId, + ); + + // Get max retention days for tier + const maxRetentionDays = this.organizationSettingsService.getMaxRetentionDays( + settings.subscriptionTier, + ); + + return { + organizationId: settings.organizationId, + subscriptionTier: settings.subscriptionTier, + analyticsRetentionDays: settings.analyticsRetentionDays, + maxRetentionDays, + createdAt: settings.createdAt, + updatedAt: settings.updatedAt, + }; + } + + @Put('settings') + @ApiOkResponse({ + description: 'Update analytics settings for the authenticated organization', + type: AnalyticsSettingsResponseDto, + }) + async updateAnalyticsSettings( + @CurrentAuth() auth: AuthContext | null, + @Body() updateDto: UpdateAnalyticsSettingsDto, + ): Promise { + // Require authentication + if (!auth || !auth.isAuthenticated) { + throw new UnauthorizedException('Authentication required'); + } + + // Require organization context + if (!auth.organizationId) { + throw new UnauthorizedException('Organization context required'); + } + + // Only org admins can update settings + if (!auth.roles.includes('ADMIN')) { + throw new ForbiddenException('Only organization admins can update analytics settings'); + } + + // Get current settings to validate against tier + const currentSettings = await this.organizationSettingsService.getOrganizationSettings( + auth.organizationId, + ); + + // Determine the tier to validate against (use new tier if provided, otherwise current) + const tierToValidate = updateDto.subscriptionTier ?? currentSettings.subscriptionTier; + + // Validate retention period is within tier limits + if (updateDto.analyticsRetentionDays !== undefined) { + if ( + typeof updateDto.analyticsRetentionDays !== 'number' || + !Number.isInteger(updateDto.analyticsRetentionDays) + ) { + throw new BadRequestException('Retention period must be an integer number of days'); + } + + const isValid = this.organizationSettingsService.validateRetentionPeriod( + tierToValidate, + updateDto.analyticsRetentionDays, + ); + + if (!isValid) { + const maxDays = TIER_LIMITS[tierToValidate].maxRetentionDays; + throw new BadRequestException( + `Retention period of ${updateDto.analyticsRetentionDays} days exceeds the limit for ${TIER_LIMITS[tierToValidate].name} tier (${maxDays} days)`, + ); + } + } + + // Update settings + const updated = await this.organizationSettingsService.updateOrganizationSettings( + auth.organizationId, + { + analyticsRetentionDays: updateDto.analyticsRetentionDays, + subscriptionTier: updateDto.subscriptionTier, + }, + ); + + // Get max retention days for updated tier + const maxRetentionDays = this.organizationSettingsService.getMaxRetentionDays( + updated.subscriptionTier, + ); + + return { + organizationId: updated.organizationId, + subscriptionTier: updated.subscriptionTier, + analyticsRetentionDays: updated.analyticsRetentionDays, + maxRetentionDays, + createdAt: updated.createdAt, + updatedAt: updated.updatedAt, + }; + } + + /** + * Ensure tenant resources exist for an organization. + * Called by worker before indexing to ensure tenant isolation is set up. + * + * Requires X-Internal-Token header for authentication (internal service-to-service). + * This endpoint is idempotent - safe to call multiple times. + */ + @Public() + @SkipThrottle() + @Post('ensure-tenant') + @ApiOkResponse({ + description: 'Ensure tenant resources exist for organization', + schema: { + type: 'object', + properties: { + success: { type: 'boolean' }, + securityEnabled: { type: 'boolean' }, + message: { type: 'string' }, + }, + }, + }) + async ensureTenant( + @Headers('x-internal-token') internalToken: string | undefined, + @Body() body: { organizationId: string }, + ): Promise<{ success: boolean; securityEnabled: boolean; message: string }> { + // Validate internal service token + if (!this.internalServiceToken) { + // Token not configured - allow in dev mode but log warning + console.warn('[ensureTenant] INTERNAL_SERVICE_TOKEN not configured'); + } else if (internalToken !== this.internalServiceToken) { + throw new UnauthorizedException('Invalid internal service token'); + } + + // Validate request body + if (!body.organizationId || typeof body.organizationId !== 'string') { + throw new BadRequestException('organizationId is required'); + } + + const orgId = body.organizationId.trim(); + if (!orgId) { + throw new BadRequestException('organizationId cannot be empty'); + } + + // Check if security mode is enabled + if (!this.openSearchTenantService.isSecurityEnabled()) { + return { + success: true, + securityEnabled: false, + message: 'Security mode disabled, tenant provisioning skipped', + }; + } + + // Provision tenant resources + const success = await this.openSearchTenantService.ensureTenantExists(orgId); + + return { + success, + securityEnabled: true, + message: success + ? `Tenant provisioned for ${orgId}` + : `Failed to provision tenant for ${orgId}`, + }; + } +} diff --git a/backend/src/analytics/analytics.module.ts b/backend/src/analytics/analytics.module.ts index ee64fd92..e77f2062 100644 --- a/backend/src/analytics/analytics.module.ts +++ b/backend/src/analytics/analytics.module.ts @@ -1,8 +1,25 @@ import { Module } from '@nestjs/common'; +import { ConfigModule } from '@nestjs/config'; import { AnalyticsService } from './analytics.service'; +import { SecurityAnalyticsService } from './security-analytics.service'; +import { OrganizationSettingsService } from './organization-settings.service'; +import { OpenSearchTenantService } from './opensearch-tenant.service'; +import { AnalyticsController } from './analytics.controller'; @Module({ - providers: [AnalyticsService], - exports: [AnalyticsService], + imports: [ConfigModule], + controllers: [AnalyticsController], + providers: [ + AnalyticsService, + SecurityAnalyticsService, + OrganizationSettingsService, + OpenSearchTenantService, + ], + exports: [ + AnalyticsService, + SecurityAnalyticsService, + OrganizationSettingsService, + OpenSearchTenantService, + ], }) export class AnalyticsModule {} diff --git a/backend/src/analytics/dto/analytics-query.dto.ts b/backend/src/analytics/dto/analytics-query.dto.ts new file mode 100644 index 00000000..969939bd --- /dev/null +++ b/backend/src/analytics/dto/analytics-query.dto.ts @@ -0,0 +1,66 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export class AnalyticsQueryRequestDto { + @ApiProperty({ + description: 'OpenSearch DSL query object', + example: { match_all: {} }, + required: false, + }) + query?: Record; + + @ApiProperty({ + description: 'Number of results to return', + example: 10, + default: 10, + minimum: 0, + maximum: 1000, + required: false, + }) + size?: number; + + @ApiProperty({ + description: 'Offset for pagination', + example: 0, + default: 0, + minimum: 0, + maximum: 10000, + required: false, + }) + from?: number; + + @ApiProperty({ + description: 'OpenSearch aggregations object', + example: { + components: { + terms: { field: 'component_id' }, + }, + }, + required: false, + }) + aggs?: Record; +} + +export class AnalyticsQueryResponseDto { + @ApiProperty({ + description: 'Total number of matching documents', + example: 100, + }) + total!: number; + + @ApiProperty({ + description: 'Search hits', + type: 'array', + items: { type: 'object' }, + }) + hits!: { + _id: string; + _source: Record; + _score?: number; + }[]; + + @ApiProperty({ + description: 'Aggregation results', + required: false, + }) + aggregations?: Record; +} diff --git a/backend/src/analytics/dto/analytics-settings.dto.ts b/backend/src/analytics/dto/analytics-settings.dto.ts new file mode 100644 index 00000000..ce34c4d3 --- /dev/null +++ b/backend/src/analytics/dto/analytics-settings.dto.ts @@ -0,0 +1,75 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsEnum, IsInt, Min, Max, IsOptional } from 'class-validator'; +import type { SubscriptionTier } from '../../database/schema/organization-settings'; + +export type { SubscriptionTier }; + +export const TIER_LIMITS: Record = { + free: { name: 'Free', maxRetentionDays: 30 }, + pro: { name: 'Pro', maxRetentionDays: 90 }, + enterprise: { name: 'Enterprise', maxRetentionDays: 365 }, +}; + +export class AnalyticsSettingsResponseDto { + @ApiProperty({ + description: 'Organization ID', + example: 'org_abc123', + }) + organizationId!: string; + + @ApiProperty({ + description: 'Subscription tier', + enum: ['free', 'pro', 'enterprise'], + example: 'free', + }) + subscriptionTier!: SubscriptionTier; + + @ApiProperty({ + description: 'Data retention period in days', + example: 30, + }) + analyticsRetentionDays!: number; + + @ApiProperty({ + description: 'Maximum retention days allowed for this tier', + example: 30, + }) + maxRetentionDays!: number; + + @ApiProperty({ + description: 'Timestamp when settings were created', + example: '2026-01-20T00:00:00.000Z', + }) + createdAt!: Date; + + @ApiProperty({ + description: 'Timestamp when settings were last updated', + example: '2026-01-20T00:00:00.000Z', + }) + updatedAt!: Date; +} + +export class UpdateAnalyticsSettingsDto { + @ApiProperty({ + description: 'Data retention period in days (must be within tier limits)', + example: 30, + minimum: 1, + maximum: 365, + required: false, + }) + @IsOptional() + @IsInt() + @Min(1) + @Max(365) + analyticsRetentionDays?: number; + + // Optional: allow updating subscription tier (if needed in the future) + @ApiProperty({ + description: 'Subscription tier (optional - usually set by billing system)', + enum: ['free', 'pro', 'enterprise'], + required: false, + }) + @IsOptional() + @IsEnum(['free', 'pro', 'enterprise']) + subscriptionTier?: SubscriptionTier; +} diff --git a/backend/src/analytics/opensearch-tenant.service.ts b/backend/src/analytics/opensearch-tenant.service.ts new file mode 100644 index 00000000..062a6dd2 --- /dev/null +++ b/backend/src/analytics/opensearch-tenant.service.ts @@ -0,0 +1,417 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; + +const MAX_RETRIES = 3; +const RETRY_BASE_DELAY_MS = 1000; + +/** + * OpenSearch Tenant Service + * + * Handles dynamic tenant provisioning for multi-tenant analytics isolation. + * Creates OpenSearch Security tenants, roles, role mappings, index templates, + * seed indices, and index patterns for new organizations. + * + * This service is idempotent - safe to call multiple times for the same org. + * Guarded by OPENSEARCH_SECURITY_ENABLED - no-op when security is disabled. + */ +@Injectable() +export class OpenSearchTenantService { + private readonly logger = new Logger(OpenSearchTenantService.name); + private readonly securityEnabled: boolean; + private readonly opensearchUrl: string; + private readonly dashboardsUrl: string; + private readonly adminUsername: string; + private readonly adminPassword: string; + + constructor(private readonly configService: ConfigService) { + this.securityEnabled = this.configService.get('OPENSEARCH_SECURITY_ENABLED') === 'true'; + this.opensearchUrl = + this.configService.get('OPENSEARCH_URL') || 'http://opensearch:9200'; + this.dashboardsUrl = + this.configService.get('OPENSEARCH_DASHBOARDS_URL') || + 'http://opensearch-dashboards:5601'; + this.adminUsername = this.configService.get('OPENSEARCH_ADMIN_USERNAME') || 'admin'; + this.adminPassword = this.configService.get('OPENSEARCH_ADMIN_PASSWORD') || ''; + + this.logger.log( + `OpenSearch tenant service initialized (security: ${this.securityEnabled}, url: ${this.opensearchUrl})`, + ); + } + + /** + * Validates organization ID format. + * Must be lowercase alphanumeric with hyphens/underscores, starting with alphanumeric. + */ + private validateOrgId(orgId: string): boolean { + return /^[a-z0-9][a-z0-9_-]*$/.test(orgId); + } + + /** + * Creates Basic Auth header for OpenSearch API calls. + */ + private getAuthHeader(): string { + return `Basic ${Buffer.from(`${this.adminUsername}:${this.adminPassword}`).toString('base64')}`; + } + + /** + * Fetch wrapper with retry logic for transient connection errors. + * Bun's fetch can fail with various messages (ConnectionRefused, "typo in url", + * "Unable to connect") during concurrent request bursts. Retry all fetch-level + * errors (not HTTP errors) with exponential backoff. + */ + private async fetchWithRetry( + url: string, + options: RequestInit, + label: string, + ): Promise { + for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) { + try { + return await fetch(url, options); + } catch (error: any) { + if (attempt === MAX_RETRIES) { + throw error; + } + + const delay = RETRY_BASE_DELAY_MS * Math.pow(2, attempt - 1); + this.logger.warn( + `${label}: fetch failed (attempt ${attempt}/${MAX_RETRIES}): ${error?.message}. Retrying in ${delay}ms`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + // Unreachable, but TypeScript needs it + throw new Error(`${label}: exhausted retries`); + } + + /** + * Ensures all tenant resources exist for the given organization. + * Creates: tenant, role, role mapping, index template, seed index, index pattern. + * + * This method is idempotent - safe to call multiple times. + * Returns true if all resources were created/verified successfully. + */ + async ensureTenantExists(orgId: string): Promise { + // No-op when security is disabled (dev mode) + if (!this.securityEnabled) { + this.logger.debug(`Tenant provisioning skipped (security disabled): ${orgId}`); + return true; + } + + // Normalize to lowercase for consistent tenant naming + const normalizedOrgId = orgId.toLowerCase(); + + // Validate format + if (!this.validateOrgId(normalizedOrgId)) { + this.logger.warn(`Invalid org ID format: ${orgId}`); + return false; + } + + this.logger.log(`Provisioning tenant for org: ${normalizedOrgId}`); + + try { + // Brief delay to let the nginx auth_request burst settle before + // making outbound connections (Bun's fetch can fail during bursts) + await new Promise((resolve) => setTimeout(resolve, 500)); + + // Step 1: Create tenant + await this.createTenant(normalizedOrgId); + + // Step 2: Create read-only role for this customer + await this.createCustomerRole(normalizedOrgId); + + // Step 3: Create role mapping + await this.createRoleMapping(normalizedOrgId); + + // Step 4: Create index template with field mappings + await this.createIndexTemplate(normalizedOrgId); + + // Step 5: Create seed index so the index pattern can resolve fields + await this.createSeedIndex(normalizedOrgId); + + // Step 6: Create index pattern in Dashboards + await this.createIndexPattern(normalizedOrgId); + + this.logger.log(`Tenant provisioned successfully: ${normalizedOrgId}`); + return true; + } catch (error: any) { + this.logger.error( + `Failed to provision tenant ${normalizedOrgId}: ${error?.message || error}`, + ); + return false; + } + } + + /** + * Creates a tenant in OpenSearch Security. + */ + private async createTenant(orgId: string): Promise { + const url = `${this.opensearchUrl}/_plugins/_security/api/tenants/${orgId}`; + + const response = await this.fetchWithRetry( + url, + { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + Authorization: this.getAuthHeader(), + }, + body: JSON.stringify({ + description: `Tenant for organization ${orgId}`, + }), + }, + `createTenant(${orgId})`, + ); + + // 200 = created, 409 = already exists (both are OK) + if (!response.ok && response.status !== 409) { + throw new Error(`Failed to create tenant: ${response.status} ${response.statusText}`); + } + + this.logger.debug(`Tenant created/verified: ${orgId}`); + } + + /** + * Creates a read-only customer role for the organization. + * Grants read-only access to security findings indices, plus the minimum + * Dashboards/Notifications permissions required for tenant-scoped UI usage. + */ + private async createCustomerRole(orgId: string): Promise { + const roleName = `customer_${orgId}_ro`; + const url = `${this.opensearchUrl}/_plugins/_security/api/roles/${roleName}`; + const tenantSavedObjectsPattern = `.kibana_*_${orgId.replace(/[^a-z0-9]/g, '')}*`; + + const roleDefinition = { + cluster_permissions: [ + 'cluster_composite_ops_ro', + // Required for Dashboards saved objects (bulk writes to .kibana_* tenant indices) + 'indices:data/write/bulk', + // Alerting: monitor CRUD, execution, alerts, and destinations (legacy endpoints) + 'cluster:admin/opendistro/alerting/monitor/get', + 'cluster:admin/opendistro/alerting/monitor/search', + 'cluster:admin/opendistro/alerting/monitor/write', + 'cluster:admin/opendistro/alerting/monitor/execute', + 'cluster:admin/opendistro/alerting/alerts/get', + 'cluster:admin/opendistro/alerting/alerts/ack', + 'cluster:admin/opendistro/alerting/destination/get', + 'cluster:admin/opendistro/alerting/destination/write', + 'cluster:admin/opendistro/alerting/destination/delete', + // Notifications plugin (OpenSearch 2.x): channel features + config CRUD + 'cluster:admin/opensearch/notifications/features', + 'cluster:admin/opensearch/notifications/configs/get', + 'cluster:admin/opensearch/notifications/configs/create', + 'cluster:admin/opensearch/notifications/configs/update', + 'cluster:admin/opensearch/notifications/configs/delete', + ], + index_permissions: [ + { + index_patterns: [`security-findings-${orgId}-*`], + allowed_actions: ['read', 'indices:data/read/*'], + }, + { + // Tenant-scoped Dashboards saved objects index alias/index + index_patterns: [tenantSavedObjectsPattern], + allowed_actions: [ + 'read', + 'write', + 'create_index', + 'indices:data/read/*', + 'indices:data/write/*', + 'indices:admin/mapping/put', + ], + }, + ], + tenant_permissions: [ + { + tenant_patterns: [orgId], + allowed_actions: ['kibana_all_write'], + }, + ], + }; + + const response = await this.fetchWithRetry( + url, + { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + Authorization: this.getAuthHeader(), + }, + body: JSON.stringify(roleDefinition), + }, + `createCustomerRole(${orgId})`, + ); + + if (!response.ok && response.status !== 409) { + throw new Error(`Failed to create role: ${response.status} ${response.statusText}`); + } + + this.logger.debug(`Role created/verified: ${roleName}`); + } + + /** + * Creates a role mapping for the customer role. + * Maps the role name to backend_roles so nginx proxy auth works. + */ + private async createRoleMapping(orgId: string): Promise { + const roleName = `customer_${orgId}_ro`; + const url = `${this.opensearchUrl}/_plugins/_security/api/rolesmapping/${roleName}`; + + const mappingDefinition = { + backend_roles: [roleName], + description: `Role mapping for ${orgId} read-only access`, + }; + + const response = await this.fetchWithRetry( + url, + { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + Authorization: this.getAuthHeader(), + }, + body: JSON.stringify(mappingDefinition), + }, + `createRoleMapping(${orgId})`, + ); + + if (!response.ok && response.status !== 409) { + throw new Error(`Failed to create role mapping: ${response.status} ${response.statusText}`); + } + + this.logger.debug(`Role mapping created/verified: ${roleName}`); + } + + /** + * Creates an index template so all future security-findings-{orgId}-* indices + * get proper field mappings automatically. + */ + private async createIndexTemplate(orgId: string): Promise { + const templateName = `security-findings-${orgId}`; + const url = `${this.opensearchUrl}/_index_template/${templateName}`; + + const templateDefinition = { + index_patterns: [`security-findings-${orgId}-*`], + template: { + mappings: { + properties: { + '@timestamp': { type: 'date' }, + workflow_id: { type: 'keyword' }, + workflow_name: { type: 'keyword' }, + run_id: { type: 'keyword' }, + node_ref: { type: 'keyword' }, + component_id: { type: 'keyword' }, + asset_key: { type: 'keyword' }, + }, + }, + }, + priority: 100, + }; + + const response = await this.fetchWithRetry( + url, + { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + Authorization: this.getAuthHeader(), + }, + body: JSON.stringify(templateDefinition), + }, + `createIndexTemplate(${orgId})`, + ); + + if (!response.ok) { + throw new Error(`Failed to create index template: ${response.status} ${response.statusText}`); + } + + this.logger.debug(`Index template created/verified: ${templateName}`); + } + + /** + * Creates a seed index with explicit mappings so the Dashboards index pattern + * can resolve fields (especially @timestamp) before any real data is ingested. + */ + private async createSeedIndex(orgId: string): Promise { + const indexName = `security-findings-${orgId}-seed`; + const url = `${this.opensearchUrl}/${indexName}`; + + const response = await this.fetchWithRetry( + url, + { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + Authorization: this.getAuthHeader(), + }, + body: JSON.stringify({ + mappings: { + properties: { + '@timestamp': { type: 'date' }, + workflow_id: { type: 'keyword' }, + workflow_name: { type: 'keyword' }, + run_id: { type: 'keyword' }, + node_ref: { type: 'keyword' }, + component_id: { type: 'keyword' }, + asset_key: { type: 'keyword' }, + }, + }, + }), + }, + `createSeedIndex(${orgId})`, + ); + + // 200 = created, 400 with "already exists" = OK + if (!response.ok && response.status !== 400) { + throw new Error(`Failed to create seed index: ${response.status} ${response.statusText}`); + } + + this.logger.debug(`Seed index created/verified: ${indexName}`); + } + + /** + * Creates an index pattern in OpenSearch Dashboards for this tenant. + */ + private async createIndexPattern(orgId: string): Promise { + const patternId = `security-findings-${orgId}-*`; + const url = `${this.dashboardsUrl}/analytics/api/saved_objects/index-pattern/${encodeURIComponent(patternId)}`; + + const response = await this.fetchWithRetry( + url, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'osd-xsrf': 'true', + securitytenant: orgId, // Create in tenant's namespace + 'x-proxy-user': this.adminUsername, // Required for Dashboards proxy auth mode + 'x-proxy-roles': 'platform_admin', + 'x-forwarded-for': '127.0.0.1', // Required for proxy auth trust chain + }, + body: JSON.stringify({ + attributes: { + title: patternId, + timeFieldName: '@timestamp', + }, + }), + }, + `createIndexPattern(${orgId})`, + ); + + // 200 = created, 409 = already exists (both are OK) + if (!response.ok && response.status !== 409) { + const body = await response.text().catch(() => ''); + throw new Error( + `Failed to create index pattern: ${response.status} ${response.statusText} - ${body}`, + ); + } + + this.logger.debug(`Index pattern created/verified: ${patternId}`); + } + + /** + * Check if security mode is enabled. + */ + isSecurityEnabled(): boolean { + return this.securityEnabled; + } +} diff --git a/backend/src/analytics/organization-settings.service.ts b/backend/src/analytics/organization-settings.service.ts new file mode 100644 index 00000000..c33adf69 --- /dev/null +++ b/backend/src/analytics/organization-settings.service.ts @@ -0,0 +1,101 @@ +import { Inject, Injectable, Logger } from '@nestjs/common'; +import { eq } from 'drizzle-orm'; +import { type NodePgDatabase } from 'drizzle-orm/node-postgres'; + +import { DRIZZLE_TOKEN } from '../database/database.module'; +import { + organizationSettingsTable, + OrganizationSettings, + SubscriptionTier, +} from '../database/schema/organization-settings'; +import { TIER_LIMITS } from './dto/analytics-settings.dto'; +import { OpenSearchTenantService } from './opensearch-tenant.service'; + +@Injectable() +export class OrganizationSettingsService { + private readonly logger = new Logger(OrganizationSettingsService.name); + + constructor( + @Inject(DRIZZLE_TOKEN) + private readonly db: NodePgDatabase, + private readonly tenantService: OpenSearchTenantService, + ) {} + + /** + * Get or create organization settings + */ + async getOrganizationSettings(organizationId: string): Promise { + // Try to get existing settings + const [existing] = await this.db + .select() + .from(organizationSettingsTable) + .where(eq(organizationSettingsTable.organizationId, organizationId)); + + if (existing) { + return existing; + } + + // Create default settings if they don't exist + this.logger.log(`Creating default settings for organization: ${organizationId}`); + const [created] = await this.db + .insert(organizationSettingsTable) + .values({ + organizationId, + subscriptionTier: 'free', + analyticsRetentionDays: 30, + }) + .returning(); + + // Provision OpenSearch tenant for the new organization (fire-and-forget) + this.tenantService.ensureTenantExists(organizationId).catch((err) => { + this.logger.error(`Failed to provision OpenSearch tenant for ${organizationId}: ${err}`); + }); + + return created; + } + + /** + * Update organization settings + */ + async updateOrganizationSettings( + organizationId: string, + updates: { + analyticsRetentionDays?: number; + subscriptionTier?: SubscriptionTier; + }, + ): Promise { + // Ensure settings exist + await this.getOrganizationSettings(organizationId); + + // Update settings + const [updated] = await this.db + .update(organizationSettingsTable) + .set({ + ...updates, + updatedAt: new Date(), + }) + .where(eq(organizationSettingsTable.organizationId, organizationId)) + .returning(); + + this.logger.log( + `Updated settings for organization ${organizationId}: ${JSON.stringify(updates)}`, + ); + + return updated; + } + + /** + * Validate retention period is within tier limits + */ + validateRetentionPeriod(tier: SubscriptionTier, retentionDays: number): boolean { + const limit = TIER_LIMITS[tier]; + return retentionDays <= limit.maxRetentionDays && retentionDays > 0; + } + + /** + * Get max retention days for a tier + */ + getMaxRetentionDays(tier: SubscriptionTier): number { + return TIER_LIMITS[tier].maxRetentionDays; + } +} diff --git a/backend/src/analytics/security-analytics.service.ts b/backend/src/analytics/security-analytics.service.ts new file mode 100644 index 00000000..ce53a645 --- /dev/null +++ b/backend/src/analytics/security-analytics.service.ts @@ -0,0 +1,239 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { OpenSearchClient } from '../config/opensearch.client'; + +interface IndexDocumentOptions { + workflowId: string; + workflowName: string; + runId: string; + nodeRef: string; + componentId: string; + assetKeyField?: string; + indexSuffix?: string; +} + +type BulkIndexOptions = IndexDocumentOptions; + +@Injectable() +export class SecurityAnalyticsService { + private readonly logger = new Logger(SecurityAnalyticsService.name); + + constructor(private readonly openSearchClient: OpenSearchClient) {} + + /** + * Index a single document to OpenSearch with metadata + */ + async indexDocument( + orgId: string, + document: Record, + options: IndexDocumentOptions, + ): Promise { + if (!this.openSearchClient.isClientEnabled()) { + this.logger.debug('OpenSearch client not enabled, skipping indexing'); + return; + } + + const client = this.openSearchClient.getClient(); + if (!client) { + this.logger.warn('OpenSearch client is null, skipping indexing'); + return; + } + + try { + const indexName = this.buildIndexName(orgId, options.indexSuffix); + const assetKey = this.detectAssetKey(document, options.assetKeyField); + + const enrichedDocument = { + ...document, + '@timestamp': new Date().toISOString(), + workflow_id: options.workflowId, + workflow_name: options.workflowName, + run_id: options.runId, + node_ref: options.nodeRef, + component_id: options.componentId, + ...(assetKey && { asset_key: assetKey }), + }; + + await client.index({ + index: indexName, + body: enrichedDocument, + }); + + this.logger.debug(`Indexed document to ${indexName} for workflow ${options.workflowId}`); + } catch (error) { + this.logger.error(`Failed to index document: ${error}`); + throw error; + } + } + + /** + * Bulk index multiple documents to OpenSearch + */ + async bulkIndex( + orgId: string, + documents: Record[], + options: BulkIndexOptions, + ): Promise { + if (!this.openSearchClient.isClientEnabled()) { + this.logger.debug('OpenSearch client not enabled, skipping bulk indexing'); + return; + } + + const client = this.openSearchClient.getClient(); + if (!client) { + this.logger.warn('OpenSearch client is null, skipping bulk indexing'); + return; + } + + if (documents.length === 0) { + this.logger.debug('No documents to index, skipping bulk indexing'); + return; + } + + try { + const indexName = this.buildIndexName(orgId, options.indexSuffix); + + // Build bulk operations array + const bulkOps: any[] = []; + for (const document of documents) { + const assetKey = this.detectAssetKey(document, options.assetKeyField); + + const enrichedDocument = { + ...document, + '@timestamp': new Date().toISOString(), + workflow_id: options.workflowId, + workflow_name: options.workflowName, + run_id: options.runId, + node_ref: options.nodeRef, + component_id: options.componentId, + ...(assetKey && { asset_key: assetKey }), + }; + + bulkOps.push({ index: { _index: indexName } }); + bulkOps.push(enrichedDocument); + } + + const response = await client.bulk({ + body: bulkOps, + }); + + if (response.body.errors) { + const errorCount = response.body.items.filter((item: any) => item.index?.error).length; + this.logger.warn( + `Bulk indexing completed with ${errorCount} errors out of ${documents.length} documents`, + ); + } else { + this.logger.debug( + `Bulk indexed ${documents.length} documents to ${indexName} for workflow ${options.workflowId}`, + ); + } + } catch (error) { + this.logger.error(`Failed to bulk index documents: ${error}`); + throw error; + } + } + + /** + * Build the index name with org scoping and date-based rotation + * Format: security-findings-{orgId}-{YYYY.MM.DD} + */ + private buildIndexName(orgId: string, indexSuffix?: string): string { + const date = new Date(); + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, '0'); + const day = String(date.getDate()).padStart(2, '0'); + + const suffix = indexSuffix || `${year}.${month}.${day}`; + return `security-findings-${orgId}-${suffix}`; + } + + /** + * Query analytics data for an organization + */ + async query( + orgId: string, + options: { + query?: Record; + size?: number; + from?: number; + aggs?: Record; + }, + ): Promise<{ + total: number; + hits: { _id: string; _source: Record; _score?: number }[]; + aggregations?: Record; + }> { + if (!this.openSearchClient.isClientEnabled()) { + this.logger.warn('OpenSearch client not enabled, returning empty results'); + return { total: 0, hits: [], aggregations: undefined }; + } + + const client = this.openSearchClient.getClient(); + if (!client) { + this.logger.warn('OpenSearch client is null, returning empty results'); + return { total: 0, hits: [], aggregations: undefined }; + } + + try { + // Build index pattern for org: security-findings-{orgId}-* + const indexPattern = `security-findings-${orgId}-*`; + + // Execute the search + const response = await client.search({ + index: indexPattern, + body: { + query: options.query || { match_all: {} }, + size: options.size ?? 10, + from: options.from ?? 0, + ...(options.aggs && { aggs: options.aggs }), + }, + }); + + // Extract results from OpenSearch response + const total: number = + typeof response.body.hits.total === 'object' + ? (response.body.hits.total.value ?? 0) + : (response.body.hits.total ?? 0); + + const hits = response.body.hits.hits.map((hit: any) => ({ + _id: hit._id, + _source: hit._source, + ...(hit._score !== undefined && { _score: hit._score }), + })); + + return { + total, + hits, + aggregations: response.body.aggregations, + }; + } catch (error) { + this.logger.error(`Failed to query analytics data: ${error}`); + throw error; + } + } + + /** + * Auto-detect asset key from common fields + * Priority: host > domain > subdomain > url > ip > asset > target + */ + private detectAssetKey(document: Record, explicitField?: string): string | null { + // If explicit field is provided, use it + if (explicitField && document[explicitField]) { + return String(document[explicitField]); + } + + if (document.asset_key) { + return String(document.asset_key); + } + + // Auto-detect from common fields + const assetFields = ['host', 'domain', 'subdomain', 'url', 'ip', 'asset', 'target']; + + for (const field of assetFields) { + if (document[field]) { + return String(document[field]); + } + } + + return null; + } +} diff --git a/backend/src/app.controller.ts b/backend/src/app.controller.ts index 681adbac..952f87ee 100644 --- a/backend/src/app.controller.ts +++ b/backend/src/app.controller.ts @@ -1,13 +1,157 @@ -import { Controller, Get } from '@nestjs/common'; +import { Controller, Get, Logger, Post, Res, UnauthorizedException, Headers } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { SkipThrottle } from '@nestjs/throttler'; +import type { Response } from 'express'; import { AppService } from './app.service'; +import { CurrentAuth } from './auth/auth-context.decorator'; +import type { AuthContext } from './auth/types'; +import { Public } from './auth/public.decorator'; +import type { AuthConfig } from './config/auth.config'; +import { + SESSION_COOKIE_NAME, + SESSION_COOKIE_MAX_AGE, + createSessionToken, +} from './auth/session.utils'; +import { OpenSearchTenantService } from './analytics/opensearch-tenant.service'; @Controller() export class AppController { - constructor(private readonly appService: AppService) {} + private readonly logger = new Logger(AppController.name); + private readonly authCfg: AuthConfig; + /** Track org provisioning state: resolved promises for completed orgs, pending promises for in-flight */ + private readonly provisioningOrgs = new Map>(); + constructor( + private readonly appService: AppService, + private readonly configService: ConfigService, + private readonly tenantService: OpenSearchTenantService, + ) { + this.authCfg = this.configService.get('auth')!; + } + + @SkipThrottle() @Get('/health') health() { return this.appService.getHealth(); } + + /** + * Auth validation endpoint for nginx auth_request. + * Returns 200 if authenticated, 401 otherwise. + * Used by nginx to protect /analytics/* routes. + * + * Response headers (for nginx tenant isolation): + * - X-Auth-Organization-Id: lowercase normalized org ID (empty if no org context) + * - X-Auth-User-Id: user identifier + * + * Note: SkipThrottle is required because nginx sends an auth_request + * for every resource loaded from /analytics/*, which can quickly + * exceed rate limits and cause 500 errors. + */ + @SkipThrottle() + @Get('/auth/validate') + validateAuth(@CurrentAuth() auth: AuthContext | null, @Res({ passthrough: true }) res: Response) { + if (!auth || !auth.isAuthenticated) { + throw new UnauthorizedException(); + } + + // Set headers for nginx tenant isolation + // Canonicalize orgId to lowercase for consistent tenant naming + const normalizedOrgId = auth.organizationId?.toLowerCase() || ''; + res.setHeader('X-Auth-Organization-Id', normalizedOrgId); + res.setHeader('X-Auth-User-Id', auth.userId || ''); + + // Ensure OpenSearch tenant exists for this org (fire-and-forget, cached) + // Uses a Map of promises so: (1) concurrent requests share the same in-flight provisioning, + // (2) failures are removed from cache to allow retry on next auth request. + if (normalizedOrgId && !this.provisioningOrgs.has(normalizedOrgId)) { + const promise = this.tenantService.ensureTenantExists(normalizedOrgId).then( + (success) => { + if (!success) { + // Provisioning returned false (validation error, etc.) ā€” allow retry + this.provisioningOrgs.delete(normalizedOrgId); + } + return success; + }, + (err) => { + // Remove from cache so it retries next request + this.provisioningOrgs.delete(normalizedOrgId); + this.logger.error(`Failed to provision OpenSearch tenant for ${normalizedOrgId}: ${err}`); + return false; + }, + ); + this.provisioningOrgs.set(normalizedOrgId, promise); + } + + return { valid: true }; + } + + /** + * Login endpoint for local auth. + * Validates Basic auth credentials and sets a session cookie. + */ + @Public() + @Post('/auth/login') + login( + @Headers('authorization') authHeader: string | undefined, + @Res({ passthrough: true }) res: Response, + ) { + // Only for local auth provider + if (this.authCfg.provider !== 'local') { + throw new UnauthorizedException('Login endpoint only available for local auth'); + } + + // Validate Basic auth header + if (!authHeader || !authHeader.startsWith('Basic ')) { + throw new UnauthorizedException('Missing Basic Auth credentials'); + } + + const base64Credentials = authHeader.slice(6); + let username: string; + let password: string; + + try { + const credentials = Buffer.from(base64Credentials, 'base64').toString('utf-8'); + [username, password] = credentials.split(':'); + } catch { + throw new UnauthorizedException('Invalid Basic Auth format'); + } + + if (!username || !password) { + throw new UnauthorizedException('Invalid Basic Auth format'); + } + + // Validate credentials + if ( + username !== this.authCfg.local.adminUsername || + password !== this.authCfg.local.adminPassword + ) { + throw new UnauthorizedException('Invalid admin credentials'); + } + + // Create session token and set cookie + const sessionToken = createSessionToken(username); + + res.cookie(SESSION_COOKIE_NAME, sessionToken, { + httpOnly: true, + secure: process.env.NODE_ENV === 'production', + sameSite: 'lax', + maxAge: SESSION_COOKIE_MAX_AGE, + path: '/', + }); + + return { success: true, message: 'Logged in successfully' }; + } + + /** + * Logout endpoint for local auth. + * Clears the session cookie. + */ + @Public() + @Post('/auth/logout') + logout(@Res({ passthrough: true }) res: Response) { + res.clearCookie(SESSION_COOKIE_NAME, { path: '/' }); + return { success: true, message: 'Logged out successfully' }; + } } diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index 81e3e26b..c11cda7e 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -2,10 +2,15 @@ import { Module } from '@nestjs/common'; import { APP_GUARD } from '@nestjs/core'; import { ConfigModule } from '@nestjs/config'; import { join } from 'node:path'; +import { ThrottlerModule, ThrottlerGuard, seconds } from '@nestjs/throttler'; +import { ThrottlerStorageRedisService } from '@nest-lab/throttler-storage-redis'; +import Redis from 'ioredis'; import { AppController } from './app.controller'; import { AppService } from './app.service'; import { authConfig } from './config/auth.config'; +import { opensearchConfig } from './config/opensearch.config'; +import { OpenSearchModule } from './config/opensearch.module'; import { AgentsModule } from './agents/agents.module'; import { AuthModule } from './auth/auth.module'; import { AuthGuard } from './auth/auth.guard'; @@ -68,8 +73,25 @@ function getEnvFilePaths(): string[] { ConfigModule.forRoot({ isGlobal: true, envFilePath: getEnvFilePaths(), - load: [authConfig], + load: [authConfig, opensearchConfig], }), + ThrottlerModule.forRootAsync({ + useFactory: () => { + const redisUrl = process.env.REDIS_URL; + + return { + throttlers: [ + { + name: 'default', + ttl: seconds(60), // 60 seconds + limit: 100, // 100 requests per minute + }, + ], + storage: redisUrl ? new ThrottlerStorageRedisService(new Redis(redisUrl)) : undefined, // Falls back to in-memory storage if Redis not configured + }; + }, + }), + OpenSearchModule, ...coreModules, ...testingModules, ], @@ -84,6 +106,10 @@ function getEnvFilePaths(): string[] { provide: APP_GUARD, useClass: RolesGuard, }, + { + provide: APP_GUARD, + useClass: ThrottlerGuard, + }, ], }) export class AppModule {} diff --git a/backend/src/auth/providers/clerk-auth.provider.ts b/backend/src/auth/providers/clerk-auth.provider.ts index 5cb42458..d51af777 100644 --- a/backend/src/auth/providers/clerk-auth.provider.ts +++ b/backend/src/auth/providers/clerk-auth.provider.ts @@ -95,45 +95,90 @@ export class ClerkAuthProvider implements AuthProviderStrategy { } private extractBearerToken(request: Request): string | null { + // Priority 1: Authorization header (API calls from frontend) const header = request.headers.authorization ?? (request.headers.Authorization as string | undefined); - if (!header) { - return null; + if (header) { + const [scheme, token] = header.split(' '); + if (scheme && token && scheme.toLowerCase() === 'bearer') { + return token.trim(); + } } - const [scheme, token] = header.split(' '); - if (!scheme || !token) { - return null; + + // Priority 2: Clerk's __session cookie (browser navigations like /analytics/) + // Clerk's JS SDK sets this cookie on the app's domain; it contains a verifiable JWT + const sessionCookie = request.cookies?.['__session']; + if (sessionCookie) { + this.logger.log(`[AUTH] No Authorization header, falling back to Clerk __session cookie`); + return sessionCookie; } - return scheme.toLowerCase() === 'bearer' ? token.trim() : null; + + return null; } /** - * Resolves organization ID with priority: - * 1. X-Organization-Id header (user's selected org) - * 2. JWT payload org_id/organization_id - * 3. Default to "user's workspace" format: workspace-{userId} + * Resolves organization ID with validation: + * 1. If X-Organization-Id header matches JWT's org_id ā†’ trust it + * 2. If header is workspace-{userId} ā†’ trust it (personal workspace) + * 3. If header specifies different org than JWT ā†’ IGNORE it, log security warning + * 4. Fall through to JWT org or workspace default + * + * Security: This prevents spoofed X-Organization-Id headers from accessing + * other organizations' data. The JWT's org_id is the source of truth. */ private resolveOrganizationId(request: Request, payload: ClerkJwt, userId: string): string { - // Priority 1: Header (user's selected org from frontend) const headerOrg = request.headers['x-organization-id'] as string | undefined; - this.logger.log(`[AUTH] X-Organization-Id header: ${headerOrg || 'not present'}`); + const jwtOrg = payload.o?.id || payload.org_id || payload.organization_id; + const userWorkspace = `workspace-${userId}`; + this.logger.log( + `[AUTH] Resolving org - Header: ${headerOrg || 'not present'}, JWT org: ${jwtOrg || 'none'}, User: ${userId}`, + ); + + // If header is provided, validate it if (headerOrg && headerOrg.trim().length > 0) { - this.logger.log(`[AUTH] Using org from header: ${headerOrg.trim()}`); - return headerOrg.trim(); + const trimmedHeader = headerOrg.trim(); + + // Case 1: Header matches JWT org ā†’ trust it + if (jwtOrg && trimmedHeader === jwtOrg) { + this.logger.log(`[AUTH] Header matches JWT org, using: ${trimmedHeader}`); + return trimmedHeader; + } + + // Case 2: Header is user's personal workspace ā†’ trust it + if (trimmedHeader === userWorkspace) { + this.logger.log(`[AUTH] Header is user's workspace, using: ${trimmedHeader}`); + return trimmedHeader; + } + + // Case 3: Header specifies a DIFFERENT org than JWT ā†’ IGNORE and log security warning + if (jwtOrg && trimmedHeader !== jwtOrg) { + this.logger.warn( + `[AUTH] SECURITY: X-Organization-Id header "${trimmedHeader}" does not match JWT org "${jwtOrg}". ` + + `User ${userId} may be attempting cross-tenant access. Ignoring header.`, + ); + // Fall through to use JWT org + } + + // Case 4: No JWT org but header is not user's workspace ā†’ potential spoofing + if (!jwtOrg && trimmedHeader !== userWorkspace) { + this.logger.warn( + `[AUTH] SECURITY: X-Organization-Id header "${trimmedHeader}" provided without JWT org context. ` + + `User ${userId} does not have active org session. Ignoring header.`, + ); + // Fall through to workspace default + } } - // Priority 2: JWT payload org - const jwtOrg = payload.o?.id || payload.org_id || payload.organization_id; + // Use JWT org if available if (jwtOrg) { this.logger.log(`[AUTH] Using org from JWT payload: ${jwtOrg}`); return jwtOrg; } - // Priority 3: Default to user's workspace - const workspace = `workspace-${userId}`; - this.logger.log(`[AUTH] No org found, using workspace: ${workspace}`); - return workspace; + // Default to user's workspace + this.logger.log(`[AUTH] No org found, using workspace: ${userWorkspace}`); + return userWorkspace; } /** diff --git a/backend/src/auth/providers/local-auth.provider.ts b/backend/src/auth/providers/local-auth.provider.ts index f4adadea..2e2e14d4 100644 --- a/backend/src/auth/providers/local-auth.provider.ts +++ b/backend/src/auth/providers/local-auth.provider.ts @@ -1,10 +1,11 @@ import type { Request } from 'express'; -import { Injectable, UnauthorizedException } from '@nestjs/common'; +import { Injectable, UnauthorizedException, Logger } from '@nestjs/common'; import type { LocalAuthConfig } from '../../config/auth.config'; import { DEFAULT_ROLES, type AuthContext } from '../types'; import type { AuthProviderStrategy } from './auth-provider.interface'; import { DEFAULT_ORGANIZATION_ID } from '../constants'; +import { verifySessionToken, SESSION_COOKIE_NAME } from '../session.utils'; function extractBasicAuth( headerValue: string | undefined, @@ -28,6 +29,7 @@ function extractBasicAuth( @Injectable() export class LocalAuthProvider implements AuthProviderStrategy { readonly name = 'local'; + private readonly logger = new Logger(LocalAuthProvider.name); constructor(private readonly config: LocalAuthConfig) {} @@ -35,16 +37,36 @@ export class LocalAuthProvider implements AuthProviderStrategy { // Always use local-dev org ID for local auth const orgId = DEFAULT_ORGANIZATION_ID; - // Require Basic Auth (admin credentials) + // Check config if (!this.config.adminUsername || !this.config.adminPassword) { throw new UnauthorizedException('Local auth not configured - admin credentials required'); } + // Try session cookie first (for browser navigation requests like /analytics/) + const sessionCookie = request.cookies?.[SESSION_COOKIE_NAME]; + if (sessionCookie) { + const session = verifySessionToken(sessionCookie); + if (session && session.username === this.config.adminUsername) { + this.logger.debug(`Session cookie auth successful for user: ${session.username}`); + return { + userId: 'admin', + organizationId: orgId, + roles: DEFAULT_ROLES, + isAuthenticated: true, + provider: this.name, + }; + } + this.logger.debug('Session cookie invalid or username mismatch'); + } + + // Fall back to Basic Auth (for API requests) const authHeader = request.headers.authorization; const basicAuth = extractBasicAuth(authHeader); if (!basicAuth) { - throw new UnauthorizedException('Missing Basic Auth credentials'); + throw new UnauthorizedException( + 'Missing authentication - provide session cookie or Basic Auth', + ); } if ( diff --git a/backend/src/auth/session.utils.ts b/backend/src/auth/session.utils.ts new file mode 100644 index 00000000..d7b54319 --- /dev/null +++ b/backend/src/auth/session.utils.ts @@ -0,0 +1,66 @@ +import * as crypto from 'crypto'; + +// Session cookie configuration +export const SESSION_COOKIE_NAME = 'shipsec_session'; +export const SESSION_COOKIE_MAX_AGE = 7 * 24 * 60 * 60 * 1000; // 7 days + +function getSessionSecret(): string { + const secret = process.env.SESSION_SECRET; + if (!secret) { + if (process.env.NODE_ENV === 'production') { + throw new Error('SESSION_SECRET is required in production for session authentication'); + } + return 'local-dev-session-secret'; + } + return secret; +} + +export interface SessionPayload { + username: string; + ts: number; +} + +/** + * Create a signed session token for local auth. + */ +export function createSessionToken(username: string): string { + const secret = getSessionSecret(); + const payload = JSON.stringify({ username, ts: Date.now() }); + const hmac = crypto.createHmac('sha256', secret); + hmac.update(payload); + const signature = hmac.digest('hex'); + return Buffer.from(`${payload}.${signature}`).toString('base64'); +} + +/** + * Verify and decode a session token. + */ +export function verifySessionToken(token: string): SessionPayload | null { + try { + const secret = getSessionSecret(); + const decoded = Buffer.from(token, 'base64').toString('utf-8'); + const lastDot = decoded.lastIndexOf('.'); + if (lastDot === -1) return null; + + const payload = decoded.slice(0, lastDot); + const signature = decoded.slice(lastDot + 1); + + const hmac = crypto.createHmac('sha256', secret); + hmac.update(payload); + const expectedSignature = hmac.digest('hex'); + + if (signature.length !== expectedSignature.length) return null; + const signatureMatch = crypto.timingSafeEqual( + Buffer.from(signature), + Buffer.from(expectedSignature), + ); + if (!signatureMatch) return null; + + const parsed = JSON.parse(payload) as SessionPayload; + if (typeof parsed.ts !== 'number') return null; + if (Date.now() - parsed.ts > SESSION_COOKIE_MAX_AGE) return null; + return parsed; + } catch { + return null; + } +} diff --git a/backend/src/config/opensearch.client.ts b/backend/src/config/opensearch.client.ts new file mode 100644 index 00000000..bed76f8f --- /dev/null +++ b/backend/src/config/opensearch.client.ts @@ -0,0 +1,53 @@ +import { Client } from '@opensearch-project/opensearch'; +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; + +@Injectable() +export class OpenSearchClient implements OnModuleInit { + private readonly logger = new Logger(OpenSearchClient.name); + private client: Client | null = null; + private isEnabled = false; + + constructor(private readonly configService: ConfigService) {} + + onModuleInit() { + this.initializeClient(); + } + + private initializeClient() { + const url = this.configService.get('opensearch.url'); + const username = this.configService.get('opensearch.username'); + const password = this.configService.get('opensearch.password'); + + if (!url) { + this.logger.warn( + 'šŸ” OpenSearch client not configured - OPENSEARCH_URL not set. Security analytics indexing disabled.', + ); + return; + } + + try { + this.client = new Client({ + node: url, + auth: username && password ? { username, password } : undefined, + ssl: { + rejectUnauthorized: process.env.NODE_ENV === 'production', + }, + }); + + this.isEnabled = true; + this.logger.log(`šŸ” OpenSearch client initialized - Connected to ${url}`); + } catch (error) { + this.logger.error(`Failed to initialize OpenSearch client: ${error}`); + this.isEnabled = false; + } + } + + getClient(): Client | null { + return this.client; + } + + isClientEnabled(): boolean { + return this.isEnabled && this.client !== null; + } +} diff --git a/backend/src/config/opensearch.config.ts b/backend/src/config/opensearch.config.ts new file mode 100644 index 00000000..b031ad3d --- /dev/null +++ b/backend/src/config/opensearch.config.ts @@ -0,0 +1,13 @@ +import { registerAs } from '@nestjs/config'; + +export interface OpenSearchConfig { + url: string | null; + username: string | null; + password: string | null; +} + +export const opensearchConfig = registerAs('opensearch', () => ({ + url: process.env.OPENSEARCH_URL ?? null, + username: process.env.OPENSEARCH_USERNAME ?? null, + password: process.env.OPENSEARCH_PASSWORD ?? null, +})); diff --git a/backend/src/config/opensearch.module.ts b/backend/src/config/opensearch.module.ts new file mode 100644 index 00000000..b4db4b84 --- /dev/null +++ b/backend/src/config/opensearch.module.ts @@ -0,0 +1,9 @@ +import { Module, Global } from '@nestjs/common'; +import { OpenSearchClient } from './opensearch.client'; + +@Global() +@Module({ + providers: [OpenSearchClient], + exports: [OpenSearchClient], +}) +export class OpenSearchModule {} diff --git a/backend/src/database/migration.guard.ts b/backend/src/database/migration.guard.ts index e321a2db..773260b1 100644 --- a/backend/src/database/migration.guard.ts +++ b/backend/src/database/migration.guard.ts @@ -8,6 +8,7 @@ const REQUIRED_TABLES = [ 'artifacts', 'workflow_log_streams', 'workflow_traces', + 'organization_settings', ]; @Injectable() diff --git a/backend/src/database/schema/index.ts b/backend/src/database/schema/index.ts index 32f11f65..6d985721 100644 --- a/backend/src/database/schema/index.ts +++ b/backend/src/database/schema/index.ts @@ -19,3 +19,4 @@ export * from './agent-trace-events'; export * from './mcp-servers'; export * from './node-io'; +export * from './organization-settings'; diff --git a/backend/src/database/schema/organization-settings.ts b/backend/src/database/schema/organization-settings.ts new file mode 100644 index 00000000..b6dd7f14 --- /dev/null +++ b/backend/src/database/schema/organization-settings.ts @@ -0,0 +1,17 @@ +import { integer, pgTable, timestamp, varchar } from 'drizzle-orm/pg-core'; + +export type SubscriptionTier = 'free' | 'pro' | 'enterprise'; + +export const organizationSettingsTable = pgTable('organization_settings', { + organizationId: varchar('organization_id', { length: 191 }).primaryKey(), + subscriptionTier: varchar('subscription_tier', { length: 50 }) + .$type() + .notNull() + .default('free'), + analyticsRetentionDays: integer('analytics_retention_days').notNull().default(30), + createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), + updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow().notNull(), +}); + +export type OrganizationSettings = typeof organizationSettingsTable.$inferSelect; +export type NewOrganizationSettings = typeof organizationSettingsTable.$inferInsert; diff --git a/backend/src/dsl/validator.ts b/backend/src/dsl/validator.ts index 6ec112a9..5df2437c 100644 --- a/backend/src/dsl/validator.ts +++ b/backend/src/dsl/validator.ts @@ -170,6 +170,10 @@ function isPlaceholderIssue(issue: ZodIssue, placeholderFields: Set): bo return true; case 'too_big': return true; + case 'invalid_value': + // Enum/literal validation fails on placeholder objects with missing fields + // The actual value from upstream will have the correct enum value at runtime + return true; case 'custom': // Custom validations (from .refine()) fail on placeholders but will pass at runtime // when the actual value comes from the connected edge diff --git a/backend/src/main.ts b/backend/src/main.ts index 9347d0a8..8c8c4236 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -3,6 +3,7 @@ import 'reflect-metadata'; import { NestFactory } from '@nestjs/core'; import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; import { cleanupOpenApiDoc } from 'nestjs-zod'; +import cookieParser from 'cookie-parser'; import { isVersionCheckDisabled, performVersionCheck } from './version-check'; @@ -14,6 +15,9 @@ async function bootstrap() { logger: ['log', 'error', 'warn'], }); + // Enable cookie parsing for session auth + app.use(cookieParser()); + // Set global prefix for all routes app.setGlobalPrefix('api/v1'); @@ -40,6 +44,7 @@ async function bootstrap() { app.enableCors({ origin: [ 'http://localhost', + 'http://localhost:80', 'http://localhost:8090', 'https://studio.shipsec.ai', ...instanceOrigins, @@ -52,6 +57,9 @@ async function bootstrap() { 'Accept', 'Cache-Control', 'x-organization-id', + 'X-Real-IP', + 'X-Forwarded-For', + 'X-Forwarded-Proto', ], }); const port = Number(process.env.PORT ?? 3211); diff --git a/backend/src/mcp/__tests__/mcp-internal.integration.spec.ts b/backend/src/mcp/__tests__/mcp-internal.integration.spec.ts index 8540c6b5..7967479f 100644 --- a/backend/src/mcp/__tests__/mcp-internal.integration.spec.ts +++ b/backend/src/mcp/__tests__/mcp-internal.integration.spec.ts @@ -8,6 +8,10 @@ import { AuthService } from '../../auth/auth.service'; import { AuthGuard } from '../../auth/auth.guard'; import { ApiKeysService } from '../../api-keys/api-keys.service'; import { AnalyticsService } from '../../analytics/analytics.service'; +import { OpenSearchTenantService } from '../../analytics/opensearch-tenant.service'; +import { SecurityAnalyticsService } from '../../analytics/security-analytics.service'; +import { OrganizationSettingsService } from '../../analytics/organization-settings.service'; +import { AnalyticsModule } from '../../analytics/analytics.module'; import { AgentTraceIngestService } from '../../agent-trace/agent-trace-ingest.service'; import { EventIngestService } from '../../events/event-ingest.service'; import { LogIngestService } from '../../logging/log-ingest.service'; @@ -68,6 +72,39 @@ describe('MCP Internal API (Integration)', () => { const moduleFixture: TestingModule = await Test.createTestingModule({ imports: [ConfigModule.forRoot({ isGlobal: true, ignoreEnvFile: true }), McpModule], }) + .overrideModule(AnalyticsModule) + .useModule( + class MockAnalyticsModule { + static providers = [ + { + provide: AnalyticsService, + useValue: { + isEnabled: () => false, + track: () => {}, + trackWorkflowStarted: () => {}, + trackWorkflowCompleted: () => {}, + trackApiCall: () => {}, + trackComponentExecuted: () => {}, + }, + }, + { + provide: OpenSearchTenantService, + useValue: { provisionTenant: async () => true }, + }, + { + provide: SecurityAnalyticsService, + useValue: { indexDocument: async () => {}, bulkIndexDocuments: async () => {} }, + }, + { + provide: OrganizationSettingsService, + useValue: { + getOrganizationSettings: async () => ({}), + updateOrganizationSettings: async () => ({}), + }, + }, + ]; + }, + ) .overrideProvider(NodeIOIngestService) .useValue({ onModuleInit: async () => {}, @@ -96,15 +133,6 @@ describe('MCP Internal API (Integration)', () => { }) .overrideProvider(McpGatewayService) .useValue(mockGatewayService) - .overrideProvider(AnalyticsService) - .useValue({ - isEnabled: () => false, - track: () => {}, - trackWorkflowStarted: () => {}, - trackWorkflowCompleted: () => {}, - trackApiCall: () => {}, - trackComponentExecuted: () => {}, - }) .overrideProvider(AuthService) .useValue({ authenticate: async () => { diff --git a/backend/src/workflows/workflows.controller.ts b/backend/src/workflows/workflows.controller.ts index f0f9c41d..b6e31f86 100644 --- a/backend/src/workflows/workflows.controller.ts +++ b/backend/src/workflows/workflows.controller.ts @@ -303,6 +303,7 @@ export class WorkflowsController { properties: { id: { type: 'string' }, workflowId: { type: 'string' }, + organizationId: { type: 'string' }, status: { type: 'string', enum: [ @@ -372,6 +373,7 @@ export class WorkflowsController { properties: { id: { type: 'string' }, workflowId: { type: 'string' }, + organizationId: { type: 'string' }, status: { type: 'string', enum: [ diff --git a/backend/src/workflows/workflows.service.ts b/backend/src/workflows/workflows.service.ts index 39768c82..0ac9191f 100644 --- a/backend/src/workflows/workflows.service.ts +++ b/backend/src/workflows/workflows.service.ts @@ -8,6 +8,7 @@ import { BadRequestException, } from '@nestjs/common'; import { status as grpcStatus, type ServiceError } from '@grpc/grpc-js'; +import { WorkflowNotFoundError } from '@temporalio/client'; import { z } from 'zod'; import { compileWorkflowGraph } from '../dsl/compiler'; @@ -65,6 +66,7 @@ export interface WorkflowRunHandle { export interface WorkflowRunSummary { id: string; workflowId: string; + organizationId: string; workflowVersionId: string | null; workflowVersion: number | null; status: ExecutionStatus; @@ -562,11 +564,12 @@ export class WorkflowsService { const graph = (version?.graph ?? workflow?.graph) as { nodes?: unknown[] } | undefined; const nodeCount = graph?.nodes && Array.isArray(graph.nodes) ? graph.nodes.length : 0; - const eventCount = await this.traceRepository.countByType( - run.runId, - 'NODE_STARTED', - organizationId, - ); + // Get trace event counts for status inference + const [startedActions, completedActions, failedActions] = await Promise.all([ + this.traceRepository.countByType(run.runId, 'NODE_STARTED', organizationId), + this.traceRepository.countByType(run.runId, 'NODE_COMPLETED', organizationId), + this.traceRepository.countByType(run.runId, 'NODE_FAILED', organizationId), + ]); // Calculate duration from events (more accurate than createdAt/updatedAt) const eventTimeRange = await this.traceRepository.getEventTimeRange(run.runId, organizationId); @@ -583,22 +586,19 @@ export class WorkflowsService { }); currentStatus = this.normalizeStatus(status.status); } catch (error) { - // If Temporal can't find the workflow (NOT_FOUND), check if events have stopped - // If events stopped more than 5 minutes ago, assume the workflow completed - const isNotFound = this.isNotFoundError(error); - if (isNotFound && eventTimeRange.lastTimestamp) { - const lastEventTime = new Date(eventTimeRange.lastTimestamp); - const minutesSinceLastEvent = (Date.now() - lastEventTime.getTime()) / (1000 * 60); - if (minutesSinceLastEvent > 5) { - // Events stopped more than 5 minutes ago and Temporal can't find it - // Assume the workflow completed successfully - currentStatus = 'COMPLETED'; - this.logger.log( - `Run ${run.runId} not found in Temporal but last event was ${minutesSinceLastEvent.toFixed(1)} minutes ago, assuming COMPLETED`, - ); - } else { - this.logger.warn(`Failed to get status for run ${run.runId}: ${error}`); - } + // If Temporal can't find the workflow, infer status from trace events + if (this.isNotFoundError(error)) { + currentStatus = this.inferStatusFromTraceEvents({ + runId: run.runId, + totalActions: run.totalActions ?? nodeCount, + completedActions, + failedActions, + startedActions, + }); + this.logger.log( + `Run ${run.runId} not found in Temporal, inferred status: ${currentStatus} ` + + `(started=${startedActions}, completed=${completedActions}, failed=${failedActions})`, + ); } else { this.logger.warn(`Failed to get status for run ${run.runId}: ${error}`); } @@ -615,6 +615,7 @@ export class WorkflowsService { return { id: run.runId, workflowId: run.workflowId, + organizationId, workflowVersionId: run.workflowVersionId ?? null, workflowVersion: run.workflowVersion ?? null, status: currentStatus, @@ -622,7 +623,7 @@ export class WorkflowsService { endTime: run.updatedAt ?? null, temporalRunId: run.temporalRunId ?? undefined, workflowName, - eventCount, + eventCount: startedActions, nodeCount, duration, triggerType, @@ -1068,19 +1069,59 @@ export class WorkflowsService { this.logger.log( `Fetching status for workflow run ${runId} (temporalRunId=${temporalRunId ?? 'latest'})`, ); - const temporalStatus = await this.temporalService.describeWorkflow({ - workflowId: runId, - runId: temporalRunId, - }); const { organizationId, run } = await this.requireRunAccess(runId, auth); + let temporalStatus: Awaited>; let completedActions = 0; + let failedActions = 0; + let startedActions = 0; + + // Pre-fetch trace event counts for status inference if (run.totalActions && run.totalActions > 0) { - completedActions = await this.traceRepository.countByType( - runId, - 'NODE_COMPLETED', - organizationId, - ); + [completedActions, failedActions, startedActions] = await Promise.all([ + this.traceRepository.countByType(runId, 'NODE_COMPLETED', organizationId), + this.traceRepository.countByType(runId, 'NODE_FAILED', organizationId), + this.traceRepository.countByType(runId, 'NODE_STARTED', organizationId), + ]); + } + + try { + temporalStatus = await this.temporalService.describeWorkflow({ + workflowId: runId, + runId: temporalRunId, + }); + } catch (error) { + // If Temporal can't find the workflow, infer status from trace events + if (this.isNotFoundError(error)) { + const inferredStatus = this.inferStatusFromTraceEvents({ + runId, + totalActions: run.totalActions ?? 0, + completedActions, + failedActions, + startedActions, + }); + + this.logger.log( + `Workflow ${runId} not found in Temporal, inferred status: ${inferredStatus} ` + + `(started=${startedActions}, completed=${completedActions}, failed=${failedActions}, total=${run.totalActions})`, + ); + + temporalStatus = { + workflowId: runId, + runId: temporalRunId ?? runId, + // Cast to WorkflowExecutionStatusName - normalizeStatus handles mapping + status: inferredStatus as unknown as typeof temporalStatus.status, + startTime: run.createdAt.toISOString(), + // Only set closeTime for terminal states that actually ran + closeTime: ['COMPLETED', 'FAILED'].includes(inferredStatus) + ? new Date().toISOString() + : undefined, + historyLength: 0, + taskQueue: '', + }; + } else { + throw error; + } } const statusPayload = this.mapTemporalStatus(runId, temporalStatus, run, completedActions); @@ -1535,15 +1576,66 @@ export class WorkflowsService { } } - private isNotFoundError(error: unknown): error is ServiceError { + private isNotFoundError(error: unknown): boolean { if (!error || typeof error !== 'object') { return false; } + // Check for Temporal WorkflowNotFoundError + if (error instanceof WorkflowNotFoundError) { + return true; + } + + // Check for gRPC NOT_FOUND error const serviceError = error as ServiceError; return serviceError.code === grpcStatus.NOT_FOUND; } + /** + * Infer workflow status from trace events when Temporal workflow is not found. + * + * Cases: + * - No started events ā†’ STALE (orphaned record - run exists but never executed) + * - All nodes completed ā†’ COMPLETED + * - Any node failed ā†’ FAILED + * - Partial completion (some started, not all finished) ā†’ FAILED (crashed/lost) + */ + private inferStatusFromTraceEvents(params: { + runId: string; + totalActions: number; + completedActions: number; + failedActions: number; + startedActions: number; + }): ExecutionStatus { + const { totalActions, completedActions, failedActions, startedActions } = params; + + // Case 1: No events at all - orphaned record (DB/Temporal mismatch) + // This indicates data inconsistency - run record exists but workflow never executed + if (startedActions === 0) { + return 'STALE'; + } + + // Case 2: Any node failed explicitly + if (failedActions > 0) { + return 'FAILED'; + } + + // Case 3: All nodes completed successfully + if (totalActions > 0 && completedActions >= totalActions) { + return 'COMPLETED'; + } + + // Case 4: Some nodes started but not all completed and no failures + // This means the workflow crashed or was lost - treat as FAILED + if (startedActions > 0 && completedActions < totalActions) { + return 'FAILED'; + } + + // Fallback: we have events but can't determine status + // This shouldn't happen normally, but default to FAILED for safety + return 'FAILED'; + } + private buildFailure(status: ExecutionStatus, failure?: unknown): FailureSummary | undefined { if (!['FAILED', 'TERMINATED', 'TIMED_OUT'].includes(status)) { return undefined; diff --git a/bun.lock b/bun.lock index 4538ff20..368f5dfe 100644 --- a/bun.lock +++ b/bun.lock @@ -33,12 +33,15 @@ "@clerk/backend": "^2.29.5", "@clerk/types": "^4.101.13", "@grpc/grpc-js": "^1.14.3", + "@nest-lab/throttler-storage-redis": "^1.1.0", "@nestjs/common": "^10.4.22", "@nestjs/config": "^3.3.0", "@nestjs/core": "^10.4.22", "@nestjs/microservices": "^11.1.13", "@nestjs/platform-express": "^10.4.22", "@nestjs/swagger": "^11.2.5", + "@nestjs/throttler": "^6.5.0", + "@opensearch-project/opensearch": "^3.5.1", "@shipsec/backend-client": "workspace:*", "@shipsec/component-sdk": "workspace:*", "@shipsec/shared": "workspace:*", @@ -75,6 +78,7 @@ "@eslint/js": "^9.39.2", "@nestjs/testing": "^10.4.22", "@types/bcryptjs": "^3.0.0", + "@types/cookie-parser": "^1.4.10", "@types/express-serve-static-core": "^4.19.8", "@types/har-format": "^1.2.16", "@types/multer": "^2.0.0", @@ -254,6 +258,7 @@ "@grpc/grpc-js": "^1.14.3", "@modelcontextprotocol/sdk": "^1.25.1", "@okta/okta-sdk-nodejs": "^7.3.0", + "@opensearch-project/opensearch": "^3.5.1", "@shipsec/component-sdk": "*", "@shipsec/contracts": "*", "@shipsec/shared": "*", @@ -641,6 +646,8 @@ "@monaco-editor/react": ["@monaco-editor/react@4.7.0", "", { "dependencies": { "@monaco-editor/loader": "^1.5.0" }, "peerDependencies": { "monaco-editor": ">= 0.25.0 < 1", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-cyzXQCtO47ydzxpQtCGSQGOC8Gk3ZUeBXFAxD+CWXYFo5OqZyZUonFl0DwUlTyAfRHntBfw2p3w4s9R6oe1eCA=="], + "@nest-lab/throttler-storage-redis": ["@nest-lab/throttler-storage-redis@1.2.0", "", { "dependencies": { "tslib": "^2.3.0" }, "peerDependencies": { "@nestjs/common": "^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0 || ^11.0.0", "@nestjs/core": "^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0 || ^11.0.0", "@nestjs/throttler": ">=6.0.0", "ioredis": ">=5.0.0", "reflect-metadata": "^0.2.1" } }, "sha512-tMkUyo68NCKTR+zILk+EC35SMYBtDPZY2mCj7ZaCietWGVTnuP4zwq9ERYfvU6kJv6h8teNZrC6MJCmY6/dljw=="], + "@nestjs/common": ["@nestjs/common@10.4.22", "", { "dependencies": { "file-type": "20.4.1", "iterare": "1.2.1", "tslib": "2.8.1", "uid": "2.0.2" }, "peerDependencies": { "class-transformer": "*", "class-validator": "*", "reflect-metadata": "^0.1.12 || ^0.2.0", "rxjs": "^7.1.0" }, "optionalPeers": ["class-transformer", "class-validator"] }, "sha512-fxJ4v85nDHaqT1PmfNCQ37b/jcv2OojtXTaK1P2uAXhzLf9qq6WNUOFvxBrV4fhQek1EQoT1o9oj5xAZmv3NRw=="], "@nestjs/config": ["@nestjs/config@3.3.0", "", { "dependencies": { "dotenv": "16.4.5", "dotenv-expand": "10.0.0", "lodash": "4.17.21" }, "peerDependencies": { "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", "rxjs": "^7.1.0" } }, "sha512-pdGTp8m9d0ZCrjTpjkUbZx6gyf2IKf+7zlkrPNMsJzYZ4bFRRTpXrnj+556/5uiI6AfL5mMrJc2u7dB6bvM+VA=="], @@ -657,6 +664,8 @@ "@nestjs/testing": ["@nestjs/testing@10.4.22", "", { "dependencies": { "tslib": "2.8.1" }, "peerDependencies": { "@nestjs/common": "^10.0.0", "@nestjs/core": "^10.0.0", "@nestjs/microservices": "^10.0.0", "@nestjs/platform-express": "^10.0.0" }, "optionalPeers": ["@nestjs/microservices", "@nestjs/platform-express"] }, "sha512-HO9aPus3bAedAC+jKVAA8jTdaj4fs5M9fing4giHrcYV2txe9CvC1l1WAjwQ9RDhEHdugjY4y+FZA/U/YqPZrA=="], + "@nestjs/throttler": ["@nestjs/throttler@6.5.0", "", { "peerDependencies": { "@nestjs/common": "^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0 || ^11.0.0", "@nestjs/core": "^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0 || ^11.0.0", "reflect-metadata": "^0.1.13 || ^0.2.0" } }, "sha512-9j0ZRfH0QE1qyrj9JjIRDz5gQLPqq9yVC2nHsrosDVAfI5HHw08/aUAWx9DZLSdQf4HDkmhTTEGLrRFHENvchQ=="], + "@noble/hashes": ["@noble/hashes@1.8.0", "", {}, "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A=="], "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], @@ -669,6 +678,8 @@ "@okta/okta-sdk-nodejs": ["@okta/okta-sdk-nodejs@7.3.0", "", { "dependencies": { "@types/node-forge": "^1.3.1", "deep-copy": "^1.4.2", "eckles": "^1.4.1", "form-data": "^4.0.4", "https-proxy-agent": "^5.0.0", "js-yaml": "^4.1.0", "lodash": "^4.17.20", "njwt": "^2.0.1", "node-fetch": "^2.6.7", "node-jose": "^2.2.0", "parse-link-header": "^2.0.0", "rasha": "^1.2.5", "safe-flat": "^2.0.2", "url-parse": "^1.5.10", "uuid": "^11.1.0" } }, "sha512-6J3VV+8fBOqIXDqb3t2sBeXj1WOEZL6wP2AcGRzvMRMb2WL7JKR6ZDrt/1Kk7j4seXCKMpZrHsPYYdfRXwkSKQ=="], + "@opensearch-project/opensearch": ["@opensearch-project/opensearch@3.5.1", "", { "dependencies": { "aws4": "^1.11.0", "debug": "^4.3.1", "hpagent": "^1.2.0", "json11": "^2.0.0", "ms": "^2.1.3", "secure-json-parse": "^2.4.0" } }, "sha512-6bf+HcuERzAtHZxrm6phjref54ABse39BpkDie/YO3AUFMCBrb3SK5okKSdT5n3+nDRuEEQLhQCl0RQV3s1qpA=="], + "@opentelemetry/api": ["@opentelemetry/api@1.9.0", "", {}, "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="], "@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.208.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg=="], @@ -1085,6 +1096,8 @@ "@types/connect": ["@types/connect@3.4.38", "", { "dependencies": { "@types/node": "*" } }, "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug=="], + "@types/cookie-parser": ["@types/cookie-parser@1.4.10", "", { "peerDependencies": { "@types/express": "*" } }, "sha512-B4xqkqfZ8Wek+rCOeRxsjMS9OgvzebEzzLYw7NHYuvzb7IdxOkI0ZHGgeEBX4PUM7QGVvNSK60T3OvWj3YfBRg=="], + "@types/cookiejar": ["@types/cookiejar@2.1.5", "", {}, "sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q=="], "@types/d3": ["@types/d3@7.4.3", "", { "dependencies": { "@types/d3-array": "*", "@types/d3-axis": "*", "@types/d3-brush": "*", "@types/d3-chord": "*", "@types/d3-color": "*", "@types/d3-contour": "*", "@types/d3-delaunay": "*", "@types/d3-dispatch": "*", "@types/d3-drag": "*", "@types/d3-dsv": "*", "@types/d3-ease": "*", "@types/d3-fetch": "*", "@types/d3-force": "*", "@types/d3-format": "*", "@types/d3-geo": "*", "@types/d3-hierarchy": "*", "@types/d3-interpolate": "*", "@types/d3-path": "*", "@types/d3-polygon": "*", "@types/d3-quadtree": "*", "@types/d3-random": "*", "@types/d3-scale": "*", "@types/d3-scale-chromatic": "*", "@types/d3-selection": "*", "@types/d3-shape": "*", "@types/d3-time": "*", "@types/d3-time-format": "*", "@types/d3-timer": "*", "@types/d3-transition": "*", "@types/d3-zoom": "*" } }, "sha512-lZXZ9ckh5R8uiFVt8ogUNf+pIrK4EsWrx2Np75WvF/eTpJ0FMHNhjXk8CKEx/+gpHbNQyJWehbFaTvqmHWB3ww=="], @@ -1389,6 +1402,8 @@ "available-typed-arrays": ["available-typed-arrays@1.0.7", "", { "dependencies": { "possible-typed-array-names": "^1.0.0" } }, "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ=="], + "aws4": ["aws4@1.13.2", "", {}, "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw=="], + "bail": ["bail@2.0.2", "", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="], "balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], @@ -1949,6 +1964,8 @@ "hosted-git-info": ["hosted-git-info@4.1.0", "", { "dependencies": { "lru-cache": "^6.0.0" } }, "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA=="], + "hpagent": ["hpagent@1.2.0", "", {}, "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA=="], + "html-encoding-sniffer": ["html-encoding-sniffer@6.0.0", "", { "dependencies": { "@exodus/bytes": "^1.6.0" } }, "sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg=="], "html-url-attributes": ["html-url-attributes@3.0.1", "", {}, "sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ=="], @@ -2113,6 +2130,8 @@ "json-stringify-safe": ["json-stringify-safe@5.0.1", "", {}, "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA=="], + "json11": ["json11@2.0.2", "", { "bin": { "json11": "dist/cli.mjs" } }, "sha512-HIrd50UPYmP6sqLuLbFVm75g16o0oZrVfxrsY0EEys22klz8mRoWlX9KAEDOSOR9Q34rcxsyC8oDveGrCz5uLQ=="], + "json5": ["json5@2.2.3", "", { "bin": { "json5": "lib/cli.js" } }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="], "jsx-ast-utils": ["jsx-ast-utils@3.3.5", "", { "dependencies": { "array-includes": "^3.1.6", "array.prototype.flat": "^1.3.1", "object.assign": "^4.1.4", "object.values": "^1.1.6" } }, "sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ=="], @@ -2325,7 +2344,7 @@ "mqtt-packet": ["mqtt-packet@9.0.2", "", { "dependencies": { "bl": "^6.0.8", "debug": "^4.3.4", "process-nextick-args": "^2.0.1" } }, "sha512-MvIY0B8/qjq7bKxdN1eD+nrljoeaai+qjLJgfRn3TiMuz0pamsIWY2bFODPZMSNmabsLANXsLl4EMoWvlaTZWA=="], - "ms": ["ms@3.0.0-canary.1", "", {}, "sha512-kh8ARjh8rMN7Du2igDRO9QJnqCb2xYTJxyQYK7vJJS4TvLLmsbyhiKpSW+t+y26gyOyMd0riphX0GeWKU3ky5g=="], + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], "multer": ["multer@2.0.2", "", { "dependencies": { "append-field": "^1.0.0", "busboy": "^1.6.0", "concat-stream": "^2.0.0", "mkdirp": "^0.5.6", "object-assign": "^4.1.1", "type-is": "^1.6.18", "xtend": "^4.0.2" } }, "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw=="], @@ -2697,6 +2716,8 @@ "schema-utils": ["schema-utils@4.3.3", "", { "dependencies": { "@types/json-schema": "^7.0.9", "ajv": "^8.9.0", "ajv-formats": "^2.1.1", "ajv-keywords": "^5.1.0" } }, "sha512-eflK8wEtyOE6+hsaRVPxvUKYCpRgzLqDTb8krvAsRIwOGlHoSgYLgBXoubGgLd2fT41/OUYdb48v4k4WWHQurA=="], + "secure-json-parse": ["secure-json-parse@2.7.0", "", {}, "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="], + "semver": ["semver@7.7.2", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA=="], "send": ["send@1.2.1", "", { "dependencies": { "debug": "^4.4.3", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", "http-errors": "^2.0.1", "mime-types": "^3.0.2", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", "statuses": "^2.0.2" } }, "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ=="], @@ -3227,6 +3248,8 @@ "@shipsec/studio-worker/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], + "@temporalio/common/ms": ["ms@3.0.0-canary.1", "", {}, "sha512-kh8ARjh8rMN7Du2igDRO9QJnqCb2xYTJxyQYK7vJJS4TvLLmsbyhiKpSW+t+y26gyOyMd0riphX0GeWKU3ky5g=="], + "@temporalio/worker/supports-color": ["supports-color@8.1.1", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q=="], "@testing-library/dom/aria-query": ["aria-query@5.3.0", "", { "dependencies": { "dequal": "^2.0.3" } }, "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A=="], @@ -3297,8 +3320,6 @@ "data-urls/whatwg-mimetype": ["whatwg-mimetype@5.0.0", "", {}, "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw=="], - "debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], - "decamelize-keys/decamelize": ["decamelize@1.2.0", "", {}, "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA=="], "decamelize-keys/map-obj": ["map-obj@1.0.1", "", {}, "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg=="], @@ -3433,8 +3454,6 @@ "schema-utils/ajv-formats": ["ajv-formats@2.1.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA=="], - "send/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], - "slice-ansi/ansi-styles": ["ansi-styles@6.2.3", "", {}, "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg=="], "source-map-loader/iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], @@ -3581,20 +3600,14 @@ "@pm2/agent/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], - "@pm2/agent/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], - "@pm2/agent/semver/lru-cache": ["lru-cache@6.0.0", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA=="], "@pm2/io/async/lodash": ["lodash@4.17.23", "", {}, "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w=="], - "@pm2/io/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], - "@pm2/io/semver/lru-cache": ["lru-cache@6.0.0", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA=="], "@pm2/js-api/async/lodash": ["lodash@4.17.23", "", {}, "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w=="], - "@pm2/js-api/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], - "@redocly/openapi-core/minimatch/brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="], "@shipsec/component-sdk/@types/node/undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], @@ -3717,8 +3730,6 @@ "multer/type-is/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], - "needle/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], - "node-fetch/whatwg-url/tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="], "node-fetch/whatwg-url/webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], @@ -3757,8 +3768,6 @@ "@nestjs/platform-express/express/send/mime": ["mime@1.6.0", "", { "bin": { "mime": "cli.js" } }, "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="], - "@nestjs/platform-express/express/send/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], - "@nestjs/platform-express/express/type-is/media-typer": ["media-typer@0.3.0", "", {}, "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ=="], "@nestjs/platform-express/express/type-is/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], diff --git a/docker/PRODUCTION.md b/docker/PRODUCTION.md new file mode 100644 index 00000000..dd5908d0 --- /dev/null +++ b/docker/PRODUCTION.md @@ -0,0 +1,223 @@ +# Production Deployment Guide + +This guide covers deploying the analytics infrastructure with security and SaaS multitenancy enabled. + +## Overview + +| Environment | Security | Multitenancy | Use Case | +|-------------|----------|--------------|----------| +| Development | Disabled | No | Local development, fast iteration | +| Production | Enabled | Yes (Strict) | Multi-tenant SaaS deployment | + +## SaaS Multitenancy Model + +**Key Principles:** +- Each customer gets complete data isolation by default +- No shared dashboards - sharing is explicitly opt-in +- Each customer has their own index pattern (`{customer_id}-*`) +- Tenants, roles, and users are created dynamically via backend + +**Index Naming Convention:** +``` +{customer_id}-analytics-* # Analytics data +{customer_id}-workflows-* # Workflow results +{customer_id}-scans-* # Scan results +``` + +## Quick Start (Production) + +```bash +# 1. Generate TLS certificates +./scripts/generate-certs.sh + +# 2. Set required environment variables +export OPENSEARCH_ADMIN_PASSWORD="your-secure-admin-password" +export OPENSEARCH_DASHBOARDS_PASSWORD="your-secure-dashboards-password" + +# 3. Start with production configuration +docker compose -f docker-compose.infra.yml -f docker-compose.prod.yml up -d +``` + +## Files Overview + +| File | Purpose | +|------|---------| +| `docker-compose.infra.yml` | Base infrastructure (dev mode, PM2 on host) | +| `docker-compose.full.yml` | Full stack containerized (simple prod, no security) | +| `docker-compose.prod.yml` | Security overlay (combines with infra.yml for SaaS) | +| `nginx/nginx.dev.conf` | Nginx routing to host (PM2 services) | +| `nginx/nginx.prod.conf` | Nginx routing to containers | +| `opensearch-dashboards.yml` | Dashboards config (dev) | +| `opensearch-dashboards.prod.yml` | Dashboards config (prod with multitenancy) | +| `scripts/generate-certs.sh` | TLS certificate generator | +| `opensearch-security/` | Security plugin configuration | +| `certs/` | Generated certificates (gitignored) | + +See [README.md](README.md) for detailed usage of each compose file. + +## Customer Provisioning (Backend Integration) + +When a new customer is onboarded, the backend must create: + +### 1. Create Customer Tenant +```bash +PUT /_plugins/_security/api/tenants/{customer_id} +{ + "description": "Tenant for customer {customer_id}" +} +``` + +### 2. Create Customer Role (with Index Isolation) +```bash +PUT /_plugins/_security/api/roles/customer_{customer_id}_rw +{ + "cluster_permissions": ["cluster_composite_ops_ro"], + "index_permissions": [{ + "index_patterns": ["{customer_id}-*"], + "allowed_actions": ["read", "write", "create_index", "indices:data/read/*", "indices:data/write/*"] + }], + "tenant_permissions": [{ + "tenant_patterns": ["{customer_id}"], + "allowed_actions": ["kibana_all_write"] + }] +} +``` + +### 3. Create Customer User +```bash +PUT /_plugins/_security/api/internalusers/{user_email} +{ + "password": "hashed_password", + "backend_roles": ["customer_{customer_id}"], + "attributes": { + "customer_id": "{customer_id}", + "email": "{user_email}" + } +} +``` + +### 4. Map User to Role +```bash +PUT /_plugins/_security/api/rolesmapping/customer_{customer_id}_rw +{ + "users": ["{user_email}"], + "backend_roles": ["customer_{customer_id}"] +} +``` + +## Security Configuration + +### TLS Certificates + +The `scripts/generate-certs.sh` script generates: + +- **root-ca.pem** - Root certificate authority +- **node.pem / node-key.pem** - OpenSearch node certificate +- **admin.pem / admin-key.pem** - Admin certificate for cluster management + +For production: +- Use a proper CA (Let's Encrypt, internal PKI) +- Store private keys in a secrets manager (Vault, AWS Secrets Manager) +- Set up certificate rotation before expiration + +### System Users + +Only two system users are defined (in `internal_users.yml`): + +| User | Purpose | +|------|---------| +| `admin` | Platform operations - DO NOT give to customers | +| `kibanaserver` | Dashboards backend communication | + +Customer users are created dynamically via the Security REST API. + +### Password Hashing + +Generate password hashes for users: +```bash +docker run -it opensearchproject/opensearch:2.11.1 \ + /usr/share/opensearch/plugins/opensearch-security/tools/hash.sh -p YOUR_PASSWORD +``` + +## Data Isolation Verification + +After setting up a customer, verify isolation: + +```bash +# As customer user - should only see their data +curl -u user@customer.com:password \ + "https://localhost:9200/{customer_id}-*/_search" + +# Should NOT be able to access other customer's data (403 Forbidden) +curl -u user@customer.com:password \ + "https://localhost:9200/other_customer-*/_search" +``` + +## Environment Variables + +| Variable | Required | Description | +|----------|----------|-------------| +| `OPENSEARCH_ADMIN_PASSWORD` | Yes | Admin user password | +| `OPENSEARCH_DASHBOARDS_PASSWORD` | Yes | kibanaserver user password | + +## Updating Security Configuration + +After modifying security files, apply changes: + +```bash +docker exec -it shipsec-opensearch \ + /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh \ + -cd /usr/share/opensearch/config/opensearch-security \ + -icl -nhnv \ + -cacert /usr/share/opensearch/config/certs/root-ca.pem \ + -cert /usr/share/opensearch/config/certs/admin.pem \ + -key /usr/share/opensearch/config/certs/admin-key.pem +``` + +## Troubleshooting + +### Container fails to start + +Check logs: +```bash +docker logs shipsec-opensearch +docker logs shipsec-opensearch-dashboards +``` + +Common issues: +- Certificate permissions (should be 600 for keys, 644 for certs) +- Missing environment variables +- Incorrect certificate paths + +### Cannot connect to secured cluster + +```bash +# Test with curl +curl -k -u admin:PASSWORD https://localhost:9200/_cluster/health +``` + +### Customer cannot see their dashboards + +1. Verify tenant was created for customer +2. Check user has correct backend_roles +3. Verify role has correct tenant_permissions +4. Check index pattern matches customer's indices + +### Cross-tenant data leak + +If a customer can see another customer's data: +1. Verify index_patterns in role are correctly scoped to `{customer_id}-*` +2. Check role mapping is correct +3. Ensure user's backend_roles match their customer ID + +## Switching Between Environments + +**Development (no security):** +```bash +docker compose -f docker-compose.infra.yml up -d +``` + +**Production (with security):** +```bash +docker compose -f docker-compose.infra.yml -f docker-compose.prod.yml up -d +``` diff --git a/docker/README.md b/docker/README.md index a0b7bce2..126191f8 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,73 +1,159 @@ -# MCP Docker Images +# Docker Configuration -This directory contains Docker images for the MCP (Model Context Protocol) Library implementation. +This directory contains Docker Compose configurations for running ShipSec Studio in different environments. -## Overview +## Docker Compose Files -The MCP Docker images provide a standardized way to deploy MCP servers with a built-in stdio proxy. Each image includes: +| File | Purpose | When to Use | +| -------------------------- | ---------------------------- | ------------------------------------------------------ | +| `docker-compose.infra.yml` | Infrastructure services only | Development with PM2 (frontend/backend on host) | +| `docker-compose.full.yml` | Full stack in containers | Self-hosted deployment, all services containerized | +| `docker-compose.prod.yml` | Security overlay | Production SaaS with multitenancy (overlays infra.yml) | -1. **Base stdio proxy** (`mcp-stdio-proxy`): A Node.js HTTP server that acts as a bridge between MCP stdio servers and HTTP clients -2. **Provider suites** (`mcp-aws-suite`): Bundled MCP servers for specific cloud providers +## Environment Modes -## Quick Start with Docker Compose - -Start all services: +### Development Mode (`just dev`) ```bash -cd docker -docker-compose up -d +just dev ``` -Check health of services: +- **Compose file**: `docker-compose.infra.yml` +- **Frontend/Backend**: Run via PM2 on host machine +- **Infrastructure**: Runs in Docker (Postgres, Redis, Temporal, OpenSearch, etc.) +- **Nginx**: Uses `nginx.dev.conf` pointing to `host.docker.internal` +- **Security**: Disabled for fast iteration + +**Access:** + +- Frontend: http://localhost:5173 +- Backend: http://localhost:3211 +- Analytics: http://localhost:5601/analytics/ + +### Production Mode (`just prod`) ```bash -curl http://localhost:8080/health # stdio proxy -curl http://localhost:8081/health # AWS suite +just prod ``` -## Individual Services +- **Compose file**: `docker-compose.full.yml` +- **All services**: Run as Docker containers +- **Nginx**: Unified entry point on port 80 +- **Security**: Disabled (simple deployment) -### stdio proxy +**Access (all via port 80):** -- Port: 8080 -- Purpose: Base HTTP to stdio proxy for MCP servers +- Frontend: http://localhost/ +- Backend API: http://localhost/api/ +- Analytics: http://localhost/analytics/ -### AWS Suite +**Nginx Routing (nginx.prod.conf):** -- Port: 8081 -- MCP servers: CloudTrail, CloudWatch, EC2, S3 -- Environment: `MCP_COMMAND`, `MCP_ARGS` +| Path | Target Container | Port | +| -------------- | --------------------- | ---- | +| `/analytics/*` | opensearch-dashboards | 5601 | +| `/api/*` | backend | 3211 | +| `/*` | frontend | 8080 | -## Customization +> **Note:** Frontend and backend containers only expose ports internally. All external traffic flows through nginx on port 80. -You can override the default MCP server using environment variables: +### Production Secure Mode (`just prod-secure`) ```bash -# Using docker-compose -docker-compose up -e MCP_COMMAND=awslabs.cloudwatch-mcp-server +just generate-certs +export OPENSEARCH_ADMIN_PASSWORD='secure-password' +export OPENSEARCH_DASHBOARDS_PASSWORD='secure-password' +just prod-secure +``` + +- **Compose files**: `docker-compose.infra.yml` + `docker-compose.prod.yml` (overlay) +- **Security**: TLS enabled, authentication required +- **Multitenancy**: Strict SaaS isolation per customer +- **Nginx**: Uses `nginx.prod.conf` with container networking + +**Access:** + +- Analytics: https://localhost/analytics (auth required) +- OpenSearch: https://localhost:9200 (TLS) + +## Nginx Configuration + +| File | Target Services | Use Case | +| ----------------------- | ------------------------------------------------------------- | ---------------------------------------- | +| `nginx/nginx.dev.conf` | `host.docker.internal:5173/3211` | Dev (PM2 on host) | +| `nginx/nginx.prod.conf` | `frontend:8080`, `backend:3211`, `opensearch-dashboards:5601` | Container mode (full stack + production) | + +### Routing Architecture -# Or with docker run -docker run -e MCP_COMMAND=github.github-issue-mcp-server shipsec/mcp-aws-suite:latest +All modes use nginx as a reverse proxy with unified routing: + +``` +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ Nginx (port 80/443) ā”‚ +ā”œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤ +ā”‚ /analytics/* ā†’ OpenSearch Dashboards:5601 ā”‚ +ā”‚ /api/* ā†’ Backend:3211 ā”‚ +ā”‚ /* ā†’ Frontend:8080 ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ``` -## Build Instructions +### OpenSearch Dashboards BasePath -Build individual images: +OpenSearch Dashboards is configured with `server.basePath: "/analytics"` to work behind nginx: -```bash -# Build stdio proxy -cd mcp-stdio-proxy -docker build -t shipsec/mcp-stdio-proxy:latest . +- Incoming requests: `/analytics/app/discover` ā†’ internally processed as `/app/discover` +- Outgoing URLs: Automatically prefixed with `/analytics` -# Build AWS suite -cd mcp-aws-suite -docker build -t shipsec/mcp-aws-suite:latest . +## Analytics Pipeline +The worker service writes analytics data to OpenSearch via the Analytics Sink component. + +**Required Environment Variable:** + +```yaml +OPENSEARCH_URL=http://opensearch:9200 ``` -## Authentication +This is pre-configured in `docker-compose.full.yml`. For detailed analytics documentation, see [docs/analytics.md](../docs/analytics.md). + +## Directory Structure + +``` +docker/ +ā”œā”€ā”€ docker-compose.infra.yml # Infrastructure (dev base) +ā”œā”€ā”€ docker-compose.full.yml # Full stack containerized +ā”œā”€ā”€ docker-compose.prod.yml # Security overlay for prod +ā”œā”€ā”€ nginx/ +ā”‚ ā”œā”€ā”€ nginx.dev.conf # Routes to host (PM2) +ā”‚ ā””ā”€ā”€ nginx.prod.conf # Routes to containers +ā”œā”€ā”€ opensearch-dashboards.yml # Dev dashboards config +ā”œā”€ā”€ opensearch-dashboards.prod.yml # Prod dashboards config +ā”œā”€ā”€ opensearch-security/ # Security plugin configs +ā”‚ ā”œā”€ā”€ internal_users.yml +ā”‚ ā”œā”€ā”€ roles.yml +ā”‚ ā”œā”€ā”€ roles_mapping.yml +ā”‚ ā””ā”€ā”€ tenants.yml +ā”œā”€ā”€ scripts/ +ā”‚ ā””ā”€ā”€ generate-certs.sh # TLS certificate generator +ā”œā”€ā”€ certs/ # Generated certs (gitignored) +ā”œā”€ā”€ PRODUCTION.md # Production deployment guide +ā””ā”€ā”€ README.md # This file +``` + +## Quick Reference + +| Command | Description | +| --------------------- | ------------------------------------------ | +| `just dev` | Start dev environment (PM2 + Docker infra) | +| `just dev stop` | Stop dev environment | +| `just prod` | Start full stack in Docker | +| `just prod stop` | Stop production | +| `just prod-secure` | Start with security & multitenancy | +| `just generate-certs` | Generate TLS certificates | +| `just infra up` | Start infrastructure only | +| `just help` | Show all available commands | -Each suite requires specific authentication: +## See Also -- **AWS Suite**: AWS credentials (access key, secret key, or credentials file) - See individual suite README for detailed authentication instructions. +- [PRODUCTION.md](PRODUCTION.md) - Detailed production deployment and customer provisioning guide +- [docs/analytics.md](../docs/analytics.md) - Analytics pipeline and OpenSearch configuration diff --git a/docker/SECURE-DEV-MODE.md b/docker/SECURE-DEV-MODE.md new file mode 100644 index 00000000..f20acccb --- /dev/null +++ b/docker/SECURE-DEV-MODE.md @@ -0,0 +1,126 @@ +# Secure Development Mode + +This document describes the secure development environment setup with OpenSearch Security enabled for multi-tenant isolation. + +## Overview + +The `just dev` command now starts the development environment with full OpenSearch Security enabled, matching the production security model. This provides: + +- **TLS encryption** for all OpenSearch communication +- **Multi-tenant isolation** - each organization's data is isolated +- **Authentication required** - no anonymous access +- **Same security model as production** - test security features locally + +## Quick Start + +```bash +# Start secure dev environment (recommended) +just dev + +# Start without security (faster, for quick iteration) +just dev-insecure +``` + +## Architecture + +### Docker Compose Files + +| File | Purpose | +|------|---------| +| `docker-compose.infra.yml` | Base infrastructure (Postgres, Redis, Temporal, etc.) | +| `docker-compose.dev-secure.yml` | Security overlay for development | +| `docker-compose.prod.yml` | Production security configuration | + +### Security Configuration Files + +Located in `docker/opensearch-security/`: + +| File | Purpose | +|------|---------| +| `config.yml` | Authentication/authorization backends (proxy auth) | +| `internal_users.yml` | System users (admin, kibanaserver, worker) | +| `roles.yml` | Role definitions with index permissions | +| `roles_mapping.yml` | User-to-role mappings | +| `action_groups.yml` | Permission groups for roles | +| `tenants.yml` | Tenant definitions | +| `audit.yml` | Audit logging configuration | + +### TLS Certificates + +Certificates are auto-generated on first run and stored in `docker/certs/`: + +- `root-ca.pem` / `root-ca-key.pem` - Certificate Authority +- `admin.pem` / `admin-key.pem` - Admin certificate for securityadmin tool +- `node.pem` / `node-key.pem` - OpenSearch node certificate + +## Default Credentials + +For development convenience, default passwords are set: + +| User | Password | Purpose | +|------|----------|---------| +| `admin` | `admin` | Platform administrator | +| `kibanaserver` | `admin` | Dashboards backend communication | +| `worker` | `admin` | Worker service for indexing | + +**Important**: Change these in production via environment variables: +- `OPENSEARCH_ADMIN_PASSWORD` +- `OPENSEARCH_DASHBOARDS_PASSWORD` + +## Multi-Tenant Isolation + +### How It Works + +1. **Index Pattern**: Each organization's data is stored in indices prefixed with their org ID: + - `security-findings-{org_id}-*` + +2. **Tenant Isolation**: OpenSearch Dashboards uses tenants to isolate saved objects (dashboards, visualizations) + +3. **Role-Based Access**: Dynamic roles are created per customer restricting access to their indices only + +### Dynamic Provisioning + +When a new customer is onboarded, the backend creates: +1. A tenant for their organization +2. A role with permissions scoped to their indices +3. User-to-role mappings + +## Troubleshooting + +### Check Container Health + +```bash +just dev status +docker logs shipsec-opensearch +docker logs shipsec-opensearch-dashboards +``` + +### Reset Security Configuration + +```bash +# Clean everything and restart +just dev clean && just dev + +# Or manually run securityadmin +docker exec shipsec-opensearch /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh \ + -cd /usr/share/opensearch/config/opensearch-security \ + -icl -nhnv \ + -cacert /usr/share/opensearch/config/certs/root-ca.pem \ + -cert /usr/share/opensearch/config/certs/admin.pem \ + -key /usr/share/opensearch/config/certs/admin-key.pem +``` + +### Regenerate Certificates + +```bash +rm -rf docker/certs +just generate-certs +just dev clean && just dev +``` + +## Changes from Previous Setup + +1. **`just dev`** now runs with security enabled (was insecure) +2. **`just dev-insecure`** is the new command for fast, insecure development +3. Certificates are auto-generated if missing +4. Environment variable `OPENSEARCH_SECURITY_ENABLED=true` is set for backend/worker diff --git a/docker/certs/.gitignore b/docker/certs/.gitignore new file mode 100644 index 00000000..5ae618b7 --- /dev/null +++ b/docker/certs/.gitignore @@ -0,0 +1,7 @@ +# Ignore generated certificates and private keys +# These should NEVER be committed to version control +*.pem +*.key +*.crt +*.csr +*.srl diff --git a/docker/docker-compose.dev-ports.yml b/docker/docker-compose.dev-ports.yml new file mode 100644 index 00000000..0da3bd05 --- /dev/null +++ b/docker/docker-compose.dev-ports.yml @@ -0,0 +1,54 @@ +# Development Ports Overlay +# +# WARNING: These ports bypass ALL nginx authentication! +# Use ONLY for local development where direct service access is needed. +# +# Usage: +# docker compose -f docker-compose.infra.yml -f docker-compose.dev-ports.yml up -d +# +# This overlay exposes all service ports on loopback (127.0.0.1) only, +# preventing external network access while allowing local development tools to connect. + +services: + postgres: + ports: + - "127.0.0.1:5433:5432" + + temporal: + ports: + - "127.0.0.1:7233:7233" + + temporal-ui: + ports: + - "127.0.0.1:8081:8080" + + minio: + ports: + - "127.0.0.1:9000:9000" + - "127.0.0.1:9001:9001" + + redis: + ports: + - "127.0.0.1:6379:6379" + + loki: + ports: + - "127.0.0.1:3100:3100" + + redpanda: + ports: + - "127.0.0.1:9092:9092" + - "127.0.0.1:9644:9644" + + redpanda-console: + ports: + - "127.0.0.1:8082:8080" + + opensearch: + ports: + - "127.0.0.1:9200:9200" + - "127.0.0.1:9600:9600" + + opensearch-dashboards: + ports: + - "127.0.0.1:5601:5601" diff --git a/docker/docker-compose.dev-secure.yml b/docker/docker-compose.dev-secure.yml new file mode 100644 index 00000000..1411e382 --- /dev/null +++ b/docker/docker-compose.dev-secure.yml @@ -0,0 +1,75 @@ +# Development Docker Compose with Security & Multitenancy +# +# Usage: +# docker compose -f docker-compose.infra.yml -f docker-compose.dev-secure.yml up -d +# +# This overlay enables OpenSearch Security plugin for development: +# - TLS encryption +# - Multi-tenant isolation +# - Same security model as production +# +# Requires: +# 1. TLS certificates in docker/certs/ (run: just generate-certs) +# 2. Environment variables (auto-set by just dev for convenience): +# - OPENSEARCH_ADMIN_PASSWORD +# - OPENSEARCH_DASHBOARDS_PASSWORD + +services: + opensearch: + # Custom entrypoint for proxy auth config templating + entrypoint: ["/usr/share/opensearch/config/opensearch-security/docker-entrypoint-security.sh"] + environment: + # Enable security plugin (override infra.yml settings) + - DISABLE_SECURITY_PLUGIN=false + - DISABLE_INSTALL_DEMO_CONFIG=true + # Admin password for healthcheck (default: admin) + - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-admin} + # Proxy auth - trusted proxy IP regex (Docker networks: 172.x, 192.168.x, 10.x) + # NOTE: Double backslashes required - sed consumes one level during config templating + - OPENSEARCH_INTERNAL_PROXIES=(172|192|10)\\.\\d+\\.\\d+\\.\\d+ + volumes: + - opensearch_data:/usr/share/opensearch/data + - ./certs:/usr/share/opensearch/config/certs:ro + - ./opensearch-security:/usr/share/opensearch/config/opensearch-security:ro + # Custom config file with admin_dn as YAML array (env vars don't support arrays) + - ./opensearch.dev-secure.yml:/usr/share/opensearch/config/opensearch.yml:ro + healthcheck: + test: ["CMD-SHELL", "curl -sf -u admin:$${OPENSEARCH_ADMIN_PASSWORD:-admin} --cacert /usr/share/opensearch/config/certs/root-ca.pem https://localhost:9200/_cluster/health || exit 1"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 60s + + opensearch-dashboards: + environment: + # Enable security plugin (override infra.yml settings) + - DISABLE_SECURITY_DASHBOARDS_PLUGIN=false + - OPENSEARCH_HOSTS=["https://opensearch:9200"] + - OPENSEARCH_DASHBOARDS_PASSWORD=${OPENSEARCH_DASHBOARDS_PASSWORD:-admin} + volumes: + - ./opensearch-dashboards.prod.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml:ro + - ./certs:/usr/share/opensearch-dashboards/config/certs:ro + healthcheck: + # Check if server responds (401 is fine - means server is up, security just requires auth) + test: ["CMD-SHELL", "curl -sf -o /dev/null -w '%{http_code}' http://localhost:5601/analytics/api/status | grep -qE '(200|401)' || exit 1"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 60s + + # Override init script to work with secured cluster + opensearch-init: + environment: + - OPENSEARCH_SECURITY_ENABLED=true + - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-admin} + - OPENSEARCH_CA_CERT=/certs/root-ca.pem + volumes: + - ./opensearch-init.sh:/init.sh:ro + - ./certs:/certs:ro + +volumes: + opensearch_data: + +networks: + default: + name: shipsec-network diff --git a/docker/docker-compose.full.yml b/docker/docker-compose.full.yml index 26c5b584..48314ea0 100644 --- a/docker/docker-compose.full.yml +++ b/docker/docker-compose.full.yml @@ -1,25 +1,3 @@ -# ShipSec Studio - Production Docker Compose -# -# Required Setup: -# Run 'just prod-init' to generate required secrets in docker/.env -# -# Required Environment Variables (set in docker/.env): -# - INTERNAL_SERVICE_TOKEN: Service-to-service auth token (auto-generated by prod-init) -# - SECRET_STORE_MASTER_KEY: Encryption key for secrets, must be exactly 32 characters (auto-generated by prod-init) -# -# Optional Environment Variables: -# - AUTH_PROVIDER: Authentication provider (default: clerk) -# - CLERK_PUBLISHABLE_KEY: Clerk public key (required for Clerk auth) -# - CLERK_SECRET_KEY: Clerk secret key (required for Clerk auth) -# - VITE_API_URL: Frontend API URL (default: http://localhost:3211) -# - VITE_BACKEND_URL: Frontend backend URL (default: http://localhost:3211) -# - SHIPSEC_TAG: Docker image tag (default: latest) -# -# Usage: -# just prod-init # Initialize secrets (first time only) -# just prod start-latest # Pull and run latest release -# just prod start # Start with current images - services: # Infrastructure postgres: @@ -30,8 +8,9 @@ services: POSTGRES_PASSWORD: shipsec POSTGRES_DB: shipsec POSTGRES_MULTIPLE_DATABASES: temporal - ports: - - '5433:5432' + # Internal only - no direct port access in production + expose: + - '5432' volumes: - postgres_data:/var/lib/postgresql/data - ./init-db:/docker-entrypoint-initdb.d @@ -56,13 +35,13 @@ services: - POSTGRES_PWD=shipsec - POSTGRES_SEEDS=postgres - AUTO_SETUP=true - ports: - - '7233:7233' + expose: + - '7233' volumes: - temporal_data:/var/lib/temporal restart: unless-stopped healthcheck: - test: ['CMD', 'tctl', '--address', 'localhost:7233', 'cluster', 'health'] + test: ['CMD-SHELL', 'tctl --address $(hostname -i):7233 cluster health'] interval: 30s timeout: 10s retries: 5 @@ -73,13 +52,13 @@ services: environment: - TEMPORAL_ADDRESS=temporal:7233 - TEMPORAL_NAMESPACE=default - ports: - - '8081:8080' + expose: + - '8080' depends_on: - temporal restart: unless-stopped healthcheck: - test: ['CMD', 'wget', '--no-verbose', '--tries=1', '--spider', 'http://localhost:8080'] + test: ['CMD', 'curl', '-sf', 'http://localhost:8080'] interval: 30s timeout: 10s retries: 5 @@ -91,9 +70,9 @@ services: environment: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin - ports: - - '9000:9000' - - '9001:9001' + expose: + - '9000' + - '9001' volumes: - minio_data:/data restart: unless-stopped @@ -107,8 +86,8 @@ services: image: grafana/loki:3.2.1 container_name: shipsec-loki command: -config.file=/etc/loki/local-config.yaml - ports: - - '3100:3100' + expose: + - '3100' volumes: - ./loki/loki-config.yaml:/etc/loki/local-config.yaml - loki_data:/loki @@ -122,8 +101,8 @@ services: redis: image: redis:latest container_name: shipsec-redis - ports: - - '6379:6379' + expose: + - '6379' volumes: - redis_data:/data restart: unless-stopped @@ -146,9 +125,9 @@ services: - --node-id=0 - --check=false - --advertise-kafka-addr=redpanda:9092 - ports: - - '9092:9092' - - '9644:9644' + expose: + - '9092' + - '9644' volumes: - redpanda_data:/var/lib/redpanda/data restart: unless-stopped @@ -165,12 +144,74 @@ services: - redpanda environment: CONFIG_FILEPATH: /etc/redpanda/console-config.yaml - ports: - - '8082:8080' + expose: + - '8080' volumes: - ./redpanda-console-config.yaml:/etc/redpanda/console-config.yaml:ro restart: unless-stopped + opensearch: + image: opensearchproject/opensearch:2.11.1 + container_name: shipsec-opensearch + environment: + - discovery.type=single-node + - bootstrap.memory_lock=true + - 'OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m' + - DISABLE_SECURITY_PLUGIN=true + - DISABLE_INSTALL_DEMO_CONFIG=true + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + expose: + - '9200' + - '9600' + volumes: + - opensearch_data:/usr/share/opensearch/data + restart: unless-stopped + healthcheck: + test: ['CMD-SHELL', 'curl -f http://localhost:9200/_cluster/health || exit 1'] + interval: 30s + timeout: 10s + retries: 5 + + opensearch-dashboards: + build: + context: . + dockerfile: opensearch-dashboards.Dockerfile + image: shipsec-opensearch-dashboards:2.11.1 + container_name: shipsec-opensearch-dashboards + depends_on: + opensearch: + condition: service_healthy + environment: + - OPENSEARCH_HOSTS=["http://opensearch:9200"] + - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true + expose: + - '5601' + volumes: + - ./opensearch-dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml:ro + restart: unless-stopped + healthcheck: + test: ['CMD-SHELL', 'curl -f http://localhost:5601/analytics/api/status || exit 1'] + interval: 30s + timeout: 10s + retries: 5 + + opensearch-init: + image: curlimages/curl:8.5.0 + container_name: shipsec-opensearch-init + depends_on: + opensearch-dashboards: + condition: service_healthy + volumes: + - ./opensearch-init.sh:/init.sh:ro + entrypoint: ['/bin/sh', '/init.sh'] + restart: 'no' + # Applications dind: image: docker:27-dind @@ -220,14 +261,20 @@ services: - AUTH_PROVIDER=${AUTH_PROVIDER:-local} - CLERK_PUBLISHABLE_KEY=${CLERK_PUBLISHABLE_KEY:-} - CLERK_SECRET_KEY=${CLERK_SECRET_KEY:-} + - SESSION_SECRET=${SESSION_SECRET:-} # Set to 'true' to disable analytics - DISABLE_ANALYTICS=${DISABLE_ANALYTICS:-false} - # Secret encryption key (must be exactly 32 characters, NOT hex-encoded) - - SECRET_STORE_MASTER_KEY=${SECRET_STORE_MASTER_KEY:-CHANGE_ME_32_CHAR_SECRET_KEY!!!!} - # Internal service-to-service auth token (must match worker) - - INTERNAL_SERVICE_TOKEN=${INTERNAL_SERVICE_TOKEN:-internal-service-token} - ports: - - '3211:3211' + # Internal service token for worker->backend auth + - INTERNAL_SERVICE_TOKEN=${INTERNAL_SERVICE_TOKEN:-} + # OpenSearch tenant provisioning + - OPENSEARCH_SECURITY_ENABLED=${OPENSEARCH_SECURITY_ENABLED:-false} + - OPENSEARCH_URL=http://opensearch:9200 + - OPENSEARCH_DASHBOARDS_URL=http://opensearch-dashboards:5601/analytics + - OPENSEARCH_ADMIN_USERNAME=${OPENSEARCH_ADMIN_USERNAME:-admin} + - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-} + # Internal only - accessed via nginx at /api/ + expose: + - '3211' depends_on: postgres: condition: service_healthy @@ -246,30 +293,31 @@ services: dockerfile: Dockerfile target: frontend args: - VITE_API_URL: ${VITE_API_URL:-http://localhost:3211} - VITE_BACKEND_URL: ${VITE_BACKEND_URL:-http://localhost:3211} - VITE_AUTH_PROVIDER: ${VITE_AUTH_PROVIDER:-clerk} - VITE_DEFAULT_ORG_ID: ${VITE_DEFAULT_ORG_ID:-} + VITE_API_URL: ${VITE_API_URL:-http://localhost} + VITE_BACKEND_URL: ${VITE_BACKEND_URL:-http://localhost} + VITE_AUTH_PROVIDER: ${VITE_AUTH_PROVIDER:-local} + VITE_DEFAULT_ORG_ID: ${VITE_DEFAULT_ORG_ID:-local-dev} VITE_CLERK_PUBLISHABLE_KEY: ${VITE_CLERK_PUBLISHABLE_KEY:-} VITE_GIT_SHA: ${GIT_SHA:-unknown} VITE_PUBLIC_POSTHOG_KEY: ${VITE_PUBLIC_POSTHOG_KEY:-} VITE_PUBLIC_POSTHOG_HOST: ${VITE_PUBLIC_POSTHOG_HOST:-} + VITE_OPENSEARCH_DASHBOARDS_URL: ${VITE_OPENSEARCH_DASHBOARDS_URL:-/analytics} container_name: shipsec-frontend - # NOTE: Auth defaults to Clerk intentionally - production requires Clerk authentication. - # Set VITE_AUTH_PROVIDER=local in .env only for local development without Clerk. environment: - - VITE_API_URL=${VITE_API_URL:-http://localhost:3211} - - VITE_BACKEND_URL=${VITE_BACKEND_URL:-http://localhost:3211} - - VITE_AUTH_PROVIDER=${VITE_AUTH_PROVIDER:-clerk} - - VITE_DEFAULT_ORG_ID=${VITE_DEFAULT_ORG_ID:-} + - VITE_API_URL=${VITE_API_URL:-http://localhost} + - VITE_BACKEND_URL=${VITE_BACKEND_URL:-http://localhost} + - VITE_AUTH_PROVIDER=${VITE_AUTH_PROVIDER:-local} + - VITE_DEFAULT_ORG_ID=${VITE_DEFAULT_ORG_ID:-local-dev} - VITE_CLERK_PUBLISHABLE_KEY=${VITE_CLERK_PUBLISHABLE_KEY:-} - ports: - - '8090:8080' + - VITE_OPENSEARCH_DASHBOARDS_URL=${VITE_OPENSEARCH_DASHBOARDS_URL:-/analytics} + # Internal only - accessed via nginx at / + expose: + - '8080' depends_on: - backend restart: unless-stopped healthcheck: - test: ['CMD', 'curl', '-f', 'http://localhost:8080'] + test: ['CMD', 'curl', '-sf', 'http://localhost:8080'] interval: 30s timeout: 10s retries: 5 @@ -301,12 +349,13 @@ services: - LOG_KAFKA_CLIENT_ID=shipsec-worker - EVENT_KAFKA_TOPIC=telemetry.events - EVENT_KAFKA_CLIENT_ID=shipsec-worker-events - # Secret encryption key (must be exactly 32 characters, NOT hex-encoded) - - SECRET_STORE_MASTER_KEY=${SECRET_STORE_MASTER_KEY:-CHANGE_ME_32_CHAR_SECRET_KEY!!!!} - # Internal service-to-service auth token (must match backend) - - INTERNAL_SERVICE_TOKEN=${INTERNAL_SERVICE_TOKEN:-internal-service-token} - # Backend URL for internal API calls - - STUDIO_API_BASE_URL=http://backend:3211/api/v1 + # OpenSearch for Analytics Sink + - OPENSEARCH_URL=http://opensearch:9200 + - OPENSEARCH_DASHBOARDS_URL=http://opensearch-dashboards:5601/analytics + # Tenant provisioning (for multi-tenant security mode) + - BACKEND_URL=http://backend:3211 + - INTERNAL_SERVICE_TOKEN=${INTERNAL_SERVICE_TOKEN:-} + - OPENSEARCH_SECURITY_ENABLED=${OPENSEARCH_SECURITY_ENABLED:-false} depends_on: postgres: condition: service_healthy @@ -320,6 +369,8 @@ services: condition: service_healthy redpanda: condition: service_healthy + opensearch: + condition: service_healthy restart: unless-stopped healthcheck: test: ['CMD', 'node', '-e', 'process.exit(0)'] @@ -327,6 +378,28 @@ services: timeout: 10s retries: 5 + # Nginx reverse proxy - unified entry point + nginx: + image: nginx:1.25-alpine + container_name: shipsec-nginx + depends_on: + frontend: + condition: service_healthy + backend: + condition: service_started + opensearch-dashboards: + condition: service_healthy + ports: + - '80:80' + volumes: + - ./nginx/nginx.prod.conf:/etc/nginx/nginx.conf:ro + restart: unless-stopped + healthcheck: + test: ['CMD', 'curl', '-sf', 'http://localhost/health'] + interval: 30s + timeout: 10s + retries: 5 + volumes: postgres_data: minio_data: @@ -335,6 +408,7 @@ volumes: docker_data: redis_data: redpanda_data: + opensearch_data: networks: default: diff --git a/docker/docker-compose.infra.yml b/docker/docker-compose.infra.yml index 1022273f..1e9e619c 100644 --- a/docker/docker-compose.infra.yml +++ b/docker/docker-compose.infra.yml @@ -1,18 +1,20 @@ services: postgres: image: postgres:16-alpine + container_name: shipsec-postgres environment: POSTGRES_USER: shipsec POSTGRES_PASSWORD: shipsec POSTGRES_DB: shipsec POSTGRES_MULTIPLE_DATABASES: temporal - ports: - - '5433:5432' + # Internal only - use docker-compose.dev-ports.yml overlay for local dev access + expose: + - "5432" volumes: - postgres_data:/var/lib/postgresql/data - ./init-db:/docker-entrypoint-initdb.d healthcheck: - test: ['CMD-SHELL', 'pg_isready -U shipsec'] + test: ["CMD-SHELL", "pg_isready -U shipsec"] interval: 5s timeout: 3s retries: 10 @@ -20,6 +22,7 @@ services: temporal: image: temporalio/auto-setup:latest + container_name: shipsec-temporal depends_on: postgres: condition: service_healthy @@ -31,72 +34,76 @@ services: - POSTGRES_PWD=shipsec - POSTGRES_SEEDS=postgres - AUTO_SETUP=true - ports: - - '7233:7233' + expose: + - "7233" volumes: - temporal_data:/var/lib/temporal restart: unless-stopped temporal-ui: image: temporalio/ui:latest + container_name: shipsec-temporal-ui depends_on: - temporal environment: - TEMPORAL_ADDRESS=temporal:7233 - # Include several common dev frontend ports. - - TEMPORAL_CORS_ORIGINS=http://localhost:5173,http://localhost:5273,http://localhost:5373 - ports: - - '8081:8080' + - TEMPORAL_CORS_ORIGINS=http://localhost:5173 + expose: + - "8080" restart: unless-stopped minio: image: minio/minio:RELEASE.2024-10-02T17-50-41Z + container_name: shipsec-minio command: server /data --console-address ":9001" environment: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin - ports: - - '9000:9000' - - '9001:9001' + expose: + - "9000" + - "9001" volumes: - minio_data:/data restart: unless-stopped healthcheck: - test: ['CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live'] + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] interval: 30s timeout: 10s retries: 5 redis: image: redis:latest - ports: - - '6379:6379' + container_name: shipsec-redis + expose: + - "6379" volumes: - redis_data:/data restart: unless-stopped healthcheck: - test: ['CMD', 'redis-cli', 'ping'] + test: ["CMD", "redis-cli", "ping"] interval: 30s timeout: 10s retries: 5 loki: image: grafana/loki:3.2.1 + container_name: shipsec-loki command: -config.file=/etc/loki/local-config.yaml - ports: - - '3100:3100' + expose: + - "3100" volumes: - ./loki/loki-config.yaml:/etc/loki/local-config.yaml - loki_data:/loki restart: unless-stopped healthcheck: - test: ['CMD', 'wget', '--no-verbose', '--tries=1', '--spider', 'http://localhost:3100/ready'] + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3100/ready"] interval: 30s timeout: 10s retries: 5 redpanda: image: redpandadata/redpanda:v24.2.5 + container_name: shipsec-redpanda command: - redpanda - start @@ -106,34 +113,121 @@ services: - --overprovisioned - --node-id=0 - --check=false - # Internal listener for Docker network containers (redpanda-console, etc.) - - --kafka-addr=internal://0.0.0.0:9092,external://0.0.0.0:19092 - - --advertise-kafka-addr=internal://redpanda:9092,external://localhost:19092 - ports: - - '19092:19092' # External Kafka port for host apps - - '9092:9092' # Internal port (for Docker-to-Docker only, maps for debugging) - - '9644:9644' + - --advertise-kafka-addr=localhost:9092 + expose: + - "9092" + - "9644" volumes: - redpanda_data:/var/lib/redpanda/data restart: unless-stopped healthcheck: - test: ['CMD', 'curl', '-f', 'http://localhost:9644/v1/status/ready'] + test: ["CMD", "curl", "-f", "http://localhost:9644/v1/status/ready"] interval: 30s timeout: 10s retries: 5 redpanda-console: image: redpandadata/console:v2.7.2 + container_name: shipsec-redpanda-console depends_on: - redpanda environment: CONFIG_FILEPATH: /etc/redpanda/console-config.yaml - ports: - - '8082:8080' + expose: + - "8080" volumes: - ./redpanda-console-config.yaml:/etc/redpanda/console-config.yaml:ro restart: unless-stopped + opensearch: + image: opensearchproject/opensearch:2.11.1 + container_name: shipsec-opensearch + environment: + - discovery.type=single-node + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - DISABLE_SECURITY_PLUGIN=true + - DISABLE_INSTALL_DEMO_CONFIG=true + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + # Ports exposed only within Docker network (not to host) + # Use docker-compose.dev-ports.yml overlay for local dev access + expose: + - "9200" + - "9600" + volumes: + - opensearch_data:/usr/share/opensearch/data + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:9200/_cluster/health || exit 1"] + interval: 30s + timeout: 10s + retries: 5 + + opensearch-dashboards: + build: + context: . + dockerfile: opensearch-dashboards.Dockerfile + image: shipsec-opensearch-dashboards:2.11.1 + container_name: shipsec-opensearch-dashboards + depends_on: + opensearch: + condition: service_healthy + environment: + - OPENSEARCH_HOSTS=["http://opensearch:9200"] + - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true + # Ports exposed only within Docker network (not to host) + # Use docker-compose.dev-ports.yml overlay for local dev access + # Production uses nginx reverse proxy at /analytics + expose: + - "5601" + volumes: + - ./opensearch-dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml:ro + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:5601/analytics/api/status || exit 1"] + interval: 30s + timeout: 10s + retries: 5 + + # Initialize OpenSearch Dashboards with default index patterns + opensearch-init: + image: curlimages/curl:8.5.0 + container_name: shipsec-opensearch-init + depends_on: + opensearch-dashboards: + condition: service_healthy + volumes: + - ./opensearch-init.sh:/init.sh:ro + entrypoint: ["/bin/sh", "/init.sh"] + restart: "no" + + # Nginx reverse proxy - unified entry point + # DEV MODE: Uses nginx.dev.conf which points to host.docker.internal for PM2 services + nginx: + image: nginx:1.25-alpine + container_name: shipsec-nginx + depends_on: + opensearch-dashboards: + condition: service_healthy + ports: + - "80:80" + volumes: + - ./nginx/nginx.dev.conf:/etc/nginx/nginx.conf:ro + extra_hosts: + - "host.docker.internal:host-gateway" + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost/health"] + interval: 30s + timeout: 10s + retries: 5 + volumes: postgres_data: minio_data: @@ -141,3 +235,8 @@ volumes: temporal_data: redis_data: redpanda_data: + opensearch_data: + +networks: + default: + name: shipsec-network diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml new file mode 100644 index 00000000..fe81b009 --- /dev/null +++ b/docker/docker-compose.prod.yml @@ -0,0 +1,97 @@ +# Production Docker Compose - OpenSearch with Security & Multitenancy +# +# Usage: +# docker compose -f docker-compose.infra.yml -f docker-compose.prod.yml up -d +# +# Prerequisites: +# 1. Generate TLS certificates: ./scripts/generate-certs.sh +# 2. Set environment variables in .env.prod or export them: +# - OPENSEARCH_ADMIN_PASSWORD (required) +# - OPENSEARCH_DASHBOARDS_PASSWORD (required) +# +# This file overrides the development infrastructure with: +# - Security plugin enabled +# - TLS encryption for transport and HTTP +# - Multitenancy support in OpenSearch Dashboards + +services: + opensearch: + # Custom entrypoint for proxy auth config templating + entrypoint: ["/usr/share/opensearch/config/opensearch-security/docker-entrypoint-security.sh"] + environment: + # Remove security disable flags (override dev settings) + - DISABLE_SECURITY_PLUGIN=false + - DISABLE_INSTALL_DEMO_CONFIG=true + # Admin password for healthcheck + - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-admin} + # Proxy auth - trusted proxy IP regex (Docker bridge network) + - OPENSEARCH_INTERNAL_PROXIES=172\.\d+\.\d+\.\d+ + # Security configuration + - plugins.security.ssl.transport.pemcert_filepath=certs/node.pem + - plugins.security.ssl.transport.pemkey_filepath=certs/node-key.pem + - plugins.security.ssl.transport.pemtrustedcas_filepath=certs/root-ca.pem + - plugins.security.ssl.transport.enforce_hostname_verification=false + - plugins.security.ssl.http.enabled=true + - plugins.security.ssl.http.pemcert_filepath=certs/node.pem + - plugins.security.ssl.http.pemkey_filepath=certs/node-key.pem + - plugins.security.ssl.http.pemtrustedcas_filepath=certs/root-ca.pem + - plugins.security.allow_unsafe_democertificates=false + - plugins.security.allow_default_init_securityindex=true + - plugins.security.authcz.admin_dn=CN=admin,OU=ShipSec,O=ShipSecAI,L=SF,ST=CA,C=US + - plugins.security.audit.type=internal_opensearch + - plugins.security.enable_snapshot_restore_privilege=true + - plugins.security.check_snapshot_restore_write_privileges=true + - plugins.security.restapi.roles_enabled=["all_access", "security_rest_api_access"] + - cluster.name=shipsec-prod + - node.name=opensearch-node1 + volumes: + - opensearch_data:/usr/share/opensearch/data + - ./certs:/usr/share/opensearch/config/certs:ro + - ./opensearch-security:/usr/share/opensearch/config/opensearch-security:ro + healthcheck: + test: ["CMD-SHELL", "curl -sf -u admin:$${OPENSEARCH_ADMIN_PASSWORD:-admin} --cacert /usr/share/opensearch/config/certs/root-ca.pem https://localhost:9200/_cluster/health || exit 1"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 60s + + opensearch-dashboards: + environment: + # Remove security disable flag (override dev settings) + - DISABLE_SECURITY_DASHBOARDS_PLUGIN=false + - OPENSEARCH_HOSTS=["https://opensearch:9200"] + volumes: + - ./opensearch-dashboards.prod.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml:ro + - ./certs:/usr/share/opensearch-dashboards/config/certs:ro + healthcheck: + # Check if server responds (401 is fine - means server is up, security just requires auth) + test: ["CMD-SHELL", "curl -sf -o /dev/null -w '%{http_code}' http://localhost:5601/analytics/api/status | grep -qE '(200|401)' || exit 1"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 60s + + # Override init script to work with secured cluster + opensearch-init: + environment: + - OPENSEARCH_SECURITY_ENABLED=true + - OPENSEARCH_CA_CERT=/certs/root-ca.pem + volumes: + - ./opensearch-init.sh:/init.sh:ro + - ./certs:/certs:ro + + # Nginx with production config (container service names) + nginx: + volumes: + - ./nginx/nginx.prod.conf:/etc/nginx/nginx.conf:ro + - ./certs:/etc/nginx/certs:ro + ports: + - "80:80" + - "443:443" + +volumes: + opensearch_data: + +networks: + default: + name: shipsec-network diff --git a/docker/init-db/01-create-instance-databases.sh b/docker/init-db/01-create-instance-databases.sh index f31e3b5a..4540fa63 100755 --- a/docker/init-db/01-create-instance-databases.sh +++ b/docker/init-db/01-create-instance-databases.sh @@ -1,16 +1,30 @@ #!/bin/bash -# Create instance-specific PostgreSQL databases +# Create additional PostgreSQL databases required by ShipSec # This script is run automatically by PostgreSQL init-entrypoint +# +# Creates: +# - temporal: Required by Temporal workflow engine +# - shipsec_instance_0..9: Multi-instance dev databases set -e -echo "šŸ—„ļø Creating instance-specific databases..." +# --- Temporal database (required for workflow engine) --- +echo "šŸ—„ļø Creating Temporal database..." +if psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d postgres -lqt | cut -d \| -f 1 | grep -qw "temporal"; then + echo " Database temporal already exists, skipping..." +else + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d postgres <<-EOSQL + CREATE DATABASE temporal OWNER "$POSTGRES_USER"; + GRANT ALL PRIVILEGES ON DATABASE temporal TO "$POSTGRES_USER"; +EOSQL + echo " āœ… temporal created" +fi -# Create databases for instances 0-9 +# --- Instance-specific databases (for multi-instance dev) --- +echo "šŸ—„ļø Creating instance-specific databases..." for i in {0..9}; do DB_NAME="shipsec_instance_$i" - - # Check if database already exists + if psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d postgres -lqt | cut -d \| -f 1 | grep -qw "$DB_NAME"; then echo " Database $DB_NAME already exists, skipping..." else @@ -22,7 +36,4 @@ EOSQL fi done -echo "āœ… Instance-specific databases created successfully" -echo "" -echo "Available databases:" -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d postgres -c "\\l" | grep shipsec_instance +echo "āœ… All databases created successfully" diff --git a/docker/init-db/create-multiple-postgresql-databases.sh b/docker/init-db/create-multiple-postgresql-databases.sh deleted file mode 100755 index 34368865..00000000 --- a/docker/init-db/create-multiple-postgresql-databases.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e - -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - CREATE DATABASE temporal; - GRANT ALL PRIVILEGES ON DATABASE temporal TO $POSTGRES_USER; -EOSQL \ No newline at end of file diff --git a/docker/nginx/nginx.dev.conf b/docker/nginx/nginx.dev.conf new file mode 100644 index 00000000..bceaf670 --- /dev/null +++ b/docker/nginx/nginx.dev.conf @@ -0,0 +1,231 @@ +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + # Debug log format for analytics auth issues + log_format auth_debug '$remote_addr [$time_local] "$request" $status ' + 'auth_org_id="$auth_org_id" auth_user="$auth_user_id"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + + # Gzip compression + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_types text/plain text/css text/xml application/json application/javascript + application/rss+xml application/atom+xml image/svg+xml; + + # ================================================================= + # DEVELOPMENT MODE - Frontend & Backend run on host via PM2 + # Uses host.docker.internal to reach host machine from container + # ================================================================= + + # Upstream definitions - pointing to host machine (PM2 services) + upstream frontend { + # Vite dev server on host + server host.docker.internal:5173; + keepalive 32; + } + + upstream backend { + # NestJS backend on host + server host.docker.internal:3211; + keepalive 32; + } + + # OpenSearch Dashboards runs in Docker + upstream opensearch-dashboards { + server opensearch-dashboards:5601; + keepalive 32; + } + + # WebSocket connection upgrade map + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 80; + server_name _; + + # Client request body size (for file uploads) + client_max_body_size 100M; + client_body_buffer_size 10M; + + # Proxy buffer settings + proxy_buffer_size 128k; + proxy_buffers 4 256k; + proxy_busy_buffers_size 256k; + + # Common proxy headers + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + + # ================================================================= + # Auth validation endpoint (public, proxied to backend) + # ================================================================= + location = /auth/validate { + proxy_pass http://backend/api/v1/auth/validate; + proxy_set_header Cookie $http_cookie; + proxy_set_header Authorization $http_authorization; + } + + # ================================================================= + # Internal auth validation endpoint for auth_request + # ================================================================= + location = /_auth { + internal; + # Debug: log internal auth requests + access_log /var/log/nginx/auth_internal.log main; + + proxy_pass http://backend/api/v1/auth/validate; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URI $request_uri; + # Pass cookies for session auth + proxy_set_header Cookie $http_cookie; + # Pass Authorization header for API key/token auth + proxy_set_header Authorization $http_authorization; + } + + # ================================================================= + # Dashboards SaaS Lockdown - Whitelist allowed app pages + # Unwanted plugins are removed from the image (opensearch-dashboards.Dockerfile) + # This regex is defense-in-depth against any remaining/future plugins + # Admin: use direct Dashboards port (5601) bypassing nginx + # ================================================================= + location ~ ^/analytics/app/(?!(discover|visualize|dashboards?|dev_tools|alerting|notifications|management|data-explorer|home)($|/|\?|#)) { + return 403; + } + + # ================================================================= + # OpenSearch Dashboards - /analytics/* (PROTECTED + TENANT ISOLATED) + # ================================================================= + location /analytics/ { + # Debug logging for auth issues + access_log /var/log/nginx/analytics_auth.log auth_debug; + + # Require authentication before proxying + auth_request /_auth; + # On auth failure, redirect to login page + error_page 401 = @auth_redirect; + + # Capture org/user context from auth response headers + auth_request_set $auth_org_id $upstream_http_x_auth_organization_id; + auth_request_set $auth_user_id $upstream_http_x_auth_user_id; + + # NOTE: Cannot use 'if ($auth_org_id = "")' here because nginx's 'if' runs + # in the rewrite phase BEFORE auth_request completes in the access phase. + # OpenSearch Security will reject requests with invalid/missing proxy auth headers. + # For fail-closed behavior, the auth backend should return 401 if org context is missing. + + proxy_pass http://opensearch-dashboards; + + # Standard forwarding headers (must be repeated here because nginx + # proxy_set_header in a location block overrides ALL parent-level directives) + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # WebSocket support for dashboards real-time features + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + + # Timeouts for dashboards (can be slow for large queries) + proxy_connect_timeout 60s; + proxy_send_timeout 120s; + proxy_read_timeout 120s; + + # Dashboards-specific headers + proxy_set_header osd-xsrf "true"; + + # OpenSearch Security proxy auth headers + # proxy_set_header REPLACES any client-supplied headers (prevents spoofing) + proxy_set_header x-proxy-user $auth_org_id; + proxy_set_header x-proxy-roles "customer_${auth_org_id}_ro"; + proxy_set_header securitytenant $auth_org_id; + + # Preserve cookies + proxy_cookie_path /analytics/ /analytics/; + + # No redirect rewriting needed - we preserve the path + proxy_redirect off; + } + + # Auth redirect handler - redirect to home with return URL + location @auth_redirect { + return 302 /?returnTo=$request_uri; + } + + # Exact match for /analytics without trailing slash + location = /analytics { + return 301 /analytics/; + } + + # ================================================================= + # Backend API - /api/* + # ================================================================= + location /api/ { + proxy_pass http://backend/api/; + + # WebSocket support for terminal/streaming endpoints + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + + # API timeouts + proxy_connect_timeout 30s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; + + # Don't buffer API responses (important for streaming) + proxy_buffering off; + } + + # ================================================================= + # Frontend (SPA) - /* (catch-all) + # ================================================================= + location / { + proxy_pass http://frontend/; + + # WebSocket support for Vite HMR in development + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + + # Frontend timeouts - longer read timeout for HMR WebSocket + proxy_connect_timeout 30s; + proxy_send_timeout 30s; + proxy_read_timeout 86400s; # 24 hours - keep HMR WebSocket alive + } + + # Health check endpoint + location /health { + access_log off; + return 200 "healthy\n"; + add_header Content-Type text/plain; + } + } +} diff --git a/docker/nginx/nginx.prod.conf b/docker/nginx/nginx.prod.conf new file mode 100644 index 00000000..3677280a --- /dev/null +++ b/docker/nginx/nginx.prod.conf @@ -0,0 +1,208 @@ +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + + # Gzip compression + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_types text/plain text/css text/xml application/json application/javascript + application/rss+xml application/atom+xml image/svg+xml; + + # Upstream definitions + upstream frontend { + server frontend:8080; + keepalive 32; + } + + upstream backend { + server backend:3211; + keepalive 32; + } + + upstream opensearch-dashboards { + server opensearch-dashboards:5601; + keepalive 32; + } + + # WebSocket connection upgrade map + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 80; + server_name _; + + # Client request body size (for file uploads) + client_max_body_size 100M; + client_body_buffer_size 10M; + + # Proxy buffer settings + proxy_buffer_size 128k; + proxy_buffers 4 256k; + proxy_busy_buffers_size 256k; + + # Common proxy headers + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + + # ================================================================= + # Auth validation endpoint (public, proxied to backend) + # ================================================================= + location = /auth/validate { + proxy_pass http://backend/api/v1/auth/validate; + proxy_set_header Cookie $http_cookie; + proxy_set_header Authorization $http_authorization; + } + + # ================================================================= + # Internal auth validation endpoint for auth_request + # ================================================================= + location = /_auth { + internal; + proxy_pass http://backend/api/v1/auth/validate; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URI $request_uri; + # Pass cookies for session auth + proxy_set_header Cookie $http_cookie; + # Pass Authorization header for API key/token auth + proxy_set_header Authorization $http_authorization; + } + + # ================================================================= + # Dashboards SaaS Lockdown - Whitelist allowed app pages + # Unwanted plugins are removed from the image (opensearch-dashboards.Dockerfile) + # This regex is defense-in-depth against any remaining/future plugins + # Admin: use direct Dashboards port (5601) bypassing nginx + # ================================================================= + location ~ ^/analytics/app/(?!(discover|visualize|dashboards?|dev_tools|alerting|notifications|management|data-explorer|home)($|/|\?|#)) { + return 403; + } + + # ================================================================= + # OpenSearch Dashboards - /analytics/* (PROTECTED + TENANT ISOLATED) + # ================================================================= + location /analytics/ { + # Require authentication before proxying + auth_request /_auth; + # On auth failure, redirect to login page + error_page 401 = @auth_redirect; + + # Capture org/user context from auth response headers + auth_request_set $auth_org_id $upstream_http_x_auth_organization_id; + auth_request_set $auth_user_id $upstream_http_x_auth_user_id; + + # FAIL-CLOSED: Reject if org context is missing + # This prevents unauthenticated or org-less sessions from reaching Dashboards + if ($auth_org_id = "") { + return 403; + } + + proxy_pass http://opensearch-dashboards/; + + # WebSocket support for dashboards real-time features + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + + # Timeouts for dashboards (can be slow for large queries) + proxy_connect_timeout 60s; + proxy_send_timeout 120s; + proxy_read_timeout 120s; + + # Dashboards-specific headers + proxy_set_header osd-xsrf "true"; + + # OpenSearch Security proxy auth headers + # proxy_set_header REPLACES any client-supplied headers (prevents spoofing) + proxy_set_header x-proxy-user $auth_org_id; + proxy_set_header x-proxy-roles "customer_${auth_org_id}_ro"; + proxy_set_header securitytenant $auth_org_id; + + # Preserve cookies + proxy_cookie_path / /analytics/; + + # Handle redirects from dashboards + proxy_redirect / /analytics/; + proxy_redirect http://opensearch-dashboards:5601/ /analytics/; + } + + # Auth redirect handler - redirect to home with return URL + location @auth_redirect { + return 302 /?returnTo=$request_uri; + } + + # Exact match for /analytics without trailing slash + location = /analytics { + return 301 /analytics/; + } + + # ================================================================= + # Backend API - /api/* + # ================================================================= + location /api/ { + proxy_pass http://backend/api/; + + # WebSocket support for terminal/streaming endpoints + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + + # API timeouts + proxy_connect_timeout 30s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; + + # Don't buffer API responses (important for streaming) + proxy_buffering off; + } + + # ================================================================= + # Frontend (SPA) - /* (catch-all) + # ================================================================= + location / { + proxy_pass http://frontend/; + + # WebSocket support for Vite HMR in development + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + + # Frontend timeouts + proxy_connect_timeout 30s; + proxy_send_timeout 30s; + proxy_read_timeout 30s; + } + + # Health check endpoint + location /health { + access_log off; + return 200 "healthy\n"; + add_header Content-Type text/plain; + } + } +} diff --git a/docker/opensearch-dashboards.Dockerfile b/docker/opensearch-dashboards.Dockerfile new file mode 100644 index 00000000..65613ffb --- /dev/null +++ b/docker/opensearch-dashboards.Dockerfile @@ -0,0 +1,18 @@ +# Custom OpenSearch Dashboards image for SaaS tenant lockdown +# Source: https://github.com/ShipSecAI/tools/tree/main/misc/opensearch-dashboards-saas +# +# Removes unwanted plugins from sidebar. Config-level disabling is NOT possible +# because OSD 2.x plugins don't register an "enabled" config key (fatal error). +# See the tools repo README for full documentation. + +FROM opensearchproject/opensearch-dashboards:2.11.1 + +RUN /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove queryWorkbenchDashboards && \ + /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove reportsDashboards && \ + /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove anomalyDetectionDashboards && \ + /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove customImportMapDashboards && \ + /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityAnalyticsDashboards && \ + /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove searchRelevanceDashboards && \ + /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove mlCommonsDashboards && \ + /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove indexManagementDashboards && \ + /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove observabilityDashboards diff --git a/docker/opensearch-dashboards.prod.yml b/docker/opensearch-dashboards.prod.yml new file mode 100644 index 00000000..c9007b13 --- /dev/null +++ b/docker/opensearch-dashboards.prod.yml @@ -0,0 +1,66 @@ +# OpenSearch Dashboards Production Configuration +# Mount this file to /usr/share/opensearch-dashboards/config/opensearch_dashboards.yml +# +# This configuration enables: +# - Security plugin with authentication +# - Multitenancy for tenant isolation +# - TLS for secure communication with OpenSearch + +server.host: "0.0.0.0" +server.port: 5601 + +# Base path configuration for reverse proxy +server.basePath: "/analytics" +server.rewriteBasePath: true + +# OpenSearch connection (HTTPS for production) +opensearch.hosts: ["https://opensearch:9200"] + +# TLS Configuration - trust the CA certificate +opensearch.ssl.verificationMode: certificate +opensearch.ssl.certificateAuthorities: ["/usr/share/opensearch-dashboards/config/certs/root-ca.pem"] + +# Authentication - proxy auth from nginx (primary) + basic auth for kibanaserver +# Note: OpenSearch Dashboards doesn't support env var interpolation in YAML +# In production, use a secrets manager or pre-process this file +opensearch.username: "kibanaserver" +opensearch.password: "admin" +opensearch.requestHeadersAllowlist: ["securitytenant", "Authorization", "x-forwarded-for", "x-proxy-user", "x-proxy-roles"] + +# Proxy Authentication Configuration +# Nginx sets x-proxy-user/x-proxy-roles headers after validating user session via auth_request. +# Dashboards trusts these headers (no login form). Users must log in via the main app first. +# The kibanaserver user (above) is still used for Dashboards' own backend connection to OpenSearch. +opensearch_security.auth.type: "proxy" +opensearch_security.proxycache.user_header: "x-proxy-user" +opensearch_security.proxycache.roles_header: "x-proxy-roles" + +# Security Plugin Configuration - SaaS Multitenancy +# Each customer gets their own isolated tenant - no shared data by default +# Tenant is forced via nginx securitytenant header (per-org), no tenant picker shown +opensearch_security.multitenancy.enabled: true +opensearch_security.multitenancy.tenants.enable_global: false +opensearch_security.multitenancy.tenants.enable_private: false +opensearch_security.multitenancy.tenants.preferred: ["Custom"] +opensearch_security.multitenancy.show_roles: false +opensearch_security.multitenancy.enable_filter: false +opensearch_security.readonly_mode.roles: ["kibana_read_only"] +opensearch_security.cookie.secure: true +opensearch_security.cookie.isSameSite: "Strict" + +# Session configuration +opensearch_security.session.ttl: 3600000 +opensearch_security.session.keepalive: true + +# Default landing page - Discover instead of Home (which shows all plugin links) +uiSettings.overrides.defaultRoute: "/app/discover" + +# Logging +logging.dest: stdout +logging.silent: false +logging.quiet: false +logging.verbose: false + +# CSP headers for security +csp.strict: true +csp.warnLegacyBrowsers: true diff --git a/docker/opensearch-dashboards.yml b/docker/opensearch-dashboards.yml new file mode 100644 index 00000000..7c24007a --- /dev/null +++ b/docker/opensearch-dashboards.yml @@ -0,0 +1,33 @@ +# OpenSearch Dashboards configuration +# Mount this file to /usr/share/opensearch-dashboards/config/opensearch_dashboards.yml +# +# SECURITY NOTE: +# - Local development: Security plugin is disabled (DISABLE_SECURITY_DASHBOARDS_PLUGIN=true in docker-compose) +# - Production: Enable security plugin and configure multitenancy: +# 1. Remove DISABLE_SECURITY_PLUGIN=true from OpenSearch +# 2. Remove DISABLE_SECURITY_DASHBOARDS_PLUGIN=true from Dashboards +# 3. Configure TLS certificates and authentication +# 4. Add: opensearch_security.multitenancy.enabled: true +# 5. Add: opensearch_security.multitenancy.tenants.preferred: ["Private", "Global"] + +server.host: "0.0.0.0" +server.port: 5601 + +# Base path configuration for reverse proxy +server.basePath: "/analytics" +server.rewriteBasePath: true + +# OpenSearch connection +opensearch.hosts: ["http://opensearch:9200"] + +# Logging +logging.dest: stdout +logging.silent: false +logging.quiet: false +logging.verbose: false + +# Default landing page - Discover instead of Home (which shows all plugin links) +uiSettings.overrides.defaultRoute: "/app/discover" + +# CSP - relaxed for development (inline scripts needed by dashboards) +csp.strict: false diff --git a/docker/opensearch-init.sh b/docker/opensearch-init.sh new file mode 100755 index 00000000..1d9d97b1 --- /dev/null +++ b/docker/opensearch-init.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# OpenSearch Dashboards initialization script +# Creates default index patterns and saved objects +# +# Environment variables: +# OPENSEARCH_DASHBOARDS_URL - Dashboards URL (default: http://opensearch-dashboards:5601) +# OPENSEARCH_SECURITY_ENABLED - Enable security mode (default: false) +# OPENSEARCH_ADMIN_PASSWORD - Admin password (not used with proxy auth, kept for reference) +# OPENSEARCH_CA_CERT - Path to CA cert for TLS (optional, for https) + +set -e + +# Note: Use /analytics prefix since dashboards is configured with server.basePath=/analytics +DASHBOARDS_URL="${OPENSEARCH_DASHBOARDS_URL:-http://opensearch-dashboards:5601}" +DASHBOARDS_BASE_PATH="/analytics" +MAX_RETRIES=30 +RETRY_INTERVAL=5 +SECURITY_ENABLED="${OPENSEARCH_SECURITY_ENABLED:-false}" + +# Wrapper function for authenticated curl requests +# When security is enabled, Dashboards uses proxy auth (not basic auth) +# We send x-proxy-user and x-proxy-roles headers to authenticate +auth_curl() { + if [ "$SECURITY_ENABLED" = "true" ]; then + curl -H "x-proxy-user: admin" -H "x-proxy-roles: admin,all_access" "$@" + else + curl "$@" + fi +} + +echo "[opensearch-init] Security mode: ${SECURITY_ENABLED}" +echo "[opensearch-init] Waiting for OpenSearch Dashboards to be ready..." + +# Wait for Dashboards to be healthy (use basePath) +# Accept 200 or 401 as "ready" - 401 means server is up but requires auth +# Note: Don't use -f flag as we want to capture 4xx status codes without curl failing +for i in $(seq 1 $MAX_RETRIES); do + HTTP_CODE=$(curl -s -o /dev/null -w '%{http_code}' "${DASHBOARDS_URL}${DASHBOARDS_BASE_PATH}/api/status" 2>/dev/null || echo "000") + + if [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "401" ]; then + echo "[opensearch-init] OpenSearch Dashboards is ready! (HTTP $HTTP_CODE)" + break + fi + + if [ $i -eq $MAX_RETRIES ]; then + echo "[opensearch-init] ERROR: OpenSearch Dashboards not ready after $((MAX_RETRIES * RETRY_INTERVAL)) seconds (last HTTP code: $HTTP_CODE)" + exit 1 + fi + + echo "[opensearch-init] Waiting for Dashboards... (attempt $i/$MAX_RETRIES, HTTP $HTTP_CODE)" + sleep $RETRY_INTERVAL +done + +# In secure mode, skip index pattern creation via Dashboards API +# Reason: Dashboards uses proxy auth which requires requests to come through nginx +# Index patterns will be created when users first access Dashboards through the normal flow +if [ "$SECURITY_ENABLED" = "true" ]; then + echo "[opensearch-init] Security mode enabled - skipping index pattern creation" + echo "[opensearch-init] Index patterns will be created on first user access via nginx" + echo "[opensearch-init] Initialization complete!" + exit 0 +fi + +# Check if index pattern already exists (insecure mode only) +echo "[opensearch-init] Checking for existing index patterns..." +EXISTING=$(auth_curl -sf "${DASHBOARDS_URL}${DASHBOARDS_BASE_PATH}/api/saved_objects/_find?type=index-pattern&search_fields=title&search=security-findings-*" \ + -H "osd-xsrf: true" 2>/dev/null || echo '{"total":0}') + +TOTAL=$(echo "$EXISTING" | grep -o '"total":[0-9]*' | grep -o '[0-9]*' || echo "0") + +if [ "$TOTAL" -gt 0 ]; then + echo "[opensearch-init] Index pattern 'security-findings-*' already exists, skipping creation" +else + echo "[opensearch-init] Creating index pattern 'security-findings-*'..." + + # Use specific ID so dashboards can reference it consistently + RESPONSE=$(auth_curl -sf -X POST "${DASHBOARDS_URL}${DASHBOARDS_BASE_PATH}/api/saved_objects/index-pattern/security-findings-*" \ + -H "Content-Type: application/json" \ + -H "osd-xsrf: true" \ + -d '{ + "attributes": { + "title": "security-findings-*", + "timeFieldName": "@timestamp" + } + }' 2>&1) + + if echo "$RESPONSE" | grep -q '"type":"index-pattern"'; then + echo "[opensearch-init] Successfully created index pattern 'security-findings-*'" + else + echo "[opensearch-init] WARNING: Failed to create index pattern. Response: $RESPONSE" + # Don't fail - the pattern might be created later when data exists + fi +fi + +# Set as default index pattern (optional, helps UX) +echo "[opensearch-init] Setting default index pattern..." +auth_curl -sf -X POST "${DASHBOARDS_URL}${DASHBOARDS_BASE_PATH}/api/opensearch-dashboards/settings" \ + -H "Content-Type: application/json" \ + -H "osd-xsrf: true" \ + -d '{"changes":{"defaultIndex":"security-findings-*"}}' > /dev/null 2>&1 || true + +echo "[opensearch-init] Initialization complete!" diff --git a/docker/opensearch-security/action_groups.yml b/docker/opensearch-security/action_groups.yml new file mode 100644 index 00000000..1b4a5179 --- /dev/null +++ b/docker/opensearch-security/action_groups.yml @@ -0,0 +1,61 @@ +# OpenSearch Security - Action Groups +# +# Action groups bundle permissions together for easier role assignment. +# Most common groups are built-in, but custom ones can be defined here. +# +# Built-in action groups (no need to redefine): +# - cluster_all, cluster_monitor, cluster_composite_ops, cluster_composite_ops_ro +# - indices_all, indices_monitor, read, write, delete, create_index, manage +# - kibana_all_read, kibana_all_write +# +# Reference: https://opensearch.org/docs/latest/security/access-control/default-action-groups/ + +--- +_meta: + type: "actiongroups" + config_version: 2 + +# ============================================================================= +# CUSTOM ACTION GROUPS +# ============================================================================= + +# Index management for security findings +security_findings_write: + reserved: false + static: false + allowed_actions: + - "indices:data/write/index" + - "indices:data/write/bulk*" + - "indices:data/write/update" + - "indices:data/write/delete" + - "indices:admin/create" + - "indices:admin/mapping/put" + description: "Write access to security findings indices" + +security_findings_read: + reserved: false + static: false + allowed_actions: + - "indices:data/read/search*" + - "indices:data/read/get*" + - "indices:data/read/mget*" + - "indices:data/read/msearch*" + - "indices:data/read/scroll*" + - "indices:admin/mappings/get" + - "indices:admin/resolve/index" + description: "Read access to security findings indices" + +# Dashboard access for customers +dashboards_read: + reserved: false + static: false + allowed_actions: + - "kibana_all_read" + description: "Read-only access to dashboards" + +dashboards_write: + reserved: false + static: false + allowed_actions: + - "kibana_all_write" + description: "Write access to dashboards" diff --git a/docker/opensearch-security/allowlist.yml b/docker/opensearch-security/allowlist.yml new file mode 100644 index 00000000..cd934c4a --- /dev/null +++ b/docker/opensearch-security/allowlist.yml @@ -0,0 +1,13 @@ +# OpenSearch Security - API Allowlist +# +# Controls which REST APIs can be accessed. +# Disabled by default (all APIs allowed based on role permissions). + +--- +_meta: + type: "allowlist" + config_version: 2 + +config: + enabled: false + requests: {} diff --git a/docker/opensearch-security/audit.yml b/docker/opensearch-security/audit.yml new file mode 100644 index 00000000..f8fae321 --- /dev/null +++ b/docker/opensearch-security/audit.yml @@ -0,0 +1,30 @@ +# OpenSearch Security - Audit Configuration +# +# Audit logging configuration for security events. + +--- +_meta: + type: "audit" + config_version: 2 + +config: + # Enable audit logging + enabled: true + audit: + # Log successful authentication + enable_rest: true + # Log transport layer (disabled for dev to reduce noise) + enable_transport: false + # What to log + resolve_bulk_requests: false + log_request_body: false + resolve_indices: true + exclude_sensitive_headers: true + # Ignore system indices + ignore_users: + - "kibanaserver" + ignore_requests: + - "SearchRequest" + - "indices:data/read/*" + compliance: + enabled: false diff --git a/docker/opensearch-security/config.yml b/docker/opensearch-security/config.yml new file mode 100644 index 00000000..7af3c434 --- /dev/null +++ b/docker/opensearch-security/config.yml @@ -0,0 +1,47 @@ +# OpenSearch Security Configuration +# +# This file configures authentication domains for the security plugin. +# Proxy auth is used for nginx-authenticated requests (Dashboards access). +# Basic auth is used for direct API access (admin, worker). + +--- +_meta: + type: "config" + config_version: 2 + +config: + dynamic: + http: + xff: + enabled: true + # Trusted proxy IPs - templated at container start by docker-entrypoint-security.sh + # Default matches Docker bridge network (172.x.x.x) + internalProxies: '__INTERNAL_PROXIES__' + remoteIpHeader: 'X-Forwarded-For' + + authc: + # Proxy authentication for nginx-authenticated requests + # Nginx sets x-proxy-user and x-proxy-roles headers after auth validation + proxy_auth_domain: + http_enabled: true + transport_enabled: true + order: 0 + http_authenticator: + type: proxy + challenge: false + config: + user_header: "x-proxy-user" + roles_header: "x-proxy-roles" + authentication_backend: + type: noop + + # Basic auth fallback for direct API access (admin, worker) + basic_internal_auth_domain: + http_enabled: true + transport_enabled: true + order: 1 + http_authenticator: + type: basic + challenge: true + authentication_backend: + type: intern diff --git a/docker/opensearch-security/docker-entrypoint-security.sh b/docker/opensearch-security/docker-entrypoint-security.sh new file mode 100755 index 00000000..83314aab --- /dev/null +++ b/docker/opensearch-security/docker-entrypoint-security.sh @@ -0,0 +1,108 @@ +#!/bin/sh +# OpenSearch Security Entrypoint (Production-Ready) +# +# This entrypoint: +# 1. Templates the internalProxies regex in config.yml +# 2. Launches a background process to initialize security after OpenSearch starts +# 3. Uses a marker file to avoid re-initializing on every restart +# +# Environment variables: +# OPENSEARCH_INTERNAL_PROXIES - Trusted proxy IP regex (default: Docker bridge) +# SECURITY_AUTO_INIT - Auto-initialize security index (default: true) + +set -e + +# Configuration +INTERNAL_PROXIES="${OPENSEARCH_INTERNAL_PROXIES:-(172|192|10)\\.\\d+\\.\\d+\\.\\d+}" +SECURITY_AUTO_INIT="${SECURITY_AUTO_INIT:-true}" +SECURITY_INIT_MARKER="/usr/share/opensearch/data/.security_initialized" + +SRC_CONFIG="/usr/share/opensearch/config/opensearch-security/config.yml" +DEST_DIR="/usr/share/opensearch/config/opensearch-security-templated" +DEST_CONFIG="${DEST_DIR}/config.yml" + +echo "[opensearch-security] Templating internalProxies: ${INTERNAL_PROXIES}" + +if [ -f "${SRC_CONFIG}" ]; then + # Create destination directory if needed + mkdir -p "${DEST_DIR}" + + # Copy and template the config file + sed "s/__INTERNAL_PROXIES__/${INTERNAL_PROXIES}/g" "${SRC_CONFIG}" > "${DEST_CONFIG}" + + # Copy other security config files to the templated directory + for file in /usr/share/opensearch/config/opensearch-security/*.yml; do + filename=$(basename "$file") + if [ "$filename" != "config.yml" ]; then + cp "$file" "${DEST_DIR}/${filename}" + fi + done + + echo "[opensearch-security] Config templating complete" +else + echo "[opensearch-security] WARNING: Config file not found at ${SRC_CONFIG}" +fi + +# Background security initialization function +security_init_background() { + # Wait for OpenSearch to be ready + echo "[opensearch-security] Waiting for OpenSearch to be ready..." + ADMIN_PASSWORD="${OPENSEARCH_ADMIN_PASSWORD:-admin}" + MAX_RETRIES=60 + RETRY_COUNT=0 + + while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do + # Use admin credentials - OpenSearch rejects unauthenticated requests + # even before security is fully initialized + if curl -sf -u "admin:${ADMIN_PASSWORD}" \ + --cacert /usr/share/opensearch/config/certs/root-ca.pem \ + https://localhost:9200/_cluster/health > /dev/null 2>&1; then + echo "[opensearch-security] OpenSearch is ready" + break + fi + RETRY_COUNT=$((RETRY_COUNT + 1)) + sleep 2 + done + + if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then + echo "[opensearch-security] ERROR: OpenSearch not ready after $MAX_RETRIES attempts" + return 1 + fi + + # Always run securityadmin.sh to apply our templated config. + # OpenSearch may auto-init security from the raw config dir (with __INTERNAL_PROXIES__ + # placeholder), so we must overwrite it with the properly templated version. + # The marker file (checked at the outer level) prevents re-runs on subsequent restarts. + echo "[opensearch-security] Applying templated security config with securityadmin.sh..." + /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh \ + -cd "${DEST_DIR}" \ + -icl \ + -nhnv \ + -cacert /usr/share/opensearch/config/certs/root-ca.pem \ + -cert /usr/share/opensearch/config/certs/admin.pem \ + -key /usr/share/opensearch/config/certs/admin-key.pem + + if [ $? -eq 0 ]; then + echo "[opensearch-security] Security initialization complete" + touch "$SECURITY_INIT_MARKER" + else + echo "[opensearch-security] ERROR: Security initialization failed" + return 1 + fi +} + +# Launch background security initialization if enabled and not already done +if [ "${SECURITY_AUTO_INIT}" = "true" ]; then + if [ -f "$SECURITY_INIT_MARKER" ]; then + echo "[opensearch-security] Security previously initialized (marker exists)" + else + echo "[opensearch-security] Will initialize security after OpenSearch starts..." + # Run in background so OpenSearch can start + security_init_background & + fi +else + echo "[opensearch-security] Auto-init disabled (SECURITY_AUTO_INIT=${SECURITY_AUTO_INIT})" +fi + +# Execute the original OpenSearch entrypoint +exec /usr/share/opensearch/opensearch-docker-entrypoint.sh "$@" diff --git a/docker/opensearch-security/internal_users.yml b/docker/opensearch-security/internal_users.yml new file mode 100644 index 00000000..fdb93d83 --- /dev/null +++ b/docker/opensearch-security/internal_users.yml @@ -0,0 +1,72 @@ +# OpenSearch Security - Internal Users (SaaS Model) +# +# USER PROVISIONING STRATEGY: +# Customer users are created dynamically via the Security REST API +# when users are added to the platform. This file only contains +# system users required for platform operations. +# +# Customer user creation example (via backend): +# PUT /_plugins/_security/api/internalusers/{user_email} +# { +# "password": "hashed_password", +# "backend_roles": ["customer_{customer_id}"], +# "attributes": { +# "customer_id": "{customer_id}", +# "email": "{user_email}" +# } +# } +# +# Password hashing: +# docker run -it opensearchproject/opensearch:2.11.1 \ +# /usr/share/opensearch/plugins/opensearch-security/tools/hash.sh -p + +--- +_meta: + type: "internalusers" + config_version: 2 + +# ============================================================================= +# SYSTEM USERS (Platform Operations) +# ============================================================================= + +# Platform admin - for internal operations only +admin: + # Default password: admin (CHANGE IN PRODUCTION!) + hash: "$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG" + reserved: true + backend_roles: + - "admin" + attributes: + role: "system" + description: "Platform administrator - internal use only" + +# Dashboards server user - used by OpenSearch Dashboards +kibanaserver: + # Default password: admin (matches OPENSEARCH_DASHBOARDS_PASSWORD default) + hash: "$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG" + reserved: true + attributes: + role: "system" + description: "Dashboards backend communication user" + +# Worker service user - for indexing security findings from worker processes +worker: + # Default password: worker (CHANGE IN PRODUCTION!) + hash: "$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG" + reserved: false + backend_roles: + - "worker_write" + attributes: + role: "system" + description: "Worker service for indexing security findings" + +# ============================================================================= +# CUSTOMER USERS +# Note: Customer users are created dynamically by the backend when users +# register or are invited to the platform. +# +# Each customer user will have: +# - backend_roles: ["customer_{customer_id}"] +# - attributes.customer_id: their customer ID +# - Mapped to customer-specific role for index isolation +# ============================================================================= diff --git a/docker/opensearch-security/nodes_dn.yml b/docker/opensearch-security/nodes_dn.yml new file mode 100644 index 00000000..566555f1 --- /dev/null +++ b/docker/opensearch-security/nodes_dn.yml @@ -0,0 +1,12 @@ +# OpenSearch Security - Node Distinguished Names +# +# For single-node development, this file is empty. +# In production multi-node clusters, list the DNs of all nodes. + +--- +_meta: + type: "nodesdn" + config_version: 2 + +# Allow all nodes with certificates signed by our CA +# (In production, specify exact node DNs for tighter security) diff --git a/docker/opensearch-security/roles.yml b/docker/opensearch-security/roles.yml new file mode 100644 index 00000000..f2d44c4d --- /dev/null +++ b/docker/opensearch-security/roles.yml @@ -0,0 +1,177 @@ +# OpenSearch Security - Roles Configuration (SaaS Model) +# +# INDEX ISOLATION STRATEGY: +# Each customer's data is stored in indices prefixed with their customer ID: +# {customer_id}-analytics-* +# {customer_id}-workflows-* +# {customer_id}-scans-* +# +# Roles are created dynamically per customer with index patterns that +# restrict access to only their data. This file defines role templates +# and system roles. +# +# Dynamic role creation example (via backend): +# PUT /_plugins/_security/api/roles/customer_{customer_id} +# { +# "cluster_permissions": ["cluster_composite_ops_ro"], +# "index_permissions": [{ +# "index_patterns": ["{customer_id}-*"], +# "allowed_actions": ["read", "indices:data/read/*"] +# }], +# "tenant_permissions": [{ +# "tenant_patterns": ["{customer_id}"], +# "allowed_actions": ["kibana_all_write"] +# }] +# } + +--- +_meta: + type: "roles" + config_version: 2 + +# ============================================================================= +# SYSTEM ROLES (Platform Operations) +# ============================================================================= + +# Platform admin - full access for operators +platform_admin: + reserved: true + cluster_permissions: + - "*" + index_permissions: + - index_patterns: + - "*" + allowed_actions: + - "*" + tenant_permissions: + - tenant_patterns: + - "*" + allowed_actions: + - "kibana_all_write" + +# Worker write role - for indexing security findings from worker processes +# Write-only access to security-findings-* indices (no read of other orgs' data) +worker_write: + reserved: false + description: "Worker service role for indexing security findings" + cluster_permissions: + - "cluster_composite_ops_ro" + - "indices:data/write/*" + index_permissions: + - index_patterns: + - "security-findings-*" + allowed_actions: + - "write" + - "create_index" + - "indices:data/write/*" + - "indices:admin/mapping/put" + +# ============================================================================= +# CUSTOMER ROLE TEMPLATE +# These are templates - actual roles are created dynamically per customer +# ============================================================================= + +# Template: Customer read-write access (for active users) +# Actual role name: customer_{customer_id}_rw +# Index pattern: {customer_id}-* +customer_template_rw: + reserved: false + description: "Template for customer read-write roles - DO NOT USE DIRECTLY" + cluster_permissions: + - "cluster_composite_ops_ro" + - "indices:data/read/scroll*" + # Required for Dashboards saved objects (bulk writes to .kibana_* tenant indices) + - "indices:data/write/bulk" + # Alerting: monitor CRUD, execution, alerts, and destinations (legacy endpoints) + - "cluster:admin/opendistro/alerting/monitor/get" + - "cluster:admin/opendistro/alerting/monitor/search" + - "cluster:admin/opendistro/alerting/monitor/write" + - "cluster:admin/opendistro/alerting/monitor/execute" + - "cluster:admin/opendistro/alerting/alerts/get" + - "cluster:admin/opendistro/alerting/alerts/ack" + - "cluster:admin/opendistro/alerting/destination/get" + - "cluster:admin/opendistro/alerting/destination/write" + - "cluster:admin/opendistro/alerting/destination/delete" + # Notifications plugin (OpenSearch 2.x): channel features + config CRUD + - "cluster:admin/opensearch/notifications/features" + - "cluster:admin/opensearch/notifications/configs/get" + - "cluster:admin/opensearch/notifications/configs/create" + - "cluster:admin/opensearch/notifications/configs/update" + - "cluster:admin/opensearch/notifications/configs/delete" + index_permissions: + - index_patterns: + - "CUSTOMER_ID_PLACEHOLDER-*" + allowed_actions: + - "read" + - "write" + - "create_index" + - "indices:data/read/*" + - "indices:data/write/*" + - "indices:admin/mapping/put" + - index_patterns: + - ".kibana*" + allowed_actions: + - "read" + - "write" + - "create_index" + - "indices:data/read/*" + - "indices:data/write/*" + - "indices:admin/mapping/put" + tenant_permissions: + - tenant_patterns: + - "CUSTOMER_ID_PLACEHOLDER" + allowed_actions: + - "kibana_all_write" + +# Template: Customer read-only access (for viewers) +# Actual role name: customer_{customer_id}_ro +# Index pattern: {customer_id}-* +customer_template_ro: + reserved: false + description: "Template for customer read-only roles - DO NOT USE DIRECTLY" + cluster_permissions: + - "cluster_composite_ops_ro" + # Required for Dashboards saved objects (bulk writes to .kibana_* tenant indices) + - "indices:data/write/bulk" + # Alerting: monitor CRUD, execution, alerts, and destinations (legacy endpoints) + - "cluster:admin/opendistro/alerting/monitor/get" + - "cluster:admin/opendistro/alerting/monitor/search" + - "cluster:admin/opendistro/alerting/monitor/write" + - "cluster:admin/opendistro/alerting/monitor/execute" + - "cluster:admin/opendistro/alerting/alerts/get" + - "cluster:admin/opendistro/alerting/alerts/ack" + - "cluster:admin/opendistro/alerting/destination/get" + - "cluster:admin/opendistro/alerting/destination/write" + - "cluster:admin/opendistro/alerting/destination/delete" + # Notifications plugin (OpenSearch 2.x): channel features + config CRUD + - "cluster:admin/opensearch/notifications/features" + - "cluster:admin/opensearch/notifications/configs/get" + - "cluster:admin/opensearch/notifications/configs/create" + - "cluster:admin/opensearch/notifications/configs/update" + - "cluster:admin/opensearch/notifications/configs/delete" + index_permissions: + - index_patterns: + - "CUSTOMER_ID_PLACEHOLDER-*" + allowed_actions: + - "read" + - "indices:data/read/*" + - index_patterns: + - ".kibana*" + allowed_actions: + - "read" + - "write" + - "create_index" + - "indices:data/read/*" + - "indices:data/write/*" + - "indices:admin/mapping/put" + tenant_permissions: + - tenant_patterns: + - "CUSTOMER_ID_PLACEHOLDER" + allowed_actions: + - "kibana_all_write" + +# ============================================================================= +# DASHBOARDS INTERNAL ROLES +# Note: kibana_server and kibana_read_only are built-in static roles +# Do NOT redefine them here - they are managed by OpenSearch Security plugin +# ============================================================================= diff --git a/docker/opensearch-security/roles_mapping.yml b/docker/opensearch-security/roles_mapping.yml new file mode 100644 index 00000000..69636259 --- /dev/null +++ b/docker/opensearch-security/roles_mapping.yml @@ -0,0 +1,73 @@ +# OpenSearch Security - Roles Mapping (SaaS Model) +# +# DYNAMIC ROLE MAPPING: +# Customer role mappings are created dynamically when users are provisioned. +# Each customer user is mapped to their customer-specific role. +# +# Example dynamic mapping creation (via backend): +# PUT /_plugins/_security/api/rolesmapping/customer_{customer_id}_rw +# { +# "users": ["user@customer.com"], +# "backend_roles": ["customer_{customer_id}"] +# } +# +# The backend should: +# 1. Create customer tenant when customer onboards +# 2. Create customer role (customer_{id}_rw or customer_{id}_ro) +# 3. Map user to customer role when user is added + +--- +_meta: + type: "rolesmapping" + config_version: 2 + +# ============================================================================= +# SYSTEM ROLE MAPPINGS +# ============================================================================= + +# Platform admin mapping - internal operators only +platform_admin: + reserved: true + users: + - "admin" + backend_roles: + - "platform_admin" + description: "Platform administrators with full system access" + +# Dashboards server mapping +kibana_server: + reserved: true + users: + - "kibanaserver" + description: "OpenSearch Dashboards server user" + +# Security REST API access - for admin operations +security_rest_api_access: + reserved: true + users: + - "admin" + backend_roles: + - "platform_admin" + description: "Access to Security REST API for tenant/role management" + +# Worker service mapping - for indexing security findings +worker_write: + reserved: false + users: + - "worker" + backend_roles: + - "worker_write" + description: "Worker service for indexing security findings" + +# ============================================================================= +# CUSTOMER ROLE MAPPINGS +# Note: Customer-specific mappings are created dynamically by the backend +# when customers and users are provisioned. +# +# Pattern for dynamic mappings: +# Role: customer_{customer_id}_rw +# Users: [list of customer's users with write access] +# +# Role: customer_{customer_id}_ro +# Users: [list of customer's users with read-only access] +# ============================================================================= diff --git a/docker/opensearch-security/tenants.yml b/docker/opensearch-security/tenants.yml new file mode 100644 index 00000000..ae9d0393 --- /dev/null +++ b/docker/opensearch-security/tenants.yml @@ -0,0 +1,28 @@ +# OpenSearch Security - Tenants Configuration (SaaS Model) +# +# TENANT ISOLATION STRATEGY: +# Each customer gets their own isolated tenant and index pattern. +# No shared/global dashboards - sharing is explicitly opt-in. +# +# Tenants are created dynamically via the Security REST API when +# a new customer is onboarded. Tenant name = customer ID. +# +# Index naming convention: {customer_id}-analytics-* +# Each customer's role restricts access to only their indices. +# +# Example dynamic tenant creation (via backend): +# POST /_plugins/_security/api/tenants/{customer_id} +# { "description": "Tenant for customer {customer_id}" } + +--- +_meta: + type: "tenants" + config_version: 2 + +# NOTE: Customer tenants are created dynamically by the application backend +# when customers are onboarded. This file only contains system tenants. + +# Admin tenant - for platform operators only (not customers) +__platform_admin: + reserved: true + description: "Platform administration - internal use only" diff --git a/docker/opensearch-security/whitelist.yml b/docker/opensearch-security/whitelist.yml new file mode 100644 index 00000000..cb55f2a9 --- /dev/null +++ b/docker/opensearch-security/whitelist.yml @@ -0,0 +1,13 @@ +# OpenSearch Security - API Whitelist (legacy name for allowlist) +# +# This file is required by securityadmin.sh even in OpenSearch 2.x. +# Actual configuration is in allowlist.yml. + +--- +_meta: + type: 'whitelist' + config_version: 2 + +config: + enabled: false + requests: {} diff --git a/docker/opensearch.dev-secure.yml b/docker/opensearch.dev-secure.yml new file mode 100644 index 00000000..f9f0494a --- /dev/null +++ b/docker/opensearch.dev-secure.yml @@ -0,0 +1,35 @@ +# OpenSearch Development Configuration with Security +# Mount to: /usr/share/opensearch/config/opensearch.yml + +cluster.name: shipsec-dev-secure +node.name: opensearch-node1 +network.host: 0.0.0.0 + +# Single-node mode +discovery.type: single-node +bootstrap.memory_lock: true + +# Security Plugin Configuration +plugins.security.ssl.transport.pemcert_filepath: certs/node.pem +plugins.security.ssl.transport.pemkey_filepath: certs/node-key.pem +plugins.security.ssl.transport.pemtrustedcas_filepath: certs/root-ca.pem +plugins.security.ssl.transport.enforce_hostname_verification: false + +plugins.security.ssl.http.enabled: true +plugins.security.ssl.http.pemcert_filepath: certs/node.pem +plugins.security.ssl.http.pemkey_filepath: certs/node-key.pem +plugins.security.ssl.http.pemtrustedcas_filepath: certs/root-ca.pem + +plugins.security.allow_unsafe_democertificates: false +plugins.security.allow_default_init_securityindex: true + +# Admin DN - Required for securityadmin.sh and REST API access +plugins.security.authcz.admin_dn: + - "CN=admin,OU=ShipSec,O=ShipSecAI,L=SF,ST=CA,C=US" + +plugins.security.audit.type: internal_opensearch +plugins.security.enable_snapshot_restore_privilege: true +plugins.security.check_snapshot_restore_write_privileges: true +plugins.security.restapi.roles_enabled: + - "all_access" + - "security_rest_api_access" diff --git a/docker/scripts/generate-certs.sh b/docker/scripts/generate-certs.sh new file mode 100755 index 00000000..4a04c2fd --- /dev/null +++ b/docker/scripts/generate-certs.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Generate TLS certificates for OpenSearch production deployment +# +# This script creates: +# - Root CA certificate and key +# - Node certificate for OpenSearch (server) +# - Admin certificate for cluster management +# +# Usage: ./generate-certs.sh [output-dir] +# +# Requirements: openssl + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUTPUT_DIR="${1:-$SCRIPT_DIR/../certs}" +DAYS_VALID=365 + +# Certificate Subject fields +COUNTRY="US" +STATE="CA" +LOCALITY="SF" +ORGANIZATION="ShipSecAI" +ORG_UNIT="ShipSec" + +echo "=== OpenSearch Certificate Generator ===" +echo "Output directory: $OUTPUT_DIR" +echo "" + +# Create output directory +mkdir -p "$OUTPUT_DIR" +cd "$OUTPUT_DIR" + +# Check if certificates already exist +if [[ -f "root-ca.pem" ]]; then + echo "WARNING: Certificates already exist in $OUTPUT_DIR" + read -p "Overwrite existing certificates? (y/N): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "Aborted." + exit 1 + fi +fi + +echo "1. Generating Root CA..." +openssl genrsa -out root-ca-key.pem 2048 +openssl req -new -x509 -sha256 -key root-ca-key.pem -out root-ca.pem -days $DAYS_VALID \ + -subj "/C=$COUNTRY/ST=$STATE/L=$LOCALITY/O=$ORGANIZATION/OU=$ORG_UNIT/CN=Root CA" + +echo "2. Generating Admin Certificate..." +openssl genrsa -out admin-key-temp.pem 2048 +openssl pkcs8 -inform PEM -outform PEM -in admin-key-temp.pem -topk8 -nocrypt -out admin-key.pem +openssl req -new -key admin-key.pem -out admin.csr \ + -subj "/C=$COUNTRY/ST=$STATE/L=$LOCALITY/O=$ORGANIZATION/OU=$ORG_UNIT/CN=admin" +openssl x509 -req -in admin.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial \ + -sha256 -out admin.pem -days $DAYS_VALID +rm admin-key-temp.pem admin.csr + +echo "3. Generating Node Certificate..." +# Create extension file for SAN (Subject Alternative Names) +cat > node-ext.cnf << EOF +subjectAltName = DNS:localhost, DNS:opensearch, DNS:opensearch-node1, IP:127.0.0.1 +EOF + +openssl genrsa -out node-key-temp.pem 2048 +openssl pkcs8 -inform PEM -outform PEM -in node-key-temp.pem -topk8 -nocrypt -out node-key.pem +openssl req -new -key node-key.pem -out node.csr \ + -subj "/C=$COUNTRY/ST=$STATE/L=$LOCALITY/O=$ORGANIZATION/OU=$ORG_UNIT/CN=opensearch-node1" +openssl x509 -req -in node.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial \ + -sha256 -out node.pem -days $DAYS_VALID -extfile node-ext.cnf +rm node-key-temp.pem node.csr node-ext.cnf + +echo "4. Setting permissions..." +chmod 600 *-key.pem +chmod 644 *.pem + +echo "" +echo "=== Certificates Generated Successfully ===" +echo "" +echo "Files created in $OUTPUT_DIR:" +ls -la "$OUTPUT_DIR" +echo "" +echo "Next steps:" +echo " 1. Review the certificates" +echo " 2. Set OPENSEARCH_ADMIN_PASSWORD and OPENSEARCH_DASHBOARDS_PASSWORD environment variables" +echo " 3. Run: docker compose -f docker-compose.infra.yml -f docker-compose.prod.yml up -d" +echo "" +echo "For production deployments:" +echo " - Use proper certificate authority (e.g., Let's Encrypt, internal CA)" +echo " - Store private keys securely (e.g., HashiCorp Vault, AWS Secrets Manager)" +echo " - Rotate certificates before expiration ($DAYS_VALID days)" diff --git a/docker/scripts/hash-password.sh b/docker/scripts/hash-password.sh new file mode 100755 index 00000000..22ba22ce --- /dev/null +++ b/docker/scripts/hash-password.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Generate BCrypt password hash for OpenSearch Security internal users +# +# Usage: ./hash-password.sh [password] +# +# If password is not provided, it will be read from stdin (useful for piping) +# The hash can be used in opensearch-security/internal_users.yml +# +# Example: +# ./hash-password.sh mySecurePassword123 +# echo "myPassword" | ./hash-password.sh + +set -euo pipefail + +OPENSEARCH_IMAGE="${OPENSEARCH_IMAGE:-opensearchproject/opensearch:2.11.1}" + +if [ $# -ge 1 ]; then + PASSWORD="$1" +elif [ ! -t 0 ]; then + # Read from stdin if piped + read -r PASSWORD +else + # Interactive prompt + echo -n "Enter password to hash: " >&2 + read -rs PASSWORD + echo >&2 +fi + +if [ -z "$PASSWORD" ]; then + echo "Error: Password cannot be empty" >&2 + exit 1 +fi + +# Use OpenSearch's built-in hash.sh tool to generate BCrypt hash +docker run --rm -i "$OPENSEARCH_IMAGE" \ + /usr/share/opensearch/plugins/opensearch-security/tools/hash.sh \ + -p "$PASSWORD" 2>/dev/null | tail -1 diff --git a/docker/scripts/security-init.sh b/docker/scripts/security-init.sh new file mode 100755 index 00000000..0e8cd3e0 --- /dev/null +++ b/docker/scripts/security-init.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# Initialize OpenSearch Security index using securityadmin.sh +# +# This script properly initializes the security configuration without using +# the deprecated demo installer. It should be run: +# - After first-time OpenSearch startup +# - After modifying security configuration files +# - When migrating from demo to production security +# +# Prerequisites: +# - OpenSearch must be running with TLS enabled +# - Admin certificates must exist in docker/certs/ +# - Security config files in docker/opensearch-security/ +# +# Usage: +# ./security-init.sh # Use defaults +# ./security-init.sh --force # Force reinitialize (overwrites existing) +# OPENSEARCH_HOST=my-host ./security-init.sh # Custom host + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DOCKER_DIR="$SCRIPT_DIR/.." + +# Configuration +OPENSEARCH_HOST="${OPENSEARCH_HOST:-opensearch}" +OPENSEARCH_PORT="${OPENSEARCH_PORT:-9200}" +CERTS_DIR="${CERTS_DIR:-$DOCKER_DIR/certs}" +SECURITY_CONFIG_DIR="${SECURITY_CONFIG_DIR:-$DOCKER_DIR/opensearch-security}" +CONTAINER_NAME="${OPENSEARCH_CONTAINER:-shipsec-opensearch}" + +# Parse arguments +FORCE_INIT=false +while [[ $# -gt 0 ]]; do + case $1 in + --force|-f) + FORCE_INIT=true + shift + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +echo "=== OpenSearch Security Initialization ===" +echo "" +echo "Configuration:" +echo " Container: $CONTAINER_NAME" +echo " Certs dir: $CERTS_DIR" +echo " Security dir: $SECURITY_CONFIG_DIR" +echo " Force init: $FORCE_INIT" +echo "" + +# Verify prerequisites +if [ ! -f "$CERTS_DIR/admin.pem" ] || [ ! -f "$CERTS_DIR/admin-key.pem" ]; then + echo "Error: Admin certificates not found in $CERTS_DIR" + echo "Run: just generate-certs" + exit 1 +fi + +if [ ! -f "$CERTS_DIR/root-ca.pem" ]; then + echo "Error: Root CA certificate not found in $CERTS_DIR" + exit 1 +fi + +if [ ! -d "$SECURITY_CONFIG_DIR" ]; then + echo "Error: Security config directory not found: $SECURITY_CONFIG_DIR" + exit 1 +fi + +# Check if OpenSearch container is running +if ! docker ps --filter "name=$CONTAINER_NAME" --format "{{.Names}}" | grep -q "$CONTAINER_NAME"; then + echo "Error: OpenSearch container '$CONTAINER_NAME' is not running" + echo "Start it first with: just dev or just prod-secure" + exit 1 +fi + +# Wait for OpenSearch to be ready +echo "Waiting for OpenSearch to be ready..." +MAX_RETRIES=30 +for i in $(seq 1 $MAX_RETRIES); do + if docker exec "$CONTAINER_NAME" curl -sf \ + --cacert /usr/share/opensearch/config/certs/root-ca.pem \ + https://localhost:9200/_cluster/health > /dev/null 2>&1; then + echo "OpenSearch is ready!" + break + fi + + if [ $i -eq $MAX_RETRIES ]; then + echo "Error: OpenSearch not ready after $MAX_RETRIES attempts" + exit 1 + fi + + echo " Waiting... (attempt $i/$MAX_RETRIES)" + sleep 2 +done + +# Check if security index already exists +echo "" +echo "Checking security index status..." +SECURITY_STATUS=$(docker exec "$CONTAINER_NAME" curl -sf \ + --cacert /usr/share/opensearch/config/certs/root-ca.pem \ + https://localhost:9200/_plugins/_security/health 2>/dev/null || echo "not_initialized") + +if echo "$SECURITY_STATUS" | grep -q '"status":"UP"'; then + if [ "$FORCE_INIT" != "true" ]; then + echo "Security index already initialized." + echo "Use --force to reinitialize (this will overwrite existing configuration)" + exit 0 + fi + echo "Security index exists, but --force specified. Reinitializing..." +else + echo "Security index not initialized. Proceeding with initialization..." +fi + +# Copy security config files to container (in case they've been updated) +echo "" +echo "Copying security configuration to container..." +docker cp "$SECURITY_CONFIG_DIR/." "$CONTAINER_NAME:/usr/share/opensearch/config/opensearch-security-init/" + +# Run securityadmin.sh +echo "" +echo "Running securityadmin.sh to initialize security index..." +docker exec "$CONTAINER_NAME" /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh \ + -cd /usr/share/opensearch/config/opensearch-security-init \ + -icl \ + -nhnv \ + -cacert /usr/share/opensearch/config/certs/root-ca.pem \ + -cert /usr/share/opensearch/config/certs/admin.pem \ + -key /usr/share/opensearch/config/certs/admin-key.pem + +# Verify initialization +echo "" +echo "Verifying security initialization..." +sleep 2 +FINAL_STATUS=$(docker exec "$CONTAINER_NAME" curl -sf \ + --cacert /usr/share/opensearch/config/certs/root-ca.pem \ + https://localhost:9200/_plugins/_security/health 2>/dev/null || echo "{}") + +if echo "$FINAL_STATUS" | grep -q '"status":"UP"'; then + echo "" + echo "=== Security Initialization Complete ===" + echo "" + echo "Security plugin status: UP" + echo "" + echo "Next steps:" + echo " - Test authentication: curl -u admin:PASSWORD --cacert docker/certs/root-ca.pem https://localhost:9200" + echo " - Update internal_users.yml with production password hashes" + echo " - Re-run this script with --force after updating passwords" +else + echo "" + echo "Warning: Security initialization may have failed" + echo "Check OpenSearch logs: docker logs $CONTAINER_NAME" + exit 1 +fi diff --git a/docs/analytics.md b/docs/analytics.md new file mode 100644 index 00000000..2deb769e --- /dev/null +++ b/docs/analytics.md @@ -0,0 +1,380 @@ +# Analytics Pipeline + +This document describes the analytics infrastructure for ShipSec Studio, including OpenSearch for data storage, OpenSearch Dashboards for visualization, and the routing architecture. + +## Architecture Overview + +``` +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ Nginx (port 80) ā”‚ +ā”‚ ā”‚ +ā”‚ /analytics/* ā”€ā”€ā”€ā”€ā”€ā”€ā–ŗ OpenSearch Dashboards (5601) ā”‚ +ā”‚ /api/* ā”€ā”€ā”€ā”€ā”€ā”€ā–ŗ Backend API (3211) ā”‚ +ā”‚ /* ā”€ā”€ā”€ā”€ā”€ā”€ā–ŗ Frontend SPA (8080) ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ + ā–¼ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ Worker Service ā”‚ +ā”‚ ā”‚ +ā”‚ Analytics Sink Component ā”€ā”€ā”€ā”€ā”€ā”€ā–ŗ OpenSearch (9200) ā”‚ +ā”‚ (OPENSEARCH_URL env var) ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ +``` + +## Components + +### OpenSearch (Port 9200) + +Time-series database for storing security findings and workflow analytics. + +**Configuration:** + +- Single-node deployment (dev/simple prod) +- Security plugin disabled for development +- Index pattern: `security-findings-{org-id}-{date}` + +### OpenSearch Dashboards (Port 5601) + +Web UI for exploring and visualizing analytics data. + +**Configuration (`opensearch-dashboards.yml`):** + +```yaml +server.basePath: '/analytics' +server.rewriteBasePath: true +opensearch.hosts: ['http://opensearch:9200'] +``` + +**Key Settings:** + +- `basePath: "/analytics"` - All URLs are prefixed with `/analytics` +- `rewriteBasePath: true` - Strips `/analytics` from incoming requests, adds it back to responses + +### Analytics Sink (Worker Component) + +The `core.analytics.sink` component writes workflow results to OpenSearch. + +**Input Ports:** + +- Ships with a default `input1` port so at least one connector is always available. +- Users can configure additional input ports via the **Data Inputs** parameter + (e.g., to aggregate results from multiple scanners into one index). +- Extra ports are resolved dynamically through the `resolvePorts` mechanism. When + loading a saved workflow the backend calls `resolveGraphPorts()` server-side; + when importing from a JSON file the frontend calls `resolvePorts` per-node to + ensure all dynamic handles are present before rendering. + +**Environment Variable:** + +```yaml +OPENSEARCH_URL=http://opensearch:9200 +``` + +**Document Structure:** + +```json +{ + "@timestamp": "2026-01-25T01:22:43.783Z", + "title": "Finding title", + "severity": "high", + "description": "...", + "shipsec": { + "organization_id": "local-dev", + "run_id": "shipsec-run-xxx", + "workflow_id": "workflow-xxx", + "workflow_name": "My Workflow", + "component_id": "core.analytics.sink", + "node_ref": "analytics-sink-123" + } +} +``` + +## Nginx Routing + +All traffic flows through Nginx on port 80: + +| Path | Target | Description | +| -------------- | ---------------------------- | ------------------------ | +| `/analytics/*` | `opensearch-dashboards:5601` | Analytics dashboard UI | +| `/api/*` | `backend:3211` | Backend REST API | +| `/*` | `frontend:8080` | Frontend SPA (catch-all) | + +### OpenSearch Dashboards Routing Details + +The `/analytics` route requires special handling: + +1. **Authentication**: Routes are protected - users must be logged in to access +2. **Session Cookies**: Backend validates session cookies for analytics route auth +3. **BasePath Configuration**: OpenSearch Dashboards is configured with `server.basePath: "/analytics"` +4. **Proxy Pass**: Nginx forwards requests to OpenSearch Dashboards without path rewriting +5. **rewriteBasePath**: OpenSearch Dashboards strips `/analytics` internally and adds it back to URLs + +```nginx +location /analytics/ { + proxy_pass http://opensearch-dashboards; + proxy_set_header osd-xsrf "true"; + proxy_cookie_path /analytics/ /analytics/; +} +``` + +## Frontend Integration + +The frontend links to OpenSearch Dashboards Discover app with pre-filtered queries: + +```typescript +const baseUrl = '/analytics'; +// Use .keyword fields for exact match filtering +const filterQuery = `shipsec.run_id.keyword:"${runId}"`; + +// Build Discover URL with proper state format +const gParam = encodeURIComponent('(time:(from:now-7d,to:now))'); +const aParam = encodeURIComponent( + `(columns:!(_source),index:'security-findings-*',interval:auto,query:(language:kuery,query:'${filterQuery}'),sort:!('@timestamp',desc))`, +); +const url = `${baseUrl}/app/discover#/?_g=${gParam}&_a=${aParam}`; + +// Open in new tab +window.open(url, '_blank', 'noopener,noreferrer'); +``` + +**Key points:** + +- Use `.keyword` fields (e.g., `shipsec.run_id.keyword`) for exact match filtering +- Use Discover app (`/app/discover`) for viewing raw data without saved views +- Include `index`, `columns`, `interval`, and `sort` in the `_a` param + +**Environment Variable:** + +``` +VITE_OPENSEARCH_DASHBOARDS_URL=/analytics +``` + +## Data Flow + +1. **Workflow Execution**: Worker runs workflow with Analytics Sink component +2. **Data Enrichment**: Analytics Sink adds `shipsec.*` metadata fields +3. **Indexing**: Documents bulk-indexed to OpenSearch via `OPENSEARCH_URL` +4. **Visualization**: Users explore data in OpenSearch Dashboards at `/analytics` + +## Analytics API Limits + +To protect OpenSearch and keep queries responsive: + +- `size` must be a non-negative integer and is capped at **1000** +- `from` must be a non-negative integer and is capped at **10000** + +Requests exceeding these limits return `400 Bad Request`. + +## Analytics Settings Updates + +The analytics settings update API supports **partial updates**: + +- `analyticsRetentionDays` is optional +- `subscriptionTier` is optional + +Omit fields you donā€™t want to change. The backend validates the retention days only when provided. + +## Multi-Tenant Architecture + +When the OpenSearch Security plugin is enabled, each organization gets an isolated tenant with its own dashboards, index patterns, and saved objects. + +### How Tenant Identity Is Resolved + +The tenant identity for OpenSearch Dashboards is determined through a proxy auth flow: + +1. **Browser navigation** to `/analytics/` sends the Clerk `__session` cookie +2. **nginx** sends an `auth_request` to the backend (`/api/v1/auth/validate`) +3. **Backend** decodes the JWT from the cookie and resolves the organization: + - If the JWT contains `org_id` (active Clerk organization session) ā†’ uses `org_id` as tenant + - If the JWT has no `org_id` ā†’ falls back to `workspace-{userId}` (personal workspace) +4. **nginx** forwards the resolved identity via `x-proxy-user`, `x-proxy-roles`, and `securitytenant` headers + +### Important: Clerk Active Organization Session + +The Clerk JWT `__session` cookie only contains `org_id` when the user has an **active organization session**. This is different from organization membership: + +| Concept | Source | Contains org info? | +| ------------------------- | --------------------------------------------- | ---------------------------------- | +| `organizationMemberships` | Clerk User object (frontend SDK) | Lists ALL orgs the user belongs to | +| JWT `org_id` | `__session` cookie (cryptographically signed) | Only the ACTIVE org, if any | + +If a user is a member of an organization but hasn't activated it (via Clerk's `OrganizationSwitcher` or `setActive()`), their JWT won't contain `org_id`, and they'll land in a personal workspace tenant instead of their organization's tenant. + +### Tenant Provisioning + +When a new `org_id` is seen during auth validation, the backend automatically provisions: + +- An OpenSearch **tenant** named after the org ID +- A **role** (`customer_{orgId}_ro`) with read access to `security-findings-{orgId}-*` indices and `kibana_all_write` tenant permissions +- A **role mapping** linking the role to the proxy auth backend role +- An **index template** and **seed index** with field mappings so index patterns resolve correctly +- A default **index pattern** (`security-findings-{orgId}-*`) in the tenant's saved objects + +### Security Guarantees + +- The JWT is cryptographically signed by Clerk ā€” `org_id` cannot be forged +- The backend validates `X-Organization-Id` headers against the JWT's `org_id` ā€” cross-tenant header spoofing is rejected +- Each tenant has isolated roles, index patterns, and saved objects +- The `workspace-{userId}` fallback creates an isolated personal sandbox ā€” no data leaks between tenants + +## Troubleshooting + +### OpenSearch Dashboards Shows `workspace-user_...` Instead of Organization Name + +**Symptom:** The user profile dropdown in OpenSearch Dashboards shows `workspace-user_{clerkUserId}` instead of the organization ID (e.g., `org_...`). The dashboard appears empty because all indexed data is under the organization's tenant, not the personal workspace. + +**Expected (org tenant active):** + +![OpenSearch showing org ID as tenant](media/opensearch-tenant-org-id.png) + +**Broken (workspace fallback):** + +![OpenSearch showing workspace-user fallback](media/opensearch-tenant-workspace-fallback.png) + +**Root Cause:** The user's Clerk session does not have an active organization. The Clerk JWT (`__session` cookie) only includes `org_id` when the organization is explicitly activated via `OrganizationSwitcher` or `clerk.setActive({ organization: orgId })`. Without an active org, the backend falls back to `workspace-{userId}`. + +This can happen when: + +- The user signed up and was added to an org but never selected it in the UI +- The user's Clerk session expired and was recreated without org context +- The frontend didn't call `setActive()` after login + +**Clerk Dashboard ā€” user in "local" org (but no active session, causing fallback):** + +![Clerk user in local org](media/clerk-user-local-org.png) + +**Clerk Dashboard ā€” user in "Test Organization" (active session, working correctly):** + +![Clerk user in test org](media/clerk-user-test-org.png) + +**Diagnosis:** + +```bash +# Check backend logs for the auth resolution path +docker logs shipsec-backend 2>&1 | grep -E "\[AUTH\].*Resolving org|No org found|Using org" + +# Example log when org is missing from JWT: +# [AUTH] Resolving org - Header: not present, JWT org: none, User: user_39ey3oxc0... +# [AUTH] No org found, using workspace: workspace-user_39ey3oxc0... + +# Example log when org is correctly resolved: +# [AUTH] Resolving org - Header: not present, JWT org: org_30cuor7xe..., User: user_abc... +# [AUTH] Using org from JWT payload: org_30cuor7xe... +``` + +**Solution:** + +1. Have the user switch to their organization using the Organization Switcher in the app UI +2. Ensure the frontend calls `clerk.setActive({ organization: orgId })` after login when the user belongs to an organization +3. After switching, refresh the `/analytics/` page ā€” the tenant should now show the org ID + +**Security Note:** This is a UX issue, not a security vulnerability. The `workspace-user_...` fallback creates an isolated empty sandbox. No data leaks between tenants. See the [Security Guarantees](#security-guarantees) section above. + +### Accessing OpenSearch Dashboards as Admin (Maintenance) + +For maintenance tasks (managing indices, debugging tenant provisioning, viewing all tenants, etc.), you need admin-level access to OpenSearch Dashboards. + +**Why normal `/analytics/` access won't work as admin:** +The nginx `/analytics/` route always injects org-scoped proxy headers (`x-proxy-user: org__user`). Since proxy auth (order 0) takes priority over basic auth (order 1), OpenSearch ignores any admin credentials and authenticates you as the org-scoped user instead. + +**How to access as admin:** + +Access Dashboards directly on port 5601, bypassing nginx entirely. Without proxy headers, the basic auth fallback activates. + +**Development (port already exposed):** + +``` +http://localhost:5601 +``` + +Log in with the admin credentials defined in `docker/opensearch-security/internal_users.yml` (default: `admin` / `admin`). + +**Production (port not publicly exposed):** + +Use SSH port forwarding to tunnel to the server's Dashboards port: + +```bash +ssh -L 5601:localhost:5601 user@your-production-server +``` + +Then open `http://localhost:5601` locally. + +If the Dashboards container doesn't bind to the host network, find its Docker IP first: + +```bash +# On the production server +docker inspect opensearch-dashboards | grep IPAddress + +# Then tunnel to that IP +ssh -L 5601::5601 user@your-production-server +``` + +**Admin capabilities:** + +- View and manage all tenants +- Inspect index mappings and document counts +- Debug role mappings and security configuration +- Manage ISM policies and index lifecycle +- Access the Security plugin UI at `/app/security-dashboards-plugin` + +### Analytics Sink Not Writing Data + +**Symptom:** New workflow runs don't appear in OpenSearch + +**Check:** + +```bash +# Verify worker has OPENSEARCH_URL set +docker exec shipsec-worker env | grep OPENSEARCH + +# Check worker logs for indexing errors +docker logs shipsec-worker 2>&1 | grep -i "analytics\|indexing" +``` + +**Solution:** Ensure `OPENSEARCH_URL=http://opensearch:9200` is set in worker environment. + +### OpenSearch Dashboards Shows Blank Page + +**Symptom:** Page loads but content area is empty + +**Check:** + +1. Browser console for JavaScript errors +2. Time range filter (data might be outside selected range) +3. Index pattern selection + +**Solution:** + +- Set time range to "Last 30 days" or wider +- Ensure `security-findings-*` index pattern is selected + +### Query Returns No Results + +**Check if data exists:** + +```bash +# Count documents +curl -s "http://localhost:9200/security-findings-*/_count" | jq '.count' + +# List run_ids with data +curl -s "http://localhost:9200/security-findings-*/_search" \ + -H "Content-Type: application/json" \ + -d '{"size":0,"aggs":{"run_ids":{"terms":{"field":"shipsec.run_id.keyword"}}}}' \ + | jq '.aggregations.run_ids.buckets' +``` + +## Environment Variables + +| Variable | Service | Description | +| -------------------------------- | -------- | ----------------------------- | +| `OPENSEARCH_URL` | Worker | OpenSearch connection URL | +| `OPENSEARCH_USERNAME` | Worker | Optional: OpenSearch username | +| `OPENSEARCH_PASSWORD` | Worker | Optional: OpenSearch password | +| `VITE_OPENSEARCH_DASHBOARDS_URL` | Frontend | Dashboard URL for links | + +## See Also + +- [Docker README](../docker/README.md) - Docker deployment configurations +- [nginx.prod.conf](../docker/nginx/nginx.prod.conf) - Container-mode nginx routing (full stack + production) +- [opensearch-dashboards.yml](../docker/opensearch-dashboards.yml) - Dashboard configuration diff --git a/docs/components/core.mdx b/docs/components/core.mdx index 101e79e0..3ad0c765 100644 --- a/docs/components/core.mdx +++ b/docs/components/core.mdx @@ -205,3 +205,44 @@ Provides AWS credentials for S3 operations. | Output | Type | Description | |--------|------|-------------| | `credentials` | Object | Credential object for S3 components | + +--- + +## Analytics + +### Analytics Sink + +Indexes workflow output data into OpenSearch for analytics dashboards, queries, and alerts. Connect the `results` port from upstream security scanners. + +| Input | Type | Description | +|-------|------|-------------| +| `data` | Any | Data to index. Works best with `list` from scanner `results` ports. | + +| Output | Type | Description | +|--------|------|-------------| +| `indexed` | Boolean | Whether data was successfully indexed | +| `documentCount` | Number | Number of documents indexed | +| `indexName` | String | Name of the OpenSearch index used | + +| Parameter | Type | Description | +|-----------|------|-------------| +| `indexSuffix` | String | Custom suffix for the index name. Defaults to slugified workflow name. | +| `assetKeyField` | Select | Field to use as asset identifier. Options: auto, asset_key, host, domain, subdomain, url, ip, asset, target, custom | +| `customAssetKeyField` | String | Custom field name when assetKeyField is "custom" | +| `failOnError` | Boolean | When enabled, workflow stops if indexing fails. Default: false (fire-and-forget) | + +**How it works:** + +1. Each item in the input array becomes a separate document +2. Workflow context is added under `shipsec.*` namespace +3. Nested objects are serialized to JSON strings (prevents field explosion) +4. All documents get the same `@timestamp` + +**Example use cases:** +- Index Nuclei scan results for trend analysis +- Store TruffleHog secrets for tracking over time +- Aggregate vulnerability data across workflows + + + See [Workflow Analytics](/development/workflow-analytics) for detailed setup and querying guide. + diff --git a/docs/development/analytics.mdx b/docs/development/analytics.mdx index 69ee9956..a4d91a3f 100644 --- a/docs/development/analytics.mdx +++ b/docs/development/analytics.mdx @@ -1,5 +1,5 @@ --- -title: "Analytics" +title: "Product Analytics (PostHog)" description: "PostHog integration for product analytics and session recording" --- diff --git a/docs/development/component-development.mdx b/docs/development/component-development.mdx index 853151f1..77151fdb 100644 --- a/docs/development/component-development.mdx +++ b/docs/development/component-development.mdx @@ -405,7 +405,166 @@ async execute({ inputs }, context) { --- +## Analytics Output Port (Results) +Security components should include a `results` output port for analytics integration. This port outputs structured findings that can be indexed into OpenSearch via the Analytics Sink. + +### Schema Requirements + +The `results` port must output `list` (array of records): + +```typescript +outputs: outputs({ + // ... other outputs ... + + results: port(z.array(z.record(z.string(), z.unknown())), { + label: 'Results', + description: + 'Analytics-ready findings array. Each item includes scanner name and asset key. Connect to Analytics Sink.', + connectionType: { kind: 'list', element: { kind: 'primitive', name: 'json' } }, + }), +}), +``` + +### Required Fields + +Each finding in the results array **must** include: + +| Field | Type | Description | +|-------|------|-------------| +| `scanner` | string | Scanner identifier (e.g., `'nuclei'`, `'trufflehog'`, `'supabase-scanner'`) | +| `asset_key` | string | Primary asset identifier (host, domain, target, etc.) | +| `finding_hash` | string | Stable hash for deduplication (16-char hex from SHA-256) | + +Additional fields from the scanner output should be spread into the finding object. + +### Finding Hash + +The `finding_hash` is a stable identifier that enables deduplication across workflow runs. It should be generated from the key identifying fields of each finding. + +**Purpose:** +- Track if a finding is **new** or **recurring** across scans +- Deduplicate findings in dashboards +- Calculate **first-seen** and **last-seen** timestamps +- Identify which findings have been **resolved** (no longer appearing) + +**How to generate:** + +Import from the component SDK: + +```typescript +import { generateFindingHash } from '@shipsec/component-sdk'; + +// Usage +const hash = generateFindingHash(finding.templateId, finding.host, finding.matchedAt); +``` + +**Key fields per scanner:** + +| Scanner | Fields Used | +|---------|-------------| +| Nuclei | `templateId + host + matchedAt` | +| TruffleHog | `DetectorType + Redacted + filePath` | +| Supabase Scanner | `check_id + projectRef + resource` | + +Choose fields that uniquely identify a finding but remain stable across runs (avoid timestamps, random IDs, etc.). + +### Example Implementation + +```typescript +import { generateFindingHash } from '@shipsec/component-sdk'; + +async execute({ inputs, params }, context) { + // ... run scanner and get findings ... + + // Build analytics-ready results with scanner metadata + const results: Record[] = findings.map((finding) => ({ + ...finding, // Spread all finding fields + scanner: 'my-scanner', // Scanner identifier + asset_key: finding.host ?? inputs.target, // Primary asset + finding_hash: generateFindingHash( // Stable deduplication hash + finding.ruleId, + finding.host, + finding.matchedAt + ), + })); + + return { + findings, // Original findings array + results, // Analytics-ready array for Analytics Sink + rawOutput, // Raw output for debugging + }; +} +``` + +### How It Works + +1. **Component outputs `results`**: Each scanner outputs its findings with `scanner` and `asset_key` fields +2. **Connect to Analytics Sink**: In the workflow canvas, connect the `results` port to Analytics Sink's `data` input +3. **Indexed to OpenSearch**: Each item in the array becomes a separate document with: + - Finding data at root level (nested objects serialized to JSON strings) + - Workflow context under `shipsec.*` namespace + - Consistent `@timestamp` for all findings in the batch + +### Document Structure in OpenSearch + +```json +{ + "check_id": "DB_RLS_DISABLED", + "severity": "CRITICAL", + "title": "RLS Disabled", + "metadata": "{\"table\":\"users\"}", + "scanner": "supabase-scanner", + "asset_key": "abc123xyz", + "finding_hash": "a1b2c3d4e5f67890", + "shipsec": { + "organization_id": "org_123", + "run_id": "run_abc123", + "workflow_id": "wf_xyz789", + "workflow_name": "Supabase Security Audit", + "component_id": "core.analytics.sink", + "node_ref": "analytics-sink-1" + }, + "@timestamp": "2024-01-21T10:30:00Z" +} +``` + +### `shipsec` Context Fields + +The Analytics Sink automatically adds workflow context under the `shipsec` namespace: + +| Field | Description | +|-------|-------------| +| `organization_id` | Organization that owns the workflow | +| `run_id` | Unique identifier for this workflow execution | +| `workflow_id` | ID of the workflow definition | +| `workflow_name` | Human-readable workflow name | +| `component_id` | Component type (e.g., `core.analytics.sink`) | +| `node_ref` | Node reference in the workflow graph | +| `asset_key` | Auto-detected or specified asset identifier | + +### Example Queries + +``` +# Find all findings for an asset +asset_key: "api.example.com" + +# Find new findings (first seen today) +finding_hash: X AND @timestamp: [now-1d TO now] AND NOT (finding_hash: X AND @timestamp: [* TO now-1d]) + +# All findings from a specific workflow run +shipsec.run_id: "run_abc123" + +# Aggregate findings by scanner +scanner: * | stats count() by scanner + +# Track recurring findings +finding_hash: "a1b2c3d4" | sort @timestamp +``` + + + Nested objects in findings are automatically serialized to JSON strings to prevent OpenSearch field explosion (1000 field limit). + --- diff --git a/docs/development/workflow-analytics.mdx b/docs/development/workflow-analytics.mdx new file mode 100644 index 00000000..52b612e5 --- /dev/null +++ b/docs/development/workflow-analytics.mdx @@ -0,0 +1,373 @@ +--- +title: "Workflow Analytics" +description: "Index security findings into OpenSearch for dashboards, queries, and alerting" +--- + +ShipSec Studio includes a workflow analytics system that indexes security findings into OpenSearch. This enables real-time dashboards, historical trend analysis, and alerting on security data. + +--- + +## Overview + +The analytics system consists of: + +1. **Analytics Sink component** - Indexes workflow output data into OpenSearch +2. **Results output port** - Structured findings from security scanners +3. **OpenSearch storage** - Time-series index for querying and visualization +4. **View Analytics button** - Quick access to filtered dashboards + +--- + +## Architecture + +```mermaid +flowchart LR + subgraph Scanners + N[Nuclei Scan] + T[TruffleHog] + S[Supabase Scanner] + end + + AS[Analytics Sink] + OS[(OpenSearch)] + + subgraph Dashboard[OpenSearch Dashboards] + V[Visualizations] + A[Alerts] + Q[Queries] + end + + N -->|results port| AS + T -->|results port| AS + S -->|results port| AS + AS --> OS + OS --> Dashboard +``` + +Each scanner outputs findings through its `results` port, which connects to the Analytics Sink. The sink indexes each finding as a separate document with workflow metadata. + +--- + +## Document Structure + +Indexed documents follow this structure: + +```json +{ + "check_id": "DB_RLS_DISABLED", + "severity": "CRITICAL", + "title": "RLS Disabled on Table: users", + "resource": "public.users", + "metadata": "{\"schema\":\"public\",\"table\":\"users\"}", + "scanner": "supabase-scanner", + "asset_key": "abcdefghij1234567890", + "finding_hash": "a1b2c3d4e5f67890", + "shipsec": { + "organization_id": "org_123", + "run_id": "shipsec-run-xxx", + "workflow_id": "d1d33161-929f-4af4-9a64-xxx", + "workflow_name": "Supabase Security Audit", + "component_id": "core.analytics.sink", + "node_ref": "analytics-sink-1" + }, + "@timestamp": "2025-01-21T10:30:00.000Z" +} +``` + +### Field Categories + +| Category | Fields | Description | +|----------|--------|-------------| +| Finding data | `check_id`, `severity`, `title`, etc. | Scanner-specific fields at root level | +| Asset tracking | `scanner`, `asset_key`, `finding_hash` | Required fields for analytics | +| Workflow context | `shipsec.*` | Automatic metadata from the workflow | +| Timestamp | `@timestamp` | Indexing timestamp | + + + Nested objects in findings are automatically serialized to JSON strings to prevent OpenSearch field explosion (1000 field limit). + + +--- + +## `shipsec` context fields + +The Analytics Sink automatically adds workflow context under the `shipsec` namespace: + +| Field | Description | +|-------|-------------| +| `organization_id` | Organization that owns the workflow | +| `run_id` | Unique identifier for this workflow execution | +| `workflow_id` | ID of the workflow definition | +| `workflow_name` | Human-readable workflow name | +| `component_id` | Component type (always `core.analytics.sink`) | +| `node_ref` | Node reference in the workflow graph | +| `asset_key` | Auto-detected or specified asset identifier | + +--- + +## Finding Hash for Deduplication + +The `finding_hash` is a stable 16-character identifier that enables tracking findings across workflow runs. + +### Purpose + +- **New vs recurring**: Determine if a finding appeared before +- **First-seen / last-seen**: Track when findings were first and last detected +- **Resolution tracking**: Findings that stop appearing may be resolved +- **Deduplication**: Remove duplicates in dashboards across runs + +### Generation + +Each scanner generates the hash from key identifying fields: + +| Scanner | Hash Fields | +|---------|-------------| +| Nuclei | `templateId + host + matchedAt` | +| TruffleHog | `DetectorType + Redacted + filePath` | +| Supabase Scanner | `check_id + projectRef + resource` | + +Fields are normalized (lowercase, trimmed) and hashed with SHA-256, truncated to 16 hex characters. + +--- + +## Querying Data + +### Basic Queries (KQL) + +``` +# Find all findings for an asset +asset_key: "api.example.com" + +# Filter by severity +severity: "CRITICAL" OR severity: "HIGH" + +# Filter by scanner +scanner: "nuclei" + +# All findings from a specific workflow run +shipsec.run_id: "shipsec-run-abc123" + +# Filter by organization +shipsec.organization_id: "org_123" + +# Filter by workflow +shipsec.workflow_id: "d1d33161-929f-4af4-9a64-xxx" +``` + +### Tracking Findings Over Time + +``` +# Track a specific finding across runs +finding_hash: "a1b2c3d4e5f67890" + +# Find recurring findings (multiple runs) +# Use aggregation: group by finding_hash, count occurrences +``` + +### Common Aggregations + +| Aggregation | Use Case | +|-------------|----------| +| `terms` on `severity` | Count findings by severity | +| `terms` on `scanner` | Count findings by scanner | +| `terms` on `asset_key` | Most vulnerable assets | +| `date_histogram` on `@timestamp` | Findings over time | +| `cardinality` on `finding_hash` | Unique findings count | + +--- + +## Setting Up OpenSearch + +### Environment Variables + +Set these in your `worker/.env`: + +```bash +OPENSEARCH_URL=http://localhost:9200 +OPENSEARCH_USERNAME=admin +OPENSEARCH_PASSWORD=admin +``` + +Set this in your `frontend/.env`: + +```bash +VITE_OPENSEARCH_DASHBOARDS_URL=http://localhost:5601 +``` + +### Docker Compose + +The infrastructure stack includes OpenSearch and OpenSearch Dashboards: + +```bash +docker compose -f docker/docker-compose.infra.yml up -d opensearch opensearch-dashboards +``` + +### Index Pattern + +After indexing data, create an index pattern in OpenSearch Dashboards: + +1. Go to **Dashboards Management** > **Index Patterns** +2. Create pattern: `security-findings-*` +3. Select `@timestamp` as the time field +4. Click **Create index pattern** + + + If you don't see `shipsec.*` fields in Available Fields after indexing, refresh the index pattern field list in Dashboards Management. + + +--- + +## Analytics API Limits + +The analytics query API enforces sane bounds to protect OpenSearch: + +- `size` must be a non-negative integer and is capped at **1000** +- `from` must be a non-negative integer and is capped at **10000** + +Requests above these limits return `400 Bad Request`. + +## Analytics Settings Updates + +The analytics settings API supports partial updates: + +- `analyticsRetentionDays` is optional +- `subscriptionTier` is optional + +Omit fields you donā€™t want to change. The backend validates retention days only when provided. + +--- + +## Using Analytics Sink + +### Basic Workflow + +1. Add a security scanner to your workflow (Nuclei, TruffleHog, etc.) +2. Add an Analytics Sink component +3. Connect the scanner's `results` port to the Analytics Sink's `data` input +4. Run the workflow + +### Component Parameters + +| Parameter | Description | +|-----------|-------------| +| **Index Suffix** | Custom suffix for the index name. Defaults to slugified workflow name. | +| **Asset Key Field** | Field to use as asset identifier. Auto-detect checks: asset_key > host > domain > subdomain > url > ip > asset > target | +| **Custom Field Name** | Custom field when Asset Key Field is "custom" | +| **Fail on Error** | When enabled, workflow stops if indexing fails. Default: fire-and-forget. | + +### Fire-and-Forget Mode + +By default, Analytics Sink operates in fire-and-forget mode: +- Indexing errors are logged but don't stop the workflow +- Useful for non-critical analytics that shouldn't block security scans +- Enable "Fail on Error" for strict indexing requirements + +--- + +## View Analytics Button + +The workflow builder includes a "View Analytics" button that opens OpenSearch Dashboards with pre-filtered data: + +- **When a run is selected**: Filters by `shipsec.run_id` +- **When no run is selected**: Filters by `shipsec.workflow_id` +- **Time range**: Last 7 days + +The button only appears when `VITE_OPENSEARCH_DASHBOARDS_URL` is configured. + +--- + +## Index Naming + +Indexes follow the pattern: `security-findings-{orgId}-{suffix}` + +| Component | Value | +|-----------|-------| +| `orgId` | Organization ID from workflow context | +| `suffix` | Custom suffix parameter, or date (`YYYY.MM.DD`) | + +Example: `security-findings-org_abc123-2025.01.21` + +--- + +## Building Dashboards + +### Recommended Visualizations + +| Visualization | Description | +|---------------|-------------| +| **Findings Over Time** | Line chart with `@timestamp` on X-axis, count on Y-axis | +| **Severity Distribution** | Pie chart with `terms` on `severity` | +| **Top Vulnerable Assets** | Bar chart with `terms` on `asset_key` | +| **Findings by Scanner** | Bar chart with `terms` on `scanner` | +| **New vs Recurring** | Use `finding_hash` cardinality vs total count | + +### Alert Examples + +| Alert | Query | +|-------|-------| +| Critical finding detected | `severity: "CRITICAL"` | +| New secrets exposed | `scanner: "trufflehog"` | +| RLS disabled | `check_id: "DB_RLS_DISABLED"` | + +--- + +## Troubleshooting + +### Data not appearing in OpenSearch + +1. Check worker logs for `[OpenSearchIndexer]` messages +2. Verify `OPENSEARCH_URL` is set in worker environment +3. Ensure Analytics Sink is connected to a `results` port +4. Check if OpenSearch is running: `curl http://localhost:9200/_cluster/health` + +### Field mapping errors + +If you see "Limit of total fields [1000] has been exceeded": +1. Delete the problematic index: `curl -X DELETE "http://localhost:9200/security-findings-*"` +2. Re-run the workflow (new index will use correct schema) + +### shipsec fields not visible + +1. Fields starting with `_` are hidden in OpenSearch UI +2. Ensure you're using `shipsec.*` (no underscore prefix) +3. Refresh the index pattern in Dashboards Management + +### pm2 not loading environment variables + +pm2's `env_file` doesn't auto-inject variables. The worker uses a custom `loadWorkerEnv()` function in `pm2.config.cjs`. After changing `worker/.env`: + +```bash +pm2 delete shipsec-worker +pm2 start pm2.config.cjs --only shipsec-worker +``` + +--- + +## Best Practices + +### Do + +- Connect `results` ports (not `rawOutput`) to Analytics Sink +- Use meaningful index suffixes for organization +- Monitor index size and implement retention policies +- Create saved searches for common queries + +### Don't + +- Don't connect deeply nested JSON (causes field explosion) +- Don't rely on analytics for critical workflow logic +- Don't store PII or secrets in indexed findings + +--- + +## Component Author Guidelines + +If you're building a security scanner component, see [Analytics Output Port](/development/component-development#analytics-output-port-results) for implementation details on adding the `results` output port. + +--- + +## Related + +- [Component Development](/development/component-development) - Building scanner components +- [Core Components](/components/core) - Analytics Sink reference +- [Analytics (PostHog)](/development/analytics) - Product analytics (different system) diff --git a/docs/docs.json b/docs/docs.json index d5d0f744..046e4014 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -42,6 +42,7 @@ "pages": [ "development/component-development", "development/isolated-volumes", + "development/workflow-analytics", "development/analytics", "development/release-process" ] diff --git a/docs/installation.mdx b/docs/installation.mdx index c99d868f..2e56a34d 100644 --- a/docs/installation.mdx +++ b/docs/installation.mdx @@ -74,6 +74,68 @@ The `just dev` command automatically: --- +## Analytics Stack (Optional) + +ShipSec Studio includes an optional analytics stack powered by OpenSearch for indexing and visualizing workflow execution data. + +### Starting the Analytics Stack + +The analytics services are included in the infrastructure docker-compose file: + +```bash +# Start infrastructure including OpenSearch +just infra up +``` + +This will start: +- **OpenSearch** on port `9200` - Search and analytics engine +- **OpenSearch Dashboards** on port `5601` - Visualization and query UI + +### Configuring Analytics + +Add these environment variables to your backend and worker `.env` files: + +```bash +# Backend (.env) +OPENSEARCH_URL=http://localhost:9200 +OPENSEARCH_USERNAME=admin +OPENSEARCH_PASSWORD=admin +OPENSEARCH_DASHBOARDS_URL=http://localhost:5601 + +# Frontend (.env) +VITE_OPENSEARCH_DASHBOARDS_URL=http://localhost:5601 +``` + +### Setting Up the Index Template + +After starting OpenSearch, create the security findings index template: + +```bash +cd backend +bun run setup:opensearch +``` + +This creates the `security-findings-*` index template with proper mappings for workflow execution data. + +### Using Analytics + +1. **Analytics Sink Component**: Add the "Analytics Sink" component to your workflows to index output data +2. **Dashboards Link**: Access OpenSearch Dashboards from the Studio sidebar +3. **Query API**: Use the `/api/analytics/query` endpoint to query indexed data programmatically + +### Analytics Service Endpoints + +| Service | URL | Notes | +|---------|-----|-------| +| OpenSearch | http://localhost:9200 | Search engine API | +| OpenSearch Dashboards | http://localhost:5601 | Visualization UI | + + + The analytics stack is optional. If OpenSearch is not configured, the Analytics Sink component will gracefully skip indexing and log a warning. + + +--- + ## Production Deployment For production, use the Docker-based deployment: diff --git a/docs/media/clerk-user-local-org.png b/docs/media/clerk-user-local-org.png new file mode 100644 index 00000000..381734f1 Binary files /dev/null and b/docs/media/clerk-user-local-org.png differ diff --git a/docs/media/clerk-user-test-org.png b/docs/media/clerk-user-test-org.png new file mode 100644 index 00000000..686eb959 Binary files /dev/null and b/docs/media/clerk-user-test-org.png differ diff --git a/docs/media/opensearch-tenant-org-id.png b/docs/media/opensearch-tenant-org-id.png new file mode 100644 index 00000000..58499a32 Binary files /dev/null and b/docs/media/opensearch-tenant-org-id.png differ diff --git a/docs/media/opensearch-tenant-workspace-fallback.png b/docs/media/opensearch-tenant-workspace-fallback.png new file mode 100644 index 00000000..fc422bb5 Binary files /dev/null and b/docs/media/opensearch-tenant-workspace-fallback.png differ diff --git a/docs/workflows/execution-status.md b/docs/workflows/execution-status.md new file mode 100644 index 00000000..1ac61aba --- /dev/null +++ b/docs/workflows/execution-status.md @@ -0,0 +1,101 @@ +# Workflow Execution Status + +This document describes the different execution statuses a workflow run can have and when each status applies. + +## Status Overview + +| Status | Color | Description | +|--------|-------|-------------| +| `QUEUED` | Blue | Workflow is waiting to be executed | +| `RUNNING` | Blue | Workflow is actively executing | +| `COMPLETED` | Green | Workflow finished successfully - all nodes completed | +| `FAILED` | Red | Workflow failed - at least one node failed or workflow crashed | +| `CANCELLED` | Gray | Workflow was cancelled by user | +| `TERMINATED` | Gray | Workflow was forcefully terminated | +| `TIMED_OUT` | Amber | Workflow exceeded maximum execution time | +| `AWAITING_INPUT` | Purple | Workflow is paused waiting for human input | +| `STALE` | Amber | Orphaned record - data inconsistency (see below) | + +## Status Transitions + +``` +QUEUED ā†’ RUNNING ā†’ COMPLETED + ā†’ FAILED + ā†’ CANCELLED + ā†’ TERMINATED + ā†’ TIMED_OUT + ā†’ AWAITING_INPUT ā†’ RUNNING (when input provided) +``` + +## Detailed Status Descriptions + +### QUEUED +The workflow run has been created and is waiting to start execution. This is the initial state before the Temporal worker picks up the workflow. + +### RUNNING +The workflow is actively executing. At least one node has started processing. + +### COMPLETED +All nodes in the workflow have finished successfully. This is a terminal state. + +**Conditions:** +- All expected nodes have `COMPLETED` trace events +- No `FAILED` trace events + +### FAILED +The workflow encountered an error during execution. This is a terminal state. + +**Conditions:** +- At least one node has a `FAILED` trace event, OR +- Some nodes started but not all completed (workflow crashed/lost) + +### CANCELLED +The user manually cancelled the workflow execution. This is a terminal state. + +### TERMINATED +The workflow was forcefully terminated (e.g., via Temporal API). This is a terminal state. + +### TIMED_OUT +The workflow exceeded its maximum allowed execution time. This is a terminal state. + +### AWAITING_INPUT +The workflow has reached a human input node and is waiting for user interaction. The workflow will resume to `RUNNING` when input is provided. + +### STALE +**Special Status - Data Inconsistency Warning** + +The run record exists in the database but there's no evidence it ever executed: +- No trace events in the database +- Temporal has no record of this workflow + +**Common Causes:** +1. **Fresh Temporal instance with old database** - The Temporal server was reset/reinstalled but the application database retained old run records +2. **Failed workflow start** - The backend created a run record but the Temporal workflow failed to start (network error, Temporal unavailable, etc.) +3. **Data migration issues** - Database was migrated without corresponding Temporal data + +**Recommended Action:** +- Review these records and delete them if they represent stale data +- Investigate why the data inconsistency occurred to prevent future occurrences + +## Status Determination Logic + +When querying run status, the system follows this logic: + +1. **Query Temporal** - Get the workflow status from Temporal server +2. **If Temporal returns status** - Use the normalized Temporal status +3. **If Temporal returns NOT_FOUND** - Infer status from trace events: + - No `STARTED` events ā†’ `STALE` (orphaned record) + - Any `FAILED` events ā†’ `FAILED` + - All nodes have `COMPLETED` events ā†’ `COMPLETED` + - Some `STARTED` but incomplete ā†’ `FAILED` (crashed) + +## Frontend Badge Colors + +Status badges use these colors for visual distinction: + +- **Blue** (active): `QUEUED`, `RUNNING` +- **Green** (success): `COMPLETED` +- **Red** (error): `FAILED` +- **Amber** (warning): `TIMED_OUT`, `STALE` +- **Gray** (neutral): `CANCELLED`, `TERMINATED` +- **Purple** (attention): `AWAITING_INPUT` diff --git a/e2e-tests/analytics.test.ts b/e2e-tests/analytics.test.ts new file mode 100644 index 00000000..eed2911d --- /dev/null +++ b/e2e-tests/analytics.test.ts @@ -0,0 +1,274 @@ +/** + * E2E Tests - Workflow Analytics + * + * Validates analytics sink ingestion into OpenSearch and analytics query API. + * + * Requirements: + * - Backend API running on http://localhost:3211 + * - Worker running and component registry loaded + * - OpenSearch running on http://localhost:9200 + */ + +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; + +const API_BASE = 'http://localhost:3211/api/v1'; +const OPENSEARCH_URL = process.env.OPENSEARCH_URL ?? 'http://localhost:9200'; +const HEADERS = { + 'Content-Type': 'application/json', + 'x-internal-token': 'local-internal-token', +}; + +const runE2E = process.env.RUN_E2E === 'true'; + +const servicesAvailableSync = (() => { + if (!runE2E) return false; + try { + const backend = Bun.spawnSync( + [ + 'curl', + '-sf', + '--max-time', + '1', + '-H', + `x-internal-token: ${HEADERS['x-internal-token']}`, + `${API_BASE}/health`, + ], + { stdout: 'pipe', stderr: 'pipe' }, + ); + if (backend.exitCode !== 0) return false; + + const opensearch = Bun.spawnSync( + ['curl', '-sf', '--max-time', '1', `${OPENSEARCH_URL}/_cluster/health`], + { stdout: 'pipe', stderr: 'pipe' }, + ); + return opensearch.exitCode === 0; + } catch { + return false; + } +})(); + +async function checkServicesAvailable(): Promise { + if (!runE2E) return false; + try { + const healthRes = await fetch(`${API_BASE}/health`, { + headers: HEADERS, + signal: AbortSignal.timeout(2000), + }); + if (!healthRes.ok) return false; + + const osRes = await fetch(`${OPENSEARCH_URL}/_cluster/health`, { + signal: AbortSignal.timeout(2000), + }); + return osRes.ok; + } catch { + return false; + } +} + +const e2eDescribe = runE2E && servicesAvailableSync ? describe : describe.skip; + +function e2eTest( + name: string, + optionsOrFn: { timeout?: number } | (() => void | Promise), + fn?: () => void | Promise, +): void { + if (runE2E && servicesAvailableSync) { + if (typeof optionsOrFn === 'function') { + test(name, optionsOrFn); + } else if (fn) { + (test as any)(name, optionsOrFn, fn); + } + } else { + const actualFn = typeof optionsOrFn === 'function' ? optionsOrFn : fn!; + test.skip(name, actualFn); + } +} + +async function pollRunStatus(runId: string, timeoutMs = 180000): Promise<{ status: string }> { + const startTime = Date.now(); + const pollInterval = 1000; + + while (Date.now() - startTime < timeoutMs) { + const res = await fetch(`${API_BASE}/workflows/runs/${runId}/status`, { headers: HEADERS }); + const s = await res.json(); + if (['COMPLETED', 'FAILED', 'CANCELLED'].includes(s.status)) { + return s; + } + await new Promise((resolve) => setTimeout(resolve, pollInterval)); + } + + throw new Error(`Workflow run ${runId} did not complete within ${timeoutMs}ms`); +} + +async function createWorkflow(workflow: any): Promise { + const res = await fetch(`${API_BASE}/workflows`, { + method: 'POST', + headers: HEADERS, + body: JSON.stringify(workflow), + }); + if (!res.ok) { + const error = await res.text(); + throw new Error(`Workflow creation failed: ${res.status} - ${error}`); + } + const { id } = await res.json(); + return id; +} + +async function runWorkflow(workflowId: string, inputs: Record = {}): Promise { + const res = await fetch(`${API_BASE}/workflows/${workflowId}/run`, { + method: 'POST', + headers: HEADERS, + body: JSON.stringify({ inputs }), + }); + if (!res.ok) { + const error = await res.text(); + throw new Error(`Workflow run failed: ${res.status} - ${error}`); + } + const { runId } = await res.json(); + return runId; +} + +async function pollOpenSearch(runId: string, timeoutMs = 60000): Promise { + const startTime = Date.now(); + const pollInterval = 2000; + + const query = { + size: 1, + query: { + term: { + 'shipsec.run_id': runId, + }, + }, + }; + + while (Date.now() - startTime < timeoutMs) { + const res = await fetch(`${OPENSEARCH_URL}/security-findings-*/_search`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(query), + }); + + if (res.ok) { + const body = await res.json(); + const total = + typeof body?.hits?.total === 'object' + ? body.hits.total.value ?? 0 + : body?.hits?.total ?? 0; + + if (total > 0) { + return total; + } + } + + await new Promise((resolve) => setTimeout(resolve, pollInterval)); + } + + throw new Error(`OpenSearch documents not indexed for runId ${runId} within ${timeoutMs}ms`); +} + +let servicesAvailable = false; + +beforeAll(async () => { + if (!runE2E) { + console.log('\n Analytics E2E: Skipping (RUN_E2E not set)'); + return; + } + + console.log('\n Analytics E2E: Verifying services...'); + servicesAvailable = await checkServicesAvailable(); + if (!servicesAvailable) { + console.log(' Required services are not available. Tests will be skipped.'); + return; + } + console.log(' Backend API and OpenSearch are running'); +}); + +afterAll(async () => { + console.log('\n Cleanup: Run "bun e2e-tests/cleanup.ts" to remove test workflows'); +}); + +e2eDescribe('Workflow Analytics E2E Tests', () => { + e2eTest('Analytics Sink indexes results into OpenSearch', { timeout: 180000 }, async () => { + console.log('\n Test: Analytics Sink indexing'); + + const workflow = { + name: 'Test: Analytics Sink E2E', + nodes: [ + { + id: 'start', + type: 'core.workflow.entrypoint', + position: { x: 0, y: 0 }, + data: { + label: 'Start', + config: { params: { runtimeInputs: [] } }, + }, + }, + { + id: 'fixture', + type: 'test.analytics.fixture', + position: { x: 200, y: 0 }, + data: { + label: 'Analytics Fixture', + config: { + params: {}, + }, + }, + }, + { + id: 'sink', + type: 'core.analytics.sink', + position: { x: 400, y: 0 }, + data: { + label: 'Analytics Sink', + config: { + params: { + dataInputs: [ + { id: 'results', label: 'Results', sourceTag: 'fixture' }, + ], + assetKeyField: 'auto', + failOnError: true, + }, + }, + }, + }, + ], + edges: [ + { id: 'e1', source: 'start', target: 'fixture' }, + { id: 'e2', source: 'fixture', target: 'sink' }, + { + id: 'e3', + source: 'fixture', + target: 'sink', + sourceHandle: 'results', + targetHandle: 'results', + }, + ], + }; + + const workflowId = await createWorkflow(workflow); + const runId = await runWorkflow(workflowId); + + const status = await pollRunStatus(runId); + expect(status.status).toBe('COMPLETED'); + + const total = await pollOpenSearch(runId); + expect(total).toBeGreaterThan(0); + + const analyticsRes = await fetch(`${API_BASE}/analytics/query`, { + method: 'POST', + headers: HEADERS, + body: JSON.stringify({ + query: { + term: { + 'shipsec.run_id': runId, + }, + }, + size: 5, + }), + }); + + expect(analyticsRes.ok).toBe(true); + const analyticsBody = await analyticsRes.json(); + expect(analyticsBody.total).toBeGreaterThan(0); + }); +}); diff --git a/frontend/.env.example b/frontend/.env.example index 7bc89a98..8089b93e 100644 --- a/frontend/.env.example +++ b/frontend/.env.example @@ -1,5 +1,5 @@ -# API Configuration -VITE_API_URL=http://localhost:3211 +# API Configuration (nginx /api) +VITE_API_URL=http://localhost # Application Configuration VITE_APP_NAME=Security Workflow Builder @@ -21,3 +21,8 @@ VITE_PUBLIC_POSTHOG_HOST= # Logo.dev public key for brand logos VITE_LOGO_DEV_PUBLIC_KEY= + +# OpenSearch Dashboards (Optional - for Analytics features) +# Leave empty to hide Dashboards navigation link +# For dev/prod: http://localhost/analytics (nginx in dev/prod) +VITE_OPENSEARCH_DASHBOARDS_URL=http://localhost/analytics diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 54d9a741..87fea899 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -13,6 +13,7 @@ import { WebhookEditorPage } from '@/pages/WebhookEditorPage'; import { SchedulesPage } from '@/pages/SchedulesPage'; import { ActionCenterPage } from '@/pages/ActionCenterPage'; import { RunRedirect } from '@/pages/RunRedirect'; +import { AnalyticsSettingsPage } from '@/pages/AnalyticsSettingsPage'; import { ToastProvider } from '@/components/ui/toast-provider'; import { AppLayout } from '@/components/layout/AppLayout'; import { AuthProvider } from '@/auth/auth-context'; @@ -85,6 +86,7 @@ function App() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/frontend/src/auth/AuthProvider.tsx b/frontend/src/auth/AuthProvider.tsx index 062f7932..44a9ac32 100644 --- a/frontend/src/auth/AuthProvider.tsx +++ b/frontend/src/auth/AuthProvider.tsx @@ -7,14 +7,14 @@ import { GlobalAuthContext } from './auth-context-def'; // Auth provider registry - easy to add new providers // Determine which provider to use based on environment function getAuthProviderName(): string { - // Priority: explicit 'local' > dev mode (always local) > environment variable + // Priority: explicit env var > dev mode default (local) > auto-detect const envProvider = import.meta.env.VITE_AUTH_PROVIDER; const hasClerkKey = typeof import.meta.env.VITE_CLERK_PUBLISHABLE_KEY === 'string' && import.meta.env.VITE_CLERK_PUBLISHABLE_KEY.trim().length > 0; - // In dev mode, always use local auth for testing (ignore Clerk settings) - if (import.meta.env.DEV) { + // In dev mode, default to local auth unless VITE_AUTH_PROVIDER is explicitly set + if (import.meta.env.DEV && !envProvider) { return 'local'; } @@ -93,7 +93,17 @@ const LocalAuthProvider: FrontendAuthProviderComponent = ({ signUp: () => { console.warn('Local auth: signUp not implemented'); }, - signOut: () => { + signOut: async () => { + // Clear session cookie via backend logout endpoint + // Use relative path to ensure we hit the same origin as login + try { + await fetch('/api/v1/auth/logout', { + method: 'POST', + credentials: 'include', + }); + } catch (error) { + console.warn('Failed to clear session cookie:', error); + } // Clear admin credentials from store useAuthStore.getState().clear(); }, diff --git a/frontend/src/components/auth/AdminLoginForm.tsx b/frontend/src/components/auth/AdminLoginForm.tsx index 3840dff6..094e3311 100644 --- a/frontend/src/components/auth/AdminLoginForm.tsx +++ b/frontend/src/components/auth/AdminLoginForm.tsx @@ -3,9 +3,8 @@ import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; import { Label } from '@/components/ui/label'; import { useAuthStore } from '@/store/authStore'; -import { useNavigate } from 'react-router-dom'; +import { useNavigate, useSearchParams } from 'react-router-dom'; import { LogIn } from 'lucide-react'; -import { API_V1_URL } from '@/services/api'; export function AdminLoginForm() { const [username, setUsername] = useState(''); @@ -14,6 +13,8 @@ export function AdminLoginForm() { const [isLoading, setIsLoading] = useState(false); const setAdminCredentials = useAuthStore((state) => state.setAdminCredentials); const navigate = useNavigate(); + const [searchParams] = useSearchParams(); + const returnTo = searchParams.get('returnTo'); const handleSubmit = async (e: React.FormEvent) => { e.preventDefault(); @@ -31,28 +32,42 @@ export function AdminLoginForm() { setIsLoading(true); try { - // Test the credentials by making a simple API call - // If it fails, the credentials are invalid + // Validate credentials and set session cookie via /auth/login endpoint + // This sets an httpOnly cookie for browser navigation to protected routes (e.g., /analytics/) + // Use relative path to ensure cookie is set via nginx (same origin as /analytics/* routes) const credentials = btoa(`${trimmedUsername}:${trimmedPassword}`); - const response = await fetch(`${API_V1_URL}/workflows`, { + const loginResponse = await fetch('/api/v1/auth/login', { + method: 'POST', headers: { Authorization: `Basic ${credentials}`, 'Content-Type': 'application/json', }, + credentials: 'include', // Important: include cookies in the response }); - if (!response.ok) { - if (response.status === 401) { + if (!loginResponse.ok) { + if (loginResponse.status === 401) { throw new Error('Invalid username or password'); } - throw new Error(`Authentication failed: ${response.status} ${response.statusText}`); + throw new Error( + `Authentication failed: ${loginResponse.status} ${loginResponse.statusText}`, + ); } - // Store credentials only after verification succeeds + // Store credentials for API requests (Basic auth header) setAdminCredentials(trimmedUsername, trimmedPassword); - // Success - navigate to home - navigate('/'); + // Success - redirect to returnTo URL or home + if (returnTo) { + // For paths like /analytics/*, use full page navigation since they're served by nginx + if (returnTo.startsWith('/analytics')) { + window.location.href = returnTo; + } else { + navigate(returnTo); + } + } else { + navigate('/'); + } } catch (err) { // Clear credentials on error useAuthStore.getState().clear(); diff --git a/frontend/src/components/layout/AppLayout.tsx b/frontend/src/components/layout/AppLayout.tsx index 1204679e..564c4d25 100644 --- a/frontend/src/components/layout/AppLayout.tsx +++ b/frontend/src/components/layout/AppLayout.tsx @@ -24,6 +24,7 @@ import { Zap, Webhook, ServerCog, + BarChart3, Settings, ChevronDown, } from 'lucide-react'; @@ -293,6 +294,16 @@ export function AppLayout({ children }: AppLayoutProps) { href: '/artifacts', icon: Archive, }, + ...(env.VITE_OPENSEARCH_DASHBOARDS_URL + ? [ + { + name: 'Dashboards', + href: env.VITE_OPENSEARCH_DASHBOARDS_URL, + icon: BarChart3, + external: true, + }, + ] + : []), ]; const settingsItems = [ @@ -311,6 +322,15 @@ export function AppLayout({ children }: AppLayoutProps) { href: '/mcp-library', icon: ServerCog, }, + ...(env.VITE_OPENSEARCH_DASHBOARDS_URL + ? [ + { + name: 'Analytics Settings', + href: '/analytics-settings', + icon: Settings, + }, + ] + : []), ]; const isActive = (path: string) => { @@ -411,6 +431,50 @@ export function AppLayout({ children }: AppLayoutProps) { {navigationItems.map((item) => { const Icon = item.icon; const active = isActive(item.href); + const isExternal = 'external' in item && item.external; + const openInNewTab = isExternal && 'newTab' in item ? item.newTab !== false : true; + + // Render external link + if (isExternal) { + return ( + { + // Close sidebar on mobile after clicking + if (isMobile) { + setSidebarOpen(false); + } + }} + > + + + + {item.name} + + + + ); + } + + // Render internal link (React Router) return ( void; onSave: () => Promise | void; @@ -47,6 +52,8 @@ const DEFAULT_WORKFLOW_NAME = 'Untitled Workflow'; export function TopBar({ workflowId, selectedRunId, + selectedRunStatus, + selectedRunOrgId, onRun, onSave, onImport, @@ -68,6 +75,11 @@ export function TopBar({ const { metadata, isDirty, setWorkflowName } = useWorkflowStore(); const mode = useWorkflowUiStore((state) => state.mode); + const organizationId = useAuthStore((s) => s.organizationId); + const authProvider = useAuthStore((s) => s.provider); + // For Clerk auth, org context is ready when organizationId is set to a real value (not default) + // For other providers (local, custom), org context is always ready + const isOrgReady = authProvider !== 'clerk' || organizationId !== DEFAULT_ORG_ID; const canEdit = Boolean(canManageWorkflows); const handleChangeWorkflowName = () => { @@ -454,6 +466,53 @@ export function TopBar({ )} + {env.VITE_OPENSEARCH_DASHBOARDS_URL && + workflowId && + (!selectedRunId || (selectedRunStatus && selectedRunStatus !== 'RUNNING')) && ( + + )} + + + + {inputs.length === 0 ? ( +
+

No data inputs configured

+

+ Configure input ports to receive analytics results from different scanner components. + Each input creates a corresponding input port on this node. +

+ +
+ ) : ( +
+ {inputs.map((input, index) => ( +
+ {/* Header with drag handle and delete */} +
+ + Input {index + 1} + +
+ + {/* ID Field */} +
+ + updateInput(index, 'id', e.target.value)} + placeholder="e.g., nucleiResults" + className="h-8 text-xs font-mono" + /> +

+ Unique identifier (becomes input port ID) +

+
+ + {/* Label Field */} +
+ + updateInput(index, 'label', e.target.value)} + placeholder="e.g., Nuclei Results" + className="h-8 text-xs" + /> +

Display name in workflow editor

+
+ + {/* Source Tag Field */} +
+ + updateInput(index, 'sourceTag', e.target.value)} + placeholder="e.g., nuclei-scan" + className="h-8 text-xs font-mono" + /> +

+ Added to indexed documents as 'source_input' for filtering in dashboards + (optional) +

+
+ + {/* Input Port Preview */} +
+
+
+ + Input port: {input.id} + +
+
+
+ ))} +
+ )} + + {/* Summary */} + {inputs.length > 0 && ( +
+

+ {inputs.length} data input + {inputs.length !== 1 ? 's' : ''} configured +

+

+ {inputs.length} input port{inputs.length !== 1 ? 's' : ''} will be created on this node +

+
+ )} + + ); +} diff --git a/frontend/src/config/env.ts b/frontend/src/config/env.ts index b39537fc..a9eb9fa5 100644 --- a/frontend/src/config/env.ts +++ b/frontend/src/config/env.ts @@ -9,6 +9,7 @@ interface FrontendEnv { VITE_ENABLE_CONNECTIONS: boolean; VITE_ENABLE_IT_OPS: boolean; VITE_API_URL: string; + VITE_OPENSEARCH_DASHBOARDS_URL: string; } export const env: FrontendEnv = { @@ -19,4 +20,6 @@ export const env: FrontendEnv = { VITE_ENABLE_CONNECTIONS: import.meta.env.VITE_ENABLE_CONNECTIONS === 'true', VITE_ENABLE_IT_OPS: import.meta.env.VITE_ENABLE_IT_OPS === 'true', VITE_API_URL: (import.meta.env.VITE_API_URL as string | undefined) ?? '', + VITE_OPENSEARCH_DASHBOARDS_URL: + (import.meta.env.VITE_OPENSEARCH_DASHBOARDS_URL as string | undefined) ?? '', }; diff --git a/frontend/src/features/workflow-builder/WorkflowBuilder.tsx b/frontend/src/features/workflow-builder/WorkflowBuilder.tsx index 53632e79..c40140fd 100644 --- a/frontend/src/features/workflow-builder/WorkflowBuilder.tsx +++ b/frontend/src/features/workflow-builder/WorkflowBuilder.tsx @@ -905,6 +905,8 @@ function WorkflowBuilderContent() { { + const componentId = + (node.data as FrontendNodeData).componentId ?? + (node.data as FrontendNodeData).componentSlug; + if (!componentId) return node; + + try { + const params = node.data.config?.params ?? {}; + const inputOverrides = node.data.config?.inputOverrides ?? {}; + const result = await api.components.resolvePorts(componentId, { + ...params, + ...inputOverrides, + }); + if (!result) return node; + return { + ...node, + data: { + ...node.data, + ...(result.inputs ? { dynamicInputs: result.inputs } : {}), + ...(result.outputs ? { dynamicOutputs: result.outputs } : {}), + }, + }; + } catch { + return node; + } + }), + ); + resetWorkflow(); - setDesignNodes(normalizedNodes); + setDesignNodes(resolvedNodes as ReactFlowNode[]); setDesignEdges(normalizedEdges); - setExecutionNodes(cloneNodes(normalizedNodes)); + setExecutionNodes(cloneNodes(resolvedNodes as ReactFlowNode[])); setExecutionEdges(cloneEdges(normalizedEdges)); setMetadata({ id: null, diff --git a/frontend/src/pages/AnalyticsSettingsPage.tsx b/frontend/src/pages/AnalyticsSettingsPage.tsx new file mode 100644 index 00000000..3a20ab6b --- /dev/null +++ b/frontend/src/pages/AnalyticsSettingsPage.tsx @@ -0,0 +1,258 @@ +import { useEffect, useState } from 'react'; +import { Button } from '@/components/ui/button'; +import { Label } from '@/components/ui/label'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { Badge } from '@/components/ui/badge'; +import { BarChart3, Database, Calendar, AlertCircle } from 'lucide-react'; +import { useAuthStore } from '@/store/authStore'; +import { hasAdminRole } from '@/utils/auth'; + +// Retention period options in days +const RETENTION_PERIODS = [ + { value: '7', label: '7 days', days: 7 }, + { value: '14', label: '14 days', days: 14 }, + { value: '30', label: '30 days', days: 30 }, + { value: '60', label: '60 days', days: 60 }, + { value: '90', label: '90 days', days: 90 }, + { value: '180', label: '180 days (6 months)', days: 180 }, + { value: '365', label: '365 days (1 year)', days: 365 }, +]; + +// Subscription tier limits +const TIER_LIMITS = { + free: { name: 'Free', maxRetentionDays: 30 }, + pro: { name: 'Pro', maxRetentionDays: 90 }, + enterprise: { name: 'Enterprise', maxRetentionDays: 365 }, +}; + +type SubscriptionTier = keyof typeof TIER_LIMITS; + +export function AnalyticsSettingsPage() { + const roles = useAuthStore((state) => state.roles); + const canManageSettings = hasAdminRole(roles); + const isReadOnly = !canManageSettings; + + // Mock data - will be replaced with actual API calls in US-013 + const [currentTier] = useState('free'); + const [retentionPeriod, setRetentionPeriod] = useState('30'); + const [isSubmitting, setIsSubmitting] = useState(false); + const [error, setError] = useState(null); + const [successMessage, setSuccessMessage] = useState(null); + const [storageUsage] = useState<{ used: string; total: string } | null>(null); + + const tierInfo = TIER_LIMITS[currentTier]; + const maxAllowedRetention = tierInfo.maxRetentionDays; + + // Filter retention periods based on tier limit + const availableRetentionPeriods = RETENTION_PERIODS.filter( + (period) => period.days <= maxAllowedRetention, + ); + + useEffect(() => { + // TODO: US-013 will implement API call to fetch current settings + // fetchSettings().catch(console.error); + }, []); + + const handleSave = async () => { + if (isReadOnly) return; + + setError(null); + setSuccessMessage(null); + setIsSubmitting(true); + + try { + // TODO: US-013 will implement API call to save settings + // await api.analytics.updateSettings({ retentionDays: parseInt(retentionPeriod) }); + + // Simulate API call + await new Promise((resolve) => setTimeout(resolve, 500)); + + setSuccessMessage('Analytics settings updated successfully'); + + // Clear success message after 5 seconds + setTimeout(() => setSuccessMessage(null), 5000); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to update analytics settings'); + } finally { + setIsSubmitting(false); + } + }; + + return ( +
+
+ {/* Header */} +
+
+

Analytics Settings

+

+ Configure data retention and storage settings for workflow analytics +

+
+
+ + {/* Read-only warning */} + {isReadOnly && ( +
+
+ +
+

+ Read-Only Access +

+

+ You need admin privileges to modify analytics settings. +

+
+
+
+ )} + + {/* Error Message */} + {error && ( +
+ {error} +
+ )} + + {/* Success Message */} + {successMessage && ( +
+ {successMessage} +
+ )} + + {/* Settings Cards */} +
+ {/* Subscription Tier Card */} +
+
+
+ +
+
+

Subscription Tier

+

+ Your current subscription tier and analytics limits +

+
+ + {tierInfo.name} + + + Maximum retention: {maxAllowedRetention} days + +
+
+
+
+ + {/* Data Retention Card */} +
+
+
+ +
+
+

Data Retention Period

+

+ How long to keep analytics data before automatic deletion +

+ +
+ + +

+ Analytics data older than {retentionPeriod} days will be automatically deleted. +

+
+ +
+ +
+
+
+
+ + {/* Storage Usage Card */} +
+
+
+ +
+
+

Storage Usage

+

+ Current storage consumption for analytics data +

+ +
+ {storageUsage ? ( +
+
+ Used + {storageUsage.used} +
+
+ Total Available + {storageUsage.total} +
+ {/* Progress bar could be added here */} +
+ ) : ( +
+ Storage usage information will be available once analytics data is collected. +
+ )} +
+
+
+
+
+ + {/* Info Box */} +
+

About Analytics Data

+
    +
  • + ā€¢ Analytics data is indexed from workflow executions using the Analytics Sink + component +
    • +
  • ā€¢ Data includes security findings, scan results, and other workflow outputs
    • +
  • ā€¢ You can query this data via the API or view it in OpenSearch Dashboards
    • +
  • ā€¢ Retention settings apply organization-wide and cannot exceed your tier limit
    • +
+
+
+
+ ); +} diff --git a/frontend/src/store/runStore.ts b/frontend/src/store/runStore.ts index 4cc61a55..cd3141de 100644 --- a/frontend/src/store/runStore.ts +++ b/frontend/src/store/runStore.ts @@ -7,6 +7,7 @@ import type { ExecutionTriggerType, ExecutionInputPreview } from '@shipsec/share export interface ExecutionRun { id: string; workflowId: string; + organizationId?: string; workflowName: string; status: ExecutionStatus; startTime: string; @@ -107,6 +108,7 @@ const normalizeRun = (run: any): ExecutionRun => { return { id: String(run.id ?? ''), workflowId: String(run.workflowId ?? ''), + organizationId: typeof run.organizationId === 'string' ? run.organizationId : undefined, workflowName: String(run.workflowName ?? 'Untitled workflow'), status, startTime, diff --git a/frontend/src/utils/statusBadgeStyles.ts b/frontend/src/utils/statusBadgeStyles.ts index ada7699a..6036f4c7 100644 --- a/frontend/src/utils/statusBadgeStyles.ts +++ b/frontend/src/utils/statusBadgeStyles.ts @@ -17,6 +17,7 @@ export const STATUS_COLOR_MAP: Record = { TERMINATED: 'gray', TIMED_OUT: 'amber', AWAITING_INPUT: 'purple', + STALE: 'amber', // Orphaned record - data inconsistency warning }; /** diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index e8818ab3..00d75468 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -23,9 +23,21 @@ export default defineConfig({ host: '0.0.0.0', port: 5173, open: false, - allowedHosts: ['studio.shipsec.ai'], + allowedHosts: ['studio.shipsec.ai', 'frontend'], + proxy: { + '/api': { + target: 'http://localhost:3211', + changeOrigin: true, + secure: false, + }, + '/analytics': { + target: 'http://localhost:5601', + changeOrigin: true, + secure: false, + }, + }, }, preview: { - allowedHosts: ['studio.shipsec.ai'], + allowedHosts: ['studio.shipsec.ai', 'frontend'], }, }) diff --git a/justfile b/justfile index 145cf510..37c64463 100644 --- a/justfile +++ b/justfile @@ -6,26 +6,12 @@ default: @just help -# Set/show the workspace "active" instance used when you run `just dev` without an explicit instance. -# This is stored in `.shipsec-instance` (gitignored). -instance action="show" value="": - #!/usr/bin/env bash - set -euo pipefail - case "{{action}}" in - show) - ./scripts/active-instance.sh get - ;; - use|set) - ./scripts/active-instance.sh set "{{value}}" - ;; - *) - echo "Usage: just instance [show|use] [0-9]" - exit 1 - ;; - esac - # === Development (recommended for contributors) === +# Default dev passwords for convenience (override with env vars for real security) +export OPENSEARCH_ADMIN_PASSWORD := env_var_or_default("OPENSEARCH_ADMIN_PASSWORD", "admin") +export OPENSEARCH_DASHBOARDS_PASSWORD := env_var_or_default("OPENSEARCH_DASHBOARDS_PASSWORD", "admin") + # Initialize environment files from examples init: #!/usr/bin/env bash @@ -52,82 +38,28 @@ init: echo " Then run: just dev" # Start development environment with hot-reload -# Usage: just dev [instance] [action] -# Examples: just dev, just dev 1, just dev 2 start, just dev 1 logs, just dev stop all -dev *args: +# Auto-detects auth mode: if CLERK_SECRET_KEY is set in backend/.env ā†’ secure mode (Clerk + OpenSearch Security) +# Otherwise ā†’ local auth mode (faster startup, no multi-tenant isolation) +dev action="start": #!/usr/bin/env bash set -euo pipefail - - # Parse arguments: instance can be 0-9, action is start/stop/logs/status/clean - INSTANCE="$(./scripts/active-instance.sh get)" - ACTION="start" - INFRA_PROJECT_NAME="shipsec-infra" - - # Process arguments - for arg in {{args}}; do - case "$arg" in - [0-9]) - INSTANCE="$arg" - ;; - all) - # Special instance selector for bulk operations (e.g. `just dev stop all`) - INSTANCE="all" - ;; - start|stop|logs|status|clean|all) - ACTION="$arg" - ;; - *) - echo "āŒ Unknown argument: $arg" - echo "Usage: just dev [instance] [action]" - echo " instance: 0-9 (default: 0)" - echo " action: start|stop|logs|status|clean" - exit 1 - ;; - esac - done - - # Handle special case: dev stop all - if [ "$ACTION" = "all" ]; then - ACTION="stop" - fi - - # Handle "just dev stop" as "just dev 0 stop" - if [ "$ACTION" = "stop" ] && [ "$INSTANCE" = "0" ] && [ -z "{{args}}" ]; then - true # Keep defaults - fi - # Validate "all" usage - if [ "$INSTANCE" = "all" ] && [ "$ACTION" != "stop" ] && [ "$ACTION" != "status" ] && [ "$ACTION" != "logs" ] && [ "$ACTION" != "clean" ]; then - echo "āŒ Instance 'all' is only supported for: stop|status|logs|clean" - exit 1 + # Auto-detect auth mode from backend/.env + CLERK_KEY="" + if [ -f "backend/.env" ]; then + CLERK_KEY=$(grep -E '^CLERK_SECRET_KEY=' backend/.env | cut -d= -f2- | tr -d '"' | tr -d "'" | xargs || true) fi - - # Get ports for this instance (skip for "all") - if [ "$INSTANCE" != "all" ]; then - eval "$(./scripts/dev-instance-manager.sh ports "$INSTANCE")" - INSTANCE_DIR=".instances/instance-$INSTANCE" - export FRONTEND BACKEND + + if [ -n "$CLERK_KEY" ]; then + SECURE_MODE=true + else + SECURE_MODE=false fi - - case "$ACTION" in + + case "{{action}}" in start) - echo "šŸš€ Starting development environment (instance $INSTANCE)..." - - # Initialize instance if needed - if [ ! -d "$INSTANCE_DIR" ]; then - ./scripts/dev-instance-manager.sh init "$INSTANCE" - fi - # Check for required env files - if [ ! -f "$INSTANCE_DIR/backend.env" ] || [ ! -f "$INSTANCE_DIR/worker.env" ] || [ ! -f "$INSTANCE_DIR/frontend.env" ]; then - echo "āŒ Environment files not found in $INSTANCE_DIR!" - echo "" - echo " Attempting to initialize instance $INSTANCE..." - ./scripts/dev-instance-manager.sh init "$INSTANCE" - fi - - # Check for original env files if instance is 0 - if [ "$INSTANCE" = "0" ] && { [ ! -f "backend/.env" ] || [ ! -f "worker/.env" ] || [ ! -f "frontend/.env" ]; }; then + if [ ! -f "backend/.env" ] || [ ! -f "worker/.env" ] || [ ! -f "frontend/.env" ]; then echo "āŒ Environment files not found!" echo "" echo " Run this first: just init" @@ -135,218 +67,184 @@ dev *args: echo " This will create .env files from the example templates." exit 1 fi - - # Start shared infrastructure (one stack for all instances) - echo "ā³ Starting shared infrastructure..." - docker compose -f docker/docker-compose.infra.yml \ - --project-name="$INFRA_PROJECT_NAME" \ - up -d - - # Wait for Postgres - echo "ā³ Waiting for infrastructure..." - POSTGRES_CONTAINER="$(docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" ps -q postgres)" - if [ -n "$POSTGRES_CONTAINER" ]; then - timeout 30s bash -c "until docker exec $POSTGRES_CONTAINER pg_isready -U shipsec >/dev/null 2>&1; do sleep 1; done" || true + + if [ "$SECURE_MODE" = "true" ]; then + echo "šŸ” Starting development environment (Clerk auth detected)..." + + # Auto-generate certificates if they don't exist + if [ ! -f "docker/certs/root-ca.pem" ]; then + echo "šŸ” Generating TLS certificates..." + chmod +x docker/scripts/generate-certs.sh + docker/scripts/generate-certs.sh + echo "āœ… Certificates generated" + fi + + # Start infrastructure with security enabled + # Note: dev-ports.yml exposes OpenSearch on localhost for backend tenant provisioning + echo "šŸš€ Starting infrastructure with OpenSearch Security..." + docker compose -f docker/docker-compose.infra.yml -f docker/docker-compose.dev-secure.yml -f docker/docker-compose.dev-ports.yml up -d + + # Wait for Postgres + echo "ā³ Waiting for infrastructure..." + timeout 30s bash -c 'until docker exec shipsec-postgres pg_isready -U shipsec >/dev/null 2>&1; do sleep 1; done' || true + + # Wait for OpenSearch to be healthy (security init takes longer) + echo "ā³ Waiting for OpenSearch security initialization..." + timeout 120s bash -c 'until docker exec shipsec-opensearch curl -sf -u admin:${OPENSEARCH_ADMIN_PASSWORD:-admin} --cacert /usr/share/opensearch/config/certs/root-ca.pem https://localhost:9200/_cluster/health >/dev/null 2>&1; do sleep 2; done' || true + + # Update git SHA and start PM2 with security enabled + ./scripts/set-git-sha.sh || true + SHIPSEC_ENV=development NODE_ENV=development OPENSEARCH_SECURITY_ENABLED=true NODE_TLS_REJECT_UNAUTHORIZED=0 \ + pm2 startOrReload pm2.config.cjs --only shipsec-frontend-0,shipsec-backend-0,shipsec-worker-0 --update-env + + echo "" + echo "āœ… Development environment ready (secure mode)" + echo " App: http://localhost (via nginx)" + echo " API: http://localhost/api" + echo " Analytics: http://localhost/analytics (requires login)" + echo " Temporal UI: http://localhost:8081" + echo "" + echo "šŸ” OpenSearch Security: ENABLED (multi-tenant isolation active)" + echo " OpenSearch admin: admin / ${OPENSEARCH_ADMIN_PASSWORD:-admin}" + echo "" + echo "šŸ’” Direct ports (debugging): Frontend :5173, Backend :3211" + else + echo "šŸš€ Starting development environment (local auth)..." + + # Start infrastructure (no security, with dev ports for analytics) + docker compose -f docker/docker-compose.infra.yml -f docker/docker-compose.dev-ports.yml up -d + + # Wait for Postgres + echo "ā³ Waiting for infrastructure..." + timeout 30s bash -c 'until docker exec shipsec-postgres pg_isready -U shipsec >/dev/null 2>&1; do sleep 1; done' || true + + # Update git SHA and start PM2 + ./scripts/set-git-sha.sh || true + SHIPSEC_ENV=development NODE_ENV=development OPENSEARCH_SECURITY_ENABLED=false \ + OPENSEARCH_URL=http://localhost:9200 \ + pm2 startOrReload pm2.config.cjs --only shipsec-frontend-0,shipsec-backend-0,shipsec-worker-0 --update-env + + echo "" + echo "āœ… Development environment ready (local auth)" + echo " App: http://localhost (via nginx)" + echo " Analytics: http://localhost/analytics" + echo " Temporal UI: http://localhost:8081" + echo "" + echo "šŸ’” Direct ports (debugging): Frontend :5173, Backend :3211, OpenSearch :9200, Dashboards :5601" + echo "" + echo "šŸ’” To enable Clerk auth + OpenSearch Security:" + echo " Set CLERK_SECRET_KEY in backend/.env, then restart" fi - # Ensure instance-specific DB/namespace exists and migrations are applied. - ./scripts/instance-bootstrap.sh "$INSTANCE" - - # Prepare PM2 environment variables - export SHIPSEC_INSTANCE="$INSTANCE" - export SHIPSEC_ENV=development - export NODE_ENV=development - export TERMINAL_REDIS_URL="redis://localhost:6379" - export LOG_KAFKA_BROKERS="localhost:19092" - export EVENT_KAFKA_BROKERS="localhost:19092" - - # Update git SHA and start PM2 with instance-specific config - ./scripts/set-git-sha.sh || true - - pm2 startOrReload pm2.config.cjs \ - --only "shipsec-frontend-$INSTANCE,shipsec-backend-$INSTANCE,shipsec-worker-$INSTANCE" \ - --update-env - - echo "" - echo "āœ… Development environment ready (instance $INSTANCE)" - ./scripts/dev-instance-manager.sh info "$INSTANCE" echo "" - echo "šŸ’” just dev $INSTANCE logs - View application logs" - echo "šŸ’” just dev $INSTANCE stop - Stop this instance" + echo "šŸ’” just dev logs - View application logs" + echo "šŸ’” just dev stop - Stop everything" + echo "šŸ’” just dev clean - Stop and remove all data" echo "" - + # Version check bun backend/scripts/version-check-summary.ts 2>/dev/null || true ;; stop) - if [ "$INSTANCE" = "all" ]; then - echo "šŸ›‘ Stopping all development environments..." - - # Stop all PM2 apps - pm2 delete shipsec-{frontend,backend,worker}-{0..9} 2>/dev/null || true - pm2 delete shipsec-test-worker 2>/dev/null || true - - # Stop shared infrastructure - just infra down - - echo "āœ… All development environments stopped" + echo "šŸ›‘ Stopping development environment..." + pm2 delete shipsec-frontend-0 shipsec-backend-0 shipsec-worker-0 shipsec-test-worker 2>/dev/null || true + if [ "$SECURE_MODE" = "true" ]; then + docker compose -f docker/docker-compose.infra.yml -f docker/docker-compose.dev-secure.yml -f docker/docker-compose.dev-ports.yml down else - echo "šŸ›‘ Stopping development environment (instance $INSTANCE)..." - - # Stop PM2 apps for this instance - pm2 delete shipsec-{frontend,backend,worker}-"$INSTANCE" 2>/dev/null || true - - echo "āœ… Instance $INSTANCE stopped" + docker compose -f docker/docker-compose.infra.yml -f docker/docker-compose.dev-ports.yml down fi + echo "āœ… Stopped" ;; logs) - if [ "$INSTANCE" = "all" ]; then - echo "šŸ“‹ Viewing logs for all instances..." - pm2 logs - else - echo "šŸ“‹ Viewing logs for instance $INSTANCE..." - pm2 logs "shipsec-frontend-$INSTANCE|shipsec-backend-$INSTANCE|shipsec-worker-$INSTANCE" - fi + pm2 logs ;; status) - if [ "$INSTANCE" = "all" ]; then - just status + pm2 status + if [ "$SECURE_MODE" = "true" ]; then + docker compose -f docker/docker-compose.infra.yml -f docker/docker-compose.dev-secure.yml -f docker/docker-compose.dev-ports.yml ps else - echo "šŸ“Š Status of instance $INSTANCE:" - echo "" - pm2 status 2>/dev/null | grep -E "shipsec-(frontend|backend|worker)-$INSTANCE|error" || echo "(Instance $INSTANCE not running in PM2)" - echo "" - just status + docker compose -f docker/docker-compose.infra.yml -f docker/docker-compose.dev-ports.yml ps fi ;; clean) - if [ "$INSTANCE" = "all" ]; then - echo "šŸ§¹ Cleaning all instances (0-9)..." - - # Stop all instance-specific PM2 apps - pm2 delete shipsec-{frontend,backend,worker}-{0..9} 2>/dev/null || true - - # Clean infra state for each instance - for i in {0..9}; do - if [ -d ".instances/instance-$i" ] || [ "$i" = "0" ]; then - echo " - Instance $i..." - ./scripts/instance-clean.sh "$i" >/dev/null 2>&1 || true - rm -rf ".instances/instance-$i" - fi - done - - # Cleanup root level instance marker if it exists - rm -f .shipsec-instance - - # Also clean global infra if requested? - # User usually runs `just infra clean` for that, but let's remind them. - echo "" - echo "šŸ’” To also wipe all Docker volumes (PSQL, Kafka, etc.), run: just infra clean" - echo "āœ… All instance-specific state cleaned" + echo "šŸ§¹ Cleaning development environment..." + pm2 delete shipsec-frontend-0 shipsec-backend-0 shipsec-worker-0 shipsec-test-worker 2>/dev/null || true + if [ "$SECURE_MODE" = "true" ]; then + docker compose -f docker/docker-compose.infra.yml -f docker/docker-compose.dev-secure.yml -f docker/docker-compose.dev-ports.yml down -v else - echo "šŸ§¹ Cleaning instance $INSTANCE..." - - # Stop PM2 apps - pm2 delete shipsec-{frontend,backend,worker}-"$INSTANCE" 2>/dev/null || true - - # Remove instance-specific infra state (DB + Temporal namespace + topics, etc.) - ./scripts/instance-clean.sh "$INSTANCE" || true - - # Remove instance directory - rm -rf "$INSTANCE_DIR" - - echo "āœ… Instance $INSTANCE cleaned" + docker compose -f docker/docker-compose.infra.yml -f docker/docker-compose.dev-ports.yml down -v fi + echo "āœ… Development environment cleaned (PM2 stopped, infrastructure volumes removed)" ;; *) - echo "Usage: just dev [instance] [action]" - echo " instance: 0-9 (default: 0)" - echo " action: start|stop|logs|status|clean" - exit 1 + echo "Usage: just dev [start|stop|logs|status|clean]" ;; esac # === Production (Docker-based) === -# Initialize production environment with secure secrets -# Creates docker/.env with auto-generated secrets if not present -prod-init: +# Run production environment in Docker +# Auto-detects security mode: if TLS certs exist (docker/certs/root-ca.pem) ā†’ secure mode with multitenancy +# Otherwise ā†’ standard mode without OpenSearch Security +prod action="start": #!/usr/bin/env bash set -euo pipefail - ENV_FILE="docker/.env" - - echo "šŸ”§ Initializing production environment..." - - # Create docker/.env if it doesn't exist - if [ ! -f "$ENV_FILE" ]; then - echo "šŸ“ Creating $ENV_FILE..." - touch "$ENV_FILE" - fi - - # Source existing env file to check for existing values - set -a - [ -f "$ENV_FILE" ] && source "$ENV_FILE" - set +a - UPDATED=false - - # Generate INTERNAL_SERVICE_TOKEN if not set - if [ -z "${INTERNAL_SERVICE_TOKEN:-}" ]; then - TOKEN=$(openssl rand -hex 32) - echo "INTERNAL_SERVICE_TOKEN=$TOKEN" >> "$ENV_FILE" - echo "šŸ”‘ Generated INTERNAL_SERVICE_TOKEN" - UPDATED=true + # Auto-detect security mode from TLS certificates + if [ -f "docker/certs/root-ca.pem" ]; then + SECURE_MODE=true else - echo "āœ… INTERNAL_SERVICE_TOKEN already set" + SECURE_MODE=false fi - # Generate SECRET_STORE_MASTER_KEY if not set (exactly 32 characters, raw string) - if [ -z "${SECRET_STORE_MASTER_KEY:-}" ]; then - KEY=$(openssl rand -base64 24 | head -c 32) - echo "SECRET_STORE_MASTER_KEY=$KEY" >> "$ENV_FILE" - echo "šŸ”‘ Generated SECRET_STORE_MASTER_KEY" - UPDATED=true + # Compose file selection based on mode + if [ "$SECURE_MODE" = "true" ]; then + COMPOSE_CMD="docker compose -f docker/docker-compose.infra.yml -f docker/docker-compose.prod.yml" else - echo "āœ… SECRET_STORE_MASTER_KEY already set" - fi - - if [ "$UPDATED" = true ]; then - echo "" - echo "āœ… Secrets generated and saved to $ENV_FILE" - echo "āš ļø Keep this file secure and never commit it to git!" + COMPOSE_CMD="docker compose -f docker/docker-compose.full.yml" fi - echo "" - echo "šŸ“‹ Current configuration in $ENV_FILE:" - echo " Run 'cat $ENV_FILE' to view" - echo "" - echo "šŸ’” Next steps:" - echo " 1. Edit $ENV_FILE to add other required variables (CLERK keys, etc.)" - echo " 2. Run 'just prod start-latest' to start with latest release" - -# Run production environment in Docker -prod action="start": - #!/usr/bin/env bash - set -euo pipefail case "{{action}}" in start) - echo "šŸš€ Starting production environment..." - # Use --env-file if docker/.env exists - ENV_FLAG="" - [ -f "docker/.env" ] && ENV_FLAG="--env-file docker/.env" - docker compose $ENV_FLAG -f docker/docker-compose.full.yml up -d - echo "" - echo "āœ… Production environment ready" - echo " Frontend: http://localhost:8090" - echo " Backend: http://localhost:3211" - echo " Temporal UI: http://localhost:8081" - echo "" + if [ "$SECURE_MODE" = "true" ]; then + echo "šŸ” Starting production environment (secure mode)..." + + # Check for required env vars in secure mode + if [ -z "${OPENSEARCH_ADMIN_PASSWORD:-}" ] || [ -z "${OPENSEARCH_DASHBOARDS_PASSWORD:-}" ]; then + echo "āŒ Required environment variables not set!" + echo "" + echo " export OPENSEARCH_ADMIN_PASSWORD='your-secure-password'" + echo " export OPENSEARCH_DASHBOARDS_PASSWORD='your-secure-password'" + exit 1 + fi + + $COMPOSE_CMD up -d + echo "" + echo "āœ… Production environment ready (secure mode)" + echo " Analytics: https://localhost/analytics (requires auth)" + echo " OpenSearch: https://localhost:9200 (TLS enabled)" + echo "" + echo "šŸ’” See docker/PRODUCTION.md for customer provisioning" + else + echo "šŸš€ Starting production environment..." + $COMPOSE_CMD up -d + echo "" + echo "āœ… Production environment ready" + echo " App: http://localhost" + echo " API: http://localhost/api" + echo " Analytics: http://localhost/analytics" + echo "" + echo "šŸ”’ All internal service ports are disabled (no direct access)" + echo "" + echo "šŸ’” To enable security + multitenancy:" + echo " Run: just generate-certs" + fi # Version check bun backend/scripts/version-check-summary.ts 2>/dev/null || true ;; stop) - docker compose -f docker/docker-compose.full.yml down + $COMPOSE_CMD down echo "āœ… Production stopped" ;; build) @@ -362,72 +260,59 @@ prod action="start": echo "šŸ“Œ Building with commit: $GIT_SHA" fi - # Use --env-file if docker/.env exists - ENV_FLAG="" - [ -f "docker/.env" ] && ENV_FLAG="--env-file docker/.env" - docker compose $ENV_FLAG -f docker/docker-compose.full.yml up -d --build + $COMPOSE_CMD up -d --build echo "āœ… Production built and started" - echo " Frontend: http://localhost:8090" - echo " Backend: http://localhost:3211" echo "" # Version check bun backend/scripts/version-check-summary.ts 2>/dev/null || true ;; logs) - docker compose -f docker/docker-compose.full.yml logs -f + $COMPOSE_CMD logs -f ;; status) - docker compose -f docker/docker-compose.full.yml ps + $COMPOSE_CMD ps ;; clean) - docker compose -f docker/docker-compose.full.yml down -v + $COMPOSE_CMD down -v docker system prune -f echo "āœ… Production cleaned" ;; start-latest) - # Auto-initialize secrets if docker/.env doesn't exist - if [ ! -f "docker/.env" ]; then - echo "āš ļø docker/.env not found, running prod-init..." - just prod-init - fi - echo "šŸ” Fetching latest release information from GitHub API..." if ! command -v curl &> /dev/null || ! command -v jq &> /dev/null; then echo "āŒ curl or jq is not installed. Please install them first." exit 1 fi - + LATEST_TAG=$(curl -s https://api.github.com/repos/ShipSecAI/studio/releases | jq -r '.[0].tag_name') - + # Strip leading 'v' if present (v0.1-rc2 -> 0.1-rc2) LATEST_TAG="${LATEST_TAG#v}" - + if [ "$LATEST_TAG" == "null" ] || [ -z "$LATEST_TAG" ]; then echo "āŒ Could not find any releases. Please check the repository at https://github.com/ShipSecAI/studio/releases" exit 1 fi - + echo "šŸ“¦ Found latest release: $LATEST_TAG" - + echo "šŸ“„ Pulling matching images from GHCR..." docker pull ghcr.io/shipsecai/studio-backend:$LATEST_TAG docker pull ghcr.io/shipsecai/studio-frontend:$LATEST_TAG docker pull ghcr.io/shipsecai/studio-worker:$LATEST_TAG - + echo "šŸš€ Starting production environment with version $LATEST_TAG..." export SHIPSEC_TAG=$LATEST_TAG - # Use --env-file if docker/.env exists - ENV_FLAG="" - [ -f "docker/.env" ] && ENV_FLAG="--env-file docker/.env" - docker compose $ENV_FLAG -f docker/docker-compose.full.yml up -d + $COMPOSE_CMD up -d echo "" echo "āœ… ShipSec Studio $LATEST_TAG ready" - echo " Frontend: http://localhost:8090" - echo " Backend: http://localhost:3211" - echo " Temporal UI: http://localhost:8081" + echo " App: http://localhost" + echo " API: http://localhost/api" + echo " Analytics: http://localhost/analytics" echo "" + echo "šŸ”’ All internal service ports are disabled (no direct access)" echo "šŸ’” Note: Using images tagged as $LATEST_TAG" ;; *) @@ -443,12 +328,6 @@ prod-images action="start": set -euo pipefail case "{{action}}" in start) - # Auto-initialize secrets if docker/.env doesn't exist - if [ ! -f "docker/.env" ]; then - echo "āš ļø docker/.env not found, running prod-init..." - just prod-init - fi - echo "šŸš€ Starting production environment with GHCR images..." # Check if images exist locally, pull if needed @@ -471,15 +350,14 @@ prod-images action="start": fi # Start with GHCR images, fallback to local build - # Use --env-file if docker/.env exists - ENV_FLAG="" - [ -f "docker/.env" ] && ENV_FLAG="--env-file docker/.env" - DOCKER_BUILDKIT=1 docker compose $ENV_FLAG -f docker/docker-compose.full.yml up -d + DOCKER_BUILDKIT=1 docker compose -f docker/docker-compose.full.yml up -d echo "" echo "āœ… Production environment ready" - echo " Frontend: http://localhost:8090" - echo " Backend: http://localhost:3211" - echo " Temporal UI: http://localhost:8081" + echo " App: http://localhost" + echo " API: http://localhost/api" + echo " Analytics: http://localhost/analytics" + echo "" + echo "šŸ”’ All internal service ports are disabled (no direct access)" ;; stop) docker compose -f docker/docker-compose.full.yml down @@ -533,27 +411,61 @@ prod-images action="start": ;; esac +# Generate TLS certificates for production +generate-certs: + #!/usr/bin/env bash + set -euo pipefail + echo "šŸ” Generating TLS certificates..." + chmod +x docker/scripts/generate-certs.sh + docker/scripts/generate-certs.sh + echo "" + echo "āœ… Certificates generated in docker/certs/" + echo "" + echo "Next steps:" + echo " 1. export OPENSEARCH_ADMIN_PASSWORD='your-secure-password'" + echo " 2. export OPENSEARCH_DASHBOARDS_PASSWORD='your-secure-password'" + echo " 3. just prod" + +# Initialize or reinitialize OpenSearch security index +security-init *args: + #!/usr/bin/env bash + set -euo pipefail + echo "šŸ” Initializing OpenSearch Security..." + chmod +x docker/scripts/security-init.sh + docker/scripts/security-init.sh {{args}} + +# Generate BCrypt password hash for OpenSearch internal users +hash-password password="": + #!/usr/bin/env bash + set -euo pipefail + chmod +x docker/scripts/hash-password.sh + if [ -n "{{password}}" ]; then + docker/scripts/hash-password.sh "{{password}}" + else + docker/scripts/hash-password.sh + fi + # === Infrastructure Only === # Manage infrastructure containers separately infra action="up": #!/usr/bin/env bash set -euo pipefail - INFRA_PROJECT_NAME="shipsec-infra" case "{{action}}" in up) - docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" up -d + docker compose -f docker/docker-compose.infra.yml -f docker/docker-compose.dev-ports.yml up -d echo "āœ… Infrastructure started (Postgres, Temporal, MinIO, Redis)" + echo " All ports bound to 127.0.0.1 (localhost only)" ;; down) - docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" down + docker compose -f docker/docker-compose.infra.yml -f docker/docker-compose.dev-ports.yml down echo "āœ… Infrastructure stopped" ;; logs) - docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" logs -f + docker compose -f docker/docker-compose.infra.yml -f docker/docker-compose.dev-ports.yml logs -f ;; clean) - docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" down -v + docker compose -f docker/docker-compose.infra.yml -f docker/docker-compose.dev-ports.yml down -v echo "āœ… Infrastructure cleaned" ;; *) @@ -578,21 +490,17 @@ status: echo "=== Production Containers ===" docker compose -f docker/docker-compose.full.yml ps 2>/dev/null || echo " (Production not running)" -# Reset database for specific instance or all instances -# Usage: just db-reset [instance] -db-reset instance="0": +# Reset database (drops all data) +db-reset: #!/usr/bin/env bash set -euo pipefail - - if [ "{{instance}}" = "all" ]; then - echo "šŸ—‘ļø Resetting all instance databases..." - for i in {0..9}; do - ./scripts/db-reset-instance.sh "$i" 2>/dev/null || true - done - echo "āœ… All instance databases reset" - else - ./scripts/db-reset-instance.sh "{{instance}}" + if ! docker ps --filter "name=shipsec-postgres" --format "{{{{.Names}}}}" | grep -q "shipsec-postgres"; then + echo "āŒ PostgreSQL not running. Run: just dev" && exit 1 fi + docker exec shipsec-postgres psql -U shipsec -d postgres -c "DROP DATABASE IF EXISTS shipsec;" + docker exec shipsec-postgres psql -U shipsec -d postgres -c "CREATE DATABASE shipsec;" + bun --cwd=backend run migration:push + echo "āœ… Database reset" # Build production images without starting build: @@ -607,33 +515,26 @@ help: @echo "Getting Started:" @echo " just init Set up dependencies and environment files" @echo "" - @echo "Development (hot-reload, multi-instance support):" - @echo " just dev Start the active instance (default: 0)" - @echo " just instance show Show active instance" - @echo " just instance use 5 Set active instance to 5 for this workspace" - @echo " just dev 1 Start instance 1" - @echo " just dev 2 start Explicitly start instance 2" - @echo " just dev 1 stop Stop instance 1" - @echo " just dev 2 logs View instance 2 logs" - @echo " just dev 0 status Check instance 0 status" - @echo " just dev 1 clean Stop and remove instance 1 data" - @echo " just dev stop all Stop all instances at once" - @echo " just dev status all Check status of all instances" - @echo "" - @echo " Note: Instances share one Docker infra stack (Postgres/Temporal/Redpanda/Redis/etc)" - @echo " Isolation comes from per-instance DB + Temporal namespace/task-queue + Kafka topic suffix" - @echo " Instance N uses base_port + N*100 (e.g., instance 0 uses 5173, instance 1 uses 5273)" + @echo "Development (hot-reload, auto-detects auth mode):" + @echo " just dev Start dev (Clerk creds in .env ā†’ secure mode, otherwise local auth)" + @echo " just dev stop Stop everything" + @echo " just dev logs View application logs" + @echo " just dev status Check service status" + @echo " just dev clean Stop and remove all data" @echo "" - @echo "Production (Docker):" - @echo " just prod-init Generate secrets in docker/.env (run once)" - @echo " just prod Start with cached images" + @echo "Production (Docker, auto-detects security mode):" + @echo " just prod Start prod (TLS certs present ā†’ secure mode, otherwise standard)" @echo " just prod build Rebuild and start" @echo " just prod start-latest Download latest release and start" @echo " just prod stop Stop production" @echo " just prod logs View production logs" @echo " just prod status Check production status" @echo " just prod clean Remove all data" - @echo " just prod-images Start with GHCR images (uses cache)" + @echo "" + @echo "Security Management:" + @echo " just security-init Initialize OpenSearch security index" + @echo " just security-init --force Reinitialize (update config)" + @echo " just hash-password Generate BCrypt hash for passwords" @echo "" @echo "Infrastructure:" @echo " just infra up Start infrastructure only" @@ -642,8 +543,6 @@ help: @echo " just infra clean Remove infrastructure data" @echo "" @echo "Utilities:" - @echo " just status Show status of all services" - @echo " just db-reset Reset instance 0 database" - @echo " just db-reset 1 Reset instance 1 database" - @echo " just db-reset all Reset all instance databases" - @echo " just build Build images only" + @echo " just status Show status of all services" + @echo " just db-reset Reset database" + @echo " just build Build images only" diff --git a/packages/component-sdk/src/analytics.ts b/packages/component-sdk/src/analytics.ts new file mode 100644 index 00000000..2035eb2a --- /dev/null +++ b/packages/component-sdk/src/analytics.ts @@ -0,0 +1,66 @@ +/** + * Analytics helpers for component authors. + * + * These utilities help components output structured findings + * that can be indexed into OpenSearch via the Analytics Sink. + */ + +import { createHash } from 'crypto'; +import { z } from 'zod'; +import { withPortMeta } from './port-meta'; + +// Analytics Results Contract +export const analyticsResultContractName = 'core.analytics.result.v1'; + +export const severitySchema = z.enum(['critical', 'high', 'medium', 'low', 'info', 'none']); + +export const analyticsResultSchema = () => + withPortMeta( + z + .object({ + scanner: z.string().describe('Scanner/component that produced this result'), + finding_hash: z.string().describe('Stable 16-char hash for deduplication'), + severity: severitySchema.describe('Finding severity level, use "none" if not applicable'), + asset_key: z + .string() + .optional() + .describe('Primary asset identifier (auto-detected if missing)'), + }) + .passthrough(), // Allow scanner-specific fields + { schemaName: analyticsResultContractName } + ); + +export type AnalyticsResult = z.infer>; +export type Severity = z.infer; + +/** + * Generate a stable hash for finding deduplication. + * + * The hash is used to track findings across workflow runs: + * - Identify new vs recurring findings + * - Calculate first-seen / last-seen timestamps + * - Deduplicate findings in dashboards + * + * @param fields - Key identifying fields of the finding (e.g., templateId, host, matchedAt) + * @returns 16-character hex string (SHA-256 truncated) + * + * @example + * ```typescript + * // Nuclei scanner + * const hash = generateFindingHash(finding.templateId, finding.host, finding.matchedAt); + * + * // TruffleHog scanner + * const hash = generateFindingHash(secret.DetectorType, secret.Redacted, filePath); + * + * // Supabase scanner + * const hash = generateFindingHash(check.check_id, projectRef, check.resource); + * ``` + */ +export function generateFindingHash( + ...fields: (string | undefined | null)[] +): string { + const normalized = fields + .map((f) => (f ?? '').toLowerCase().trim()) + .join('|'); + return createHash('sha256').update(normalized).digest('hex').slice(0, 16); +} diff --git a/packages/component-sdk/src/context.ts b/packages/component-sdk/src/context.ts index 63185b3a..e267f869 100644 --- a/packages/component-sdk/src/context.ts +++ b/packages/component-sdk/src/context.ts @@ -45,10 +45,13 @@ export interface CreateContextOptions { logCollector?: (entry: LogEventInput) => void; terminalCollector?: (chunk: TerminalChunkInput) => void; agentTracePublisher?: AgentTracePublisher; + workflowId?: string; + workflowName?: string; + organizationId?: string | null; } export function createExecutionContext(options: CreateContextOptions): ExecutionContext { - const { runId, componentRef, metadata: metadataInput, storage, secrets, artifacts, trace, logCollector, terminalCollector, agentTracePublisher } = + const { runId, componentRef, metadata: metadataInput, storage, secrets, artifacts, trace, logCollector, terminalCollector, agentTracePublisher, workflowId, workflowName, organizationId } = options; const metadata = createMetadata(runId, componentRef, metadataInput); const scopedTrace = trace ? createScopedTrace(trace, metadata) : undefined; @@ -145,6 +148,9 @@ export function createExecutionContext(options: CreateContextOptions): Execution terminalCollector, metadata, agentTracePublisher, + workflowId, + workflowName, + organizationId, http: undefined as unknown as ExecutionContext['http'], }; diff --git a/packages/component-sdk/src/index.ts b/packages/component-sdk/src/index.ts index 7e914b2a..edd46ce6 100644 --- a/packages/component-sdk/src/index.ts +++ b/packages/component-sdk/src/index.ts @@ -36,3 +36,6 @@ export * from './zod-parameters'; export * from './json-schema'; export * from './schema-validation'; export * from './zod-coerce'; + +// Analytics helpers for component authors +export * from './analytics'; diff --git a/packages/component-sdk/src/types.ts b/packages/component-sdk/src/types.ts index 19ae0c78..c9cd317b 100644 --- a/packages/component-sdk/src/types.ts +++ b/packages/component-sdk/src/types.ts @@ -271,7 +271,8 @@ export type ComponentParameterType = | 'artifact' | 'variable-list' | 'form-fields' - | 'selection-options'; + | 'selection-options' + | 'analytics-inputs'; export interface ComponentParameterOption { label: string; @@ -377,6 +378,11 @@ export interface ExecutionContext { metadata: ExecutionContextMetadata; agentTracePublisher?: AgentTracePublisher; + // Workflow context (optional, available when running in workflow) + workflowId?: string; + workflowName?: string; + organizationId?: string | null; + // Service interfaces - implemented by adapters storage?: IFileStorageService; secrets?: ISecretsService; diff --git a/packages/shared/src/execution.ts b/packages/shared/src/execution.ts index 59b7e7ff..83b8e6b9 100644 --- a/packages/shared/src/execution.ts +++ b/packages/shared/src/execution.ts @@ -1,5 +1,20 @@ import { z } from 'zod'; +/** + * Workflow execution status values. + * + * @see docs/workflows/execution-status.md for detailed documentation + * + * - QUEUED: Waiting to execute + * - RUNNING: Actively executing + * - COMPLETED: All nodes finished successfully + * - FAILED: Execution failed (node failure or crash) + * - CANCELLED: User cancelled + * - TERMINATED: Forcefully terminated + * - TIMED_OUT: Exceeded max execution time + * - AWAITING_INPUT: Paused for human input + * - STALE: Orphaned record (data inconsistency) + */ export const EXECUTION_STATUS = [ 'QUEUED', 'RUNNING', @@ -8,7 +23,8 @@ export const EXECUTION_STATUS = [ 'CANCELLED', 'TERMINATED', 'TIMED_OUT', - 'AWAITING_INPUT' + 'AWAITING_INPUT', + 'STALE', ] as const; export type ExecutionStatus = (typeof EXECUTION_STATUS)[number]; diff --git a/pm2.config.cjs b/pm2.config.cjs index 71452c39..b06c78a8 100644 --- a/pm2.config.cjs +++ b/pm2.config.cjs @@ -196,6 +196,41 @@ function loadFrontendEnv() { const frontendEnv = loadFrontendEnv(); +// Load worker .env file for OpenSearch and other worker-specific variables +function loadWorkerEnv() { + const envPath = path.join(__dirname, 'worker', '.env'); + const env = {}; + + try { + if (fs.existsSync(envPath)) { + const envContent = fs.readFileSync(envPath, 'utf-8'); + envContent.split('\n').forEach((line) => { + const trimmed = line.trim(); + // Skip comments and empty lines + if (!trimmed || trimmed.startsWith('#')) { + return; + } + const match = trimmed.match(/^([^=]+)=(.*)$/); + if (match) { + const key = match[1].trim(); + let value = match[2].trim(); + // Remove surrounding quotes if present + if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) { + value = value.slice(1, -1); + } + env[key] = value; + } + }); + } + } catch (err) { + console.warn('Failed to load worker .env file:', err.message); + } + + return env; +} + +const workerEnv = loadWorkerEnv(); + // Determine environment from NODE_ENV or SHIPSEC_ENV const environment = process.env.SHIPSEC_ENV || process.env.NODE_ENV || 'development'; const isProduction = environment === 'production'; @@ -264,7 +299,7 @@ module.exports = { // Ensure instance DB isolation even if dotenv auto-loads a workspace/default `.env`. ...devInstanceEnv, TERMINAL_REDIS_URL: process.env.TERMINAL_REDIS_URL || 'redis://localhost:6379', - LOG_KAFKA_BROKERS: process.env.LOG_KAFKA_BROKERS || 'localhost:19092', + LOG_KAFKA_BROKERS: process.env.LOG_KAFKA_BROKERS || 'localhost:9092', LOG_KAFKA_TOPIC: process.env.LOG_KAFKA_TOPIC || 'telemetry.logs', LOG_KAFKA_CLIENT_ID: process.env.LOG_KAFKA_CLIENT_ID || `shipsec-backend-${instanceNum}`, LOG_KAFKA_GROUP_ID: process.env.LOG_KAFKA_GROUP_ID || `shipsec-backend-log-consumer-${instanceNum}`, @@ -305,13 +340,14 @@ module.exports = { env_file: resolveEnvFile('worker', instanceNum), env: Object.assign( { + ...workerEnv, // Load worker .env file (includes OPENSEARCH_URL, etc.) ...currentEnvConfig, NAPI_RS_FORCE_WASI: '1', INTERNAL_SERVICE_TOKEN: process.env.INTERNAL_SERVICE_TOKEN || 'local-internal-token', STUDIO_API_BASE_URL: process.env.STUDIO_API_BASE_URL || `http://localhost:${getInstancePort(3211, instanceNum)}/api/v1`, ...devInstanceEnv, TERMINAL_REDIS_URL: process.env.TERMINAL_REDIS_URL || 'redis://localhost:6379', - LOG_KAFKA_BROKERS: process.env.LOG_KAFKA_BROKERS || 'localhost:19092', + LOG_KAFKA_BROKERS: process.env.LOG_KAFKA_BROKERS || 'localhost:9092', LOG_KAFKA_TOPIC: process.env.LOG_KAFKA_TOPIC || 'telemetry.logs', LOG_KAFKA_CLIENT_ID: process.env.LOG_KAFKA_CLIENT_ID || `shipsec-worker-${instanceNum}`, EVENT_KAFKA_TOPIC: process.env.EVENT_KAFKA_TOPIC || 'telemetry.events', @@ -335,6 +371,7 @@ module.exports = { env_file: __dirname + '/worker/.env', env: Object.assign( { + ...workerEnv, // Load worker .env file (includes OPENSEARCH_URL, etc.) TEMPORAL_TASK_QUEUE: 'test-worker-integration', TEMPORAL_NAMESPACE: 'shipsec-dev', NODE_ENV: 'development', diff --git a/scripts/dev-instance-manager.sh b/scripts/dev-instance-manager.sh deleted file mode 100755 index d86d0f21..00000000 --- a/scripts/dev-instance-manager.sh +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env bash -# Multi-instance dev stack manager for ShipSec Studio -# Handles isolated Docker containers and PM2 processes per instance - -set -euo pipefail - -# Configuration -INSTANCES_DIR=".instances" - -# Base port mappings (plain variables for bash 3.2 compatibility on macOS) -BASE_PORT_FRONTEND=5173 -BASE_PORT_BACKEND=3211 - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' - -# Helper functions -log_info() { - echo -e "${BLUE}ā„¹${NC} $*" -} - -log_success() { - echo -e "${GREEN}āœ…${NC} $*" -} - -log_warn() { - echo -e "${YELLOW}āš ļø${NC} $*" -} - -log_error() { - echo -e "${RED}āŒ${NC} $*" -} - -get_instance_dir() { - local instance=$1 - echo "$INSTANCES_DIR/instance-$instance" -} - -get_port() { - local port_name=$1 - local instance=$2 - local base_port="" - - case "$port_name" in - FRONTEND) base_port=$BASE_PORT_FRONTEND ;; - BACKEND) base_port=$BASE_PORT_BACKEND ;; - *) - log_error "Unknown port: $port_name" - return 1 - ;; - esac - - # Port offset: instance N uses base_port + N*100 - echo $((base_port + instance * 100)) -} - -ensure_instance_dir() { - local instance=$1 - local inst_dir=$(get_instance_dir "$instance") - - if [ ! -d "$inst_dir" ]; then - mkdir -p "$inst_dir" - log_info "Created instance directory: $inst_dir" - fi -} - -copy_env_files() { - local instance=$1 - local inst_dir=$(get_instance_dir "$instance") - local root_env=".env" - - # Read KEY from repo root .env (first match). Prints empty string if missing. - get_root_env_value() { - local key=$1 - if [ ! -f "$root_env" ]; then - echo "" - return 0 - fi - # Keep everything after the first '=' (values can contain '=') - rg -m1 "^${key}=" "$root_env" 2>/dev/null | sed -E "s/^${key}=//" || true - } - - # Append KEY=VALUE to dest if KEY is not already present. - ensure_env_key() { - local dest=$1 - local key=$2 - local value=$3 - if [ -z "$value" ]; then - return 0 - fi - if rg -q "^${key}=" "$dest" 2>/dev/null; then - return 0 - fi - echo "${key}=${value}" >> "$dest" - } - - # Copy and modify .env files for this instance - for app_dir in backend worker frontend; do - local src_file="$app_dir/.env" - if [ -f "$src_file" ]; then - local dest="$inst_dir/${app_dir}.env" - cp "$src_file" "$dest" - - # Ensure each instance points at its own Postgres database. - # Backend uses quoted DATABASE_URL, worker typically does not. - if [ "$app_dir" = "backend" ] || [ "$app_dir" = "worker" ]; then - sed -i.bak -E \ - -e "s|(DATABASE_URL=.*\\/)(shipsec)(\"?)$|\\1shipsec_instance_${instance}\\3|" \ - "$dest" - fi - - # Ensure secrets/internal auth keys exist for dev processes. These live in repo root `.env`. - # Keep instance env self-contained so backend/worker don't need to load root `.env` (which - # contains a default DATABASE_URL and would break isolation). - if [ "$app_dir" = "backend" ] || [ "$app_dir" = "worker" ]; then - ensure_env_key "$dest" "INTERNAL_SERVICE_TOKEN" "$(get_root_env_value INTERNAL_SERVICE_TOKEN)" - ensure_env_key "$dest" "SECRET_STORE_MASTER_KEY" "$(get_root_env_value SECRET_STORE_MASTER_KEY)" - fi - - rm -f "$dest.bak" - log_success "Created $dest" - fi - done -} - -get_docker_compose_project_name() { - local instance=$1 - echo "shipsec-dev-$instance" -} - -validate_instance_setup() { - local instance=$1 - local inst_dir=$(get_instance_dir "$instance") - - # Check that all required env files exist - for env_file in backend worker frontend; do - if [ ! -f "$inst_dir/${env_file}.env" ]; then - log_error "Missing $inst_dir/${env_file}.env" - return 1 - fi - done - - log_success "Instance $instance configuration validated" - return 0 -} - -show_instance_info() { - local instance=$1 - - echo "" - echo -e "${BLUE}=== Instance $instance ===${NC}" - echo "Directory: $(get_instance_dir "$instance")" - echo "" - echo "Ports:" - echo " Frontend: http://localhost:$(get_port FRONTEND $instance)" - echo " Backend: http://localhost:$(get_port BACKEND $instance)" - echo " Temporal UI: http://localhost:8081" - echo "" - echo "Database: postgresql://shipsec:shipsec@localhost:5433/shipsec_instance_$instance" - echo "MinIO API: http://localhost:9000" - echo "MinIO UI: http://localhost:9001" - echo "Redis: redis://localhost:6379" - echo "" -} - -initialize_instance() { - local instance=$1 - - log_info "Initializing instance $instance..." - ensure_instance_dir "$instance" - copy_env_files "$instance" - - if validate_instance_setup "$instance"; then - show_instance_info "$instance" - log_success "Instance $instance initialized successfully" - return 0 - else - log_error "Instance $instance initialization failed" - return 1 - fi -} - -# Main command handler -main() { - local command=${1:-help} - local instance=${2:-0} - - case "$command" in - init) - initialize_instance "$instance" - ;; - info) - show_instance_info "$instance" - ;; - ports) - echo "FRONTEND=$(get_port FRONTEND $instance)" - echo "BACKEND=$(get_port BACKEND $instance)" - ;; - project-name) - get_docker_compose_project_name "$instance" - ;; - *) - log_error "Unknown command: $command" - echo "Usage: $0 {init|info|ports|project-name} [instance]" - exit 1 - ;; - esac -} - -main "$@" diff --git a/scripts/instance-bootstrap.sh b/scripts/instance-bootstrap.sh deleted file mode 100755 index 845c88ce..00000000 --- a/scripts/instance-bootstrap.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env bash -# Bootstrap shared infra resources for a specific instance. -# - Ensure instance DB exists -# - Run migrations against that DB -# - Ensure Temporal namespace exists -# - Ensure Kafka topics exist (best-effort) -# -# Usage: ./scripts/instance-bootstrap.sh [instance_number] - -set -euo pipefail - -INSTANCE="${1:-0}" -INFRA_PROJECT_NAME="shipsec-infra" -DB_NAME="shipsec_instance_${INSTANCE}" -NAMESPACE="shipsec-dev-${INSTANCE}" -TEMPORAL_ADDRESS="127.0.0.1:7233" - -BLUE='\033[0;34m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -RED='\033[0;31m' -NC='\033[0m' - -log_info() { echo -e "${BLUE}ā„¹${NC} $*"; } -log_success() { echo -e "${GREEN}āœ…${NC} $*"; } -log_warn() { echo -e "${YELLOW}āš ļø${NC} $*"; } -log_error() { echo -e "${RED}āŒ${NC} $*"; } - -POSTGRES_CONTAINER="$( - docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" ps -q postgres 2>/dev/null || true -)" - -if [ -z "$POSTGRES_CONTAINER" ]; then - log_error "Postgres container not found (infra project: $INFRA_PROJECT_NAME). Is infra running?" - exit 1 -fi - -log_info "Ensuring database exists: $DB_NAME" -docker exec "$POSTGRES_CONTAINER" \ - psql -v ON_ERROR_STOP=1 -U shipsec -d postgres </dev/null 2>&1; then - log_success "Migrations completed" -else - log_error "Migrations failed" - exit 1 -fi - -if ! command -v temporal >/dev/null 2>&1; then - log_info "temporal CLI not found; skipping Temporal namespace bootstrap" -else - log_info "Ensuring Temporal namespace exists: $NAMESPACE" - # Temporal can take a few seconds to accept CLI requests after the container is "Started". - for _ in {1..30}; do - if temporal operator namespace list --address "$TEMPORAL_ADDRESS" >/dev/null 2>&1; then - break - fi - sleep 1 - done - - if temporal operator namespace describe --address "$TEMPORAL_ADDRESS" --namespace "$NAMESPACE" >/dev/null 2>&1; then - log_success "Temporal namespace exists" - else - # Create is not idempotent; it errors if the namespace already exists. Treat that as success. - if temporal operator namespace create --address "$TEMPORAL_ADDRESS" --namespace "$NAMESPACE" --retention 72h >/dev/null 2>&1; then - log_success "Temporal namespace created" - elif temporal operator namespace describe --address "$TEMPORAL_ADDRESS" --namespace "$NAMESPACE" >/dev/null 2>&1; then - log_success "Temporal namespace exists" - else - log_warn "Unable to ensure Temporal namespace (will likely break worker); continuing anyway" - fi - fi -fi - -# Best-effort Kafka topic creation in shared Redpanda. -REDPANDA_CONTAINER="$( - docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" ps -q redpanda 2>/dev/null || true -)" -if [ -n "$REDPANDA_CONTAINER" ]; then - log_info "Ensuring Kafka topics exist for instance $INSTANCE (best-effort)..." - for base in telemetry.logs telemetry.events telemetry.agent-trace telemetry.node-io; do - topic="${base}.instance-${INSTANCE}" - docker exec "$REDPANDA_CONTAINER" rpk topic create "$topic" --brokers redpanda:9092 >/dev/null 2>&1 || true - done - log_success "Kafka topics ensured" -fi diff --git a/worker/.env.example b/worker/.env.example index 65c71cc9..a1c6b7f9 100644 --- a/worker/.env.example +++ b/worker/.env.example @@ -24,5 +24,16 @@ LOKI_PASSWORD= # Generate with: openssl rand -base64 24 | head -c 32 SECRET_STORE_MASTER_KEY=CHANGE_ME_32_CHAR_SECRET_KEY!!!! +# OpenSearch Configuration (Optional - for Analytics Sink component) +# Leave empty to disable analytics indexing +OPENSEARCH_URL=http://localhost:9200 +OPENSEARCH_USERNAME= +OPENSEARCH_PASSWORD= +# OpenSearch Dashboards URL for auto-refreshing index patterns after indexing +# - Local dev (worker outside Docker): http://localhost:5601/analytics +# - Docker (worker inside Docker): http://opensearch-dashboards:5601/analytics (set in docker-compose) +# Note: Include the basePath (/analytics) as configured in opensearch_dashboards.yml +OPENSEARCH_DASHBOARDS_URL=http://localhost:5601/analytics + # Kafka / Redpanda configuration for log and event ingestion LOG_KAFKA_BROKERS=localhost:9092 diff --git a/worker/package.json b/worker/package.json index 191d1361..632d0914 100644 --- a/worker/package.json +++ b/worker/package.json @@ -27,6 +27,7 @@ "@grpc/grpc-js": "^1.14.3", "@modelcontextprotocol/sdk": "^1.25.1", "@okta/okta-sdk-nodejs": "^7.3.0", + "@opensearch-project/opensearch": "^3.5.1", "@shipsec/component-sdk": "*", "@shipsec/contracts": "*", "@shipsec/shared": "*", diff --git a/worker/src/components/core/analytics-sink.ts b/worker/src/components/core/analytics-sink.ts new file mode 100644 index 00000000..3a173034 --- /dev/null +++ b/worker/src/components/core/analytics-sink.ts @@ -0,0 +1,373 @@ +import { z } from 'zod'; +import { + componentRegistry, + defineComponent, + inputs, + outputs, + parameters, + port, + param, + analyticsResultSchema, + withPortMeta, + ValidationError, +} from '@shipsec/component-sdk'; + +// Schema for defining a data input port +const dataInputDefinitionSchema = z.object({ + id: z.string().describe('Unique identifier for this input (becomes input port ID)'), + label: z.string().describe('Display label for the input in the UI'), + sourceTag: z + .string() + .optional() + .describe('Tag added to indexed documents for filtering by source in dashboards'), +}); + +type DataInputDefinition = z.infer; + +// Base input schema with a default input port. +// resolvePorts adds extra ports when users configure multiple data inputs. +const baseInputSchema = inputs({ + input1: port(z.array(analyticsResultSchema()).optional(), { + label: 'Input 1', + description: 'Analytics results to index.', + }), +}); + +const outputSchema = outputs({ + indexed: port(z.boolean(), { + label: 'Indexed', + description: 'Indicates whether the data was successfully indexed to OpenSearch.', + }), + documentCount: port(z.number(), { + label: 'Document Count', + description: 'Number of documents indexed (1 for objects, array length for arrays).', + }), + indexName: port(z.string(), { + label: 'Index Name', + description: 'Name of the OpenSearch index where data was stored.', + }), +}); + +const parameterSchema = parameters({ + dataInputs: param( + z + .array(dataInputDefinitionSchema) + .default([{ id: 'input1', label: 'Input 1', sourceTag: 'input_1' }]) + .describe('Define multiple data inputs from different scanner components'), + { + label: 'Data Inputs', + editor: 'analytics-inputs', + description: + 'Configure input ports for different scanner results. Each input creates a corresponding input port.', + helpText: + 'Each input accepts AnalyticsResult[] and can be tagged for filtering in dashboards.', + }, + ), + indexSuffix: param( + z + .string() + .optional() + .describe( + 'Optional suffix to append to the index name. Defaults to date (YYYY.MM.DD) if not provided.', + ), + { + label: 'Index Suffix', + editor: 'text', + placeholder: 'YYYY.MM.DD (default)', + description: + 'Custom suffix for the index name (e.g., "subdomain-enum"). Defaults to date-based sharding (YYYY.MM.DD) if not provided.', + }, + ), + assetKeyField: param( + z + .enum([ + 'auto', + 'asset_key', + 'host', + 'domain', + 'subdomain', + 'url', + 'ip', + 'asset', + 'target', + 'custom', + ]) + .default('auto') + .describe( + 'Field name to use as the asset_key. Auto-detect checks common fields (asset_key, host, domain, subdomain, url, ip, asset, target) in priority order.', + ), + { + label: 'Asset Key Field', + editor: 'select', + options: [ + { label: 'Auto-detect', value: 'auto' }, + { label: 'asset_key', value: 'asset_key' }, + { label: 'host', value: 'host' }, + { label: 'domain', value: 'domain' }, + { label: 'subdomain', value: 'subdomain' }, + { label: 'url', value: 'url' }, + { label: 'ip', value: 'ip' }, + { label: 'asset', value: 'asset' }, + { label: 'target', value: 'target' }, + { label: 'Custom field name', value: 'custom' }, + ], + description: + 'Specify which field to use as the asset identifier. Auto-detect uses priority: asset_key > host > domain > subdomain > url > ip > asset > target.', + }, + ), + customAssetKeyField: param( + z + .string() + .optional() + .describe('Custom field name to use as asset_key when assetKeyField is set to "custom".'), + { + label: 'Custom Field Name', + editor: 'text', + placeholder: 'e.g., hostname, endpoint, etc.', + description: 'Enter the custom field name to use as the asset identifier.', + visibleWhen: { assetKeyField: 'custom' }, + }, + ), + failOnError: param( + z + .boolean() + .default(false) + .describe( + 'Strict mode: requires all configured inputs to have data and validates all documents before indexing. Default is lenient (fire-and-forget).', + ), + { + label: 'Strict Mode (Fail on Error)', + editor: 'boolean', + description: + 'When enabled: requires ALL configured inputs to have data, validates ALL documents before indexing, and fails the workflow if any check fails. When disabled: skips missing inputs and logs errors without failing.', + }, + ), +}); + +const definition = defineComponent({ + id: 'core.analytics.sink', + label: 'Analytics Sink', + category: 'output', + runner: { kind: 'inline' }, + inputs: baseInputSchema, + outputs: outputSchema, + parameters: parameterSchema, + docs: 'Indexes structured analytics results into OpenSearch for dashboards, queries, and alerts. Configure multiple data inputs to aggregate results from different scanner components. Each input can be tagged with a sourceTag for filtering in dashboards. Supports lenient (fire-and-forget) and strict (all-or-nothing) modes via the failOnError parameter.', + ui: { + slug: 'analytics-sink', + version: '2.0.0', + type: 'output', + category: 'output', + description: + 'Index security findings from multiple scanners into OpenSearch for analytics, dashboards, and alerting.', + icon: 'BarChart3', + author: { + name: 'ShipSecAI', + type: 'shipsecai', + }, + isLatest: true, + deprecated: false, + examples: [ + 'Aggregate findings from Nuclei, Subfinder, and Prowler into a unified security dashboard.', + 'Index subdomain enumeration results for tracking asset discovery over time.', + 'Store vulnerability scan findings for correlation and trend analysis.', + ], + }, + resolvePorts(params: z.infer) { + const dataInputs = Array.isArray(params.dataInputs) ? params.dataInputs : []; + + const inputShape: Record = {}; + + // Create dynamic input ports from dataInputs parameter + for (const input of dataInputs) { + const id = typeof input?.id === 'string' ? input.id.trim() : ''; + if (!id) { + continue; + } + + const label = typeof input?.label === 'string' ? input.label : id; + const sourceTag = typeof input?.sourceTag === 'string' ? input.sourceTag : undefined; + + const description = sourceTag + ? `Analytics results tagged with '${sourceTag}' in indexed documents.` + : `Analytics results from ${label}.`; + + // Each input port accepts an optional array of analytics results + inputShape[id] = withPortMeta(z.array(analyticsResultSchema()).optional(), { + label, + description, + }); + } + + return { + inputs: inputs(inputShape), + outputs: outputSchema, + }; + }, + async execute({ inputs, params }, context) { + const { getOpenSearchIndexer } = await import('../../utils/opensearch-indexer'); + const indexer = getOpenSearchIndexer(); + + const dataInputsMap = new Map( + (params.dataInputs ?? []).map((d) => [d.id, d]), + ); + + // Check if indexing is enabled + if (!indexer.isEnabled()) { + context.logger.debug( + '[Analytics Sink] OpenSearch not configured, skipping indexing (fire-and-forget)', + ); + return { + indexed: false, + documentCount: 0, + indexName: '', + }; + } + + // Validate required workflow context + if (!context.workflowId || !context.workflowName || !context.organizationId) { + const error = new Error( + 'Analytics Sink requires workflow context (workflowId, workflowName, organizationId)', + ); + context.logger.error(`[Analytics Sink] ${error.message}`); + if (params.failOnError) { + throw error; + } + return { + indexed: false, + documentCount: 0, + indexName: '', + }; + } + + // STRICT MODE: Require all configured inputs to be present + if (params.failOnError) { + for (const inputDef of params.dataInputs ?? []) { + const inputData = (inputs as Record)[inputDef.id]; + if (!inputData || !Array.isArray(inputData) || inputData.length === 0) { + throw new ValidationError( + `Required input '${inputDef.label}' (${inputDef.id}) is missing or empty. ` + + `All configured inputs must provide data when strict mode is enabled.`, + { + fieldErrors: { [inputDef.id]: ['This input is required but has no data'] }, + }, + ); + } + } + } + + // Aggregate all documents from all inputs + const allDocuments: Record[] = []; + const inputsRecord = inputs as Record; + + for (const [inputId, inputData] of Object.entries(inputsRecord)) { + if (!inputData || !Array.isArray(inputData)) { + if (!params.failOnError) { + context.logger.warn( + `[Analytics Sink] Input '${inputId}' is empty or undefined, skipping`, + ); + } + continue; + } + + const inputDef = dataInputsMap.get(inputId); + const sourceTag = inputDef?.sourceTag; + + for (const doc of inputData) { + // STRICT MODE: Validate each document against analytics schema + if (params.failOnError) { + const validated = analyticsResultSchema().safeParse(doc); + if (!validated.success) { + throw new ValidationError( + `Document from input '${inputDef?.label ?? inputId}' failed validation: ${validated.error.message}`, + { + fieldErrors: { [inputId]: [validated.error.message] }, + }, + ); + } + } + + // Add source_input field if sourceTag is defined + const enrichedDoc = sourceTag ? { ...doc, source_input: sourceTag } : { ...doc }; + allDocuments.push(enrichedDoc); + } + } + + const documentCount = allDocuments.length; + + if (documentCount === 0) { + context.logger.info('[Analytics Sink] No documents to index from any input'); + return { + indexed: false, + documentCount: 0, + indexName: '', + }; + } + + // LENIENT MODE: Validate all documents (but don't fail, just log warnings) + if (!params.failOnError) { + const validated = z.array(analyticsResultSchema()).safeParse(allDocuments); + if (!validated.success) { + context.logger.warn( + `[Analytics Sink] Some documents have validation issues: ${validated.error.message}`, + ); + // Continue anyway in lenient mode + } + } + + try { + // Determine the actual asset key field to use + let assetKeyField: string | undefined; + if (params.assetKeyField === 'auto') { + assetKeyField = undefined; + } else if (params.assetKeyField === 'custom') { + assetKeyField = params.customAssetKeyField; + } else { + assetKeyField = params.assetKeyField; + } + + const fallbackIndexSuffix = params.indexSuffix || undefined; + + const indexOptions = { + workflowId: context.workflowId, + workflowName: context.workflowName, + runId: context.runId, + nodeRef: context.componentRef, + componentId: 'core.analytics.sink', + assetKeyField, + indexSuffix: fallbackIndexSuffix, + trace: context.trace, + }; + + context.logger.info( + `[Analytics Sink] Bulk indexing ${documentCount} documents from ${dataInputsMap.size} input(s)`, + ); + const result = await indexer.bulkIndex(context.organizationId, allDocuments, indexOptions); + + context.logger.info( + `[Analytics Sink] Successfully indexed ${result.documentCount} document(s) to ${result.indexName}`, + ); + return { + indexed: true, + documentCount: result.documentCount, + indexName: result.indexName, + }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error during indexing'; + context.logger.error(`[Analytics Sink] Indexing failed: ${errorMessage}`); + + if (params.failOnError) { + throw error; + } + + // Fire-and-forget mode: log error but don't fail workflow + return { + indexed: false, + documentCount, + indexName: '', + }; + } + }, +}); + +componentRegistry.register(definition); diff --git a/worker/src/components/index.ts b/worker/src/components/index.ts index e2d4d533..a1d2f1e1 100644 --- a/worker/src/components/index.ts +++ b/worker/src/components/index.ts @@ -28,6 +28,7 @@ import './core/destination-s3'; import './core/text-block'; import './core/workflow-call'; import './core/mcp-library'; +import './core/analytics-sink'; // Manual Action components import './manual-action/manual-approval'; import './manual-action/manual-selection'; @@ -69,6 +70,7 @@ import './it-automation/okta-user-offboard'; import './test/sleep-parallel'; import './test/live-event-heartbeat'; import './test/simple-http-mcp'; +import './test/analytics-fixture'; // Export registry for external use export { componentRegistry } from '@shipsec/component-sdk'; diff --git a/worker/src/components/security/__tests__/dnsx.test.ts b/worker/src/components/security/__tests__/dnsx.test.ts index 5fad599c..2674493b 100644 --- a/worker/src/components/security/__tests__/dnsx.test.ts +++ b/worker/src/components/security/__tests__/dnsx.test.ts @@ -83,9 +83,11 @@ describe.skip('dnsx component', () => { expect(result.domainCount).toBe(1); expect(result.recordCount).toBe(2); - expect(result.results).toHaveLength(2); - expect(result.results[0].host).toBe('example.com'); - const aggregatedAnswers = result.results.flatMap((entry) => entry.answers.a ?? []); + expect(result.dnsRecords).toHaveLength(2); + expect((result.dnsRecords[0] as { host: string }).host).toBe('example.com'); + const aggregatedAnswers = result.dnsRecords.flatMap( + (entry) => (entry as { answers: { a?: string[] } }).answers.a ?? [], + ); expect(aggregatedAnswers).toEqual(['23.215.0.138', '23.215.0.136']); expect(result.recordTypes).toEqual(['A']); expect(result.resolvedHosts).toEqual(['example.com']); diff --git a/worker/src/components/security/__tests__/httpx.test.ts b/worker/src/components/security/__tests__/httpx.test.ts index 83282c84..8f7f45f8 100644 --- a/worker/src/components/security/__tests__/httpx.test.ts +++ b/worker/src/components/security/__tests__/httpx.test.ts @@ -85,7 +85,7 @@ describeHttpx('httpx component', () => { }); const payload: HttpxOutput = { - results: [ + responses: [ { url: 'https://example.com', host: 'example.com', @@ -105,6 +105,7 @@ describeHttpx('httpx component', () => { timestamp: '2023-01-01T00:00:00Z', }, ], + results: [], rawOutput: '{"url":"https://example.com","host":"example.com","status-code":200,"title":"Example Domain","tech":["HTTP","CDN"]}', targetCount: 1, diff --git a/worker/src/components/security/__tests__/nuclei.test.ts b/worker/src/components/security/__tests__/nuclei.test.ts index 9bd6ee20..4208cb42 100644 --- a/worker/src/components/security/__tests__/nuclei.test.ts +++ b/worker/src/components/security/__tests__/nuclei.test.ts @@ -147,6 +147,7 @@ describe('Nuclei Component', () => { timestamp: '2024-12-04T10:00:00Z', }, ], + results: [], rawOutput: '{"template-id":"CVE-2024-1234"}', targetCount: 1, findingCount: 1, @@ -174,6 +175,7 @@ describe('Nuclei Component', () => { timestamp: '2024-12-04T10:00:00Z', }, ], + results: [], rawOutput: '', targetCount: 1, findingCount: 1, @@ -200,6 +202,7 @@ describe('Nuclei Component', () => { ip: '1.2.3.4', }, ], + results: [], rawOutput: '', targetCount: 1, findingCount: 1, diff --git a/worker/src/components/security/__tests__/subfinder.test.ts b/worker/src/components/security/__tests__/subfinder.test.ts index 4e590dee..6f6b07d1 100644 --- a/worker/src/components/security/__tests__/subfinder.test.ts +++ b/worker/src/components/security/__tests__/subfinder.test.ts @@ -73,13 +73,19 @@ describe.skip('subfinder component', () => { rawOutput: 'api.example.com', domainCount: 1, subdomainCount: 1, + results: [], }; vi.spyOn(sdk, 'runComponentWithRunner').mockResolvedValue(payload); const result = component.outputs.parse(await component.execute(executePayload, context)); - expect(result).toEqual(component.outputs.parse(payload)); + expect(result.subdomains).toEqual(['api.example.com']); + expect(result.domainCount).toBe(1); + expect(result.subdomainCount).toBe(1); + expect(result.results).toHaveLength(1); + expect(result.results[0].scanner).toBe('subfinder'); + expect(result.results[0].severity).toBe('info'); }); it('should accept a single domain string and normalise to array', () => { diff --git a/worker/src/components/security/__tests__/trufflehog.test.ts b/worker/src/components/security/__tests__/trufflehog.test.ts index d3c4e13a..e9bfff11 100644 --- a/worker/src/components/security/__tests__/trufflehog.test.ts +++ b/worker/src/components/security/__tests__/trufflehog.test.ts @@ -97,6 +97,18 @@ describe('trufflehog component', () => { secretCount: 1, verifiedCount: 1, hasVerifiedSecrets: true, + results: [ + { + DetectorType: 'AWS', + DetectorName: 'AWS', + Verified: true, + Raw: 'AKIAIOSFODNN7EXAMPLE', + scanner: 'trufflehog', + severity: 'high', + finding_hash: 'abc123def456abcd', + asset_key: 'https://github.com/test/repo', + }, + ], }; vi.spyOn(sdk, 'runComponentWithRunner').mockResolvedValue(JSON.stringify(mockOutput)); @@ -107,6 +119,9 @@ describe('trufflehog component', () => { expect(result.verifiedCount).toBe(1); expect(result.hasVerifiedSecrets).toBe(true); expect(result.secrets).toHaveLength(1); + expect(result.results).toHaveLength(1); + expect(result.results[0].scanner).toBe('trufflehog'); + expect(result.results[0].severity).toBe('high'); }); it('should handle no secrets found', async () => { @@ -135,6 +150,7 @@ describe('trufflehog component', () => { secretCount: 0, verifiedCount: 0, hasVerifiedSecrets: false, + results: [], }; vi.spyOn(sdk, 'runComponentWithRunner').mockResolvedValue(JSON.stringify(mockOutput)); @@ -145,6 +161,7 @@ describe('trufflehog component', () => { expect(result.verifiedCount).toBe(0); expect(result.hasVerifiedSecrets).toBe(false); expect(result.secrets).toHaveLength(0); + expect(result.results).toHaveLength(0); }); it('should support different scan types', () => { @@ -234,6 +251,26 @@ describe('trufflehog component', () => { secretCount: 2, verifiedCount: 1, hasVerifiedSecrets: true, + results: [ + { + DetectorType: 'Generic', + Verified: false, + Raw: 'potential_secret_123', + scanner: 'trufflehog', + severity: 'high', + finding_hash: 'def456abc789def0', + asset_key: 'https://github.com/test/repo', + }, + { + DetectorType: 'AWS', + Verified: true, + Raw: 'AKIAIOSFODNN7EXAMPLE', + scanner: 'trufflehog', + severity: 'high', + finding_hash: 'abc123def456abcd', + asset_key: 'https://github.com/test/repo', + }, + ], }; vi.spyOn(sdk, 'runComponentWithRunner').mockResolvedValue(JSON.stringify(mockOutput)); @@ -243,6 +280,7 @@ describe('trufflehog component', () => { expect(result.secretCount).toBe(2); expect(result.verifiedCount).toBe(1); expect(result.hasVerifiedSecrets).toBe(true); + expect(result.results).toHaveLength(2); }); it('should handle parse errors gracefully', async () => { diff --git a/worker/src/components/security/abuseipdb.ts b/worker/src/components/security/abuseipdb.ts index c9a5a044..6dec02eb 100644 --- a/worker/src/components/security/abuseipdb.ts +++ b/worker/src/components/security/abuseipdb.ts @@ -13,6 +13,9 @@ import { param, coerceBooleanFromText, coerceNumberFromText, + generateFindingHash, + analyticsResultSchema, + type AnalyticsResult, } from '@shipsec/component-sdk'; const inputSchema = inputs({ @@ -102,6 +105,11 @@ const outputSchema = outputs({ reason: 'Full AbuseIPDB response payload varies by plan and API version.', connectionType: { kind: 'primitive', name: 'json' }, }), + results: port(z.array(analyticsResultSchema()), { + label: 'Results', + description: + 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', + }), }); const abuseIPDBRetryPolicy: ComponentRetryPolicy = { @@ -177,6 +185,7 @@ const definition = defineComponent({ context.logger.warn(`[AbuseIPDB] IP not found: ${ipAddress}`); return { ipAddress, + results: [], abuseConfidenceScore: 0, full_report: { error: 'Not Found' }, }; @@ -190,14 +199,46 @@ const definition = defineComponent({ const data = (await response.json()) as Record; const info = (data.data || {}) as Record; - context.logger.info(`[AbuseIPDB] Score for ${ipAddress}: ${info.abuseConfidenceScore}`); + const abuseConfidenceScore = info.abuseConfidenceScore as number; + + context.logger.info(`[AbuseIPDB] Score for ${ipAddress}: ${abuseConfidenceScore}`); + + // Determine severity based on abuse confidence score + let severity: 'critical' | 'high' | 'medium' | 'low' | 'info' | 'none' = 'none'; + if (abuseConfidenceScore >= 90) { + severity = 'critical'; + } else if (abuseConfidenceScore >= 70) { + severity = 'high'; + } else if (abuseConfidenceScore >= 50) { + severity = 'medium'; + } else if (abuseConfidenceScore >= 25) { + severity = 'low'; + } else if (abuseConfidenceScore > 0) { + severity = 'info'; + } + + // Build analytics-ready results + const analyticsResults: AnalyticsResult[] = [ + { + scanner: 'abuseipdb', + finding_hash: generateFindingHash('ip-reputation', ipAddress, String(abuseConfidenceScore)), + severity, + asset_key: ipAddress, + ip_address: ipAddress, + abuse_confidence_score: abuseConfidenceScore, + country_code: info.countryCode as string | undefined, + isp: info.isp as string | undefined, + total_reports: info.totalReports as number | undefined, + }, + ]; return { ipAddress: info.ipAddress as string, + results: analyticsResults, isPublic: info.isPublic as boolean | undefined, ipVersion: info.ipVersion as number | undefined, isWhitelisted: info.isWhitelisted as boolean | undefined, - abuseConfidenceScore: info.abuseConfidenceScore as number, + abuseConfidenceScore, countryCode: info.countryCode as string | undefined, usageType: info.usageType as string | undefined, isp: info.isp as string | undefined, diff --git a/worker/src/components/security/amass.ts b/worker/src/components/security/amass.ts index 16386347..16fe1094 100644 --- a/worker/src/components/security/amass.ts +++ b/worker/src/components/security/amass.ts @@ -11,6 +11,11 @@ import { port, param, type DockerRunnerConfig, + generateFindingHash, + analyticsResultSchema, + type AnalyticsResult, + type ExecutionContext, + type ExecutionPayload, } from '@shipsec/component-sdk'; import { IsolatedContainerVolume } from '../../utils/isolated-volume'; @@ -278,6 +283,11 @@ const outputSchema = outputs({ connectionType: { kind: 'primitive', name: 'json' }, }, ), + results: port(z.array(analyticsResultSchema()), { + label: 'Results', + description: + 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', + }), }); // Split custom CLI flags into an array of arguments @@ -452,7 +462,7 @@ const amassRetryPolicy: ComponentRetryPolicy = { nonRetryableErrorTypes: ['ContainerError', 'ValidationError', 'ConfigurationError'], }; -const definition = defineComponent({ +const definition = (defineComponent as any)({ id: 'shipsec.amass.enum', label: 'Amass Enumeration', category: 'security', @@ -506,7 +516,13 @@ const definition = defineComponent({ 'Perform quick passive reconnaissance using custom CLI flags like --passive.', ], }, - async execute({ inputs, params }, context) { + async execute( + { + inputs, + params, + }: ExecutionPayload, z.infer>, + context: ExecutionContext, + ) { const parsedParams = parameterSchema.parse(params); const { passive, @@ -718,12 +734,23 @@ const definition = defineComponent({ }); } + // Build analytics-ready results with scanner metadata + const analyticsResults: AnalyticsResult[] = subdomains.map((subdomain) => ({ + scanner: 'amass', + finding_hash: generateFindingHash('subdomain-discovery', subdomain, inputs.domains.join(',')), + severity: 'info' as const, + asset_key: subdomain, + subdomain, + parent_domains: inputs.domains, + })); + return { subdomains, rawOutput, domainCount, subdomainCount, options: optionsSummary, + results: analyticsResults, }; }, }); diff --git a/worker/src/components/security/dnsx.ts b/worker/src/components/security/dnsx.ts index 2dcf2f12..11c690ba 100644 --- a/worker/src/components/security/dnsx.ts +++ b/worker/src/components/security/dnsx.ts @@ -11,6 +11,9 @@ import { parameters, port, param, + generateFindingHash, + analyticsResultSchema, + type AnalyticsResult, } from '@shipsec/component-sdk'; import { IsolatedContainerVolume } from '../../utils/isolated-volume'; @@ -235,8 +238,8 @@ const dnsxLineSchema = z .passthrough(); const outputSchema = outputs({ - results: port(z.array(z.any()), { - label: 'Results', + dnsRecords: port(z.array(z.any()), { + label: 'DNS Records', description: 'DNS resolution results returned by dnsx.', allowAny: true, reason: 'dnsx returns heterogeneous record payloads.', @@ -270,6 +273,11 @@ const outputSchema = outputs({ label: 'Errors', description: 'Errors encountered during dnsx execution.', }), + results: port(z.array(analyticsResultSchema()), { + label: 'Results', + description: + 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', + }), }); const splitCliArgs = (input: string): string[] => { @@ -566,6 +574,7 @@ const definition = defineComponent({ if (domainCount === 0) { context.logger.info('[DNSX] Skipping dnsx execution because no domains were provided.'); return outputSchema.parse({ + dnsRecords: [], results: [], rawOutput: '', domainCount: 0, @@ -770,8 +779,24 @@ const definition = defineComponent({ .filter((host): host is string => typeof host === 'string' && host.length > 0), ); + // Build analytics-ready results with scanner metadata + const analyticsResults: AnalyticsResult[] = normalisedRecords.map((record) => ({ + scanner: 'dnsx', + finding_hash: generateFindingHash( + 'dns-resolution', + record.host, + JSON.stringify(record.answers), + ), + severity: 'info' as const, + asset_key: record.host, + host: record.host, + record_types: Object.keys(record.answers), + answers: record.answers, + })); + return { - results: normalisedRecords, + dnsRecords: normalisedRecords, + results: analyticsResults, rawOutput: params.rawOutput, domainCount: params.domainCount, recordCount: params.recordCount, @@ -810,6 +835,7 @@ const definition = defineComponent({ if (trimmed.length === 0) { return { + dnsRecords: [], results: [], rawOutput, domainCount: domainCount, @@ -834,6 +860,7 @@ const definition = defineComponent({ ? (record.domainCount as number) : domainCount; return { + dnsRecords: [], results: [], rawOutput: trimmed, domainCount: errorDomainCount, @@ -848,10 +875,10 @@ const definition = defineComponent({ const validated = outputSchema.safeParse(record); if (validated.success) { return buildOutput({ - records: validated.data.results as z.infer[], + records: validated.data.dnsRecords as z.infer[], rawOutput: validated.data.rawOutput ?? rawOutput, domainCount: validated.data.domainCount ?? domainCount, - recordCount: validated.data.recordCount ?? validated.data.results.length, + recordCount: validated.data.recordCount ?? validated.data.dnsRecords.length, recordTypes: validated.data.recordTypes ?? recordTypes, resolvers: validated.data.resolvers ?? resolverList, errors: validated.data.errors, @@ -869,6 +896,7 @@ const definition = defineComponent({ if (lines.length === 0) { return { + dnsRecords: [], results: [], rawOutput, domainCount: domainCount, @@ -895,8 +923,24 @@ const definition = defineComponent({ }; }); + // Build analytics-ready results + const analyticsResults: AnalyticsResult[] = silentRecords.map((record) => ({ + scanner: 'dnsx', + finding_hash: generateFindingHash( + 'dns-resolution', + record.host, + JSON.stringify(record.answers), + ), + severity: 'info' as const, + asset_key: record.host, + host: record.host, + record_types: Object.keys(record.answers), + answers: record.answers, + })); + return { - results: silentRecords, + dnsRecords: silentRecords, + results: analyticsResults, rawOutput, domainCount: domainCount, recordCount: silentRecords.length, @@ -919,6 +963,7 @@ const definition = defineComponent({ if (trimmed.length === 0) { return { + dnsRecords: [], results: [], rawOutput, domainCount: domainCount, @@ -975,8 +1020,24 @@ const definition = defineComponent({ }; }); + // Build analytics-ready results + const analyticsResults: AnalyticsResult[] = fallbackResults.map((record) => ({ + scanner: 'dnsx', + finding_hash: generateFindingHash( + 'dns-resolution', + record.host, + JSON.stringify(record.answers), + ), + severity: 'info' as const, + asset_key: record.host, + host: record.host, + record_types: Object.keys(record.answers), + answers: record.answers, + })); + return { - results: fallbackResults, + dnsRecords: fallbackResults, + results: analyticsResults, rawOutput, domainCount: domainCount, recordCount: fallbackResults.length, @@ -1016,6 +1077,7 @@ const definition = defineComponent({ : JSON.stringify(rawPayload, null, 2).slice(0, 5000); return { + dnsRecords: [], results: [], rawOutput, domainCount: domainCount, @@ -1028,10 +1090,10 @@ const definition = defineComponent({ } return buildOutput({ - records: safeResult.data.results as z.infer[], + records: safeResult.data.dnsRecords as z.infer[], rawOutput: safeResult.data.rawOutput, domainCount: safeResult.data.domainCount ?? domainCount, - recordCount: safeResult.data.recordCount ?? safeResult.data.results.length, + recordCount: safeResult.data.recordCount ?? safeResult.data.dnsRecords.length, recordTypes: safeResult.data.recordTypes, resolvers: safeResult.data.resolvers, errors: safeResult.data.errors, diff --git a/worker/src/components/security/httpx.ts b/worker/src/components/security/httpx.ts index 84198ea5..f885d6ab 100644 --- a/worker/src/components/security/httpx.ts +++ b/worker/src/components/security/httpx.ts @@ -10,6 +10,9 @@ import { parameters, port, param, + generateFindingHash, + analyticsResultSchema, + type AnalyticsResult, } from '@shipsec/component-sdk'; import { IsolatedContainerVolume } from '../../utils/isolated-volume'; @@ -145,7 +148,7 @@ const findingSchema = z.object({ type Finding = z.infer; const outputSchema = outputs({ - results: port(z.array(findingSchema), { + responses: port(z.array(findingSchema), { label: 'HTTP Responses', description: 'Structured metadata for each responsive endpoint.', connectionType: { kind: 'list', element: { kind: 'primitive', name: 'json' } }, @@ -178,6 +181,11 @@ const outputSchema = outputs({ connectionType: { kind: 'primitive', name: 'json' }, }, ), + results: port(z.array(analyticsResultSchema()), { + label: 'Results', + description: + 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', + }), }); const httpxRunnerOutputSchema = z.object({ @@ -272,6 +280,7 @@ const definition = defineComponent({ if (runnerParams.targets.length === 0) { context.logger.info('[httpx] Skipping httpx probe because no targets were provided.'); const emptyOutput: Output = { + responses: [], results: [], rawOutput: '', targetCount: 0, @@ -395,8 +404,27 @@ const definition = defineComponent({ `[httpx] Completed probe with ${findings.length} result(s) from ${runnerParams.targets.length} target(s)`, ); + // Build analytics-ready results with scanner metadata + const analyticsResults: AnalyticsResult[] = findings.map((finding) => ({ + scanner: 'httpx', + finding_hash: generateFindingHash( + 'http-endpoint', + finding.url, + String(finding.statusCode ?? 0), + ), + severity: 'info' as const, + asset_key: finding.url, + url: finding.url, + host: finding.host, + status_code: finding.statusCode, + title: finding.title, + webserver: finding.webserver, + technologies: finding.technologies, + })); + const output: Output = { - results: findings, + responses: findings, + results: analyticsResults, rawOutput: runnerOutput, targetCount: runnerParams.targets.length, resultCount: findings.length, diff --git a/worker/src/components/security/naabu.ts b/worker/src/components/security/naabu.ts index 052aa16a..8f5de0bf 100644 --- a/worker/src/components/security/naabu.ts +++ b/worker/src/components/security/naabu.ts @@ -8,6 +8,9 @@ import { parameters, port, param, + generateFindingHash, + analyticsResultSchema, + type AnalyticsResult, } from '@shipsec/component-sdk'; const inputSchema = inputs({ @@ -160,6 +163,11 @@ const outputSchema = outputs({ connectionType: { kind: 'primitive', name: 'json' }, }, ), + results: port(z.array(analyticsResultSchema()), { + label: 'Results', + description: + 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', + }), }); type Finding = z.infer; @@ -338,8 +346,22 @@ eval "$CMD" if (typeof result === 'string') { const findings = parseNaabuOutput(result); + + // Build analytics-ready results with scanner metadata + const analyticsResults: AnalyticsResult[] = findings.map((finding) => ({ + scanner: 'naabu', + finding_hash: generateFindingHash('open-port', finding.host, String(finding.port)), + severity: 'info' as const, + asset_key: `${finding.host}:${finding.port}`, + host: finding.host, + port: finding.port, + protocol: finding.protocol, + ip: finding.ip, + })); + const output: Output = { findings, + results: analyticsResults, rawOutput: result, targetCount: runnerParams.targets.length, openPortCount: findings.length, @@ -365,6 +387,7 @@ eval "$CMD" return { findings: [], + results: [], rawOutput: typeof result === 'string' ? result : '', targetCount: runnerParams.targets.length, openPortCount: 0, diff --git a/worker/src/components/security/nuclei.ts b/worker/src/components/security/nuclei.ts index c6bbab5c..84e51e70 100644 --- a/worker/src/components/security/nuclei.ts +++ b/worker/src/components/security/nuclei.ts @@ -12,6 +12,9 @@ import { parameters, port, param, + generateFindingHash, + analyticsResultSchema, + type AnalyticsResult, } from '@shipsec/component-sdk'; import { IsolatedContainerVolume } from '../../utils/isolated-volume'; import * as yaml from 'js-yaml'; @@ -203,6 +206,11 @@ const outputSchema = outputs({ description: 'Array of detected vulnerabilities with severity, tags, and matched URLs.', connectionType: { kind: 'list', element: { kind: 'primitive', name: 'json' } }, }), + results: port(z.array(analyticsResultSchema()), { + label: 'Results', + description: + 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', + }), rawOutput: port(z.string(), { label: 'Raw Output', description: 'Complete JSONL output from nuclei for downstream processing.', @@ -549,8 +557,17 @@ const definition = defineComponent({ `[Nuclei] Scan complete: ${findings.length} finding(s) from ${parsedInputs.targets.length} target(s)`, ); + // Build analytics-ready results with scanner metadata (follows core.analytics.result.v1 contract) + const results: AnalyticsResult[] = findings.map((finding) => ({ + ...finding, + scanner: 'nuclei', + asset_key: finding.host ?? finding.matchedAt, + finding_hash: generateFindingHash(finding.templateId, finding.host, finding.matchedAt), + })); + const output = { findings, + results, rawOutput: stdout, targetCount: parsedInputs.targets.length, findingCount: findings.length, diff --git a/worker/src/components/security/prowler-scan.ts b/worker/src/components/security/prowler-scan.ts index bc299fd1..589f9d22 100644 --- a/worker/src/components/security/prowler-scan.ts +++ b/worker/src/components/security/prowler-scan.ts @@ -14,6 +14,9 @@ import { parameters, port, param, + analyticsResultSchema, + generateFindingHash, + type AnalyticsResult, } from '@shipsec/component-sdk'; import type { DockerRunnerConfig } from '@shipsec/component-sdk'; @@ -247,6 +250,11 @@ const outputSchema = outputs({ 'Array of normalized findings derived from Prowler ASFF output (includes severity, resource id, remediation).', connectionType: { kind: 'list', element: { kind: 'primitive', name: 'json' } }, }), + results: port(z.array(analyticsResultSchema()), { + label: 'Results', + description: + 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', + }), rawOutput: port(z.string(), { label: 'Raw Output', description: 'Raw Prowler output for debugging.', @@ -734,9 +742,29 @@ const definition = defineComponent({ const scanId = buildScanId(parsedInputs.accountId, parsedParams.scanMode); + // Build analytics-ready results (follows core.analytics.result.v1 contract) + const results: AnalyticsResult[] = findings.map((finding) => ({ + scanner: 'prowler', + finding_hash: generateFindingHash( + finding.id, + finding.resourceId ?? finding.accountId ?? '', + finding.title ?? '', + ), + severity: mapToAnalyticsSeverity(finding.severity), + asset_key: finding.resourceId ?? finding.accountId ?? undefined, + // Include additional context for analytics + title: finding.title, + description: finding.description, + region: finding.region, + status: finding.status, + remediationText: finding.remediationText, + recommendationUrl: finding.recommendationUrl, + })); + const output: Output = { scanId, findings, + results, rawOutput: rawSegments.join('\n'), summary: { totalFindings: findings.length, @@ -943,6 +971,31 @@ function extractRegionFromArn(resourceId?: string): string | null { return null; } +/** + * Maps Prowler severity levels to analytics severity enum. + * Prowler: critical, high, medium, low, informational, unknown + * Analytics: critical, high, medium, low, info, none + */ +function mapToAnalyticsSeverity( + prowlerSeverity: NormalisedSeverity, +): 'critical' | 'high' | 'medium' | 'low' | 'info' | 'none' { + switch (prowlerSeverity) { + case 'critical': + return 'critical'; + case 'high': + return 'high'; + case 'medium': + return 'medium'; + case 'low': + return 'low'; + case 'informational': + return 'info'; + case 'unknown': + default: + return 'none'; + } +} + componentRegistry.register(definition); // Create local type aliases for backward compatibility diff --git a/worker/src/components/security/shuffledns-massdns.ts b/worker/src/components/security/shuffledns-massdns.ts index fff8e396..d3690bea 100644 --- a/worker/src/components/security/shuffledns-massdns.ts +++ b/worker/src/components/security/shuffledns-massdns.ts @@ -12,6 +12,9 @@ import { parameters, port, param, + generateFindingHash, + analyticsResultSchema, + type AnalyticsResult, } from '@shipsec/component-sdk'; import { IsolatedContainerVolume } from '../../utils/isolated-volume'; @@ -162,6 +165,11 @@ const outputSchema = outputs({ label: 'Subdomain Count', description: 'Number of unique subdomains discovered.', }), + results: port(z.array(analyticsResultSchema()), { + label: 'Results', + description: + 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', + }), }); const definition = defineComponent({ @@ -371,8 +379,19 @@ const definition = defineComponent({ const deduped = Array.from(new Set(subdomains)); + // Build analytics-ready results with scanner metadata + const analyticsResults: AnalyticsResult[] = deduped.map((subdomain) => ({ + scanner: 'shuffledns', + finding_hash: generateFindingHash('subdomain-discovery', subdomain, domains.join(',')), + severity: 'info' as const, + asset_key: subdomain, + subdomain, + parent_domains: domains, + })); + return outputSchema.parse({ subdomains: deduped, + results: analyticsResults, rawOutput, domainCount: domains.length, subdomainCount: deduped.length, @@ -397,17 +416,31 @@ const definition = defineComponent({ .map((line) => line.trim()) .filter((line) => line.length > 0); + const deduped = Array.from(new Set(subdomainsValue)); + + // Build analytics-ready results + const analyticsResults: AnalyticsResult[] = deduped.map((subdomain) => ({ + scanner: 'shuffledns', + finding_hash: generateFindingHash('subdomain-discovery', subdomain, domains.join(',')), + severity: 'info' as const, + asset_key: subdomain, + subdomain, + parent_domains: domains, + })); + return outputSchema.parse({ - subdomains: Array.from(new Set(subdomainsValue)), + subdomains: deduped, + results: analyticsResults, rawOutput: maybeRaw || subdomainsValue.join('\n'), domainCount: domains.length, - subdomainCount: subdomainsValue.length, + subdomainCount: deduped.length, }); } // Fallback ā€“ empty return outputSchema.parse({ subdomains: [], + results: [], rawOutput: '', domainCount: domains.length, subdomainCount: 0, diff --git a/worker/src/components/security/subfinder.ts b/worker/src/components/security/subfinder.ts index 0dce2f4a..ddfbf12b 100644 --- a/worker/src/components/security/subfinder.ts +++ b/worker/src/components/security/subfinder.ts @@ -11,6 +11,9 @@ import { parameters, port, param, + generateFindingHash, + analyticsResultSchema, + type AnalyticsResult, } from '@shipsec/component-sdk'; import { IsolatedContainerVolume } from '../../utils/isolated-volume'; @@ -123,6 +126,11 @@ const outputSchema = outputs({ label: 'Subdomain Count', description: 'Number of subdomains discovered.', }), + results: port(z.array(analyticsResultSchema()), { + label: 'Results', + description: + 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', + }), }); // Split custom CLI flags into an array of arguments @@ -360,6 +368,7 @@ const definition = defineComponent({ context.logger.info('[Subfinder] Skipping execution because no domains were provided.'); return { subdomains: [], + results: [], rawOutput: '', domainCount: 0, subdomainCount: 0, @@ -511,11 +520,22 @@ const definition = defineComponent({ }); } + // Build analytics-ready results with scanner metadata + const analyticsResults: AnalyticsResult[] = subdomains.map((subdomain) => ({ + scanner: 'subfinder', + finding_hash: generateFindingHash('subdomain-discovery', subdomain, domains.join(',')), + severity: 'info' as const, + asset_key: subdomain, + subdomain, + parent_domains: domains, + })); + return { subdomains, rawOutput, domainCount, subdomainCount, + results: analyticsResults, }; }, }); diff --git a/worker/src/components/security/supabase-scanner.ts b/worker/src/components/security/supabase-scanner.ts index 4bd0d167..5b1d3c96 100644 --- a/worker/src/components/security/supabase-scanner.ts +++ b/worker/src/components/security/supabase-scanner.ts @@ -10,8 +10,11 @@ import { parameters, port, param, + generateFindingHash, + analyticsResultSchema, + type AnalyticsResult, + type DockerRunnerConfig, } from '@shipsec/component-sdk'; -import type { DockerRunnerConfig } from '@shipsec/component-sdk'; import { IsolatedContainerVolume } from '../../utils/isolated-volume'; // Extract Supabase project ref from a standard URL like https://.supabase.co @@ -150,6 +153,11 @@ const outputSchema = outputs({ reason: 'Scanner issue payloads can vary by Supabase project configuration.', connectionType: { kind: 'list', element: { kind: 'primitive', name: 'json' } }, }), + results: port(z.array(analyticsResultSchema()), { + label: 'Results', + description: + 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', + }), report: port(z.unknown(), { label: 'Scanner Report', description: 'Full JSON report produced by the scanner.', @@ -329,6 +337,19 @@ const definition = defineComponent({ } catch (err) { const msg = (err as Error)?.message ?? 'Unknown error'; context.logger.error(`[SupabaseScanner] Scanner failed: ${msg}`); + + // Check if this is a fatal Docker error (image pull failure, container start failure) + // These should fail hard, not gracefully degrade + if ( + msg.includes('exit code 125') || + msg.includes('Unable to find image') || + msg.includes('permission denied') || + msg.includes('authentication required') + ) { + throw err; + } + + // For other errors (scanner runtime errors), allow graceful degradation errors.push(msg); } @@ -357,6 +378,22 @@ const definition = defineComponent({ } catch (err) { const msg = (err as Error)?.message ?? 'Unknown error'; context.logger.error(`[SupabaseScanner] Scanner failed: ${msg}`); + + // Check if this is a fatal Docker error that should fail the workflow + if ( + msg.includes('exit code 125') || + msg.includes('Unable to find image') || + msg.includes('permission denied') || + msg.includes('authentication required') + ) { + // Cleanup volume before throwing + if (volumeInitialized) { + await volume.cleanup(); + context.logger.info('[SupabaseScanner] Cleaned up isolated volume'); + } + throw err; + } + errors.push(msg); } finally { if (volumeInitialized) { @@ -365,11 +402,34 @@ const definition = defineComponent({ } } + // Build analytics-ready results with scanner metadata (follows core.analytics.result.v1 contract) + const results: AnalyticsResult[] = (issues ?? []).map((issue) => { + const issueObj = typeof issue === 'object' && issue !== null ? issue : { raw: issue }; + const issueRecord = issueObj as Record; + // Extract check_id and resource for deduplication hash + const checkId = issueRecord.check_id as string | undefined; + const resource = issueRecord.resource as string | undefined; + // Map severity from scanner output or default to 'medium' for security issues + const rawSeverity = (issueRecord.severity as string | undefined)?.toLowerCase(); + const validSeverities = ['critical', 'high', 'medium', 'low', 'info', 'none'] as const; + const severity = validSeverities.includes(rawSeverity as (typeof validSeverities)[number]) + ? (rawSeverity as (typeof validSeverities)[number]) + : 'medium'; + return { + ...issueObj, + scanner: 'supabase-scanner', + severity, + asset_key: projectRef ?? undefined, + finding_hash: generateFindingHash(checkId, projectRef, resource), + }; + }); + const output: Output = { projectRef: projectRef ?? null, score, summary, issues, + results, report, rawOutput: stdoutCombined ?? '', errors: errors.length > 0 ? errors : undefined, diff --git a/worker/src/components/security/trufflehog.ts b/worker/src/components/security/trufflehog.ts index 3ed88eef..3388ad4a 100644 --- a/worker/src/components/security/trufflehog.ts +++ b/worker/src/components/security/trufflehog.ts @@ -12,6 +12,9 @@ import { parameters, port, param, + generateFindingHash, + analyticsResultSchema, + type AnalyticsResult, } from '@shipsec/component-sdk'; import { IsolatedContainerVolume } from '../../utils/isolated-volume'; @@ -186,6 +189,11 @@ const outputSchema = outputs({ label: 'Has Verified Secrets', description: 'True when any verified secrets are detected.', }), + results: port(z.array(analyticsResultSchema()), { + label: 'Results', + description: + 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', + }), }); // Helper function to build TruffleHog command arguments @@ -256,6 +264,7 @@ function parseRawOutput(rawOutput: string): Output { secretCount: 0, verifiedCount: 0, hasVerifiedSecrets: false, + results: [], }; } @@ -294,6 +303,7 @@ function parseRawOutput(rawOutput: string): Output { secretCount: secrets.length, verifiedCount, hasVerifiedSecrets: verifiedCount > 0, + results: [], // Populated in execute() with scanner metadata }; } @@ -493,7 +503,23 @@ const definition = defineComponent({ }); } - return output; + // Build analytics-ready results with scanner metadata (follows core.analytics.result.v1 contract) + const results: AnalyticsResult[] = output.secrets.map((secret: Secret) => { + // Extract file path from source metadata for hashing + const filePath = + secret.SourceMetadata?.Data?.Git?.file ?? + secret.SourceMetadata?.Data?.Filesystem?.file ?? + ''; + return { + ...secret, + scanner: 'trufflehog', + severity: 'high' as const, // Secrets are always high severity + asset_key: runnerPayload.scanTarget, + finding_hash: generateFindingHash(secret.DetectorType, secret.Redacted, filePath), + }; + }); + + return { ...output, results }; } finally { // Always cleanup volume if it was created if (volume) { diff --git a/worker/src/components/security/virustotal.ts b/worker/src/components/security/virustotal.ts index 76bcad0d..f9699ea2 100644 --- a/worker/src/components/security/virustotal.ts +++ b/worker/src/components/security/virustotal.ts @@ -11,6 +11,9 @@ import { parameters, port, param, + generateFindingHash, + analyticsResultSchema, + type AnalyticsResult, } from '@shipsec/component-sdk'; const inputSchema = inputs({ @@ -64,6 +67,11 @@ const outputSchema = outputs({ connectionType: { kind: 'primitive', name: 'json' }, }, ), + results: port(z.array(analyticsResultSchema()), { + label: 'Results', + description: + 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', + }), }); // Retry policy for VirusTotal API - handles rate limits and transient failures @@ -167,6 +175,7 @@ const definition = defineComponent({ suspicious: 0, harmless: 0, tags: [], + results: [], full_report: { error: 'Not Found in VirusTotal' }, }; } @@ -190,12 +199,44 @@ const definition = defineComponent({ `[VirusTotal] Results for ${indicator}: ${malicious} malicious, ${suspicious} suspicious.`, ); + // Determine severity based on malicious/suspicious counts + let severity: 'critical' | 'high' | 'medium' | 'low' | 'info' | 'none' = 'none'; + if (malicious >= 10) { + severity = 'critical'; + } else if (malicious >= 5) { + severity = 'high'; + } else if (malicious >= 1 || suspicious >= 5) { + severity = 'medium'; + } else if (suspicious >= 1) { + severity = 'low'; + } else { + severity = 'info'; + } + + // Build analytics-ready results + const analyticsResults: AnalyticsResult[] = [ + { + scanner: 'virustotal', + finding_hash: generateFindingHash('threat-intelligence', indicator, type), + severity, + asset_key: indicator, + indicator, + indicator_type: type, + malicious_count: malicious, + suspicious_count: suspicious, + harmless_count: harmless, + reputation, + tags, + }, + ]; + return { malicious, suspicious, harmless, tags, reputation, + results: analyticsResults, full_report: data, }; }, diff --git a/worker/src/components/test/__tests__/analytics-fixture.test.ts b/worker/src/components/test/__tests__/analytics-fixture.test.ts new file mode 100644 index 00000000..30d7ff33 --- /dev/null +++ b/worker/src/components/test/__tests__/analytics-fixture.test.ts @@ -0,0 +1,36 @@ +import { beforeAll, describe, expect, it } from 'bun:test'; +import { componentRegistry, createExecutionContext } from '@shipsec/component-sdk'; +import type { AnalyticsFixtureInput, AnalyticsFixtureOutput } from '../analytics-fixture'; + +describe('test.analytics.fixture component', () => { + beforeAll(async () => { + await import('../../index'); + }); + + it('should be registered', () => { + const component = componentRegistry.get( + 'test.analytics.fixture', + ); + expect(component).toBeDefined(); + expect(component!.label).toBe('Analytics Fixture (Test)'); + }); + + it('should emit deterministic analytics results', async () => { + const component = componentRegistry.get( + 'test.analytics.fixture', + ); + if (!component) { + throw new Error('test.analytics.fixture not registered'); + } + + const context = createExecutionContext({ + runId: 'analytics-fixture-test', + componentRef: 'fixture-node', + }); + + const result = await component.execute({ inputs: {}, params: {} }, context); + expect(Array.isArray(result.results)).toBe(true); + expect(result.results.length).toBeGreaterThanOrEqual(2); + expect(result.results[0].scanner).toBe('analytics-fixture'); + }); +}); diff --git a/worker/src/components/test/analytics-fixture.ts b/worker/src/components/test/analytics-fixture.ts new file mode 100644 index 00000000..f50e482e --- /dev/null +++ b/worker/src/components/test/analytics-fixture.ts @@ -0,0 +1,79 @@ +import { z } from 'zod'; +import { + componentRegistry, + defineComponent, + inputs, + outputs, + port, + analyticsResultSchema, + generateFindingHash, + type ExecutionContext, +} from '@shipsec/component-sdk'; + +const inputSchema = inputs({ + trigger: port(z.any().optional(), { + label: 'Trigger', + description: 'Optional trigger input to allow wiring from entrypoint.', + allowAny: true, + reason: 'Test fixture accepts any trigger input for E2E testing flexibility.', + }), +}); + +const outputSchema = outputs({ + results: port(z.array(analyticsResultSchema()), { + label: 'Results', + description: 'Deterministic analytics results for E2E testing.', + }), +}); + +const definition = (defineComponent as any)({ + id: 'test.analytics.fixture', + label: 'Analytics Fixture (Test)', + category: 'transform', + runner: { kind: 'inline' }, + inputs: inputSchema, + outputs: outputSchema, + docs: 'Emits deterministic analytics results for end-to-end tests.', + ui: { + slug: 'analytics-fixture', + version: '1.0.0', + type: 'process', + category: 'transform', + description: 'Test-only component that emits deterministic analytics results.', + author: { + name: 'ShipSecAI', + type: 'shipsecai', + }, + }, + async execute(_payload: z.infer, _context: ExecutionContext) { + const results = [ + { + scanner: 'analytics-fixture', + severity: 'high', + title: 'Fixture Finding 1', + asset_key: 'fixture.local', + host: 'fixture.local', + finding_hash: generateFindingHash('fixture', 'finding-1'), + }, + { + scanner: 'analytics-fixture', + severity: 'low', + title: 'Fixture Finding 2', + asset_key: 'fixture.local', + host: 'fixture.local', + finding_hash: generateFindingHash('fixture', 'finding-2'), + }, + ]; + + return { results }; + }, +}); + +if (!componentRegistry.has(definition.id)) { + componentRegistry.register(definition); +} + +type Input = typeof inputSchema; +type Output = typeof outputSchema; + +export type { Input as AnalyticsFixtureInput, Output as AnalyticsFixtureOutput }; diff --git a/worker/src/temporal/__tests__/optional-input-handling.test.ts b/worker/src/temporal/__tests__/optional-input-handling.test.ts new file mode 100644 index 00000000..98d20a65 --- /dev/null +++ b/worker/src/temporal/__tests__/optional-input-handling.test.ts @@ -0,0 +1,279 @@ +/** + * Tests for optional input handling in component execution + * + * Validates that components with optional inputs (required: false or connectionType.kind === 'any') + * can proceed when upstream components return undefined values, instead of failing with + * a ValidationError. + * + * This tests the fix for workflows getting stuck in infinite retry loops when an upstream + * component fails gracefully and returns undefined for some outputs. + */ + +import { describe, expect, it, beforeAll } from 'bun:test'; +import { z } from 'zod'; +import { + componentRegistry, + defineComponent, + inputs, + outputs, + port, + createExecutionContext, + extractPorts, + type ComponentPortMetadata, +} from '@shipsec/component-sdk'; + +describe('Optional Input Handling', () => { + beforeAll(() => { + // Register test component with optional input (required: false) + if (!componentRegistry.has('test.optional.required-false')) { + const component = defineComponent({ + id: 'test.optional.required-false', + label: 'Optional Input (required: false)', + category: 'transform', + runner: { kind: 'inline' }, + inputs: inputs({ + requiredInput: port(z.string(), { + label: 'Required Input', + description: 'This input is required', + }), + optionalInput: port(z.string().optional(), { + label: 'Optional Input', + description: 'This input is optional', + }), + }), + outputs: outputs({ + result: port(z.string(), { label: 'Result' }), + }), + async execute({ inputs }) { + return { + result: `required: ${inputs.requiredInput}, optional: ${inputs.optionalInput ?? 'undefined'}`, + }; + }, + }); + componentRegistry.register(component); + } + + // Register test component with allowAny input (connectionType.kind === 'any') + if (!componentRegistry.has('test.optional.allow-any')) { + const component = defineComponent({ + id: 'test.optional.allow-any', + label: 'Optional Input (allowAny)', + category: 'transform', + runner: { kind: 'inline' }, + inputs: inputs({ + requiredInput: port(z.string(), { + label: 'Required Input', + description: 'This input is required', + }), + anyInput: port(z.any(), { + label: 'Any Input', + description: 'This input accepts any type including undefined', + allowAny: true, + reason: 'Accepts arbitrary data for testing', + connectionType: { kind: 'any' }, + }), + }), + outputs: outputs({ + result: port(z.string(), { label: 'Result' }), + }), + async execute({ inputs }) { + return { + result: `required: ${inputs.requiredInput}, any: ${inputs.anyInput ?? 'undefined'}`, + }; + }, + }); + componentRegistry.register(component); + } + + // Register test component with all required inputs + if (!componentRegistry.has('test.all-required')) { + const component = defineComponent({ + id: 'test.all-required', + label: 'All Required Inputs', + category: 'transform', + runner: { kind: 'inline' }, + inputs: inputs({ + input1: port(z.string(), { + label: 'Input 1', + description: 'Required input 1', + }), + input2: port(z.string(), { + label: 'Input 2', + description: 'Required input 2', + }), + }), + outputs: outputs({ + result: port(z.string(), { label: 'Result' }), + }), + async execute({ inputs }) { + return { result: `${inputs.input1} + ${inputs.input2}` }; + }, + }); + componentRegistry.register(component); + } + }); + + describe('extractPorts identifies optional inputs correctly', () => { + it('identifies required: false as optional', () => { + const component = componentRegistry.get('test.optional.required-false'); + expect(component).toBeDefined(); + + const ports = extractPorts(component!.inputs); + const optionalPort = ports.find((p: ComponentPortMetadata) => p.id === 'optionalInput'); + + expect(optionalPort).toBeDefined(); + expect(optionalPort!.required).toBe(false); + }); + + it('identifies connectionType.kind === "any" as optional', () => { + const component = componentRegistry.get('test.optional.allow-any'); + expect(component).toBeDefined(); + + const ports = extractPorts(component!.inputs); + const anyPort = ports.find((p: ComponentPortMetadata) => p.id === 'anyInput'); + + expect(anyPort).toBeDefined(); + expect(anyPort!.connectionType?.kind).toBe('any'); + }); + + it('identifies regular inputs as required', () => { + const component = componentRegistry.get('test.all-required'); + expect(component).toBeDefined(); + + const ports = extractPorts(component!.inputs); + + for (const port of ports) { + // Required is either undefined (defaults to true) or explicitly true + expect(port.required).not.toBe(false); + expect(port.connectionType?.kind).not.toBe('any'); + } + }); + }); + + describe('filterRequiredMissingInputs logic', () => { + /** + * This test validates the core logic used in run-component.activity.ts + * to filter out optional inputs from the missing inputs list. + */ + it('filters out optional inputs from missing list', () => { + const component = componentRegistry.get('test.optional.required-false'); + expect(component).toBeDefined(); + + const inputPorts = extractPorts(component!.inputs); + + // Simulate warnings for both inputs being undefined + const warningsToReport = [ + { target: 'requiredInput', sourceRef: 'upstream', sourceHandle: 'output' }, + { target: 'optionalInput', sourceRef: 'upstream', sourceHandle: 'output' }, + ]; + + // Apply the filtering logic from run-component.activity.ts + const requiredMissingInputs = warningsToReport.filter((warning) => { + const portMeta = inputPorts.find((p: ComponentPortMetadata) => p.id === warning.target); + if (!portMeta) return true; + if (portMeta.required === false) return false; + if (portMeta.connectionType?.kind === 'any') return false; + return true; + }); + + // Only requiredInput should be in the filtered list + expect(requiredMissingInputs).toHaveLength(1); + expect(requiredMissingInputs[0].target).toBe('requiredInput'); + }); + + it('filters out allowAny inputs from missing list', () => { + const component = componentRegistry.get('test.optional.allow-any'); + expect(component).toBeDefined(); + + const inputPorts = extractPorts(component!.inputs); + + // Simulate warnings for both inputs being undefined + const warningsToReport = [ + { target: 'requiredInput', sourceRef: 'upstream', sourceHandle: 'output' }, + { target: 'anyInput', sourceRef: 'upstream', sourceHandle: 'output' }, + ]; + + // Apply the filtering logic from run-component.activity.ts + const requiredMissingInputs = warningsToReport.filter((warning) => { + const portMeta = inputPorts.find((p: ComponentPortMetadata) => p.id === warning.target); + if (!portMeta) return true; + if (portMeta.required === false) return false; + if (portMeta.connectionType?.kind === 'any') return false; + return true; + }); + + // Only requiredInput should be in the filtered list + expect(requiredMissingInputs).toHaveLength(1); + expect(requiredMissingInputs[0].target).toBe('requiredInput'); + }); + + it('keeps all required inputs in missing list', () => { + const component = componentRegistry.get('test.all-required'); + expect(component).toBeDefined(); + + const inputPorts = extractPorts(component!.inputs); + + // Simulate warnings for both inputs being undefined + const warningsToReport = [ + { target: 'input1', sourceRef: 'upstream', sourceHandle: 'output' }, + { target: 'input2', sourceRef: 'upstream', sourceHandle: 'output' }, + ]; + + // Apply the filtering logic from run-component.activity.ts + const requiredMissingInputs = warningsToReport.filter((warning) => { + const portMeta = inputPorts.find((p: ComponentPortMetadata) => p.id === warning.target); + if (!portMeta) return true; + if (portMeta.required === false) return false; + if (portMeta.connectionType?.kind === 'any') return false; + return true; + }); + + // Both inputs should be in the filtered list + expect(requiredMissingInputs).toHaveLength(2); + }); + }); + + describe('component execution with optional inputs', () => { + it('executes component with undefined optional input (required: false)', async () => { + const component = componentRegistry.get('test.optional.required-false'); + expect(component).toBeDefined(); + + const context = createExecutionContext({ + runId: 'test-run', + componentRef: 'test-node', + }); + + // Execute with only the required input + const result = await component!.execute!( + { + inputs: { requiredInput: 'hello', optionalInput: undefined }, + params: {}, + }, + context, + ); + + expect(result).toEqual({ result: 'required: hello, optional: undefined' }); + }); + + it('executes component with undefined allowAny input', async () => { + const component = componentRegistry.get('test.optional.allow-any'); + expect(component).toBeDefined(); + + const context = createExecutionContext({ + runId: 'test-run', + componentRef: 'test-node', + }); + + // Execute with only the required input + const result = await component!.execute!( + { + inputs: { requiredInput: 'hello', anyInput: undefined }, + params: {}, + }, + context, + ); + + expect(result).toEqual({ result: 'required: hello, any: undefined' }); + }); + }); +}); diff --git a/worker/src/temporal/activities/run-component.activity.ts b/worker/src/temporal/activities/run-component.activity.ts index 803594cc..94c19658 100644 --- a/worker/src/temporal/activities/run-component.activity.ts +++ b/worker/src/temporal/activities/run-component.activity.ts @@ -154,6 +154,9 @@ export async function runComponentActivity( const context = createExecutionContext({ runId: input.runId, componentRef: action.ref, + workflowId: input.workflowId, + workflowName: input.workflowName, + organizationId: input.organizationId ?? null, metadata: { activityId: activityInfo.activityId, attempt: activityInfo.attempt, @@ -383,21 +386,53 @@ export async function runComponentActivity( await resolveSecretParams(resolvedParams, input.rawParams ?? {}); + // Get input port metadata to check which inputs are truly required + let inputsSchemaForValidation = component.inputs; + if (typeof component.resolvePorts === 'function') { + try { + const resolved = component.resolvePorts(resolvedParams); + if (resolved?.inputs) { + inputsSchemaForValidation = resolved.inputs; + } + } catch { + // If port resolution fails, use the base schema + } + } + const inputPorts = inputsSchemaForValidation ? extractPorts(inputsSchemaForValidation) : []; + + // Filter warnings to only those for truly required inputs + // An input is NOT required if: + // - Its schema allows undefined/null (required: false) + // - It accepts any type (connectionType.kind === 'any') which includes undefined + const requiredMissingInputs = warningsToReport.filter((warning) => { + const portMeta = inputPorts.find((p: ComponentPortMetadata) => p.id === warning.target); + // If we can't find the port metadata, assume it's required to be safe + if (!portMeta) return true; + // If marked as not required, it's optional + if (portMeta.required === false) return false; + // If connectionType is 'any', it accepts undefined + if (portMeta.connectionType?.kind === 'any') return false; + return true; + }); + + // Log warnings for all undefined inputs (even optional ones) for (const warning of warningsToReport) { + const isRequired = requiredMissingInputs.some((r) => r.target === warning.target); context.trace?.record({ type: 'NODE_PROGRESS', timestamp: new Date().toISOString(), message: `Input '${warning.target}' mapped from ${warning.sourceRef}.${warning.sourceHandle} was undefined`, - level: 'warn', + level: isRequired ? 'error' : 'warn', data: warning, }); } - if (warningsToReport.length > 0) { - const missing = warningsToReport.map((warning) => `'${warning.target}'`).join(', '); + // Only throw if there are truly missing required inputs + if (requiredMissingInputs.length > 0) { + const missing = requiredMissingInputs.map((warning) => `'${warning.target}'`).join(', '); throw new ValidationError(`Missing required inputs for ${action.ref}: ${missing}`, { fieldErrors: Object.fromEntries( - warningsToReport.map((w) => [ + requiredMissingInputs.map((w) => [ w.target, [`mapped from ${w.sourceRef}.${w.sourceHandle} was undefined`], ]), diff --git a/worker/src/temporal/types.ts b/worker/src/temporal/types.ts index 591ca991..e75daef3 100644 --- a/worker/src/temporal/types.ts +++ b/worker/src/temporal/types.ts @@ -74,6 +74,7 @@ export interface WorkflowDefinition { export interface RunComponentActivityInput { runId: string; workflowId: string; + workflowName?: string; workflowVersionId?: string | null; organizationId?: string | null; action: { diff --git a/worker/src/temporal/workflow-runner.ts b/worker/src/temporal/workflow-runner.ts index ab5cef3e..166e99fc 100644 --- a/worker/src/temporal/workflow-runner.ts +++ b/worker/src/temporal/workflow-runner.ts @@ -304,6 +304,9 @@ export async function executeWorkflow( artifacts: scopedArtifacts, trace: options.trace, logCollector: forwardLog, + workflowId: options.workflowId, + workflowName: definition.title, + organizationId: options.organizationId, }); try { diff --git a/worker/src/temporal/workflows/index.ts b/worker/src/temporal/workflows/index.ts index ad3e97f6..8b041b07 100644 --- a/worker/src/temporal/workflows/index.ts +++ b/worker/src/temporal/workflows/index.ts @@ -642,6 +642,7 @@ export async function shipsecWorkflowRun( const activityInput: RunComponentActivityInput = { runId: input.runId, workflowId: input.workflowId, + workflowName: input.definition.title, workflowVersionId: input.workflowVersionId ?? null, organizationId: input.organizationId ?? null, action: { diff --git a/worker/src/utils/opensearch-indexer.ts b/worker/src/utils/opensearch-indexer.ts new file mode 100644 index 00000000..ad033875 --- /dev/null +++ b/worker/src/utils/opensearch-indexer.ts @@ -0,0 +1,542 @@ +import { Client } from '@opensearch-project/opensearch'; +import type { IScopedTraceService } from '@shipsec/component-sdk'; + +interface IndexOptions { + workflowId: string; + workflowName: string; + runId: string; + nodeRef: string; + componentId: string; + assetKeyField?: string; + indexSuffix?: string; + trace?: IScopedTraceService; +} + +/** + * Retry helper with exponential backoff + * Attempts: 3, delays: 1s, 2s, 4s + */ +async function retryWithBackoff(operation: () => Promise, operationName: string): Promise { + const maxAttempts = 3; + const delays = [1000, 2000, 4000]; // milliseconds + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + try { + return await operation(); + } catch (error) { + const isLastAttempt = attempt === maxAttempts - 1; + + if (isLastAttempt) { + throw error; // Re-throw on last attempt + } + + const delay = delays[attempt]; + console.warn( + `[OpenSearchIndexer] ${operationName} failed (attempt ${attempt + 1}/${maxAttempts}), ` + + `retrying in ${delay}ms...`, + error, + ); + + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + + // This should never be reached, but TypeScript requires it + throw new Error(`${operationName} failed after ${maxAttempts} attempts`); +} + +// TTL for tenant provisioning cache (1 hour in milliseconds) +const TENANT_CACHE_TTL_MS = 60 * 60 * 1000; + +export class OpenSearchIndexer { + private client: Client | null = null; + private enabled = false; + private dashboardsUrl: string | null = null; + private dashboardsAuth: { username: string; password: string } | null = null; + private securityEnabled = false; + private backendUrl: string | null = null; + private internalServiceToken: string | null = null; + + // Cache of provisioned org IDs with timestamp + private provisionedOrgs = new Map(); + + constructor() { + const url = process.env.OPENSEARCH_URL; + const username = process.env.OPENSEARCH_USERNAME; + const password = process.env.OPENSEARCH_PASSWORD; + + // OpenSearch Dashboards URL for index pattern management + this.dashboardsUrl = process.env.OPENSEARCH_DASHBOARDS_URL || null; + if (username && password) { + this.dashboardsAuth = { username, password }; + } + + // Security mode configuration + this.securityEnabled = process.env.OPENSEARCH_SECURITY_ENABLED === 'true'; + this.backendUrl = process.env.BACKEND_URL || 'http://localhost:3211'; + this.internalServiceToken = process.env.INTERNAL_SERVICE_TOKEN || null; + + if (url) { + try { + this.client = new Client({ + node: url, + ...(username && + password && { + auth: { + username, + password, + }, + }), + ssl: { + rejectUnauthorized: process.env.NODE_ENV === 'production', + }, + }); + this.enabled = true; + console.log( + `[OpenSearchIndexer] Client initialized (security enabled: ${this.securityEnabled})`, + ); + } catch (error) { + console.warn('[OpenSearchIndexer] Failed to initialize client:', error); + } + } else { + console.debug('[OpenSearchIndexer] OpenSearch URL not configured, indexing disabled'); + } + } + + isEnabled(): boolean { + return this.enabled && this.client !== null; + } + + /** + * Ensure tenant is provisioned in OpenSearch Security. + * Caches provisioned orgs with 1-hour TTL to avoid redundant calls. + * On failure: logs error but returns true to allow indexing to continue. + */ + private async ensureTenantProvisioned(orgId: string): Promise { + // Skip if security is disabled + if (!this.securityEnabled) { + return true; + } + + // Check cache + const cachedTimestamp = this.provisionedOrgs.get(orgId); + if (cachedTimestamp && Date.now() - cachedTimestamp < TENANT_CACHE_TTL_MS) { + console.debug(`[OpenSearchIndexer] Tenant already provisioned (cached): ${orgId}`); + return true; + } + + // Call backend to provision tenant + try { + const url = `${this.backendUrl}/api/v1/analytics/ensure-tenant`; + const headers: Record = { + 'Content-Type': 'application/json', + }; + + if (this.internalServiceToken) { + headers['X-Internal-Token'] = this.internalServiceToken; + } + + const response = await fetch(url, { + method: 'POST', + headers, + body: JSON.stringify({ organizationId: orgId }), + }); + + if (!response.ok) { + console.error( + `[OpenSearchIndexer] Failed to provision tenant ${orgId}: ${response.status} ${response.statusText}`, + ); + // Continue with indexing anyway - tenant might already exist + return true; + } + + const result = (await response.json()) as { success: boolean; message: string }; + if (result.success) { + // Cache the successful provisioning + this.provisionedOrgs.set(orgId, Date.now()); + console.log(`[OpenSearchIndexer] Tenant provisioned: ${orgId}`); + } else { + console.warn(`[OpenSearchIndexer] Tenant provisioning returned failure: ${result.message}`); + } + + return true; + } catch (error) { + // Log error but don't block indexing + const message = error instanceof Error ? error.message : String(error); + console.error(`[OpenSearchIndexer] Error provisioning tenant ${orgId}: ${message}`); + return true; // Continue with indexing + } + } + + /** + * Serialize nested objects and arrays to JSON strings to prevent field explosion. + * Preserves primitive values (string, number, boolean, null) as-is. + */ + private serializeNestedFields(document: Record): Record { + // Pass through as-is - let OpenSearch handle dynamic mapping + return { ...document }; + } + + /** + * Build the enriched document structure with _shipsec context. + * - Component data fields at root level (nested objects serialized) + * - Workflow context under _shipsec namespace (prevents field collision) + */ + private buildEnrichedDocument( + document: Record, + options: IndexOptions, + orgId: string, + timestamp: string, + assetKey: string | null, + ): Record { + // Serialize nested objects in the document to prevent field explosion + const serializedDocument = this.serializeNestedFields(document); + + return { + // Component data at root level (serialized) + ...serializedDocument, + + // Workflow context under shipsec namespace (no underscore prefix for UI visibility) + shipsec: { + organization_id: orgId, + run_id: options.runId, + workflow_id: options.workflowId, + workflow_name: options.workflowName, + component_id: options.componentId, + node_ref: options.nodeRef, + ...(assetKey && { asset_key: assetKey }), + }, + + // Standard timestamp + '@timestamp': timestamp, + }; + } + + async indexDocument( + orgId: string, + document: Record, + options: IndexOptions, + ): Promise { + if (!this.isEnabled() || !this.client) { + console.debug('[OpenSearchIndexer] Indexing skipped, client not enabled'); + throw new Error('OpenSearch client not enabled'); + } + + const indexName = this.buildIndexName(orgId, options.indexSuffix); + const assetKey = this.detectAssetKey(document, options.assetKeyField); + const timestamp = new Date().toISOString(); + + const enrichedDocument = this.buildEnrichedDocument( + document, + options, + orgId, + timestamp, + assetKey, + ); + + try { + await retryWithBackoff(async () => { + await this.client!.index({ + index: indexName, + body: enrichedDocument, + }); + }, `Index document to ${indexName}`); + + console.debug(`[OpenSearchIndexer] Indexed document to ${indexName}`); + + // Log successful indexing to trace + if (options.trace) { + options.trace.record({ + type: 'NODE_PROGRESS', + level: 'info', + message: `Successfully indexed 1 document to ${indexName}`, + data: { + indexName, + documentCount: 1, + assetKey: assetKey ?? undefined, + }, + }); + } + + return indexName; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + console.error(`[OpenSearchIndexer] Failed to index document after retries:`, error); + + // Log indexing error to trace + if (options.trace) { + options.trace.record({ + type: 'NODE_PROGRESS', + level: 'error', + message: `Failed to index document to ${indexName}`, + error: errorMessage, + data: { + indexName, + documentCount: 1, + }, + }); + } + + throw error; + } + } + + async bulkIndex( + orgId: string, + documents: Record[], + options: IndexOptions, + ): Promise<{ indexName: string; documentCount: number }> { + if (!this.isEnabled() || !this.client) { + console.debug('[OpenSearchIndexer] Bulk indexing skipped, client not enabled'); + throw new Error('OpenSearch client not enabled'); + } + + if (documents.length === 0) { + console.debug('[OpenSearchIndexer] No documents to index'); + return { indexName: '', documentCount: 0 }; + } + + // Ensure tenant is provisioned before indexing (for multi-tenant security) + await this.ensureTenantProvisioned(orgId); + + const indexName = this.buildIndexName(orgId, options.indexSuffix); + + // Use same timestamp for all documents in this batch + // (they all came from the same component execution) + const timestamp = new Date().toISOString(); + + // Build bulk operations array + const bulkOps: any[] = []; + for (const document of documents) { + const assetKey = this.detectAssetKey(document, options.assetKeyField); + + const enrichedDocument = this.buildEnrichedDocument( + document, + options, + orgId, + timestamp, + assetKey, + ); + + bulkOps.push({ index: { _index: indexName } }); + bulkOps.push(enrichedDocument); + } + + try { + const response = await retryWithBackoff(async () => { + return await this.client!.bulk({ + body: bulkOps, + }); + }, `Bulk index ${documents.length} documents to ${indexName}`); + + if (response.body.errors) { + const failedItems = response.body.items.filter((item: any) => item.index?.error); + const errorCount = failedItems.length; + + // Log first 3 error details for debugging + const errorSamples = failedItems.slice(0, 3).map((item: any) => ({ + type: item.index?.error?.type, + reason: item.index?.error?.reason, + })); + + console.warn( + `[OpenSearchIndexer] Bulk indexing completed with ${errorCount} errors out of ${documents.length} documents`, + ); + console.warn(`[OpenSearchIndexer] Error samples:`, JSON.stringify(errorSamples, null, 2)); + + // Log partial failure to trace + if (options.trace) { + options.trace.record({ + type: 'NODE_PROGRESS', + level: 'warn', + message: `Bulk indexed with ${errorCount} errors out of ${documents.length} documents to ${indexName}`, + data: { + indexName, + documentCount: documents.length, + errorCount, + errorSamples, + }, + }); + } + } else { + console.debug( + `[OpenSearchIndexer] Bulk indexed ${documents.length} documents to ${indexName}`, + ); + + // Log successful bulk indexing to trace + if (options.trace) { + options.trace.record({ + type: 'NODE_PROGRESS', + level: 'info', + message: `Successfully bulk indexed ${documents.length} documents to ${indexName}`, + data: { + indexName, + documentCount: documents.length, + }, + }); + } + } + + // Refresh index pattern in OpenSearch Dashboards to make new fields visible + // Skip when security is enabled - patterns are created per-tenant by the provisioning service + if (!this.securityEnabled) { + await this.refreshIndexPattern(); + } + + return { indexName, documentCount: documents.length }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + console.error(`[OpenSearchIndexer] Failed to bulk index after retries:`, error); + + // Log bulk indexing error to trace + if (options.trace) { + options.trace.record({ + type: 'NODE_PROGRESS', + level: 'error', + message: `Failed to bulk index ${documents.length} documents to ${indexName}`, + error: errorMessage, + data: { + indexName, + documentCount: documents.length, + }, + }); + } + + throw error; + } + } + + /** + * Refresh the index pattern in OpenSearch Dashboards to make new fields visible. + * Two-step process: + * 1. Get fresh field mappings from OpenSearch via _fields_for_wildcard API + * 2. Update the saved index pattern object with the new fields + * Fails silently if Dashboards URL is not configured or refresh fails. + */ + private async refreshIndexPattern(): Promise { + if (!this.dashboardsUrl) { + console.debug( + '[OpenSearchIndexer] Dashboards URL not configured, skipping index pattern refresh', + ); + return; + } + + const indexPatternId = 'security-findings-*'; + + try { + const headers: Record = { + 'Content-Type': 'application/json', + 'osd-xsrf': 'true', // Required by OpenSearch Dashboards + }; + + // Add basic auth if credentials are available + if (this.dashboardsAuth) { + const authString = Buffer.from( + `${this.dashboardsAuth.username}:${this.dashboardsAuth.password}`, + ).toString('base64'); + headers['Authorization'] = `Basic ${authString}`; + } + + // Step 1: Get fresh fields from OpenSearch via Dashboards API + const fieldsUrl = `${this.dashboardsUrl}/api/index_patterns/_fields_for_wildcard?pattern=${encodeURIComponent(indexPatternId)}&meta_fields=_source&meta_fields=_id&meta_fields=_type&meta_fields=_index&meta_fields=_score`; + const fieldsResponse = await fetch(fieldsUrl, { method: 'GET', headers }); + + if (!fieldsResponse.ok) { + console.warn(`[OpenSearchIndexer] Failed to get fresh fields: ${fieldsResponse.status}`); + return; + } + + const fieldsData = (await fieldsResponse.json()) as { fields?: unknown[] }; + const freshFields = fieldsData.fields || []; + + // Step 2: Get current index pattern to preserve other attributes + const patternUrl = `${this.dashboardsUrl}/api/saved_objects/index-pattern/${encodeURIComponent(indexPatternId)}`; + const patternResponse = await fetch(patternUrl, { method: 'GET', headers }); + + if (!patternResponse.ok) { + console.warn(`[OpenSearchIndexer] Index pattern not found: ${patternResponse.status}`); + return; + } + + const patternData = (await patternResponse.json()) as { + attributes: { title: string; timeFieldName: string }; + version: string; + }; + + // Step 3: Update the index pattern with fresh fields + // Include version for optimistic concurrency control (matches UI behavior) + const updateResponse = await fetch(patternUrl, { + method: 'PUT', + headers, + body: JSON.stringify({ + attributes: { + title: patternData.attributes.title, + timeFieldName: patternData.attributes.timeFieldName, + fields: JSON.stringify(freshFields), + }, + version: patternData.version, + }), + }); + + if (updateResponse.ok) { + console.debug( + `[OpenSearchIndexer] Index pattern fields refreshed (${freshFields.length} fields)`, + ); + } else { + console.warn( + `[OpenSearchIndexer] Failed to update index pattern: ${updateResponse.status}`, + ); + } + } catch (error) { + // Non-critical failure - log but don't throw + console.warn('[OpenSearchIndexer] Failed to refresh index pattern:', error); + } + } + + private buildIndexName(orgId: string, indexSuffix?: string): string { + const date = new Date(); + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, '0'); + const day = String(date.getDate()).padStart(2, '0'); + + const suffix = indexSuffix || `${year}.${month}.${day}`; + return `security-findings-${orgId}-${suffix}`.toLowerCase(); + } + + private detectAssetKey(document: Record, explicitField?: string): string | null { + // If explicit field is provided, use it + if (explicitField && document[explicitField]) { + return String(document[explicitField]); + } + + // Auto-detect from common fields + const assetFields = [ + 'asset_key', + 'host', + 'domain', + 'subdomain', + 'url', + 'ip', + 'asset', + 'target', + ]; + + for (const field of assetFields) { + if (document[field]) { + return String(document[field]); + } + } + + return null; + } +} + +// Singleton instance +let indexerInstance: OpenSearchIndexer | null = null; + +export function getOpenSearchIndexer(): OpenSearchIndexer { + if (!indexerInstance) { + indexerInstance = new OpenSearchIndexer(); + } + return indexerInstance; +}