diff --git a/docs/components/security.mdx b/docs/components/security.mdx index acf28a06..38c39655 100644 --- a/docs/components/security.mdx +++ b/docs/components/security.mdx @@ -12,7 +12,7 @@ Security components wrap popular open-source tools for subdomain discovery, DNS ### Subfinder -[GitHub](https://github.com/projectdiscovery/subfinder) · Docker: `projectdiscovery/subfinder` +[GitHub](https://github.com/projectdiscovery/subfinder) · Docker: `ghcr.io/shipsecai/subfinder` Discovers subdomains using passive sources. @@ -35,7 +35,7 @@ Discovers subdomains using passive sources. ### Amass -[GitHub](https://github.com/owasp-amass/amass) · Docker: `owaspamass/amass` +[GitHub](https://github.com/owasp-amass/amass) · Docker: `ghcr.io/shipsecai/amass` Active and passive subdomain enumeration. @@ -73,7 +73,7 @@ High-performance DNS bruteforcing and resolution. This is a combined image that ### DNSX -[GitHub](https://github.com/projectdiscovery/dnsx) · Docker: `projectdiscovery/dnsx` +[GitHub](https://github.com/projectdiscovery/dnsx) · Docker: `ghcr.io/shipsecai/dnsx` Resolves DNS records with support for multiple record types and custom resolvers. @@ -105,7 +105,7 @@ Resolves DNS records with support for multiple record types and custom resolvers ### httpx -[GitHub](https://github.com/projectdiscovery/httpx) · Docker: `projectdiscovery/httpx` +[GitHub](https://github.com/projectdiscovery/httpx) · Docker: `ghcr.io/shipsecai/httpx` Probes hosts for live HTTP services and captures response metadata. @@ -137,7 +137,7 @@ Probes hosts for live HTTP services and captures response metadata. ### Naabu -[GitHub](https://github.com/projectdiscovery/naabu) · Docker: `projectdiscovery/naabu` +[GitHub](https://github.com/projectdiscovery/naabu) · Docker: `ghcr.io/shipsecai/naabu` Fast active port scanning using SYN/CONNECT probes. @@ -196,7 +196,7 @@ Template-based vulnerability scanning. This is nuclei custom image with nuclei-t ### TruffleHog -[GitHub](https://github.com/trufflesecurity/trufflehog) · Docker: `trufflesecurity/trufflehog` +[GitHub](https://github.com/trufflesecurity/trufflehog) · Docker: `ghcr.io/shipsecai/trufflehog` Scans for leaked credentials across repositories, filesystems, and cloud storage. @@ -227,7 +227,7 @@ Scans for leaked credentials across repositories, filesystems, and cloud storage ### Prowler Scan -[GitHub](https://github.com/prowler-cloud/prowler) · Docker: `prowlercloud/prowler` +[GitHub](https://github.com/prowler-cloud/prowler) · Docker: `ghcr.io/shipsecai/prowler` Cloud (AWS, Azure, GCP) security posture management. Best practices auditing. @@ -262,7 +262,7 @@ Scans Supabase instances for misconfigurations. ### Notify -[GitHub](https://github.com/projectdiscovery/notify) · Docker: `projectdiscovery/notify` +[GitHub](https://github.com/projectdiscovery/notify) · Docker: `ghcr.io/shipsecai/notify` Sends alerts to Slack, Discord, Telegram, or email. diff --git a/docs/development/component-development.mdx b/docs/development/component-development.mdx index 853151f1..6fc85988 100644 --- a/docs/development/component-development.mdx +++ b/docs/development/component-development.mdx @@ -733,7 +733,7 @@ export default defineComponent({ category: 'security', runner: { kind: 'docker', - image: 'projectdiscovery/dnsx:latest', + image: 'ghcr.io/shipsecai/dnsx:latest', entrypoint: 'sh', command: ['-c', 'dnsx "$@"', '--'], network: 'bridge', diff --git a/scratch/opencode-mcp-test/run_test.sh b/scratch/opencode-mcp-test/run_test.sh index 4809bf67..ecd4f040 100755 --- a/scratch/opencode-mcp-test/run_test.sh +++ b/scratch/opencode-mcp-test/run_test.sh @@ -21,7 +21,7 @@ docker run --rm \ --network host \ -v "$DIR:/workspace" \ -e OPENROUTER_API_KEY="$OPENROUTER_API_KEY" \ - ghcr.io/anomalyco/opencode \ + ghcr.io/shipsecai/opencode:1.1.53 \ run --log-level INFO "$(cat prompt.txt)" # Kill MCP server diff --git a/worker/README.md b/worker/README.md index 972ce137..f38aa010 100644 --- a/worker/README.md +++ b/worker/README.md @@ -3,11 +3,13 @@ Node.js component execution engine with Temporal.io integration for running security workflows in isolated environments. ## Prerequisites + - Bun latest (see root `README.md` for install instructions) - Infrastructure services running (`just dev` from repo root) - Docker for containerized component execution ## Development Commands + ```bash # Install workspace dependencies (run once from repo root) bun install @@ -26,6 +28,7 @@ bun run test ## Architecture Overview ### Core Technologies + - **Node.js** with TypeScript for component execution - **Temporal.io** for workflow orchestration and activities - **Docker** for isolated component execution @@ -56,26 +59,31 @@ const runComponentActivity = async (componentId, input, context) => { The worker provides concrete implementations of SDK interfaces: #### File Storage Adapter + - **MinIO Integration**: S3-compatible object storage - **PostgreSQL Metadata**: File metadata and organization - **Artifact Management**: Component outputs and execution results #### Secrets Adapter + - **HashiCorp Vault**: Enterprise-grade secret management - **AES-256 Encryption**: Secure secret storage - **Version Control**: Multiple secret versions with rollback #### Trace Adapter + - **Event Streaming**: Kafka-based event publishing - **Redis Transport**: Real-time event delivery - **Timeline Generation**: Sequential event numbering #### Logging Adapters + - **Kafka Log Transport**: Structured log streaming - **Loki Integration**: Log aggregation and querying - **PostgreSQL Persistence**: Log metadata and indexing #### Terminal Adapter + - **Redis Streams**: Real-time terminal output streaming - **Base64 Encoding**: Efficient binary data transport - **Monotonic Timestamps**: Precise chronological ordering @@ -85,12 +93,14 @@ The worker provides concrete implementations of SDK interfaces: ### Component Categories #### Core Components + - **file-loader**: File upload and content extraction - **trigger-manual**: Manual workflow execution trigger - **text-block**: Markdown documentation and notes - **text-joiner**: Text concatenation and formatting #### Security Components + - **subfinder**: Subdomain discovery - **dnsx**: DNS resolution and enumeration - **nmap**: Network scanning and discovery @@ -106,7 +116,7 @@ const definition: ComponentDefinition = { id: 'security.subfinder', label: 'Subfinder', category: 'discovery', - runner: { kind: 'docker', image: 'projectdiscovery/subfinder' }, + runner: { kind: 'docker', image: 'ghcr.io/shipsecai/subfinder' }, inputSchema: z.object({ domain: z.string(), timeout: z.number().default(30), @@ -122,17 +132,20 @@ componentRegistry.register(definition); ## Temporal Integration ### Activities + - **runComponentActivity**: Execute individual components - **setRunMetadataActivity**: Store workflow execution metadata - **finalizeRunActivity**: Complete workflow and cleanup resources ### Workflows + - **Workflow Orchestration**: Topological sorting and dependency resolution - **Join Strategies**: Handle multiple parent dependencies (all, any, first) - **Error Handling**: Retry policies and graceful degradation - **Heartbeating**: Long-running activity support ### Worker Configuration + ```typescript const worker = await Worker.create({ connection, @@ -148,6 +161,7 @@ const worker = await Worker.create({ ``` ## Project Structure + ``` src/ ├── components/ # Component implementations @@ -169,12 +183,14 @@ src/ ## Container Execution ### Docker Integration + - **Isolated Execution**: Components run in isolated Docker containers - **Resource Limits**: CPU and memory constraints enforced - **Network Isolation**: Bridge network configuration - **Volume Management**: Isolated storage volumes for file operations ### Terminal Capture + - **PTY Allocation**: Pseudo-terminal for interactive tools - **Stream Multiplexing**: stdout, stderr, and PTY stream capture - **Chunk Encoding**: Base64 encoding for efficient transport @@ -189,6 +205,7 @@ src/ 5. **Validation**: Check types with `bun run typecheck` ## Where To Read More + - **[Architecture Overview](../docs/architecture.md)** - Complete system design and component execution - **[Component Development](../docs/component-development.md)** - Building security components -- **[Component SDK](../packages/component-sdk)** - Framework-agnostic component interfaces \ No newline at end of file +- **[Component SDK](../packages/component-sdk)** - Framework-agnostic component interfaces diff --git a/worker/src/components/ai/__tests__/opencode.test.ts b/worker/src/components/ai/__tests__/opencode.test.ts index 7c410f83..3fb16d85 100644 --- a/worker/src/components/ai/__tests__/opencode.test.ts +++ b/worker/src/components/ai/__tests__/opencode.test.ts @@ -93,7 +93,7 @@ describe('shipsec.opencode.agent', () => { expect(runSpy).toHaveBeenCalled(); const runnerCall = runSpy.mock.calls[0][0]; - expect(runnerCall.image).toBe('ghcr.io/anomalyco/opencode'); + expect(runnerCall.image).toBe('ghcr.io/shipsecai/opencode:1.1.53'); expect(runnerCall.network).toBe('host'); expect(runnerCall.env.OPENAI_API_KEY).toBe('sk-test'); }); diff --git a/worker/src/components/ai/opencode.ts b/worker/src/components/ai/opencode.ts index c9f41967..d23b56aa 100644 --- a/worker/src/components/ai/opencode.ts +++ b/worker/src/components/ai/opencode.ts @@ -99,7 +99,7 @@ const definition = defineComponent({ category: 'ai', runner: { kind: 'docker', - image: 'ghcr.io/anomalyco/opencode', + image: 'ghcr.io/shipsecai/opencode:1.1.53', entrypoint: 'opencode', // We will override this in execution network: 'host' as const, // Required to access localhost gateway command: ['help'], diff --git a/worker/src/components/security/__tests__/amass.test.ts b/worker/src/components/security/__tests__/amass.test.ts index afc058e5..ac070fd9 100644 --- a/worker/src/components/security/__tests__/amass.test.ts +++ b/worker/src/components/security/__tests__/amass.test.ts @@ -107,13 +107,13 @@ describe.skip('amass component', () => { expect(result.rawOutput).toBe(rawOutput); }); - it('should configure docker runner for owaspamass/amass image', () => { + it('should configure docker runner for ghcr.io/shipsecai/amass image', () => { const component = componentRegistry.get('shipsec.amass.enum'); if (!component) throw new Error('Component not registered'); expect(component.runner.kind).toBe('docker'); if (component.runner.kind === 'docker') { - expect(component.runner.image).toBe('owaspamass/amass:v5.0.1'); + expect(component.runner.image).toBe('ghcr.io/shipsecai/amass:v5.0.1'); expect(component.runner.entrypoint).toBe('sh'); expect(component.runner.command).toBeInstanceOf(Array); } diff --git a/worker/src/components/security/__tests__/dnsx.test.ts b/worker/src/components/security/__tests__/dnsx.test.ts index 5fad599c..7cf0f091 100644 --- a/worker/src/components/security/__tests__/dnsx.test.ts +++ b/worker/src/components/security/__tests__/dnsx.test.ts @@ -195,7 +195,7 @@ describe.skip('dnsx component', () => { expect(component.runner.kind).toBe('docker'); if (component.runner.kind === 'docker') { - expect(component.runner.image).toBe('projectdiscovery/dnsx:v1.2.2'); + expect(component.runner.image).toBe('ghcr.io/shipsecai/dnsx:v1.2.2'); expect(component.runner.entrypoint).toBe('sh'); } }); diff --git a/worker/src/components/security/__tests__/naabu.test.ts b/worker/src/components/security/__tests__/naabu.test.ts index 52813e3f..9f300910 100644 --- a/worker/src/components/security/__tests__/naabu.test.ts +++ b/worker/src/components/security/__tests__/naabu.test.ts @@ -105,7 +105,7 @@ describe('naabu component', () => { expect(component.runner.kind).toBe('docker'); if (component.runner.kind === 'docker') { - expect(component.runner.image).toBe('projectdiscovery/naabu:v2.3.7'); + expect(component.runner.image).toBe('ghcr.io/shipsecai/naabu:v2.3.7'); expect(component.runner.entrypoint).toBe('sh'); expect(component.runner.command).toBeInstanceOf(Array); } diff --git a/worker/src/components/security/__tests__/notify-registration.test.ts b/worker/src/components/security/__tests__/notify-registration.test.ts index 9eafdaf9..83604fc8 100644 --- a/worker/src/components/security/__tests__/notify-registration.test.ts +++ b/worker/src/components/security/__tests__/notify-registration.test.ts @@ -14,7 +14,7 @@ describeNotify('Notify component registration', () => { expect(component!.ui?.slug).toBe('notify'); if (component!.runner.kind === 'docker') { - expect(component!.runner.image).toContain('projectdiscovery/notify'); + expect(component!.runner.image).toContain('ghcr.io/shipsecai/notify'); expect(component!.runner.entrypoint).toBe('sh'); } else { throw new Error('Expected docker runner for notify component'); diff --git a/worker/src/components/security/__tests__/subfinder.test.ts b/worker/src/components/security/__tests__/subfinder.test.ts index 4e590dee..b64c1e27 100644 --- a/worker/src/components/security/__tests__/subfinder.test.ts +++ b/worker/src/components/security/__tests__/subfinder.test.ts @@ -160,7 +160,7 @@ describe.skip('subfinder component', () => { expect(component.runner.kind).toBe('docker'); if (component.runner.kind === 'docker') { - expect(component.runner.image).toBe('projectdiscovery/subfinder:v2.12.0'); + expect(component.runner.image).toBe('ghcr.io/shipsecai/subfinder:v2.12.0'); } }); }); diff --git a/worker/src/components/security/__tests__/trufflehog.test.ts b/worker/src/components/security/__tests__/trufflehog.test.ts index d3c4e13a..6fa065d5 100644 --- a/worker/src/components/security/__tests__/trufflehog.test.ts +++ b/worker/src/components/security/__tests__/trufflehog.test.ts @@ -29,7 +29,7 @@ describe('trufflehog component', () => { expect(component.runner.kind).toBe('docker'); if (component.runner.kind === 'docker') { - expect(component.runner.image).toBe('trufflesecurity/trufflehog:v3.92.1'); + expect(component.runner.image).toBe('ghcr.io/shipsecai/trufflehog:v3.93.1'); } }); diff --git a/worker/src/components/security/amass.ts b/worker/src/components/security/amass.ts index 16386347..f964064a 100644 --- a/worker/src/components/security/amass.ts +++ b/worker/src/components/security/amass.ts @@ -14,7 +14,7 @@ import { } from '@shipsec/component-sdk'; import { IsolatedContainerVolume } from '../../utils/isolated-volume'; -const AMASS_IMAGE = 'owaspamass/amass:v5.0.1'; +const AMASS_IMAGE = 'ghcr.io/shipsecai/amass:v5.0.1'; const AMASS_TIMEOUT_SECONDS = (() => { const raw = process.env.AMASS_TIMEOUT_SECONDS; const parsed = raw ? Number.parseInt(raw, 10) : NaN; diff --git a/worker/src/components/security/dnsx.ts b/worker/src/components/security/dnsx.ts index 2dcf2f12..aa3cd203 100644 --- a/worker/src/components/security/dnsx.ts +++ b/worker/src/components/security/dnsx.ts @@ -32,7 +32,7 @@ const recordTypeEnum = z.enum([ const outputModeEnum = z.enum(['silent', 'json']); -const DNSX_IMAGE = 'projectdiscovery/dnsx:v1.2.2'; +const DNSX_IMAGE = 'ghcr.io/shipsecai/dnsx:v1.2.2'; const DNSX_TIMEOUT_SECONDS = 180; const INPUT_MOUNT_NAME = 'inputs'; const CONTAINER_INPUT_DIR = `/${INPUT_MOUNT_NAME}`; diff --git a/worker/src/components/security/httpx.ts b/worker/src/components/security/httpx.ts index 84198ea5..b1f4812a 100644 --- a/worker/src/components/security/httpx.ts +++ b/worker/src/components/security/httpx.ts @@ -202,7 +202,7 @@ const definition = defineComponent({ category: 'security', runner: { kind: 'docker', - image: 'projectdiscovery/httpx:v1.7.4', + image: 'ghcr.io/shipsecai/httpx:v1.7.4', entrypoint: 'httpx', network: 'bridge', timeoutSeconds: dockerTimeoutSeconds, diff --git a/worker/src/components/security/naabu.ts b/worker/src/components/security/naabu.ts index 052aa16a..20b18d6c 100644 --- a/worker/src/components/security/naabu.ts +++ b/worker/src/components/security/naabu.ts @@ -179,7 +179,7 @@ const definition = defineComponent({ category: 'security', runner: { kind: 'docker', - image: 'projectdiscovery/naabu:v2.3.7', + image: 'ghcr.io/shipsecai/naabu:v2.3.7', entrypoint: 'sh', network: 'bridge', timeoutSeconds: dockerTimeoutSeconds, diff --git a/worker/src/components/security/notify.ts b/worker/src/components/security/notify.ts index 6dcd643a..917c3d96 100644 --- a/worker/src/components/security/notify.ts +++ b/worker/src/components/security/notify.ts @@ -213,7 +213,7 @@ const definition = defineComponent({ category: 'security', runner: { kind: 'docker', - image: 'projectdiscovery/notify:v1.0.7', + image: 'ghcr.io/shipsecai/notify:v1.0.7', entrypoint: 'sh', network: 'bridge', timeoutSeconds: dockerTimeoutSeconds, diff --git a/worker/src/components/security/prowler-scan.ts b/worker/src/components/security/prowler-scan.ts index bc299fd1..fac802d5 100644 --- a/worker/src/components/security/prowler-scan.ts +++ b/worker/src/components/security/prowler-scan.ts @@ -399,14 +399,14 @@ const definition = defineComponent({ retryPolicy: prowlerRetryPolicy, runner: { kind: 'docker', - image: 'prowlercloud/prowler:5.14.2', + image: 'ghcr.io/shipsecai/prowler:5.14.2', platform: 'linux/amd64', command: [], // Placeholder - actual command built dynamically in execute() }, inputs: inputSchema, outputs: outputSchema, parameters: parameterSchema, - docs: 'Execute Prowler inside Docker using `prowlercloud/prowler` (amd64 enforced on ARM hosts). Supports AWS account scans and the multi-cloud `prowler cloud` overview, with optional CLI flag customisation.', + docs: 'Execute Prowler inside Docker using `ghcr.io/shipsecai/prowler` (amd64 enforced on ARM hosts). Supports AWS account scans and the multi-cloud `prowler cloud` overview, with optional CLI flag customisation.', ui: { slug: 'prowler-scan', version: '2.0.0', @@ -566,7 +566,7 @@ const definition = defineComponent({ // Prepare a one-off runner with dynamic command and volume const dockerRunner: DockerRunnerConfig = { kind: 'docker', - image: 'prowlercloud/prowler:5.14.2', + image: 'ghcr.io/shipsecai/prowler:5.14.2', platform: 'linux/amd64', network: 'bridge', timeoutSeconds: 900, diff --git a/worker/src/components/security/subfinder.ts b/worker/src/components/security/subfinder.ts index 0dce2f4a..800b5d4e 100644 --- a/worker/src/components/security/subfinder.ts +++ b/worker/src/components/security/subfinder.ts @@ -14,7 +14,7 @@ import { } from '@shipsec/component-sdk'; import { IsolatedContainerVolume } from '../../utils/isolated-volume'; -const SUBFINDER_IMAGE = 'projectdiscovery/subfinder:v2.12.0'; +const SUBFINDER_IMAGE = 'ghcr.io/shipsecai/subfinder:v2.12.0'; const SUBFINDER_TIMEOUT_SECONDS = 1800; // 30 minutes const INPUT_MOUNT_NAME = 'inputs'; const CONTAINER_INPUT_DIR = `/${INPUT_MOUNT_NAME}`; @@ -280,7 +280,7 @@ const definition = defineComponent({ inputs: inputSchema, outputs: outputSchema, parameters: parameterSchema, - docs: 'Runs projectdiscovery/subfinder to discover subdomains for a given domain. Optionally accepts a provider config secret to enable authenticated sources.', + docs: 'Runs subfinder to discover subdomains for a given domain. Optionally accepts a provider config secret to enable authenticated sources.', ui: { slug: 'subfinder', version: '1.0.0', diff --git a/worker/src/components/security/trufflehog.ts b/worker/src/components/security/trufflehog.ts index 3ed88eef..197c9ba2 100644 --- a/worker/src/components/security/trufflehog.ts +++ b/worker/src/components/security/trufflehog.ts @@ -303,7 +303,7 @@ const definition = defineComponent({ category: 'security', runner: { kind: 'docker', - image: 'trufflesecurity/trufflehog:v3.92.1', + image: 'ghcr.io/shipsecai/trufflehog:v3.93.1', entrypoint: 'trufflehog', network: 'bridge', command: [], // Will be built dynamically in execute