diff --git a/.changeset/cap-controller-resources.md b/.changeset/cap-controller-resources.md new file mode 100644 index 00000000..32c81c70 --- /dev/null +++ b/.changeset/cap-controller-resources.md @@ -0,0 +1,14 @@ +--- +"@prover-coder-ai/docker-git": patch +--- + +feat: cap controller container CPU, memory, and PID consumption + +Adds default `cpus`, `mem_limit`, `memswap_limit`, and `pids_limit` to the +`docker-git-api` controller in `docker-compose.yml` and +`docker-compose.api.yml`. Each value is parameterized so operators can +override it via `DOCKER_GIT_CONTROLLER_CPUS`, `DOCKER_GIT_CONTROLLER_MEMORY`, +and `DOCKER_GIT_CONTROLLER_PIDS`, or via `--controller-cpu`, +`--controller-ram`, and `--controller-pids` on the host CLI. Defaults resolve +to 90% CPU, 90% RAM/swap, and 4096 PIDs. This complements the existing +per-project caps so a runaway controller cannot consume the entire host. diff --git a/README.md b/README.md index 44c37fe3..d4ec5b36 100644 --- a/README.md +++ b/README.md @@ -113,3 +113,30 @@ When the CLI cannot acquire Docker access it now prints a message that names the specific failure mode, restates the host-Docker contract, and lists remediation steps for that exact mode. Implementation lives in `packages/app/src/docker-git/controller-docker-diagnostics.ts`. + +## Resource limits + +`docker-git` caps host resource consumption at two layers so a runaway +project (or the controller itself) cannot consume the entire system. + +- **Per-project containers** ship with a default limit of `30%` CPU and + `30%` RAM (resolved against the host on `apply`). Override via + `--cpu` / `--ram` (or per-project `docker-git.json`). +- **Controller container** (`docker-git-api`) is capped in + `docker-compose.yml` and `docker-compose.api.yml`. When started through + `docker-git` or `./ctl`, the default CPU/RAM cap is resolved to `90%` of + host resources. Override with global CLI flags: + + ```bash + docker-git --controller-cpu 75% --controller-ram 8g --controller-pids 8192 ps + ./ctl up --cpu 75% --ram 8g --pids 8192 + ``` + + The same values can be provided through env vars before `docker-git` or + `./ctl up`: + + | Variable | Default | Purpose | + | ------------------------------ | ------- | ------------------------------------ | + | `DOCKER_GIT_CONTROLLER_CPUS` | `90%` | CPU percent or cores for the controller | + | `DOCKER_GIT_CONTROLLER_MEMORY` | `90%` | RAM percent or size; swap is matched | + | `DOCKER_GIT_CONTROLLER_PIDS` | `4096` | Maximum PIDs inside the controller | diff --git a/ctl b/ctl index da9c909e..475e3d05 100755 --- a/ctl +++ b/ctl @@ -18,10 +18,14 @@ API_PORT="${DOCKER_GIT_API_PORT:-3334}" API_HOST="${DOCKER_GIT_API_BIND_HOST:-127.0.0.1}" API_BASE_URL="http://127.0.0.1:${API_PORT}" DOCKER_CMD=() +PARSED_ARGS=() +CONTROLLER_CPU_LIMIT="${DOCKER_GIT_CONTROLLER_CPUS:-}" +CONTROLLER_RAM_LIMIT="${DOCKER_GIT_CONTROLLER_MEMORY:-}" +CONTROLLER_PIDS_LIMIT="${DOCKER_GIT_CONTROLLER_PIDS:-}" usage() { cat <<'USAGE' -Usage: ./ctl +Usage: ./ctl [controller options] Controller: up Build and start the API controller @@ -41,6 +45,11 @@ API: ./ctl request POST /projects '{"repoUrl":"https://github.com/org/repo.git"}' ./ctl request POST /projects//up +Controller options: + --cpu, --cpus, --controller-cpu CPU cap intent, percent or cores (default: 90%) + --ram, --memory, --controller-ram RAM cap intent, percent or size (default: 90%) + --pids, --controller-pids PID cap (default: 4096) + USAGE } @@ -62,6 +71,76 @@ prepare_controller_revision() { export DOCKER_GIT_CONTROLLER_REV="$revision" } +parse_controller_limit_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --cpu|--cpus|--controller-cpu|--controller-cpus) + if [[ $# -lt 2 ]]; then + echo "Missing value for option: $1" >&2 + exit 1 + fi + CONTROLLER_CPU_LIMIT="$2" + shift 2 + ;; + --cpu=*|--cpus=*|--controller-cpu=*|--controller-cpus=*) + CONTROLLER_CPU_LIMIT="${1#*=}" + shift + ;; + --ram|--memory|--controller-ram|--controller-memory) + if [[ $# -lt 2 ]]; then + echo "Missing value for option: $1" >&2 + exit 1 + fi + CONTROLLER_RAM_LIMIT="$2" + shift 2 + ;; + --ram=*|--memory=*|--controller-ram=*|--controller-memory=*) + CONTROLLER_RAM_LIMIT="${1#*=}" + shift + ;; + --pids|--controller-pids) + if [[ $# -lt 2 ]]; then + echo "Missing value for option: $1" >&2 + exit 1 + fi + CONTROLLER_PIDS_LIMIT="$2" + shift 2 + ;; + --pids=*|--controller-pids=*) + CONTROLLER_PIDS_LIMIT="${1#*=}" + shift + ;; + *) + PARSED_ARGS+=("$1") + shift + ;; + esac + done +} + +prepare_controller_resource_limits() { + local env_output + env_output="$( + DOCKER_GIT_CONTROLLER_CPUS="$CONTROLLER_CPU_LIMIT" \ + DOCKER_GIT_CONTROLLER_MEMORY="$CONTROLLER_RAM_LIMIT" \ + DOCKER_GIT_CONTROLLER_PIDS="$CONTROLLER_PIDS_LIMIT" \ + bun --cwd "$ROOT/packages/app" scripts/print-controller-resource-env.ts + )" + + local key + local value + while IFS='=' read -r key value; do + if [[ -z "$key" ]]; then + continue + fi + case "$key" in + DOCKER_GIT_CONTROLLER_CPUS|DOCKER_GIT_CONTROLLER_MEMORY|DOCKER_GIT_CONTROLLER_PIDS) + export "$key=$value" + ;; + esac + done <<< "$env_output" +} + require_running() { if ! "${DOCKER_CMD[@]}" ps --format '{{.Names}}' | grep -Fxq "$CONTAINER_NAME"; then echo "Controller is not running. Start it with: ./ctl up" >&2 @@ -192,8 +271,12 @@ resolve_docker_cmd() { resolve_docker_cmd +parse_controller_limit_args "$@" +set -- "${PARSED_ARGS[@]}" + case "${1:-}" in up) + prepare_controller_resource_limits prepare_controller_revision compose up -d --build wait_for_health @@ -209,6 +292,7 @@ case "${1:-}" in compose logs -f --tail=200 ;; restart) + prepare_controller_resource_limits prepare_controller_revision compose up -d --build --force-recreate wait_for_health diff --git a/docker-compose.api.yml b/docker-compose.api.yml index bee3eb1b..f078cf92 100644 --- a/docker-compose.api.yml +++ b/docker-compose.api.yml @@ -37,6 +37,10 @@ services: cgroup: host init: true restart: unless-stopped + cpus: ${DOCKER_GIT_CONTROLLER_CPUS:-0.9} + mem_limit: ${DOCKER_GIT_CONTROLLER_MEMORY:-921m} + memswap_limit: ${DOCKER_GIT_CONTROLLER_MEMORY:-921m} + pids_limit: ${DOCKER_GIT_CONTROLLER_PIDS:-4096} volumes: docker_git_projects: diff --git a/docker-compose.yml b/docker-compose.yml index e955aff2..876236db 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,6 +39,10 @@ services: cgroup: host init: true restart: unless-stopped + cpus: ${DOCKER_GIT_CONTROLLER_CPUS:-0.9} + mem_limit: ${DOCKER_GIT_CONTROLLER_MEMORY:-921m} + memswap_limit: ${DOCKER_GIT_CONTROLLER_MEMORY:-921m} + pids_limit: ${DOCKER_GIT_CONTROLLER_PIDS:-4096} volumes: docker_git_projects: diff --git a/packages/app/scripts/print-controller-resource-env.ts b/packages/app/scripts/print-controller-resource-env.ts new file mode 100644 index 00000000..8f45cf96 --- /dev/null +++ b/packages/app/scripts/print-controller-resource-env.ts @@ -0,0 +1,67 @@ +import { NodeRuntime } from "@effect/platform-node" +import { Effect, Either } from "effect" + +import { + controllerCpuLimitEnvKey, + controllerMemoryLimitEnvKey, + controllerPidsLimitEnvKey, + resolveControllerResourceLimitEnv +} from "../src/docker-git/controller-resource-limits.js" +import { formatParseError } from "../src/docker-git/frontend-lib/core/parse-errors.js" + +const fallbackControllerHostResources = { + cpuCount: 1, + totalMemoryBytes: 1024 ** 3 +} + +const loadControllerHostResources = Effect.tryPromise({ + try: () => import("node:os"), + catch: (error) => new Error(String(error)) +}).pipe( + Effect.map((os) => ({ + cpuCount: os.availableParallelism(), + totalMemoryBytes: os.totalmem() + })), + Effect.match({ + onFailure: () => fallbackControllerHostResources, + onSuccess: (value) => value + }) +) + +const renderEnv = ( + env: { + readonly cpus: string + readonly memory: string + readonly pids: string + } +): string => + [ + `${controllerCpuLimitEnvKey}=${env.cpus}`, + `${controllerMemoryLimitEnvKey}=${env.memory}`, + `${controllerPidsLimitEnvKey}=${env.pids}` + ].join("\n") + +const program = Effect.gen(function*(_) { + const hostResources = yield* _(loadControllerHostResources) + const resolved = resolveControllerResourceLimitEnv( + { + cpuLimit: process.env[controllerCpuLimitEnvKey], + ramLimit: process.env[controllerMemoryLimitEnvKey], + pidsLimit: process.env[controllerPidsLimitEnvKey] + }, + hostResources + ) + + if (Either.isLeft(resolved)) { + return yield* _(Effect.fail(new Error(formatParseError(resolved.left)))) + } + + yield* _( + Effect.sync(() => { + process.stdout.write(renderEnv(resolved.right)) + process.stdout.write("\n") + }) + ) +}) + +NodeRuntime.runMain(program) diff --git a/packages/app/src/docker-git/cli/read-command.ts b/packages/app/src/docker-git/cli/read-command.ts index fcc0fdf7..fc801e72 100644 --- a/packages/app/src/docker-git/cli/read-command.ts +++ b/packages/app/src/docker-git/cli/read-command.ts @@ -1,9 +1,39 @@ import { Effect, Either, pipe } from "effect" +import { + controllerCpuLimitEnvKey, + controllerMemoryLimitEnvKey, + controllerPidsLimitEnvKey, + type ControllerResourceLimitArgParse, + controllerResourceLimitEnvAssignments, + controllerResourceLimitsForceRecreateEnvKey, + shouldForceRecreateForControllerResourceLimitIntent, + stripControllerResourceLimitArgs +} from "../controller-resource-limits.js" import { type Command, type ParseError } from "../frontend-lib/core/domain.js" import { parseArgs } from "./parser.js" +const applyControllerResourceLimitEnv = ( + parsed: ControllerResourceLimitArgParse +): Effect.Effect> => + Effect.sync(() => { + const assignments = controllerResourceLimitEnvAssignments(parsed.controllerResourceLimits) + const forceRecreate = shouldForceRecreateForControllerResourceLimitIntent(parsed.controllerResourceLimits, { + cpuLimit: process.env[controllerCpuLimitEnvKey], + ramLimit: process.env[controllerMemoryLimitEnvKey], + pidsLimit: process.env[controllerPidsLimitEnvKey] + }) + + for (const assignment of assignments) { + process.env[assignment.key] = assignment.value + } + + process.env[controllerResourceLimitsForceRecreateEnvKey] = forceRecreate ? "1" : "0" + + return parsed.args + }) + // CHANGE: read and parse CLI arguments from process.argv // WHY: keep IO at the boundary and delegate parsing to CORE // QUOTE(ТЗ): "Надо написать CLI команду" @@ -16,6 +46,13 @@ import { parseArgs } from "./parser.js" // COMPLEXITY: O(n) where n = |argv| export const readCommand: Effect.Effect = pipe( Effect.sync(() => process.argv.slice(2)), + Effect.map((args) => stripControllerResourceLimitArgs(args)), + Effect.flatMap((result) => + Either.match(result, { + onLeft: (error) => Effect.fail(error), + onRight: (parsed) => applyControllerResourceLimitEnv(parsed) + }) + ), Effect.map((args) => parseArgs(args)), Effect.flatMap((result) => Either.match(result, { diff --git a/packages/app/src/docker-git/cli/usage.ts b/packages/app/src/docker-git/cli/usage.ts index 3eab8c86..31cc085d 100644 --- a/packages/app/src/docker-git/cli/usage.ts +++ b/packages/app/src/docker-git/cli/usage.ts @@ -54,6 +54,9 @@ Options: --codex-home Container path for Codex auth (default: /home/dev/.codex) --cpu CPU limit: percent or cores (examples: 30%, 1.5; default: 30%) --ram RAM limit: percent or size (examples: 30%, 512m, 4g; default: 30%) + --controller-cpu Controller CPU cap intent: percent or cores (default: 90%) + --controller-ram Controller RAM cap intent: percent or size (default: 90%) + --controller-pids Controller PID cap (default: 4096) --playwright-cpu CPU limit for the MCP Playwright browser sidecar (default: 30% or --cpu when set) --playwright-ram RAM limit for the MCP Playwright browser sidecar (default: 30% or --ram when set) --network-mode Compose network mode: shared|project (default: shared) diff --git a/packages/app/src/docker-git/controller-resource-limits-shell.ts b/packages/app/src/docker-git/controller-resource-limits-shell.ts new file mode 100644 index 00000000..6b8ab0be --- /dev/null +++ b/packages/app/src/docker-git/controller-resource-limits-shell.ts @@ -0,0 +1,84 @@ +import { Effect, Either } from "effect" + +import { + controllerCpuLimitEnvKey, + controllerMemoryLimitEnvKey, + controllerPidsLimitEnvKey, + controllerResourceLimitsForceRecreateEnvKey, + resolveControllerResourceLimitEnv +} from "./controller-resource-limits.js" +import { formatParseError } from "./frontend-lib/core/parse-errors.js" +import type { ControllerBootstrapError } from "./host-errors.js" + +const controllerBootstrapError = (message: string): ControllerBootstrapError => ({ + _tag: "ControllerBootstrapError", + message +}) + +const fallbackControllerHostResources = { + cpuCount: 1, + totalMemoryBytes: 1024 ** 3 +} + +const loadControllerHostResources = (): Effect.Effect< + { readonly cpuCount: number; readonly totalMemoryBytes: number } +> => + Effect.tryPromise({ + try: () => import("node:os"), + catch: (error) => new Error(String(error)) + }).pipe( + Effect.map((os) => ({ + cpuCount: os.availableParallelism(), + totalMemoryBytes: os.totalmem() + })), + Effect.match({ + onFailure: () => fallbackControllerHostResources, + onSuccess: (value) => value + }) + ) + +const currentControllerResourceLimitIntent = () => ({ + cpuLimit: process.env[controllerCpuLimitEnvKey], + ramLimit: process.env[controllerMemoryLimitEnvKey], + pidsLimit: process.env[controllerPidsLimitEnvKey] +}) + +export const shouldForceRecreateForControllerResourceLimits = (): boolean => + process.env[controllerResourceLimitsForceRecreateEnvKey]?.trim() === "1" + +// CHANGE: resolve controller resource limits before invoking docker compose. +// WHY: compose requires concrete cpus/memory values, while docker-git accepts 90% defaults and percentage CLI/env intent. +// QUOTE(ТЗ): "по дефолту он должен иметь возможность к 90% лимитов" +// REF: issue-260-pr-comment-4429205358 +// SOURCE: https://github.com/ProverCoderAI/docker-git/pull/263#issuecomment-4429205358 +// FORMAT THEOREM: forall h: prepare(h) -> env(cpus,memory,pids) are compose-compatible +// PURITY: SHELL +// EFFECT: Effect +// INVARIANT: docker compose never receives percentage memory values +// COMPLEXITY: O(1) +export const prepareControllerResourceLimitEnv = (): Effect.Effect => + Effect.gen(function*(_) { + const hostResources = yield* _(loadControllerHostResources()) + const resolved = resolveControllerResourceLimitEnv(currentControllerResourceLimitIntent(), hostResources) + + if (Either.isLeft(resolved)) { + return yield* _( + Effect.fail( + controllerBootstrapError( + [ + "Invalid docker-git controller resource limit.", + formatParseError(resolved.left) + ].join("\n") + ) + ) + ) + } + + yield* _( + Effect.sync(() => { + process.env[controllerCpuLimitEnvKey] = resolved.right.cpus + process.env[controllerMemoryLimitEnvKey] = resolved.right.memory + process.env[controllerPidsLimitEnvKey] = resolved.right.pids + }) + ) + }) diff --git a/packages/app/src/docker-git/controller-resource-limits.ts b/packages/app/src/docker-git/controller-resource-limits.ts new file mode 100644 index 00000000..78118747 --- /dev/null +++ b/packages/app/src/docker-git/controller-resource-limits.ts @@ -0,0 +1,310 @@ +import { Either, Match } from "effect" + +import type { ParseError } from "./frontend-lib/core/domain.js" +import { + normalizeCpuLimit, + normalizeRamLimit, + resolveComposeResourceLimits +} from "./frontend-lib/core/resource-limits.js" + +export const controllerCpuLimitEnvKey = "DOCKER_GIT_CONTROLLER_CPUS" +export const controllerMemoryLimitEnvKey = "DOCKER_GIT_CONTROLLER_MEMORY" +export const controllerPidsLimitEnvKey = "DOCKER_GIT_CONTROLLER_PIDS" +export const controllerResourceLimitsForceRecreateEnvKey = "DOCKER_GIT_CONTROLLER_RESOURCE_LIMITS_FORCE_RECREATE" + +export const defaultControllerCpuLimit = "90%" +export const defaultControllerRamLimit = "90%" +export const defaultControllerPidsLimit = "4096" + +export const controllerCpuOption = "--controller-cpu" +export const controllerRamOption = "--controller-ram" +export const controllerPidsOption = "--controller-pids" + +type HostResources = { + readonly cpuCount: number + readonly totalMemoryBytes: number +} + +export type ControllerResourceLimitIntent = { + readonly cpuLimit?: string | undefined + readonly ramLimit?: string | undefined + readonly pidsLimit?: string | undefined +} + +export type ControllerResourceLimitEnv = { + readonly cpus: string + readonly memory: string + readonly pids: string +} + +export type ControllerResourceLimitArgParse = { + readonly args: ReadonlyArray + readonly controllerResourceLimits: ControllerResourceLimitIntent +} + +type ControllerResourceLimitKey = "cpuLimit" | "ramLimit" | "pidsLimit" + +type ControllerValueOptionSpec = { + readonly flag: string + readonly key: ControllerResourceLimitKey +} + +type EnvAssignment = { + readonly key: string + readonly value: string +} + +const controllerValueOptionSpecs: ReadonlyArray = [ + { flag: "--controller-cpu", key: "cpuLimit" }, + { flag: "--controller-cpus", key: "cpuLimit" }, + { flag: "--controller-ram", key: "ramLimit" }, + { flag: "--controller-memory", key: "ramLimit" }, + { flag: "--controller-pids", key: "pidsLimit" } +] + +const controllerValueOptionSpecByFlag: ReadonlyMap = new Map( + controllerValueOptionSpecs.map((spec) => [spec.flag, spec]) +) + +const pidsLimitPattern = /^[1-9]\d*$/u + +const rewriteInvalidOption = (error: ParseError, option: string): ParseError => + error._tag === "InvalidOption" + ? { _tag: "InvalidOption", option, reason: error.reason } + : error + +const requireNormalizedValue = ( + value: string | undefined, + option: string +): Either.Either => + value === undefined + ? Either.left({ _tag: "MissingOptionValue", option }) + : Either.right(value) + +const normalizeControllerMeasuredLimit = ( + normalize: (value: string, option: string) => Either.Either, + value: string, + option: string +): Either.Either => { + const normalized = normalize(value, option) + if (Either.isLeft(normalized)) { + return Either.left(rewriteInvalidOption(normalized.left, option)) + } + return requireNormalizedValue(normalized.right, option) +} + +const normalizeControllerCpuLimit = ( + value: string, + option: string +): Either.Either => normalizeControllerMeasuredLimit(normalizeCpuLimit, value, option) + +const normalizeControllerRamLimit = ( + value: string, + option: string +): Either.Either => normalizeControllerMeasuredLimit(normalizeRamLimit, value, option) + +const normalizeControllerPidsLimit = ( + value: string, + option: string +): Either.Either => { + const candidate = value.trim() + if (!pidsLimitPattern.test(candidate)) { + return Either.left({ + _tag: "InvalidOption", + option, + reason: "expected positive integer PID limit like 4096" + }) + } + return Either.right(candidate) +} + +const nonEmptyOrDefault = (value: string | undefined, defaultValue: string): string => { + const candidate = value?.trim() ?? "" + return candidate.length === 0 ? defaultValue : candidate +} + +const normalizeControllerValue = ( + key: ControllerResourceLimitKey, + value: string, + option: string +): Either.Either => + Match.value(key).pipe( + Match.when("cpuLimit", () => normalizeControllerCpuLimit(value, option)), + Match.when("ramLimit", () => normalizeControllerRamLimit(value, option)), + Match.when("pidsLimit", () => normalizeControllerPidsLimit(value, option)), + Match.exhaustive + ) + +const withControllerValue = ( + intent: ControllerResourceLimitIntent, + key: ControllerResourceLimitKey, + value: string +): ControllerResourceLimitIntent => + Match.value(key).pipe( + Match.when("cpuLimit", () => ({ ...intent, cpuLimit: value })), + Match.when("ramLimit", () => ({ ...intent, ramLimit: value })), + Match.when("pidsLimit", () => ({ ...intent, pidsLimit: value })), + Match.exhaustive + ) + +const applyControllerValueOption = ( + intent: ControllerResourceLimitIntent, + spec: ControllerValueOptionSpec, + value: string +): Either.Either => { + const normalized = normalizeControllerValue(spec.key, value, spec.flag) + if (Either.isLeft(normalized)) { + return Either.left(normalized.left) + } + return Either.right(withControllerValue(intent, spec.key, normalized.right)) +} + +const splitInlineControllerValueToken = ( + token: string +): { readonly flag: string; readonly value: string } | null => { + if (!token.startsWith("-")) { + return null + } + + const equalIndex = token.indexOf("=") + return equalIndex <= 0 + ? null + : { flag: token.slice(0, equalIndex), value: token.slice(equalIndex + 1) } +} + +const parseInlineControllerValueToken = ( + token: string +): { readonly spec: ControllerValueOptionSpec; readonly value: string } | null => { + const inline = splitInlineControllerValueToken(token) + if (inline === null) { + return null + } + + const spec = controllerValueOptionSpecByFlag.get(inline.flag) + if (spec === undefined) { + return null + } + + return { spec, value: inline.value } +} + +export const hasControllerResourceLimitOverrides = ( + intent: ControllerResourceLimitIntent +): boolean => intent.cpuLimit !== undefined || intent.ramLimit !== undefined || intent.pidsLimit !== undefined + +// CHANGE: decide whether resource-limit intent changes require controller recreate. +// WHY: compose applies caps only at container creation, so CLI/env overrides must bypass healthy-controller reuse. +// QUOTE(ТЗ): "можно настраивать и больше и меньше с помощью cli параметров" +// REF: issue-260-pr-comment-4429205358 +// SOURCE: https://github.com/ProverCoderAI/docker-git/pull/263#issuecomment-4429205358 +// FORMAT THEOREM: hasOverrides(cli) || hasOverrides(env) -> recreate(controller) +// PURITY: CORE +// EFFECT: n/a +// INVARIANT: no configured limit intent is silently ignored by controller reuse +// COMPLEXITY: O(1) +export const shouldForceRecreateForControllerResourceLimitIntent = ( + cliIntent: ControllerResourceLimitIntent, + envIntent: ControllerResourceLimitIntent +): boolean => hasControllerResourceLimitOverrides(cliIntent) || hasControllerResourceLimitOverrides(envIntent) + +export const controllerResourceLimitEnvAssignments = ( + intent: ControllerResourceLimitIntent +): ReadonlyArray => [ + ...(intent.cpuLimit === undefined ? [] : [{ key: controllerCpuLimitEnvKey, value: intent.cpuLimit }]), + ...(intent.ramLimit === undefined ? [] : [{ key: controllerMemoryLimitEnvKey, value: intent.ramLimit }]), + ...(intent.pidsLimit === undefined ? [] : [{ key: controllerPidsLimitEnvKey, value: intent.pidsLimit }]), + ...(hasControllerResourceLimitOverrides(intent) + ? [{ key: controllerResourceLimitsForceRecreateEnvKey, value: "1" }] + : []) +] + +// CHANGE: strip controller-specific resource flags before normal command parsing. +// WHY: controller limits are global bootstrap options, not per-project template options. +// QUOTE(ТЗ): "можно настраивать и больше и меньше с помощью cli параметров" +// REF: issue-260-pr-comment-4429205358 +// SOURCE: https://github.com/ProverCoderAI/docker-git/pull/263#issuecomment-4429205358 +// FORMAT THEOREM: forall argv: strip(argv).args contains no controller-limit flags +// PURITY: CORE +// EFFECT: n/a +// INVARIANT: extracted values are normalized before reaching the shell boundary +// COMPLEXITY: O(n) where n = |argv| +export const stripControllerResourceLimitArgs = ( + args: ReadonlyArray +): Either.Either => { + const strippedArgs: Array = [] + let controllerResourceLimits: ControllerResourceLimitIntent = {} + let index = 0 + + while (index < args.length) { + const token = args[index] ?? "" + const inline = parseInlineControllerValueToken(token) + if (inline !== null) { + const parsed = applyControllerValueOption(controllerResourceLimits, inline.spec, inline.value) + if (Either.isLeft(parsed)) { + return Either.left(parsed.left) + } + controllerResourceLimits = parsed.right + index += 1 + continue + } + + const spec = controllerValueOptionSpecByFlag.get(token) + if (spec !== undefined) { + const value = args[index + 1] + if (value === undefined) { + return Either.left({ _tag: "MissingOptionValue", option: token }) + } + const parsed = applyControllerValueOption(controllerResourceLimits, spec, value) + if (Either.isLeft(parsed)) { + return Either.left(parsed.left) + } + controllerResourceLimits = parsed.right + index += 2 + continue + } + + strippedArgs.push(token) + index += 1 + } + + return Either.right({ + args: strippedArgs, + controllerResourceLimits + }) +} + +// CHANGE: resolve controller resource intent into Docker Compose-compatible values. +// WHY: compose cannot consume percentage memory limits, so percentages must be resolved at the host boundary. +// QUOTE(ТЗ): "по дефолту он должен иметь возможность к 90% лимитов" +// REF: issue-260-pr-comment-4429205358 +// SOURCE: https://github.com/ProverCoderAI/docker-git/pull/263#issuecomment-4429205358 +// FORMAT THEOREM: forall host: cpu=90% -> cpus=0.9*host.cpuCount, ram=90% -> memory=floor(0.9*host.ramMiB)m +// PURITY: CORE +// EFFECT: n/a +// INVARIANT: returned cpus/memory/pids are valid docker compose resource values +// COMPLEXITY: O(1) +export const resolveControllerResourceLimitEnv = ( + intent: ControllerResourceLimitIntent, + hostResources: HostResources +): Either.Either => + Either.gen(function*(_) { + const cpuLimit = yield* _( + normalizeControllerCpuLimit(nonEmptyOrDefault(intent.cpuLimit, defaultControllerCpuLimit), controllerCpuOption) + ) + const ramLimit = yield* _( + normalizeControllerRamLimit(nonEmptyOrDefault(intent.ramLimit, defaultControllerRamLimit), controllerRamOption) + ) + const pidsLimit = yield* _( + normalizeControllerPidsLimit( + nonEmptyOrDefault(intent.pidsLimit, defaultControllerPidsLimit), + controllerPidsOption + ) + ) + const resolved = resolveComposeResourceLimits({ cpuLimit, ramLimit }, hostResources) + + return { + cpus: String(resolved.cpuLimit), + memory: resolved.ramLimit, + pids: pidsLimit + } + }) diff --git a/packages/app/src/docker-git/controller.ts b/packages/app/src/docker-git/controller.ts index 4b2b6bd5..c8812745 100644 --- a/packages/app/src/docker-git/controller.ts +++ b/packages/app/src/docker-git/controller.ts @@ -23,6 +23,10 @@ import { resolveExplicitApiBaseUrl, trimTrailingSlashes } from "./controller-reachability.js" +import { + prepareControllerResourceLimitEnv, + shouldForceRecreateForControllerResourceLimits +} from "./controller-resource-limits-shell.js" import { shouldForceRecreateController } from "./controller-revision.js" import type { ControllerBootstrapError } from "./host-errors.js" @@ -149,22 +153,21 @@ const loadControllerBootstrapContext = (): Effect.Effect< ControllerRuntime > => Effect.gen(function*(_) { + yield* _(prepareControllerResourceLimitEnv()) const explicitApiBaseUrl = resolveExplicitApiBaseUrl() const localControllerRevision = yield* _(prepareLocalControllerRevision()) const currentControllerExists = yield* _(controllerExists()) const currentControllerRevision = yield* _(inspectControllerRevision()) const currentContainerNetworks = yield* _(resolveCurrentContainerNetworks()) const initialControllerNetworks = yield* _(inspectContainerNetworks(controllerContainerName)) + const forceRecreateForResourceLimits = shouldForceRecreateForControllerResourceLimits() return { explicitApiBaseUrl, localControllerRevision, currentControllerRevision, - forceRecreateController: shouldForceRecreateController( - currentControllerExists, - localControllerRevision, - currentControllerRevision - ), + forceRecreateController: forceRecreateForResourceLimits || + shouldForceRecreateController(currentControllerExists, localControllerRevision, currentControllerRevision), currentContainerNetworks, initialControllerNetworks } @@ -255,6 +258,7 @@ export const ensureControllerReady = (): Effect.Effect = ["docker-compose.yml", "docker-compose.api.yml"] + +const readComposeFile = (relativePath: string): Effect.Effect => + Effect.gen(function*(_) { + const fs = yield* _(FileSystem.FileSystem) + const path = yield* _(Path.Path) + return yield* _(fs.readFileString(path.join("..", "..", relativePath))) + }).pipe( + Effect.provide(NodeContext.layer), + Effect.orDie + ) + +describe("controller compose resource limits", () => { + for (const composeFile of composeFiles) { + describe(composeFile, () => { + it.effect("caps controller CPU usage", () => + Effect.gen(function*(_) { + const contents = yield* _(readComposeFile(composeFile)) + expect(contents).toContain("cpus: ${DOCKER_GIT_CONTROLLER_CPUS:-0.9}") + })) + + it.effect("caps controller memory and swap together", () => + Effect.gen(function*(_) { + const contents = yield* _(readComposeFile(composeFile)) + expect(contents).toContain("mem_limit: ${DOCKER_GIT_CONTROLLER_MEMORY:-921m}") + expect(contents).toContain("memswap_limit: ${DOCKER_GIT_CONTROLLER_MEMORY:-921m}") + })) + + it.effect("caps controller PIDs to prevent fork bombs", () => + Effect.gen(function*(_) { + const contents = yield* _(readComposeFile(composeFile)) + expect(contents).toMatch(/pids_limit: \$\{DOCKER_GIT_CONTROLLER_PIDS:-\d+\}/u) + })) + }) + } +}) + +describe("controller resource limit resolution", () => { + it.effect("resolves CPU and RAM defaults to 90% of host resources", () => + Effect.sync(() => { + const resolved = resolveControllerResourceLimitEnv( + {}, + { + cpuCount: 8, + totalMemoryBytes: 16 * 1024 ** 3 + } + ) + + Either.match(resolved, { + onLeft: (error) => { + throw new Error(`unexpected parse error ${error._tag}`) + }, + onRight: (env) => { + expect(env).toEqual({ + cpus: "7.2", + memory: "14745m", + pids: "4096" + }) + } + }) + })) + + it.effect("allows controller CLI flags before and after the command", () => + Effect.sync(() => { + const parsed = stripControllerResourceLimitArgs([ + "--controller-cpu", + "75%", + "clone", + "https://github.com/org/repo.git", + "--controller-ram=8g", + "--controller-pids", + "8192" + ]) + + Either.match(parsed, { + onLeft: (error) => { + throw new Error(`unexpected parse error ${error._tag}`) + }, + onRight: (result) => { + expect(result.args).toEqual(["clone", "https://github.com/org/repo.git"]) + expect(result.controllerResourceLimits).toEqual({ + cpuLimit: "75%", + ramLimit: "8g", + pidsLimit: "8192" + }) + } + }) + })) + + it.effect("emits compose env intent and recreate marker for controller CLI overrides", () => + Effect.sync(() => { + expect( + controllerResourceLimitEnvAssignments({ + cpuLimit: "4", + ramLimit: "16g", + pidsLimit: "8192" + }) + ).toEqual([ + { key: controllerCpuLimitEnvKey, value: "4" }, + { key: controllerMemoryLimitEnvKey, value: "16g" }, + { key: controllerPidsLimitEnvKey, value: "8192" }, + { key: controllerResourceLimitsForceRecreateEnvKey, value: "1" } + ]) + })) + + it.effect("forces controller recreate for either CLI or env limit intent", () => + Effect.sync(() => { + expect( + shouldForceRecreateForControllerResourceLimitIntent( + { cpuLimit: "75%" }, + {} + ) + ).toBe(true) + expect( + shouldForceRecreateForControllerResourceLimitIntent( + {}, + { ramLimit: "8g" } + ) + ).toBe(true) + expect(shouldForceRecreateForControllerResourceLimitIntent({}, {})).toBe(false) + })) +})