From 708b4388ca869aa0ece204673a557c261cb1a29d Mon Sep 17 00:00:00 2001 From: Vignesh Date: Wed, 14 Oct 2026 19:57:11 +0530 Subject: [PATCH 1/3] feat(agents): make agent model, output budget, and compression env-tunable Agent model was hardcoded to openai/gpt-4o-mini and output caps/compression were fixed in source. Add opt-in env overrides so a free OpenRouter model and looser budgets can be used without editing code: - STACKFORGE_MODEL / STACKFORGE_MODEL_: per-agent or global model - STACKFORGE_OUTPUT_SCALE: scale factor on maxOutputTokens and tokenBudget - STACKFORGE_COMPRESSION: global compression level override Defaults are unchanged when env is unset, so existing behaviour and tests are preserved. Document the new vars in .env.example. --- apps/api/.env.example | 14 +++ packages/agents/src/config/agent.configs.ts | 110 +++++++++++++++----- 2 files changed, 96 insertions(+), 28 deletions(-) diff --git a/apps/api/.env.example b/apps/api/.env.example index d93872d..49c71cf 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -16,6 +16,20 @@ STACKFORGE_PROVIDER=auto # Codegen provider: default | nvidia STACKFORGE_CODEGEN_PROVIDER=default +# ─── Per-agent model & budget tuning (all optional) ────────────────────────── +# Override the model for every agent at once (e.g. a free OpenRouter model): +# STACKFORGE_MODEL=meta-llama/llama-3.3-70b-instruct:free +STACKFORGE_MODEL= +# Or override a single agent (planner|schema|api|frontend|devops|reviewer|codegen): +# STACKFORGE_MODEL_PLANNER=... +# STACKFORGE_MODEL_CODEGEN=... +# Scale factor applied to every agent's output token budget (default 1). +# Raise it to give weaker models more room, e.g. 1.5: +STACKFORGE_OUTPUT_SCALE= +# Global compression override: low | medium | high (default: per-agent setting). +# "low" reduces context loss between agents at higher token cost: +STACKFORGE_COMPRESSION= + # OpenRouter API Keys (all 3 required for key rotation) OPENROUTER_API_KEY_1= OPENROUTER_API_KEY_2= diff --git a/packages/agents/src/config/agent.configs.ts b/packages/agents/src/config/agent.configs.ts index c98764b..52567ef 100644 --- a/packages/agents/src/config/agent.configs.ts +++ b/packages/agents/src/config/agent.configs.ts @@ -1,94 +1,148 @@ import type { AgentConfig, AgentName } from "@stackforge/shared"; +/** + * Resolve the model for an agent, allowing env overrides. + * + * Precedence (highest first): + * 1. STACKFORGE_MODEL_ e.g. STACKFORGE_MODEL_PLANNER + * 2. STACKFORGE_MODEL global override for every agent + * 3. the hardcoded default passed in + * + * This lets a free OpenRouter model be swapped in without editing source, + * either per-agent or across the whole pipeline. + */ +function resolveModel(agent: AgentName, fallback: string): string { + const perAgent = process.env[`STACKFORGE_MODEL_${agent.toUpperCase()}`]?.trim(); + if (perAgent !== undefined && perAgent.length > 0) { + return perAgent; + } + + const global = process.env["STACKFORGE_MODEL"]?.trim(); + if (global !== undefined && global.length > 0) { + return global; + } + + return fallback; +} + +/** + * Scale factor applied to every agent's output budget (maxOutputTokens and + * tokenBudget). Lets a demo give models more room without editing per-agent + * numbers. Defaults to 1 (no change), so existing behaviour and tests are + * unaffected. Set e.g. STACKFORGE_OUTPUT_SCALE=1.6 to loosen caps. + */ +const OUTPUT_SCALE = (() => { + const raw = Number.parseFloat(process.env["STACKFORGE_OUTPUT_SCALE"] ?? ""); + return Number.isFinite(raw) && raw > 0 ? raw : 1; +})(); + +/** + * Global override for every agent's compression aggressiveness. When unset, + * each agent keeps its own default level. Set STACKFORGE_COMPRESSION=low to + * reduce context loss (better output on weak models, higher token cost). + */ +const COMPRESSION_OVERRIDE: AgentConfig["compressionLevel"] | undefined = (() => { + const raw = (process.env["STACKFORGE_COMPRESSION"] ?? "").trim().toLowerCase(); + return raw === "low" || raw === "medium" || raw === "high" ? raw : undefined; +})(); + +function scaleOutput(value: number): number { + return Math.round(value * OUTPUT_SCALE); +} + +function resolveCompression(fallback: AgentConfig["compressionLevel"]): AgentConfig["compressionLevel"] { + return COMPRESSION_OVERRIDE ?? fallback; +} + export const AGENT_CONFIGS: Record = { planner: { name: "planner", description: "Resolves stack, project name, and top-level folder structure", - tokenBudget: 1000, - model: "openai/gpt-4o-mini", + tokenBudget: scaleOutput(1000), + model: resolveModel("planner", "openai/gpt-4o-mini"), maxInputTokens: 900, - maxOutputTokens: 700, + maxOutputTokens: scaleOutput(700), minOutputTokens: 220, temperature: 0.2, - compressionLevel: "high", + compressionLevel: resolveCompression("high"), budgetOverflowRetries: 2, retries: 1, }, schema: { name: "schema", description: "Designs entities, fields, indexes, and relationships", - tokenBudget: 1500, - model: "openai/gpt-4o-mini", + tokenBudget: scaleOutput(1500), + model: resolveModel("schema", "openai/gpt-4o-mini"), maxInputTokens: 1200, - maxOutputTokens: 1100, + maxOutputTokens: scaleOutput(1100), minOutputTokens: 320, temperature: 0.1, - compressionLevel: "medium", + compressionLevel: resolveCompression("medium"), budgetOverflowRetries: 2, retries: 1, }, api: { name: "api", description: "Plans REST routes, request/response shapes, and auth requirements", - tokenBudget: 1600, - model: "openai/gpt-4o-mini", + tokenBudget: scaleOutput(1600), + model: resolveModel("api", "openai/gpt-4o-mini"), maxInputTokens: 950, - maxOutputTokens: 1200, + maxOutputTokens: scaleOutput(1200), minOutputTokens: 280, temperature: 0.1, - compressionLevel: "medium", + compressionLevel: resolveCompression("medium"), budgetOverflowRetries: 2, retries: 1, }, frontend: { name: "frontend", description: "Plans pages, React component trees, and routing", - tokenBudget: 1200, - model: "openai/gpt-4o-mini", + tokenBudget: scaleOutput(1200), + model: resolveModel("frontend", "openai/gpt-4o-mini"), maxInputTokens: 900, - maxOutputTokens: 900, + maxOutputTokens: scaleOutput(900), minOutputTokens: 250, temperature: 0.2, - compressionLevel: "high", + compressionLevel: resolveCompression("high"), budgetOverflowRetries: 2, retries: 1, }, devops: { name: "devops", description: "Plans CI/CD pipelines, deployment targets, Docker, and env vars", - tokenBudget: 1000, - model: "openai/gpt-4o-mini", + tokenBudget: scaleOutput(1000), + model: resolveModel("devops", "openai/gpt-4o-mini"), maxInputTokens: 700, - maxOutputTokens: 850, + maxOutputTokens: scaleOutput(850), minOutputTokens: 180, temperature: 0.2, - compressionLevel: "high", + compressionLevel: resolveCompression("high"), budgetOverflowRetries: 3, retries: 1, }, reviewer: { name: "reviewer", description: "Performs consistency checks and flags issues across the full blueprint", - tokenBudget: 1200, - model: "openai/gpt-4o-mini", + tokenBudget: scaleOutput(1200), + model: resolveModel("reviewer", "openai/gpt-4o-mini"), maxInputTokens: 850, - maxOutputTokens: 900, + maxOutputTokens: scaleOutput(900), minOutputTokens: 220, temperature: 0, - compressionLevel: "high", + compressionLevel: resolveCompression("high"), budgetOverflowRetries: 2, retries: 1, }, codegen: { name: "codegen", description: "Generates concrete source files from the finalized blueprint", - tokenBudget: 2200, - model: "openai/gpt-4o-mini", + tokenBudget: scaleOutput(2200), + model: resolveModel("codegen", "openai/gpt-4o-mini"), maxInputTokens: 1100, - maxOutputTokens: 1200, + maxOutputTokens: scaleOutput(1200), minOutputTokens: 280, temperature: 0.1, - compressionLevel: "medium", + compressionLevel: resolveCompression("medium"), budgetOverflowRetries: 3, retries: 1, }, From a62843300dde4730930c8196ff6df745d2944198 Mon Sep 17 00:00:00 2001 From: Vignesh Date: Wed, 14 Oct 2026 19:57:18 +0530 Subject: [PATCH 2/3] feat(workflow): add single self-correction pass for invalid agent output Weak/free models frequently emit malformed or incomplete JSON, which previously failed the entire job on the first validation error. When validateStepOutput throws, run one repair call that shows the model its own output plus the validation error and asks for corrected strict JSON, then re-validate. If the repair also fails, the original failure propagates as before. Add a test proving the engine recovers from an invalid planner output via exactly one repair pass with no agent_failed event emitted. --- packages/agents/src/workflow/engine.ts | 71 ++++++++++++++++++- .../agents/test/orchestrator-failure.test.ts | 58 +++++++++++++++ 2 files changed, 127 insertions(+), 2 deletions(-) diff --git a/packages/agents/src/workflow/engine.ts b/packages/agents/src/workflow/engine.ts index 17ec630..23d3b21 100644 --- a/packages/agents/src/workflow/engine.ts +++ b/packages/agents/src/workflow/engine.ts @@ -1,4 +1,4 @@ -import type { AgentName, SSEEvent } from "@stackforge/shared"; +import type { AgentName, AgentConfig, SSEEvent } from "@stackforge/shared"; import { AGENT_CONFIGS } from "../config/agent.configs.js"; import type { AgentCache } from "../cache/agent.cache.js"; import type { LLMProvider, ProviderCallOutput } from "../provider/provider.interface.js"; @@ -357,7 +357,28 @@ export class WorkflowEngine { } } - validateStepOutput(step, primary.output); + try { + validateStepOutput(step, primary.output); + } catch (validationError) { + // One self-correction pass: weak/free models frequently emit malformed + // or incomplete JSON. Rather than fail the whole job, show the model its + // own output plus the validation error and ask for a corrected version. + const repaired = await this.attemptRepair( + workflow, + step, + config, + userPrompt, + primary.output, + validationError instanceof Error ? validationError.message : String(validationError), + ); + primary = { + ...primary, + output: repaired.output, + tokensUsed: primary.tokensUsed + repaired.tokensUsed, + }; + this.options.cache.set(primaryCacheKey, primary.output); + validateStepOutput(step, primary.output); + } // Convert raw code string to structured format for codegen let normalizedOutput = primary.output; @@ -426,6 +447,52 @@ export class WorkflowEngine { } } + private async attemptRepair( + workflow: WorkflowDefinition, + step: WorkflowStep, + config: AgentConfig, + originalUserPrompt: string, + badOutput: unknown, + validationError: string, + ): Promise<{ output: unknown; tokensUsed: number }> { + const repairPrompt = [ + "Your previous response for this workflow step was invalid and must be corrected.", + "", + `Validation error: ${validationError}`, + "", + "Your previous (invalid) output was:", + serializeOutput(badOutput), + "", + "Original step instructions:", + originalUserPrompt, + "", + "Return ONLY the corrected, strict JSON object for this step. No markdown, no code fences, no commentary.", + ].join("\n"); + + const response = await this.options.provider.call({ + agentName: step.agent, + input: { + workflowId: workflow.id, + stepId: step.id, + repair: true, + }, + options: { + systemPrompt: + "You repair invalid workflow-step output. Return strict JSON object output only that satisfies the stated schema.", + userPrompt: repairPrompt, + model: config.model, + maxInputTokens: config.maxInputTokens, + maxOutputTokens: config.maxOutputTokens, + temperature: 0, + }, + }); + + return { + output: response.output, + tokensUsed: response.tokensUsed, + }; + } + private async compressStepOutput( workflow: WorkflowDefinition, step: WorkflowStep, diff --git a/packages/agents/test/orchestrator-failure.test.ts b/packages/agents/test/orchestrator-failure.test.ts index d008cb4..38ca753 100644 --- a/packages/agents/test/orchestrator-failure.test.ts +++ b/packages/agents/test/orchestrator-failure.test.ts @@ -148,6 +148,38 @@ class CompressionFailureProvider extends FixedTokenProvider { } } +/** + * Returns an invalid planner payload on the first attempt, then a valid payload + * once the engine issues a repair call (flagged via `repair: true` in input). + * Exercises the self-correction retry path in the workflow engine. + */ +class RepairablePlannerProvider extends FixedTokenProvider { + repairCalls = 0; + + override async call(input: ProviderCallInput): Promise { + const payload = input.input as Record; + + if (input.agentName === "planner" && "compressionTarget" in payload === false) { + if (payload["repair"] === true) { + this.repairCalls += 1; + return super.call(input); + } + + // First (pre-repair) planner attempt: deliberately invalid shape. + return { + output: { bad: true }, + tokensUsed: 10, + durationMs: 1, + inputTokens: 5, + outputTokens: 5, + model: input.options.model, + }; + } + + return super.call(input); + } +} + describe("Orchestrator failure propagation", () => { it("emits agent_failed and rejects when agent output schema validation fails", async () => { const events: SSEEvent[] = []; @@ -228,4 +260,30 @@ describe("Orchestrator failure propagation", () => { expect(blueprint.reviewerNotes.length).toBeGreaterThan(0); expect((blueprint.generatedSourceFiles?.length ?? 0) > 0).toBe(true); }); + + it("recovers from invalid agent output via a single repair pass", async () => { + const events: SSEEvent[] = []; + const provider = new RepairablePlannerProvider(); + + const blueprint = await runOrchestrator({ + jobId: crypto.randomUUID(), + prompt: "Build a task management platform", + projectName: "taskflow", + execution: { + enableCodeGeneration: false, + }, + emit: (event) => events.push(event), + provider, + cache: new AgentCache(), + }); + + // Repair was triggered exactly once and the job still completed. + expect(provider.repairCalls).toBe(1); + expect(blueprint.projectName.length).toBeGreaterThan(0); + expect(blueprint.routePlan.length).toBeGreaterThan(0); + + // No agent_failed should be emitted since the repair succeeded. + const failed = events.find((event) => event.type === "agent_failed"); + expect(failed).toBeUndefined(); + }); }); From 75ac30b5ad6e991117c1af7a216aebede3521d13 Mon Sep 17 00:00:00 2001 From: Vignesh Date: Wed, 14 Oct 2026 20:01:12 +0530 Subject: [PATCH 3/3] chore: sync bun.lock after husky removal The 'remove husky pre-commit hooks' change dropped husky from package.json but left it in bun.lock, so 'bun install --frozen-lockfile' (used in CI) failed. Regenerate the lockfile to drop the stale husky entry. --- bun.lock | 3 --- 1 file changed, 3 deletions(-) diff --git a/bun.lock b/bun.lock index a53c940..9abee9f 100644 --- a/bun.lock +++ b/bun.lock @@ -4,7 +4,6 @@ "": { "name": "stackforge", "devDependencies": { - "husky": "^9.1.7", "turbo": "^2.3.3", "typescript": "^5.7.3", }, @@ -464,8 +463,6 @@ "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="], - "husky": ["husky@9.1.7", "", { "bin": { "husky": "bin.js" } }, "sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA=="], - "iconv-lite": ["iconv-lite@0.4.24", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3" } }, "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA=="], "immediate": ["immediate@3.0.6", "", {}, "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ=="],