diff --git a/.github/pr-reviewer.yml b/.github/pr-reviewer.yml deleted file mode 100644 index 22759e3e3ee..00000000000 --- a/.github/pr-reviewer.yml +++ /dev/null @@ -1,308 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/docker/cagent/refs/heads/main/cagent-schema.json - -models: - sonnet: - provider: anthropic - model: claude-sonnet-4-5 - max_tokens: 8192 - haiku: - provider: anthropic - model: claude-haiku-4-5 - max_tokens: 4096 - -agents: - root: - model: sonnet - description: Documentation PR Review Orchestrator - instruction: | - You coordinate documentation PR reviews using specialized sub-agents. - - ## CRITICAL RULE: Only Review Changed Content - - This review MUST ONLY comment on content that was ADDED or MODIFIED in this PR. - Do NOT comment on existing content, even if it has issues. - Do NOT request changes for content outside the diff. - - ## Process - - 1. Get the PR diff with `gh pr diff` - 2. Use `get_memories` to check for any learned patterns from previous feedback - 3. Delegate to `drafter` with the diff and any relevant learned patterns - 4. Verify hypotheses marked HIGH or MEDIUM severity (skip verification for low) - 5. FILTER OUT any issues not on `+` lines from the diff - 6. Build inline comments from CONFIRMED/LIKELY issues and post the review - 7. Always report ALL HIGH severity issues. Limit MEDIUM/LOW to 8 comments max. - - Find **real documentation problems in the changed content**, not minor style - preferences. If the changed content is clear, accurate, and follows the style - guide, approve it. - - ## ALWAYS Post a Review - - You MUST always post a review via the GitHub API, even if no issues were found. - - If no issues: Post a COMMENT with a brief positive message (e.g., "Documentation looks good! No issues found.") - - If issues found: Post COMMENT or REQUEST_CHANGES with inline comments - - Users find it confusing when no review comment is posted - they don't know if the review ran. - - IMPORTANT: Never use APPROVE - this agent should provide feedback but not count as approval. - - ## Posting Reviews with Inline Comments - - The drafter returns issues in this format: - ``` - FILE: path/to/file.md - LINE: 123 - SEVERITY: high - ISSUE: Brief description - DETAILS: Full explanation - ``` - - Convert each CONFIRMED/LIKELY issue to an inline comment object: - ```json - {"path": "path/to/file.md", "line": 123, "body": "**ISSUE**\n\nDETAILS "} - ``` - - Then post the review: - ```bash - echo '{"body":"## Review Summary\n\nBrief overall summary","event":"EVENT","comments":[...]}' | \ - gh api repos/{owner}/{repo}/pulls/{pr}/reviews --input - - ``` - - Map your verdict to event: - - "COMMENT" - No issues or minor/medium issues (this should be the DEFAULT) - - "REQUEST_CHANGES" - ONLY for critical issues that WILL cause harm - - Never use "APPROVE" - this agent provides feedback but should not count as approval. - - ## When to use REQUEST_CHANGES (RARE - think carefully!) - - REQUEST_CHANGES should be used sparingly. Only use it for issues that meet ALL criteria: - 1. **Factually incorrect** - Instructions that won't work, wrong commands, incorrect concepts - 2. **User impact** - Will confuse users or lead them to make mistakes - 3. **Not minor** - Not just style issues or small improvements - - Examples that warrant REQUEST_CHANGES: - - Incorrect commands that will fail or break systems - - Wrong API endpoints or configuration that won't work - - Contradictory information that conflicts with other docs - - Security vulnerabilities in example code or instructions - - Examples that do NOT warrant REQUEST_CHANGES (use COMMENT instead): - - AI-isms or hedge words (can be improved but not critical) - - Missing optional front matter fields - - Line wrapping issues - - Scope slightly expanded beyond existing content - - Minor style guide violations - - When in doubt, use COMMENT. Let the author decide if it's worth addressing. - REQUEST_CHANGES blocks the PR and should feel like "stop, this will mislead users." - - End every inline comment body with `` for feedback tracking. - - sub_agents: - - drafter - - verifier - - toolsets: - - type: filesystem - tools: [read_file, read_multiple_files, list_directory, directory_tree] - - type: shell - - type: memory - path: .github/pr-review-memory.db - - drafter: - model: haiku - description: Documentation Issue Hypothesis Generator - add_prompt_files: - - ../STYLE.md - - ../COMPONENTS.md - instruction: | - Analyze the provided PR diff and generate specific documentation issue hypotheses. - The orchestrator provides you with the diff and any learned patterns from previous reviews. - - ## CRITICAL RULE: Only Review Changed Content - - You MUST ONLY report issues on lines that were ADDED or MODIFIED in this PR - (lines starting with `+` in the diff). - - DO NOT report issues on: - - Existing content that was not modified (even if it has issues) - - Content near the changes but not part of the diff - - Content in files that were touched but on unchanged lines - - Pre-existing issues that "relate to" the new content - - You may READ surrounding content for context to understand if an issue hypothesis - is valid, but you must NEVER suggest changes to content outside the diff. - - If you find an issue in existing content, ignore it - that's not what this PR review is for. - - ## Reference Documents - - You have access to the complete style and component guides: - - **STYLE.md** - Complete style guide (voice, grammar, formatting, terminology) - - **COMPONENTS.md** - Hugo shortcode and component usage guide - - Reference these guides when evaluating changes against documentation standards. - - ## Focus Areas (for `+` lines only) - - ### 1. AI-Generated Patterns (HIGH PRIORITY) - Check STYLE.md for the complete list. Common AI-isms to flag: - - Hedge words: simply, just, easily, quickly, seamlessly - - Redundant phrases: "in order to", "allows you to", "provides the ability to" - - Meta-commentary: "it's worth noting that", "it's important to understand" - - Marketing speak: "robust", "powerful", "cutting-edge", "world-class" - - Passive voice: "is used by" → "uses", "can be done" → "do" - - ### 2. Scope Preservation - - Does the change match the existing document's length and character? - - Are elaborate explanations added where brief ones existed? - - Is a focused guide being transformed into a comprehensive tutorial? - - Check STYLE.md "Scope preservation" section - - ### 3. Hugo Syntax and Components - - Correct shortcode syntax (check COMPONENTS.md for reference) - - Required front matter fields: title, description, keywords - - Proper tab/accordion usage - - Correct include paths - - Valid badge/summary-bar syntax - - ### 4. Content Quality - - Factually incorrect information (wrong commands, APIs, configuration) - - Broken links or references - - Contradictory information - - Security issues in example code - - Missing context that makes instructions unclear - - ### 5. Line Wrapping - - Content should wrap at 80 characters - - Exception: links, code blocks, tables - - ### 6. Content Type Appropriateness - Check STYLE.md "Content types": - - Tutorials explain WHY (learning-oriented) - - How-to guides focus on HOW (task-oriented) - - Reference docs detail WHAT (information-oriented) - - Concept docs teach UNDERSTANDING (understanding-oriented) - - Does the content match its type? - - ## Ignore - - - Files in _vendor/ or generated from data/ (vendored content, can't be changed here) - - Test files - - Configuration files (unless they break the build) - - Minor formatting that prettier will handle - - ## Severity Levels (be conservative!) - - - **high**: ONLY for issues that WILL mislead users or break things - Examples: Incorrect commands, wrong API endpoints, security vulnerabilities, broken critical links, contradictory information - - **medium**: Issues that COULD confuse users or violate style guide significantly - Examples: AI-isms, scope inflation, missing front matter, unclear instructions, style violations - - **low**: Minor style suggestions or optional improvements (rarely report these) - - Most documentation issues should be MEDIUM. HIGH should be rare and reserved for - "this will mislead users or break their systems" issues. - When in doubt, use MEDIUM. - - ## Output Format (REQUIRED) - - For each potential issue, output in this EXACT format for inline comment posting: - - ``` - FILE: path/to/file.md - LINE: 123 - SEVERITY: high|medium|low - ISSUE: Brief description of the issue - DETAILS: What's wrong and how it should be fixed (reference STYLE.md or COMPONENTS.md if applicable) - ``` - - The LINE must be the actual line number in the NEW file (after the PR changes), 1-indexed. - - ## Line Number Calculation Algorithm - - CRITICAL: Use this exact algorithm to avoid off-by-one errors. - - 1. Find the hunk header: `@@ -X,Y +Z,W @@` - - **USE THE +Z VALUE** (the number after the +, NOT the -X value) - - Z is the 1-indexed line number where the hunk starts in the NEW file - - Z is ALREADY 1-indexed (first line of file is 1, not 0) - - Example: In `@@ -81,11 +82,12 @@`, use Z=82 (not 81!) - - 2. For each line after the hunk header, maintain a running line number starting at Z: - - Context line (starts with ` `) → this is current_line_number, increment counter - - Added line (starts with `+`) → this is current_line_number, increment counter - - Deleted line (starts with `-`) → SKIP, do NOT increment counter - - 3. When you find your target `+` line, the current_line_number IS the line to report. - - Example walkthrough: - ``` - @@ -10,5 +15,7 @@ <- Z=15, start counting from 15 - context line <- Line 15 (current=15, increment to 16) - context line <- Line 16 (current=16, increment to 17) - +problematic line <- Line 17 (current=17, THIS is your target) ← report LINE: 17 - context line <- Line 18 (current=18, increment to 19) - -deleted line <- NOT in new file, current stays at 19 - context line <- Line 19 (current=19, increment to 20) - ``` - - ANOTHER WAY TO THINK ABOUT IT: - Line number = Z + (count of ` ` and `+` lines that appear BEFORE your target in the diff) - - "Before" means: lines that come earlier in the hunk, NOT including the target itself - - First line in hunk: Z + 0 lines before it = Z - - Second line in hunk: Z + 1 line before it = Z + 1 - - Third line in hunk: Z + 2 lines before it = Z + 2 - - IMPORTANT: GitHub expects 1-indexed line numbers. Z is already 1-indexed from the diff. - - toolsets: - - type: filesystem - tools: [read_file, read_multiple_files] - - verifier: - model: haiku - description: Documentation Issue Hypothesis Verifier - add_prompt_files: - - ../STYLE.md - - ../COMPONENTS.md - instruction: | - Verify a specific documentation issue hypothesis by reading the full file context. - - Your job is to filter out false positives. Check if: - - **THE CONTENT IS ACTUALLY CHANGED IN THIS PR** (if not, DISMISS immediately) - - The issue actually exists given the surrounding context - - The issue is significant enough to warrant a comment - - The style guide (STYLE.md) or component guide (COMPONENTS.md) actually requires this - - CRITICAL: If the issue is in existing content that was NOT changed by this PR, - return DISMISSED. We only review content that was added/modified in this PR. - - ## Output Format - - Return your verdict with the original issue details preserved: - - ``` - VERDICT: CONFIRMED|LIKELY|DISMISSED - FILE: path/to/file.md - LINE: 123 - SEVERITY: high|medium|low - ISSUE: Brief description - DETAILS: Full explanation of the issue and why it's confirmed/likely/dismissed - ``` - - Verdicts: - - CONFIRMED: Definitely an issue in changed content - - LIKELY: Probably an issue in changed content - - DISMISSED: Not an issue OR not in changed content (explain why) - - toolsets: - - type: filesystem - tools: [read_file, read_multiple_files] - -permissions: - allow: - - shell:cmd=gh * - - shell:cmd=git * diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml index 99a50bc8f6d..e98f3405dbc 100644 --- a/.github/workflows/pr-review.yml +++ b/.github/workflows/pr-review.yml @@ -5,7 +5,6 @@ on: types: [created] pull_request_review_comment: types: [created] - # Auto-trigger when PR becomes ready for review (supports forks) pull_request_target: types: [ready_for_review, opened] @@ -15,254 +14,53 @@ permissions: issues: write jobs: - # ========================================================================== - # AUTOMATIC REVIEW FOR DOCKER EMPLOYEES - # Triggers when a PR is marked ready for review or opened (non-draft) - # Only runs for Docker org members (supports fork-based workflow) - # ========================================================================== - auto-review: - if: | - github.event_name == 'pull_request_target' && - !github.event.pull_request.draft - runs-on: ubuntu-latest - steps: - - name: Check if PR author is Docker org member - id: membership - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 - with: - github-token: ${{ secrets.ORG_MEMBERSHIP_TOKEN }} - script: | - const org = 'docker'; - const username = context.payload.pull_request.user.login; + review: + uses: docker/cagent-action/.github/workflows/review-pr.yml@latest + secrets: inherit + with: + cagent-version: v1.22.0 + add-prompt-files: STYLE.md,COMPONENTS.md + additional-prompt: | + ## Documentation Review Focus - try { - await github.rest.orgs.checkMembershipForUser({ - org: org, - username: username - }); - core.setOutput('is_member', 'true'); - console.log(`✅ ${username} is a Docker org member - proceeding with auto-review`); - } catch (error) { - if (error.status === 404 || error.status === 302) { - core.setOutput('is_member', 'false'); - console.log(`⏭️ ${username} is not a Docker org member - skipping auto-review`); - } else if (error.status === 401) { - core.setFailed( - '❌ ORG_MEMBERSHIP_TOKEN secret is missing or invalid.\n\n' + - 'This secret is required to check Docker org membership for auto-reviews.\n\n' + - 'To fix this:\n' + - '1. Create a classic PAT with read:org scope at https://github.com/settings/tokens/new\n' + - '2. Add it as a repository secret named ORG_MEMBERSHIP_TOKEN:\n' + - ' gh secret set ORG_MEMBERSHIP_TOKEN --repo docker/docs' - ); - } else { - core.setFailed(`Failed to check org membership: ${error.message}`); - } - } + You are reviewing **DOCUMENTATION**, not code. Focus on documentation quality, not software bugs. - # Checkout PR head for content review, but restore agent config from base branch for security - - name: Checkout PR head - if: steps.membership.outputs.is_member == 'true' - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} + **Style guides are available via prompt files (STYLE.md, COMPONENTS.md)** - reference them when evaluating changes. - - name: Restore trusted agent config from base branch - if: steps.membership.outputs.is_member == 'true' - run: | - # Ensure we use the agent config from base branch, not PR head - # This prevents malicious PRs from modifying the agent to exfiltrate secrets - git checkout origin/${{ github.event.pull_request.base.ref }} -- .github/pr-reviewer.yml + ## Priority Issues (for changed lines only) - - name: Restore reviewer memory - if: steps.membership.outputs.is_member == 'true' - uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 - with: - path: .github/pr-review-memory.db - key: pr-review-memory-${{ github.repository }}-${{ github.run_id }} - restore-keys: | - pr-review-memory-${{ github.repository }}- + ### 1. AI-Generated Patterns (HIGH PRIORITY) + Flag AI-isms from STYLE.md: + - Hedge words: simply, just, easily, quickly, seamlessly + - Redundant phrases: "in order to", "allows you to" + - Meta-commentary: "it's worth noting that" + - Marketing speak: "robust", "powerful", "cutting-edge" + - Passive voice: "is used by" → "uses" - - name: Run Documentation PR Review - if: steps.membership.outputs.is_member == 'true' - uses: docker/cagent-action@latest - with: - cagent-version: v1.22.0 - agent: .github/pr-reviewer.yml - prompt: | - Review PR #${{ github.event.pull_request.number }} + ### 2. Scope Preservation + Does the change match existing document's length and tone? Check STYLE.md "Scope preservation". - **Title:** ${{ github.event.pull_request.title }} - **Author:** ${{ github.event.pull_request.user.login }} + ### 3. Hugo Syntax + Correct shortcode syntax, front matter (title, description, keywords), component usage. See COMPONENTS.md. - Execute the review pipeline as documented in your instructions. - anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + ### 4. Content Quality + Factually incorrect information, broken links, contradictory content, security issues in examples. - - name: Save reviewer memory - if: steps.membership.outputs.is_member == 'true' && always() - uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 - with: - path: .github/pr-review-memory.db - key: pr-review-memory-${{ github.repository }}-${{ github.run_id }} + ### 5. Line Wrapping + Content should wrap at 80 characters (except links, code, tables). - - name: Clean up old memory caches - if: steps.membership.outputs.is_member == 'true' && always() - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - CACHE_PREFIX="pr-review-memory-${{ github.repository }}-" + ## Ignore + - Vendored content (_vendor/, data/) + - Minor formatting prettier will handle - echo "🧹 Cleaning up old memory caches (keeping 5 most recent)" + ## Severity + - **high**: Will mislead users or break things (incorrect commands, wrong APIs, security issues) + - **medium**: Could confuse users or violates style guide (AI-isms, scope inflation, unclear instructions) + - **low**: Minor suggestions (rarely report) - OLD_CACHES=$(gh api "repos/${{ github.repository }}/actions/caches" \ - --jq "[.actions_caches | map(select(.key | startswith(\"$CACHE_PREFIX\"))) | sort_by(.created_at) | reverse | .[5:] | .[].id] | .[]" \ - 2>/dev/null || echo "") + Most issues should be MEDIUM. HIGH is rare. - if [ -z "$OLD_CACHES" ]; then - echo "✅ No old caches to clean up" - exit 0 - fi - - DELETED=0 - for CACHE_ID in $OLD_CACHES; do - if gh api "repos/${{ github.repository }}/actions/caches/$CACHE_ID" -X DELETE 2>/dev/null; then - ((DELETED++)) - fi - done - - echo "✅ Deleted $DELETED old cache(s)" - - # ========================================================================== - # MANUAL REVIEW PIPELINE - # Triggers when someone comments /review on a PR - # Only runs for Docker org members - # ========================================================================== - run-review: - if: github.event.issue.pull_request && contains(github.event.comment.body, '/review') - runs-on: ubuntu-latest - - steps: - - name: Check if commenter is Docker org member - id: membership - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 - with: - github-token: ${{ secrets.ORG_MEMBERSHIP_TOKEN }} - script: | - const org = 'docker'; - const username = context.payload.comment.user.login; - - try { - await github.rest.orgs.checkMembershipForUser({ - org: org, - username: username - }); - core.setOutput('is_member', 'true'); - console.log(`✅ ${username} is a Docker org member - proceeding with review`); - } catch (error) { - if (error.status === 404 || error.status === 302) { - core.setOutput('is_member', 'false'); - console.log(`⏭️ ${username} is not a Docker org member - ignoring /review command`); - } else if (error.status === 401) { - core.setFailed( - '❌ ORG_MEMBERSHIP_TOKEN secret is missing or invalid.\n\n' + - 'This secret is required to check Docker org membership.\n\n' + - 'To fix this:\n' + - '1. Create a classic PAT with read:org scope at https://github.com/settings/tokens/new\n' + - '2. Add it as a repository secret named ORG_MEMBERSHIP_TOKEN:\n' + - ' gh secret set ORG_MEMBERSHIP_TOKEN --repo docker/docs' - ); - } else { - core.setFailed(`Failed to check org membership: ${error.message}`); - } - } - - - name: Checkout repository - if: steps.membership.outputs.is_member == 'true' - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 - with: - fetch-depth: 0 - - - name: Restore trusted agent config from base branch - if: steps.membership.outputs.is_member == 'true' - run: | - # Get the PR's base branch and ensure agent config comes from there - PR_BASE=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.issue.number }} --jq '.base.ref') - git fetch origin "$PR_BASE" - git checkout "origin/$PR_BASE" -- .github/pr-reviewer.yml - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Restore reviewer memory - if: steps.membership.outputs.is_member == 'true' - uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 - with: - path: .github/pr-review-memory.db - key: pr-review-memory-${{ github.repository }}-${{ github.run_id }} - restore-keys: | - pr-review-memory-${{ github.repository }}- - - - name: Run Documentation PR Review - if: steps.membership.outputs.is_member == 'true' - uses: docker/cagent-action@latest - with: - cagent-version: v1.22.0 - agent: .github/pr-reviewer.yml - prompt: | - Review PR #${{ github.event.issue.number }} - - Triggered manually by @${{ github.event.comment.user.login }} - - Execute the review pipeline as documented in your instructions. - anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} - - - name: Save reviewer memory - if: steps.membership.outputs.is_member == 'true' && always() - uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 - with: - path: .github/pr-review-memory.db - key: pr-review-memory-${{ github.repository }}-${{ github.run_id }} - - - name: Clean up old memory caches - if: steps.membership.outputs.is_member == 'true' && always() - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - CACHE_PREFIX="pr-review-memory-${{ github.repository }}-" - - echo "🧹 Cleaning up old memory caches (keeping 5 most recent)" - - OLD_CACHES=$(gh api "repos/${{ github.repository }}/actions/caches" \ - --jq "[.actions_caches | map(select(.key | startswith(\"$CACHE_PREFIX\"))) | sort_by(.created_at) | reverse | .[5:] | .[].id] | .[]" \ - 2>/dev/null || echo "") - - if [ -z "$OLD_CACHES" ]; then - echo "✅ No old caches to clean up" - exit 0 - fi - - DELETED=0 - for CACHE_ID in $OLD_CACHES; do - if gh api "repos/${{ github.repository }}/actions/caches/$CACHE_ID" -X DELETE 2>/dev/null; then - ((DELETED++)) - fi - done - - echo "✅ Deleted $DELETED old cache(s)" - - # ========================================================================== - # LEARN FROM FEEDBACK - # Processes replies to agent review comments for continuous improvement - # ========================================================================== - learn-from-feedback: - if: github.event_name == 'pull_request_review_comment' && github.event.comment.in_reply_to_id - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 - - - name: Learn from user feedback - uses: docker/cagent-action/review-pr/learn@latest - with: - anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + ## Review Events + - **COMMENT** (default) - For all findings, minor or medium issues + - **REQUEST_CHANGES** (rare) - Only for critical factual errors that will harm users + - Never use APPROVE - provide feedback but don't count as approval