diff --git a/.github/scripts/sweep-recent-review-feedback.rb b/.github/scripts/sweep-recent-review-feedback.rb index cba1e19..fcf4408 100644 --- a/.github/scripts/sweep-recent-review-feedback.rb +++ b/.github/scripts/sweep-recent-review-feedback.rb @@ -126,9 +126,12 @@ def search_recent_prs(owner:, since:, limit: 100) JSON.parse(stdout) end - def feedback_items(owner:, since:, min_severity:, pr_limit: 100) - search_recent_prs(owner: owner, since: since, limit: pr_limit).flat_map do |pr| + def feedback_items(owner:, since:, min_severity:, pr_limit: 100, progress: false) + prs = search_recent_prs(owner: owner, since: since, limit: pr_limit) + warn "review feedback sweep: inspecting #{prs.length} merged PRs since #{since}" if progress + prs.each_with_index.flat_map do |pr, index| repo = pr.fetch("repository").fetch("nameWithOwner") + warn "review feedback sweep: #{index + 1}/#{prs.length} #{repo}##{pr.fetch("number")}" if progress payload = EvalOpsReviewThreadGuard.fetch_payload(repo: repo, pr: pr.fetch("number")) EvalOpsReviewThreadGuard.blocking_feedback(payload, min_severity: min_severity).map do |item| item.merge( @@ -925,6 +928,7 @@ def upsert_issue(repo:, title:, body:) guardrail_repo_issues: false, weekly_report_issue_repo: nil, weekly_report_issue_title: EvalOpsReviewFeedbackSweep::DEFAULT_WEEKLY_REPORT_TITLE, + progress: false, pr_limit: 100, dry_run: false } @@ -944,6 +948,7 @@ def upsert_issue(repo:, title:, body:) parser.on("--guardrail-lifecycle-json-output PATH", "Write guardrail issue lifecycle JSON to this path") { |value| options[:guardrail_lifecycle_json_output] = value } parser.on("--weekly-report-issue-repo OWNER/REPO", "Create or comment on this issue repo with the guardrail report") { |value| options[:weekly_report_issue_repo] = value } parser.on("--weekly-report-issue-title TITLE", "Issue title for the guardrail report") { |value| options[:weekly_report_issue_title] = value } + parser.on("--progress", "Print PR inspection progress to stderr") { options[:progress] = true } parser.on("--dry-run", "Print report and skip issue writes") { options[:dry_run] = true } end.parse! @@ -957,7 +962,8 @@ def upsert_issue(repo:, title:, body:) owner: options.fetch(:owner), since: since, min_severity: options.fetch(:min_severity), - pr_limit: options.fetch(:pr_limit) + pr_limit: options.fetch(:pr_limit), + progress: options.fetch(:progress) ) ledger = EvalOpsReviewFeedbackSweep.ledger_json( items, diff --git a/.github/workflows/review-feedback-backfill.yml b/.github/workflows/review-feedback-backfill.yml index cc2cbe5..3e8411b 100644 --- a/.github/workflows/review-feedback-backfill.yml +++ b/.github/workflows/review-feedback-backfill.yml @@ -16,7 +16,7 @@ on: pr_limit: description: "Maximum merged PRs to inspect" required: false - default: "1000" + default: "250" permissions: contents: read @@ -31,7 +31,7 @@ jobs: GH_TOKEN: ${{ secrets.EVALOPS_REVIEW_GUARD_TOKEN || secrets.EVALOPS_ORG_READ_TOKEN }} SINCE_HOURS: ${{ inputs.since_hours || '720' }} MIN_SEVERITY: ${{ inputs.min_severity || 'high' }} - PR_LIMIT: ${{ inputs.pr_limit || '1000' }} + PR_LIMIT: ${{ inputs.pr_limit || '250' }} steps: - uses: actions/checkout@v5 @@ -51,6 +51,7 @@ jobs: --since-hours "${SINCE_HOURS}" \ --pr-limit "${PR_LIMIT}" \ --min-severity "${MIN_SEVERITY}" \ + --progress \ --json-output review-feedback-30d-ledger.json \ --guardrail-backlog-output review-feedback-30d-guardrail-backlog.md \ --guardrail-backlog-json-output review-feedback-30d-guardrail-backlog.json \ diff --git a/test/sweep_recent_review_feedback_test.rb b/test/sweep_recent_review_feedback_test.rb index 90ce273..20c2716 100644 --- a/test/sweep_recent_review_feedback_test.rb +++ b/test/sweep_recent_review_feedback_test.rb @@ -86,6 +86,62 @@ def test_ledger_json_records_empty_sweeps assert_equal [], ledger.fetch("findings") end + def test_feedback_items_progress_reports_search_and_pr_numbers + prs = [ + { + "repository" => { + "nameWithOwner" => "evalops/deploy" + }, + "number" => 2390, + "title" => "fix parser", + "url" => "https://github.com/evalops/deploy/pull/2390", + "closedAt" => "2026-05-10T05:00:00Z" + } + ] + payload = { + "repository" => { + "pullRequest" => { + "reviewThreads" => { + "nodes" => [] + }, + "comments" => { + "nodes" => [] + }, + "reviews" => { + "nodes" => [] + } + } + } + } + original_search = EvalOpsReviewFeedbackSweep.method(:search_recent_prs) + original_fetch = EvalOpsReviewThreadGuard.method(:fetch_payload) + EvalOpsReviewFeedbackSweep.define_singleton_method(:search_recent_prs) { |owner:, since:, limit:| prs } + EvalOpsReviewThreadGuard.define_singleton_method(:fetch_payload) { |repo:, pr:| payload } + + _stdout, stderr = capture_io do + assert_equal( + [], + EvalOpsReviewFeedbackSweep.feedback_items( + owner: "evalops", + since: "2026-04-10", + min_severity: "high", + pr_limit: 10, + progress: true + ) + ) + end + + assert_includes stderr, "review feedback sweep: inspecting 1 merged PRs since 2026-04-10" + assert_includes stderr, "review feedback sweep: 1/1 evalops/deploy#2390" + ensure + EvalOpsReviewFeedbackSweep.define_singleton_method(:search_recent_prs) do |owner:, since:, limit: 100| + original_search.call(owner: owner, since: since, limit: limit) + end + EvalOpsReviewThreadGuard.define_singleton_method(:fetch_payload) do |repo:, pr:| + original_fetch.call(repo: repo, pr: pr) + end + end + def test_guardrail_backlog_ranks_recurring_feedback_classes ledger = { "schema_version" => "evalops.review_feedback_ledger.v1",