From cb17657b39432a27ae672f5ac6abedd2acb774f5 Mon Sep 17 00:00:00 2001 From: Andy Ye <35905412+TurboTheTurtle@users.noreply.github.com> Date: Wed, 1 Jul 2026 12:29:54 -0700 Subject: [PATCH 1/2] Add bounded PR detail fill command --- internal/cli/app.go | 206 +++++++++++++++++++++++++++ internal/cli/app_test.go | 129 +++++++++++++++++ internal/store/pull_request_fill.go | 61 ++++++++ internal/store/pull_requests_test.go | 56 ++++++++ 4 files changed, 452 insertions(+) create mode 100644 internal/store/pull_request_fill.go diff --git a/internal/cli/app.go b/internal/cli/app.go index aad709f..5835360 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -157,6 +157,8 @@ func (a *App) Run(ctx context.Context, args []string) error { return a.runStatus(ctx, rest[1:]) case "sync": return a.runSync(ctx, rest[1:]) + case "fill-pr-details": + return a.runFillPRDetails(ctx, rest[1:]) case "threads": return a.runThreads(ctx, rest[1:]) case "close-thread": @@ -2674,6 +2676,204 @@ type syncOptions struct { Quiet bool } +type fillPRDetailsResult struct { + Repository string `json:"repository"` + Selected int `json:"selected"` + Filled int `json:"filled"` + Remaining int `json:"remaining"` + Numbers []int `json:"numbers,omitempty"` + Order string `json:"order"` + Limit int `json:"limit,omitempty"` + BatchSize int `json:"batch_size"` + StoppedReason string `json:"stopped_reason,omitempty"` + RateLimit *fillRateLimitResult `json:"rate_limit,omitempty"` + StartedAt string `json:"started_at"` + FinishedAt string `json:"finished_at"` + Batches []fillPRDetailsBatch `json:"batches,omitempty"` + IncludeComments bool `json:"include_comments,omitempty"` + JSONProgress bool `json:"json_progress,omitempty"` + ReserveRateLimit int `json:"reserve_rate_limit,omitempty"` +} + +type fillPRDetailsBatch struct { + Index int `json:"index"` + Numbers []int `json:"numbers"` + PullRequestsSynced int `json:"pull_requests_synced"` + PRDetailsSynced int `json:"pr_details_synced"` + RateLimit *fillRateLimitResult `json:"rate_limit,omitempty"` +} + +type fillRateLimitResult struct { + Host string `json:"host,omitempty"` + Resource string `json:"resource,omitempty"` + Limit int `json:"limit,omitempty"` + Remaining int `json:"remaining"` + ResetAt string `json:"reset_at,omitempty"` + Low bool `json:"low,omitempty"` +} + +func (a *App) runFillPRDetails(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("fill-pr-details", flag.ContinueOnError) + fs.SetOutput(io.Discard) + limitRaw := fs.String("limit", "", "maximum missing PR details to hydrate") + order := fs.String("order", "newest-first", "missing PR order: newest-first|oldest-first|open-first") + batchSizeRaw := fs.String("batch-size", "50", "PRs to hydrate per sync batch") + reserveRaw := fs.String("reserve-rate-limit", "", "stop before starting a batch when remaining GitHub quota is at or below this value") + jsonProgress := fs.Bool("json-progress", false, "write newline JSON progress events to stderr") + includeComments := fs.Bool("include-comments", false, "also hydrate issue comments and PR reviews while filling details") + jsonOut := fs.Bool("json", false, "write JSON output") + if err := fs.Parse(normalizeCommandArgs(args, map[string]bool{"limit": true, "order": true, "batch-size": true, "reserve-rate-limit": true})); err != nil { + return usageErr(err) + } + a.applyCommandJSON(*jsonOut) + if fs.NArg() != 1 { + return usageErr(fmt.Errorf("fill-pr-details requires owner/repo")) + } + owner, repoName, err := parseOwnerRepo(fs.Arg(0)) + if err != nil { + return usageErr(err) + } + limit, err := parseOptionalPositiveInt(*limitRaw) + if err != nil { + return usageErr(err) + } + batchSize, err := parseOptionalPositiveInt(*batchSizeRaw) + if err != nil { + return usageErr(err) + } + if batchSize <= 0 { + batchSize = 50 + } + reserve, err := parseOptionalPositiveInt(*reserveRaw) + if err != nil { + return usageErr(err) + } + startedAt := time.Now().UTC().Format(time.RFC3339Nano) + rt, err := a.openLocalRuntime(ctx) + if err != nil { + return err + } + repo, err := rt.repository(ctx, owner, repoName) + if err != nil { + _ = rt.Store.Close() + return fmt.Errorf("repository %s/%s is not in the local archive; run gitcrawl sync first: %w", owner, repoName, err) + } + numbers, err := rt.Store.MissingPullRequestDetailNumbers(ctx, repo.ID, store.MissingPullRequestDetailOptions{Limit: limit, Order: *order}) + if closeErr := rt.Store.Close(); err == nil && closeErr != nil { + err = closeErr + } + if err != nil { + return err + } + result := fillPRDetailsResult{ + Repository: repo.FullName, + Selected: len(numbers), + Numbers: numbers, + Order: strings.TrimSpace(*order), + Limit: limit, + BatchSize: batchSize, + StartedAt: startedAt, + IncludeComments: *includeComments, + JSONProgress: *jsonProgress, + ReserveRateLimit: reserve, + } + for i := 0; i < len(numbers); i += batchSize { + if reserve > 0 { + if rate, ok := a.currentFillRateLimit(ctx); ok && rate.Remaining <= reserve { + result.StoppedReason = "rate-limit-reserve" + result.RateLimit = &rate + break + } + } + end := i + batchSize + if end > len(numbers) { + end = len(numbers) + } + batchNumbers := append([]int(nil), numbers[i:end]...) + if *jsonProgress { + a.writeFillPRDetailsProgress(fillPRDetailsProgressEvent{ + Event: "batch_start", + Repository: repo.FullName, + Batch: len(result.Batches) + 1, + Numbers: batchNumbers, + }) + } + stats, err := a.syncRepository(ctx, owner, repoName, syncOptions{ + Numbers: batchNumbers, + IncludeComments: *includeComments, + IncludePRDetails: true, + Quiet: *jsonProgress, + }) + if err != nil { + return err + } + rate, hasRate := a.currentFillRateLimit(ctx) + batch := fillPRDetailsBatch{ + Index: len(result.Batches) + 1, + Numbers: batchNumbers, + PullRequestsSynced: stats.PullRequestsSynced, + PRDetailsSynced: stats.PRDetailsSynced, + } + if hasRate { + batch.RateLimit = &rate + result.RateLimit = &rate + } + result.Batches = append(result.Batches, batch) + result.Filled += stats.PRDetailsSynced + if *jsonProgress { + a.writeFillPRDetailsProgress(fillPRDetailsProgressEvent{ + Event: "batch_done", + Repository: repo.FullName, + Batch: batch.Index, + Numbers: batchNumbers, + Filled: result.Filled, + RateLimit: batch.RateLimit, + }) + } + } + result.Remaining = result.Selected - result.Filled + if result.Remaining < 0 { + result.Remaining = 0 + } + result.FinishedAt = time.Now().UTC().Format(time.RFC3339Nano) + return a.writeOutput("fill-pr-details", result, true) +} + +type fillPRDetailsProgressEvent struct { + Event string `json:"event"` + Repository string `json:"repository"` + Batch int `json:"batch"` + Numbers []int `json:"numbers"` + Filled int `json:"filled,omitempty"` + RateLimit *fillRateLimitResult `json:"rate_limit,omitempty"` +} + +func (a *App) writeFillPRDetailsProgress(event fillPRDetailsProgressEvent) { + data, err := json.Marshal(event) + if err != nil { + return + } + fmt.Fprintln(a.Stderr, string(data)) +} + +func (a *App) currentFillRateLimit(ctx context.Context) (fillRateLimitResult, bool) { + state, ok := a.sharedRateLimitState(ctx) + if !ok { + return fillRateLimitResult{}, false + } + result := fillRateLimitResult{ + Host: state.Host, + Resource: state.Resource, + Limit: state.Limit, + Remaining: state.Remaining, + Low: state.Low, + } + if !state.ResetAt.IsZero() { + result.ResetAt = state.ResetAt.Format(time.RFC3339) + } + return result, true +} + func parseSyncWith(value string) (map[string]bool, error) { out := map[string]bool{} for _, part := range strings.Split(value, ",") { @@ -4323,6 +4523,7 @@ Core commands: init create config, optionally from a portable store doctor check config, token, and database readiness sync sync GitHub issue and pull request metadata + fill-pr-details hydrate locally missing pull request detail rows refresh run sync, enrichment, embedding, and clustering pipeline embed generate OpenAI embeddings for local thread documents threads list local issue and pull request rows @@ -4406,6 +4607,11 @@ Usage: Usage: gitcrawl sync owner/repo [--state open|closed|all] [--numbers refs] [--with pr-details] [--include-pr-details] [--json] +`, + "fill-pr-details": `gitcrawl fill-pr-details hydrates locally missing pull request detail rows. + +Usage: + gitcrawl fill-pr-details owner/repo [--limit N] [--order newest-first|oldest-first|open-first] [--batch-size N] [--reserve-rate-limit N] [--json-progress] [--json] `, "refresh": `gitcrawl refresh runs sync, enrichment, embedding, and clustering. diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index e901d5a..1cb6dd5 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -3193,6 +3193,135 @@ func TestSyncCommandUsesConfiguredGitHubBaseURLAndHydratesComments(t *testing.T) } } +func TestFillPRDetailsHydratesMissingPullRequestDetails(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + init := New() + if err := init.Run(ctx, []string{"--config", configPath, "init", "--db", dbPath}); err != nil { + t.Fatalf("init: %v", err) + } + st, err := store.Open(ctx, dbPath) + if err != nil { + t.Fatalf("open store: %v", err) + } + now := "2026-06-20T12:00:00Z" + repoID, err := st.UpsertRepository(ctx, store.Repository{Owner: "openclaw", Name: "gitcrawl", FullName: "openclaw/gitcrawl", RawJSON: "{}", UpdatedAt: now}) + if err != nil { + t.Fatalf("repo: %v", err) + } + if _, err := st.UpsertThread(ctx, store.Thread{ + RepoID: repoID, GitHubID: "102", Number: 102, Kind: "pull_request", State: "open", + Title: "Fill details", HTMLURL: "https://github.com/openclaw/gitcrawl/pull/102", + LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "h102", + UpdatedAtGitHub: now, UpdatedAt: now, + }); err != nil { + t.Fatalf("thread: %v", err) + } + if err := st.Close(); err != nil { + t.Fatalf("close seed store: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "Bearer test-gh-token" { + t.Fatalf("authorization = %q", got) + } + w.Header().Set("X-RateLimit-Limit", "5000") + w.Header().Set("X-RateLimit-Remaining", "4990") + w.Header().Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(time.Hour).Unix(), 10)) + switch r.URL.Path { + case "/repos/openclaw/gitcrawl": + _ = json.NewEncoder(w).Encode(map[string]any{"id": 12345, "full_name": "openclaw/gitcrawl"}) + case "/repos/openclaw/gitcrawl/issues/102": + row := githubIssueJSON(102, "pull_request", "Fill details") + row["html_url"] = "https://github.com/openclaw/gitcrawl/pull/102" + row["pull_request"] = map[string]any{"url": githubServerURL(r) + "/repos/openclaw/gitcrawl/pulls/102"} + _ = json.NewEncoder(w).Encode(row) + case "/graphql": + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": map[string]any{ + "repository": map[string]any{ + "pullRequest": map[string]any{ + "reviewThreads": map[string]any{ + "nodes": []map[string]any{}, + "pageInfo": map[string]any{"hasNextPage": false, "endCursor": ""}, + }, + }, + }, + }, + }) + case "/repos/openclaw/gitcrawl/pulls/102": + _ = json.NewEncoder(w).Encode(map[string]any{ + "number": 102, + "base": map[string]any{"sha": "base-sha"}, + "head": map[string]any{"sha": "head-sha", "ref": "fill-details", "repo": map[string]any{"full_name": "openclaw/gitcrawl"}}, + "mergeable_state": "clean", + "additions": 3, + "deletions": 1, + "changed_files": 1, + }) + case "/repos/openclaw/gitcrawl/pulls/102/files": + _ = json.NewEncoder(w).Encode([]map[string]any{{"filename": "internal/cli/app.go", "status": "modified", "additions": 3, "deletions": 1, "changes": 4}}) + case "/repos/openclaw/gitcrawl/pulls/102/commits": + _ = json.NewEncoder(w).Encode([]map[string]any{{"sha": "commit-sha", "commit": map[string]any{"message": "feat: fill details", "author": map[string]any{"name": "Andy", "date": now}}, "html_url": "https://github.com/openclaw/gitcrawl/commit/commit-sha"}}) + case "/repos/openclaw/gitcrawl/commits/head-sha/check-runs": + _ = json.NewEncoder(w).Encode(map[string]any{"check_runs": []map[string]any{{"name": "CI", "status": "completed", "conclusion": "success", "details_url": "https://example.invalid/check"}}}) + case "/repos/openclaw/gitcrawl/actions/runs": + if r.URL.Query().Get("head_sha") != "head-sha" { + t.Fatalf("workflow runs query = %s", r.URL.RawQuery) + } + _ = json.NewEncoder(w).Encode(map[string]any{"workflow_runs": []map[string]any{{"id": 99, "run_number": 7, "head_sha": "head-sha", "status": "completed", "conclusion": "success", "name": "CI", "html_url": "https://example.invalid/run"}}}) + default: + t.Fatalf("unexpected GitHub path: %s", r.URL.String()) + } + })) + defer server.Close() + t.Setenv("GITHUB_TOKEN", "test-gh-token") + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + + run := New() + var stdout, stderr bytes.Buffer + run.Stdout = &stdout + run.Stderr = &stderr + if err := run.Run(ctx, []string{"--config", configPath, "fill-pr-details", "openclaw/gitcrawl", "--limit", "1", "--batch-size", "1", "--json-progress", "--json"}); err != nil { + t.Fatalf("fill-pr-details: %v\nstderr=%s", err, stderr.String()) + } + var result struct { + Selected int `json:"selected"` + Filled int `json:"filled"` + } + if err := json.Unmarshal(stdout.Bytes(), &result); err != nil { + t.Fatalf("decode fill result: %v\n%s", err, stdout.String()) + } + if result.Selected != 1 || result.Filled != 1 { + t.Fatalf("fill result = %+v", result) + } + if !strings.Contains(stderr.String(), `"event":"batch_start"`) || !strings.Contains(stderr.String(), `"event":"batch_done"`) { + t.Fatalf("missing json progress events: %s", stderr.String()) + } + st, err = store.Open(ctx, dbPath) + if err != nil { + t.Fatalf("reopen store: %v", err) + } + defer st.Close() + cache, err := st.PullRequestCache(ctx, repoID, 102) + if err != nil { + t.Fatalf("pull request cache: %v", err) + } + if cache.Detail.HeadSHA != "head-sha" || len(cache.Files) != 1 || len(cache.Commits) != 1 || len(cache.Checks) != 1 { + t.Fatalf("cache after fill = %+v", cache) + } +} + +func githubServerURL(r *http.Request) string { + scheme := "http" + if r.TLS != nil { + scheme = "https" + } + return scheme + "://" + r.Host +} + func TestRefreshRunsSyncEmbedAndClusterWithLocalServers(t *testing.T) { ctx := context.Background() dir := t.TempDir() diff --git a/internal/store/pull_request_fill.go b/internal/store/pull_request_fill.go new file mode 100644 index 0000000..7973ac0 --- /dev/null +++ b/internal/store/pull_request_fill.go @@ -0,0 +1,61 @@ +package store + +import ( + "context" + "fmt" + "strings" +) + +type MissingPullRequestDetailOptions struct { + Limit int + Order string +} + +func (s *Store) MissingPullRequestDetailNumbers(ctx context.Context, repoID int64, options MissingPullRequestDetailOptions) ([]int, error) { + orderClause, err := missingPullRequestDetailOrderClause(options.Order) + if err != nil { + return nil, err + } + limit := options.Limit + if limit <= 0 { + limit = -1 + } + rows, err := s.q().QueryContext(ctx, ` +select t.number +from threads t +left join pull_request_details d on d.thread_id = t.id +where t.repo_id = ? + and t.kind = 'pull_request' + and d.thread_id is null +order by `+orderClause+` +limit ?`, repoID, limit) + if err != nil { + return nil, fmt.Errorf("list missing pull request details: %w", err) + } + defer rows.Close() + var out []int + for rows.Next() { + var number int + if err := rows.Scan(&number); err != nil { + return nil, fmt.Errorf("scan missing pull request detail number: %w", err) + } + out = append(out, number) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("scan missing pull request detail numbers: %w", err) + } + return out, nil +} + +func missingPullRequestDetailOrderClause(order string) (string, error) { + switch strings.TrimSpace(strings.ToLower(order)) { + case "", "newest-first": + return "coalesce(t.updated_at_gh, t.updated_at, '') desc, t.number desc", nil + case "oldest-first": + return "coalesce(t.updated_at_gh, t.updated_at, '') asc, t.number asc", nil + case "open-first": + return "case when t.state = 'open' then 0 else 1 end, coalesce(t.updated_at_gh, t.updated_at, '') desc, t.number desc", nil + default: + return "", fmt.Errorf("unsupported fill-pr-details order %q", order) + } +} diff --git a/internal/store/pull_requests_test.go b/internal/store/pull_requests_test.go index 2f98211..4594a86 100644 --- a/internal/store/pull_requests_test.go +++ b/internal/store/pull_requests_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "path/filepath" "testing" + "time" ) func TestPullRequestCacheRoundTripAndWorkflowFilters(t *testing.T) { @@ -94,6 +95,61 @@ func TestPullRequestCacheRoundTripAndWorkflowFilters(t *testing.T) { } } +func TestMissingPullRequestDetailNumbersSelectsOnlyUnfilledPRs(t *testing.T) { + ctx := context.Background() + st, err := Open(ctx, filepath.Join(t.TempDir(), "gitcrawl.db")) + if err != nil { + t.Fatalf("open store: %v", err) + } + defer st.Close() + now := time.Date(2026, 6, 20, 12, 0, 0, 0, time.UTC).Format(time.RFC3339Nano) + repoID, err := st.UpsertRepository(ctx, Repository{Owner: "openclaw", Name: "gitcrawl", FullName: "openclaw/gitcrawl", RawJSON: "{}", UpdatedAt: now}) + if err != nil { + t.Fatalf("repo: %v", err) + } + threads := []Thread{ + {RepoID: repoID, GitHubID: "11", Number: 11, Kind: "pull_request", State: "closed", Title: "old missing", HTMLURL: "https://github.com/openclaw/gitcrawl/pull/11", LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "h11", UpdatedAtGitHub: "2026-06-01T00:00:00Z", UpdatedAt: now}, + {RepoID: repoID, GitHubID: "12", Number: 12, Kind: "pull_request", State: "open", Title: "new missing", HTMLURL: "https://github.com/openclaw/gitcrawl/pull/12", LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "h12", UpdatedAtGitHub: "2026-06-03T00:00:00Z", UpdatedAt: now}, + {RepoID: repoID, GitHubID: "13", Number: 13, Kind: "pull_request", State: "open", Title: "already filled", HTMLURL: "https://github.com/openclaw/gitcrawl/pull/13", LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "h13", UpdatedAtGitHub: "2026-06-04T00:00:00Z", UpdatedAt: now}, + {RepoID: repoID, GitHubID: "14", Number: 14, Kind: "issue", State: "open", Title: "plain issue", HTMLURL: "https://github.com/openclaw/gitcrawl/issues/14", LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "h14", UpdatedAtGitHub: "2026-06-05T00:00:00Z", UpdatedAt: now}, + } + var filledThreadID int64 + for _, thread := range threads { + id, err := st.UpsertThread(ctx, thread) + if err != nil { + t.Fatalf("thread %d: %v", thread.Number, err) + } + if thread.Number == 13 { + filledThreadID = id + } + } + if err := st.UpsertPullRequestCache(ctx, PullRequestDetail{ThreadID: filledThreadID, RepoID: repoID, Number: 13, RawJSON: "{}", FetchedAt: now, UpdatedAt: now}, nil, nil, nil, nil); err != nil { + t.Fatalf("fill pr detail: %v", err) + } + + got, err := st.MissingPullRequestDetailNumbers(ctx, repoID, MissingPullRequestDetailOptions{Order: "newest-first"}) + if err != nil { + t.Fatalf("missing newest: %v", err) + } + if len(got) != 2 || got[0] != 12 || got[1] != 11 { + t.Fatalf("newest missing = %v", got) + } + got, err = st.MissingPullRequestDetailNumbers(ctx, repoID, MissingPullRequestDetailOptions{Order: "oldest-first", Limit: 1}) + if err != nil { + t.Fatalf("missing oldest: %v", err) + } + if len(got) != 1 || got[0] != 11 { + t.Fatalf("oldest limited missing = %v", got) + } + got, err = st.MissingPullRequestDetailNumbers(ctx, repoID, MissingPullRequestDetailOptions{Order: "open-first"}) + if err != nil { + t.Fatalf("missing open-first: %v", err) + } + if len(got) != 2 || got[0] != 12 || got[1] != 11 { + t.Fatalf("open-first missing = %v", got) + } +} + func TestPullRequestCacheAllowsDuplicateFilePaths(t *testing.T) { ctx := context.Background() st, err := Open(ctx, filepath.Join(t.TempDir(), "gitcrawl.db")) From e0621ab1cb09c29c1e620eb3d549e41f103d29df Mon Sep 17 00:00:00 2001 From: Andy Ye <35905412+TurboTheTurtle@users.noreply.github.com> Date: Wed, 1 Jul 2026 12:54:13 -0700 Subject: [PATCH 2/2] Ignore stale fill rate limit cache --- internal/cli/app.go | 13 +++++ internal/cli/app_test.go | 110 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 122 insertions(+), 1 deletion(-) diff --git a/internal/cli/app.go b/internal/cli/app.go index 5835360..f40df51 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -2861,6 +2861,9 @@ func (a *App) currentFillRateLimit(ctx context.Context) (fillRateLimitResult, bo if !ok { return fillRateLimitResult{}, false } + if !fillRateLimitStateFresh(state, time.Now().UTC()) { + return fillRateLimitResult{}, false + } result := fillRateLimitResult{ Host: state.Host, Resource: state.Resource, @@ -2874,6 +2877,16 @@ func (a *App) currentFillRateLimit(ctx context.Context) (fillRateLimitResult, bo return result, true } +func fillRateLimitStateFresh(state ghSharedRateLimitState, now time.Time) bool { + if !state.ResetAt.IsZero() && !now.Before(state.ResetAt) { + return false + } + if !state.UpdatedAt.IsZero() && now.Sub(state.UpdatedAt) > ghRateLimitStateMaxAge() { + return false + } + return true +} + func parseSyncWith(value string) (map[string]bool, error) { out := map[string]bool{} for _, part := range strings.Split(value, ",") { diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 1cb6dd5..12087c8 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -3198,10 +3198,12 @@ func TestFillPRDetailsHydratesMissingPullRequestDetails(t *testing.T) { dir := t.TempDir() configPath := filepath.Join(dir, "config.toml") dbPath := filepath.Join(dir, "gitcrawl.db") + cacheDir := filepath.Join(dir, "cache") init := New() if err := init.Run(ctx, []string{"--config", configPath, "init", "--db", dbPath}); err != nil { t.Fatalf("init: %v", err) } + configureTestGitHubCache(t, configPath, cacheDir, "test-gh-token") st, err := store.Open(ctx, dbPath) if err != nil { t.Fatalf("open store: %v", err) @@ -3279,12 +3281,14 @@ func TestFillPRDetailsHydratesMissingPullRequestDetails(t *testing.T) { defer server.Close() t.Setenv("GITHUB_TOKEN", "test-gh-token") t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + t.Setenv("GITCRAWL_GH_RATE_LIMIT_MAX_AGE", "1m") + writeFillPRDetailsRateLimitState(t, cacheDir, "test-gh-token", 1, time.Now().Add(-time.Minute), time.Now().Add(-time.Hour)) run := New() var stdout, stderr bytes.Buffer run.Stdout = &stdout run.Stderr = &stderr - if err := run.Run(ctx, []string{"--config", configPath, "fill-pr-details", "openclaw/gitcrawl", "--limit", "1", "--batch-size", "1", "--json-progress", "--json"}); err != nil { + if err := run.Run(ctx, []string{"--config", configPath, "fill-pr-details", "openclaw/gitcrawl", "--limit", "1", "--batch-size", "1", "--reserve-rate-limit", "10", "--json-progress", "--json"}); err != nil { t.Fatalf("fill-pr-details: %v\nstderr=%s", err, stderr.String()) } var result struct { @@ -3314,6 +3318,110 @@ func TestFillPRDetailsHydratesMissingPullRequestDetails(t *testing.T) { } } +func TestFillPRDetailsReserveRateLimitStopsOnFreshLowCache(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + cacheDir := filepath.Join(dir, "cache") + init := New() + if err := init.Run(ctx, []string{"--config", configPath, "init", "--db", dbPath}); err != nil { + t.Fatalf("init: %v", err) + } + configureTestGitHubCache(t, configPath, cacheDir, "test-gh-token") + st, err := store.Open(ctx, dbPath) + if err != nil { + t.Fatalf("open store: %v", err) + } + now := "2026-06-20T12:00:00Z" + repoID, err := st.UpsertRepository(ctx, store.Repository{Owner: "openclaw", Name: "gitcrawl", FullName: "openclaw/gitcrawl", RawJSON: "{}", UpdatedAt: now}) + if err != nil { + t.Fatalf("repo: %v", err) + } + if _, err := st.UpsertThread(ctx, store.Thread{ + RepoID: repoID, GitHubID: "103", Number: 103, Kind: "pull_request", State: "open", + Title: "Fill details", HTMLURL: "https://github.com/openclaw/gitcrawl/pull/103", + LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "h103", + UpdatedAtGitHub: now, UpdatedAt: now, + }); err != nil { + t.Fatalf("thread: %v", err) + } + if err := st.Close(); err != nil { + t.Fatalf("close seed store: %v", err) + } + + t.Setenv("GITHUB_TOKEN", "test-gh-token") + t.Setenv("GITCRAWL_GH_RATE_LIMIT_MAX_AGE", "1m") + writeFillPRDetailsRateLimitState(t, cacheDir, "test-gh-token", 5, time.Now().Add(time.Hour), time.Now()) + + run := New() + var stdout bytes.Buffer + run.Stdout = &stdout + if err := run.Run(ctx, []string{"--config", configPath, "fill-pr-details", "openclaw/gitcrawl", "--limit", "1", "--reserve-rate-limit", "10", "--json"}); err != nil { + t.Fatalf("fill-pr-details: %v", err) + } + var result struct { + Selected int `json:"selected"` + Filled int `json:"filled"` + Remaining int `json:"remaining"` + StoppedReason string `json:"stopped_reason"` + RateLimit *fillRateLimitResult `json:"rate_limit"` + } + if err := json.Unmarshal(stdout.Bytes(), &result); err != nil { + t.Fatalf("decode fill result: %v\n%s", err, stdout.String()) + } + if result.Selected != 1 || result.Filled != 0 || result.Remaining != 1 || result.StoppedReason != "rate-limit-reserve" { + t.Fatalf("fill result = %+v", result) + } + if result.RateLimit == nil || result.RateLimit.Remaining != 5 { + t.Fatalf("rate limit = %+v", result.RateLimit) + } +} + +func configureTestGitHubCache(t *testing.T, configPath, cacheDir, token string) { + t.Helper() + cfg, err := config.Load(configPath) + if err != nil { + t.Fatalf("load config: %v", err) + } + cfg.CacheDir = cacheDir + if cfg.Env == nil { + cfg.Env = map[string]string{} + } + cfg.Env["GITHUB_TOKEN"] = token + if err := config.Save(configPath, cfg); err != nil { + t.Fatalf("save config: %v", err) + } +} + +func writeFillPRDetailsRateLimitState(t *testing.T, cacheDir, token string, remaining int, resetAt, updatedAt time.Time) { + t.Helper() + dir := filepath.Join(cacheDir, "octopool-migrated-gh") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatalf("create cache dir: %v", err) + } + state := ghSharedRateLimitState{ + Host: "github.com", + TokenHash: ghRateLimitTokenHash(token), + Limit: 5000, + Remaining: remaining, + ResetAt: resetAt.UTC(), + Resource: "core", + UpdatedAt: updatedAt.UTC(), + Source: "test", + Low: true, + Threshold: 10, + } + data, err := json.Marshal(state) + if err != nil { + t.Fatalf("marshal rate limit state: %v", err) + } + path := filepath.Join(dir, ghSharedRateLimitFilePrefix+"github.com_"+ghRateLimitTokenHash(token)+".json") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write rate limit state: %v", err) + } +} + func githubServerURL(r *http.Request) string { scheme := "http" if r.TLS != nil {