diff --git a/internal/cli/app.go b/internal/cli/app.go index 617df04..d616f2b 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -161,6 +161,8 @@ func (a *App) Run(ctx context.Context, args []string) error { return a.runStatus(ctx, rest[1:]) case "sync": return a.runSync(ctx, rest[1:]) + case "fill-pr-details": + return a.runFillPRDetails(ctx, rest[1:]) case "threads": return a.runThreads(ctx, rest[1:]) case "close-thread": @@ -2802,6 +2804,217 @@ type syncOptions struct { Quiet bool } +type fillPRDetailsResult struct { + Repository string `json:"repository"` + Selected int `json:"selected"` + Filled int `json:"filled"` + Remaining int `json:"remaining"` + Numbers []int `json:"numbers,omitempty"` + Order string `json:"order"` + Limit int `json:"limit,omitempty"` + BatchSize int `json:"batch_size"` + StoppedReason string `json:"stopped_reason,omitempty"` + RateLimit *fillRateLimitResult `json:"rate_limit,omitempty"` + StartedAt string `json:"started_at"` + FinishedAt string `json:"finished_at"` + Batches []fillPRDetailsBatch `json:"batches,omitempty"` + IncludeComments bool `json:"include_comments,omitempty"` + JSONProgress bool `json:"json_progress,omitempty"` + ReserveRateLimit int `json:"reserve_rate_limit,omitempty"` +} + +type fillPRDetailsBatch struct { + Index int `json:"index"` + Numbers []int `json:"numbers"` + PullRequestsSynced int `json:"pull_requests_synced"` + PRDetailsSynced int `json:"pr_details_synced"` + RateLimit *fillRateLimitResult `json:"rate_limit,omitempty"` +} + +type fillRateLimitResult struct { + Host string `json:"host,omitempty"` + Resource string `json:"resource,omitempty"` + Limit int `json:"limit,omitempty"` + Remaining int `json:"remaining"` + ResetAt string `json:"reset_at,omitempty"` + Low bool `json:"low,omitempty"` +} + +func (a *App) runFillPRDetails(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("fill-pr-details", flag.ContinueOnError) + fs.SetOutput(io.Discard) + limitRaw := fs.String("limit", "", "maximum missing PR details to hydrate") + order := fs.String("order", "newest-first", "missing PR order: newest-first|oldest-first|open-first") + batchSizeRaw := fs.String("batch-size", "50", "PRs to hydrate per sync batch") + reserveRaw := fs.String("reserve-rate-limit", "", "stop before starting a batch when remaining GitHub quota is at or below this value") + jsonProgress := fs.Bool("json-progress", false, "write newline JSON progress events to stderr") + includeComments := fs.Bool("include-comments", false, "also hydrate issue comments and PR reviews while filling details") + jsonOut := fs.Bool("json", false, "write JSON output") + if err := fs.Parse(normalizeCommandArgs(args, map[string]bool{"limit": true, "order": true, "batch-size": true, "reserve-rate-limit": true})); err != nil { + return usageErr(err) + } + a.applyCommandJSON(*jsonOut) + if fs.NArg() != 1 { + return usageErr(fmt.Errorf("fill-pr-details requires owner/repo")) + } + owner, repoName, err := parseOwnerRepo(fs.Arg(0)) + if err != nil { + return usageErr(err) + } + limit, err := parseOptionalPositiveInt(*limitRaw) + if err != nil { + return usageErr(err) + } + batchSize, err := parseOptionalPositiveInt(*batchSizeRaw) + if err != nil { + return usageErr(err) + } + if batchSize <= 0 { + batchSize = 50 + } + reserve, err := parseOptionalPositiveInt(*reserveRaw) + if err != nil { + return usageErr(err) + } + startedAt := time.Now().UTC().Format(time.RFC3339Nano) + rt, err := a.openLocalRuntime(ctx) + if err != nil { + return err + } + repo, err := rt.repository(ctx, owner, repoName) + if err != nil { + _ = rt.Store.Close() + return fmt.Errorf("repository %s/%s is not in the local archive; run gitcrawl sync first: %w", owner, repoName, err) + } + numbers, err := rt.Store.MissingPullRequestDetailNumbers(ctx, repo.ID, store.MissingPullRequestDetailOptions{Limit: limit, Order: *order}) + if closeErr := rt.Store.Close(); err == nil && closeErr != nil { + err = closeErr + } + if err != nil { + return err + } + result := fillPRDetailsResult{ + Repository: repo.FullName, + Selected: len(numbers), + Numbers: numbers, + Order: strings.TrimSpace(*order), + Limit: limit, + BatchSize: batchSize, + StartedAt: startedAt, + IncludeComments: *includeComments, + JSONProgress: *jsonProgress, + ReserveRateLimit: reserve, + } + for i := 0; i < len(numbers); i += batchSize { + if reserve > 0 { + if rate, ok := a.currentFillRateLimit(ctx); ok && rate.Remaining <= reserve { + result.StoppedReason = "rate-limit-reserve" + result.RateLimit = &rate + break + } + } + end := i + batchSize + if end > len(numbers) { + end = len(numbers) + } + batchNumbers := append([]int(nil), numbers[i:end]...) + if *jsonProgress { + a.writeFillPRDetailsProgress(fillPRDetailsProgressEvent{ + Event: "batch_start", + Repository: repo.FullName, + Batch: len(result.Batches) + 1, + Numbers: batchNumbers, + }) + } + stats, err := a.syncRepository(ctx, owner, repoName, syncOptions{ + Numbers: batchNumbers, + IncludeComments: *includeComments, + IncludePRDetails: true, + Quiet: *jsonProgress, + }) + if err != nil { + return err + } + rate, hasRate := a.currentFillRateLimit(ctx) + batch := fillPRDetailsBatch{ + Index: len(result.Batches) + 1, + Numbers: batchNumbers, + PullRequestsSynced: stats.PullRequestsSynced, + PRDetailsSynced: stats.PRDetailsSynced, + } + if hasRate { + batch.RateLimit = &rate + result.RateLimit = &rate + } + result.Batches = append(result.Batches, batch) + result.Filled += stats.PRDetailsSynced + if *jsonProgress { + a.writeFillPRDetailsProgress(fillPRDetailsProgressEvent{ + Event: "batch_done", + Repository: repo.FullName, + Batch: batch.Index, + Numbers: batchNumbers, + Filled: result.Filled, + RateLimit: batch.RateLimit, + }) + } + } + result.Remaining = result.Selected - result.Filled + if result.Remaining < 0 { + result.Remaining = 0 + } + result.FinishedAt = time.Now().UTC().Format(time.RFC3339Nano) + return a.writeOutput("fill-pr-details", result, true) +} + +type fillPRDetailsProgressEvent struct { + Event string `json:"event"` + Repository string `json:"repository"` + Batch int `json:"batch"` + Numbers []int `json:"numbers"` + Filled int `json:"filled,omitempty"` + RateLimit *fillRateLimitResult `json:"rate_limit,omitempty"` +} + +func (a *App) writeFillPRDetailsProgress(event fillPRDetailsProgressEvent) { + data, err := json.Marshal(event) + if err != nil { + return + } + fmt.Fprintln(a.Stderr, string(data)) +} + +func (a *App) currentFillRateLimit(ctx context.Context) (fillRateLimitResult, bool) { + state, ok := a.sharedRateLimitState(ctx) + if !ok { + return fillRateLimitResult{}, false + } + if !fillRateLimitStateFresh(state, time.Now().UTC()) { + return fillRateLimitResult{}, false + } + result := fillRateLimitResult{ + Host: state.Host, + Resource: state.Resource, + Limit: state.Limit, + Remaining: state.Remaining, + Low: state.Low, + } + if !state.ResetAt.IsZero() { + result.ResetAt = state.ResetAt.Format(time.RFC3339) + } + return result, true +} + +func fillRateLimitStateFresh(state ghSharedRateLimitState, now time.Time) bool { + if !state.ResetAt.IsZero() && !now.Before(state.ResetAt) { + return false + } + if !state.UpdatedAt.IsZero() && now.Sub(state.UpdatedAt) > ghRateLimitStateMaxAge() { + return false + } + return true +} + func parseSyncWith(value string) (map[string]bool, error) { out := map[string]bool{} for _, part := range strings.Split(value, ",") { @@ -4553,6 +4766,7 @@ Core commands: init create config, optionally from a portable store doctor check config, token, and database readiness sync sync GitHub issue and pull request metadata + fill-pr-details hydrate locally missing pull request detail rows coverage report local archive PR-detail completeness refresh run sync, enrichment, embedding, and clustering pipeline embed generate OpenAI embeddings for local thread documents @@ -4637,6 +4851,11 @@ Usage: Usage: gitcrawl sync owner/repo [--state open|closed|all] [--numbers refs] [--with pr-details] [--include-pr-details] [--json] +`, + "fill-pr-details": `gitcrawl fill-pr-details hydrates locally missing pull request detail rows. + +Usage: + gitcrawl fill-pr-details owner/repo [--limit N] [--order newest-first|oldest-first|open-first] [--batch-size N] [--reserve-rate-limit N] [--json-progress] [--json] `, "coverage": `gitcrawl coverage reports local archive completeness by repository. diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 7b98030..1f2f1f4 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -3794,6 +3794,243 @@ func TestSyncCommandUsesConfiguredGitHubBaseURLAndHydratesComments(t *testing.T) } } +func TestFillPRDetailsHydratesMissingPullRequestDetails(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + cacheDir := filepath.Join(dir, "cache") + init := New() + if err := init.Run(ctx, []string{"--config", configPath, "init", "--db", dbPath}); err != nil { + t.Fatalf("init: %v", err) + } + configureTestGitHubCache(t, configPath, cacheDir, "test-gh-token") + st, err := store.Open(ctx, dbPath) + if err != nil { + t.Fatalf("open store: %v", err) + } + now := "2026-06-20T12:00:00Z" + repoID, err := st.UpsertRepository(ctx, store.Repository{Owner: "openclaw", Name: "gitcrawl", FullName: "openclaw/gitcrawl", RawJSON: "{}", UpdatedAt: now}) + if err != nil { + t.Fatalf("repo: %v", err) + } + if _, err := st.UpsertThread(ctx, store.Thread{ + RepoID: repoID, GitHubID: "102", Number: 102, Kind: "pull_request", State: "open", + Title: "Fill details", HTMLURL: "https://github.com/openclaw/gitcrawl/pull/102", + LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "h102", + UpdatedAtGitHub: now, UpdatedAt: now, + }); err != nil { + t.Fatalf("thread: %v", err) + } + if err := st.Close(); err != nil { + t.Fatalf("close seed store: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "Bearer test-gh-token" { + t.Fatalf("authorization = %q", got) + } + w.Header().Set("X-RateLimit-Limit", "5000") + w.Header().Set("X-RateLimit-Remaining", "4990") + w.Header().Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(time.Hour).Unix(), 10)) + switch r.URL.Path { + case "/repos/openclaw/gitcrawl": + _ = json.NewEncoder(w).Encode(map[string]any{"id": 12345, "full_name": "openclaw/gitcrawl"}) + case "/repos/openclaw/gitcrawl/issues/102": + row := githubIssueJSON(102, "pull_request", "Fill details") + row["html_url"] = "https://github.com/openclaw/gitcrawl/pull/102" + row["pull_request"] = map[string]any{"url": githubServerURL(r) + "/repos/openclaw/gitcrawl/pulls/102"} + _ = json.NewEncoder(w).Encode(row) + case "/graphql": + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": map[string]any{ + "repository": map[string]any{ + "pullRequest": map[string]any{ + "reviewThreads": map[string]any{ + "nodes": []map[string]any{}, + "pageInfo": map[string]any{"hasNextPage": false, "endCursor": ""}, + }, + }, + }, + }, + }) + case "/repos/openclaw/gitcrawl/pulls/102": + _ = json.NewEncoder(w).Encode(map[string]any{ + "number": 102, + "base": map[string]any{"sha": "base-sha"}, + "head": map[string]any{"sha": "head-sha", "ref": "fill-details", "repo": map[string]any{"full_name": "openclaw/gitcrawl"}}, + "mergeable_state": "clean", + "additions": 3, + "deletions": 1, + "changed_files": 1, + }) + case "/repos/openclaw/gitcrawl/pulls/102/files": + _ = json.NewEncoder(w).Encode([]map[string]any{{"filename": "internal/cli/app.go", "status": "modified", "additions": 3, "deletions": 1, "changes": 4}}) + case "/repos/openclaw/gitcrawl/pulls/102/commits": + _ = json.NewEncoder(w).Encode([]map[string]any{{"sha": "commit-sha", "commit": map[string]any{"message": "feat: fill details", "author": map[string]any{"name": "Andy", "date": now}}, "html_url": "https://github.com/openclaw/gitcrawl/commit/commit-sha"}}) + case "/repos/openclaw/gitcrawl/commits/head-sha/check-runs": + _ = json.NewEncoder(w).Encode(map[string]any{"check_runs": []map[string]any{{"name": "CI", "status": "completed", "conclusion": "success", "details_url": "https://example.invalid/check"}}}) + case "/repos/openclaw/gitcrawl/actions/runs": + if r.URL.Query().Get("head_sha") != "head-sha" { + t.Fatalf("workflow runs query = %s", r.URL.RawQuery) + } + _ = json.NewEncoder(w).Encode(map[string]any{"workflow_runs": []map[string]any{{"id": 99, "run_number": 7, "head_sha": "head-sha", "status": "completed", "conclusion": "success", "name": "CI", "html_url": "https://example.invalid/run"}}}) + default: + t.Fatalf("unexpected GitHub path: %s", r.URL.String()) + } + })) + defer server.Close() + t.Setenv("GITHUB_TOKEN", "test-gh-token") + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) + t.Setenv("GITCRAWL_GH_RATE_LIMIT_MAX_AGE", "1m") + writeFillPRDetailsRateLimitState(t, cacheDir, "test-gh-token", 1, time.Now().Add(-time.Minute), time.Now().Add(-time.Hour)) + + run := New() + var stdout, stderr bytes.Buffer + run.Stdout = &stdout + run.Stderr = &stderr + if err := run.Run(ctx, []string{"--config", configPath, "fill-pr-details", "openclaw/gitcrawl", "--limit", "1", "--batch-size", "1", "--reserve-rate-limit", "10", "--json-progress", "--json"}); err != nil { + t.Fatalf("fill-pr-details: %v\nstderr=%s", err, stderr.String()) + } + var result struct { + Selected int `json:"selected"` + Filled int `json:"filled"` + } + if err := json.Unmarshal(stdout.Bytes(), &result); err != nil { + t.Fatalf("decode fill result: %v\n%s", err, stdout.String()) + } + if result.Selected != 1 || result.Filled != 1 { + t.Fatalf("fill result = %+v", result) + } + if !strings.Contains(stderr.String(), `"event":"batch_start"`) || !strings.Contains(stderr.String(), `"event":"batch_done"`) { + t.Fatalf("missing json progress events: %s", stderr.String()) + } + st, err = store.Open(ctx, dbPath) + if err != nil { + t.Fatalf("reopen store: %v", err) + } + defer st.Close() + cache, err := st.PullRequestCache(ctx, repoID, 102) + if err != nil { + t.Fatalf("pull request cache: %v", err) + } + if cache.Detail.HeadSHA != "head-sha" || len(cache.Files) != 1 || len(cache.Commits) != 1 || len(cache.Checks) != 1 { + t.Fatalf("cache after fill = %+v", cache) + } +} + +func TestFillPRDetailsReserveRateLimitStopsOnFreshLowCache(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + cacheDir := filepath.Join(dir, "cache") + init := New() + if err := init.Run(ctx, []string{"--config", configPath, "init", "--db", dbPath}); err != nil { + t.Fatalf("init: %v", err) + } + configureTestGitHubCache(t, configPath, cacheDir, "test-gh-token") + st, err := store.Open(ctx, dbPath) + if err != nil { + t.Fatalf("open store: %v", err) + } + now := "2026-06-20T12:00:00Z" + repoID, err := st.UpsertRepository(ctx, store.Repository{Owner: "openclaw", Name: "gitcrawl", FullName: "openclaw/gitcrawl", RawJSON: "{}", UpdatedAt: now}) + if err != nil { + t.Fatalf("repo: %v", err) + } + if _, err := st.UpsertThread(ctx, store.Thread{ + RepoID: repoID, GitHubID: "103", Number: 103, Kind: "pull_request", State: "open", + Title: "Fill details", HTMLURL: "https://github.com/openclaw/gitcrawl/pull/103", + LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "h103", + UpdatedAtGitHub: now, UpdatedAt: now, + }); err != nil { + t.Fatalf("thread: %v", err) + } + if err := st.Close(); err != nil { + t.Fatalf("close seed store: %v", err) + } + + t.Setenv("GITHUB_TOKEN", "test-gh-token") + t.Setenv("GITCRAWL_GH_RATE_LIMIT_MAX_AGE", "1m") + writeFillPRDetailsRateLimitState(t, cacheDir, "test-gh-token", 5, time.Now().Add(time.Hour), time.Now()) + + run := New() + var stdout bytes.Buffer + run.Stdout = &stdout + if err := run.Run(ctx, []string{"--config", configPath, "fill-pr-details", "openclaw/gitcrawl", "--limit", "1", "--reserve-rate-limit", "10", "--json"}); err != nil { + t.Fatalf("fill-pr-details: %v", err) + } + var result struct { + Selected int `json:"selected"` + Filled int `json:"filled"` + Remaining int `json:"remaining"` + StoppedReason string `json:"stopped_reason"` + RateLimit *fillRateLimitResult `json:"rate_limit"` + } + if err := json.Unmarshal(stdout.Bytes(), &result); err != nil { + t.Fatalf("decode fill result: %v\n%s", err, stdout.String()) + } + if result.Selected != 1 || result.Filled != 0 || result.Remaining != 1 || result.StoppedReason != "rate-limit-reserve" { + t.Fatalf("fill result = %+v", result) + } + if result.RateLimit == nil || result.RateLimit.Remaining != 5 { + t.Fatalf("rate limit = %+v", result.RateLimit) + } +} + +func configureTestGitHubCache(t *testing.T, configPath, cacheDir, token string) { + t.Helper() + cfg, err := config.Load(configPath) + if err != nil { + t.Fatalf("load config: %v", err) + } + cfg.CacheDir = cacheDir + if cfg.Env == nil { + cfg.Env = map[string]string{} + } + cfg.Env["GITHUB_TOKEN"] = token + if err := config.Save(configPath, cfg); err != nil { + t.Fatalf("save config: %v", err) + } +} + +func writeFillPRDetailsRateLimitState(t *testing.T, cacheDir, token string, remaining int, resetAt, updatedAt time.Time) { + t.Helper() + dir := filepath.Join(cacheDir, "octopool-migrated-gh") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatalf("create cache dir: %v", err) + } + state := ghSharedRateLimitState{ + Host: "github.com", + TokenHash: ghRateLimitTokenHash(token), + Limit: 5000, + Remaining: remaining, + ResetAt: resetAt.UTC(), + Resource: "core", + UpdatedAt: updatedAt.UTC(), + Source: "test", + Low: true, + Threshold: 10, + } + data, err := json.Marshal(state) + if err != nil { + t.Fatalf("marshal rate limit state: %v", err) + } + path := filepath.Join(dir, ghSharedRateLimitFilePrefix+"github.com_"+ghRateLimitTokenHash(token)+".json") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write rate limit state: %v", err) + } +} + +func githubServerURL(r *http.Request) string { + scheme := "http" + if r.TLS != nil { + scheme = "https" + } + return scheme + "://" + r.Host +} + func TestRefreshRunsSyncEmbedAndClusterWithLocalServers(t *testing.T) { ctx := context.Background() dir := t.TempDir() diff --git a/internal/store/pull_request_fill.go b/internal/store/pull_request_fill.go new file mode 100644 index 0000000..7973ac0 --- /dev/null +++ b/internal/store/pull_request_fill.go @@ -0,0 +1,61 @@ +package store + +import ( + "context" + "fmt" + "strings" +) + +type MissingPullRequestDetailOptions struct { + Limit int + Order string +} + +func (s *Store) MissingPullRequestDetailNumbers(ctx context.Context, repoID int64, options MissingPullRequestDetailOptions) ([]int, error) { + orderClause, err := missingPullRequestDetailOrderClause(options.Order) + if err != nil { + return nil, err + } + limit := options.Limit + if limit <= 0 { + limit = -1 + } + rows, err := s.q().QueryContext(ctx, ` +select t.number +from threads t +left join pull_request_details d on d.thread_id = t.id +where t.repo_id = ? + and t.kind = 'pull_request' + and d.thread_id is null +order by `+orderClause+` +limit ?`, repoID, limit) + if err != nil { + return nil, fmt.Errorf("list missing pull request details: %w", err) + } + defer rows.Close() + var out []int + for rows.Next() { + var number int + if err := rows.Scan(&number); err != nil { + return nil, fmt.Errorf("scan missing pull request detail number: %w", err) + } + out = append(out, number) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("scan missing pull request detail numbers: %w", err) + } + return out, nil +} + +func missingPullRequestDetailOrderClause(order string) (string, error) { + switch strings.TrimSpace(strings.ToLower(order)) { + case "", "newest-first": + return "coalesce(t.updated_at_gh, t.updated_at, '') desc, t.number desc", nil + case "oldest-first": + return "coalesce(t.updated_at_gh, t.updated_at, '') asc, t.number asc", nil + case "open-first": + return "case when t.state = 'open' then 0 else 1 end, coalesce(t.updated_at_gh, t.updated_at, '') desc, t.number desc", nil + default: + return "", fmt.Errorf("unsupported fill-pr-details order %q", order) + } +} diff --git a/internal/store/pull_requests_test.go b/internal/store/pull_requests_test.go index 2f98211..4594a86 100644 --- a/internal/store/pull_requests_test.go +++ b/internal/store/pull_requests_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "path/filepath" "testing" + "time" ) func TestPullRequestCacheRoundTripAndWorkflowFilters(t *testing.T) { @@ -94,6 +95,61 @@ func TestPullRequestCacheRoundTripAndWorkflowFilters(t *testing.T) { } } +func TestMissingPullRequestDetailNumbersSelectsOnlyUnfilledPRs(t *testing.T) { + ctx := context.Background() + st, err := Open(ctx, filepath.Join(t.TempDir(), "gitcrawl.db")) + if err != nil { + t.Fatalf("open store: %v", err) + } + defer st.Close() + now := time.Date(2026, 6, 20, 12, 0, 0, 0, time.UTC).Format(time.RFC3339Nano) + repoID, err := st.UpsertRepository(ctx, Repository{Owner: "openclaw", Name: "gitcrawl", FullName: "openclaw/gitcrawl", RawJSON: "{}", UpdatedAt: now}) + if err != nil { + t.Fatalf("repo: %v", err) + } + threads := []Thread{ + {RepoID: repoID, GitHubID: "11", Number: 11, Kind: "pull_request", State: "closed", Title: "old missing", HTMLURL: "https://github.com/openclaw/gitcrawl/pull/11", LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "h11", UpdatedAtGitHub: "2026-06-01T00:00:00Z", UpdatedAt: now}, + {RepoID: repoID, GitHubID: "12", Number: 12, Kind: "pull_request", State: "open", Title: "new missing", HTMLURL: "https://github.com/openclaw/gitcrawl/pull/12", LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "h12", UpdatedAtGitHub: "2026-06-03T00:00:00Z", UpdatedAt: now}, + {RepoID: repoID, GitHubID: "13", Number: 13, Kind: "pull_request", State: "open", Title: "already filled", HTMLURL: "https://github.com/openclaw/gitcrawl/pull/13", LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "h13", UpdatedAtGitHub: "2026-06-04T00:00:00Z", UpdatedAt: now}, + {RepoID: repoID, GitHubID: "14", Number: 14, Kind: "issue", State: "open", Title: "plain issue", HTMLURL: "https://github.com/openclaw/gitcrawl/issues/14", LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: "h14", UpdatedAtGitHub: "2026-06-05T00:00:00Z", UpdatedAt: now}, + } + var filledThreadID int64 + for _, thread := range threads { + id, err := st.UpsertThread(ctx, thread) + if err != nil { + t.Fatalf("thread %d: %v", thread.Number, err) + } + if thread.Number == 13 { + filledThreadID = id + } + } + if err := st.UpsertPullRequestCache(ctx, PullRequestDetail{ThreadID: filledThreadID, RepoID: repoID, Number: 13, RawJSON: "{}", FetchedAt: now, UpdatedAt: now}, nil, nil, nil, nil); err != nil { + t.Fatalf("fill pr detail: %v", err) + } + + got, err := st.MissingPullRequestDetailNumbers(ctx, repoID, MissingPullRequestDetailOptions{Order: "newest-first"}) + if err != nil { + t.Fatalf("missing newest: %v", err) + } + if len(got) != 2 || got[0] != 12 || got[1] != 11 { + t.Fatalf("newest missing = %v", got) + } + got, err = st.MissingPullRequestDetailNumbers(ctx, repoID, MissingPullRequestDetailOptions{Order: "oldest-first", Limit: 1}) + if err != nil { + t.Fatalf("missing oldest: %v", err) + } + if len(got) != 1 || got[0] != 11 { + t.Fatalf("oldest limited missing = %v", got) + } + got, err = st.MissingPullRequestDetailNumbers(ctx, repoID, MissingPullRequestDetailOptions{Order: "open-first"}) + if err != nil { + t.Fatalf("missing open-first: %v", err) + } + if len(got) != 2 || got[0] != 12 || got[1] != 11 { + t.Fatalf("open-first missing = %v", got) + } +} + func TestPullRequestCacheAllowsDuplicateFilePaths(t *testing.T) { ctx := context.Background() st, err := Open(ctx, filepath.Join(t.TempDir(), "gitcrawl.db"))