Track failed sync hydration attempts

internal/cli/app.go

@@ -179,6 +179,8 @@ func (a *App) Run(ctx context.Context, args []string) error {
 		return a.runSetClusterCanonical(ctx, rest[1:])
 	case "runs":
 		return a.runRuns(ctx, rest[1:])
+	case "sync-failures":
+		return a.runSyncFailures(ctx, rest[1:])
 	case "coverage":
 		return a.runCoverage(ctx, rest[1:])
 	case "search":
@@ -2262,6 +2264,53 @@ func (a *App) runRuns(ctx context.Context, args []string) error {
 	}, true)
 }
 
+func (a *App) runSyncFailures(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("sync-failures", flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+	includeResolved := fs.Bool("include-resolved", false, "include resolved failure rows")
+	limitRaw := fs.String("limit", "", "maximum failure rows")
+	jsonOut := fs.Bool("json", false, "write JSON output")
+	if err := fs.Parse(normalizeCommandArgs(args, map[string]bool{"limit": true})); err != nil {
+		return usageErr(err)
+	}
+	a.applyCommandJSON(*jsonOut)
+	if fs.NArg() != 1 {
+		return usageErr(fmt.Errorf("sync-failures requires owner/repo"))
+	}
+	owner, repoName, err := parseOwnerRepo(fs.Arg(0))
+	if err != nil {
+		return usageErr(err)
+	}
+	limit, err := parseOptionalPositiveInt(*limitRaw)
+	if err != nil {
+		return usageErr(err)
+	}
+
+	rt, err := a.openLocalRuntimeReadOnly(ctx)
+	if err != nil {
+		return err
+	}
+	defer rt.Store.Close()
+
+	repo, err := rt.repository(ctx, owner, repoName)
+	if err != nil {
+		return err
+	}
+	failures, err := rt.Store.ListSyncAttemptFailures(ctx, store.SyncAttemptFailureListOptions{
+		RepoID:          repo.ID,
+		IncludeResolved: *includeResolved,
+		Limit:           limit,
+	})
+	if err != nil {
+		return err
+	}
+	return a.writeOutput("sync-failures", map[string]any{
+		"repository":       repo.FullName,
+		"include_resolved": *includeResolved,
+		"failures":         failures,
+	}, true)
+}
+
 func (a *App) runCoverage(ctx context.Context, args []string) error {
 	fs := flag.NewFlagSet("coverage", flag.ContinueOnError)
 	fs.SetOutput(io.Discard)
@@ -2327,7 +2376,7 @@ func (a *App) runCoverage(ctx context.Context, args []string) error {
 	payload := map[string]any{
 		"repository_filters":           resolvedFilters,
 		"min_missing_pr_details":       minMissing,
-		"hydration_failures_available": false,
+		"hydration_failures_available": coverage.Totals.HydrationFailuresSupported,
 		"repositories":                 coverage.Rows,
 		"totals":                       coverage.Totals,
 	}
@@ -4553,6 +4602,7 @@ Core commands:
   init                 create config, optionally from a portable store
   doctor               check config, token, and database readiness
   sync                 sync GitHub issue and pull request metadata
+  sync-failures        list failed sync hydration attempts
   coverage             report local archive PR-detail completeness
   refresh              run sync, enrichment, embedding, and clustering pipeline
   embed                generate OpenAI embeddings for local thread documents
@@ -4637,6 +4687,11 @@ Usage:
 
 Usage:
   gitcrawl sync owner/repo [--state open|closed|all] [--numbers refs] [--with pr-details] [--include-pr-details] [--json]
+`,
+	"sync-failures": `gitcrawl sync-failures lists failed sync hydration attempts.
+
+Usage:
+  gitcrawl sync-failures owner/repo [--include-resolved] [--limit N] [--json]
 `,
 	"coverage": `gitcrawl coverage reports local archive completeness by repository.
 

internal/cli/app_test.go

@@ -2161,10 +2161,10 @@ func TestDoctorJSONReportsCurrentSchemaDiagnosticsWithoutMutation(t *testing.T)
 	if got := schema["state"]; got != "current" {
 		t.Fatalf("db_schema.state = %#v, payload=%#v", got, schema)
 	}
-	if got := schema["current_version"]; got != float64(4) {
+	if got := schema["current_version"]; got != float64(5) {
 		t.Fatalf("db_schema.current_version = %#v, payload=%#v", got, schema)
 	}
-	if got := schema["supported_version"]; got != float64(4) {
+	if got := schema["supported_version"]; got != float64(5) {
 		t.Fatalf("db_schema.supported_version = %#v, payload=%#v", got, schema)
 	}
 	prDetails := doctorMap(t, schema, "pr_details")
@@ -2423,7 +2423,7 @@ func TestDoctorJSONReportsLegacyPendingSchemaWithoutMutation(t *testing.T) {
 		t.Fatalf("pr_details.duplicate_path_files_supported = %#v, payload=%#v", got, prDetails)
 	}
 	pending := doctorStringList(t, schema, "pending_migrations")
-	if !doctorListContains(pending, "schema_version_3_to_4") || !doctorListContains(pending, "pull_request_files_position_key") {
+	if !doctorListContains(pending, "schema_version_3_to_5") || !doctorListContains(pending, "pull_request_files_position_key") {
 		t.Fatalf("pending_migrations = %#v", pending)
 	}
 	if len(doctorStringList(t, schema, "next_steps")) == 0 {
@@ -3085,6 +3085,85 @@ func TestTUIInfersRepository(t *testing.T) {
 	}
 }
 
+func TestSyncFailuresListsUnresolvedAndHistory(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	configPath := filepath.Join(dir, "config.toml")
+	dbPath := filepath.Join(dir, "gitcrawl.db")
+	app := New()
+	if err := app.Run(ctx, []string{"--config", configPath, "init", "--db", dbPath}); err != nil {
+		t.Fatalf("init: %v", err)
+	}
+	st, err := store.Open(ctx, dbPath)
+	if err != nil {
+		t.Fatalf("open store: %v", err)
+	}
+	repoID, err := st.UpsertRepository(ctx, store.Repository{
+		Owner:     "openclaw",
+		Name:      "gitcrawl",
+		FullName:  "openclaw/gitcrawl",
+		RawJSON:   "{}",
+		UpdatedAt: "2026-06-06T00:00:00Z",
+	})
+	if err != nil {
+		t.Fatalf("seed repository: %v", err)
+	}
+	for _, number := range []int{83, 84} {
+		threadID, err := st.UpsertThread(ctx, store.Thread{
+			RepoID: repoID, GitHubID: strconv.Itoa(number), Number: number, Kind: "pull_request", State: "open",
+			Title: "hydration failure", HTMLURL: fmt.Sprintf("https://github.com/openclaw/gitcrawl/pull/%d", number),
+			LabelsJSON: "[]", AssigneesJSON: "[]", RawJSON: "{}", ContentHash: fmt.Sprintf("h%d", number), UpdatedAt: "2026-06-06T00:00:00Z",
+		})
+		if err != nil {
+			t.Fatalf("seed thread %d: %v", number, err)
+		}
+		if _, err := st.RecordSyncAttemptFailure(ctx, store.SyncAttemptFailure{
+			RepoID: repoID, ThreadID: threadID, Number: number, Operation: "pull_request_details", ErrorClass: "error",
+			ErrorMessage: fmt.Sprintf("failure %d", number), FirstSeenAt: "2026-06-06T00:00:00Z", LastSeenAt: "2026-06-06T00:00:00Z",
+		}); err != nil {
+			t.Fatalf("seed failure %d: %v", number, err)
+		}
+		if number == 83 {
+			if _, err := st.ResolveSyncAttemptFailures(ctx, repoID, number, "2026-06-06T00:05:00Z"); err != nil {
+				t.Fatalf("resolve failure %d: %v", number, err)
+			}
+		}
+	}
+	if err := st.Close(); err != nil {
+		t.Fatalf("close store: %v", err)
+	}
+
+	run := New()
+	var stdout bytes.Buffer
+	run.Stdout = &stdout
+	if err := run.Run(ctx, []string{"--config", configPath, "--json", "sync-failures", "openclaw/gitcrawl"}); err != nil {
+		t.Fatalf("sync-failures: %v", err)
+	}
+	var active struct {
+		Failures []store.SyncAttemptFailure `json:"failures"`
+	}
+	if err := json.Unmarshal(stdout.Bytes(), &active); err != nil {
+		t.Fatalf("decode active failures: %v\n%s", err, stdout.String())
+	}
+	if len(active.Failures) != 1 || active.Failures[0].Number != 84 || active.Failures[0].ResolvedAt != "" {
+		t.Fatalf("active failures = %+v", active.Failures)
+	}
+
+	stdout.Reset()
+	if err := run.Run(ctx, []string{"--config", configPath, "--json", "sync-failures", "openclaw/gitcrawl", "--include-resolved"}); err != nil {
+		t.Fatalf("sync-failures history: %v", err)
+	}
+	var history struct {
+		Failures []store.SyncAttemptFailure `json:"failures"`
+	}
+	if err := json.Unmarshal(stdout.Bytes(), &history); err != nil {
+		t.Fatalf("decode failure history: %v\n%s", err, stdout.String())
+	}
+	if len(history.Failures) != 2 {
+		t.Fatalf("history failures = %+v", history.Failures)
+	}
+}
+
 func TestTUIJSONUsesDefaultsWhenConfigMissing(t *testing.T) {
 	ctx := context.Background()
 	dir := t.TempDir()

internal/cli/archive_coverage_test.go

@@ -46,9 +46,12 @@ func TestCoverageCommandJSONAndTable(t *testing.T) {
 	if row.Comments != 2 || row.PRReviews != 1 {
 		t.Fatalf("comment coverage = %+v", row)
 	}
-	if row.HydrationFailuresSupported || row.KnownFailedHydrations != nil {
+	if !row.HydrationFailuresSupported || row.KnownFailedHydrations == nil || *row.KnownFailedHydrations != 1 {
 		t.Fatalf("failure ledger fields = %+v", row)
 	}
+	if !payload.Totals.HydrationFailuresSupported || payload.Totals.KnownFailedHydrations == nil || *payload.Totals.KnownFailedHydrations != 1 {
+		t.Fatalf("failure ledger totals = %+v", payload.Totals)
+	}
 
 	tableRun := New()
 	var tableOut bytes.Buffer
@@ -176,6 +179,7 @@ func TestCoverageCommandSupportsArchiveWithoutOptionalCoverageTables(t *testing.
 		"github_workflow_runs",
 		"sync_runs",
 		"repo_sync_state",
+		"sync_attempt_failures",
 	} {
 		if _, err := st.DB().ExecContext(ctx, `drop table `+table); err != nil {
 			st.Close()
@@ -287,6 +291,32 @@ func seedCoverageStore(t *testing.T, ctx context.Context, dbPath string) {
 			t.Fatalf("comment: %v", err)
 		}
 	}
+	if _, err := st.RecordSyncAttemptFailure(ctx, store.SyncAttemptFailure{
+		RepoID:       gitcrawlID,
+		ThreadID:     detailedPRID,
+		Number:       4,
+		Operation:    "pull_request_details",
+		ErrorClass:   "api",
+		ErrorMessage: "api failure",
+		FirstSeenAt:  "2026-07-03T00:01:00Z",
+		LastSeenAt:   "2026-07-03T00:01:00Z",
+	}); err != nil {
+		t.Fatalf("record failure: %v", err)
+	}
+	if _, err := st.RecordSyncAttemptFailure(ctx, store.SyncAttemptFailure{
+		RepoID:       gitcrawlID,
+		Number:       2,
+		Operation:    "pull_request_details",
+		ErrorClass:   "temporary",
+		ErrorMessage: "temporary failure",
+		FirstSeenAt:  "2026-07-03T00:02:00Z",
+		LastSeenAt:   "2026-07-03T00:02:00Z",
+	}); err != nil {
+		t.Fatalf("record resolved failure: %v", err)
+	}
+	if _, err := st.ResolveSyncAttemptFailures(ctx, gitcrawlID, 2, "2026-07-03T00:03:00Z"); err != nil {
+		t.Fatalf("resolve failure: %v", err)
+	}
 }
 
 func coverageThread(repoID int64, number int, kind string) store.Thread {

internal/cli/gh_compat_helpers_test.go

@@ -2,10 +2,14 @@ package cli
 
 import (
 	"bytes"
+	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/openclaw/gitcrawl/internal/store"
 )
 
 func TestGHAPIRouteNormalization(t *testing.T) {
@@ -76,3 +80,65 @@ func TestWriteJSONValueWithoutJQ(t *testing.T) {
 		t.Fatalf("json output = %s", out.String())
 	}
 }
+
+func TestWriteJSONValueErrors(t *testing.T) {
+	app := New()
+	if err := app.writeJSONValue(map[string]any{"bad": func() {}}, ""); err == nil {
+		t.Fatalf("expected marshal error")
+	}
+	t.Setenv("PATH", "")
+	if err := app.writeJSONValue(map[string]any{"ok": true}, ".ok"); !errors.Is(err, errLocalGHUnsupported) {
+		t.Fatalf("jq error = %v, want local gh unsupported", err)
+	}
+}
+
+func TestLocalGHUnsupportedAndShim(t *testing.T) {
+	if err := localGHUnsupported(nil); !errors.Is(err, errLocalGHUnsupported) {
+		t.Fatalf("nil wrapped error = %v", err)
+	}
+	if err := localGHUnsupported(errors.New("missing")); !errors.Is(err, errLocalGHUnsupported) || !strings.Contains(err.Error(), "missing") {
+		t.Fatalf("wrapped error = %v", err)
+	}
+
+	app := New()
+	var stderr bytes.Buffer
+	app.Stderr = &stderr
+	err := app.runGHShim(nil, nil)
+	if err == nil || !strings.Contains(err.Error(), "gitcrawl gh moved to octopool") {
+		t.Fatalf("runGHShim error = %v", err)
+	}
+	if !strings.Contains(stderr.String(), "octopool login") {
+		t.Fatalf("stderr = %s", stderr.String())
+	}
+}
+
+func TestClusterAndRuntimeHelperBranches(t *testing.T) {
+	clusterCases := []struct {
+		source string
+		want   string
+		ok     bool
+	}{
+		{source: "", want: "", ok: true},
+		{source: " AUTO ", want: "", ok: true},
+		{source: "raw", want: store.ClusterSourceRun, ok: true},
+		{source: "durable", want: store.ClusterSourceDurable, ok: true},
+		{source: "unknown", ok: false},
+	}
+	for _, tc := range clusterCases {
+		got, err := parseClusterDetailSource(tc.source)
+		if tc.ok && (err != nil || got != tc.want) {
+			t.Fatalf("parseClusterDetailSource(%q) = %q, %v; want %q", tc.source, got, err, tc.want)
+		}
+		if !tc.ok && err == nil {
+			t.Fatalf("parseClusterDetailSource(%q) unexpectedly succeeded", tc.source)
+		}
+	}
+
+	err := neighborEmbeddingRecoveryError("openclaw", "gitcrawl", 42, true, fmt.Errorf("missing vector"))
+	if !strings.Contains(err.Error(), "gitcrawl embed openclaw/gitcrawl --number 42 --limit 1 --include-closed") {
+		t.Fatalf("neighbor recovery error = %v", err)
+	}
+	if isGitIndexLockError(nil) || isGitIndexLockError(errors.New("index.lock")) || !isGitIndexLockError(errors.New("fatal: Unable to create '.git/index.lock': File exists.")) {
+		t.Fatalf("index lock classifier failed")
+	}
+}