diff --git a/CHANGELOG.md b/CHANGELOG.md index 99964c6..e48d98b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## 0.7.2 - Unreleased +- Require every Gitcrawl cloud publisher manifest to opt into snapshot staging so `--stage-only` and normal publish-then-cutover runs cannot activate serving state through the legacy compatibility path. +- Update CrawlKit to v0.14.2 and bind publisher resume plus post-cutover verification to the exact snapshot ID, so a newer concurrently staged candidate cannot be mistaken for the snapshot this publisher owns. +- Fail cloud publication before any remote mutation when local enrichment coverage is incomplete unless operators explicitly pass `--allow-incomplete`. + ## 0.7.1 - 2026-07-09 - Add local, fail-closed packaging and independent verification for official macOS archives signed as `org.openclaw.gitcrawl` by `Developer ID Application: OpenClaw Foundation (FWJYW4S8P8)`, while keeping CI and cross-platform snapshots credential-free and non-publishing. diff --git a/README.md b/README.md index 23dd60a..d101708 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,34 @@ gitcrawl tui owner/repo The remote service is deployed separately from gitcrawl in `openclaw/crawl-remote` with Wrangler. gitcrawl only stores the Worker endpoint/archive in config and calls that service. `gitcrawl remote login` starts the Worker GitHub OAuth flow, verifies org/team membership server-side, and stores the signed bearer token in the OS keyring. Use `gitcrawl remote login --github-token-env GITHUB_TOKEN` for non-browser bootstrap; the Worker verifies that GitHub token against the same org/team policy and stores only the returned remote session token locally. -`gitcrawl cloud publish` sends the local SQLite repositories and thread rows to a Worker archive through the role-gated ingest endpoint. +`gitcrawl cloud publish` freezes and sanitizes one local SQLite image, uses its +SHA-256 as the snapshot identity, exports repositories, threads, revisions, +fingerprints, summaries, durable clusters, and PR detail/file rows from that +same image, negotiates the remote snapshot-provenance contract before touching +R2, uploads its digest-scoped bundle, and completes staged D1 coverage through +row- and encoded-byte-bounded ingest requests. +Publishing moves unpinned reads to the complete snapshot by default, preserving +the existing reader-refresh behavior. The remote must advertise +`gitcrawl.snapshot.staging.v1`; `--stage-only` keeps the immutable snapshot +staged without changing serving state. A later publish verifies the +candidate through the publisher-only status projection, skips repeated ingest +when its digest, source sync, schema, resolved publication profile, persisted +generation timestamp, and coverage match, then cuts it over. Cutover requires +the remote contract to advertise reader-authenticated `GET /sqlite`; Gitcrawl +validates the cutover acknowledgement, retries the scoped reader projection +until its digest, profile, generation, and dataset coverage are exact, rechecks +the exact publisher metadata, and hashes the downloaded bound SQLite image +before reporting success. Before any upload or ingest, Gitcrawl verifies +the configured credential through the advertised `/v1/whoami` route and +requires both publisher and reader roles, including for stage-only publication. +Incomplete local enrichment fails before any remote mutation; +`--allow-incomplete` is an explicit escape hatch, and `--observation-order` +publishes durable fetch ordering after the remote operator fence is enabled. +Digest-scoped SQLite bundles can contain private issue and pull-request text. +Gitcrawl intentionally exposes no remote deletion command: operators must only +enable publication against a remote deployment with bounded lifecycle rules for +failed, superseded, and uncut staged bundles. `--stage-only` does not transfer +that retention responsibility back to the client. `gitcrawl clusters-report` writes a Markdown report for the top clusters using the same display view, with an at-a-glance table, per-cluster metadata, member tables, and key snippets. Use `--json` for the hydrated report payload. `gitcrawl cluster` and `gitcrawl refresh` build ghcrawl-shaped durable clusters by default (`--threshold 0.80`, `--min-size 1`, `--max-cluster-size 40`, `--k 16`, `--cross-kind-threshold 0.93`): every active vector-backed thread is represented, singleton rows use `singleton_orphan`, multi-member rows use `duplicate_candidate`, and stable IDs are derived from the representative thread. They also add deterministic GitHub reference evidence for direct issue/PR links such as `#123`, `issues/123`, and `pull/123`. Weak embedding edges need concrete title-token overlap unless their similarity is already high, which keeps generic low-confidence bridges from forming unrelated clusters. `gitcrawl tui` infers the most recently updated local repository when `owner/repo` is omitted. `serve` is intentionally not part of `gitcrawl`. diff --git a/go.mod b/go.mod index 4d4b7f4..4c403a0 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 github.com/charmbracelet/x/ansi v0.11.7 github.com/mattn/go-isatty v0.0.22 - github.com/openclaw/crawlkit v0.13.4 + github.com/openclaw/crawlkit v0.14.2 github.com/zalando/go-keyring v0.2.8 golang.org/x/sys v0.46.0 modernc.org/sqlite v1.53.0 diff --git a/go.sum b/go.sum index 9b1a907..2169d41 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/openclaw/crawlkit v0.13.4 h1:6KZaIzs5BQxBjCeOwg1u2XPLNbsOkrRPQGPtTkxZaYE= -github.com/openclaw/crawlkit v0.13.4/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= +github.com/openclaw/crawlkit v0.14.2 h1:NA0+fDlyYnQjrI6krkhajHDKMTOt9q8JTlMFsGH37pg= +github.com/openclaw/crawlkit v0.14.2/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/internal/cli/app.go b/internal/cli/app.go index 05b8fa8..76ce7ec 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -4770,7 +4770,7 @@ Usage: "cloud": `gitcrawl cloud manages Worker-backed remote archives. Usage: - gitcrawl cloud publish --remote URL --archive id [--json] + gitcrawl cloud publish --remote URL --archive id [--allow-incomplete] [--observation-order] [--stage-only] [--json] `, "whoami": `gitcrawl whoami prints the configured remote archive identity. diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 028676b..0806e2c 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -13,6 +13,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "slices" "sort" "strconv" "strings" @@ -366,6 +367,73 @@ func TestCloudSQLiteSnapshotDropsLocalCodeCorpus(t *testing.T) { }, []store.CodeDocument{{Path: "secret.txt", Language: "txt", ContentHash: "h", Text: "secret", ByteSize: 6, UpdatedAt: "2026-06-06T00:00:00Z"}}); err != nil { t.Fatalf("replace code snapshot: %v", err) } + if _, err := st.DB().ExecContext(ctx, ` + insert into threads( + id, repo_id, github_id, number, kind, state, title, body, html_url, + labels_json, assignees_json, raw_json, content_hash, updated_at + ) values( + 1, ?, 'pr-1', 1, 'pull_request', 'open', 'PR', 'body', + 'https://github.com/openclaw/gitcrawl/pull/1', '[]', '[]', + '{"secret":"thread"}', 'hash', '2026-06-06T00:00:00Z' + ); + insert into pull_request_details( + thread_id, repo_id, number, raw_json, fetched_at, updated_at + ) values( + 1, ?, 1, '{"secret":"detail"}', + '2026-06-06T00:00:00Z', '2026-06-06T00:00:00Z' + ); + insert into pull_request_files( + thread_id, position, path, patch, raw_json, fetched_at + ) values( + 1, 0, 'secret.go', '@@ private source', '{"secret":"file"}', + '2026-06-06T00:00:00Z' + ); + insert into pull_request_review_threads( + thread_id, review_thread_id, path, comments_json, raw_json, fetched_at + ) values( + 1, 'review-thread-1', 'secret.go', + '[{"body":"review body","diffHunk":"@@ private review source"}]', + '{"secret":"review-thread"}', '2026-06-06T00:00:00Z' + ); + insert into blobs( + sha256, media_type, compression, size_bytes, storage_kind, inline_text, created_at + ) values( + 'blob-hash', 'application/json', 'none', 6, 'inline', 'secret', + '2026-06-06T00:00:00Z' + ); + insert into sync_runs( + repo_id, scope, status, started_at, finished_at, stats_json, error_text + ) values( + ?, 'all', 'failed', '2026-06-06T00:00:00Z', + '2026-06-06T00:01:00Z', + '{"message":"/private/host/token","threads":1}', '/private/host/token' + ); + insert into summary_runs( + repo_id, scope, status, started_at, finished_at, stats_json, error_text + ) values( + ?, 'all', 'failed', '2026-06-06T00:00:00Z', + '2026-06-06T00:01:00Z', + '{"failures":[{"message":"provider secret"}]}', 'provider secret' + ); + insert into embedding_runs( + repo_id, scope, status, started_at, finished_at, stats_json, error_text + ) values( + ?, 'all', 'failed', '2026-06-06T00:00:00Z', + '2026-06-06T00:01:00Z', + '{"failures":[{"message":"/private/embed"}]}', '/private/embed' + ); + insert into cluster_runs( + repo_id, scope, status, started_at, finished_at, stats_json, error_text + ) values( + ?, 'all', 'failed', '2026-06-06T00:00:00Z', + '2026-06-06T00:01:00Z', + '{"message":"/private/cluster"}', '/private/cluster' + ); + create table portable_metadata(key text primary key, value text not null); + insert into portable_metadata(key, value) values('source_path', '/private/archive.db') + `, repoID, repoID, repoID, repoID, repoID, repoID); err != nil { + t.Fatalf("seed private cloud payloads: %v", err) + } snapshotPath, cleanup, err := cloudSQLiteSnapshotPath(ctx, st.DB(), dbPath) if err != nil { t.Fatalf("cloud snapshot: %v", err) @@ -376,7 +444,14 @@ func TestCloudSQLiteSnapshotDropsLocalCodeCorpus(t *testing.T) { t.Fatalf("open cloud snapshot: %v", err) } defer snapshotDB.Close() - for _, table := range []string{"code_snapshots", "code_documents", "code_documents_fts"} { + for _, table := range []string{ + "code_snapshots", + "code_documents", + "code_documents_fts", + "thread_code_snapshots", + "thread_changed_files", + "thread_hunk_signatures", + } { var count int if err := snapshotDB.QueryRowContext(ctx, `select count(*) from sqlite_schema where name = ?`, table).Scan(&count); err != nil { t.Fatalf("inspect %s: %v", table, err) @@ -385,6 +460,65 @@ func TestCloudSQLiteSnapshotDropsLocalCodeCorpus(t *testing.T) { t.Fatalf("cloud snapshot retained %s", table) } } + var repositoryRawJSON, threadRawJSON, detailRawJSON, filePatch, fileRawJSON string + if err := snapshotDB.QueryRowContext(ctx, ` + select repository.raw_json, thread.raw_json, detail.raw_json, file.patch, file.raw_json + from repositories repository + join threads thread on thread.repo_id = repository.id + join pull_request_details detail on detail.thread_id = thread.id + join pull_request_files file on file.thread_id = thread.id + `).Scan(&repositoryRawJSON, &threadRawJSON, &detailRawJSON, &filePatch, &fileRawJSON); err != nil { + t.Fatalf("inspect scrubbed cloud payloads: %v", err) + } + if repositoryRawJSON != "" || threadRawJSON != "" || detailRawJSON != "" || filePatch != "" || fileRawJSON != "" { + t.Fatalf( + "cloud snapshot retained private payloads: repository=%q thread=%q detail=%q patch=%q file=%q", + repositoryRawJSON, + threadRawJSON, + detailRawJSON, + filePatch, + fileRawJSON, + ) + } + var blobCount int + if err := snapshotDB.QueryRowContext(ctx, `select count(*) from blobs`).Scan(&blobCount); err != nil { + t.Fatalf("inspect cloud blobs: %v", err) + } + if blobCount != 0 { + t.Fatalf("cloud snapshot retained %d blobs", blobCount) + } + var reviewComments string + if err := snapshotDB.QueryRowContext(ctx, ` + select comments_json from pull_request_review_threads limit 1 + `).Scan(&reviewComments); err != nil { + t.Fatalf("inspect scrubbed review comments: %v", err) + } + if reviewComments != "" { + t.Fatalf("cloud snapshot retained review diff payloads: %q", reviewComments) + } + for _, table := range []string{"sync_runs", "summary_runs", "embedding_runs", "cluster_runs"} { + var statsJSON, errorText string + if err := snapshotDB.QueryRowContext(ctx, ` + select coalesce(stats_json, ''), coalesce(error_text, '') from `+table+` limit 1 + `).Scan(&statsJSON, &errorText); err != nil { + t.Fatalf("inspect scrubbed cloud %s payload: %v", table, err) + } + if statsJSON != "" || errorText != "" { + t.Fatalf( + "cloud snapshot retained %s failure payloads: stats=%q error=%q", + table, + statsJSON, + errorText, + ) + } + } + var sourcePath string + if err := snapshotDB.QueryRowContext(ctx, `select value from portable_metadata where key = 'source_path'`).Scan(&sourcePath); err != nil { + t.Fatalf("inspect scrubbed cloud source path: %v", err) + } + if sourcePath != "" { + t.Fatalf("cloud snapshot retained private source path: %q", sourcePath) + } var sourceCount int if err := st.DB().QueryRowContext(ctx, `select count(*) from code_documents`).Scan(&sourceCount); err != nil { t.Fatalf("source code documents: %v", err) @@ -560,7 +694,7 @@ func TestRemoteCloudModeDoesNotCreateLocalDB(t *testing.T) { t.Setenv("HOME", dir) t.Setenv("CRAWL_REMOTE_TOKEN", "test-token") - var sawQuery bool + var queryArgSets [][]string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if got := r.Header.Get("authorization"); got != "Bearer test-token" { http.Error(w, "missing bearer", http.StatusUnauthorized) @@ -589,7 +723,12 @@ func TestRemoteCloudModeDoesNotCreateLocalDB(t *testing.T) { http.Error(w, "unexpected query", http.StatusBadRequest) return } - sawQuery = true + argNames := make([]string, 0, len(req.Args)) + for name := range req.Args { + argNames = append(argNames, name) + } + sort.Strings(argNames) + queryArgSets = append(queryArgSets, argNames) _ = json.NewEncoder(w).Encode(crawlremote.QueryResult{ Values: []map[string]any{{ "thread_id": 10, @@ -656,8 +795,10 @@ func TestRemoteCloudModeDoesNotCreateLocalDB(t *testing.T) { if err := searchApp.Run(ctx, []string{"--config", configPath, "--json", "search", "openclaw/openclaw", "--query", "remote"}); err != nil { t.Fatalf("remote search: %v", err) } - if !sawQuery || !strings.Contains(searchOut.String(), `"remote search"`) { - t.Fatalf("remote search did not use worker query: saw=%v out=%s", sawQuery, searchOut.String()) + if len(queryArgSets) != 1 || + !slices.Equal(queryArgSets[0], []string{"limit", "mode", "owner", "query", "repo"}) || + !strings.Contains(searchOut.String(), `"remote search"`) { + t.Fatalf("remote search query args=%v out=%s", queryArgSets, searchOut.String()) } ghSearchApp := New() var ghSearchOut bytes.Buffer @@ -668,6 +809,10 @@ func TestRemoteCloudModeDoesNotCreateLocalDB(t *testing.T) { if !strings.Contains(ghSearchOut.String(), `"number": 42`) || !strings.Contains(ghSearchOut.String(), `"url": "https://github.com/openclaw/openclaw/issues/42"`) { t.Fatalf("remote gh search output = %s", ghSearchOut.String()) } + if len(queryArgSets) != 2 || + !slices.Equal(queryArgSets[1], []string{"kind", "limit", "owner", "query", "repo", "state"}) { + t.Fatalf("remote gh search query args=%v", queryArgSets) + } doctorApp := New() var doctorOut bytes.Buffer doctorApp.Stdout = &doctorOut @@ -692,6 +837,246 @@ func TestRemoteCloudModeDoesNotCreateLocalDB(t *testing.T) { } } +func testSnapshotPublishContract() crawlremote.Contract { + contract := crawlremote.BaseContract() + contract.Routes = append(contract.Routes, crawlremote.RouteSpec{ + Method: http.MethodGet, + Path: "/v1/apps/:app/archives/:archive/sqlite", + Auth: crawlremote.AuthReader, + }) + contract.Apps = []crawlremote.AppSpec{{ + App: "gitcrawl", + Queries: []crawlremote.QuerySpec{ + {Name: "gitcrawl.threads.search", Args: []string{"limit", "mode", "state", "kind", "query", "repo", "owner"}}, + {Name: "gitcrawl.clusters.related", Args: []string{"owner", "repo", "number"}}, + {Name: "gitcrawl.clusters.list", Args: []string{"owner", "repo", "status", "min_size"}}, + {Name: "gitcrawl.clusters.members", Args: []string{"owner", "repo", "cluster_id"}}, + {Name: "gitcrawl.pull_requests.review_context", Args: []string{"owner", "repo", "number"}}, + {Name: "gitcrawl.coverage", Args: []string{"dataset"}}, + }, + IngestTables: []crawlremote.IngestTableSpec{ + {Name: "repositories", Columns: gitcrawlRepositoryColumns}, + {Name: "threads", Columns: gitcrawlThreadColumns}, + {Name: "thread_revisions", Columns: []string{ + "id", "thread_id", "source_updated_at", "content_hash", "title_hash", + "body_hash", "labels_hash", "created_at", + }}, + {Name: "thread_fingerprints", Columns: []string{ + "id", "thread_revision_id", "algorithm_version", "fingerprint_hash", + "fingerprint_slug", "body_token_hash", "file_set_hash", "simhash64", "created_at", + }}, + {Name: "thread_key_summaries", Columns: []string{ + "id", "thread_revision_id", "summary_kind", "prompt_version", "provider", + "model", "input_hash", "output_hash", "key_text", "created_at", + }}, + {Name: "cluster_groups", Columns: []string{ + "id", "repo_id", "stable_key", "stable_slug", "status", "cluster_type", + "representative_thread_id", "title", "member_count", "created_at", "updated_at", + "closed_at", + }}, + {Name: "cluster_memberships", Columns: []string{ + "cluster_id", "thread_id", "role", "state", "score_to_representative", + "created_at", "updated_at", "removed_at", + }}, + {Name: "pull_request_details", Columns: []string{ + "thread_id", "repo_id", "number", "base_sha", "head_sha", "head_ref", + "head_repo_full_name", "mergeable_state", "additions", "deletions", + "changed_files", "fetched_at", "updated_at", + }}, + {Name: "pull_request_files", Columns: []string{ + "thread_id", "position", "path", "status", "additions", "deletions", + "changes", "previous_path", "fetched_at", + }}, + {Name: "dataset_coverage", Columns: gitcrawlCloudCoverageColumns}, + }, + Capabilities: []string{ + gitcrawlSnapshotAtomicCapability, + gitcrawlSnapshotCutoverCapability, + gitcrawlSnapshotProvenanceCapability, + gitcrawlSnapshotStagingCapability, + sqliteBundleGzipUploadCapability, + }, + }} + return contract +} + +func TestCloudPublishRejectsPublisherOnlyTokenBeforeMutation(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + cfg := config.Default() + cfg.DBPath = dbPath + if err := config.Save(configPath, cfg); err != nil { + t.Fatalf("save config: %v", err) + } + seedCommandFlowStore(t, dbPath) + + const tokenEnv = "GITCRAWL_TEST_PUBLISHER_ONLY_TOKEN" + t.Setenv(tokenEnv, "publisher-only-token") + whoamiRequests := 0 + mutations := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract": + w.Header().Set("content-type", "application/json") + _ = json.NewEncoder(w).Encode(testSnapshotPublishContract()) + case r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/whoami": + whoamiRequests++ + if got := r.Header.Get("authorization"); got != "Bearer publisher-only-token" { + http.Error(w, "missing bearer", http.StatusUnauthorized) + return + } + w.Header().Set("content-type", "application/json") + _ = json.NewEncoder(w).Encode(crawlremote.Identity{ + Owner: "openclaw", + Org: "openclaw", + Login: "publisher", + Roles: []string{crawlremote.AuthPublisher}, + }) + default: + if r.Method == http.MethodPost || + r.Method == http.MethodPut || + r.Method == http.MethodPatch || + r.Method == http.MethodDelete { + mutations++ + } + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + err := New().Run(ctx, []string{ + "--config", configPath, + "cloud", "publish", + "--remote", server.URL, + "--archive", "gitcrawl/openclaw__openclaw", + "--token-env", tokenEnv, + "--allow-incomplete", + "--json", + }) + if err == nil || !strings.Contains(err.Error(), "missing reader") { + t.Fatalf("cloud publish error = %v, want missing reader role", err) + } + if whoamiRequests != 1 { + t.Fatalf("whoami requests = %d, want 1", whoamiRequests) + } + if mutations != 0 { + t.Fatalf("remote mutations = %d, want 0", mutations) + } +} + +func TestGitcrawlPublisherStatusMatchesExactMetadata(t *testing.T) { + snapshotID := strings.Repeat("a", 64) + manifest := crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__openclaw", + SchemaName: gitcrawlCloudSchemaName, + SchemaVersion: gitcrawlCloudSchemaVersion, + SchemaHash: gitcrawlCloudSchemaHash, + SourceSyncAt: "2026-07-12T12:00:00Z", + SnapshotID: snapshotID, + SourceSHA256: snapshotID, + } + publicationCapabilities := gitcrawlCloudPublicationCapabilities(manifest.Capabilities) + status := crawlremote.PublisherStatus{ + App: "gitcrawl", + Archive: manifest.Archive, + ActiveSnapshotID: snapshotID, + CoverageComplete: true, + Snapshot: &crawlremote.ArchiveSnapshot{ + ID: snapshotID, + SourceSHA256: snapshotID, + SourceSyncAt: manifest.SourceSyncAt, + DatasetGeneratedAt: "2026-07-12T12:01:00Z", + SchemaName: gitcrawlCloudSchemaName, + SchemaVersion: gitcrawlCloudSchemaVersion, + SchemaHash: gitcrawlCloudSchemaHash, + Capabilities: publicationCapabilities, + CoverageComplete: true, + }, + } + if !gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) { + t.Fatal("exact staged snapshot did not match") + } + + activeMismatch := status + activeMismatch.ActiveSnapshotID = strings.Repeat("f", 64) + if gitcrawlPublisherStatusMatches(activeMismatch, manifest, publicationCapabilities) { + t.Fatal("publisher status candidate mismatch was reused") + } + + sourceMismatch := status + sourceMismatch.Snapshot = &crawlremote.ArchiveSnapshot{} + *sourceMismatch.Snapshot = *status.Snapshot + sourceMismatch.Snapshot.SourceSHA256 = strings.Repeat("b", 64) + if gitcrawlPublisherStatusMatches(sourceMismatch, manifest, publicationCapabilities) { + t.Fatal("source digest mismatch was reused") + } + + sourceSyncMismatch := status + sourceSyncMismatch.Snapshot = &crawlremote.ArchiveSnapshot{} + *sourceSyncMismatch.Snapshot = *status.Snapshot + sourceSyncMismatch.Snapshot.SourceSyncAt = "2026-07-12T12:00:01Z" + if gitcrawlPublisherStatusMatches(sourceSyncMismatch, manifest, publicationCapabilities) { + t.Fatal("source sync mismatch was reused") + } + + schemaMismatch := status + schemaMismatch.Snapshot = &crawlremote.ArchiveSnapshot{} + *schemaMismatch.Snapshot = *status.Snapshot + schemaMismatch.Snapshot.SchemaVersion++ + if gitcrawlPublisherStatusMatches(schemaMismatch, manifest, publicationCapabilities) { + t.Fatal("schema mismatch was reused") + } + + missingGeneratedAt := status + missingGeneratedAt.Snapshot = &crawlremote.ArchiveSnapshot{} + *missingGeneratedAt.Snapshot = *status.Snapshot + missingGeneratedAt.Snapshot.DatasetGeneratedAt = "" + if gitcrawlPublisherStatusMatches(missingGeneratedAt, manifest, publicationCapabilities) { + t.Fatal("snapshot without its staged generation timestamp was reused") + } + + status.CoverageComplete = false + if gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) { + t.Fatal("incomplete publisher status was reused") + } + status.CoverageComplete = true + + nestedIncomplete := status + nestedIncomplete.Snapshot = &crawlremote.ArchiveSnapshot{} + *nestedIncomplete.Snapshot = *status.Snapshot + nestedIncomplete.Snapshot.CoverageComplete = false + if gitcrawlPublisherStatusMatches(nestedIncomplete, manifest, publicationCapabilities) { + t.Fatal("snapshot with nested incomplete coverage was reused") + } + + manifest.Capabilities = []string{gitcrawlObservationOrderCapability} + publicationCapabilities = gitcrawlCloudPublicationCapabilities(manifest.Capabilities) + if gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) { + t.Fatal("staged snapshot reused without snapshot capability proof") + } + status.Snapshot.Capabilities = slices.Clone(publicationCapabilities) + if !gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) { + t.Fatal("staged snapshot with requested capability did not match") + } + status.Snapshot.Capabilities = append( + slices.Clone(publicationCapabilities), + "gitcrawl.unrequested-profile.v1", + ) + if gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) { + t.Fatal("snapshot with a broader publication profile was reused") + } + status.Snapshot.Capabilities = append( + slices.Clone(publicationCapabilities), + gitcrawlObservationOrderCapability, + ) + if gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) { + t.Fatal("snapshot with duplicate publication capabilities was reused") + } +} + func TestCloudPublishSendsLocalRows(t *testing.T) { ctx := context.Background() dir := t.TempDir() @@ -709,11 +1094,33 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { seenTables := map[string]crawlremote.IngestRequest{} var sawSQLitePart bool var sawSQLiteManifest bool + var sawCutover bool + var sawSQLiteRead bool + var snapshotID string + var sqliteImage []byte + var publishedSnapshot *crawlremote.ArchiveSnapshot + var publishedDatasets []crawlremote.DatasetCoverage + var publisherStatusSnapshotIDs []string + mutationCounter := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { + w.Header().Set("content-type", "application/json") + _ = json.NewEncoder(w).Encode(testSnapshotPublishContract()) + return + } if got := r.Header.Get("authorization"); got != "Bearer publish-token" { http.Error(w, "missing bearer", http.StatusUnauthorized) return } + if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/whoami" { + _ = json.NewEncoder(w).Encode(crawlremote.Identity{ + Owner: "openclaw", + Org: "openclaw", + Login: "publisher", + Roles: []string{crawlremote.AuthPublisher, crawlremote.AuthReader}, + }) + return + } if r.Method == http.MethodPut && r.URL.EscapedPath() == "/v1/apps/gitcrawl/archives/gitcrawl%2Fopenclaw__openclaw/sqlite" { uploadKind := r.Header.Get("x-crawl-sqlite-upload") payload, err := io.ReadAll(r.Body) @@ -725,7 +1132,17 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { switch uploadKind { case "bundle-part": sawSQLitePart = true - if r.Header.Get("content-type") != "application/gzip" || r.Header.Get("x-crawl-content-sha256") == "" { + if got := r.Header.Get("x-crawl-snapshot-id"); len(got) != 64 { + http.Error(w, "missing snapshot id", http.StatusBadRequest) + return + } else { + snapshotID = got + } + partSHA := r.Header.Get("x-crawl-content-sha256") + partIndex, err := strconv.Atoi(r.Header.Get("x-crawl-bundle-part-index")) + if r.Header.Get("content-type") != "application/gzip" || + partSHA == "" || + err != nil { http.Error(w, "bad bundle part headers", http.StatusBadRequest) return } @@ -750,11 +1167,21 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { http.Error(w, "bundle did not contain sqlite", http.StatusBadRequest) return } + sqliteImage = decompressed _ = json.NewEncoder(w).Encode(crawlremote.SQLiteUploadResult{ App: "gitcrawl", Archive: "gitcrawl/openclaw__openclaw", Complete: false, - Object: &crawlremote.SQLiteObject{Key: "v1/gitcrawl/gitcrawl%2Fopenclaw__openclaw/sqlite/chunks/current.db.gz.part-0000", Size: int64(len(payload))}, + Object: &crawlremote.SQLiteObject{ + Key: crawlremote.SQLiteSnapshotBundlePartKey( + "gitcrawl", + "gitcrawl/openclaw__openclaw", + snapshotID, + partSHA, + partIndex, + ), + Size: int64(len(payload)), + }, }) case "bundle-manifest": sawSQLiteManifest = true @@ -771,17 +1198,97 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { http.Error(w, "bundle privacy should disclose message bodies", http.StatusBadRequest) return } + if manifest.SnapshotID != snapshotID || + manifest.Object.Key != crawlremote.SQLiteSnapshotObjectKey("gitcrawl", "gitcrawl/openclaw__openclaw", snapshotID) { + http.Error(w, "bundle snapshot mismatch", http.StatusBadRequest) + return + } _ = json.NewEncoder(w).Encode(crawlremote.SQLiteBundleUploadResult{ App: "gitcrawl", Archive: "gitcrawl/openclaw__openclaw", Complete: true, - Bundle: &crawlremote.SQLiteBundle{Key: "v1/gitcrawl/gitcrawl%2Fopenclaw__openclaw/sqlite/current.manifest.json", Manifest: &manifest}, + Bundle: &crawlremote.SQLiteBundle{ + Key: crawlremote.SQLiteSnapshotBundleManifestKey("gitcrawl", "gitcrawl/openclaw__openclaw", snapshotID), + Manifest: &manifest, + }, }) default: http.Error(w, "missing upload kind", http.StatusBadRequest) } return } + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status") { + publisherStatusSnapshotIDs = append( + publisherStatusSnapshotIDs, + r.URL.Query().Get("snapshot_id"), + ) + if publishedSnapshot == nil { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(crawlremote.PublisherStatus{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__openclaw", + ActiveSnapshotID: snapshotID, + CoverageComplete: true, + Snapshot: publishedSnapshot, + }) + return + } + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/status") { + status := crawlremote.Status{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__openclaw", + } + if sawCutover && publishedSnapshot != nil { + status.Mode = "cloud" + status.SnapshotMode = "snapshot" + status.SnapshotCutoverAt = "2026-07-12T09:00:00Z" + status.SchemaName = publishedSnapshot.SchemaName + status.SchemaVersion = publishedSnapshot.SchemaVersion + status.SchemaHash = publishedSnapshot.SchemaHash + status.Capabilities = slices.Clone(publishedSnapshot.Capabilities) + status.ActiveSnapshotID = publishedSnapshot.ID + status.SourceSyncAt = publishedSnapshot.SourceSyncAt + status.DatasetGeneratedAt = publishedSnapshot.DatasetGeneratedAt + status.CoverageComplete = true + status.Datasets = slices.Clone(publishedDatasets) + snapshot := *publishedSnapshot + snapshot.CutoverAt = status.SnapshotCutoverAt + status.Snapshot = &snapshot + } + _ = json.NewEncoder(w).Encode(status) + return + } + if r.Method == http.MethodPost && strings.HasSuffix(r.URL.EscapedPath(), "/cutover") { + var request struct { + SnapshotID string `json:"snapshot_id"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil || request.SnapshotID != snapshotID { + http.Error(w, "cutover snapshot mismatch", http.StatusBadRequest) + return + } + sawCutover = true + _ = json.NewEncoder(w).Encode(crawlremote.CutoverResult{ + Archive: "gitcrawl/openclaw__openclaw", + SnapshotID: snapshotID, + SnapshotMode: "snapshot", + CutoverAt: "2026-07-12T09:00:00Z", + }) + return + } + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/sqlite") { + if !sawCutover || len(sqliteImage) == 0 { + http.Error(w, "bound snapshot unavailable", http.StatusConflict) + return + } + sawSQLiteRead = true + w.Header().Set("content-type", "application/vnd.sqlite3") + w.Header().Set("content-length", strconv.Itoa(len(sqliteImage))) + w.Header().Set("x-crawl-content-sha256", snapshotID) + _, _ = w.Write(sqliteImage) + return + } if r.Method != http.MethodPost || r.URL.EscapedPath() != "/v1/apps/gitcrawl/archives/gitcrawl%2Fopenclaw__openclaw/ingest" { http.NotFound(w, r) return @@ -791,13 +1298,53 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { http.Error(w, err.Error(), http.StatusBadRequest) return } - if body.Manifest.App != "gitcrawl" || body.Manifest.Archive != "gitcrawl/openclaw__openclaw" { + if body.Manifest.App != "gitcrawl" || + body.Manifest.Archive != "gitcrawl/openclaw__openclaw" || + body.Manifest.SnapshotID != snapshotID || + body.Manifest.SourceSHA256 != snapshotID { http.Error(w, "manifest mismatch", http.StatusBadRequest) return } + if body.Table == "dataset_coverage" { + if body.MutationToken == "" || !body.Final || len(body.Rows) != 9 { + http.Error(w, "coverage activation mismatch", http.StatusBadRequest) + return + } + publishedDatasets = testGitcrawlDatasetCoverageRows(t, body.Rows) + for _, row := range body.Rows { + if row[len(row)-1] != body.MutationToken { + http.Error(w, "coverage token mismatch", http.StatusBadRequest) + return + } + } + publishedSnapshot = &crawlremote.ArchiveSnapshot{ + ID: body.Manifest.SnapshotID, + SourceSHA256: body.Manifest.SourceSHA256, + SchemaName: body.Manifest.SchemaName, + SchemaVersion: body.Manifest.SchemaVersion, + SchemaHash: body.Manifest.SchemaHash, + Capabilities: gitcrawlCloudPublicationCapabilities( + body.Manifest.Capabilities, + ), + SourceSyncAt: body.Manifest.SourceSyncAt, + DatasetGeneratedAt: fmt.Sprint(body.Rows[0][5]), + CoverageComplete: true, + } + } seenTables[body.Table] = body + mutationCounter++ + token := fmt.Sprintf("mutation-%d", mutationCounter) + if body.Table == "dataset_coverage" { + token = body.MutationToken + } w.Header().Set("content-type", "application/json") - _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{Table: body.Table, RowsAccepted: int64(len(body.Rows)), Complete: body.Final}) + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + Table: body.Table, + SnapshotID: snapshotID, + MutationToken: token, + RowsAccepted: int64(len(body.Rows)), + Complete: body.Final, + }) })) defer server.Close() @@ -810,13 +1357,14 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { "--remote", server.URL, "--archive", "gitcrawl/openclaw__openclaw", "--token-env", tokenEnv, + "--allow-incomplete", "--json", }); err != nil { t.Fatalf("cloud publish: %v", err) } - if len(seenTables) != 2 { - t.Fatalf("tables = %v, want repositories and threads", seenTables) + if len(seenTables) != 10 { + t.Fatalf("tables = %v, want all cloud-v2 datasets and coverage", seenTables) } if got := len(seenTables["repositories"].Rows); got != 1 { t.Fatalf("repositories rows = %d, want 1", got) @@ -824,28 +1372,954 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { if got := len(seenTables["threads"].Rows); got != 3 { t.Fatalf("threads rows = %d, want 3", got) } - if !seenTables["threads"].Final { - t.Fatalf("threads batch should finish ingest") + if seenTables["threads"].Final { + t.Fatalf("threads batch should remain staged") + } + if !seenTables["dataset_coverage"].Final { + t.Fatalf("coverage batch should activate the snapshot") } if !sawSQLitePart || !sawSQLiteManifest { t.Fatalf("cloud publish did not upload compressed sqlite bundle: part=%v manifest=%v", sawSQLitePart, sawSQLiteManifest) } + if !sawCutover { + t.Fatal("cloud publish did not cut over the activated snapshot") + } + if !sawSQLiteRead { + t.Fatal("cloud publish did not verify the readable bound SQLite snapshot") + } var payload map[string]any if err := json.Unmarshal(out.Bytes(), &payload); err != nil { t.Fatalf("decode output: %v\n%s", err, out.String()) } - if payload["repositories"] != float64(1) || payload["threads"] != float64(3) { + datasets, ok := payload["datasets"].(map[string]any) + if !ok || datasets["repositories"] != float64(1) || datasets["threads"] != float64(3) { t.Fatalf("unexpected output: %#v", payload) } + if payload["snapshot_id"] != snapshotID || payload["source_sha256"] != snapshotID { + t.Fatalf("missing snapshot provenance: %#v", payload) + } if payload["sqlite_bundle"] == nil { t.Fatalf("missing sqlite bundle output: %#v", payload) } + for _, requestedSnapshotID := range publisherStatusSnapshotIDs { + if requestedSnapshotID != snapshotID { + t.Fatalf( + "publisher status requested snapshot %q, want %q", + requestedSnapshotID, + snapshotID, + ) + } + } privacy, ok := payload["sqlite_bundle_privacy"].(map[string]any) if !ok || privacy["includes_private_messages"] != true { t.Fatalf("missing sqlite bundle privacy output: %#v", payload) } } +func TestCloudPublishRejectsMissingSnapshotCapabilityBeforeUpload(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + cfg := config.Default() + cfg.DBPath = dbPath + if err := config.Save(configPath, cfg); err != nil { + t.Fatalf("save config: %v", err) + } + seedCommandFlowStore(t, dbPath) + + const tokenEnv = "GITCRAWL_TEST_CONTRACT_TOKEN" + t.Setenv(tokenEnv, "publish-token") + mutations := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { + contract := testSnapshotPublishContract() + contract.Apps[0].Capabilities = []string{ + gitcrawlSnapshotAtomicCapability, + sqliteBundleGzipUploadCapability, + } + w.Header().Set("content-type", "application/json") + _ = json.NewEncoder(w).Encode(contract) + return + } + mutations++ + http.Error(w, "unexpected mutation", http.StatusInternalServerError) + })) + defer server.Close() + + err := New().Run(ctx, []string{ + "--config", configPath, + "cloud", "publish", + "--remote", server.URL, + "--archive", "gitcrawl/openclaw__openclaw", + "--token-env", tokenEnv, + "--allow-incomplete", + "--json", + }) + if err == nil || !strings.Contains(err.Error(), gitcrawlSnapshotProvenanceCapability) { + t.Fatalf("cloud publish error = %v", err) + } + if mutations != 0 { + t.Fatalf("remote mutations = %d, want 0", mutations) + } +} + +func TestCloudPublishRejectsMissingRequestedCapabilityBeforeUpload(t *testing.T) { + tests := []struct { + name string + args []string + missingCapability string + }{ + { + name: "observation order", + args: []string{"--observation-order", "--stage-only"}, + missingCapability: gitcrawlObservationOrderCapability, + }, + { + name: "stage isolation", + args: []string{"--stage-only"}, + missingCapability: gitcrawlSnapshotStagingCapability, + }, + { + name: "cutover", + missingCapability: gitcrawlSnapshotCutoverCapability, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + cfg := config.Default() + cfg.DBPath = dbPath + if err := config.Save(configPath, cfg); err != nil { + t.Fatalf("save config: %v", err) + } + seedCommandFlowStore(t, dbPath) + + const tokenEnv = "GITCRAWL_TEST_OPTION_CONTRACT_TOKEN" + t.Setenv(tokenEnv, "publish-token") + mutations := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { + contract := testSnapshotPublishContract() + capabilities := make([]string, 0, len(contract.Apps[0].Capabilities)) + for _, capability := range contract.Apps[0].Capabilities { + if capability != test.missingCapability { + capabilities = append(capabilities, capability) + } + } + contract.Apps[0].Capabilities = capabilities + w.Header().Set("content-type", "application/json") + _ = json.NewEncoder(w).Encode(contract) + return + } + mutations++ + http.Error(w, "unexpected mutation", http.StatusInternalServerError) + })) + defer server.Close() + + args := []string{ + "--config", configPath, + "cloud", "publish", + "--remote", server.URL, + "--archive", "gitcrawl/openclaw__openclaw", + "--token-env", tokenEnv, + "--allow-incomplete", + "--json", + } + args = append(args, test.args...) + err := New().Run(ctx, args) + if err == nil || !strings.Contains(err.Error(), test.missingCapability) { + t.Fatalf("cloud publish error = %v", err) + } + if mutations != 0 { + t.Fatalf("remote mutations = %d, want 0", mutations) + } + }) + } +} + +func TestCloudPublishRejectsIncompleteRemoteSurfaceBeforeUpload(t *testing.T) { + type remoteSurfaceTest struct { + name string + want string + mutate func(*crawlremote.Contract) + extraArgs []string + } + tests := []remoteSurfaceTest{ + { + name: "missing authenticated whoami route", + want: "GET /v1/whoami", + mutate: func(contract *crawlremote.Contract) { + contract.Routes = slices.DeleteFunc( + contract.Routes, + func(route crawlremote.RouteSpec) bool { + return route.Method == http.MethodGet && + route.Path == "/v1/whoami" + }, + ) + }, + }, + { + name: "missing ingest column", + want: "pull_request_files is missing required column position", + mutate: func(contract *crawlremote.Contract) { + for tableIndex := range contract.Apps[0].IngestTables { + table := &contract.Apps[0].IngestTables[tableIndex] + if table.Name == "pull_request_files" { + table.Columns = slices.DeleteFunc( + table.Columns, + func(column string) bool { return column == "position" }, + ) + } + } + }, + }, + { + name: "missing cutover route", + want: "POST /v1/apps/:app/archives/:archive/cutover", + mutate: func(contract *crawlremote.Contract) { + contract.Routes = slices.DeleteFunc( + contract.Routes, + func(route crawlremote.RouteSpec) bool { + return route.Method == http.MethodPost && + route.Path == "/v1/apps/:app/archives/:archive/cutover" + }, + ) + }, + }, + { + name: "missing publisher status route", + want: "GET /v1/apps/:app/archives/:archive/publish-status", + mutate: func(contract *crawlremote.Contract) { + contract.Routes = slices.DeleteFunc( + contract.Routes, + func(route crawlremote.RouteSpec) bool { + return route.Method == http.MethodGet && + route.Path == "/v1/apps/:app/archives/:archive/publish-status" + }, + ) + }, + }, + { + name: "publisher status route with reader auth", + want: "GET /v1/apps/:app/archives/:archive/publish-status", + mutate: func(contract *crawlremote.Contract) { + for index := range contract.Routes { + route := &contract.Routes[index] + if route.Method == http.MethodGet && + route.Path == "/v1/apps/:app/archives/:archive/publish-status" { + route.Auth = crawlremote.AuthReader + } + } + }, + }, + { + name: "missing reader query route", + want: "POST /v1/apps/:app/archives/:archive/query", + mutate: func(contract *crawlremote.Contract) { + contract.Routes = slices.DeleteFunc( + contract.Routes, + func(route crawlremote.RouteSpec) bool { + return route.Method == http.MethodPost && + route.Path == "/v1/apps/:app/archives/:archive/query" + }, + ) + }, + }, + { + name: "missing SQLite hydration route", + want: "GET /v1/apps/:app/archives/:archive/sqlite", + mutate: func(contract *crawlremote.Contract) { + contract.Routes = slices.DeleteFunc( + contract.Routes, + func(route crawlremote.RouteSpec) bool { + return route.Method == http.MethodGet && + route.Path == "/v1/apps/:app/archives/:archive/sqlite" + }, + ) + }, + }, + { + name: "SQLite hydration route with publisher auth", + want: "GET /v1/apps/:app/archives/:archive/sqlite", + mutate: func(contract *crawlremote.Contract) { + for index := range contract.Routes { + route := &contract.Routes[index] + if route.Method == http.MethodGet && + route.Path == "/v1/apps/:app/archives/:archive/sqlite" { + route.Auth = crawlremote.AuthPublisher + } + } + }, + }, + { + name: "zero reader queries", + want: "required reader query gitcrawl.threads.search", + mutate: func(contract *crawlremote.Contract) { + contract.Apps[0].Queries = nil + }, + }, + { + name: "thread search missing mode and limit", + want: "reader query gitcrawl.threads.search has arguments", + mutate: func(contract *crawlremote.Contract) { + contract.Apps[0].Queries[0].Args = []string{ + "owner", "repo", "query", "kind", "state", + } + }, + }, + { + name: "thread search duplicate argument", + want: "reader query gitcrawl.threads.search has arguments", + mutate: func(contract *crawlremote.Contract) { + contract.Apps[0].Queries[0].Args = []string{ + "owner", "repo", "query", "kind", "state", "mode", "mode", + } + }, + }, + { + name: "duplicate reader query", + want: "required reader query gitcrawl.threads.search more than once", + mutate: func(contract *crawlremote.Contract) { + contract.Apps[0].Queries = append( + contract.Apps[0].Queries, + contract.Apps[0].Queries[0], + ) + }, + }, + } + for _, required := range gitcrawlCloudReaderQuerySpecs() { + tests = append(tests, remoteSurfaceTest{ + name: "missing reader query " + required.Name, + want: "required reader query " + required.Name, + mutate: func(contract *crawlremote.Contract) { + contract.Apps[0].Queries = slices.DeleteFunc( + contract.Apps[0].Queries, + func(query crawlremote.QuerySpec) bool { + return query.Name == required.Name + }, + ) + }, + }) + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + cfg := config.Default() + cfg.DBPath = dbPath + if err := config.Save(configPath, cfg); err != nil { + t.Fatalf("save config: %v", err) + } + seedCommandFlowStore(t, dbPath) + + const tokenEnv = "GITCRAWL_TEST_SURFACE_CONTRACT_TOKEN" + t.Setenv(tokenEnv, "publish-token") + mutations := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { + contract := testSnapshotPublishContract() + test.mutate(&contract) + w.Header().Set("content-type", "application/json") + _ = json.NewEncoder(w).Encode(contract) + return + } + mutations++ + http.Error(w, "unexpected mutation", http.StatusInternalServerError) + })) + defer server.Close() + + args := []string{ + "--config", configPath, + "cloud", "publish", + "--remote", server.URL, + "--archive", "gitcrawl/openclaw__openclaw", + "--token-env", tokenEnv, + "--allow-incomplete", + "--json", + } + args = append(args, test.extraArgs...) + err := New().Run(ctx, args) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("cloud publish error = %v", err) + } + if mutations != 0 { + t.Fatalf("remote mutations = %d, want 0", mutations) + } + }) + } +} + +func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + cfg := config.Default() + cfg.DBPath = dbPath + if err := config.Save(configPath, cfg); err != nil { + t.Fatalf("save config: %v", err) + } + seedCommandFlowStore(t, dbPath) + + const tokenEnv = "GITCRAWL_TEST_RESUME_TOKEN" + const concurrentDatasetGeneratedAt = "2000-01-02T03:04:05.678901234Z" + t.Setenv(tokenEnv, "publish-token") + var snapshotID string + var stagedSnapshot *crawlremote.ArchiveSnapshot + var stagedDatasets []crawlremote.DatasetCoverage + servingSnapshotID := strings.Repeat("f", 64) + var sqliteImage []byte + uploadRequests := 0 + ingestRequests := 0 + publisherStatusRequests := 0 + var publisherStatusSnapshotIDs []string + snapshotMismatchResponsesRemaining := 1 + readerStatusRequests := 0 + sqliteReadRequests := 0 + cutovers := 0 + hydrationFailuresRemaining := 1 + concurrentFinalCompletionsRemaining := 1 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { + contract := testSnapshotPublishContract() + contract.Apps[0].Capabilities = append( + contract.Apps[0].Capabilities, + gitcrawlObservationOrderCapability, + ) + for index := range contract.Apps[0].IngestTables { + table := &contract.Apps[0].IngestTables[index] + if table.Name == "threads" || table.Name == "thread_revisions" { + table.Columns = append(table.Columns, "observation_sequence") + } + } + w.Header().Set("content-type", "application/json") + _ = json.NewEncoder(w).Encode(contract) + return + } + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/status") { + readerStatusRequests++ + if r.Header.Get("authorization") != "Bearer publish-token" { + http.Error(w, "missing dual-role bearer", http.StatusForbidden) + return + } + status := crawlremote.Status{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__openclaw", + ActiveSnapshotID: servingSnapshotID, + } + readerProjectionExact := false + switch readerStatusRequests { + case 3, 5, 6, 7: + readerProjectionExact = true + } + if servingSnapshotID == snapshotID && + readerProjectionExact && + stagedSnapshot != nil { + status.Mode = "cloud" + status.SnapshotMode = "snapshot" + status.SnapshotCutoverAt = "2026-07-12T11:00:00Z" + status.SchemaName = stagedSnapshot.SchemaName + status.SchemaVersion = stagedSnapshot.SchemaVersion + status.SchemaHash = stagedSnapshot.SchemaHash + status.Capabilities = slices.Clone(stagedSnapshot.Capabilities) + status.SourceSyncAt = stagedSnapshot.SourceSyncAt + status.DatasetGeneratedAt = stagedSnapshot.DatasetGeneratedAt + status.CoverageComplete = true + status.Datasets = slices.Clone(stagedDatasets) + snapshot := *stagedSnapshot + snapshot.CutoverAt = status.SnapshotCutoverAt + status.Snapshot = &snapshot + } + _ = json.NewEncoder(w).Encode(status) + return + } + if got := r.Header.Get("authorization"); got != "Bearer publish-token" { + http.Error(w, "missing bearer", http.StatusUnauthorized) + return + } + if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/whoami" { + _ = json.NewEncoder(w).Encode(crawlremote.Identity{ + Owner: "openclaw", + Org: "openclaw", + Login: "publisher", + Roles: []string{crawlremote.AuthPublisher, crawlremote.AuthReader}, + }) + return + } + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/sqlite") { + sqliteReadRequests++ + if servingSnapshotID != snapshotID || len(sqliteImage) == 0 { + http.Error(w, "bound snapshot unavailable", http.StatusConflict) + return + } + if hydrationFailuresRemaining > 0 { + hydrationFailuresRemaining-- + http.Error(w, "temporary hydration failure", http.StatusConflict) + return + } + w.Header().Set("content-type", "application/vnd.sqlite3") + w.Header().Set("content-length", strconv.Itoa(len(sqliteImage))) + w.Header().Set("x-crawl-content-sha256", snapshotID) + _, _ = w.Write(sqliteImage) + return + } + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status") { + publisherStatusRequests++ + requestedSnapshotID := r.URL.Query().Get("snapshot_id") + publisherStatusSnapshotIDs = append( + publisherStatusSnapshotIDs, + requestedSnapshotID, + ) + if requestedSnapshotID != "" && snapshotMismatchResponsesRemaining > 0 { + snapshotMismatchResponsesRemaining-- + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(map[string]any{ + "error": "snapshot_mismatch", + "message": "snapshot ingest is incomplete", + }) + return + } + if stagedSnapshot == nil { + http.NotFound(w, r) + return + } + statusSnapshot := stagedSnapshot + if requestedSnapshotID == "" { + concurrent := *stagedSnapshot + concurrent.ID = strings.Repeat("c", 64) + concurrent.SourceSHA256 = concurrent.ID + statusSnapshot = &concurrent + } + activeSnapshotID := "" + if statusSnapshot != nil { + activeSnapshotID = statusSnapshot.ID + } + _ = json.NewEncoder(w).Encode(crawlremote.PublisherStatus{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__openclaw", + ActiveSnapshotID: activeSnapshotID, + CoverageComplete: statusSnapshot != nil, + Snapshot: statusSnapshot, + }) + return + } + if r.Method == http.MethodPut && strings.HasSuffix(r.URL.EscapedPath(), "/sqlite") { + uploadRequests++ + w.Header().Set("content-type", "application/json") + switch r.Header.Get("x-crawl-sqlite-upload") { + case "bundle-part": + snapshotID = r.Header.Get("x-crawl-snapshot-id") + partSHA := r.Header.Get("x-crawl-content-sha256") + partIndex, err := strconv.Atoi(r.Header.Get("x-crawl-bundle-part-index")) + if len(snapshotID) != 64 || len(partSHA) != 64 || err != nil { + http.Error(w, "invalid bundle part", http.StatusBadRequest) + return + } + compressed, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + reader, err := gzip.NewReader(bytes.NewReader(compressed)) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + sqliteImage, err = io.ReadAll(reader) + if closeErr := reader.Close(); err == nil { + err = closeErr + } + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + _ = json.NewEncoder(w).Encode(crawlremote.SQLiteUploadResult{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__openclaw", + Complete: false, + Object: &crawlremote.SQLiteObject{ + Key: crawlremote.SQLiteSnapshotBundlePartKey( + "gitcrawl", + "gitcrawl/openclaw__openclaw", + snapshotID, + partSHA, + partIndex, + ), + }, + }) + case "bundle-manifest": + var manifest crawlremote.SQLiteBundleManifest + if err := json.NewDecoder(r.Body).Decode(&manifest); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if manifest.SnapshotID != snapshotID { + http.Error(w, "snapshot mismatch", http.StatusBadRequest) + return + } + _ = json.NewEncoder(w).Encode(crawlremote.SQLiteBundleUploadResult{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__openclaw", + Complete: true, + Bundle: &crawlremote.SQLiteBundle{ + Key: crawlremote.SQLiteSnapshotBundleManifestKey("gitcrawl", "gitcrawl/openclaw__openclaw", snapshotID), + Manifest: &manifest, + }, + }) + default: + http.Error(w, "missing upload kind", http.StatusBadRequest) + } + return + } + if r.Method == http.MethodPost && strings.HasSuffix(r.URL.EscapedPath(), "/ingest") { + ingestRequests++ + if stagedSnapshot != nil { + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(map[string]any{ + "error": "snapshot_active", + "message": "activated Gitcrawl snapshots are immutable", + }) + return + } + var body crawlremote.IngestRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if body.Manifest.SnapshotID != snapshotID || + body.Manifest.SourceSHA256 != snapshotID { + http.Error(w, "ingest provenance mismatch", http.StatusBadRequest) + return + } + mutationToken := fmt.Sprintf("resume-mutation-%d", ingestRequests) + if body.Final { + if body.Table != "dataset_coverage" || body.MutationToken == "" { + http.Error(w, "invalid snapshot activation", http.StatusBadRequest) + return + } + mutationToken = body.MutationToken + stagedDatasets = testGitcrawlDatasetCoverageRows(t, body.Rows) + if got := fmt.Sprint(body.Rows[0][5]); got == concurrentDatasetGeneratedAt { + t.Fatalf("loser generation unexpectedly equals independent winner generation %q", got) + } + for index := range stagedDatasets { + stagedDatasets[index].DatasetGeneratedAt = concurrentDatasetGeneratedAt + } + stagedSnapshot = &crawlremote.ArchiveSnapshot{ + ID: body.Manifest.SnapshotID, + SourceSHA256: body.Manifest.SourceSHA256, + SourceSyncAt: body.Manifest.SourceSyncAt, + DatasetGeneratedAt: concurrentDatasetGeneratedAt, + SchemaName: body.Manifest.SchemaName, + SchemaVersion: body.Manifest.SchemaVersion, + SchemaHash: body.Manifest.SchemaHash, + Capabilities: gitcrawlCloudPublicationCapabilities( + body.Manifest.Capabilities, + ), + CoverageComplete: true, + } + if concurrentFinalCompletionsRemaining > 0 { + concurrentFinalCompletionsRemaining-- + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(map[string]any{ + "error": "snapshot_active", + "message": "a concurrent publisher completed the snapshot", + }) + return + } + } + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + Table: body.Table, + SnapshotID: body.Manifest.SnapshotID, + MutationToken: mutationToken, + RowsAccepted: int64(len(body.Rows)), + Complete: body.Final, + }) + return + } + if r.Method == http.MethodPost && strings.HasSuffix(r.URL.EscapedPath(), "/cutover") { + var request struct { + SnapshotID string `json:"snapshot_id"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil || + request.SnapshotID != snapshotID { + http.Error(w, "cutover snapshot mismatch", http.StatusBadRequest) + return + } + if stagedSnapshot == nil || stagedSnapshot.ID != request.SnapshotID { + http.Error(w, "snapshot is not staged", http.StatusConflict) + return + } + cutovers++ + servingSnapshotID = request.SnapshotID + _ = json.NewEncoder(w).Encode(crawlremote.CutoverResult{ + Archive: "gitcrawl/openclaw__openclaw", + SnapshotID: snapshotID, + SnapshotMode: "snapshot", + CutoverAt: "2026-07-12T11:00:00Z", + }) + return + } + http.NotFound(w, r) + })) + defer server.Close() + + baseArgs := []string{ + "--config", configPath, + "cloud", "publish", + "--remote", server.URL, + "--archive", "gitcrawl/openclaw__openclaw", + "--token-env", tokenEnv, + "--allow-incomplete", + "--observation-order", + "--json", + } + stageIngestRequests := 0 + stageUploadRequests := 0 + originalServingSnapshotID := servingSnapshotID + var stagedDatasetGeneratedAt string + for index, extra := range [][]string{{"--stage-only"}, nil, nil, nil} { + app := New() + var output bytes.Buffer + app.Stdout = &output + args := append(append([]string(nil), baseArgs...), extra...) + err := app.Run(ctx, args) + if index == 1 { + if err == nil || !strings.Contains(err.Error(), "temporary hydration failure") { + t.Fatalf("first cutover hydration error = %v", err) + } + if servingSnapshotID != snapshotID { + t.Fatalf("failed hydration did not leave snapshot serving: %q", servingSnapshotID) + } + continue + } + if err != nil { + t.Fatalf("cloud publish run %d: %v", index+1, err) + } + var result struct { + DatasetGeneratedAt string `json:"dataset_generated_at"` + AlreadyStaged bool `json:"already_staged"` + AlreadyCutOver bool `json:"already_cut_over"` + MutationToken string `json:"mutation_token"` + Cutover *crawlremote.CutoverResult `json:"cutover"` + } + if err := json.Unmarshal(output.Bytes(), &result); err != nil { + t.Fatalf("decode run %d output: %v\n%s", index+1, err, output.String()) + } + if index == 0 && result.Cutover != nil { + t.Fatalf("stage-only publish unexpectedly cut over: %#v", result.Cutover) + } + if index == 0 && !result.AlreadyStaged { + t.Fatal("initial stage-only publish did not recover concurrent completion") + } + if index == 0 { + stageIngestRequests = ingestRequests + stageUploadRequests = uploadRequests + if stageIngestRequests == 0 || stagedSnapshot == nil { + t.Fatal("stage-only publish did not complete the candidate snapshot") + } + if servingSnapshotID != originalServingSnapshotID { + t.Fatalf("stage-only changed serving snapshot to %q", servingSnapshotID) + } + stagedDatasetGeneratedAt = result.DatasetGeneratedAt + if stagedDatasetGeneratedAt != concurrentDatasetGeneratedAt || + result.MutationToken != "" { + t.Fatalf("stage-only did not bind its generation: %#v", result) + } + time.Sleep(2 * time.Millisecond) + } + if index == 2 { + if result.AlreadyCutOver || result.Cutover == nil { + t.Fatalf("incomplete reader status incorrectly skipped cutover: %s", output.String()) + } + } + if index == 3 { + if !result.AlreadyStaged || !result.AlreadyCutOver || result.Cutover != nil { + t.Fatalf("retry did not reuse the staged serving snapshot: %s", output.String()) + } + if result.DatasetGeneratedAt != stagedDatasetGeneratedAt { + t.Fatalf( + "resumed generation = %q, want staged generation %q", + result.DatasetGeneratedAt, + stagedDatasetGeneratedAt, + ) + } + if result.MutationToken != "" { + t.Fatalf("resumed publish minted mutation token %q", result.MutationToken) + } + } + } + if ingestRequests != stageIngestRequests { + t.Fatalf( + "ingest requests = %d after complete resume, want stage-only count %d", + ingestRequests, + stageIngestRequests, + ) + } + if uploadRequests != stageUploadRequests { + t.Fatalf( + "upload requests = %d after staged resume, want stage-only count %d", + uploadRequests, + stageUploadRequests, + ) + } + if cutovers != 2 { + t.Fatalf("cutovers = %d, want 2 after incomplete reader status", cutovers) + } + if servingSnapshotID != snapshotID { + t.Fatalf("serving snapshot = %q, want staged snapshot %q", servingSnapshotID, snapshotID) + } + if publisherStatusRequests != 8 { + t.Fatalf( + "publisher status requests = %d, want 8 including concurrent completion recovery", + publisherStatusRequests, + ) + } + for _, requestedSnapshotID := range publisherStatusSnapshotIDs { + if requestedSnapshotID != snapshotID { + t.Fatalf( + "publisher status requested snapshot %q, want %q", + requestedSnapshotID, + snapshotID, + ) + } + } + if readerStatusRequests != 7 { + t.Fatalf( + "reader status requests = %d, want 7 pre-cutover and exact post-cutover checks", + readerStatusRequests, + ) + } + if sqliteReadRequests != 3 { + t.Fatalf("SQLite read requests = %d, want failed hydration plus two retries", sqliteReadRequests) + } +} + +func testGitcrawlDatasetCoverageRows( + t *testing.T, + rows [][]any, +) []crawlremote.DatasetCoverage { + t.Helper() + datasets := make([]crawlremote.DatasetCoverage, 0, len(rows)) + for _, row := range rows { + if len(row) != len(gitcrawlCloudCoverageColumns) { + t.Fatalf("coverage row columns = %d, want %d", len(row), len(gitcrawlCloudCoverageColumns)) + } + number := func(value any) int64 { + t.Helper() + switch typed := value.(type) { + case float64: + return int64(typed) + case int64: + return typed + default: + t.Fatalf("coverage count type = %T", value) + return 0 + } + } + complete, ok := row[6].(bool) + if !ok { + t.Fatalf("coverage complete type = %T", row[6]) + } + covered := number(row[3]) + datasets = append(datasets, crawlremote.DatasetCoverage{ + Dataset: fmt.Sprint(row[0]), + RowCount: number(row[1]), + EligibleCount: number(row[2]), + CoveredCount: covered, + FreshCount: covered, + MaxSourceAt: fmt.Sprint(row[4]), + DatasetGeneratedAt: fmt.Sprint(row[5]), + Complete: complete, + }) + } + return datasets +} + +func TestCloudPublishRejectsIncompleteHydrationBeforeRemoteMutation(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + cfg := config.Default() + cfg.DBPath = dbPath + if err := config.Save(cfgPath, cfg); err != nil { + t.Fatalf("save config: %v", err) + } + seedCommandFlowStore(t, dbPath) + + tokenEnv := "GITCRAWL_TEST_INCOMPLETE_PUBLISH_TOKEN" + t.Setenv(tokenEnv, "publish-token") + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + http.Error(w, "unexpected request", http.StatusInternalServerError) + })) + defer server.Close() + + err := New().Run(ctx, []string{ + "--config", cfgPath, + "cloud", "publish", + "--remote", server.URL, + "--archive", "gitcrawl/openclaw__openclaw", + "--token-env", tokenEnv, + "--json", + }) + if err == nil || !strings.Contains(err.Error(), "cloud snapshot enrichment is incomplete") { + t.Fatalf("cloud publish error = %v", err) + } + if requests != 0 { + t.Fatalf("remote requests = %d, want 0", requests) + } +} + +func TestCloudPublishRejectsLegacyExportSchemaBeforeRemoteMutation(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + cfg := config.Default() + cfg.DBPath = dbPath + if err := config.Save(configPath, cfg); err != nil { + t.Fatalf("save config: %v", err) + } + seedCommandFlowStore(t, dbPath) + seedDoctorLegacyPullRequestFilesSchema(t, ctx, dbPath) + + const tokenEnv = "GITCRAWL_TEST_LEGACY_EXPORT_TOKEN" + t.Setenv(tokenEnv, "publish-token") + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + http.Error(w, "unexpected remote request", http.StatusInternalServerError) + })) + defer server.Close() + + err := New().Run(ctx, []string{ + "--config", configPath, + "cloud", "publish", + "--remote", server.URL, + "--archive", "gitcrawl/openclaw__openclaw", + "--token-env", tokenEnv, + "--allow-incomplete", + "--json", + }) + if err == nil || + !strings.Contains(err.Error(), "prepare cloud dataset pull_request_files export") || + !strings.Contains(err.Error(), "no such column: position") { + t.Fatalf("cloud publish error = %v", err) + } + if requests != 0 { + t.Fatalf("remote requests = %d, want 0", requests) + } +} + func TestRemoteLoginStoresKeyringToken(t *testing.T) { ctx := context.Background() dir := t.TempDir() diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index bb796c9..3c64475 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -4,13 +4,17 @@ import ( "context" "crypto/sha256" "database/sql" + "encoding/json" "errors" "flag" "fmt" "io" "net/http" + "net/url" "os" "path/filepath" + "slices" + "strconv" "strings" "time" @@ -19,10 +23,45 @@ import ( ) const ( - gitcrawlCloudBatchSize = 250 - gitcrawlCloudSQLiteBundleChunkSize = int64(64 * 1024 * 1024) + gitcrawlCloudBatchSize = 250 + gitcrawlCloudIngestRequestMaxBytes = int64(4 * 1024 * 1024) + gitcrawlCloudSQLiteBundleChunkSize = int64(64 * 1024 * 1024) + gitcrawlCloudPublishPreflightTimeout = 30 * time.Second + gitcrawlCloudPostCutoverStatusAttempts = 5 + gitcrawlCloudPostCutoverStatusRetryDelay = 100 * time.Millisecond + gitcrawlCloudHydrationTimeout = 10 * time.Minute + + gitcrawlSnapshotAtomicCapability = "gitcrawl.snapshot.atomic" + gitcrawlSnapshotCutoverCapability = "gitcrawl.snapshot.cutover" + gitcrawlSnapshotProvenanceCapability = "gitcrawl.snapshot.provenance.v1" + gitcrawlSnapshotStagingCapability = "gitcrawl.snapshot.staging.v1" + sqliteBundleGzipUploadCapability = "sqlite.bundle.gzip.upload" ) +var gitcrawlCloudCoverageColumns = []string{ + "dataset", "row_count", "eligible_count", "covered_count", + "max_source_at", "dataset_generated_at", "complete", "mutation_token", +} + +func gitcrawlCloudReaderQuerySpecs() []crawlremote.QuerySpec { + return []crawlremote.QuerySpec{ + {Name: "gitcrawl.threads.search", Args: []string{"owner", "repo", "query", "kind", "state", "mode", "limit"}}, + {Name: "gitcrawl.clusters.related", Args: []string{"owner", "repo", "number"}}, + {Name: "gitcrawl.clusters.list", Args: []string{"owner", "repo", "status", "min_size"}}, + {Name: "gitcrawl.clusters.members", Args: []string{"owner", "repo", "cluster_id"}}, + {Name: "gitcrawl.pull_requests.review_context", Args: []string{"owner", "repo", "number"}}, + {Name: "gitcrawl.coverage", Args: []string{"dataset"}}, + } +} + +func gitcrawlCloudPublicationCapabilities(requested []string) []string { + capabilities := make([]string, 0, len(gitcrawlCloudReaderQuerySpecs())+len(requested)) + for _, query := range gitcrawlCloudReaderQuerySpecs() { + capabilities = append(capabilities, query.Name) + } + return append(capabilities, requested...) +} + func (a *App) runCloud(ctx context.Context, args []string) error { if len(args) == 0 { return usageErr(fmt.Errorf("cloud requires a subcommand")) @@ -41,6 +80,9 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { remoteEndpoint := fs.String("remote", "", "remote archive endpoint") archive := fs.String("archive", "", "remote archive id") tokenEnv := fs.String("token-env", "", "remote token environment variable") + allowIncomplete := fs.Bool("allow-incomplete", false, "publish even when local enrichment coverage is incomplete") + observationOrder := fs.Bool("observation-order", false, "publish durable observation ordering when the remote fence is enabled") + stageOnly := fs.Bool("stage-only", false, "stage the immutable snapshot without moving unpinned reads") jsonOut := fs.Bool("json", false, "write JSON output") if err := fs.Parse(normalizeCommandArgs(args, map[string]bool{"remote": true, "archive": true, "token-env": true})); err != nil { return usageErr(err) @@ -49,6 +91,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { if fs.NArg() != 0 { return usageErr(fmt.Errorf("cloud publish takes flags only")) } + cutover := !*stageOnly cfg, err := config.LoadRuntime(a.configPath) if err != nil { @@ -75,123 +118,743 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { Archive: archiveID, TokenEnv: firstNonEmpty(*tokenEnv, cfg.Remote.TokenEnv, crawlremote.DefaultTokenEnv), } + httpClient := &http.Client{Timeout: 10 * time.Minute} + tokenProvider := crawlremote.EnvTokenProvider{Name: remoteCfg.TokenEnv} client, err := crawlremote.NewClientFromConfig(remoteCfg, crawlremote.Options{ - UserAgent: "gitcrawl/" + version, - HTTPClient: &http.Client{Timeout: 10 * time.Minute}, + UserAgent: "gitcrawl/" + version, + HTTPClient: httpClient, + TokenProvider: tokenProvider, }) if err != nil { return err } - manifest := crawlremote.IngestManifest{ - App: "gitcrawl", - Archive: archiveID, - SchemaName: "gitcrawl-cloud-v1", - SchemaVersion: 1, - SchemaHash: "gitcrawl-cloud-v1", - Mode: crawlremote.ModePublisher, - Source: "sqlite", - } - repoRows, err := publishRows(ctx, rt.Store.DB(), gitcrawlRepositoryExportSQL, func(values []any) []any { return values }) + snapshotPath, cleanupSnapshot, err := cloudSQLiteSnapshotPath(ctx, rt.Store.DB(), rt.Store.Path()) if err != nil { return err } - threadSQL, err := gitcrawlThreadExportSQL(ctx, rt.Store.DB()) + defer cleanupSnapshot() + snapshotDB, err := sql.Open("sqlite", snapshotPath) if err != nil { - return err + return fmt.Errorf("open frozen cloud snapshot: %w", err) } - threadRows, err := publishRows(ctx, rt.Store.DB(), threadSQL, func(values []any) []any { return values }) + defer snapshotDB.Close() + snapshot, err := buildGitcrawlCloudSnapshot( + ctx, + snapshotDB, + snapshotPath, + *allowIncomplete, + *observationOrder, + ) if err != nil { return err } - repoAccepted, err := sendIngestRows(ctx, client, "gitcrawl", archiveID, manifest, "repositories", gitcrawlRepositoryColumns, repoRows, false) - if err != nil { + manifest := gitcrawlCloudManifest(archiveID, snapshot) + publicationCapabilities := gitcrawlCloudPublicationCapabilities(manifest.Capabilities) + counts := gitcrawlCloudDatasetCounts(snapshot) + if err := requireGitcrawlSnapshotPublishContract( + ctx, + client, + snapshot, + cutover, + ); err != nil { return err } - threadAccepted, err := sendIngestRows(ctx, client, "gitcrawl", archiveID, manifest, "threads", gitcrawlThreadColumns, threadRows, true) - if err != nil { + if err := requireGitcrawlCloudPublishRoles(ctx, client); err != nil { return err } - sqliteBundle, err := uploadSQLiteArchive(ctx, client, "gitcrawl", archiveID, rt.Store.DB(), rt.Store.Path(), manifest, map[string]int64{ - "repositories": repoAccepted, - "threads": threadAccepted, - }) + snapshotInfo, err := os.Stat(snapshotPath) if err != nil { - return err + return fmt.Errorf("stat frozen cloud snapshot: %w", err) + } + sqliteSourceSize := snapshotInfo.Size() + if sqliteSourceSize <= 0 { + return fmt.Errorf("frozen cloud snapshot has invalid size %d", sqliteSourceSize) + } + + alreadyStaged := false + status, statusErr := client.PublishStatusForSnapshot( + ctx, + "gitcrawl", + archiveID, + snapshot.ID, + ) + if statusErr == nil { + alreadyStaged = gitcrawlPublisherStatusMatches( + status, + manifest, + publicationCapabilities, + ) + if alreadyStaged { + snapshot.DatasetGeneratedAt = status.Snapshot.DatasetGeneratedAt + } + } else if !remoteNotFound(statusErr) && !remoteSnapshotIncomplete(statusErr) { + return statusErr + } + var sqliteBundle *crawlremote.SQLiteBundle + var mutationToken string + if !alreadyStaged { + uploadedBundle, uploadedSourceSize, err := uploadSQLiteSnapshotArchive( + ctx, + client, + "gitcrawl", + archiveID, + snapshotPath, + snapshot.ID, + counts, + ) + if err != nil { + return err + } + if uploadedSourceSize != sqliteSourceSize { + return fmt.Errorf( + "SQLite bundle source size changed from %d to %d", + sqliteSourceSize, + uploadedSourceSize, + ) + } + sqliteBundle = uploadedBundle + completedConcurrently := false + for _, dataset := range snapshot.Datasets { + progress, err := sendSnapshotIngestDataset( + ctx, + snapshotDB, + client, + "gitcrawl", + archiveID, + manifest, + dataset, + mutationToken, + ) + if err != nil { + recoveredGeneration, recoveryErr := recoverConcurrentGitcrawlSnapshot( + ctx, + client, + archiveID, + snapshot, + manifest, + publicationCapabilities, + err, + ) + if recoveryErr != nil { + return fmt.Errorf( + "publish cloud dataset %s: %w", + dataset.Name, + recoveryErr, + ) + } + if recoveredGeneration != "" { + snapshot.DatasetGeneratedAt = recoveredGeneration + mutationToken = "" + alreadyStaged = true + completedConcurrently = true + break + } + return fmt.Errorf("publish cloud dataset %s: %w", dataset.Name, err) + } + counts[dataset.Name] = progress.RowsAccepted + mutationToken = progress.MutationToken + } + if !completedConcurrently { + progress, err := completeGitcrawlSnapshotStaging( + ctx, + client, + "gitcrawl", + archiveID, + manifest, + snapshot, + mutationToken, + ) + if err != nil { + recoveredGeneration, recoveryErr := recoverConcurrentGitcrawlSnapshot( + ctx, + client, + archiveID, + snapshot, + manifest, + publicationCapabilities, + err, + ) + if recoveryErr != nil { + return fmt.Errorf("complete cloud snapshot staging: %w", recoveryErr) + } + if recoveredGeneration == "" { + return fmt.Errorf("complete cloud snapshot staging: %w", err) + } + snapshot.DatasetGeneratedAt = recoveredGeneration + mutationToken = "" + alreadyStaged = true + } else { + mutationToken = progress.MutationToken + } + } + } + var cutoverResult *crawlremote.CutoverResult + alreadyCutOver := false + if cutover { + readerStatus, err := client.Status(ctx, "gitcrawl", archiveID) + if err != nil { + return fmt.Errorf("read serving cloud snapshot status: %w", err) + } + if readerStatus.App != "gitcrawl" || readerStatus.Archive != archiveID { + return fmt.Errorf( + "serving cloud snapshot status returned app=%q archive=%q", + readerStatus.App, + readerStatus.Archive, + ) + } + alreadyCutOver = gitcrawlReaderStatusMatches( + readerStatus, + snapshot, + manifest, + publicationCapabilities, + "", + ) + expectedCutoverAt := "" + if !alreadyCutOver { + result, err := client.Cutover(ctx, "gitcrawl", archiveID, snapshot.ID) + if err != nil { + return fmt.Errorf("cut over cloud snapshot: %w", err) + } + if err := validateGitcrawlCutoverResult(result, archiveID, snapshot.ID); err != nil { + return fmt.Errorf("validate cloud snapshot cutover: %w", err) + } + cutoverResult = &result + expectedCutoverAt = result.CutoverAt + } + if err := verifyGitcrawlSnapshotPublication( + ctx, + client, + httpClient, + tokenProvider, + endpoint, + archiveID, + snapshot, + manifest, + publicationCapabilities, + expectedCutoverAt, + sqliteSourceSize, + ); err != nil { + return fmt.Errorf("verify published cloud snapshot: %w", err) + } } sqliteBundlePrivacy := gitcrawlCloudSQLiteBundlePrivacy() return a.writeOutput("cloud publish", map[string]any{ "remote": strings.TrimRight(endpoint, "/"), "archive": archiveID, - "repositories": repoAccepted, - "threads": threadAccepted, + "snapshot_id": snapshot.ID, + "source_sha256": snapshot.ID, + "source_sync_at": snapshot.SourceSyncAt, + "dataset_generated_at": snapshot.DatasetGeneratedAt, + "capabilities": manifest.Capabilities, + "datasets": counts, + "hydration": snapshot.Hydration, + "already_staged": alreadyStaged, + "already_cut_over": alreadyCutOver, + "mutation_token": mutationToken, + "cutover": cutoverResult, "sqlite_bundle": sqliteBundle, "sqlite_bundle_privacy": sqliteBundlePrivacy, }, true) } -func publishRows(ctx context.Context, db *sql.DB, query string, mapRow func([]any) []any) ([][]any, error) { - rows, err := db.QueryContext(ctx, query) +type ingestProgress struct { + RowsAccepted int64 + MutationToken string + Result crawlremote.IngestResult +} + +type gitcrawlIngestBatchSizer struct { + baseBytes int64 + finalBaseBytes int64 + rowBytes int64 + rowCount int +} + +func newGitcrawlIngestBatchSizer( + app, archive string, + manifest crawlremote.IngestManifest, + table string, + columns []string, + cursor, mutationToken string, +) (gitcrawlIngestBatchSizer, error) { + manifest.App = strings.TrimSpace(app) + manifest.Archive = strings.TrimSpace(archive) + request := crawlremote.IngestRequest{ + Manifest: manifest, + Table: table, + Columns: columns, + Rows: [][]any{}, + Cursor: cursor, + MutationToken: mutationToken, + } + baseBytes, err := encodedGitcrawlIngestRequestBytes(request) if err != nil { - return nil, err + return gitcrawlIngestBatchSizer{}, err + } + request.Final = true + finalBaseBytes, err := encodedGitcrawlIngestRequestBytes(request) + if err != nil { + return gitcrawlIngestBatchSizer{}, err + } + return gitcrawlIngestBatchSizer{ + baseBytes: baseBytes, + finalBaseBytes: finalBaseBytes, + }, nil +} + +func encodedGitcrawlIngestRequestBytes(request crawlremote.IngestRequest) (int64, error) { + encoded, err := json.Marshal(request) + if err != nil { + return 0, fmt.Errorf("encode ingest request envelope: %w", err) + } + // crawlremote uses json.Encoder, which appends one newline byte. + return int64(len(encoded)) + 1, nil +} + +func (s *gitcrawlIngestBatchSizer) add(row []any, final bool) (bool, int64, error) { + encoded, err := json.Marshal(row) + if err != nil { + return false, 0, fmt.Errorf("encode ingest row: %w", err) + } + separatorBytes := int64(0) + if s.rowCount > 0 { + separatorBytes = 1 + } + baseBytes := s.baseBytes + if final { + baseBytes = s.finalBaseBytes + } + encodedBytes := baseBytes + s.rowBytes + separatorBytes + int64(len(encoded)) + if encodedBytes > gitcrawlCloudIngestRequestMaxBytes { + return false, encodedBytes, nil + } + s.rowBytes += separatorBytes + int64(len(encoded)) + s.rowCount++ + return true, encodedBytes, nil +} + +func (s gitcrawlIngestBatchSizer) encodedBytes(final bool) int64 { + baseBytes := s.baseBytes + if final { + baseBytes = s.finalBaseBytes + } + return baseBytes + s.rowBytes +} + +func sendSnapshotIngestDataset( + ctx context.Context, + db *sql.DB, + client *crawlremote.Client, + app, archive string, + manifest crawlremote.IngestManifest, + dataset gitcrawlCloudDataset, + mutationToken string, +) (ingestProgress, error) { + rows, err := db.QueryContext(ctx, dataset.Query) + if err != nil { + return ingestProgress{}, err } defer rows.Close() - cols, err := rows.Columns() + columns, err := rows.Columns() if err != nil { - return nil, err + return ingestProgress{}, err + } + if len(columns) != len(dataset.Columns) { + return ingestProgress{}, fmt.Errorf( + "dataset query returned %d columns, want %d", + len(columns), + len(dataset.Columns), + ) + } + + batch := make([][]any, 0, gitcrawlCloudBatchSize) + var scanned int64 + var accepted int64 + var batchStart int64 + var batchSizer *gitcrawlIngestBatchSizer + ensureBatchSizer := func() error { + if batchSizer != nil { + return nil + } + sizer, err := newGitcrawlIngestBatchSizer( + app, + archive, + manifest, + dataset.Name, + dataset.Columns, + cursorFor(batchStart), + mutationToken, + ) + if err != nil { + return err + } + batchSizer = &sizer + return nil } - out := make([][]any, 0) + flush := func() error { + result, err := sendIngestBatch( + ctx, + client, + app, + archive, + manifest, + dataset.Name, + dataset.Columns, + batch, + batchStart, + mutationToken, + false, + ) + if err != nil { + return err + } + if result.RowsAccepted != int64(len(batch)) { + return fmt.Errorf( + "remote accepted %d rows from a %d-row batch", + result.RowsAccepted, + len(batch), + ) + } + accepted += result.RowsAccepted + mutationToken = result.MutationToken + batch = batch[:0] + batchStart = scanned + batchSizer = nil + return nil + } + for rows.Next() { - values := make([]any, len(cols)) - ptrs := make([]any, len(cols)) - for i := range values { - ptrs[i] = &values[i] + values := make([]any, len(columns)) + pointers := make([]any, len(columns)) + for index := range values { + pointers[index] = &values[index] } - if err := rows.Scan(ptrs...); err != nil { - return nil, err + if err := rows.Scan(pointers...); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err } - for i, value := range values { + for index, value := range values { if bytes, ok := value.([]byte); ok { - values[i] = string(bytes) + values[index] = string(bytes) } } - out = append(out, mapRow(values)) + if err := ensureBatchSizer(); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + fits, encodedBytes, err := batchSizer.add(values, false) + if err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + if !fits && len(batch) > 0 { + if err := flush(); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + if err := ensureBatchSizer(); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + fits, encodedBytes, err = batchSizer.add(values, false) + if err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + } + if !fits { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, fmt.Errorf( + "dataset %s row %d encoded ingest request is %d bytes, limit %d", + dataset.Name, + scanned, + encodedBytes, + gitcrawlCloudIngestRequestMaxBytes, + ) + } + batch = append(batch, values) + scanned++ + if len(batch) == gitcrawlCloudBatchSize { + if err := flush(); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + } + } + if err := rows.Err(); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + if len(batch) > 0 { + if err := flush(); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } } - return out, rows.Err() + if scanned == 0 { + if err := ensureBatchSizer(); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + if encodedBytes := batchSizer.encodedBytes(false); encodedBytes > gitcrawlCloudIngestRequestMaxBytes { + return ingestProgress{}, fmt.Errorf( + "dataset %s empty encoded ingest request is %d bytes, limit %d", + dataset.Name, + encodedBytes, + gitcrawlCloudIngestRequestMaxBytes, + ) + } + if err := flush(); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + } + if scanned != dataset.RowCount { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, fmt.Errorf( + "dataset row count changed from preflight %d to stream %d", + dataset.RowCount, + scanned, + ) + } + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, nil } -func sendIngestRows(ctx context.Context, client *crawlremote.Client, app, archive string, manifest crawlremote.IngestManifest, table string, columns []string, rows [][]any, final bool) (int64, error) { +func sendIngestRows( + ctx context.Context, + client *crawlremote.Client, + app, archive string, + manifest crawlremote.IngestManifest, + table string, + columns []string, + rows [][]any, + final bool, +) (int64, error) { + progress, err := sendSnapshotIngestRows( + ctx, + client, + app, + archive, + manifest, + table, + columns, + rows, + "", + final, + ) + return progress.RowsAccepted, err +} + +func sendSnapshotIngestRows( + ctx context.Context, + client *crawlremote.Client, + app, archive string, + manifest crawlremote.IngestManifest, + table string, + columns []string, + rows [][]any, + mutationToken string, + final bool, +) (ingestProgress, error) { var total int64 if len(rows) == 0 { - result, err := sendIngestBatch(ctx, client, app, archive, manifest, table, columns, [][]any{}, 0, final) - return result.RowsAccepted, err + sizer, sizeErr := newGitcrawlIngestBatchSizer( + app, + archive, + manifest, + table, + columns, + "", + mutationToken, + ) + if sizeErr != nil { + return ingestProgress{}, sizeErr + } + if encodedBytes := sizer.encodedBytes(final); encodedBytes > gitcrawlCloudIngestRequestMaxBytes { + return ingestProgress{}, fmt.Errorf( + "ingest table %s empty encoded request is %d bytes, limit %d", + table, + encodedBytes, + gitcrawlCloudIngestRequestMaxBytes, + ) + } + result, err := sendIngestBatch( + ctx, + client, + app, + archive, + manifest, + table, + columns, + [][]any{}, + 0, + mutationToken, + final, + ) + return ingestProgress{ + RowsAccepted: result.RowsAccepted, + MutationToken: result.MutationToken, + Result: result, + }, err } - for start := 0; start < len(rows); start += gitcrawlCloudBatchSize { - end := start + gitcrawlCloudBatchSize - if end > len(rows) { - end = len(rows) + var lastResult crawlremote.IngestResult + for start := 0; start < len(rows); { + sizer, err := newGitcrawlIngestBatchSizer( + app, + archive, + manifest, + table, + columns, + cursorFor(int64(start)), + mutationToken, + ) + if err != nil { + return ingestProgress{ + RowsAccepted: total, + MutationToken: mutationToken, + Result: lastResult, + }, err } - result, err := sendIngestBatch(ctx, client, app, archive, manifest, table, columns, rows[start:end], start, final && end == len(rows)) + end := start + for end < len(rows) && end-start < gitcrawlCloudBatchSize { + fits, encodedBytes, err := sizer.add(rows[end], final && end == len(rows)-1) + if err != nil { + return ingestProgress{ + RowsAccepted: total, + MutationToken: mutationToken, + Result: lastResult, + }, err + } + if !fits { + if end == start { + return ingestProgress{ + RowsAccepted: total, + MutationToken: mutationToken, + Result: lastResult, + }, fmt.Errorf( + "ingest table %s row %d encoded request is %d bytes, limit %d", + table, + start, + encodedBytes, + gitcrawlCloudIngestRequestMaxBytes, + ) + } + break + } + end++ + } + result, err := sendIngestBatch( + ctx, + client, + app, + archive, + manifest, + table, + columns, + rows[start:end], + int64(start), + mutationToken, + final && end == len(rows), + ) if err != nil { - return total, err + return ingestProgress{ + RowsAccepted: total, + MutationToken: mutationToken, + Result: lastResult, + }, err } total += result.RowsAccepted + mutationToken = result.MutationToken + lastResult = result + start = end } - return total, nil + return ingestProgress{ + RowsAccepted: total, + MutationToken: mutationToken, + Result: lastResult, + }, nil } -func sendIngestBatch(ctx context.Context, client *crawlremote.Client, app, archive string, manifest crawlremote.IngestManifest, table string, columns []string, rows [][]any, cursor int, final bool) (crawlremote.IngestResult, error) { +func completeGitcrawlSnapshotStaging( + ctx context.Context, + client *crawlremote.Client, + app, archive string, + manifest crawlremote.IngestManifest, + snapshot gitcrawlCloudSnapshot, + mutationToken string, +) (ingestProgress, error) { + progress, err := sendSnapshotIngestRows( + ctx, + client, + app, + archive, + manifest, + "dataset_coverage", + gitcrawlCloudCoverageColumns, + gitcrawlCloudCoverageRows(snapshot, mutationToken), + mutationToken, + true, + ) + if err != nil { + return progress, err + } + result := progress.Result + expectedRows := int64(len(snapshot.Datasets)) + if result.Table != "dataset_coverage" { + return progress, fmt.Errorf( + "remote completed table %q, want dataset_coverage", + result.Table, + ) + } + if result.SnapshotID != snapshot.ID { + return progress, fmt.Errorf( + "remote completed snapshot %q, want %q", + result.SnapshotID, + snapshot.ID, + ) + } + if progress.RowsAccepted != expectedRows || result.RowsAccepted != expectedRows { + return progress, fmt.Errorf( + "remote accepted %d coverage rows with final batch count %d, want %d datasets", + progress.RowsAccepted, + result.RowsAccepted, + expectedRows, + ) + } + if result.MutationToken != mutationToken { + return progress, fmt.Errorf( + "remote completed mutation token %q, want %q", + result.MutationToken, + mutationToken, + ) + } + if !result.Complete { + return progress, fmt.Errorf("remote did not complete snapshot %s", snapshot.ID) + } + return progress, nil +} + +func sendIngestBatch( + ctx context.Context, + client *crawlremote.Client, + app, archive string, + manifest crawlremote.IngestManifest, + table string, + columns []string, + rows [][]any, + cursor int64, + mutationToken string, + final bool, +) (crawlremote.IngestResult, error) { for { result, err := client.Ingest(ctx, app, archive, crawlremote.IngestRequest{ - Manifest: manifest, - Table: table, - Columns: columns, - Rows: rows, - Cursor: cursorFor(cursor), - Final: final, + Manifest: manifest, + Table: table, + Columns: columns, + Rows: rows, + Cursor: cursorFor(cursor), + MutationToken: mutationToken, + Final: final, }) if err == nil { if result.ResetIncomplete { - if err := drainIngestReset(ctx, client, app, archive, manifest, table, columns); err != nil { + if err := drainIngestReset( + ctx, + client, + app, + archive, + manifest, + table, + columns, + mutationToken, + ); err != nil { return crawlremote.IngestResult{}, err } continue @@ -201,19 +864,37 @@ func sendIngestBatch(ctx context.Context, client *crawlremote.Client, app, archi if !isResetIncomplete(err) { return crawlremote.IngestResult{}, err } - if err := drainIngestReset(ctx, client, app, archive, manifest, table, columns); err != nil { + if err := drainIngestReset( + ctx, + client, + app, + archive, + manifest, + table, + columns, + mutationToken, + ); err != nil { return crawlremote.IngestResult{}, err } } } -func drainIngestReset(ctx context.Context, client *crawlremote.Client, app, archive string, manifest crawlremote.IngestManifest, table string, columns []string) error { +func drainIngestReset( + ctx context.Context, + client *crawlremote.Client, + app, archive string, + manifest crawlremote.IngestManifest, + table string, + columns []string, + mutationToken string, +) error { for { result, err := client.Ingest(ctx, app, archive, crawlremote.IngestRequest{ - Manifest: manifest, - Table: table, - Columns: columns, - Rows: [][]any{}, + Manifest: manifest, + Table: table, + Columns: columns, + Rows: [][]any{}, + MutationToken: mutationToken, }) if err != nil { return err @@ -229,7 +910,7 @@ func isResetIncomplete(err error) bool { return errors.As(err, &remoteErr) && remoteErr.Code == "reset_incomplete" } -func cursorFor(start int) string { +func cursorFor(start int64) string { if start == 0 { return "" } @@ -242,7 +923,29 @@ func uploadSQLiteArchive(ctx context.Context, client *crawlremote.Client, app, a return nil, err } defer cleanup() - bundle, err := crawlremote.BuildGzipSQLiteBundle(ctx, crawlremote.SQLiteBundleBuildOptions{ + snapshotID, err := cloudFileSHA256(snapshotPath) + if err != nil { + return nil, err + } + bundle, _, err := uploadSQLiteSnapshotArchive( + ctx, + client, + app, + archive, + snapshotPath, + snapshotID, + counts, + ) + return bundle, err +} + +func uploadSQLiteSnapshotArchive( + ctx context.Context, + client *crawlremote.Client, + app, archive, snapshotPath, expectedSnapshotID string, + counts map[string]int64, +) (*crawlremote.SQLiteBundle, int64, error) { + bundle, err := crawlremote.BuildSnapshotGzipSQLiteBundle(ctx, crawlremote.SQLiteBundleBuildOptions{ App: app, Archive: archive, SourcePath: snapshotPath, @@ -251,16 +954,712 @@ func uploadSQLiteArchive(ctx context.Context, client *crawlremote.Client, app, a Privacy: gitcrawlCloudSQLiteBundlePrivacy(), }) if err != nil { - return nil, err + return nil, 0, err } defer bundle.Cleanup() + expectedSnapshotID = strings.TrimSpace(expectedSnapshotID) + if expectedSnapshotID == "" { + return nil, 0, fmt.Errorf("selected SQLite snapshot digest is required") + } + if bundle.Manifest.SnapshotID != expectedSnapshotID || + bundle.Manifest.Object.SHA256 != expectedSnapshotID { + return nil, 0, fmt.Errorf( + "SQLite bundle source digest changed from selected snapshot %s to snapshot %s with object digest %s", + expectedSnapshotID, + bundle.Manifest.SnapshotID, + bundle.Manifest.Object.SHA256, + ) + } + sourceSize := bundle.Manifest.Object.Size + if sourceSize <= 0 { + return nil, 0, fmt.Errorf("SQLite bundle manifest has invalid source size %d", sourceSize) + } result, err := client.UploadSQLiteBundleFiles(ctx, app, archive, bundle.Manifest, bundle.Parts) if err != nil { - return nil, err + return nil, 0, err + } + uploadedBundle, err := validateGitcrawlSQLiteBundleUpload( + result, + app, + archive, + bundle.Manifest, + ) + if err != nil { + return nil, 0, err + } + return uploadedBundle, sourceSize, nil +} + +func validateGitcrawlSQLiteBundleUpload( + result crawlremote.SQLiteBundleUploadResult, + app, archive string, + expected crawlremote.SQLiteBundleManifest, +) (*crawlremote.SQLiteBundle, error) { + if result.App != app || result.Archive != archive { + return nil, fmt.Errorf( + "SQLite bundle upload returned app=%q archive=%q, want app=%q archive=%q", + result.App, + result.Archive, + app, + archive, + ) + } + if !result.Complete { + return nil, fmt.Errorf("SQLite bundle upload was not finalized") + } + if result.Bundle == nil || result.Bundle.Manifest == nil { + return nil, fmt.Errorf("SQLite bundle upload omitted the finalized manifest") + } + actual := result.Bundle.Manifest + if actual.SnapshotID != expected.SnapshotID { + return nil, fmt.Errorf( + "SQLite bundle upload acknowledged snapshot %q, want %q", + actual.SnapshotID, + expected.SnapshotID, + ) + } + if actual.Object.SHA256 != expected.Object.SHA256 { + return nil, fmt.Errorf( + "SQLite bundle upload acknowledged digest %q, want %q", + actual.Object.SHA256, + expected.Object.SHA256, + ) + } + if actual.Object.Size != expected.Object.Size { + return nil, fmt.Errorf( + "SQLite bundle upload acknowledged source size %d, want %d", + actual.Object.Size, + expected.Object.Size, + ) } return result.Bundle, nil } +func remoteNotFound(err error) bool { + var remoteErr *crawlremote.Error + return errors.As(err, &remoteErr) && remoteErr.Status == http.StatusNotFound +} + +func remoteSnapshotIncomplete(err error) bool { + var remoteErr *crawlremote.Error + return errors.As(err, &remoteErr) && + remoteErr.Status == http.StatusConflict && + remoteErr.Code == "snapshot_mismatch" +} + +func recoverConcurrentGitcrawlSnapshot( + ctx context.Context, + client *crawlremote.Client, + archive string, + snapshot gitcrawlCloudSnapshot, + manifest crawlremote.IngestManifest, + publicationCapabilities []string, + cause error, +) (string, error) { + var remoteErr *crawlremote.Error + if !errors.As(cause, &remoteErr) || + remoteErr.Status != http.StatusConflict || + remoteErr.Code != "snapshot_active" { + return "", nil + } + status, err := client.PublishStatusForSnapshot( + ctx, + "gitcrawl", + archive, + snapshot.ID, + ) + if err != nil { + return "", fmt.Errorf("re-probe concurrent snapshot completion: %w", err) + } + if !gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) { + return "", fmt.Errorf( + "concurrent active snapshot %s does not match the requested digest, profile, and coverage", + snapshot.ID, + ) + } + return status.Snapshot.DatasetGeneratedAt, nil +} + +func requireGitcrawlSnapshotPublishContract( + ctx context.Context, + client *crawlremote.Client, + snapshot gitcrawlCloudSnapshot, + cutover bool, +) error { + contract, err := client.Contract(ctx) + if err != nil { + return fmt.Errorf("read remote snapshot publish contract: %w", err) + } + if err := contract.Validate(); err != nil { + return fmt.Errorf("validate remote snapshot publish contract: %w", err) + } + var appSpec *crawlremote.AppSpec + for index := range contract.Apps { + app := &contract.Apps[index] + if app.App == "gitcrawl" { + appSpec = app + break + } + } + if appSpec == nil { + return fmt.Errorf("remote contract does not advertise the gitcrawl app") + } + requiredCapabilities := []string{ + gitcrawlSnapshotAtomicCapability, + gitcrawlSnapshotProvenanceCapability, + gitcrawlSnapshotStagingCapability, + sqliteBundleGzipUploadCapability, + } + requiredCapabilities = append(requiredCapabilities, snapshot.Capabilities...) + if cutover { + requiredCapabilities = append( + requiredCapabilities, + gitcrawlSnapshotCutoverCapability, + ) + } + for _, capability := range requiredCapabilities { + if !slices.Contains(appSpec.Capabilities, capability) { + return fmt.Errorf( + "remote does not advertise required snapshot publish capability %s", + capability, + ) + } + } + requiredRoutes := []crawlremote.RouteSpec{ + { + Method: http.MethodGet, + Path: "/v1/whoami", + Auth: crawlremote.AuthReader, + }, + { + Method: http.MethodGet, + Path: "/v1/apps/:app/archives/:archive/publish-status", + Auth: crawlremote.AuthPublisher, + }, + { + Method: http.MethodPost, + Path: "/v1/apps/:app/archives/:archive/query", + Auth: crawlremote.AuthReader, + }, + { + Method: http.MethodPost, + Path: "/v1/apps/:app/archives/:archive/ingest", + Auth: crawlremote.AuthPublisher, + }, + { + Method: http.MethodPut, + Path: "/v1/apps/:app/archives/:archive/sqlite", + Auth: crawlremote.AuthPublisher, + }, + } + if cutover { + requiredRoutes = append( + requiredRoutes, + crawlremote.RouteSpec{ + Method: http.MethodGet, + Path: "/v1/apps/:app/archives/:archive/status", + Auth: crawlremote.AuthReader, + }, + crawlremote.RouteSpec{ + Method: http.MethodPost, + Path: "/v1/apps/:app/archives/:archive/cutover", + Auth: crawlremote.AuthPublisher, + }, + crawlremote.RouteSpec{ + Method: http.MethodGet, + Path: "/v1/apps/:app/archives/:archive/sqlite", + Auth: crawlremote.AuthReader, + }, + ) + } + for _, required := range requiredRoutes { + if !slices.ContainsFunc(contract.Routes, func(route crawlremote.RouteSpec) bool { + return route == required + }) { + return fmt.Errorf( + "remote contract does not advertise required snapshot publish route %s %s with %s auth", + required.Method, + required.Path, + required.Auth, + ) + } + } + for _, required := range gitcrawlCloudReaderQuerySpecs() { + queryIndex := -1 + for index, query := range appSpec.Queries { + if query.Name != required.Name { + continue + } + if queryIndex >= 0 { + return fmt.Errorf( + "remote contract advertises required reader query %s more than once", + required.Name, + ) + } + queryIndex = index + } + if queryIndex < 0 { + return fmt.Errorf( + "remote contract does not advertise required reader query %s", + required.Name, + ) + } + remoteArgs := appSpec.Queries[queryIndex].Args + if !uniqueStringSuperset(remoteArgs, required.Args) { + return fmt.Errorf( + "remote contract reader query %s has arguments %v, missing required arguments from %v", + required.Name, + remoteArgs, + required.Args, + ) + } + } + requiredTables := make([]crawlremote.IngestTableSpec, 0, len(snapshot.Datasets)+1) + for _, dataset := range snapshot.Datasets { + requiredTables = append(requiredTables, crawlremote.IngestTableSpec{ + Name: dataset.Name, + Columns: dataset.Columns, + }) + } + requiredTables = append(requiredTables, crawlremote.IngestTableSpec{ + Name: "dataset_coverage", + Columns: gitcrawlCloudCoverageColumns, + }) + for _, required := range requiredTables { + tableIndex := slices.IndexFunc(appSpec.IngestTables, func(table crawlremote.IngestTableSpec) bool { + return table.Name == required.Name + }) + if tableIndex < 0 { + return fmt.Errorf( + "remote contract does not advertise required snapshot ingest table %s", + required.Name, + ) + } + remoteColumns := appSpec.IngestTables[tableIndex].Columns + for _, column := range required.Columns { + if !slices.Contains(remoteColumns, column) { + return fmt.Errorf( + "remote contract snapshot ingest table %s is missing required column %s", + required.Name, + column, + ) + } + } + } + return nil +} + +func requireGitcrawlCloudPublishRoles(ctx context.Context, client *crawlremote.Client) error { + preflightCtx, cancel := context.WithTimeout(ctx, gitcrawlCloudPublishPreflightTimeout) + defer cancel() + identity, err := client.Whoami(preflightCtx) + if err != nil { + return fmt.Errorf("read remote identity before snapshot publication: %w", err) + } + if slices.Contains(identity.Roles, "admin") { + return nil + } + missing := make([]string, 0, 2) + for _, role := range []string{crawlremote.AuthPublisher, crawlremote.AuthReader} { + if !slices.Contains(identity.Roles, role) { + missing = append(missing, role) + } + } + if len(missing) > 0 { + return fmt.Errorf( + "remote token must have publisher and reader roles before snapshot publication; missing %s", + strings.Join(missing, ", "), + ) + } + return nil +} + +func equalUniqueStringSet(left, right []string) bool { + if len(left) != len(right) { + return false + } + seen := make(map[string]struct{}, len(left)) + for _, value := range left { + if _, duplicate := seen[value]; duplicate { + return false + } + seen[value] = struct{}{} + } + for _, value := range right { + if _, duplicate := seen[value]; !duplicate { + return false + } + delete(seen, value) + } + return len(seen) == 0 +} + +func uniqueStringSuperset(values, required []string) bool { + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + if _, duplicate := seen[value]; duplicate { + return false + } + seen[value] = struct{}{} + } + for _, value := range required { + if _, ok := seen[value]; !ok { + return false + } + } + return true +} + +func gitcrawlPublisherStatusMatches( + status crawlremote.PublisherStatus, + manifest crawlremote.IngestManifest, + publicationCapabilities []string, +) bool { + snapshot := status.Snapshot + if status.App != manifest.App || + status.Archive != manifest.Archive || + status.ActiveSnapshotID != manifest.SnapshotID || + snapshot == nil || + snapshot.ID != manifest.SnapshotID || + snapshot.SourceSHA256 != manifest.SourceSHA256 || + snapshot.SourceSyncAt != manifest.SourceSyncAt || + snapshot.SchemaName != manifest.SchemaName || + snapshot.SchemaVersion != manifest.SchemaVersion || + snapshot.SchemaHash != manifest.SchemaHash || + strings.TrimSpace(snapshot.DatasetGeneratedAt) == "" { + return false + } + if status.CoverageComplete != snapshot.CoverageComplete || + !status.CoverageComplete { + return false + } + if !equalUniqueStringSet(snapshot.Capabilities, publicationCapabilities) { + return false + } + return true +} + +func gitcrawlReaderStatusMatches( + status crawlremote.Status, + snapshot gitcrawlCloudSnapshot, + manifest crawlremote.IngestManifest, + publicationCapabilities []string, + expectedCutoverAt string, +) bool { + if status.App != manifest.App || + status.Archive != manifest.Archive || + status.Mode != "cloud" || + status.SnapshotMode != "snapshot" || + status.ActiveSnapshotID != snapshot.ID || + status.SchemaName != manifest.SchemaName || + status.SchemaVersion != manifest.SchemaVersion || + status.SchemaHash != manifest.SchemaHash || + status.SourceSyncAt != snapshot.SourceSyncAt || + status.DatasetGeneratedAt != snapshot.DatasetGeneratedAt || + !status.CoverageComplete || + !equalUniqueStringSet(status.Capabilities, publicationCapabilities) { + return false + } + if !gitcrawlPublisherStatusMatches(crawlremote.PublisherStatus{ + App: status.App, + Archive: status.Archive, + ActiveSnapshotID: status.ActiveSnapshotID, + CoverageComplete: status.CoverageComplete, + Snapshot: status.Snapshot, + }, manifest, publicationCapabilities) || + status.Snapshot.DatasetGeneratedAt != snapshot.DatasetGeneratedAt { + return false + } + return gitcrawlReaderCutoverMatches(status, expectedCutoverAt) && + gitcrawlDatasetCoverageMatches(status.Datasets, snapshot) +} + +func gitcrawlReaderCutoverMatches(status crawlremote.Status, expectedCutoverAt string) bool { + if status.Snapshot == nil { + return false + } + if _, err := time.Parse(time.RFC3339Nano, status.SnapshotCutoverAt); err != nil { + return false + } + if _, err := time.Parse(time.RFC3339Nano, status.Snapshot.CutoverAt); err != nil || + status.SnapshotCutoverAt != status.Snapshot.CutoverAt { + return false + } + if expectedCutoverAt == "" { + return true + } + if _, err := time.Parse(time.RFC3339Nano, expectedCutoverAt); err != nil { + return false + } + return status.SnapshotCutoverAt == expectedCutoverAt +} + +func validateGitcrawlCutoverResult( + result crawlremote.CutoverResult, + archive, snapshotID string, +) error { + if result.Archive != archive { + return fmt.Errorf("cutover returned archive %q, want %q", result.Archive, archive) + } + if result.SnapshotID != snapshotID { + return fmt.Errorf("cutover returned snapshot %q, want %q", result.SnapshotID, snapshotID) + } + if result.SnapshotMode != "snapshot" { + return fmt.Errorf("cutover returned snapshot mode %q, want snapshot", result.SnapshotMode) + } + if _, err := time.Parse(time.RFC3339Nano, result.CutoverAt); err != nil { + return fmt.Errorf("cutover returned invalid timestamp %q: %w", result.CutoverAt, err) + } + return nil +} + +func verifyGitcrawlReaderProjection( + ctx context.Context, + client *crawlremote.Client, + archive string, + snapshot gitcrawlCloudSnapshot, + manifest crawlremote.IngestManifest, + publicationCapabilities []string, + expectedCutoverAt string, +) error { + return verifyGitcrawlReaderProjectionWithRetry( + ctx, + client, + archive, + snapshot, + manifest, + publicationCapabilities, + expectedCutoverAt, + gitcrawlCloudPostCutoverStatusAttempts, + gitcrawlCloudPostCutoverStatusRetryDelay, + ) +} + +func verifyGitcrawlReaderProjectionWithRetry( + ctx context.Context, + client *crawlremote.Client, + archive string, + snapshot gitcrawlCloudSnapshot, + manifest crawlremote.IngestManifest, + publicationCapabilities []string, + expectedCutoverAt string, + attempts int, + retryDelay time.Duration, +) error { + if attempts < 1 { + attempts = 1 + } + var lastErr error + for attempt := 1; attempt <= attempts; attempt++ { + status, err := client.Status(ctx, "gitcrawl", archive) + if err == nil && gitcrawlReaderStatusMatches( + status, + snapshot, + manifest, + publicationCapabilities, + expectedCutoverAt, + ) { + return nil + } + lastErr = err + if attempt == attempts { + break + } + timer := time.NewTimer(retryDelay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } + if lastErr != nil { + return fmt.Errorf( + "read post-cutover reader status after %d attempts: %w", + attempts, + lastErr, + ) + } + return fmt.Errorf( + "post-cutover reader status does not match snapshot %s digest, profile, generation, and coverage after %d attempts", + snapshot.ID, + attempts, + ) +} + +func gitcrawlDatasetCoverageMatches( + actual []crawlremote.DatasetCoverage, + snapshot gitcrawlCloudSnapshot, +) bool { + if len(actual) != len(snapshot.Datasets) { + return false + } + expected := make(map[string]gitcrawlCloudDataset, len(snapshot.Datasets)) + for _, dataset := range snapshot.Datasets { + if _, duplicate := expected[dataset.Name]; duplicate { + return false + } + expected[dataset.Name] = dataset + } + for _, dataset := range actual { + want, ok := expected[dataset.Dataset] + if !ok || + dataset.RowCount != want.RowCount || + dataset.EligibleCount != want.EligibleCount || + dataset.CoveredCount != want.CoveredCount || + dataset.FreshCount != want.CoveredCount || + dataset.MaxSourceAt != want.MaxSourceAt || + dataset.DatasetGeneratedAt != snapshot.DatasetGeneratedAt || + dataset.Complete != want.Complete { + return false + } + delete(expected, dataset.Dataset) + } + return len(expected) == 0 +} + +func verifyGitcrawlSnapshotPublication( + ctx context.Context, + client *crawlremote.Client, + httpClient *http.Client, + tokenProvider crawlremote.TokenProvider, + endpoint, archive string, + snapshot gitcrawlCloudSnapshot, + manifest crawlremote.IngestManifest, + publicationCapabilities []string, + expectedCutoverAt string, + sourceSize int64, +) error { + if err := verifyGitcrawlReaderProjection( + ctx, + client, + archive, + snapshot, + manifest, + publicationCapabilities, + expectedCutoverAt, + ); err != nil { + return err + } + status, err := client.PublishStatusForSnapshot( + ctx, + "gitcrawl", + archive, + snapshot.ID, + ) + if err != nil { + return fmt.Errorf("read post-cutover publisher status: %w", err) + } + if !gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) || + status.Snapshot.DatasetGeneratedAt != snapshot.DatasetGeneratedAt { + return fmt.Errorf( + "post-cutover publisher status does not match snapshot %s digest, profile, generation, and coverage", + snapshot.ID, + ) + } + + token, err := tokenProvider.Token(ctx) + if err != nil { + return fmt.Errorf("read remote token for snapshot hydration: %w", err) + } + sqliteURL := strings.TrimRight(endpoint, "/") + + "/v1/apps/" + url.PathEscape("gitcrawl") + + "/archives/" + url.PathEscape(archive) + + "/sqlite" + hydrationCtx, cancel := context.WithTimeout(ctx, gitcrawlCloudHydrationTimeout) + defer cancel() + request, err := http.NewRequestWithContext(hydrationCtx, http.MethodGet, sqliteURL, nil) + if err != nil { + return fmt.Errorf("build snapshot hydration request: %w", err) + } + request.Header.Set("accept", "application/vnd.sqlite3, application/octet-stream") + request.Header.Set("authorization", "Bearer "+token) + request.Header.Set("user-agent", "gitcrawl/"+version) + response, err := httpClient.Do(request) + if err != nil { + return fmt.Errorf("download bound SQLite snapshot: %w", err) + } + defer response.Body.Close() + if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { + body, _ := io.ReadAll(io.LimitReader(response.Body, 1<<20)) + return fmt.Errorf( + "download bound SQLite snapshot: status=%d body=%s", + response.StatusCode, + strings.TrimSpace(string(body)), + ) + } + return verifyGitcrawlSQLiteHydration(response, snapshot.ID, sourceSize) +} + +func verifyGitcrawlSQLiteHydration( + response *http.Response, + expectedDigest string, + expectedSize int64, +) error { + advertised := strings.TrimSpace(response.Header.Get("x-crawl-content-sha256")) + if advertised == "" { + return fmt.Errorf("downloaded SQLite snapshot is missing x-crawl-content-sha256") + } + if !strings.EqualFold(advertised, expectedDigest) { + return fmt.Errorf( + "downloaded SQLite snapshot advertises digest %s, want %s", + advertised, + expectedDigest, + ) + } + if expectedSize <= 0 { + return fmt.Errorf("uploaded SQLite manifest source size must be positive, got %d", expectedSize) + } + contentLength := int64(-1) + header := strings.TrimSpace(response.Header.Get("content-length")) + if header != "" { + parsed, err := strconv.ParseInt(header, 10, 64) + if err != nil || parsed <= 0 { + return fmt.Errorf("downloaded SQLite snapshot has invalid Content-Length %q", header) + } + contentLength = parsed + } else if response.ContentLength >= 0 { + contentLength = response.ContentLength + } + if contentLength >= 0 && contentLength != expectedSize { + return fmt.Errorf( + "downloaded SQLite snapshot Content-Length %d does not match uploaded source size %d", + contentLength, + expectedSize, + ) + } + hash := sha256.New() + written, err := io.CopyN(hash, response.Body, expectedSize) + if err != nil { + return fmt.Errorf( + "downloaded SQLite snapshot truncated after %d of %d bytes: %w", + written, + expectedSize, + err, + ) + } + var extra [1]byte + n, err := response.Body.Read(extra[:]) + if n > 0 { + return fmt.Errorf( + "downloaded SQLite snapshot exceeds uploaded source size %d", + expectedSize, + ) + } + if err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("check downloaded SQLite snapshot boundary: %w", err) + } + actual := fmt.Sprintf("%x", hash.Sum(nil)) + if actual != expectedDigest { + return fmt.Errorf( + "downloaded SQLite snapshot digest %s does not match source %s", + actual, + expectedDigest, + ) + } + return nil +} + func gitcrawlCloudSQLiteBundlePrivacy() map[string]any { return map[string]any{ "includes_private_messages": true, @@ -298,12 +1697,10 @@ func cloudSQLiteSnapshotPath(ctx context.Context, db *sql.DB, dbPath string) (st cleanup() return "", func() {}, fmt.Errorf("open cloud SQLite snapshot: %w", err) } - for _, table := range []string{"code_documents_fts", "code_documents", "code_snapshots"} { - if _, err := snapshotDB.ExecContext(ctx, `drop table if exists `+table); err != nil { - _ = snapshotDB.Close() - cleanup() - return "", func() {}, fmt.Errorf("drop local-only cloud snapshot table %s: %w", table, err) - } + if err := sanitizeCloudSQLiteSnapshot(ctx, snapshotDB); err != nil { + _ = snapshotDB.Close() + cleanup() + return "", func() {}, err } if _, err := snapshotDB.ExecContext(ctx, `vacuum`); err != nil { _ = snapshotDB.Close() @@ -317,6 +1714,102 @@ func cloudSQLiteSnapshotPath(ctx context.Context, db *sql.DB, dbPath string) (st return snapshotPath, cleanup, nil } +func sanitizeCloudSQLiteSnapshot(ctx context.Context, db *sql.DB) error { + for _, column := range []struct { + table string + name string + }{ + {table: "repositories", name: "raw_json"}, + {table: "threads", name: "raw_json"}, + {table: "comments", name: "raw_json"}, + {table: "pull_request_details", name: "raw_json"}, + {table: "pull_request_files", name: "raw_json"}, + {table: "pull_request_commits", name: "raw_json"}, + {table: "pull_request_checks", name: "raw_json"}, + {table: "pull_request_review_threads", name: "raw_json"}, + {table: "pull_request_review_threads", name: "comments_json"}, + {table: "github_workflow_runs", name: "raw_json"}, + {table: "sync_runs", name: "error_text"}, + {table: "sync_runs", name: "stats_json"}, + {table: "summary_runs", name: "error_text"}, + {table: "summary_runs", name: "stats_json"}, + {table: "embedding_runs", name: "error_text"}, + {table: "embedding_runs", name: "stats_json"}, + {table: "cluster_runs", name: "error_text"}, + {table: "cluster_runs", name: "stats_json"}, + } { + exists, err := sqliteColumnExists(ctx, db, column.table, column.name) + if err != nil { + return fmt.Errorf("inspect cloud snapshot %s.%s: %w", column.table, column.name, err) + } + if !exists { + continue + } + if _, err := db.ExecContext( + ctx, + `update `+column.table+` set `+column.name+` = '' where `+column.name+` is not null and `+column.name+` != ''`, + ); err != nil { + return fmt.Errorf("clear cloud snapshot %s.%s: %w", column.table, column.name, err) + } + } + for _, column := range []struct { + table string + name string + }{ + {table: "comments", name: "raw_json_blob_id"}, + {table: "thread_revisions", name: "raw_json_blob_id"}, + {table: "pull_request_files", name: "patch"}, + } { + exists, err := sqliteColumnExists(ctx, db, column.table, column.name) + if err != nil { + return fmt.Errorf("inspect cloud snapshot %s.%s: %w", column.table, column.name, err) + } + if !exists { + continue + } + value := "null" + if column.name == "patch" { + value = "''" + } + if _, err := db.ExecContext( + ctx, + `update `+column.table+` set `+column.name+` = `+value+` where `+column.name+` is not null`, + ); err != nil { + return fmt.Errorf("clear cloud snapshot %s.%s: %w", column.table, column.name, err) + } + } + for _, table := range []string{ + "thread_changed_files", + "thread_hunk_signatures", + "thread_code_snapshots", + "code_documents_fts", + "code_documents", + "code_snapshots", + } { + if _, err := db.ExecContext(ctx, `drop table if exists `+table); err != nil { + return fmt.Errorf("drop local-only cloud snapshot table %s: %w", table, err) + } + } + if exists, err := sqliteTableExists(ctx, db, "blobs"); err != nil { + return err + } else if exists { + if _, err := db.ExecContext(ctx, `delete from blobs`); err != nil { + return fmt.Errorf("clear cloud snapshot blobs: %w", err) + } + } + if exists, err := sqliteTableExists(ctx, db, "portable_metadata"); err != nil { + return err + } else if exists { + if _, err := db.ExecContext( + ctx, + `update portable_metadata set value = '' where key = 'source_path'`, + ); err != nil { + return fmt.Errorf("clear cloud snapshot portable source path: %w", err) + } + } + return nil +} + func sqliteSnapshotPath(ctx context.Context, db *sql.DB, dbPath string) (string, func(), error) { tmpDir, err := os.MkdirTemp("", "gitcrawl-cloud-sqlite-*") if err != nil { @@ -414,3 +1907,19 @@ func sqliteColumnExists(ctx context.Context, db *sql.DB, table, column string) ( } return false, rows.Err() } + +func sqliteTableExists(ctx context.Context, db *sql.DB, table string) (bool, error) { + var name string + err := db.QueryRowContext( + ctx, + `select name from sqlite_schema where type in ('table', 'view') and name = ?`, + table, + ).Scan(&name) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("inspect cloud snapshot table %s: %w", table, err) + } + return name == table, nil +} diff --git a/internal/cli/cloud_commands_test.go b/internal/cli/cloud_commands_test.go new file mode 100644 index 0000000..20a6624 --- /dev/null +++ b/internal/cli/cloud_commands_test.go @@ -0,0 +1,1697 @@ +package cli + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + crawlremote "github.com/openclaw/crawlkit/remote" +) + +func TestUniqueStringSupersetAllowsAdditiveArguments(t *testing.T) { + if !uniqueStringSuperset( + []string{"owner", "repo", "query", "limit", "cursor"}, + []string{"owner", "repo", "query", "limit"}, + ) { + t.Fatal("additive optional remote argument was rejected") + } +} + +func TestUniqueStringSupersetRejectsMissingOrDuplicateArguments(t *testing.T) { + for _, values := range [][]string{ + {"owner", "repo"}, + {"owner", "repo", "query", "query"}, + } { + if uniqueStringSuperset(values, []string{"owner", "repo", "query"}) { + t.Fatalf("invalid remote arguments accepted: %v", values) + } + } +} + +func TestGitcrawlCloudManifestAlwaysOptsIntoStaging(t *testing.T) { + snapshot := gitcrawlCloudSnapshot{ + ID: strings.Repeat("a", 64), + Capabilities: []string{gitcrawlObservationOrderCapability}, + } + + manifest := gitcrawlCloudManifest("gitcrawl/openclaw__gitcrawl", snapshot) + + if !slices.Equal(manifest.Capabilities, []string{ + gitcrawlObservationOrderCapability, + gitcrawlSnapshotStagingCapability, + }) { + t.Fatalf("manifest capabilities = %v", manifest.Capabilities) + } + if !slices.Equal(snapshot.Capabilities, []string{gitcrawlObservationOrderCapability}) { + t.Fatalf("snapshot capabilities mutated = %v", snapshot.Capabilities) + } + + manifest = gitcrawlCloudManifest("gitcrawl/openclaw__gitcrawl", gitcrawlCloudSnapshot{ + ID: strings.Repeat("b", 64), + Capabilities: []string{ + gitcrawlSnapshotStagingCapability, + }, + }) + if !slices.Equal(manifest.Capabilities, []string{gitcrawlSnapshotStagingCapability}) { + t.Fatalf("staging capability duplicated = %v", manifest.Capabilities) + } +} + +func TestRequireGitcrawlCloudPublishRolesAcceptsAdmin(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.EscapedPath() != "/v1/whoami" { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(crawlremote.Identity{ + Login: "admin", + Roles: []string{"admin"}, + }) + })) + defer server.Close() + + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("admin-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + if err := requireGitcrawlCloudPublishRoles(context.Background(), client); err != nil { + t.Fatalf("admin role preflight: %v", err) + } +} + +func TestSendSnapshotIngestDatasetStreamsBoundedBatches(t *testing.T) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + defer db.Close() + if _, err := db.Exec(` + create table rows(id integer primary key, value text not null); + with recursive sequence(id) as ( + select 1 + union all + select id + 1 from sequence where id < 501 + ) + insert into rows(id, value) + select id, printf('row-%03d', id) from sequence; + `); err != nil { + t.Fatalf("seed rows: %v", err) + } + + var requests []crawlremote.IngestRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var request crawlremote.IngestRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + requests = append(requests, request) + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + Table: request.Table, + SnapshotID: request.Manifest.SnapshotID, + MutationToken: fmt.Sprintf("mutation-%d", len(requests)), + RowsAccepted: int64(len(request.Rows)), + }) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("publisher-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + dataset := gitcrawlCloudDataset{ + Name: "bounded", + Columns: []string{"id", "value"}, + Query: `select id, value from rows order by id`, + RowCount: 501, + } + progress, err := sendSnapshotIngestDataset( + context.Background(), + db, + client, + "gitcrawl", + "gitcrawl/openclaw__gitcrawl", + crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__gitcrawl", + SnapshotID: strings.Repeat("a", 64), + }, + dataset, + "", + ) + if err != nil { + t.Fatalf("stream dataset: %v", err) + } + if progress.RowsAccepted != 501 || progress.MutationToken != "mutation-3" { + t.Fatalf("progress = %#v", progress) + } + if len(requests) != 3 { + t.Fatalf("requests = %d, want 3", len(requests)) + } + for index, wantRows := range []int{250, 250, 1} { + if got := len(requests[index].Rows); got != wantRows { + t.Fatalf("request %d rows = %d, want %d", index, got, wantRows) + } + } + for index, wantCursor := range []string{"", "250", "500"} { + if requests[index].Cursor != wantCursor { + t.Fatalf("request %d cursor = %q, want %q", index, requests[index].Cursor, wantCursor) + } + } +} + +func TestSendSnapshotIngestDatasetFlushesBeforeEncodedByteLimit(t *testing.T) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + defer db.Close() + if _, err := db.Exec(`create table rows(id integer primary key, value text not null)`); err != nil { + t.Fatalf("create rows: %v", err) + } + value := strings.Repeat("x", int(gitcrawlCloudIngestRequestMaxBytes/2)) + for id := 1; id <= 3; id++ { + if _, err := db.Exec(`insert into rows(id, value) values(?, ?)`, id, value); err != nil { + t.Fatalf("seed row %d: %v", id, err) + } + } + + var requests []crawlremote.IngestRequest + var encodedSizes []int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if int64(len(body)) > gitcrawlCloudIngestRequestMaxBytes { + http.Error(w, "encoded request exceeded byte budget", http.StatusRequestEntityTooLarge) + return + } + var request crawlremote.IngestRequest + if err := json.Unmarshal(body, &request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + requests = append(requests, request) + encodedSizes = append(encodedSizes, len(body)) + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + Table: request.Table, + SnapshotID: request.Manifest.SnapshotID, + MutationToken: fmt.Sprintf("mutation-%d", len(requests)), + RowsAccepted: int64(len(request.Rows)), + }) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("publisher-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + progress, err := sendSnapshotIngestDataset( + context.Background(), + db, + client, + "gitcrawl", + "gitcrawl/openclaw__gitcrawl", + crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__gitcrawl", + SnapshotID: strings.Repeat("a", 64), + }, + gitcrawlCloudDataset{ + Name: "bounded_bytes", + Columns: []string{"id", "value"}, + Query: `select id, value from rows order by id`, + RowCount: 3, + }, + "", + ) + if err != nil { + t.Fatalf("stream byte-bounded dataset: %v", err) + } + if progress.RowsAccepted != 3 || progress.MutationToken != "mutation-3" { + t.Fatalf("progress = %#v", progress) + } + if len(requests) != 3 { + t.Fatalf("requests = %d, want one per large row", len(requests)) + } + for index := range requests { + if len(requests[index].Rows) != 1 { + t.Fatalf("request %d rows = %d, want 1", index, len(requests[index].Rows)) + } + if int64(encodedSizes[index]) > gitcrawlCloudIngestRequestMaxBytes { + t.Fatalf( + "request %d encoded bytes = %d, limit %d", + index, + encodedSizes[index], + gitcrawlCloudIngestRequestMaxBytes, + ) + } + } +} + +func TestSendSnapshotIngestRowsFlushesBeforeEncodedByteLimit(t *testing.T) { + var requests []crawlremote.IngestRequest + var encodedSizes []int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if int64(len(body)) > gitcrawlCloudIngestRequestMaxBytes { + http.Error(w, "encoded request exceeded byte budget", http.StatusRequestEntityTooLarge) + return + } + var request crawlremote.IngestRequest + if err := json.Unmarshal(body, &request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + requests = append(requests, request) + encodedSizes = append(encodedSizes, len(body)) + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + Table: request.Table, + SnapshotID: request.Manifest.SnapshotID, + MutationToken: fmt.Sprintf("mutation-%d", len(requests)), + RowsAccepted: int64(len(request.Rows)), + Complete: request.Final, + }) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("publisher-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + value := strings.Repeat("x", int(gitcrawlCloudIngestRequestMaxBytes/2)) + progress, err := sendSnapshotIngestRows( + context.Background(), + client, + "gitcrawl", + "gitcrawl/openclaw__gitcrawl", + crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__gitcrawl", + SnapshotID: strings.Repeat("a", 64), + }, + "threads", + []string{"body"}, + [][]any{{value}, {value}, {value}}, + "", + true, + ) + if err != nil { + t.Fatalf("send byte-bounded rows: %v", err) + } + if progress.RowsAccepted != 3 || progress.MutationToken != "mutation-3" { + t.Fatalf("progress = %#v", progress) + } + if len(requests) != 3 { + t.Fatalf("requests = %d, want one per large row", len(requests)) + } + for index := range requests { + if len(requests[index].Rows) != 1 { + t.Fatalf("request %d rows = %d, want 1", index, len(requests[index].Rows)) + } + if int64(encodedSizes[index]) > gitcrawlCloudIngestRequestMaxBytes { + t.Fatalf( + "request %d encoded bytes = %d, limit %d", + index, + encodedSizes[index], + gitcrawlCloudIngestRequestMaxBytes, + ) + } + wantCursor := "" + if index > 0 { + wantCursor = fmt.Sprint(index) + } + if requests[index].Cursor != wantCursor { + t.Fatalf("request %d cursor = %q, want %q", index, requests[index].Cursor, wantCursor) + } + if requests[index].Final != (index == len(requests)-1) { + t.Fatalf("request %d final = %v", index, requests[index].Final) + } + } +} + +func TestIngestBatchingRejectsSingleOversizedRowBeforeRequest(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + http.Error(w, "unexpected request", http.StatusInternalServerError) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("publisher-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + _, err = sendSnapshotIngestRows( + context.Background(), + client, + "gitcrawl", + "gitcrawl/openclaw__gitcrawl", + crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__gitcrawl", + SnapshotID: strings.Repeat("a", 64), + }, + "threads", + []string{"body"}, + [][]any{{strings.Repeat("x", int(gitcrawlCloudIngestRequestMaxBytes))}}, + "", + false, + ) + if err == nil || + !strings.Contains(err.Error(), "ingest table threads row 0 encoded request") || + !strings.Contains(err.Error(), fmt.Sprintf("limit %d", gitcrawlCloudIngestRequestMaxBytes)) { + t.Fatalf("oversized row error = %v", err) + } + if requests != 0 { + t.Fatalf("remote requests = %d, want 0", requests) + } + + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + defer db.Close() + if _, err := db.Exec(`create table rows(value text not null)`); err != nil { + t.Fatalf("create rows: %v", err) + } + if _, err := db.Exec( + `insert into rows(value) values(?)`, + strings.Repeat("x", int(gitcrawlCloudIngestRequestMaxBytes)), + ); err != nil { + t.Fatalf("seed oversized row: %v", err) + } + _, err = sendSnapshotIngestDataset( + context.Background(), + db, + client, + "gitcrawl", + "gitcrawl/openclaw__gitcrawl", + crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__gitcrawl", + SnapshotID: strings.Repeat("a", 64), + }, + gitcrawlCloudDataset{ + Name: "oversized", + Columns: []string{"value"}, + Query: `select value from rows`, + RowCount: 1, + }, + "", + ) + if err == nil || + !strings.Contains(err.Error(), "dataset oversized row 0 encoded ingest request") { + t.Fatalf("oversized dataset row error = %v", err) + } + if requests != 0 { + t.Fatalf("remote requests after streamed row = %d, want 0", requests) + } +} + +func TestIngestBatchingMatchesCrawlkitEscapingAtByteBoundary(t *testing.T) { + manifest := crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__gitcrawl", + SchemaName: gitcrawlCloudSchemaName, + SchemaVersion: gitcrawlCloudSchemaVersion, + SchemaHash: gitcrawlCloudSchemaHash, + SnapshotID: strings.Repeat("a", 64), + SourceSHA256: strings.Repeat("a", 64), + } + columns := []string{"value"} + var bodies [][]byte + var requests []crawlremote.IngestRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if int64(len(body)) > gitcrawlCloudIngestRequestMaxBytes { + http.Error(w, "encoded request exceeded byte budget", http.StatusRequestEntityTooLarge) + return + } + var request crawlremote.IngestRequest + if err := json.Unmarshal(body, &request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + bodies = append(bodies, body) + requests = append(requests, request) + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + Table: request.Table, + SnapshotID: request.Manifest.SnapshotID, + MutationToken: fmt.Sprintf("mutation-%d", len(requests)), + RowsAccepted: int64(len(request.Rows)), + Complete: request.Final, + }) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("publisher-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + + fitRows, oversizedRows := escapedIngestBoundaryValues( + t, + manifest, + "escaped_rows", + columns, + true, + ) + if _, err := sendSnapshotIngestRows( + context.Background(), + client, + manifest.App, + manifest.Archive, + manifest, + "escaped_rows", + columns, + [][]any{{fitRows}}, + "", + true, + ); err != nil { + t.Fatalf("send escaped in-memory boundary row: %v", err) + } + assertCrawlkitWireSizeParity(t, requests[0], bodies[0]) + assertEscapedIngestWireFragments(t, bodies[0]) + requestCount := len(requests) + if _, err := sendSnapshotIngestRows( + context.Background(), + client, + manifest.App, + manifest.Archive, + manifest, + "escaped_rows", + columns, + [][]any{{oversizedRows}}, + "", + true, + ); err == nil || !strings.Contains(err.Error(), "encoded request") { + t.Fatalf("escaped in-memory oversized row error = %v", err) + } + if len(requests) != requestCount { + t.Fatalf("escaped in-memory oversized row sent %d requests", len(requests)-requestCount) + } + + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("open escaped stream database: %v", err) + } + defer db.Close() + if _, err := db.Exec(`create table rows(value text not null)`); err != nil { + t.Fatalf("create escaped stream rows: %v", err) + } + fitStream, oversizedStream := escapedIngestBoundaryValues( + t, + manifest, + "escaped_stream", + columns, + false, + ) + if _, err := db.Exec(`insert into rows(value) values(?)`, fitStream); err != nil { + t.Fatalf("seed escaped stream boundary row: %v", err) + } + dataset := gitcrawlCloudDataset{ + Name: "escaped_stream", + Columns: columns, + Query: `select value from rows`, + RowCount: 1, + } + if _, err := sendSnapshotIngestDataset( + context.Background(), + db, + client, + manifest.App, + manifest.Archive, + manifest, + dataset, + "", + ); err != nil { + t.Fatalf("send escaped stream boundary row: %v", err) + } + assertCrawlkitWireSizeParity(t, requests[requestCount], bodies[requestCount]) + assertEscapedIngestWireFragments(t, bodies[requestCount]) + requestCount = len(requests) + if _, err := db.Exec(`update rows set value = ?`, oversizedStream); err != nil { + t.Fatalf("seed escaped stream oversized row: %v", err) + } + if _, err := sendSnapshotIngestDataset( + context.Background(), + db, + client, + manifest.App, + manifest.Archive, + manifest, + dataset, + "", + ); err == nil || !strings.Contains(err.Error(), "encoded ingest request") { + t.Fatalf("escaped stream oversized row error = %v", err) + } + if len(requests) != requestCount { + t.Fatalf("escaped stream oversized row sent %d requests", len(requests)-requestCount) + } +} + +func escapedIngestBoundaryValues( + t *testing.T, + manifest crawlremote.IngestManifest, + table string, + columns []string, + final bool, +) (fit string, oversized string) { + t.Helper() + const pattern = "\"\\\x00\b\f\n\r\t<>&\u2028\u2029\u00e9\u65e5\u672c\u8a9e\U0001F600" + fits := func(repeats int) bool { + t.Helper() + sizer, err := newGitcrawlIngestBatchSizer( + manifest.App, + manifest.Archive, + manifest, + table, + columns, + "", + "", + ) + if err != nil { + t.Fatalf("create escaped ingest batch sizer: %v", err) + } + ok, _, err := sizer.add([]any{strings.Repeat(pattern, repeats)}, final) + if err != nil { + t.Fatalf("size escaped ingest row: %v", err) + } + return ok + } + low, high := 0, 1 + for fits(high) { + low = high + high *= 2 + } + for low+1 < high { + middle := low + (high-low)/2 + if fits(middle) { + low = middle + } else { + high = middle + } + } + return strings.Repeat(pattern, low), strings.Repeat(pattern, high) +} + +func assertCrawlkitWireSizeParity( + t *testing.T, + request crawlremote.IngestRequest, + body []byte, +) { + t.Helper() + expected, err := encodedGitcrawlIngestRequestBytes(request) + if err != nil { + t.Fatalf("size CrawlKit ingest request: %v", err) + } + if int64(len(body)) != expected { + t.Fatalf("CrawlKit wire bytes = %d, batch sizer = %d", len(body), expected) + } + if int64(len(body)) > gitcrawlCloudIngestRequestMaxBytes { + t.Fatalf( + "CrawlKit wire bytes = %d, limit %d", + len(body), + gitcrawlCloudIngestRequestMaxBytes, + ) + } +} + +func assertEscapedIngestWireFragments(t *testing.T, body []byte) { + t.Helper() + for _, fragment := range []string{ + `\"`, + `\\`, + `\u0000`, + `\b`, + `\f`, + `\n`, + `\r`, + `\t`, + `\u003c`, + `\u003e`, + `\u0026`, + `\u2028`, + `\u2029`, + } { + if !bytes.Contains(body, []byte(fragment)) { + t.Fatalf("CrawlKit wire body is missing escaped fragment %q", fragment) + } + } + for _, value := range []string{ + "\u00e9", + "\u65e5\u672c\u8a9e", + "\U0001F600", + } { + if !bytes.Contains(body, []byte(value)) { + t.Fatalf("CrawlKit wire body is missing UTF-8 value %q", value) + } + } +} + +func TestCompleteGitcrawlSnapshotStagingRequiresExactAcknowledgement(t *testing.T) { + snapshotID := strings.Repeat("a", 64) + snapshot := gitcrawlCloudSnapshot{ + ID: snapshotID, + DatasetGeneratedAt: "2026-07-12T12:01:00Z", + Datasets: []gitcrawlCloudDataset{ + {Name: "repositories", RowCount: 1, EligibleCount: 1, CoveredCount: 1, Complete: true}, + {Name: "threads", RowCount: 3, EligibleCount: 3, CoveredCount: 3, Complete: true}, + }, + } + manifest := crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__gitcrawl", + SnapshotID: snapshotID, + } + const mutationToken = "mutation-final" + + for _, test := range []struct { + name string + mutate func(*crawlremote.IngestResult) + want string + }{ + {name: "exact acknowledgement"}, + { + name: "wrong table", + mutate: func(result *crawlremote.IngestResult) { + result.Table = "threads" + }, + want: "want dataset_coverage", + }, + { + name: "wrong snapshot", + mutate: func(result *crawlremote.IngestResult) { + result.SnapshotID = strings.Repeat("b", 64) + }, + want: "want \"" + snapshotID + "\"", + }, + { + name: "wrong dataset count", + mutate: func(result *crawlremote.IngestResult) { + result.RowsAccepted-- + }, + want: "want 2 datasets", + }, + { + name: "wrong mutation token", + mutate: func(result *crawlremote.IngestResult) { + result.MutationToken = "other-mutation" + }, + want: "want \"mutation-final\"", + }, + { + name: "partial candidate", + mutate: func(result *crawlremote.IngestResult) { + result.Complete = false + }, + want: "did not complete snapshot", + }, + } { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var request crawlremote.IngestRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if request.Table != "dataset_coverage" || + !request.Final || + request.MutationToken != mutationToken || + len(request.Rows) != len(snapshot.Datasets) { + http.Error(w, "invalid final coverage request", http.StatusBadRequest) + return + } + result := crawlremote.IngestResult{ + Table: request.Table, + SnapshotID: request.Manifest.SnapshotID, + MutationToken: request.MutationToken, + RowsAccepted: int64(len(request.Rows)), + Complete: true, + } + if test.mutate != nil { + test.mutate(&result) + } + _ = json.NewEncoder(w).Encode(result) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("publisher-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + + _, err = completeGitcrawlSnapshotStaging( + context.Background(), + client, + manifest.App, + manifest.Archive, + manifest, + snapshot, + mutationToken, + ) + if test.want == "" { + if err != nil { + t.Fatalf("complete staging: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("complete staging error = %v, want %q", err, test.want) + } + }) + } +} + +func TestValidateGitcrawlSQLiteBundleUploadRequiresFinalSnapshotDigest(t *testing.T) { + expected := crawlremote.SQLiteBundleManifest{ + SnapshotID: strings.Repeat("a", 64), + Object: crawlremote.SQLiteBundleObject{ + Size: 123, + SHA256: strings.Repeat("a", 64), + }, + } + validResult := func() crawlremote.SQLiteBundleUploadResult { + manifest := expected + return crawlremote.SQLiteBundleUploadResult{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__gitcrawl", + Complete: true, + Bundle: &crawlremote.SQLiteBundle{ + Manifest: &manifest, + }, + } + } + for _, test := range []struct { + name string + mutate func(*crawlremote.SQLiteBundleUploadResult) + want string + }{ + {name: "exact acknowledgement"}, + { + name: "not finalized", + mutate: func(result *crawlremote.SQLiteBundleUploadResult) { + result.Complete = false + }, + want: "not finalized", + }, + { + name: "wrong snapshot", + mutate: func(result *crawlremote.SQLiteBundleUploadResult) { + result.Bundle.Manifest.SnapshotID = strings.Repeat("b", 64) + }, + want: "acknowledged snapshot", + }, + { + name: "wrong digest", + mutate: func(result *crawlremote.SQLiteBundleUploadResult) { + result.Bundle.Manifest.Object.SHA256 = strings.Repeat("b", 64) + }, + want: "acknowledged digest", + }, + { + name: "wrong source size", + mutate: func(result *crawlremote.SQLiteBundleUploadResult) { + result.Bundle.Manifest.Object.Size++ + }, + want: "acknowledged source size", + }, + } { + t.Run(test.name, func(t *testing.T) { + result := validResult() + if test.mutate != nil { + test.mutate(&result) + } + _, err := validateGitcrawlSQLiteBundleUpload( + result, + "gitcrawl", + "gitcrawl/openclaw__gitcrawl", + expected, + ) + if test.want == "" { + if err != nil { + t.Fatalf("validate bundle upload: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("bundle validation error = %v, want %q", err, test.want) + } + }) + } +} + +func TestUploadSQLiteSnapshotArchiveRejectsSameSizeSourceDigestDrift(t *testing.T) { + snapshotPath := filepath.Join(t.TempDir(), "snapshot.db") + selectedSource := []byte("SQLite format 3\x00selected-source") + driftedSource := []byte("SQLite format 3\x00drifted--source") + if len(selectedSource) != len(driftedSource) { + t.Fatalf("test sources differ in size: %d != %d", len(selectedSource), len(driftedSource)) + } + if err := os.WriteFile(snapshotPath, selectedSource, 0o600); err != nil { + t.Fatalf("write selected source: %v", err) + } + selectedSnapshotID, err := cloudFileSHA256(snapshotPath) + if err != nil { + t.Fatalf("hash selected source: %v", err) + } + if err := os.WriteFile(snapshotPath, driftedSource, 0o600); err != nil { + t.Fatalf("rewrite drifted source: %v", err) + } + + uploadRequests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + uploadRequests++ + http.Error(w, "unexpected upload", http.StatusInternalServerError) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("publisher-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + + _, _, err = uploadSQLiteSnapshotArchive( + context.Background(), + client, + "gitcrawl", + "gitcrawl/openclaw__gitcrawl", + snapshotPath, + selectedSnapshotID, + nil, + ) + if err == nil || !strings.Contains(err.Error(), "SQLite bundle source digest changed") { + t.Fatalf("same-size source drift error = %v", err) + } + if uploadRequests != 0 { + t.Fatalf("same-size source drift sent %d upload requests", uploadRequests) + } +} + +func TestGitcrawlReaderStatusMatchesCompleteServingSnapshot(t *testing.T) { + snapshotID := strings.Repeat("a", 64) + const cutoverAt = "2026-07-12T12:02:00.123456789Z" + snapshot := gitcrawlCloudSnapshot{ + ID: snapshotID, + SourceSyncAt: "2026-07-12T12:00:00Z", + DatasetGeneratedAt: "2026-07-12T12:01:00Z", + Datasets: []gitcrawlCloudDataset{{ + Name: "repositories", + RowCount: 1, + EligibleCount: 1, + CoveredCount: 1, + MaxSourceAt: "2026-07-12T12:00:00Z", + Complete: true, + }}, + } + manifest := gitcrawlCloudManifest("gitcrawl/openclaw__gitcrawl", snapshot) + capabilities := gitcrawlCloudPublicationCapabilities(manifest.Capabilities) + status := crawlremote.Status{ + App: manifest.App, + Archive: manifest.Archive, + Mode: "cloud", + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: capabilities, + SnapshotMode: "snapshot", + SnapshotCutoverAt: cutoverAt, + ActiveSnapshotID: snapshotID, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + CoverageComplete: true, + Datasets: []crawlremote.DatasetCoverage{{ + Dataset: "repositories", + RowCount: 1, + EligibleCount: 1, + CoveredCount: 1, + FreshCount: 1, + MaxSourceAt: "2026-07-12T12:00:00Z", + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + Complete: true, + }}, + Snapshot: &crawlremote.ArchiveSnapshot{ + ID: snapshotID, + SourceSHA256: snapshotID, + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: capabilities, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + CoverageComplete: true, + CutoverAt: cutoverAt, + }, + } + if !gitcrawlReaderStatusMatches(status, snapshot, manifest, capabilities, cutoverAt) { + t.Fatal("complete serving snapshot did not match") + } + + status.CoverageComplete = false + if gitcrawlReaderStatusMatches(status, snapshot, manifest, capabilities, cutoverAt) { + t.Fatal("top-level incomplete reader status matched") + } + status.CoverageComplete = true + status.Snapshot.CoverageComplete = false + if gitcrawlReaderStatusMatches(status, snapshot, manifest, capabilities, cutoverAt) { + t.Fatal("nested incomplete reader status matched") + } + status.Snapshot.CoverageComplete = true + status.Datasets[0].CoveredCount = 0 + if gitcrawlReaderStatusMatches(status, snapshot, manifest, capabilities, cutoverAt) { + t.Fatal("reader dataset count drift matched") + } +} + +func TestGitcrawlReaderStatusMatchesRequiresCutoverAttestation(t *testing.T) { + snapshotID := strings.Repeat("a", 64) + const cutoverAt = "2026-07-12T12:02:00.123456789Z" + snapshot := gitcrawlCloudSnapshot{ + ID: snapshotID, + SourceSyncAt: "2026-07-12T12:00:00Z", + DatasetGeneratedAt: "2026-07-12T12:01:00Z", + } + manifest := gitcrawlCloudManifest("gitcrawl/openclaw__gitcrawl", snapshot) + capabilities := gitcrawlCloudPublicationCapabilities(manifest.Capabilities) + valid := crawlremote.Status{ + App: manifest.App, + Archive: manifest.Archive, + Mode: "cloud", + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: capabilities, + SnapshotMode: "snapshot", + SnapshotCutoverAt: cutoverAt, + ActiveSnapshotID: snapshotID, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + CoverageComplete: true, + Snapshot: &crawlremote.ArchiveSnapshot{ + ID: snapshotID, + SourceSHA256: snapshotID, + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: capabilities, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + CoverageComplete: true, + CutoverAt: cutoverAt, + }, + } + + for _, test := range []struct { + name string + expectedCutoverAt string + mutate func(*crawlremote.Status) + }{ + {name: "resumed cutover", expectedCutoverAt: ""}, + {name: "fresh cutover", expectedCutoverAt: cutoverAt}, + { + name: "missing cloud mode", + mutate: func(status *crawlremote.Status) { + status.Mode = "" + }, + }, + { + name: "missing snapshot mode", + mutate: func(status *crawlremote.Status) { + status.SnapshotMode = "" + }, + }, + { + name: "legacy snapshot mode", + mutate: func(status *crawlremote.Status) { + status.SnapshotMode = "legacy" + }, + }, + { + name: "missing top-level cutover", + mutate: func(status *crawlremote.Status) { + status.SnapshotCutoverAt = "" + }, + }, + { + name: "missing nested cutover", + mutate: func(status *crawlremote.Status) { + status.Snapshot.CutoverAt = "" + }, + }, + { + name: "invalid top-level cutover", + mutate: func(status *crawlremote.Status) { + status.SnapshotCutoverAt = "later" + }, + }, + { + name: "inconsistent nested cutover", + mutate: func(status *crawlremote.Status) { + status.Snapshot.CutoverAt = "2026-07-12T12:02:01Z" + }, + }, + { + name: "equivalent nested cutover encoding", + mutate: func(status *crawlremote.Status) { + status.Snapshot.CutoverAt = "2026-07-12T12:02:00.123456789+00:00" + }, + }, + { + name: "fresh acknowledgement mismatch", + expectedCutoverAt: "2026-07-12T12:02:01Z", + }, + { + name: "equivalent acknowledgement encoding", + expectedCutoverAt: "2026-07-12T12:02:00.123456789+00:00", + }, + } { + t.Run(test.name, func(t *testing.T) { + status := valid + snapshotEnvelope := *valid.Snapshot + status.Snapshot = &snapshotEnvelope + if test.mutate != nil { + test.mutate(&status) + } + matched := gitcrawlReaderStatusMatches( + status, + snapshot, + manifest, + capabilities, + test.expectedCutoverAt, + ) + want := test.mutate == nil && + (test.expectedCutoverAt == "" || test.expectedCutoverAt == cutoverAt) + if matched != want { + t.Fatalf("cutover status match = %v, want %v", matched, want) + } + }) + } +} + +func TestValidateGitcrawlCutoverResultRequiresExactAcknowledgement(t *testing.T) { + const archive = "gitcrawl/openclaw__gitcrawl" + snapshotID := strings.Repeat("a", 64) + valid := crawlremote.CutoverResult{ + Archive: archive, + SnapshotID: snapshotID, + SnapshotMode: "snapshot", + CutoverAt: "2026-07-12T12:00:00.123456789Z", + } + for _, test := range []struct { + name string + mutate func(*crawlremote.CutoverResult) + want string + }{ + {name: "exact acknowledgement"}, + { + name: "wrong archive", + mutate: func(result *crawlremote.CutoverResult) { + result.Archive = "gitcrawl/other" + }, + want: "want \"" + archive + "\"", + }, + { + name: "wrong snapshot", + mutate: func(result *crawlremote.CutoverResult) { + result.SnapshotID = strings.Repeat("b", 64) + }, + want: "want \"" + snapshotID + "\"", + }, + { + name: "wrong mode", + mutate: func(result *crawlremote.CutoverResult) { + result.SnapshotMode = "mutable" + }, + want: "want snapshot", + }, + { + name: "invalid timestamp", + mutate: func(result *crawlremote.CutoverResult) { + result.CutoverAt = "later" + }, + want: "invalid timestamp", + }, + } { + t.Run(test.name, func(t *testing.T) { + result := valid + if test.mutate != nil { + test.mutate(&result) + } + err := validateGitcrawlCutoverResult(result, archive, snapshotID) + if test.want == "" { + if err != nil { + t.Fatalf("validate cutover: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("validation error = %v, want %q", err, test.want) + } + }) + } +} + +func TestVerifyGitcrawlReaderProjectionUsesBoundedExactRetries(t *testing.T) { + snapshotID := strings.Repeat("a", 64) + const cutoverAt = "2026-07-12T12:02:00Z" + snapshot := gitcrawlCloudSnapshot{ + ID: snapshotID, + SourceSyncAt: "2026-07-12T12:00:00Z", + DatasetGeneratedAt: "2026-07-12T12:01:00Z", + Datasets: []gitcrawlCloudDataset{{ + Name: "repositories", + RowCount: 1, + EligibleCount: 1, + CoveredCount: 1, + MaxSourceAt: "2026-07-12T12:00:00Z", + Complete: true, + }}, + } + manifest := gitcrawlCloudManifest("gitcrawl/openclaw__gitcrawl", snapshot) + capabilities := gitcrawlCloudPublicationCapabilities(manifest.Capabilities) + exactStatus := crawlremote.Status{ + App: manifest.App, + Archive: manifest.Archive, + Mode: "cloud", + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: capabilities, + SnapshotMode: "snapshot", + SnapshotCutoverAt: cutoverAt, + ActiveSnapshotID: snapshot.ID, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + CoverageComplete: true, + Datasets: []crawlremote.DatasetCoverage{{ + Dataset: "repositories", + RowCount: 1, + EligibleCount: 1, + CoveredCount: 1, + FreshCount: 1, + MaxSourceAt: "2026-07-12T12:00:00Z", + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + Complete: true, + }}, + Snapshot: &crawlremote.ArchiveSnapshot{ + ID: snapshot.ID, + SourceSHA256: snapshot.ID, + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: capabilities, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + CoverageComplete: true, + CutoverAt: cutoverAt, + }, + } + + for _, test := range []struct { + name string + exactAt int + want string + wantCalls int + }{ + { + name: "eventually exact", + exactAt: 3, + wantCalls: 3, + }, + { + name: "retry bound exhausted", + want: "after 3 attempts", + wantCalls: 3, + }, + } { + t.Run(test.name, func(t *testing.T) { + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + status := exactStatus + if test.exactAt == 0 || calls < test.exactAt { + status.ActiveSnapshotID = strings.Repeat("b", 64) + } + _ = json.NewEncoder(w).Encode(status) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("reader-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + err = verifyGitcrawlReaderProjectionWithRetry( + context.Background(), + client, + manifest.Archive, + snapshot, + manifest, + capabilities, + cutoverAt, + 3, + 0, + ) + if test.want == "" { + if err != nil { + t.Fatalf("verify reader projection: %v", err) + } + } else if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("verification error = %v, want %q", err, test.want) + } + if calls != test.wantCalls { + t.Fatalf("status calls = %d, want %d", calls, test.wantCalls) + } + }) + } +} + +func TestRecoverConcurrentGitcrawlSnapshotAdoptsOnlyMatchingCompletedSnapshot(t *testing.T) { + snapshotID := strings.Repeat("a", 64) + snapshot := gitcrawlCloudSnapshot{ + ID: snapshotID, + SourceSyncAt: "2026-07-12T12:00:00Z", + DatasetGeneratedAt: "2026-07-12T12:01:00Z", + } + manifest := gitcrawlCloudManifest("gitcrawl/openclaw__openclaw", snapshot) + publicationCapabilities := gitcrawlCloudPublicationCapabilities(manifest.Capabilities) + const winnerGeneration = "2026-07-12T12:02:00Z" + + for _, test := range []struct { + name string + activeSnapshotID string + coverageComplete bool + wantGeneration string + want string + }{ + { + name: "independent winner generation", + activeSnapshotID: snapshotID, + coverageComplete: true, + wantGeneration: winnerGeneration, + }, + { + name: "unrelated active candidate", + activeSnapshotID: strings.Repeat("b", 64), + coverageComplete: true, + want: "does not match the requested digest, profile, and coverage", + }, + { + name: "incomplete candidate", + activeSnapshotID: snapshotID, + want: "does not match the requested digest, profile, and coverage", + }, + } { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.URL.Query().Get("snapshot_id"); got != snapshotID { + http.Error(w, fmt.Sprintf("snapshot_id = %q, want %q", got, snapshotID), http.StatusBadRequest) + return + } + _ = json.NewEncoder(w).Encode(crawlremote.PublisherStatus{ + App: manifest.App, + Archive: manifest.Archive, + ActiveSnapshotID: test.activeSnapshotID, + CoverageComplete: test.coverageComplete, + Snapshot: &crawlremote.ArchiveSnapshot{ + ID: snapshotID, + SourceSHA256: snapshotID, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: winnerGeneration, + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: publicationCapabilities, + CoverageComplete: test.coverageComplete, + }, + }) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("publisher-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + + generation, err := recoverConcurrentGitcrawlSnapshot( + context.Background(), + client, + manifest.Archive, + snapshot, + manifest, + publicationCapabilities, + &crawlremote.Error{ + Status: http.StatusConflict, + Code: "snapshot_active", + Message: "a concurrent publisher completed the snapshot", + }, + ) + if test.want != "" { + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("recovery error = %v, want %q", err, test.want) + } + if generation != "" { + t.Fatalf("recovered generation = %q, want none", generation) + } + return + } + if err != nil { + t.Fatalf("recover concurrent snapshot: %v", err) + } + if generation != test.wantGeneration { + t.Fatalf("recovered generation = %q, want %q", generation, test.wantGeneration) + } + }) + } +} + +func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t *testing.T) { + source := []byte("SQLite format 3\x00bound source") + snapshotID := fmt.Sprintf("%x", sha256.Sum256(source)) + const cutoverAt = "2026-07-12T12:02:00Z" + snapshot := gitcrawlCloudSnapshot{ + ID: snapshotID, + SourceSyncAt: "2026-07-12T12:00:00Z", + DatasetGeneratedAt: "2026-07-12T12:01:00Z", + } + manifest := gitcrawlCloudManifest("gitcrawl/openclaw__openclaw", snapshot) + publicationCapabilities := gitcrawlCloudPublicationCapabilities(manifest.Capabilities) + + for _, test := range []struct { + name string + statusGeneration string + statusCode int + body []byte + want string + }{ + { + name: "publisher generation mismatch", + statusGeneration: "2026-07-12T12:02:00Z", + statusCode: http.StatusOK, + body: source, + want: "post-cutover publisher status does not match", + }, + { + name: "bound snapshot unavailable", + statusCode: http.StatusConflict, + body: []byte(`{"error":"snapshot_bundle_required"}`), + want: "status=409", + }, + { + name: "downloaded digest mismatch", + statusCode: http.StatusOK, + body: bytes.Repeat([]byte("x"), len(source)), + want: "does not match source", + }, + } { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/status"): + _ = json.NewEncoder(w).Encode(crawlremote.Status{ + App: manifest.App, + Archive: manifest.Archive, + Mode: "cloud", + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: publicationCapabilities, + SnapshotMode: "snapshot", + SnapshotCutoverAt: cutoverAt, + ActiveSnapshotID: snapshot.ID, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + CoverageComplete: true, + Snapshot: &crawlremote.ArchiveSnapshot{ + ID: snapshot.ID, + SourceSHA256: snapshot.ID, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: publicationCapabilities, + CoverageComplete: true, + CutoverAt: cutoverAt, + }, + }) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status"): + if got := r.URL.Query().Get("snapshot_id"); got != snapshotID { + http.Error( + w, + fmt.Sprintf("snapshot_id = %q, want %q", got, snapshotID), + http.StatusBadRequest, + ) + return + } + w.Header().Set("content-type", "application/json") + statusGeneration := snapshot.DatasetGeneratedAt + if test.statusGeneration != "" { + statusGeneration = test.statusGeneration + } + _ = json.NewEncoder(w).Encode(crawlremote.PublisherStatus{ + App: "gitcrawl", + Archive: manifest.Archive, + ActiveSnapshotID: snapshotID, + CoverageComplete: true, + Snapshot: &crawlremote.ArchiveSnapshot{ + ID: snapshotID, + SourceSHA256: snapshotID, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: statusGeneration, + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: publicationCapabilities, + CoverageComplete: true, + }, + }) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/sqlite"): + if test.statusCode == http.StatusOK { + w.Header().Set("content-length", fmt.Sprintf("%d", len(test.body))) + w.Header().Set("x-crawl-content-sha256", snapshotID) + } + w.WriteHeader(test.statusCode) + _, _ = w.Write(test.body) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + httpClient := &http.Client{Timeout: time.Second} + tokenProvider := crawlremote.StaticToken("reader-publisher-token") + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: httpClient, + TokenProvider: tokenProvider, + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + err = verifyGitcrawlSnapshotPublication( + context.Background(), + client, + httpClient, + tokenProvider, + server.URL, + manifest.Archive, + snapshot, + manifest, + publicationCapabilities, + cutoverAt, + int64(len(source)), + ) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("verification error = %v, want %q", err, test.want) + } + }) + } +} + +func TestVerifyGitcrawlSQLiteHydrationRejectsInvalidFramingAndDigest(t *testing.T) { + source := []byte("SQLite format 3\x00bound source") + snapshotID := fmt.Sprintf("%x", sha256.Sum256(source)) + different := bytes.Repeat([]byte("x"), len(source)) + + tests := []struct { + name string + body []byte + digest string + contentLength int64 + contentLengthHeader string + want string + }{ + { + name: "valid", + body: source, + digest: snapshotID, + contentLength: int64(len(source)), + }, + { + name: "missing digest", + body: source, + contentLength: int64(len(source)), + want: "missing x-crawl-content-sha256", + }, + { + name: "wrong digest", + body: source, + digest: strings.Repeat("f", 64), + contentLength: int64(len(source)), + want: "advertises digest", + }, + { + name: "chunked response", + body: source, + digest: snapshotID, + contentLength: -1, + }, + { + name: "malformed length", + body: source, + digest: snapshotID, + contentLength: -1, + contentLengthHeader: "not-a-number", + want: "invalid Content-Length", + }, + { + name: "wrong length", + body: source, + digest: snapshotID, + contentLength: int64(len(source) + 1), + want: "does not match uploaded source size", + }, + { + name: "truncated body", + body: source[:len(source)-1], + digest: snapshotID, + contentLength: int64(len(source)), + want: "truncated", + }, + { + name: "oversized body", + body: append(append([]byte(nil), source...), 'x'), + digest: snapshotID, + contentLength: int64(len(source)), + want: "exceeds uploaded source size", + }, + { + name: "body digest mismatch", + body: different, + digest: snapshotID, + contentLength: int64(len(source)), + want: "does not match source", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + response := &http.Response{ + Header: make(http.Header), + Body: io.NopCloser(bytes.NewReader(test.body)), + ContentLength: test.contentLength, + } + if test.digest != "" { + response.Header.Set("x-crawl-content-sha256", test.digest) + } + if test.contentLengthHeader != "" { + response.Header.Set("content-length", test.contentLengthHeader) + } + err := verifyGitcrawlSQLiteHydration(response, snapshotID, int64(len(source))) + if test.want == "" { + if err != nil { + t.Fatalf("verify hydration: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("verification error = %v, want %q", err, test.want) + } + }) + } +} + +func TestVerifyGitcrawlSQLiteHydrationAcceptsChunkedResponse(t *testing.T) { + source := []byte("SQLite format 3\x00chunked source") + snapshotID := fmt.Sprintf("%x", sha256.Sum256(source)) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("x-crawl-content-sha256", snapshotID) + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + _, _ = w.Write(source) + })) + defer server.Close() + + response, err := server.Client().Get(server.URL) + if err != nil { + t.Fatalf("download chunked hydration: %v", err) + } + defer response.Body.Close() + if response.ContentLength != -1 || !slices.Contains(response.TransferEncoding, "chunked") { + t.Fatalf( + "response framing = length %d transfer %v, want chunked", + response.ContentLength, + response.TransferEncoding, + ) + } + if err := verifyGitcrawlSQLiteHydration( + response, + snapshotID, + int64(len(source)), + ); err != nil { + t.Fatalf("verify chunked hydration: %v", err) + } +} diff --git a/internal/cli/cloud_snapshot.go b/internal/cli/cloud_snapshot.go new file mode 100644 index 0000000..4ece1b7 --- /dev/null +++ b/internal/cli/cloud_snapshot.go @@ -0,0 +1,501 @@ +package cli + +import ( + "context" + "database/sql" + "fmt" + "slices" + "strings" + "time" + + crawlremote "github.com/openclaw/crawlkit/remote" + crawlstore "github.com/openclaw/gitcrawl/internal/store" +) + +const ( + gitcrawlCloudSchemaName = "gitcrawl-cloud-v2" + gitcrawlCloudSchemaVersion = 8 + gitcrawlCloudSchemaHash = "gitcrawl-cloud-v2" + + gitcrawlObservationOrderCapability = "gitcrawl.observation-order.v1" +) + +type gitcrawlCloudDataset struct { + Name string + Columns []string + Query string + RowCount int64 + EligibleCount int64 + CoveredCount int64 + MaxSourceAt string + Complete bool +} + +type gitcrawlCloudSnapshot struct { + ID string + SourceSyncAt string + DatasetGeneratedAt string + Capabilities []string + Datasets []gitcrawlCloudDataset + Hydration crawlstore.EnrichmentCoverage +} + +func buildGitcrawlCloudSnapshot( + ctx context.Context, + db *sql.DB, + snapshotPath string, + allowIncomplete bool, + observationOrder bool, +) (gitcrawlCloudSnapshot, error) { + snapshotID, err := cloudFileSHA256(snapshotPath) + if err != nil { + return gitcrawlCloudSnapshot{}, err + } + sourceSyncAt, err := gitcrawlCloudSourceSyncAt(ctx, db) + if err != nil { + return gitcrawlCloudSnapshot{}, err + } + capabilities, err := gitcrawlCloudCapabilities(ctx, db, observationOrder) + if err != nil { + return gitcrawlCloudSnapshot{}, err + } + hydration, err := gitcrawlCloudHydration(ctx, snapshotPath) + if err != nil { + return gitcrawlCloudSnapshot{}, err + } + datasets, err := loadGitcrawlCloudDatasets( + ctx, + db, + slices.Contains(capabilities, gitcrawlObservationOrderCapability), + hydration, + ) + if err != nil { + return gitcrawlCloudSnapshot{}, err + } + if len(datasets) == 0 || datasets[0].RowCount == 0 { + return gitcrawlCloudSnapshot{}, fmt.Errorf("cloud snapshot has no repositories") + } + missing := incompleteGitcrawlCloudHydration(hydration) + if len(missing) > 0 && !allowIncomplete { + return gitcrawlCloudSnapshot{}, fmt.Errorf( + "cloud snapshot enrichment is incomplete (%s); hydrate the archive or pass --allow-incomplete", + strings.Join(missing, ", "), + ) + } + return gitcrawlCloudSnapshot{ + ID: snapshotID, + SourceSyncAt: sourceSyncAt, + DatasetGeneratedAt: time.Now().UTC().Format(time.RFC3339Nano), + Capabilities: capabilities, + Datasets: datasets, + Hydration: hydration, + }, nil +} + +func gitcrawlCloudManifest(archive string, snapshot gitcrawlCloudSnapshot) crawlremote.IngestManifest { + capabilities := slices.Clone(snapshot.Capabilities) + if !slices.Contains(capabilities, gitcrawlSnapshotStagingCapability) { + capabilities = append(capabilities, gitcrawlSnapshotStagingCapability) + } + return crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: archive, + SchemaName: gitcrawlCloudSchemaName, + SchemaVersion: gitcrawlCloudSchemaVersion, + SchemaHash: gitcrawlCloudSchemaHash, + Mode: crawlremote.ModePublisher, + Source: "sqlite", + SourceSyncAt: snapshot.SourceSyncAt, + SnapshotID: snapshot.ID, + SourceSHA256: snapshot.ID, + Capabilities: capabilities, + } +} + +func loadGitcrawlCloudDatasets( + ctx context.Context, + db *sql.DB, + observationOrder bool, + hydration crawlstore.EnrichmentCoverage, +) ([]gitcrawlCloudDataset, error) { + threadColumns := append([]string(nil), gitcrawlThreadColumns...) + threadQuery, err := gitcrawlThreadExportSQL(ctx, db) + if err != nil { + return nil, err + } + threadSelect := strings.TrimSpace(strings.TrimSuffix(threadQuery, "order by repo_id, number")) + revisionColumns := []string{ + "id", "thread_id", "source_updated_at", "content_hash", "title_hash", + "body_hash", "labels_hash", "created_at", + } + revisionSelect := ` +select id, thread_id, coalesce(source_updated_at, ''), content_hash, title_hash, + body_hash, labels_hash, created_at +from thread_revisions` + if observationOrder { + threadColumns = append(threadColumns, "observation_sequence") + threadSelect = strings.Replace( + threadSelect, + "from threads", + ", observation_sequence\nfrom threads", + 1, + ) + revisionColumns = append(revisionColumns, "observation_sequence") + revisionSelect = strings.Replace( + revisionSelect, + "from thread_revisions", + ", observation_sequence\nfrom thread_revisions", + 1, + ) + } + + specs := []struct { + name string + columns []string + query string + sourceAtQuery string + }{ + { + name: "repositories", + columns: gitcrawlRepositoryColumns, + query: gitcrawlRepositoryExportSQL, + sourceAtQuery: `select coalesce(updated_at, '') from repositories`, + }, + { + name: "threads", + columns: threadColumns, + query: threadSelect + "\norder by repo_id, number", + sourceAtQuery: `select coalesce(nullif(updated_at_gh, ''), updated_at, '') from threads`, + }, + { + name: "thread_revisions", + columns: revisionColumns, + query: revisionSelect + "\norder by id", + sourceAtQuery: `select coalesce(nullif(source_updated_at, ''), created_at, '') from thread_revisions`, + }, + { + name: "thread_fingerprints", + columns: []string{ + "id", "thread_revision_id", "algorithm_version", "fingerprint_hash", + "fingerprint_slug", "body_token_hash", "file_set_hash", "simhash64", "created_at", + }, + query: ` +select id, thread_revision_id, algorithm_version, fingerprint_hash, + fingerprint_slug, body_token_hash, file_set_hash, simhash64, created_at +from thread_fingerprints +order by id`, + sourceAtQuery: `select coalesce(created_at, '') from thread_fingerprints`, + }, + { + name: "thread_key_summaries", + columns: []string{ + "id", "thread_revision_id", "summary_kind", "prompt_version", "provider", + "model", "input_hash", "output_hash", "key_text", "created_at", + }, + query: ` +select id, thread_revision_id, summary_kind, prompt_version, provider, + model, input_hash, output_hash, key_text, created_at +from thread_key_summaries +order by id`, + sourceAtQuery: `select coalesce(created_at, '') from thread_key_summaries`, + }, + { + name: "cluster_groups", + columns: []string{ + "id", "repo_id", "stable_key", "stable_slug", "status", "cluster_type", + "representative_thread_id", "title", "member_count", "created_at", "updated_at", "closed_at", + }, + query: ` +select cluster.id, cluster.repo_id, cluster.stable_key, cluster.stable_slug, + cluster.status, coalesce(cluster.cluster_type, ''), + cluster.representative_thread_id, coalesce(cluster.title, ''), + (select count(*) from cluster_memberships membership + where membership.cluster_id = cluster.id and membership.state = 'active'), + cluster.created_at, cluster.updated_at, coalesce(cluster.closed_at, '') +from cluster_groups cluster +order by cluster.id`, + sourceAtQuery: `select coalesce(updated_at, '') from cluster_groups`, + }, + { + name: "cluster_memberships", + columns: []string{ + "cluster_id", "thread_id", "role", "state", "score_to_representative", + "created_at", "updated_at", "removed_at", + }, + query: ` +select cluster_id, thread_id, role, state, score_to_representative, + created_at, updated_at, coalesce(removed_at, '') +from cluster_memberships +order by cluster_id, thread_id`, + sourceAtQuery: `select coalesce(updated_at, '') from cluster_memberships`, + }, + { + name: "pull_request_details", + columns: []string{ + "thread_id", "repo_id", "number", "base_sha", "head_sha", "head_ref", + "head_repo_full_name", "mergeable_state", "additions", "deletions", + "changed_files", "fetched_at", "updated_at", + }, + query: ` +select thread_id, repo_id, number, coalesce(base_sha, ''), coalesce(head_sha, ''), + coalesce(head_ref, ''), coalesce(head_repo_full_name, ''), + coalesce(mergeable_state, ''), additions, deletions, changed_files, + fetched_at, updated_at +from pull_request_details +order by thread_id`, + sourceAtQuery: `select coalesce(fetched_at, '') from pull_request_details`, + }, + { + name: "pull_request_files", + columns: []string{ + "thread_id", "position", "path", "status", "additions", "deletions", + "changes", "previous_path", "fetched_at", + }, + query: ` +select thread_id, position, path, coalesce(status, ''), additions, deletions, + changes, coalesce(previous_path, ''), fetched_at +from pull_request_files +order by thread_id, position`, + sourceAtQuery: `select coalesce(fetched_at, '') from pull_request_files`, + }, + } + + datasets := make([]gitcrawlCloudDataset, 0, len(specs)) + for _, spec := range specs { + statement, err := db.PrepareContext(ctx, spec.query) + if err != nil { + return nil, fmt.Errorf("prepare cloud dataset %s export: %w", spec.name, err) + } + if err := statement.Close(); err != nil { + return nil, fmt.Errorf("close cloud dataset %s export: %w", spec.name, err) + } + var rowCount int64 + if err := db.QueryRowContext( + ctx, + "select count(*) from "+spec.name, + ).Scan(&rowCount); err != nil { + return nil, fmt.Errorf("count cloud dataset %s: %w", spec.name, err) + } + maxSourceAt, err := latestRFC3339QueryValue(ctx, db, spec.sourceAtQuery) + if err != nil { + return nil, fmt.Errorf("read cloud dataset %s freshness: %w", spec.name, err) + } + dataset := gitcrawlCloudDataset{ + Name: spec.name, + Columns: spec.columns, + Query: spec.query, + RowCount: rowCount, + EligibleCount: rowCount, + CoveredCount: rowCount, + MaxSourceAt: maxSourceAt, + Complete: true, + } + if observationOrder && spec.name == "thread_revisions" { + dataset.EligibleCount = int64(hydration.Revisions.Eligible) + dataset.CoveredCount = int64(hydration.Revisions.Fresh) + dataset.Complete = hydration.Revisions.Supported && hydration.Revisions.Complete + } + datasets = append(datasets, dataset) + } + return datasets, nil +} + +func gitcrawlCloudCapabilities(ctx context.Context, db *sql.DB, observationOrder bool) ([]string, error) { + if !observationOrder { + return nil, nil + } + threadSequence, err := sqliteColumnExists(ctx, db, "threads", "observation_sequence") + if err != nil { + return nil, err + } + revisionSequence, err := sqliteColumnExists(ctx, db, "thread_revisions", "observation_sequence") + if err != nil { + return nil, err + } + if !threadSequence || !revisionSequence { + return nil, fmt.Errorf( + "--observation-order requires observation_sequence on threads and thread_revisions", + ) + } + return []string{gitcrawlObservationOrderCapability}, nil +} + +func gitcrawlCloudSourceSyncAt(ctx context.Context, db *sql.DB) (string, error) { + queries := make([]string, 0, 4) + if ok, err := sqliteTableHasColumns( + ctx, + db, + "sync_runs", + "status", + "started_at", + "finished_at", + ); err != nil { + return "", err + } else if ok { + queries = append(queries, ` +select coalesce(finished_at, started_at, '') +from sync_runs +where status in ('success', 'completed')`) + } + if ok, err := sqliteTableHasColumns(ctx, db, "portable_metadata", "key", "value"); err != nil { + return "", err + } else if ok { + queries = append(queries, ` +select value +from portable_metadata +where key = 'exported_at'`) + } + if ok, err := sqliteTableHasColumns(ctx, db, "threads", "updated_at"); err != nil { + return "", err + } else if ok { + hasGitHubUpdatedAt, err := sqliteColumnExists(ctx, db, "threads", "updated_at_gh") + if err != nil { + return "", err + } + if hasGitHubUpdatedAt { + queries = append(queries, `select coalesce(nullif(updated_at_gh, ''), updated_at, '') from threads`) + } else { + queries = append(queries, `select coalesce(updated_at, '') from threads`) + } + } + if ok, err := sqliteTableHasColumns(ctx, db, "repositories", "updated_at"); err != nil { + return "", err + } else if ok { + queries = append(queries, `select coalesce(updated_at, '') from repositories`) + } + + var latest time.Time + for _, query := range queries { + value, err := latestRFC3339QueryValue(ctx, db, query) + if err != nil { + return "", fmt.Errorf("read cloud snapshot source sync time: %w", err) + } + if value == "" { + continue + } + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return "", fmt.Errorf("parse normalized cloud snapshot source sync time %q: %w", value, err) + } + if latest.IsZero() || parsed.After(latest) { + latest = parsed + } + } + if latest.IsZero() { + return "", nil + } + return latest.UTC().Format(time.RFC3339Nano), nil +} + +func sqliteTableHasColumns(ctx context.Context, db *sql.DB, table string, columns ...string) (bool, error) { + exists, err := sqliteTableExists(ctx, db, table) + if err != nil || !exists { + return false, err + } + for _, column := range columns { + exists, err := sqliteColumnExists(ctx, db, table, column) + if err != nil || !exists { + return false, err + } + } + return true, nil +} + +func latestRFC3339QueryValue(ctx context.Context, db *sql.DB, query string) (string, error) { + rows, err := db.QueryContext(ctx, query) + if err != nil { + return "", err + } + defer rows.Close() + + var latest time.Time + for rows.Next() { + var value string + if err := rows.Scan(&value); err != nil { + return "", err + } + value = strings.TrimSpace(value) + if value == "" { + continue + } + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return "", fmt.Errorf("parse RFC3339 timestamp %q: %w", value, err) + } + if latest.IsZero() || parsed.After(latest) { + latest = parsed + } + } + if err := rows.Err(); err != nil { + return "", err + } + if latest.IsZero() { + return "", nil + } + return latest.UTC().Format(time.RFC3339Nano), nil +} + +func gitcrawlCloudHydration(ctx context.Context, snapshotPath string) (crawlstore.EnrichmentCoverage, error) { + st, err := crawlstore.OpenReadOnly(ctx, snapshotPath) + if err != nil { + return crawlstore.EnrichmentCoverage{}, fmt.Errorf("open cloud snapshot for hydration coverage: %w", err) + } + defer st.Close() + coverage, err := st.ArchiveCoverage(ctx, crawlstore.ArchiveCoverageOptions{}) + if err != nil { + return crawlstore.EnrichmentCoverage{}, fmt.Errorf("read cloud snapshot hydration coverage: %w", err) + } + return coverage.Totals.Enrichment, nil +} + +func incompleteGitcrawlCloudHydration(coverage crawlstore.EnrichmentCoverage) []string { + metrics := []struct { + name string + metric crawlstore.EnrichmentCoverageMetric + }{ + {name: "revisions", metric: coverage.Revisions}, + {name: "fingerprints", metric: coverage.Fingerprints}, + {name: "summaries", metric: coverage.Summaries}, + {name: "clusters", metric: coverage.Clusters}, + {name: "pr_details", metric: coverage.PRDetails}, + {name: "pr_files", metric: coverage.PRFiles}, + } + var missing []string + for _, item := range metrics { + if !item.metric.Supported || !item.metric.Complete { + missing = append(missing, fmt.Sprintf( + "%s=%d/%d fresh=%d", + item.name, + item.metric.Covered, + item.metric.Eligible, + item.metric.Fresh, + )) + } + } + return missing +} + +func gitcrawlCloudCoverageRows(snapshot gitcrawlCloudSnapshot, mutationToken string) [][]any { + rows := make([][]any, 0, len(snapshot.Datasets)) + for _, dataset := range snapshot.Datasets { + rows = append(rows, []any{ + dataset.Name, + dataset.RowCount, + dataset.EligibleCount, + dataset.CoveredCount, + dataset.MaxSourceAt, + snapshot.DatasetGeneratedAt, + dataset.Complete, + mutationToken, + }) + } + return rows +} + +func gitcrawlCloudDatasetCounts(snapshot gitcrawlCloudSnapshot) map[string]int64 { + counts := make(map[string]int64, len(snapshot.Datasets)) + for _, dataset := range snapshot.Datasets { + counts[dataset.Name] = dataset.RowCount + } + return counts +} diff --git a/internal/cli/cloud_snapshot_test.go b/internal/cli/cloud_snapshot_test.go new file mode 100644 index 0000000..1872ea5 --- /dev/null +++ b/internal/cli/cloud_snapshot_test.go @@ -0,0 +1,252 @@ +package cli + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "testing" + + crawlstore "github.com/openclaw/gitcrawl/internal/store" +) + +func TestLatestRFC3339QueryValueUsesParsedTimestampOrder(t *testing.T) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + defer db.Close() + if _, err := db.Exec(` + create table timestamps(value text not null); + insert into timestamps(value) values + ('2026-07-12T01:00:00+02:00'), + ('2026-07-12T00:30:00Z'), + (''); + `); err != nil { + t.Fatalf("seed timestamps: %v", err) + } + + got, err := latestRFC3339QueryValue( + context.Background(), + db, + `select value from timestamps`, + ) + if err != nil { + t.Fatalf("latest timestamp: %v", err) + } + if got != "2026-07-12T00:30:00Z" { + t.Fatalf("latest timestamp = %q, want parsed maximum", got) + } +} + +func TestGitcrawlCloudSourceSyncAtNormalizesRFC3339Maximum(t *testing.T) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + defer db.Close() + if _, err := db.Exec(` + create table sync_runs(status text, started_at text, finished_at text); + create table threads(updated_at_gh text, updated_at text); + create table repositories(updated_at text); + insert into sync_runs(status, started_at, finished_at) + values('success', '2026-07-12T01:00:00+02:00', ''); + insert into threads(updated_at_gh, updated_at) + values('2026-07-12T00:30:00Z', '2026-07-12T00:00:00Z'); + insert into repositories(updated_at) + values('2026-07-12T00:15:00Z'); + `); err != nil { + t.Fatalf("seed source clocks: %v", err) + } + + got, err := gitcrawlCloudSourceSyncAt(context.Background(), db) + if err != nil { + t.Fatalf("source sync at: %v", err) + } + if got != "2026-07-12T00:30:00Z" { + t.Fatalf("source sync at = %q, want parsed maximum", got) + } +} + +func TestGitcrawlCloudSourceSyncAtUsesPortableMetadataWithoutSyncRuns(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "portable.db") + st, err := crawlstore.Open(ctx, path) + if err != nil { + t.Fatalf("open store: %v", err) + } + defer st.Close() + repoID, err := st.UpsertRepository(ctx, crawlstore.Repository{ + Owner: "openclaw", + Name: "gitcrawl", + FullName: "openclaw/gitcrawl", + UpdatedAt: "2026-07-11T23:00:00Z", + }) + if err != nil { + t.Fatalf("seed repository: %v", err) + } + if _, err := st.UpsertThread(ctx, crawlstore.Thread{ + RepoID: repoID, + GitHubID: "portable-1", + Number: 1, + Kind: "issue", + State: "open", + Title: "Portable source clock", + HTMLURL: "https://github.com/openclaw/gitcrawl/issues/1", + LabelsJSON: "[]", + AssigneesJSON: "[]", + ContentHash: "portable-1", + UpdatedAtGitHub: "2026-07-11T23:30:00Z", + UpdatedAt: "2026-07-11T23:30:00Z", + }); err != nil { + t.Fatalf("seed thread: %v", err) + } + if _, err := st.PrunePortablePayloads(ctx, crawlstore.PortablePruneOptions{BodyChars: 64}); err != nil { + t.Fatalf("prune portable store: %v", err) + } + hasSyncRuns, err := sqliteTableExists(ctx, st.DB(), "sync_runs") + if err != nil { + t.Fatalf("inspect sync_runs: %v", err) + } + if hasSyncRuns { + t.Fatal("portable store retained sync_runs") + } + var exportedAt string + if err := st.DB().QueryRowContext( + ctx, + `select value from portable_metadata where key = 'exported_at'`, + ).Scan(&exportedAt); err != nil { + t.Fatalf("read portable exported_at: %v", err) + } + + got, err := gitcrawlCloudSourceSyncAt(ctx, st.DB()) + if err != nil { + t.Fatalf("source sync at: %v", err) + } + if got != exportedAt { + t.Fatalf("source sync at = %q, want portable exported_at %q", got, exportedAt) + } + if err := st.Close(); err != nil { + t.Fatalf("close portable store: %v", err) + } + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatalf("reopen portable sqlite: %v", err) + } + defer db.Close() + snapshot, err := buildGitcrawlCloudSnapshot(ctx, db, path, true, false) + if err != nil { + t.Fatalf("build portable cloud snapshot: %v", err) + } + if snapshot.SourceSyncAt != exportedAt { + t.Fatalf("snapshot source sync at = %q, want %q", snapshot.SourceSyncAt, exportedAt) + } + if counts := gitcrawlCloudDatasetCounts(snapshot); counts["repositories"] != 1 || counts["threads"] != 1 { + t.Fatalf("portable snapshot counts = %#v", counts) + } +} + +func TestObservationOrderRevisionCoverageCountsFreshSelectedThreads(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "coverage.db") + st, err := crawlstore.Open(ctx, path) + if err != nil { + t.Fatalf("open store: %v", err) + } + defer st.Close() + repoID, err := st.UpsertRepository(ctx, crawlstore.Repository{ + Owner: "openclaw", + Name: "gitcrawl", + FullName: "openclaw/gitcrawl", + UpdatedAt: "2026-07-12T10:00:00Z", + }) + if err != nil { + t.Fatalf("seed repository: %v", err) + } + threadIDs := make([]int64, 0, 4) + threads := make([]crawlstore.Thread, 0, 4) + for number := 1; number <= 4; number++ { + thread := crawlstore.Thread{ + RepoID: repoID, + GitHubID: fmt.Sprintf("thread-%d", number), + Number: number, + Kind: "issue", + State: "open", + Title: "Revision coverage", + HTMLURL: fmt.Sprintf("https://github.com/openclaw/gitcrawl/issues/%d", number), + LabelsJSON: "[]", + AssigneesJSON: "[]", + ContentHash: fmt.Sprintf("thread-%d", number), + UpdatedAtGitHub: "2026-07-12T10:00:00Z", + UpdatedAt: "2026-07-12T10:00:00Z", + } + threadID, err := st.UpsertThread(ctx, thread) + if err != nil { + t.Fatalf("seed thread %d: %v", number, err) + } + thread.ID = threadID + threadIDs = append(threadIDs, threadID) + threads = append(threads, thread) + } + if _, err := st.UpsertThreadRevisionAndFingerprint( + ctx, + crawlstore.ThreadEvidence{Thread: threads[1]}, + "2026-07-12T10:01:00Z", + ); err != nil { + t.Fatalf("seed fresh selected revision: %v", err) + } + if _, err := st.DB().ExecContext(ctx, ` + insert into thread_revisions( + thread_id, source_updated_at, content_hash, title_hash, body_hash, + labels_hash, observation_sequence, created_at + ) values + (?, '2026-07-12T09:00:00Z', 'later-stale', 'later-stale', 'later-stale', 'later-stale', 99, '2026-07-12T10:02:00Z'), + (?, '2026-07-12T09:00:00Z', 'stale', 'stale', 'stale', 'stale', 1, '2026-07-12T10:03:00Z') + `, threadIDs[1], threadIDs[2]); err != nil { + t.Fatalf("seed revisions: %v", err) + } + coverage, err := st.ArchiveCoverage(ctx, crawlstore.ArchiveCoverageOptions{}) + if err != nil { + t.Fatalf("archive coverage: %v", err) + } + revisions := coverage.Totals.Enrichment.Revisions + if revisions.Eligible != 4 || revisions.Covered != 2 || revisions.Fresh != 1 { + t.Fatalf("revision hydration = %#v, want eligible=4 covered=2 fresh=1", revisions) + } + datasets, err := loadGitcrawlCloudDatasets(ctx, st.DB(), true, coverage.Totals.Enrichment) + if err != nil { + t.Fatalf("load datasets: %v", err) + } + var revisionDataset gitcrawlCloudDataset + for _, dataset := range datasets { + if dataset.Name == "thread_revisions" { + revisionDataset = dataset + break + } + } + if revisionDataset.RowCount != 3 || + revisionDataset.EligibleCount != 4 || + revisionDataset.CoveredCount != 1 || + revisionDataset.Complete { + t.Fatalf("revision dataset coverage = %#v", revisionDataset) + } +} + +func TestLatestRFC3339QueryValueRejectsInvalidTimestamp(t *testing.T) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + defer db.Close() + if _, err := db.Exec(`create table timestamps(value text); insert into timestamps values('not-a-time')`); err != nil { + t.Fatalf("seed timestamp: %v", err) + } + + if _, err := latestRFC3339QueryValue( + context.Background(), + db, + `select value from timestamps`, + ); err == nil { + t.Fatal("invalid RFC3339 timestamp was accepted") + } +} diff --git a/internal/cli/remote_commands_test.go b/internal/cli/remote_commands_test.go index 14dfff8..9d5bfbe 100644 --- a/internal/cli/remote_commands_test.go +++ b/internal/cli/remote_commands_test.go @@ -3,6 +3,7 @@ package cli import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" @@ -160,9 +161,69 @@ func TestSendIngestRowsBatchesAndFinalizes(t *testing.T) { } } +func TestSendSnapshotIngestRowsRotatesMutationTokens(t *testing.T) { + ctx := context.Background() + var requests []crawlremote.IngestRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body crawlremote.IngestRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + requests = append(requests, body) + token := fmt.Sprintf("generation-%d", len(requests)) + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + SnapshotID: body.Manifest.SnapshotID, + MutationToken: token, + RowsAccepted: int64(len(body.Rows)), + Complete: body.Final, + }) + })) + defer server.Close() + + client, err := crawlremote.NewClientFromConfig(crawlremote.Config{Endpoint: server.URL}, crawlremote.Options{ + TokenProvider: crawlremote.StaticToken("publish-token"), + }) + if err != nil { + t.Fatalf("client: %v", err) + } + rows := make([][]any, gitcrawlCloudBatchSize+1) + for i := range rows { + rows[i] = []any{i} + } + progress, err := sendSnapshotIngestRows( + ctx, + client, + "gitcrawl", + "gitcrawl/openclaw", + crawlremote.IngestManifest{App: "gitcrawl", SnapshotID: strings.Repeat("a", 64)}, + "threads", + []string{"id"}, + rows, + "previous-dataset", + true, + ) + if err != nil { + t.Fatalf("send snapshot ingest: %v", err) + } + if progress.RowsAccepted != int64(len(rows)) || progress.MutationToken != "generation-2" { + t.Fatalf("progress = %#v", progress) + } + if len(requests) != 2 { + t.Fatalf("requests = %#v", requests) + } + if requests[0].Cursor != "" || requests[0].MutationToken != "previous-dataset" { + t.Fatalf("first request = %#v", requests[0]) + } + if requests[1].Cursor != "250" || requests[1].MutationToken != "generation-1" || !requests[1].Final { + t.Fatalf("second request = %#v", requests[1]) + } +} + func TestSendIngestRowsDrainsRemoteResetBeforeRetry(t *testing.T) { ctx := context.Background() var requests []crawlremote.IngestRequest + var drainRequests []crawlremote.IngestRequest resetCalls := 0 rejectedFirstBatch := false server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -177,8 +238,13 @@ func TestSendIngestRowsDrainsRemoteResetBeforeRetry(t *testing.T) { } w.Header().Set("content-type", "application/json") if len(body.Rows) == 0 { + drainRequests = append(drainRequests, body) resetCalls++ - _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ResetIncomplete: resetCalls == 1, ResetDeleted: 10000}) + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + MutationToken: body.MutationToken, + ResetIncomplete: resetCalls == 1, + ResetDeleted: 10000, + }) return } if body.Cursor == "" && !rejectedFirstBatch { @@ -191,7 +257,11 @@ func TestSendIngestRowsDrainsRemoteResetBeforeRetry(t *testing.T) { return } requests = append(requests, body) - _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{RowsAccepted: int64(len(body.Rows)), Complete: body.Final}) + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + MutationToken: body.MutationToken, + RowsAccepted: int64(len(body.Rows)), + Complete: body.Final, + }) })) defer server.Close() @@ -201,12 +271,23 @@ func TestSendIngestRowsDrainsRemoteResetBeforeRetry(t *testing.T) { if err != nil { t.Fatalf("client: %v", err) } - accepted, err := sendIngestRows(ctx, client, "gitcrawl", "gitcrawl/openclaw", crawlremote.IngestManifest{App: "gitcrawl"}, "threads", []string{"id"}, [][]any{{1}}, true) + progress, err := sendSnapshotIngestRows( + ctx, + client, + "gitcrawl", + "gitcrawl/openclaw", + crawlremote.IngestManifest{App: "gitcrawl"}, + "threads", + []string{"id"}, + [][]any{{1}}, + "generation-current", + true, + ) if err != nil { t.Fatalf("send ingest: %v", err) } - if accepted != 1 { - t.Fatalf("accepted = %d", accepted) + if progress.RowsAccepted != 1 || progress.MutationToken != "generation-current" { + t.Fatalf("progress = %#v", progress) } if resetCalls != 2 { t.Fatalf("resetCalls = %d", resetCalls) @@ -214,6 +295,11 @@ func TestSendIngestRowsDrainsRemoteResetBeforeRetry(t *testing.T) { if len(requests) != 1 || requests[0].Cursor != "" || !requests[0].Final { t.Fatalf("data requests = %#v", requests) } + for index, request := range drainRequests { + if request.MutationToken != "generation-current" { + t.Fatalf("drain request %d mutation token = %q", index, request.MutationToken) + } + } } func TestRemoteAndCloudCommandDispatchErrors(t *testing.T) { diff --git a/internal/store/archive_coverage.go b/internal/store/archive_coverage.go index 3688e6b..b409ced 100644 --- a/internal/store/archive_coverage.go +++ b/internal/store/archive_coverage.go @@ -34,6 +34,7 @@ type EnrichmentCoverage struct { Summaries EnrichmentCoverageMetric `json:"summaries"` Clusters EnrichmentCoverageMetric `json:"clusters"` PRDetails EnrichmentCoverageMetric `json:"pr_details"` + PRFiles EnrichmentCoverageMetric `json:"pr_files"` } type ArchiveCoverageRow struct { @@ -291,6 +292,10 @@ func (s *Store) archiveEnrichmentCoverage(ctx context.Context, repoID int64) (En if err != nil { return EnrichmentCoverage{}, err } + coverage.PRFiles, err = s.archivePRFileCoverage(ctx, repoID) + if err != nil { + return EnrichmentCoverage{}, err + } syncObservedAt, ok, err := s.archiveLatestSuccessfulHydrationRunAt(ctx, repoID) if err != nil { return EnrichmentCoverage{}, err @@ -533,24 +538,8 @@ func (s *Store) archivePRDetailCoverage(ctx context.Context, repoID int64) (Enri if !s.archiveCoverageHasColumns(ctx, "pull_request_details", "thread_id", "fetched_at") { return EnrichmentCoverageMetric{}, nil } - acceptedSourceUpdatedAt := archiveThreadUpdatedAtExpression(s, ctx, "t") - acceptedObservationSequence := "0" - if s.hasColumn(ctx, "threads", "observation_sequence") { - acceptedObservationSequence = "t.observation_sequence" - } - if s.hasColumn(ctx, "threads", "evidence_observation_sequence") && - s.hasColumn(ctx, "threads", "evidence_source_updated_at") { - acceptedSourceUpdatedAt = `case - when t.evidence_observation_sequence > 0 - then coalesce(t.evidence_source_updated_at, '') - else ` + acceptedSourceUpdatedAt + ` - end` - acceptedObservationSequence = `case - when t.evidence_observation_sequence > 0 - then t.evidence_observation_sequence - else ` + acceptedObservationSequence + ` - end` - } + acceptedSourceUpdatedAt, acceptedObservationSequence := + s.archiveAcceptedThreadObservationExpressions(ctx, "t") reservationJoin := "" reservationPresent := "0" reservationSourceUpdatedAt := "''" @@ -642,6 +631,170 @@ func (s *Store) archivePRDetailCoverage(ctx context.Context, repoID int64) (Enri return metric, nil } +func (s *Store) archivePRFileCoverage( + ctx context.Context, + repoID int64, +) (EnrichmentCoverageMetric, error) { + hasPersistedProof := s.archiveCoverageHasColumns( + ctx, + "pull_request_details", + "thread_id", + "changed_files", + "fetched_at", + ) && s.archiveCoverageHasColumns( + ctx, + "pull_request_files", + "thread_id", + "fetched_at", + ) + if !hasPersistedProof { + return EnrichmentCoverageMetric{}, nil + } + hasReservations := s.archiveCoverageHasColumns( + ctx, + "thread_child_observation_reservations", + "thread_id", + "family", + "source_updated_at", + "observation_sequence", + ) + acceptedSourceUpdatedAt, acceptedObservationSequence := + s.archiveAcceptedThreadObservationExpressions(ctx, "t") + reservationJoin := "" + reservationPresent := "0" + reservationSourceUpdatedAt := "''" + reservationObservationSequence := "0" + if hasReservations { + reservationJoin = ` + left join thread_child_observation_reservations reservation + on reservation.thread_id = t.id + and reservation.family = 'pull_request_files' + ` + reservationPresent = "case when reservation.thread_id is null then 0 else 1 end" + reservationSourceUpdatedAt = "coalesce(reservation.source_updated_at, '')" + reservationObservationSequence = "coalesce(reservation.observation_sequence, 0)" + } + persistedEvidenceJoin := ` + left join pull_request_details prd on prd.thread_id = t.id + left join ( + select thread_id, + count(*) as file_count, + min(fetched_at) as oldest_fetched_at, + max(fetched_at) as latest_fetched_at + from pull_request_files + group by thread_id + ) prf on prf.thread_id = t.id + ` + rows, err := s.q().QueryContext(ctx, ` + select `+reservationPresent+`, + `+reservationSourceUpdatedAt+`, + `+reservationObservationSequence+`, + case when prd.thread_id is null then 0 else 1 end, + coalesce(prd.changed_files, 0), + coalesce(prd.fetched_at, ''), + coalesce(prf.file_count, 0), + coalesce(prf.oldest_fetched_at, ''), + coalesce(prf.latest_fetched_at, ''), + `+acceptedSourceUpdatedAt+`, + `+acceptedObservationSequence+` + from threads t + `+reservationJoin+` + `+persistedEvidenceJoin+` + where t.repo_id = ? and t.kind = 'pull_request' + `, repoID) + if err != nil { + return EnrichmentCoverageMetric{}, fmt.Errorf("archive PR file coverage: %w", err) + } + defer rows.Close() + + metric := EnrichmentCoverageMetric{Supported: true} + var latestObservedAt time.Time + for rows.Next() { + var hasReservation, hasDetail, changedFiles, files int + var reservationSequence, acceptedSequence int64 + var reservationSourceUpdatedAt, detailFetchedAt, oldestFileFetchedAt string + var latestFileFetchedAt, acceptedSource string + if err := rows.Scan( + &hasReservation, + &reservationSourceUpdatedAt, + &reservationSequence, + &hasDetail, + &changedFiles, + &detailFetchedAt, + &files, + &oldestFileFetchedAt, + &latestFileFetchedAt, + &acceptedSource, + &acceptedSequence, + ); err != nil { + return EnrichmentCoverageMetric{}, fmt.Errorf("scan archive PR file coverage: %w", err) + } + metric.Eligible++ + if hasDetail == 0 || changedFiles < 0 || files != changedFiles { + continue + } + metric.Covered++ + latestObservedAt = laterArchiveCoverageTimestamp(latestObservedAt, detailFetchedAt) + latestObservedAt = laterArchiveCoverageTimestamp(latestObservedAt, latestFileFetchedAt) + detailFresh := archiveCoverageTimestampAtOrAfter(detailFetchedAt, acceptedSource) + filesFresh := files == 0 || + archiveCoverageTimestampAtOrAfter(oldestFileFetchedAt, acceptedSource) + reservationFresh := true + if hasReservation != 0 { + reservationFresh = archiveObservationAtOrAfter( + reservationSourceUpdatedAt, + reservationSequence, + acceptedSource, + observationSequenceOrderValue(acceptedSequence), + ) + } + if detailFresh && filesFresh && reservationFresh { + metric.Fresh++ + } + } + if err := rows.Err(); err != nil { + return EnrichmentCoverageMetric{}, fmt.Errorf("iterate archive PR file coverage: %w", err) + } + if !latestObservedAt.IsZero() { + metric.LatestAt = formatArchiveCoverageTimestamp(latestObservedAt) + } + finalizeEnrichmentCoverageMetric(&metric) + return metric, nil +} + +func laterArchiveCoverageTimestamp(latest time.Time, candidate string) time.Time { + parsed, ok := parseArchiveCoverageTimestamp(candidate) + if ok && (latest.IsZero() || parsed.After(latest)) { + return parsed + } + return latest +} + +func (s *Store) archiveAcceptedThreadObservationExpressions( + ctx context.Context, + alias string, +) (sourceUpdatedAt string, observationSequence string) { + sourceUpdatedAt = archiveThreadUpdatedAtExpression(s, ctx, alias) + observationSequence = "0" + if s.hasColumn(ctx, "threads", "observation_sequence") { + observationSequence = alias + ".observation_sequence" + } + if s.hasColumn(ctx, "threads", "evidence_observation_sequence") && + s.hasColumn(ctx, "threads", "evidence_source_updated_at") { + sourceUpdatedAt = `case + when ` + alias + `.evidence_observation_sequence > 0 + then coalesce(` + alias + `.evidence_source_updated_at, '') + else ` + sourceUpdatedAt + ` + end` + observationSequence = `case + when ` + alias + `.evidence_observation_sequence > 0 + then ` + alias + `.evidence_observation_sequence + else ` + observationSequence + ` + end` + } + return sourceUpdatedAt, observationSequence +} + func archiveObservationAtOrAfter( observedSourceUpdatedAt string, observedSequence int64, @@ -743,6 +896,7 @@ func addEnrichmentCoverage(total *EnrichmentCoverage, row EnrichmentCoverage) { addEnrichmentCoverageMetric(&total.Summaries, row.Summaries) addEnrichmentCoverageMetric(&total.Clusters, row.Clusters) addEnrichmentCoverageMetric(&total.PRDetails, row.PRDetails) + addEnrichmentCoverageMetric(&total.PRFiles, row.PRFiles) } func addEnrichmentCoverageMetric(total *EnrichmentCoverageMetric, row EnrichmentCoverageMetric) { @@ -763,4 +917,5 @@ func finalizeEnrichmentCoverage(coverage *EnrichmentCoverage) { finalizeEnrichmentCoverageMetric(&coverage.Summaries) finalizeEnrichmentCoverageMetric(&coverage.Clusters) finalizeEnrichmentCoverageMetric(&coverage.PRDetails) + finalizeEnrichmentCoverageMetric(&coverage.PRFiles) } diff --git a/internal/store/archive_coverage_test.go b/internal/store/archive_coverage_test.go index a1bbbaf..9f3864c 100644 --- a/internal/store/archive_coverage_test.go +++ b/internal/store/archive_coverage_test.go @@ -218,6 +218,218 @@ func TestArchiveCoveragePRDetailFreshnessUsesAcceptedSourceObservation(t *testin } } +func TestArchiveCoveragePRFilesRequireCurrentObservation(t *testing.T) { + ctx := context.Background() + st, err := Open(ctx, filepath.Join(t.TempDir(), "gitcrawl.db")) + if err != nil { + t.Fatalf("open store: %v", err) + } + defer st.Close() + + repoID, err := st.UpsertRepository(ctx, Repository{ + Owner: "openclaw", Name: "gitcrawl", FullName: "openclaw/gitcrawl", + RawJSON: "{}", UpdatedAt: "2026-07-12T12:00:00Z", + }) + if err != nil { + t.Fatalf("repository: %v", err) + } + thread := archiveCoverageThread(repoID, 1, "pull_request") + thread.UpdatedAtGitHub = "2026-07-12T12:00:00Z" + thread.RawJSON = `{"version":1}` + thread.ContentHash = "version-1" + initial, err := st.UpsertThreadObservation(ctx, thread, UpsertThreadOptions{ + ObservationSequence: 1, + }) + if err != nil { + t.Fatalf("initial thread: %v", err) + } + thread.ID = initial.ID + + assertMetric := func(wantCovered, wantFresh, wantMissing, wantStale int, wantComplete bool) { + t.Helper() + coverage, err := st.ArchiveCoverage(ctx, ArchiveCoverageOptions{}) + if err != nil { + t.Fatalf("archive coverage: %v", err) + } + if len(coverage.Rows) != 1 { + t.Fatalf("coverage rows = %d, want 1", len(coverage.Rows)) + } + metric := coverage.Rows[0].Enrichment.PRFiles + if !metric.Supported || metric.Eligible != 1 || + metric.Covered != wantCovered || metric.Fresh != wantFresh || + metric.Missing != wantMissing || metric.Stale != wantStale || + metric.Complete != wantComplete { + t.Fatalf("PR file coverage = %+v", metric) + } + } + + assertMetric(0, 0, 1, 0, false) + if applied, err := st.ReserveThreadChildObservation( + ctx, + thread.ID, + ThreadChildPullRequestFiles, + thread.UpdatedAtGitHub, + 1, + ); err != nil || !applied { + t.Fatalf("reserve PR file observation = %t, %v", applied, err) + } + assertMetric(0, 0, 1, 0, false) + if err := st.UpsertPullRequestCacheFamilies(ctx, PullRequestDetail{ + ThreadID: thread.ID, + RepoID: repoID, + Number: thread.Number, + ChangedFiles: 1, + RawJSON: "{}", + FetchedAt: "2026-07-12T12:01:00Z", + UpdatedAt: "2026-07-12T12:01:00Z", + }, []PullRequestFile{{ + ThreadID: thread.ID, + Position: 0, + Path: "README.md", + RawJSON: "{}", + FetchedAt: "2026-07-12T12:01:00Z", + }}, nil, nil, nil, PullRequestHydrationFamilies{ + Details: true, + Files: true, + }); err != nil { + t.Fatalf("persist PR file evidence: %v", err) + } + assertMetric(1, 1, 0, 0, true) + + thread.UpdatedAtGitHub = "2026-07-12T12:03:00Z" + thread.RawJSON = `{"version":2}` + thread.ContentHash = "version-2" + if accepted, err := st.UpsertThreadObservation(ctx, thread, UpsertThreadOptions{ + ObservationSequence: 2, + }); err != nil || !accepted.EvidenceApplied { + t.Fatalf("newer accepted evidence = %+v, %v", accepted, err) + } + assertMetric(1, 0, 0, 1, false) + + if applied, err := st.ReserveThreadChildObservation( + ctx, + thread.ID, + ThreadChildPullRequestFiles, + thread.UpdatedAtGitHub, + 2, + ); err != nil || !applied { + t.Fatalf("reserve current PR file observation = %t, %v", applied, err) + } + assertMetric(1, 0, 0, 1, false) + if err := st.UpsertPullRequestCacheFamilies(ctx, PullRequestDetail{ + ThreadID: thread.ID, + RepoID: repoID, + Number: thread.Number, + ChangedFiles: 1, + RawJSON: "{}", + FetchedAt: "2026-07-12T12:04:00Z", + UpdatedAt: "2026-07-12T12:04:00Z", + }, []PullRequestFile{{ + ThreadID: thread.ID, + Position: 0, + Path: "README.md", + RawJSON: "{}", + FetchedAt: "2026-07-12T12:04:00Z", + }}, nil, nil, nil, PullRequestHydrationFamilies{ + Details: true, + Files: true, + }); err != nil { + t.Fatalf("persist ordered PR file evidence: %v", err) + } + assertMetric(1, 1, 0, 0, true) +} + +func TestArchiveCoveragePRFilesUsesLegacySnapshotProof(t *testing.T) { + ctx := context.Background() + st, err := Open(ctx, filepath.Join(t.TempDir(), "gitcrawl.db")) + if err != nil { + t.Fatalf("open store: %v", err) + } + defer st.Close() + + repoID, err := st.UpsertRepository(ctx, Repository{ + Owner: "openclaw", Name: "gitcrawl", FullName: "openclaw/gitcrawl", + RawJSON: "{}", UpdatedAt: "2026-07-12T12:00:00Z", + }) + if err != nil { + t.Fatalf("repository: %v", err) + } + thread := archiveCoverageThread(repoID, 1, "pull_request") + thread.UpdatedAtGitHub = "2026-07-12T12:00:00Z" + thread.RawJSON = `{"version":1}` + thread.ContentHash = "version-1" + initial, err := st.UpsertThreadObservation(ctx, thread, UpsertThreadOptions{ + ObservationSequence: 1, + }) + if err != nil { + t.Fatalf("initial thread: %v", err) + } + thread.ID = initial.ID + if _, err := st.DB().ExecContext(ctx, `drop table thread_child_observation_reservations`); err != nil { + t.Fatalf("drop reservation table: %v", err) + } + + upsertFiles := func(changedFiles int, files []PullRequestFile) { + t.Helper() + if err := st.UpsertPullRequestCacheFamilies(ctx, PullRequestDetail{ + ThreadID: thread.ID, + RepoID: repoID, + Number: thread.Number, + ChangedFiles: changedFiles, + RawJSON: "{}", + FetchedAt: "2026-07-12T12:05:00Z", + UpdatedAt: "2026-07-12T12:05:00Z", + }, files, nil, nil, nil, PullRequestHydrationFamilies{ + Details: true, + Files: true, + }); err != nil { + t.Fatalf("PR cache: %v", err) + } + } + assertMetric := func(wantCovered, wantFresh int, wantComplete bool) { + t.Helper() + coverage, err := st.ArchiveCoverage(ctx, ArchiveCoverageOptions{}) + if err != nil { + t.Fatalf("archive coverage: %v", err) + } + metric := coverage.Rows[0].Enrichment.PRFiles + if !metric.Supported || metric.Eligible != 1 || + metric.Covered != wantCovered || metric.Fresh != wantFresh || + metric.Complete != wantComplete { + t.Fatalf("legacy PR file coverage = %+v", metric) + } + } + + upsertFiles(0, nil) + assertMetric(1, 1, true) + + upsertFiles(2, []PullRequestFile{{ + ThreadID: thread.ID, + Path: "README.md", + RawJSON: "{}", + FetchedAt: "2026-07-12T12:05:00Z", + }}) + assertMetric(0, 0, false) + + upsertFiles(1, []PullRequestFile{{ + ThreadID: thread.ID, + Path: "README.md", + RawJSON: "{}", + FetchedAt: "2026-07-12T12:05:00Z", + }}) + assertMetric(1, 1, true) + + thread.UpdatedAtGitHub = "2026-07-12T12:06:00Z" + thread.RawJSON = `{"version":2}` + thread.ContentHash = "version-2" + if accepted, err := st.UpsertThreadObservation(ctx, thread, UpsertThreadOptions{ + ObservationSequence: 2, + }); err != nil || !accepted.EvidenceApplied { + t.Fatalf("newer accepted evidence = %+v, %v", accepted, err) + } + assertMetric(1, 0, false) +} + func TestArchiveObservationFreshnessUsesSequenceForEqualSource(t *testing.T) { const source = "2026-07-12T12:00:00Z" if archiveObservationAtOrAfter(source, 10, source, 11) {