From 8867f2d594688b6574126078d396e4aaa6dbc009 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:06:03 +0200 Subject: [PATCH 01/52] build: consume snapshot-aware crawlkit --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4d4b7f4..a32d51f 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 github.com/charmbracelet/x/ansi v0.11.7 github.com/mattn/go-isatty v0.0.22 - github.com/openclaw/crawlkit v0.13.4 + github.com/openclaw/crawlkit v0.13.5-0.20260712090520-2a5e11612f81 github.com/zalando/go-keyring v0.2.8 golang.org/x/sys v0.46.0 modernc.org/sqlite v1.53.0 diff --git a/go.sum b/go.sum index 9b1a907..0f8d8d9 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/openclaw/crawlkit v0.13.4 h1:6KZaIzs5BQxBjCeOwg1u2XPLNbsOkrRPQGPtTkxZaYE= -github.com/openclaw/crawlkit v0.13.4/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= +github.com/openclaw/crawlkit v0.13.5-0.20260712090520-2a5e11612f81 h1:npL85kzFx39d07yJx1ekh2MIFHb/2XXfIXTdwEKA1tc= +github.com/openclaw/crawlkit v0.13.5-0.20260712090520-2a5e11612f81/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= From fefc5081a78161f749e3e532257ff600917b950a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:13:16 +0200 Subject: [PATCH 02/52] feat(cloud): publish content-addressed snapshots --- internal/cli/app.go | 2 +- internal/cli/cloud_commands.go | 352 +++++++++++++++++++++++++++---- internal/cli/cloud_snapshot.go | 366 +++++++++++++++++++++++++++++++++ 3 files changed, 674 insertions(+), 46 deletions(-) create mode 100644 internal/cli/cloud_snapshot.go diff --git a/internal/cli/app.go b/internal/cli/app.go index 05b8fa8..5035e77 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -4770,7 +4770,7 @@ Usage: "cloud": `gitcrawl cloud manages Worker-backed remote archives. Usage: - gitcrawl cloud publish --remote URL --archive id [--json] + gitcrawl cloud publish --remote URL --archive id [--allow-incomplete] [--observation-order] [--cutover=false] [--json] `, "whoami": `gitcrawl whoami prints the configured remote archive identity. diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index bb796c9..e5a9fca 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -41,6 +41,9 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { remoteEndpoint := fs.String("remote", "", "remote archive endpoint") archive := fs.String("archive", "", "remote archive id") tokenEnv := fs.String("token-env", "", "remote token environment variable") + allowIncomplete := fs.Bool("allow-incomplete", false, "publish even when local enrichment coverage is incomplete") + observationOrder := fs.Bool("observation-order", false, "publish durable observation ordering when the remote fence is enabled") + cutover := fs.Bool("cutover", true, "cut over unpinned reads after activating the snapshot") jsonOut := fs.Bool("json", false, "write JSON output") if err := fs.Parse(normalizeCommandArgs(args, map[string]bool{"remote": true, "archive": true, "token-env": true})); err != nil { return usageErr(err) @@ -82,48 +85,111 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { if err != nil { return err } - manifest := crawlremote.IngestManifest{ - App: "gitcrawl", - Archive: archiveID, - SchemaName: "gitcrawl-cloud-v1", - SchemaVersion: 1, - SchemaHash: "gitcrawl-cloud-v1", - Mode: crawlremote.ModePublisher, - Source: "sqlite", - } - repoRows, err := publishRows(ctx, rt.Store.DB(), gitcrawlRepositoryExportSQL, func(values []any) []any { return values }) + snapshotPath, cleanupSnapshot, err := cloudSQLiteSnapshotPath(ctx, rt.Store.DB(), rt.Store.Path()) if err != nil { return err } - threadSQL, err := gitcrawlThreadExportSQL(ctx, rt.Store.DB()) + defer cleanupSnapshot() + snapshotDB, err := sql.Open("sqlite", snapshotPath) if err != nil { - return err - } - threadRows, err := publishRows(ctx, rt.Store.DB(), threadSQL, func(values []any) []any { return values }) + return fmt.Errorf("open frozen cloud snapshot: %w", err) + } + defer snapshotDB.Close() + snapshot, err := buildGitcrawlCloudSnapshot( + ctx, + snapshotDB, + snapshotPath, + *allowIncomplete, + *observationOrder, + ) if err != nil { return err } - repoAccepted, err := sendIngestRows(ctx, client, "gitcrawl", archiveID, manifest, "repositories", gitcrawlRepositoryColumns, repoRows, false) + manifest := gitcrawlCloudManifest(archiveID, snapshot) + counts := gitcrawlCloudDatasetCounts(snapshot) + sqliteBundle, err := uploadSQLiteSnapshotArchive( + ctx, + client, + "gitcrawl", + archiveID, + snapshotPath, + counts, + ) if err != nil { return err } - threadAccepted, err := sendIngestRows(ctx, client, "gitcrawl", archiveID, manifest, "threads", gitcrawlThreadColumns, threadRows, true) - if err != nil { - return err + + alreadyActive := false + status, statusErr := client.Status(ctx, "gitcrawl", archiveID) + if statusErr == nil { + alreadyActive = status.ActiveSnapshotID == snapshot.ID && status.CoverageComplete + } else if !remoteNotFound(statusErr) { + return statusErr + } + var mutationToken string + if !alreadyActive { + for _, dataset := range snapshot.Datasets { + progress, err := sendSnapshotIngestRows( + ctx, + client, + "gitcrawl", + archiveID, + manifest, + dataset.Name, + dataset.Columns, + dataset.Rows, + mutationToken, + false, + ) + if err != nil { + return fmt.Errorf("publish cloud dataset %s: %w", dataset.Name, err) + } + counts[dataset.Name] = progress.RowsAccepted + mutationToken = progress.MutationToken + } + coverageRows := gitcrawlCloudCoverageRows(snapshot, mutationToken) + progress, err := sendSnapshotIngestRows( + ctx, + client, + "gitcrawl", + archiveID, + manifest, + "dataset_coverage", + []string{ + "dataset", "row_count", "eligible_count", "covered_count", + "max_source_at", "dataset_generated_at", "complete", "mutation_token", + }, + coverageRows, + mutationToken, + true, + ) + if err != nil { + return fmt.Errorf("activate cloud snapshot: %w", err) + } + mutationToken = progress.MutationToken } - sqliteBundle, err := uploadSQLiteArchive(ctx, client, "gitcrawl", archiveID, rt.Store.DB(), rt.Store.Path(), manifest, map[string]int64{ - "repositories": repoAccepted, - "threads": threadAccepted, - }) - if err != nil { - return err + var cutoverResult *crawlremote.CutoverResult + if *cutover { + result, err := client.Cutover(ctx, "gitcrawl", archiveID, snapshot.ID) + if err != nil { + return fmt.Errorf("cut over cloud snapshot: %w", err) + } + cutoverResult = &result } sqliteBundlePrivacy := gitcrawlCloudSQLiteBundlePrivacy() return a.writeOutput("cloud publish", map[string]any{ "remote": strings.TrimRight(endpoint, "/"), "archive": archiveID, - "repositories": repoAccepted, - "threads": threadAccepted, + "snapshot_id": snapshot.ID, + "source_sha256": snapshot.ID, + "source_sync_at": snapshot.SourceSyncAt, + "dataset_generated_at": snapshot.DatasetGeneratedAt, + "capabilities": snapshot.Capabilities, + "datasets": counts, + "hydration": snapshot.Hydration, + "already_active": alreadyActive, + "mutation_token": mutationToken, + "cutover": cutoverResult, "sqlite_bundle": sqliteBundle, "sqlite_bundle_privacy": sqliteBundlePrivacy, }, true) @@ -159,35 +225,112 @@ func publishRows(ctx context.Context, db *sql.DB, query string, mapRow func([]an return out, rows.Err() } -func sendIngestRows(ctx context.Context, client *crawlremote.Client, app, archive string, manifest crawlremote.IngestManifest, table string, columns []string, rows [][]any, final bool) (int64, error) { +type ingestProgress struct { + RowsAccepted int64 + MutationToken string +} + +func sendIngestRows( + ctx context.Context, + client *crawlremote.Client, + app, archive string, + manifest crawlremote.IngestManifest, + table string, + columns []string, + rows [][]any, + final bool, +) (int64, error) { + progress, err := sendSnapshotIngestRows( + ctx, + client, + app, + archive, + manifest, + table, + columns, + rows, + "", + final, + ) + return progress.RowsAccepted, err +} + +func sendSnapshotIngestRows( + ctx context.Context, + client *crawlremote.Client, + app, archive string, + manifest crawlremote.IngestManifest, + table string, + columns []string, + rows [][]any, + mutationToken string, + final bool, +) (ingestProgress, error) { var total int64 if len(rows) == 0 { - result, err := sendIngestBatch(ctx, client, app, archive, manifest, table, columns, [][]any{}, 0, final) - return result.RowsAccepted, err + result, err := sendIngestBatch( + ctx, + client, + app, + archive, + manifest, + table, + columns, + [][]any{}, + 0, + mutationToken, + final, + ) + return ingestProgress{RowsAccepted: result.RowsAccepted, MutationToken: result.MutationToken}, err } for start := 0; start < len(rows); start += gitcrawlCloudBatchSize { end := start + gitcrawlCloudBatchSize if end > len(rows) { end = len(rows) } - result, err := sendIngestBatch(ctx, client, app, archive, manifest, table, columns, rows[start:end], start, final && end == len(rows)) + result, err := sendIngestBatch( + ctx, + client, + app, + archive, + manifest, + table, + columns, + rows[start:end], + start, + mutationToken, + final && end == len(rows), + ) if err != nil { - return total, err + return ingestProgress{RowsAccepted: total, MutationToken: mutationToken}, err } total += result.RowsAccepted + mutationToken = result.MutationToken } - return total, nil + return ingestProgress{RowsAccepted: total, MutationToken: mutationToken}, nil } -func sendIngestBatch(ctx context.Context, client *crawlremote.Client, app, archive string, manifest crawlremote.IngestManifest, table string, columns []string, rows [][]any, cursor int, final bool) (crawlremote.IngestResult, error) { +func sendIngestBatch( + ctx context.Context, + client *crawlremote.Client, + app, archive string, + manifest crawlremote.IngestManifest, + table string, + columns []string, + rows [][]any, + cursor int, + mutationToken string, + final bool, +) (crawlremote.IngestResult, error) { for { result, err := client.Ingest(ctx, app, archive, crawlremote.IngestRequest{ - Manifest: manifest, - Table: table, - Columns: columns, - Rows: rows, - Cursor: cursorFor(cursor), - Final: final, + Manifest: manifest, + Table: table, + Columns: columns, + Rows: rows, + Cursor: cursorFor(cursor), + MutationToken: mutationToken, + Final: final, }) if err == nil { if result.ResetIncomplete { @@ -242,6 +385,15 @@ func uploadSQLiteArchive(ctx context.Context, client *crawlremote.Client, app, a return nil, err } defer cleanup() + return uploadSQLiteSnapshotArchive(ctx, client, app, archive, snapshotPath, counts) +} + +func uploadSQLiteSnapshotArchive( + ctx context.Context, + client *crawlremote.Client, + app, archive, snapshotPath string, + counts map[string]int64, +) (*crawlremote.SQLiteBundle, error) { bundle, err := crawlremote.BuildGzipSQLiteBundle(ctx, crawlremote.SQLiteBundleBuildOptions{ App: app, Archive: archive, @@ -261,6 +413,11 @@ func uploadSQLiteArchive(ctx context.Context, client *crawlremote.Client, app, a return result.Bundle, nil } +func remoteNotFound(err error) bool { + var remoteErr *crawlremote.Error + return errors.As(err, &remoteErr) && remoteErr.Status == http.StatusNotFound +} + func gitcrawlCloudSQLiteBundlePrivacy() map[string]any { return map[string]any{ "includes_private_messages": true, @@ -298,12 +455,10 @@ func cloudSQLiteSnapshotPath(ctx context.Context, db *sql.DB, dbPath string) (st cleanup() return "", func() {}, fmt.Errorf("open cloud SQLite snapshot: %w", err) } - for _, table := range []string{"code_documents_fts", "code_documents", "code_snapshots"} { - if _, err := snapshotDB.ExecContext(ctx, `drop table if exists `+table); err != nil { - _ = snapshotDB.Close() - cleanup() - return "", func() {}, fmt.Errorf("drop local-only cloud snapshot table %s: %w", table, err) - } + if err := sanitizeCloudSQLiteSnapshot(ctx, snapshotDB); err != nil { + _ = snapshotDB.Close() + cleanup() + return "", func() {}, err } if _, err := snapshotDB.ExecContext(ctx, `vacuum`); err != nil { _ = snapshotDB.Close() @@ -317,6 +472,97 @@ func cloudSQLiteSnapshotPath(ctx context.Context, db *sql.DB, dbPath string) (st return snapshotPath, cleanup, nil } +func sanitizeCloudSQLiteSnapshot(ctx context.Context, db *sql.DB) error { + for _, column := range []struct { + table string + name string + }{ + {table: "repositories", name: "raw_json"}, + {table: "threads", name: "raw_json"}, + {table: "comments", name: "raw_json"}, + {table: "pull_request_details", name: "raw_json"}, + {table: "pull_request_files", name: "raw_json"}, + {table: "pull_request_commits", name: "raw_json"}, + {table: "pull_request_checks", name: "raw_json"}, + {table: "pull_request_review_threads", name: "raw_json"}, + {table: "github_workflow_runs", name: "raw_json"}, + {table: "sync_runs", name: "error_text"}, + {table: "summary_runs", name: "error_text"}, + {table: "embedding_runs", name: "error_text"}, + {table: "cluster_runs", name: "error_text"}, + } { + exists, err := sqliteColumnExists(ctx, db, column.table, column.name) + if err != nil { + return fmt.Errorf("inspect cloud snapshot %s.%s: %w", column.table, column.name, err) + } + if !exists { + continue + } + if _, err := db.ExecContext( + ctx, + `update `+column.table+` set `+column.name+` = '' where `+column.name+` is not null and `+column.name+` != ''`, + ); err != nil { + return fmt.Errorf("clear cloud snapshot %s.%s: %w", column.table, column.name, err) + } + } + for _, column := range []struct { + table string + name string + }{ + {table: "comments", name: "raw_json_blob_id"}, + {table: "thread_revisions", name: "raw_json_blob_id"}, + {table: "pull_request_files", name: "patch"}, + } { + exists, err := sqliteColumnExists(ctx, db, column.table, column.name) + if err != nil { + return fmt.Errorf("inspect cloud snapshot %s.%s: %w", column.table, column.name, err) + } + if !exists { + continue + } + value := "null" + if column.name == "patch" { + value = "''" + } + if _, err := db.ExecContext( + ctx, + `update `+column.table+` set `+column.name+` = `+value+` where `+column.name+` is not null`, + ); err != nil { + return fmt.Errorf("clear cloud snapshot %s.%s: %w", column.table, column.name, err) + } + } + for _, table := range []string{ + "thread_changed_files", + "thread_hunk_signatures", + "thread_code_snapshots", + "code_documents_fts", + "code_documents", + "code_snapshots", + } { + if _, err := db.ExecContext(ctx, `drop table if exists `+table); err != nil { + return fmt.Errorf("drop local-only cloud snapshot table %s: %w", table, err) + } + } + if exists, err := sqliteTableExists(ctx, db, "blobs"); err != nil { + return err + } else if exists { + if _, err := db.ExecContext(ctx, `delete from blobs`); err != nil { + return fmt.Errorf("clear cloud snapshot blobs: %w", err) + } + } + if exists, err := sqliteTableExists(ctx, db, "portable_metadata"); err != nil { + return err + } else if exists { + if _, err := db.ExecContext( + ctx, + `update portable_metadata set value = '' where key = 'source_path'`, + ); err != nil { + return fmt.Errorf("clear cloud snapshot portable source path: %w", err) + } + } + return nil +} + func sqliteSnapshotPath(ctx context.Context, db *sql.DB, dbPath string) (string, func(), error) { tmpDir, err := os.MkdirTemp("", "gitcrawl-cloud-sqlite-*") if err != nil { @@ -414,3 +660,19 @@ func sqliteColumnExists(ctx context.Context, db *sql.DB, table, column string) ( } return false, rows.Err() } + +func sqliteTableExists(ctx context.Context, db *sql.DB, table string) (bool, error) { + var name string + err := db.QueryRowContext( + ctx, + `select name from sqlite_schema where type in ('table', 'view') and name = ?`, + table, + ).Scan(&name) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("inspect cloud snapshot table %s: %w", table, err) + } + return name == table, nil +} diff --git a/internal/cli/cloud_snapshot.go b/internal/cli/cloud_snapshot.go new file mode 100644 index 0000000..68287e9 --- /dev/null +++ b/internal/cli/cloud_snapshot.go @@ -0,0 +1,366 @@ +package cli + +import ( + "context" + "database/sql" + "fmt" + "slices" + "strings" + "time" + + crawlremote "github.com/openclaw/crawlkit/remote" + crawlstore "github.com/openclaw/gitcrawl/internal/store" +) + +const ( + gitcrawlCloudSchemaName = "gitcrawl-cloud-v2" + gitcrawlCloudSchemaVersion = 8 + gitcrawlCloudSchemaHash = "gitcrawl-cloud-v2" + + gitcrawlObservationOrderCapability = "gitcrawl.observation-order.v1" +) + +type gitcrawlCloudDataset struct { + Name string + Columns []string + Rows [][]any + MaxSourceAt string +} + +type gitcrawlCloudSnapshot struct { + ID string + SourceSyncAt string + DatasetGeneratedAt string + Capabilities []string + Datasets []gitcrawlCloudDataset + Hydration crawlstore.EnrichmentCoverage +} + +func buildGitcrawlCloudSnapshot( + ctx context.Context, + db *sql.DB, + snapshotPath string, + allowIncomplete bool, + observationOrder bool, +) (gitcrawlCloudSnapshot, error) { + snapshotID, err := cloudFileSHA256(snapshotPath) + if err != nil { + return gitcrawlCloudSnapshot{}, err + } + sourceSyncAt, err := gitcrawlCloudSourceSyncAt(ctx, db) + if err != nil { + return gitcrawlCloudSnapshot{}, err + } + capabilities, err := gitcrawlCloudCapabilities(ctx, db, observationOrder) + if err != nil { + return gitcrawlCloudSnapshot{}, err + } + datasets, err := loadGitcrawlCloudDatasets(ctx, db, slices.Contains(capabilities, gitcrawlObservationOrderCapability)) + if err != nil { + return gitcrawlCloudSnapshot{}, err + } + if len(datasets) == 0 || len(datasets[0].Rows) == 0 { + return gitcrawlCloudSnapshot{}, fmt.Errorf("cloud snapshot has no repositories") + } + hydration, err := gitcrawlCloudHydration(ctx, snapshotPath) + if err != nil { + return gitcrawlCloudSnapshot{}, err + } + if missing := incompleteGitcrawlCloudHydration(hydration); len(missing) > 0 && !allowIncomplete { + return gitcrawlCloudSnapshot{}, fmt.Errorf( + "cloud snapshot enrichment is incomplete (%s); hydrate the archive or pass --allow-incomplete", + strings.Join(missing, ", "), + ) + } + return gitcrawlCloudSnapshot{ + ID: snapshotID, + SourceSyncAt: sourceSyncAt, + DatasetGeneratedAt: time.Now().UTC().Format(time.RFC3339Nano), + Capabilities: capabilities, + Datasets: datasets, + Hydration: hydration, + }, nil +} + +func gitcrawlCloudManifest(archive string, snapshot gitcrawlCloudSnapshot) crawlremote.IngestManifest { + return crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: archive, + SchemaName: gitcrawlCloudSchemaName, + SchemaVersion: gitcrawlCloudSchemaVersion, + SchemaHash: gitcrawlCloudSchemaHash, + Mode: crawlremote.ModePublisher, + Source: "sqlite", + SourceSyncAt: snapshot.SourceSyncAt, + SnapshotID: snapshot.ID, + SourceSHA256: snapshot.ID, + Capabilities: snapshot.Capabilities, + } +} + +func loadGitcrawlCloudDatasets(ctx context.Context, db *sql.DB, observationOrder bool) ([]gitcrawlCloudDataset, error) { + threadColumns := append([]string(nil), gitcrawlThreadColumns...) + threadQuery, err := gitcrawlThreadExportSQL(ctx, db) + if err != nil { + return nil, err + } + threadSelect := strings.TrimSpace(strings.TrimSuffix(threadQuery, "order by repo_id, number")) + revisionColumns := []string{ + "id", "thread_id", "source_updated_at", "content_hash", "title_hash", + "body_hash", "labels_hash", "created_at", + } + revisionSelect := ` +select id, thread_id, coalesce(source_updated_at, ''), content_hash, title_hash, + body_hash, labels_hash, created_at +from thread_revisions` + if observationOrder { + threadColumns = append(threadColumns, "observation_sequence") + threadSelect = strings.Replace( + threadSelect, + "from threads", + ", observation_sequence\nfrom threads", + 1, + ) + revisionColumns = append(revisionColumns, "observation_sequence") + revisionSelect = strings.Replace( + revisionSelect, + "from thread_revisions", + ", observation_sequence\nfrom thread_revisions", + 1, + ) + } + + specs := []struct { + name string + columns []string + query string + maxSourceAt string + }{ + { + name: "repositories", + columns: gitcrawlRepositoryColumns, + query: gitcrawlRepositoryExportSQL, + maxSourceAt: `select coalesce(max(updated_at), '') from repositories`, + }, + { + name: "threads", + columns: threadColumns, + query: threadSelect + "\norder by repo_id, number", + maxSourceAt: `select coalesce(max(coalesce(nullif(updated_at_gh, ''), updated_at)), '') from threads`, + }, + { + name: "thread_revisions", + columns: revisionColumns, + query: revisionSelect + "\norder by id", + maxSourceAt: `select coalesce(max(coalesce(nullif(source_updated_at, ''), created_at)), '') from thread_revisions`, + }, + { + name: "thread_fingerprints", + columns: []string{ + "id", "thread_revision_id", "algorithm_version", "fingerprint_hash", + "fingerprint_slug", "body_token_hash", "file_set_hash", "simhash64", "created_at", + }, + query: ` +select id, thread_revision_id, algorithm_version, fingerprint_hash, + fingerprint_slug, body_token_hash, file_set_hash, simhash64, created_at +from thread_fingerprints +order by id`, + maxSourceAt: `select coalesce(max(created_at), '') from thread_fingerprints`, + }, + { + name: "thread_key_summaries", + columns: []string{ + "id", "thread_revision_id", "summary_kind", "prompt_version", "provider", + "model", "input_hash", "output_hash", "key_text", "created_at", + }, + query: ` +select id, thread_revision_id, summary_kind, prompt_version, provider, + model, input_hash, output_hash, key_text, created_at +from thread_key_summaries +order by id`, + maxSourceAt: `select coalesce(max(created_at), '') from thread_key_summaries`, + }, + { + name: "cluster_groups", + columns: []string{ + "id", "repo_id", "stable_key", "stable_slug", "status", "cluster_type", + "representative_thread_id", "title", "member_count", "created_at", "updated_at", "closed_at", + }, + query: ` +select cluster.id, cluster.repo_id, cluster.stable_key, cluster.stable_slug, + cluster.status, coalesce(cluster.cluster_type, ''), + cluster.representative_thread_id, coalesce(cluster.title, ''), + (select count(*) from cluster_memberships membership + where membership.cluster_id = cluster.id and membership.state = 'active'), + cluster.created_at, cluster.updated_at, coalesce(cluster.closed_at, '') +from cluster_groups cluster +order by cluster.id`, + maxSourceAt: `select coalesce(max(updated_at), '') from cluster_groups`, + }, + { + name: "cluster_memberships", + columns: []string{ + "cluster_id", "thread_id", "role", "state", "score_to_representative", + "created_at", "updated_at", "removed_at", + }, + query: ` +select cluster_id, thread_id, role, state, score_to_representative, + created_at, updated_at, coalesce(removed_at, '') +from cluster_memberships +order by cluster_id, thread_id`, + maxSourceAt: `select coalesce(max(updated_at), '') from cluster_memberships`, + }, + { + name: "pull_request_details", + columns: []string{ + "thread_id", "repo_id", "number", "base_sha", "head_sha", "head_ref", + "head_repo_full_name", "mergeable_state", "additions", "deletions", + "changed_files", "fetched_at", "updated_at", + }, + query: ` +select thread_id, repo_id, number, coalesce(base_sha, ''), coalesce(head_sha, ''), + coalesce(head_ref, ''), coalesce(head_repo_full_name, ''), + coalesce(mergeable_state, ''), additions, deletions, changed_files, + fetched_at, updated_at +from pull_request_details +order by thread_id`, + maxSourceAt: `select coalesce(max(fetched_at), '') from pull_request_details`, + }, + { + name: "pull_request_files", + columns: []string{ + "thread_id", "position", "path", "status", "additions", "deletions", + "changes", "previous_path", "fetched_at", + }, + query: ` +select thread_id, position, path, coalesce(status, ''), additions, deletions, + changes, coalesce(previous_path, ''), fetched_at +from pull_request_files +order by thread_id, position`, + maxSourceAt: `select coalesce(max(fetched_at), '') from pull_request_files`, + }, + } + + datasets := make([]gitcrawlCloudDataset, 0, len(specs)) + for _, spec := range specs { + rows, err := publishRows(ctx, db, spec.query, func(values []any) []any { return values }) + if err != nil { + return nil, fmt.Errorf("export cloud dataset %s: %w", spec.name, err) + } + var maxSourceAt string + if err := db.QueryRowContext(ctx, spec.maxSourceAt).Scan(&maxSourceAt); err != nil { + return nil, fmt.Errorf("read cloud dataset %s freshness: %w", spec.name, err) + } + datasets = append(datasets, gitcrawlCloudDataset{ + Name: spec.name, + Columns: spec.columns, + Rows: rows, + MaxSourceAt: maxSourceAt, + }) + } + return datasets, nil +} + +func gitcrawlCloudCapabilities(ctx context.Context, db *sql.DB, observationOrder bool) ([]string, error) { + if !observationOrder { + return nil, nil + } + threadSequence, err := sqliteColumnExists(ctx, db, "threads", "observation_sequence") + if err != nil { + return nil, err + } + revisionSequence, err := sqliteColumnExists(ctx, db, "thread_revisions", "observation_sequence") + if err != nil { + return nil, err + } + if !threadSequence || !revisionSequence { + return nil, fmt.Errorf( + "--observation-order requires observation_sequence on threads and thread_revisions", + ) + } + return []string{gitcrawlObservationOrderCapability}, nil +} + +func gitcrawlCloudSourceSyncAt(ctx context.Context, db *sql.DB) (string, error) { + var sourceSyncAt string + err := db.QueryRowContext(ctx, ` +select coalesce(max(value), '') +from ( + select coalesce(finished_at, started_at, '') as value + from sync_runs + where status in ('success', 'completed') + union all + select coalesce(nullif(updated_at_gh, ''), updated_at, '') from threads + union all + select coalesce(updated_at, '') from repositories +)`).Scan(&sourceSyncAt) + if err != nil { + return "", fmt.Errorf("read cloud snapshot source sync time: %w", err) + } + return sourceSyncAt, nil +} + +func gitcrawlCloudHydration(ctx context.Context, snapshotPath string) (crawlstore.EnrichmentCoverage, error) { + st, err := crawlstore.OpenReadOnly(ctx, snapshotPath) + if err != nil { + return crawlstore.EnrichmentCoverage{}, fmt.Errorf("open cloud snapshot for hydration coverage: %w", err) + } + defer st.Close() + coverage, err := st.ArchiveCoverage(ctx, crawlstore.ArchiveCoverageOptions{}) + if err != nil { + return crawlstore.EnrichmentCoverage{}, fmt.Errorf("read cloud snapshot hydration coverage: %w", err) + } + return coverage.Totals.Enrichment, nil +} + +func incompleteGitcrawlCloudHydration(coverage crawlstore.EnrichmentCoverage) []string { + metrics := []struct { + name string + metric crawlstore.EnrichmentCoverageMetric + }{ + {name: "revisions", metric: coverage.Revisions}, + {name: "fingerprints", metric: coverage.Fingerprints}, + {name: "summaries", metric: coverage.Summaries}, + {name: "clusters", metric: coverage.Clusters}, + {name: "pr_details", metric: coverage.PRDetails}, + } + var missing []string + for _, item := range metrics { + if !item.metric.Supported || !item.metric.Complete { + missing = append(missing, fmt.Sprintf( + "%s=%d/%d fresh=%d", + item.name, + item.metric.Covered, + item.metric.Eligible, + item.metric.Fresh, + )) + } + } + return missing +} + +func gitcrawlCloudCoverageRows(snapshot gitcrawlCloudSnapshot, mutationToken string) [][]any { + rows := make([][]any, 0, len(snapshot.Datasets)) + for _, dataset := range snapshot.Datasets { + count := int64(len(dataset.Rows)) + rows = append(rows, []any{ + dataset.Name, + count, + count, + count, + dataset.MaxSourceAt, + snapshot.DatasetGeneratedAt, + true, + mutationToken, + }) + } + return rows +} + +func gitcrawlCloudDatasetCounts(snapshot gitcrawlCloudSnapshot) map[string]int64 { + counts := make(map[string]int64, len(snapshot.Datasets)) + for _, dataset := range snapshot.Datasets { + counts[dataset.Name] = int64(len(dataset.Rows)) + } + return counts +} From 4b006231261ba075f7ebe5cfc350b64cc5047e03 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:13:16 +0200 Subject: [PATCH 03/52] test(cloud): cover snapshot publication and privacy --- internal/cli/app_test.go | 217 +++++++++++++++++++++++++-- internal/cli/remote_commands_test.go | 60 ++++++++ 2 files changed, 267 insertions(+), 10 deletions(-) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 028676b..0012d4f 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -366,6 +366,44 @@ func TestCloudSQLiteSnapshotDropsLocalCodeCorpus(t *testing.T) { }, []store.CodeDocument{{Path: "secret.txt", Language: "txt", ContentHash: "h", Text: "secret", ByteSize: 6, UpdatedAt: "2026-06-06T00:00:00Z"}}); err != nil { t.Fatalf("replace code snapshot: %v", err) } + if _, err := st.DB().ExecContext(ctx, ` + insert into threads( + id, repo_id, github_id, number, kind, state, title, body, html_url, + labels_json, assignees_json, raw_json, content_hash, updated_at + ) values( + 1, ?, 'pr-1', 1, 'pull_request', 'open', 'PR', 'body', + 'https://github.com/openclaw/gitcrawl/pull/1', '[]', '[]', + '{"secret":"thread"}', 'hash', '2026-06-06T00:00:00Z' + ); + insert into pull_request_details( + thread_id, repo_id, number, raw_json, fetched_at, updated_at + ) values( + 1, ?, 1, '{"secret":"detail"}', + '2026-06-06T00:00:00Z', '2026-06-06T00:00:00Z' + ); + insert into pull_request_files( + thread_id, position, path, patch, raw_json, fetched_at + ) values( + 1, 0, 'secret.go', '@@ private source', '{"secret":"file"}', + '2026-06-06T00:00:00Z' + ); + insert into blobs( + sha256, media_type, compression, size_bytes, storage_kind, inline_text, created_at + ) values( + 'blob-hash', 'application/json', 'none', 6, 'inline', 'secret', + '2026-06-06T00:00:00Z' + ); + insert into sync_runs( + repo_id, scope, status, started_at, finished_at, error_text + ) values( + ?, 'all', 'failed', '2026-06-06T00:00:00Z', + '2026-06-06T00:01:00Z', '/private/host/token' + ); + create table portable_metadata(key text primary key, value text not null); + insert into portable_metadata(key, value) values('source_path', '/private/archive.db') + `, repoID, repoID, repoID); err != nil { + t.Fatalf("seed private cloud payloads: %v", err) + } snapshotPath, cleanup, err := cloudSQLiteSnapshotPath(ctx, st.DB(), dbPath) if err != nil { t.Fatalf("cloud snapshot: %v", err) @@ -376,7 +414,14 @@ func TestCloudSQLiteSnapshotDropsLocalCodeCorpus(t *testing.T) { t.Fatalf("open cloud snapshot: %v", err) } defer snapshotDB.Close() - for _, table := range []string{"code_snapshots", "code_documents", "code_documents_fts"} { + for _, table := range []string{ + "code_snapshots", + "code_documents", + "code_documents_fts", + "thread_code_snapshots", + "thread_changed_files", + "thread_hunk_signatures", + } { var count int if err := snapshotDB.QueryRowContext(ctx, `select count(*) from sqlite_schema where name = ?`, table).Scan(&count); err != nil { t.Fatalf("inspect %s: %v", table, err) @@ -385,6 +430,43 @@ func TestCloudSQLiteSnapshotDropsLocalCodeCorpus(t *testing.T) { t.Fatalf("cloud snapshot retained %s", table) } } + var repositoryRawJSON, threadRawJSON, detailRawJSON, filePatch, fileRawJSON string + if err := snapshotDB.QueryRowContext(ctx, ` + select repository.raw_json, thread.raw_json, detail.raw_json, file.patch, file.raw_json + from repositories repository + join threads thread on thread.repo_id = repository.id + join pull_request_details detail on detail.thread_id = thread.id + join pull_request_files file on file.thread_id = thread.id + `).Scan(&repositoryRawJSON, &threadRawJSON, &detailRawJSON, &filePatch, &fileRawJSON); err != nil { + t.Fatalf("inspect scrubbed cloud payloads: %v", err) + } + if repositoryRawJSON != "" || threadRawJSON != "" || detailRawJSON != "" || filePatch != "" || fileRawJSON != "" { + t.Fatalf( + "cloud snapshot retained private payloads: repository=%q thread=%q detail=%q patch=%q file=%q", + repositoryRawJSON, + threadRawJSON, + detailRawJSON, + filePatch, + fileRawJSON, + ) + } + var blobCount int + if err := snapshotDB.QueryRowContext(ctx, `select count(*) from blobs`).Scan(&blobCount); err != nil { + t.Fatalf("inspect cloud blobs: %v", err) + } + if blobCount != 0 { + t.Fatalf("cloud snapshot retained %d blobs", blobCount) + } + var errorText, sourcePath string + if err := snapshotDB.QueryRowContext(ctx, `select coalesce(error_text, '') from sync_runs limit 1`).Scan(&errorText); err != nil { + t.Fatalf("inspect scrubbed cloud run error: %v", err) + } + if err := snapshotDB.QueryRowContext(ctx, `select value from portable_metadata where key = 'source_path'`).Scan(&sourcePath); err != nil { + t.Fatalf("inspect scrubbed cloud source path: %v", err) + } + if errorText != "" || sourcePath != "" { + t.Fatalf("cloud snapshot retained private paths: error=%q source=%q", errorText, sourcePath) + } var sourceCount int if err := st.DB().QueryRowContext(ctx, `select count(*) from code_documents`).Scan(&sourceCount); err != nil { t.Fatalf("source code documents: %v", err) @@ -709,6 +791,9 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { seenTables := map[string]crawlremote.IngestRequest{} var sawSQLitePart bool var sawSQLiteManifest bool + var sawCutover bool + var snapshotID string + mutationCounter := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if got := r.Header.Get("authorization"); got != "Bearer publish-token" { http.Error(w, "missing bearer", http.StatusUnauthorized) @@ -725,6 +810,12 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { switch uploadKind { case "bundle-part": sawSQLitePart = true + if got := r.Header.Get("x-crawl-snapshot-id"); len(got) != 64 { + http.Error(w, "missing snapshot id", http.StatusBadRequest) + return + } else { + snapshotID = got + } if r.Header.Get("content-type") != "application/gzip" || r.Header.Get("x-crawl-content-sha256") == "" { http.Error(w, "bad bundle part headers", http.StatusBadRequest) return @@ -754,7 +845,10 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { App: "gitcrawl", Archive: "gitcrawl/openclaw__openclaw", Complete: false, - Object: &crawlremote.SQLiteObject{Key: "v1/gitcrawl/gitcrawl%2Fopenclaw__openclaw/sqlite/chunks/current.db.gz.part-0000", Size: int64(len(payload))}, + Object: &crawlremote.SQLiteObject{ + Key: crawlremote.SQLiteSnapshotBundlePartKey("gitcrawl", "gitcrawl/openclaw__openclaw", snapshotID, 0), + Size: int64(len(payload)), + }, }) case "bundle-manifest": sawSQLiteManifest = true @@ -771,17 +865,46 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { http.Error(w, "bundle privacy should disclose message bodies", http.StatusBadRequest) return } + if manifest.SnapshotID != snapshotID || + manifest.Object.Key != crawlremote.SQLiteSnapshotObjectKey("gitcrawl", "gitcrawl/openclaw__openclaw", snapshotID) { + http.Error(w, "bundle snapshot mismatch", http.StatusBadRequest) + return + } _ = json.NewEncoder(w).Encode(crawlremote.SQLiteBundleUploadResult{ App: "gitcrawl", Archive: "gitcrawl/openclaw__openclaw", Complete: true, - Bundle: &crawlremote.SQLiteBundle{Key: "v1/gitcrawl/gitcrawl%2Fopenclaw__openclaw/sqlite/current.manifest.json", Manifest: &manifest}, + Bundle: &crawlremote.SQLiteBundle{ + Key: crawlremote.SQLiteSnapshotBundleManifestKey("gitcrawl", "gitcrawl/openclaw__openclaw", snapshotID), + Manifest: &manifest, + }, }) default: http.Error(w, "missing upload kind", http.StatusBadRequest) } return } + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/status") { + http.NotFound(w, r) + return + } + if r.Method == http.MethodPost && strings.HasSuffix(r.URL.EscapedPath(), "/cutover") { + var request struct { + SnapshotID string `json:"snapshot_id"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil || request.SnapshotID != snapshotID { + http.Error(w, "cutover snapshot mismatch", http.StatusBadRequest) + return + } + sawCutover = true + _ = json.NewEncoder(w).Encode(crawlremote.CutoverResult{ + Archive: "gitcrawl/openclaw__openclaw", + SnapshotID: snapshotID, + SnapshotMode: "snapshot", + CutoverAt: "2026-07-12T09:00:00Z", + }) + return + } if r.Method != http.MethodPost || r.URL.EscapedPath() != "/v1/apps/gitcrawl/archives/gitcrawl%2Fopenclaw__openclaw/ingest" { http.NotFound(w, r) return @@ -791,13 +914,39 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { http.Error(w, err.Error(), http.StatusBadRequest) return } - if body.Manifest.App != "gitcrawl" || body.Manifest.Archive != "gitcrawl/openclaw__openclaw" { + if body.Manifest.App != "gitcrawl" || + body.Manifest.Archive != "gitcrawl/openclaw__openclaw" || + body.Manifest.SnapshotID != snapshotID || + body.Manifest.SourceSHA256 != snapshotID { http.Error(w, "manifest mismatch", http.StatusBadRequest) return } + if body.Table == "dataset_coverage" { + if body.MutationToken == "" || !body.Final || len(body.Rows) != 9 { + http.Error(w, "coverage activation mismatch", http.StatusBadRequest) + return + } + for _, row := range body.Rows { + if row[len(row)-1] != body.MutationToken { + http.Error(w, "coverage token mismatch", http.StatusBadRequest) + return + } + } + } seenTables[body.Table] = body + mutationCounter++ + token := fmt.Sprintf("mutation-%d", mutationCounter) + if body.Table == "dataset_coverage" { + token = body.MutationToken + } w.Header().Set("content-type", "application/json") - _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{Table: body.Table, RowsAccepted: int64(len(body.Rows)), Complete: body.Final}) + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + Table: body.Table, + SnapshotID: snapshotID, + MutationToken: token, + RowsAccepted: int64(len(body.Rows)), + Complete: body.Final, + }) })) defer server.Close() @@ -810,13 +959,14 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { "--remote", server.URL, "--archive", "gitcrawl/openclaw__openclaw", "--token-env", tokenEnv, + "--allow-incomplete", "--json", }); err != nil { t.Fatalf("cloud publish: %v", err) } - if len(seenTables) != 2 { - t.Fatalf("tables = %v, want repositories and threads", seenTables) + if len(seenTables) != 10 { + t.Fatalf("tables = %v, want all cloud-v2 datasets and coverage", seenTables) } if got := len(seenTables["repositories"].Rows); got != 1 { t.Fatalf("repositories rows = %d, want 1", got) @@ -824,19 +974,29 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { if got := len(seenTables["threads"].Rows); got != 3 { t.Fatalf("threads rows = %d, want 3", got) } - if !seenTables["threads"].Final { - t.Fatalf("threads batch should finish ingest") + if seenTables["threads"].Final { + t.Fatalf("threads batch should remain staged") + } + if !seenTables["dataset_coverage"].Final { + t.Fatalf("coverage batch should activate the snapshot") } if !sawSQLitePart || !sawSQLiteManifest { t.Fatalf("cloud publish did not upload compressed sqlite bundle: part=%v manifest=%v", sawSQLitePart, sawSQLiteManifest) } + if !sawCutover { + t.Fatal("cloud publish did not cut over the activated snapshot") + } var payload map[string]any if err := json.Unmarshal(out.Bytes(), &payload); err != nil { t.Fatalf("decode output: %v\n%s", err, out.String()) } - if payload["repositories"] != float64(1) || payload["threads"] != float64(3) { + datasets, ok := payload["datasets"].(map[string]any) + if !ok || datasets["repositories"] != float64(1) || datasets["threads"] != float64(3) { t.Fatalf("unexpected output: %#v", payload) } + if payload["snapshot_id"] != snapshotID || payload["source_sha256"] != snapshotID { + t.Fatalf("missing snapshot provenance: %#v", payload) + } if payload["sqlite_bundle"] == nil { t.Fatalf("missing sqlite bundle output: %#v", payload) } @@ -846,6 +1006,43 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { } } +func TestCloudPublishRejectsIncompleteHydrationBeforeRemoteMutation(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + cfg := config.Default() + cfg.DBPath = dbPath + if err := config.Save(cfgPath, cfg); err != nil { + t.Fatalf("save config: %v", err) + } + seedCommandFlowStore(t, dbPath) + + tokenEnv := "GITCRAWL_TEST_INCOMPLETE_PUBLISH_TOKEN" + t.Setenv(tokenEnv, "publish-token") + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + http.Error(w, "unexpected request", http.StatusInternalServerError) + })) + defer server.Close() + + err := New().Run(ctx, []string{ + "--config", cfgPath, + "cloud", "publish", + "--remote", server.URL, + "--archive", "gitcrawl/openclaw__openclaw", + "--token-env", tokenEnv, + "--json", + }) + if err == nil || !strings.Contains(err.Error(), "cloud snapshot enrichment is incomplete") { + t.Fatalf("cloud publish error = %v", err) + } + if requests != 0 { + t.Fatalf("remote requests = %d, want 0", requests) + } +} + func TestRemoteLoginStoresKeyringToken(t *testing.T) { ctx := context.Background() dir := t.TempDir() diff --git a/internal/cli/remote_commands_test.go b/internal/cli/remote_commands_test.go index 14dfff8..e8e866d 100644 --- a/internal/cli/remote_commands_test.go +++ b/internal/cli/remote_commands_test.go @@ -3,6 +3,7 @@ package cli import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" @@ -160,6 +161,65 @@ func TestSendIngestRowsBatchesAndFinalizes(t *testing.T) { } } +func TestSendSnapshotIngestRowsRotatesMutationTokens(t *testing.T) { + ctx := context.Background() + var requests []crawlremote.IngestRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body crawlremote.IngestRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + requests = append(requests, body) + token := fmt.Sprintf("generation-%d", len(requests)) + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + SnapshotID: body.Manifest.SnapshotID, + MutationToken: token, + RowsAccepted: int64(len(body.Rows)), + Complete: body.Final, + }) + })) + defer server.Close() + + client, err := crawlremote.NewClientFromConfig(crawlremote.Config{Endpoint: server.URL}, crawlremote.Options{ + TokenProvider: crawlremote.StaticToken("publish-token"), + }) + if err != nil { + t.Fatalf("client: %v", err) + } + rows := make([][]any, gitcrawlCloudBatchSize+1) + for i := range rows { + rows[i] = []any{i} + } + progress, err := sendSnapshotIngestRows( + ctx, + client, + "gitcrawl", + "gitcrawl/openclaw", + crawlremote.IngestManifest{App: "gitcrawl", SnapshotID: strings.Repeat("a", 64)}, + "threads", + []string{"id"}, + rows, + "previous-dataset", + true, + ) + if err != nil { + t.Fatalf("send snapshot ingest: %v", err) + } + if progress.RowsAccepted != int64(len(rows)) || progress.MutationToken != "generation-2" { + t.Fatalf("progress = %#v", progress) + } + if len(requests) != 2 { + t.Fatalf("requests = %#v", requests) + } + if requests[0].Cursor != "" || requests[0].MutationToken != "previous-dataset" { + t.Fatalf("first request = %#v", requests[0]) + } + if requests[1].Cursor != "250" || requests[1].MutationToken != "generation-1" || !requests[1].Final { + t.Fatalf("second request = %#v", requests[1]) + } +} + func TestSendIngestRowsDrainsRemoteResetBeforeRetry(t *testing.T) { ctx := context.Background() var requests []crawlremote.IngestRequest From d36f3743c0f0dc5c571a24d0fd4aea287140a524 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:13:16 +0200 Subject: [PATCH 04/52] docs(cloud): explain atomic snapshot publishing --- CHANGELOG.md | 8 ++++++++ README.md | 10 +++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99964c6..ed73056 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## 0.7.2 - Unreleased +- Publish Cloudflare archives from one content-addressed SQLite image: export all + cloud-v2 review, revision, fingerprint, summary, cluster, and PR datasets from + that frozen image, enforce enrichment freshness before remote mutation, + activate through mutation-token coverage, upload digest-scoped R2 bundles, + and cut over the exact snapshot. +- Strip raw GitHub JSON, file patches, blobs, and local source-code indexes from + cloud SQLite bundles before hashing and upload. + ## 0.7.1 - 2026-07-09 - Add local, fail-closed packaging and independent verification for official macOS archives signed as `org.openclaw.gitcrawl` by `Developer ID Application: OpenClaw Foundation (FWJYW4S8P8)`, while keeping CI and cross-platform snapshots credential-free and non-publishing. diff --git a/README.md b/README.md index 23dd60a..9ff3412 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,15 @@ gitcrawl tui owner/repo The remote service is deployed separately from gitcrawl in `openclaw/crawl-remote` with Wrangler. gitcrawl only stores the Worker endpoint/archive in config and calls that service. `gitcrawl remote login` starts the Worker GitHub OAuth flow, verifies org/team membership server-side, and stores the signed bearer token in the OS keyring. Use `gitcrawl remote login --github-token-env GITHUB_TOKEN` for non-browser bootstrap; the Worker verifies that GitHub token against the same org/team policy and stores only the returned remote session token locally. -`gitcrawl cloud publish` sends the local SQLite repositories and thread rows to a Worker archive through the role-gated ingest endpoint. +`gitcrawl cloud publish` freezes and sanitizes one local SQLite image, uses its +SHA-256 as the snapshot identity, exports repositories, threads, revisions, +fingerprints, summaries, durable clusters, and PR detail/file rows from that +same image, uploads its digest-scoped R2 bundle, activates complete D1 coverage, +and cuts unpinned reads over to the exact snapshot. Incomplete local enrichment +fails before any remote mutation; `--allow-incomplete` is an explicit escape +hatch, `--observation-order` publishes durable fetch ordering after the remote +operator fence is enabled, and `--cutover=false` stages and activates without +changing unpinned reads. `gitcrawl clusters-report` writes a Markdown report for the top clusters using the same display view, with an at-a-glance table, per-cluster metadata, member tables, and key snippets. Use `--json` for the hydrated report payload. `gitcrawl cluster` and `gitcrawl refresh` build ghcrawl-shaped durable clusters by default (`--threshold 0.80`, `--min-size 1`, `--max-cluster-size 40`, `--k 16`, `--cross-kind-threshold 0.93`): every active vector-backed thread is represented, singleton rows use `singleton_orphan`, multi-member rows use `duplicate_candidate`, and stable IDs are derived from the representative thread. They also add deterministic GitHub reference evidence for direct issue/PR links such as `#123`, `issues/123`, and `pull/123`. Weak embedding edges need concrete title-token overlap unless their similarity is already high, which keeps generic low-confidence bridges from forming unrelated clusters. `gitcrawl tui` infers the most recently updated local repository when `owner/repo` is omitted. `serve` is intentionally not part of `gitcrawl`. From c2553631d18d60430c67613f4d655c6f02a44db5 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 11:15:06 +0200 Subject: [PATCH 05/52] build: track reviewed crawlkit snapshot API --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a32d51f..923936a 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 github.com/charmbracelet/x/ansi v0.11.7 github.com/mattn/go-isatty v0.0.22 - github.com/openclaw/crawlkit v0.13.5-0.20260712090520-2a5e11612f81 + github.com/openclaw/crawlkit v0.13.5-0.20260712091440-1f28a67d580e github.com/zalando/go-keyring v0.2.8 golang.org/x/sys v0.46.0 modernc.org/sqlite v1.53.0 diff --git a/go.sum b/go.sum index 0f8d8d9..72b5524 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/openclaw/crawlkit v0.13.5-0.20260712090520-2a5e11612f81 h1:npL85kzFx39d07yJx1ekh2MIFHb/2XXfIXTdwEKA1tc= -github.com/openclaw/crawlkit v0.13.5-0.20260712090520-2a5e11612f81/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= +github.com/openclaw/crawlkit v0.13.5-0.20260712091440-1f28a67d580e h1:zjCy1SJwrU2Ax9YLF3nSzYzSMu19Vw4USYDe/vfzW+k= +github.com/openclaw/crawlkit v0.13.5-0.20260712091440-1f28a67d580e/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= From 8301159ff537bde9c039c6822b5b9072f64ba9c0 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:48:22 +0200 Subject: [PATCH 06/52] fix(cloud): publish scoped snapshot bundles --- go.mod | 2 +- go.sum | 4 ++-- internal/cli/app_test.go | 14 ++++++++++++-- internal/cli/cloud_commands.go | 2 +- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 923936a..5cc8990 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 github.com/charmbracelet/x/ansi v0.11.7 github.com/mattn/go-isatty v0.0.22 - github.com/openclaw/crawlkit v0.13.5-0.20260712091440-1f28a67d580e + github.com/openclaw/crawlkit v0.13.5-0.20260712095254-5d8ad9db7f08 github.com/zalando/go-keyring v0.2.8 golang.org/x/sys v0.46.0 modernc.org/sqlite v1.53.0 diff --git a/go.sum b/go.sum index 72b5524..4a97dbd 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/openclaw/crawlkit v0.13.5-0.20260712091440-1f28a67d580e h1:zjCy1SJwrU2Ax9YLF3nSzYzSMu19Vw4USYDe/vfzW+k= -github.com/openclaw/crawlkit v0.13.5-0.20260712091440-1f28a67d580e/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= +github.com/openclaw/crawlkit v0.13.5-0.20260712095254-5d8ad9db7f08 h1:JWXyF8yplATomdnYs+q36r63/Df2uui/t4l4ddKtzZY= +github.com/openclaw/crawlkit v0.13.5-0.20260712095254-5d8ad9db7f08/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 0012d4f..f010f2b 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -816,7 +816,11 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { } else { snapshotID = got } - if r.Header.Get("content-type") != "application/gzip" || r.Header.Get("x-crawl-content-sha256") == "" { + partSHA := r.Header.Get("x-crawl-content-sha256") + partIndex, err := strconv.Atoi(r.Header.Get("x-crawl-bundle-part-index")) + if r.Header.Get("content-type") != "application/gzip" || + partSHA == "" || + err != nil { http.Error(w, "bad bundle part headers", http.StatusBadRequest) return } @@ -846,7 +850,13 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { Archive: "gitcrawl/openclaw__openclaw", Complete: false, Object: &crawlremote.SQLiteObject{ - Key: crawlremote.SQLiteSnapshotBundlePartKey("gitcrawl", "gitcrawl/openclaw__openclaw", snapshotID, 0), + Key: crawlremote.SQLiteSnapshotBundlePartKey( + "gitcrawl", + "gitcrawl/openclaw__openclaw", + snapshotID, + partSHA, + partIndex, + ), Size: int64(len(payload)), }, }) diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index e5a9fca..785a17a 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -394,7 +394,7 @@ func uploadSQLiteSnapshotArchive( app, archive, snapshotPath string, counts map[string]int64, ) (*crawlremote.SQLiteBundle, error) { - bundle, err := crawlremote.BuildGzipSQLiteBundle(ctx, crawlremote.SQLiteBundleBuildOptions{ + bundle, err := crawlremote.BuildSnapshotGzipSQLiteBundle(ctx, crawlremote.SQLiteBundleBuildOptions{ App: app, Archive: archive, SourcePath: snapshotPath, From 7f461a3327627a4cfcfb7a6fcf2679f9f597d049 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:59:53 +0200 Subject: [PATCH 07/52] build: pin landed crawlkit snapshot API --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5cc8990..0f72fbe 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 github.com/charmbracelet/x/ansi v0.11.7 github.com/mattn/go-isatty v0.0.22 - github.com/openclaw/crawlkit v0.13.5-0.20260712095254-5d8ad9db7f08 + github.com/openclaw/crawlkit v0.13.5-0.20260712105859-a51e1868fb75 github.com/zalando/go-keyring v0.2.8 golang.org/x/sys v0.46.0 modernc.org/sqlite v1.53.0 diff --git a/go.sum b/go.sum index 4a97dbd..56cf667 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/openclaw/crawlkit v0.13.5-0.20260712095254-5d8ad9db7f08 h1:JWXyF8yplATomdnYs+q36r63/Df2uui/t4l4ddKtzZY= -github.com/openclaw/crawlkit v0.13.5-0.20260712095254-5d8ad9db7f08/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= +github.com/openclaw/crawlkit v0.13.5-0.20260712105859-a51e1868fb75 h1:Zrb8DB2rreGblEu3sVaBWfvSozrW2tD63XEN59DmOjQ= +github.com/openclaw/crawlkit v0.13.5-0.20260712105859-a51e1868fb75/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= From d67ca5e4f450f395debd80115c3de67f7e6c30de Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:11:50 +0200 Subject: [PATCH 08/52] fix(cloud): require current PR file hydration --- internal/cli/cloud_snapshot.go | 1 + internal/store/archive_coverage.go | 128 ++++++++++++++++++++---- internal/store/archive_coverage_test.go | 68 +++++++++++++ 3 files changed, 179 insertions(+), 18 deletions(-) diff --git a/internal/cli/cloud_snapshot.go b/internal/cli/cloud_snapshot.go index 68287e9..01b01f7 100644 --- a/internal/cli/cloud_snapshot.go +++ b/internal/cli/cloud_snapshot.go @@ -323,6 +323,7 @@ func incompleteGitcrawlCloudHydration(coverage crawlstore.EnrichmentCoverage) [] {name: "summaries", metric: coverage.Summaries}, {name: "clusters", metric: coverage.Clusters}, {name: "pr_details", metric: coverage.PRDetails}, + {name: "pr_files", metric: coverage.PRFiles}, } var missing []string for _, item := range metrics { diff --git a/internal/store/archive_coverage.go b/internal/store/archive_coverage.go index 3688e6b..f690938 100644 --- a/internal/store/archive_coverage.go +++ b/internal/store/archive_coverage.go @@ -34,6 +34,7 @@ type EnrichmentCoverage struct { Summaries EnrichmentCoverageMetric `json:"summaries"` Clusters EnrichmentCoverageMetric `json:"clusters"` PRDetails EnrichmentCoverageMetric `json:"pr_details"` + PRFiles EnrichmentCoverageMetric `json:"pr_files"` } type ArchiveCoverageRow struct { @@ -291,6 +292,10 @@ func (s *Store) archiveEnrichmentCoverage(ctx context.Context, repoID int64) (En if err != nil { return EnrichmentCoverage{}, err } + coverage.PRFiles, err = s.archivePRFileCoverage(ctx, repoID) + if err != nil { + return EnrichmentCoverage{}, err + } syncObservedAt, ok, err := s.archiveLatestSuccessfulHydrationRunAt(ctx, repoID) if err != nil { return EnrichmentCoverage{}, err @@ -533,24 +538,8 @@ func (s *Store) archivePRDetailCoverage(ctx context.Context, repoID int64) (Enri if !s.archiveCoverageHasColumns(ctx, "pull_request_details", "thread_id", "fetched_at") { return EnrichmentCoverageMetric{}, nil } - acceptedSourceUpdatedAt := archiveThreadUpdatedAtExpression(s, ctx, "t") - acceptedObservationSequence := "0" - if s.hasColumn(ctx, "threads", "observation_sequence") { - acceptedObservationSequence = "t.observation_sequence" - } - if s.hasColumn(ctx, "threads", "evidence_observation_sequence") && - s.hasColumn(ctx, "threads", "evidence_source_updated_at") { - acceptedSourceUpdatedAt = `case - when t.evidence_observation_sequence > 0 - then coalesce(t.evidence_source_updated_at, '') - else ` + acceptedSourceUpdatedAt + ` - end` - acceptedObservationSequence = `case - when t.evidence_observation_sequence > 0 - then t.evidence_observation_sequence - else ` + acceptedObservationSequence + ` - end` - } + acceptedSourceUpdatedAt, acceptedObservationSequence := + s.archiveAcceptedThreadObservationExpressions(ctx, "t") reservationJoin := "" reservationPresent := "0" reservationSourceUpdatedAt := "''" @@ -642,6 +631,107 @@ func (s *Store) archivePRDetailCoverage(ctx context.Context, repoID int64) (Enri return metric, nil } +func (s *Store) archivePRFileCoverage( + ctx context.Context, + repoID int64, +) (EnrichmentCoverageMetric, error) { + if !s.archiveCoverageHasColumns( + ctx, + "thread_child_observation_reservations", + "thread_id", + "family", + "source_updated_at", + "observation_sequence", + ) { + return EnrichmentCoverageMetric{}, nil + } + acceptedSourceUpdatedAt, acceptedObservationSequence := + s.archiveAcceptedThreadObservationExpressions(ctx, "t") + rows, err := s.q().QueryContext(ctx, ` + select case when reservation.thread_id is null then 0 else 1 end, + coalesce(reservation.source_updated_at, ''), + coalesce(reservation.observation_sequence, 0), + `+acceptedSourceUpdatedAt+`, + `+acceptedObservationSequence+` + from threads t + left join thread_child_observation_reservations reservation + on reservation.thread_id = t.id + and reservation.family = 'pull_request_files' + where t.repo_id = ? and t.kind = 'pull_request' + `, repoID) + if err != nil { + return EnrichmentCoverageMetric{}, fmt.Errorf("archive PR file coverage: %w", err) + } + defer rows.Close() + + metric := EnrichmentCoverageMetric{Supported: true} + var latestObservedAt time.Time + for rows.Next() { + var hasReservation int + var reservationSequence, acceptedSequence int64 + var reservationSourceUpdatedAt, acceptedSource string + if err := rows.Scan( + &hasReservation, + &reservationSourceUpdatedAt, + &reservationSequence, + &acceptedSource, + &acceptedSequence, + ); err != nil { + return EnrichmentCoverageMetric{}, fmt.Errorf("scan archive PR file coverage: %w", err) + } + metric.Eligible++ + if hasReservation == 0 { + continue + } + metric.Covered++ + if observedAt, ok := parseArchiveCoverageTimestamp(reservationSourceUpdatedAt); ok && + (latestObservedAt.IsZero() || observedAt.After(latestObservedAt)) { + latestObservedAt = observedAt + } + if archiveObservationAtOrAfter( + reservationSourceUpdatedAt, + reservationSequence, + acceptedSource, + observationSequenceOrderValue(acceptedSequence), + ) { + metric.Fresh++ + } + } + if err := rows.Err(); err != nil { + return EnrichmentCoverageMetric{}, fmt.Errorf("iterate archive PR file coverage: %w", err) + } + if !latestObservedAt.IsZero() { + metric.LatestAt = formatArchiveCoverageTimestamp(latestObservedAt) + } + finalizeEnrichmentCoverageMetric(&metric) + return metric, nil +} + +func (s *Store) archiveAcceptedThreadObservationExpressions( + ctx context.Context, + alias string, +) (sourceUpdatedAt string, observationSequence string) { + sourceUpdatedAt = archiveThreadUpdatedAtExpression(s, ctx, alias) + observationSequence = "0" + if s.hasColumn(ctx, "threads", "observation_sequence") { + observationSequence = alias + ".observation_sequence" + } + if s.hasColumn(ctx, "threads", "evidence_observation_sequence") && + s.hasColumn(ctx, "threads", "evidence_source_updated_at") { + sourceUpdatedAt = `case + when ` + alias + `.evidence_observation_sequence > 0 + then coalesce(` + alias + `.evidence_source_updated_at, '') + else ` + sourceUpdatedAt + ` + end` + observationSequence = `case + when ` + alias + `.evidence_observation_sequence > 0 + then ` + alias + `.evidence_observation_sequence + else ` + observationSequence + ` + end` + } + return sourceUpdatedAt, observationSequence +} + func archiveObservationAtOrAfter( observedSourceUpdatedAt string, observedSequence int64, @@ -743,6 +833,7 @@ func addEnrichmentCoverage(total *EnrichmentCoverage, row EnrichmentCoverage) { addEnrichmentCoverageMetric(&total.Summaries, row.Summaries) addEnrichmentCoverageMetric(&total.Clusters, row.Clusters) addEnrichmentCoverageMetric(&total.PRDetails, row.PRDetails) + addEnrichmentCoverageMetric(&total.PRFiles, row.PRFiles) } func addEnrichmentCoverageMetric(total *EnrichmentCoverageMetric, row EnrichmentCoverageMetric) { @@ -763,4 +854,5 @@ func finalizeEnrichmentCoverage(coverage *EnrichmentCoverage) { finalizeEnrichmentCoverageMetric(&coverage.Summaries) finalizeEnrichmentCoverageMetric(&coverage.Clusters) finalizeEnrichmentCoverageMetric(&coverage.PRDetails) + finalizeEnrichmentCoverageMetric(&coverage.PRFiles) } diff --git a/internal/store/archive_coverage_test.go b/internal/store/archive_coverage_test.go index a1bbbaf..a7d36d2 100644 --- a/internal/store/archive_coverage_test.go +++ b/internal/store/archive_coverage_test.go @@ -218,6 +218,74 @@ func TestArchiveCoveragePRDetailFreshnessUsesAcceptedSourceObservation(t *testin } } +func TestArchiveCoveragePRFilesRequireCurrentObservation(t *testing.T) { + ctx := context.Background() + st, err := Open(ctx, filepath.Join(t.TempDir(), "gitcrawl.db")) + if err != nil { + t.Fatalf("open store: %v", err) + } + defer st.Close() + + repoID, err := st.UpsertRepository(ctx, Repository{ + Owner: "openclaw", Name: "gitcrawl", FullName: "openclaw/gitcrawl", + RawJSON: "{}", UpdatedAt: "2026-07-12T12:00:00Z", + }) + if err != nil { + t.Fatalf("repository: %v", err) + } + thread := archiveCoverageThread(repoID, 1, "pull_request") + thread.UpdatedAtGitHub = "2026-07-12T12:00:00Z" + thread.RawJSON = `{"version":1}` + thread.ContentHash = "version-1" + initial, err := st.UpsertThreadObservation(ctx, thread, UpsertThreadOptions{ + ObservationSequence: 1, + }) + if err != nil { + t.Fatalf("initial thread: %v", err) + } + thread.ID = initial.ID + + assertMetric := func(wantCovered, wantFresh, wantMissing, wantStale int, wantComplete bool) { + t.Helper() + coverage, err := st.ArchiveCoverage(ctx, ArchiveCoverageOptions{}) + if err != nil { + t.Fatalf("archive coverage: %v", err) + } + if len(coverage.Rows) != 1 { + t.Fatalf("coverage rows = %d, want 1", len(coverage.Rows)) + } + metric := coverage.Rows[0].Enrichment.PRFiles + if !metric.Supported || metric.Eligible != 1 || + metric.Covered != wantCovered || metric.Fresh != wantFresh || + metric.Missing != wantMissing || metric.Stale != wantStale || + metric.Complete != wantComplete { + t.Fatalf("PR file coverage = %+v", metric) + } + } + + assertMetric(0, 0, 1, 0, false) + if applied, err := st.ReserveThreadChildObservation( + ctx, + thread.ID, + ThreadChildPullRequestFiles, + thread.UpdatedAtGitHub, + 1, + ); err != nil || !applied { + t.Fatalf("reserve PR file observation = %t, %v", applied, err) + } + assertMetric(1, 1, 0, 0, true) + + thread.UpdatedAtGitHub = "2026-07-12T12:03:00Z" + thread.RawJSON = `{"version":2}` + thread.ContentHash = "version-2" + if accepted, err := st.UpsertThreadObservation(ctx, thread, UpsertThreadOptions{ + ObservationSequence: 2, + }); err != nil || !accepted.EvidenceApplied { + t.Fatalf("newer accepted evidence = %+v, %v", accepted, err) + } + assertMetric(1, 0, 0, 1, false) +} + func TestArchiveObservationFreshnessUsesSequenceForEqualSource(t *testing.T) { const source = "2026-07-12T12:00:00Z" if archiveObservationAtOrAfter(source, 10, source, 11) { From 27e6b4fe3e6ea11ccef3dcc07b8af82ba495e3c9 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:11:53 +0200 Subject: [PATCH 09/52] fix(cloud): negotiate safe snapshot publication --- internal/cli/app_test.go | 319 ++++++++++++++++++++++++++++++--- internal/cli/cloud_commands.go | 48 ++++- 2 files changed, 343 insertions(+), 24 deletions(-) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index f010f2b..cfc1eb7 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -381,27 +381,56 @@ func TestCloudSQLiteSnapshotDropsLocalCodeCorpus(t *testing.T) { 1, ?, 1, '{"secret":"detail"}', '2026-06-06T00:00:00Z', '2026-06-06T00:00:00Z' ); - insert into pull_request_files( - thread_id, position, path, patch, raw_json, fetched_at - ) values( - 1, 0, 'secret.go', '@@ private source', '{"secret":"file"}', - '2026-06-06T00:00:00Z' - ); - insert into blobs( - sha256, media_type, compression, size_bytes, storage_kind, inline_text, created_at - ) values( + insert into pull_request_files( + thread_id, position, path, patch, raw_json, fetched_at + ) values( + 1, 0, 'secret.go', '@@ private source', '{"secret":"file"}', + '2026-06-06T00:00:00Z' + ); + insert into pull_request_review_threads( + thread_id, review_thread_id, path, comments_json, raw_json, fetched_at + ) values( + 1, 'review-thread-1', 'secret.go', + '[{"body":"review body","diffHunk":"@@ private review source"}]', + '{"secret":"review-thread"}', '2026-06-06T00:00:00Z' + ); + insert into blobs( + sha256, media_type, compression, size_bytes, storage_kind, inline_text, created_at + ) values( 'blob-hash', 'application/json', 'none', 6, 'inline', 'secret', '2026-06-06T00:00:00Z' ); - insert into sync_runs( - repo_id, scope, status, started_at, finished_at, error_text - ) values( - ?, 'all', 'failed', '2026-06-06T00:00:00Z', - '2026-06-06T00:01:00Z', '/private/host/token' - ); - create table portable_metadata(key text primary key, value text not null); - insert into portable_metadata(key, value) values('source_path', '/private/archive.db') - `, repoID, repoID, repoID); err != nil { + insert into sync_runs( + repo_id, scope, status, started_at, finished_at, stats_json, error_text + ) values( + ?, 'all', 'failed', '2026-06-06T00:00:00Z', + '2026-06-06T00:01:00Z', + '{"message":"/private/host/token","threads":1}', '/private/host/token' + ); + insert into summary_runs( + repo_id, scope, status, started_at, finished_at, stats_json, error_text + ) values( + ?, 'all', 'failed', '2026-06-06T00:00:00Z', + '2026-06-06T00:01:00Z', + '{"failures":[{"message":"provider secret"}]}', 'provider secret' + ); + insert into embedding_runs( + repo_id, scope, status, started_at, finished_at, stats_json, error_text + ) values( + ?, 'all', 'failed', '2026-06-06T00:00:00Z', + '2026-06-06T00:01:00Z', + '{"failures":[{"message":"/private/embed"}]}', '/private/embed' + ); + insert into cluster_runs( + repo_id, scope, status, started_at, finished_at, stats_json, error_text + ) values( + ?, 'all', 'failed', '2026-06-06T00:00:00Z', + '2026-06-06T00:01:00Z', + '{"message":"/private/cluster"}', '/private/cluster' + ); + create table portable_metadata(key text primary key, value text not null); + insert into portable_metadata(key, value) values('source_path', '/private/archive.db') + `, repoID, repoID, repoID, repoID, repoID, repoID); err != nil { t.Fatalf("seed private cloud payloads: %v", err) } snapshotPath, cleanup, err := cloudSQLiteSnapshotPath(ctx, st.DB(), dbPath) @@ -457,15 +486,37 @@ func TestCloudSQLiteSnapshotDropsLocalCodeCorpus(t *testing.T) { if blobCount != 0 { t.Fatalf("cloud snapshot retained %d blobs", blobCount) } - var errorText, sourcePath string - if err := snapshotDB.QueryRowContext(ctx, `select coalesce(error_text, '') from sync_runs limit 1`).Scan(&errorText); err != nil { - t.Fatalf("inspect scrubbed cloud run error: %v", err) + var reviewComments string + if err := snapshotDB.QueryRowContext(ctx, ` + select comments_json from pull_request_review_threads limit 1 + `).Scan(&reviewComments); err != nil { + t.Fatalf("inspect scrubbed review comments: %v", err) + } + if reviewComments != "" { + t.Fatalf("cloud snapshot retained review diff payloads: %q", reviewComments) + } + for _, table := range []string{"sync_runs", "summary_runs", "embedding_runs", "cluster_runs"} { + var statsJSON, errorText string + if err := snapshotDB.QueryRowContext(ctx, ` + select coalesce(stats_json, ''), coalesce(error_text, '') from `+table+` limit 1 + `).Scan(&statsJSON, &errorText); err != nil { + t.Fatalf("inspect scrubbed cloud %s payload: %v", table, err) + } + if statsJSON != "" || errorText != "" { + t.Fatalf( + "cloud snapshot retained %s failure payloads: stats=%q error=%q", + table, + statsJSON, + errorText, + ) + } } + var sourcePath string if err := snapshotDB.QueryRowContext(ctx, `select value from portable_metadata where key = 'source_path'`).Scan(&sourcePath); err != nil { t.Fatalf("inspect scrubbed cloud source path: %v", err) } - if errorText != "" || sourcePath != "" { - t.Fatalf("cloud snapshot retained private paths: error=%q source=%q", errorText, sourcePath) + if sourcePath != "" { + t.Fatalf("cloud snapshot retained private source path: %q", sourcePath) } var sourceCount int if err := st.DB().QueryRowContext(ctx, `select count(*) from code_documents`).Scan(&sourceCount); err != nil { @@ -774,6 +825,19 @@ func TestRemoteCloudModeDoesNotCreateLocalDB(t *testing.T) { } } +func testSnapshotPublishContract() crawlremote.Contract { + contract := crawlremote.BaseContract() + contract.Apps = []crawlremote.AppSpec{{ + App: "gitcrawl", + Capabilities: []string{ + gitcrawlSnapshotAtomicCapability, + gitcrawlSnapshotProvenanceCapability, + sqliteBundleGzipUploadCapability, + }, + }} + return contract +} + func TestCloudPublishSendsLocalRows(t *testing.T) { ctx := context.Background() dir := t.TempDir() @@ -795,6 +859,11 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { var snapshotID string mutationCounter := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { + w.Header().Set("content-type", "application/json") + _ = json.NewEncoder(w).Encode(testSnapshotPublishContract()) + return + } if got := r.Header.Get("authorization"); got != "Bearer publish-token" { http.Error(w, "missing bearer", http.StatusUnauthorized) return @@ -970,6 +1039,7 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { "--archive", "gitcrawl/openclaw__openclaw", "--token-env", tokenEnv, "--allow-incomplete", + "--cutover", "--json", }); err != nil { t.Fatalf("cloud publish: %v", err) @@ -1016,6 +1086,209 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { } } +func TestCloudPublishRejectsMissingSnapshotCapabilityBeforeUpload(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + cfg := config.Default() + cfg.DBPath = dbPath + if err := config.Save(configPath, cfg); err != nil { + t.Fatalf("save config: %v", err) + } + seedCommandFlowStore(t, dbPath) + + const tokenEnv = "GITCRAWL_TEST_CONTRACT_TOKEN" + t.Setenv(tokenEnv, "publish-token") + mutations := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { + contract := testSnapshotPublishContract() + contract.Apps[0].Capabilities = []string{ + gitcrawlSnapshotAtomicCapability, + sqliteBundleGzipUploadCapability, + } + w.Header().Set("content-type", "application/json") + _ = json.NewEncoder(w).Encode(contract) + return + } + mutations++ + http.Error(w, "unexpected mutation", http.StatusInternalServerError) + })) + defer server.Close() + + err := New().Run(ctx, []string{ + "--config", configPath, + "cloud", "publish", + "--remote", server.URL, + "--archive", "gitcrawl/openclaw__openclaw", + "--token-env", tokenEnv, + "--allow-incomplete", + "--json", + }) + if err == nil || !strings.Contains(err.Error(), gitcrawlSnapshotProvenanceCapability) { + t.Fatalf("cloud publish error = %v", err) + } + if mutations != 0 { + t.Fatalf("remote mutations = %d, want 0", mutations) + } +} + +func TestCloudPublishStagesByDefaultAndResumesCutover(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + cfg := config.Default() + cfg.DBPath = dbPath + if err := config.Save(configPath, cfg); err != nil { + t.Fatalf("save config: %v", err) + } + seedCommandFlowStore(t, dbPath) + + const tokenEnv = "GITCRAWL_TEST_RESUME_TOKEN" + t.Setenv(tokenEnv, "publish-token") + var snapshotID string + ingestRequests := 0 + cutovers := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { + w.Header().Set("content-type", "application/json") + _ = json.NewEncoder(w).Encode(testSnapshotPublishContract()) + return + } + if got := r.Header.Get("authorization"); got != "Bearer publish-token" { + http.Error(w, "missing bearer", http.StatusUnauthorized) + return + } + if r.Method == http.MethodPut && strings.HasSuffix(r.URL.EscapedPath(), "/sqlite") { + w.Header().Set("content-type", "application/json") + switch r.Header.Get("x-crawl-sqlite-upload") { + case "bundle-part": + snapshotID = r.Header.Get("x-crawl-snapshot-id") + partSHA := r.Header.Get("x-crawl-content-sha256") + partIndex, err := strconv.Atoi(r.Header.Get("x-crawl-bundle-part-index")) + if len(snapshotID) != 64 || len(partSHA) != 64 || err != nil { + http.Error(w, "invalid bundle part", http.StatusBadRequest) + return + } + _ = json.NewEncoder(w).Encode(crawlremote.SQLiteUploadResult{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__openclaw", + Complete: false, + Object: &crawlremote.SQLiteObject{ + Key: crawlremote.SQLiteSnapshotBundlePartKey( + "gitcrawl", + "gitcrawl/openclaw__openclaw", + snapshotID, + partSHA, + partIndex, + ), + }, + }) + case "bundle-manifest": + var manifest crawlremote.SQLiteBundleManifest + if err := json.NewDecoder(r.Body).Decode(&manifest); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if manifest.SnapshotID != snapshotID { + http.Error(w, "snapshot mismatch", http.StatusBadRequest) + return + } + _ = json.NewEncoder(w).Encode(crawlremote.SQLiteBundleUploadResult{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__openclaw", + Complete: true, + Bundle: &crawlremote.SQLiteBundle{ + Key: crawlremote.SQLiteSnapshotBundleManifestKey("gitcrawl", "gitcrawl/openclaw__openclaw", snapshotID), + Manifest: &manifest, + }, + }) + default: + http.Error(w, "missing upload kind", http.StatusBadRequest) + } + return + } + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/status") { + _ = json.NewEncoder(w).Encode(crawlremote.Status{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__openclaw", + ActiveSnapshotID: snapshotID, + CoverageComplete: true, + Snapshot: &crawlremote.ArchiveSnapshot{ + ID: snapshotID, + SourceSHA256: snapshotID, + CoverageComplete: true, + }, + }) + return + } + if r.Method == http.MethodPost && strings.HasSuffix(r.URL.EscapedPath(), "/ingest") { + ingestRequests++ + http.Error(w, "activated snapshots must not be re-ingested", http.StatusConflict) + return + } + if r.Method == http.MethodPost && strings.HasSuffix(r.URL.EscapedPath(), "/cutover") { + var request struct { + SnapshotID string `json:"snapshot_id"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil || + request.SnapshotID != snapshotID { + http.Error(w, "cutover snapshot mismatch", http.StatusBadRequest) + return + } + cutovers++ + _ = json.NewEncoder(w).Encode(crawlremote.CutoverResult{ + Archive: "gitcrawl/openclaw__openclaw", + SnapshotID: snapshotID, + SnapshotMode: "snapshot", + CutoverAt: "2026-07-12T11:00:00Z", + }) + return + } + http.NotFound(w, r) + })) + defer server.Close() + + baseArgs := []string{ + "--config", configPath, + "cloud", "publish", + "--remote", server.URL, + "--archive", "gitcrawl/openclaw__openclaw", + "--token-env", tokenEnv, + "--allow-incomplete", + "--json", + } + for index, extra := range [][]string{nil, {"--cutover"}} { + app := New() + var output bytes.Buffer + app.Stdout = &output + args := append(append([]string(nil), baseArgs...), extra...) + if err := app.Run(ctx, args); err != nil { + t.Fatalf("cloud publish run %d: %v", index+1, err) + } + var result struct { + Cutover *crawlremote.CutoverResult `json:"cutover"` + } + if err := json.Unmarshal(output.Bytes(), &result); err != nil { + t.Fatalf("decode run %d output: %v\n%s", index+1, err, output.String()) + } + if index == 0 && result.Cutover != nil { + t.Fatalf("default publish unexpectedly cut over: %#v", result.Cutover) + } + if index == 1 && result.Cutover == nil { + t.Fatalf("explicit cutover missing: %s", output.String()) + } + } + if ingestRequests != 0 { + t.Fatalf("ingest requests = %d, want 0 for active snapshot resume", ingestRequests) + } + if cutovers != 1 { + t.Fatalf("cutovers = %d, want 1", cutovers) + } +} + func TestCloudPublishRejectsIncompleteHydrationBeforeRemoteMutation(t *testing.T) { ctx := context.Background() dir := t.TempDir() diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 785a17a..e4f62b0 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -11,6 +11,7 @@ import ( "net/http" "os" "path/filepath" + "slices" "strings" "time" @@ -21,6 +22,10 @@ import ( const ( gitcrawlCloudBatchSize = 250 gitcrawlCloudSQLiteBundleChunkSize = int64(64 * 1024 * 1024) + + gitcrawlSnapshotAtomicCapability = "gitcrawl.snapshot.atomic" + gitcrawlSnapshotProvenanceCapability = "gitcrawl.snapshot.provenance.v1" + sqliteBundleGzipUploadCapability = "sqlite.bundle.gzip.upload" ) func (a *App) runCloud(ctx context.Context, args []string) error { @@ -43,7 +48,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { tokenEnv := fs.String("token-env", "", "remote token environment variable") allowIncomplete := fs.Bool("allow-incomplete", false, "publish even when local enrichment coverage is incomplete") observationOrder := fs.Bool("observation-order", false, "publish durable observation ordering when the remote fence is enabled") - cutover := fs.Bool("cutover", true, "cut over unpinned reads after activating the snapshot") + cutover := fs.Bool("cutover", false, "cut over unpinned reads after activating the snapshot") jsonOut := fs.Bool("json", false, "write JSON output") if err := fs.Parse(normalizeCommandArgs(args, map[string]bool{"remote": true, "archive": true, "token-env": true})); err != nil { return usageErr(err) @@ -107,6 +112,9 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { } manifest := gitcrawlCloudManifest(archiveID, snapshot) counts := gitcrawlCloudDatasetCounts(snapshot) + if err := requireGitcrawlSnapshotPublishContract(ctx, client); err != nil { + return err + } sqliteBundle, err := uploadSQLiteSnapshotArchive( ctx, client, @@ -418,6 +426,39 @@ func remoteNotFound(err error) bool { return errors.As(err, &remoteErr) && remoteErr.Status == http.StatusNotFound } +func requireGitcrawlSnapshotPublishContract( + ctx context.Context, + client *crawlremote.Client, +) error { + contract, err := client.Contract(ctx) + if err != nil { + return fmt.Errorf("read remote snapshot publish contract: %w", err) + } + if err := contract.Validate(); err != nil { + return fmt.Errorf("validate remote snapshot publish contract: %w", err) + } + var capabilities []string + for _, app := range contract.Apps { + if app.App == "gitcrawl" { + capabilities = app.Capabilities + break + } + } + for _, capability := range []string{ + gitcrawlSnapshotAtomicCapability, + gitcrawlSnapshotProvenanceCapability, + sqliteBundleGzipUploadCapability, + } { + if !slices.Contains(capabilities, capability) { + return fmt.Errorf( + "remote does not advertise required snapshot publish capability %s", + capability, + ) + } + } + return nil +} + func gitcrawlCloudSQLiteBundlePrivacy() map[string]any { return map[string]any{ "includes_private_messages": true, @@ -485,11 +526,16 @@ func sanitizeCloudSQLiteSnapshot(ctx context.Context, db *sql.DB) error { {table: "pull_request_commits", name: "raw_json"}, {table: "pull_request_checks", name: "raw_json"}, {table: "pull_request_review_threads", name: "raw_json"}, + {table: "pull_request_review_threads", name: "comments_json"}, {table: "github_workflow_runs", name: "raw_json"}, {table: "sync_runs", name: "error_text"}, + {table: "sync_runs", name: "stats_json"}, {table: "summary_runs", name: "error_text"}, + {table: "summary_runs", name: "stats_json"}, {table: "embedding_runs", name: "error_text"}, + {table: "embedding_runs", name: "stats_json"}, {table: "cluster_runs", name: "error_text"}, + {table: "cluster_runs", name: "stats_json"}, } { exists, err := sqliteColumnExists(ctx, db, column.table, column.name) if err != nil { From 8f7bb317f4f8839280db6fbff859995d42f4e04e Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:11:55 +0200 Subject: [PATCH 10/52] docs(cloud): document explicit snapshot cutover --- CHANGELOG.md | 10 ---------- README.md | 15 ++++++++------- internal/cli/app.go | 2 +- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed73056..ff7a6e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,5 @@ # Changelog -## 0.7.2 - Unreleased - -- Publish Cloudflare archives from one content-addressed SQLite image: export all - cloud-v2 review, revision, fingerprint, summary, cluster, and PR datasets from - that frozen image, enforce enrichment freshness before remote mutation, - activate through mutation-token coverage, upload digest-scoped R2 bundles, - and cut over the exact snapshot. -- Strip raw GitHub JSON, file patches, blobs, and local source-code indexes from - cloud SQLite bundles before hashing and upload. - ## 0.7.1 - 2026-07-09 - Add local, fail-closed packaging and independent verification for official macOS archives signed as `org.openclaw.gitcrawl` by `Developer ID Application: OpenClaw Foundation (FWJYW4S8P8)`, while keeping CI and cross-platform snapshots credential-free and non-publishing. diff --git a/README.md b/README.md index 9ff3412..5f338f6 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ gitcrawl remote login --endpoint https://crawl.openclaw.ai --json gitcrawl remote status --json gitcrawl remote archives --json gitcrawl whoami --json -gitcrawl cloud publish --remote https://crawl.openclaw.ai --archive gitcrawl/openclaw__openclaw --json +gitcrawl cloud publish --remote https://crawl.openclaw.ai --archive gitcrawl/openclaw__openclaw --cutover --json gitcrawl sync owner/repo gitcrawl sync owner/repo --state open gitcrawl sync owner/repo --numbers 123,456 --include-comments @@ -70,12 +70,13 @@ Use `gitcrawl remote login --github-token-env GITHUB_TOKEN` for non-browser boot `gitcrawl cloud publish` freezes and sanitizes one local SQLite image, uses its SHA-256 as the snapshot identity, exports repositories, threads, revisions, fingerprints, summaries, durable clusters, and PR detail/file rows from that -same image, uploads its digest-scoped R2 bundle, activates complete D1 coverage, -and cuts unpinned reads over to the exact snapshot. Incomplete local enrichment -fails before any remote mutation; `--allow-incomplete` is an explicit escape -hatch, `--observation-order` publishes durable fetch ordering after the remote -operator fence is enabled, and `--cutover=false` stages and activates without -changing unpinned reads. +same image, negotiates the remote snapshot-provenance contract before touching +R2, uploads its digest-scoped bundle, and activates complete D1 coverage. +Publishing stages the immutable snapshot by default; `--cutover` explicitly +moves unpinned reads to that snapshot and disables legacy ingest. Incomplete +local enrichment fails before any remote mutation; `--allow-incomplete` is an +explicit escape hatch, and `--observation-order` publishes durable fetch +ordering after the remote operator fence is enabled. `gitcrawl clusters-report` writes a Markdown report for the top clusters using the same display view, with an at-a-glance table, per-cluster metadata, member tables, and key snippets. Use `--json` for the hydrated report payload. `gitcrawl cluster` and `gitcrawl refresh` build ghcrawl-shaped durable clusters by default (`--threshold 0.80`, `--min-size 1`, `--max-cluster-size 40`, `--k 16`, `--cross-kind-threshold 0.93`): every active vector-backed thread is represented, singleton rows use `singleton_orphan`, multi-member rows use `duplicate_candidate`, and stable IDs are derived from the representative thread. They also add deterministic GitHub reference evidence for direct issue/PR links such as `#123`, `issues/123`, and `pull/123`. Weak embedding edges need concrete title-token overlap unless their similarity is already high, which keeps generic low-confidence bridges from forming unrelated clusters. `gitcrawl tui` infers the most recently updated local repository when `owner/repo` is omitted. `serve` is intentionally not part of `gitcrawl`. diff --git a/internal/cli/app.go b/internal/cli/app.go index 5035e77..69fa3e2 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -4770,7 +4770,7 @@ Usage: "cloud": `gitcrawl cloud manages Worker-backed remote archives. Usage: - gitcrawl cloud publish --remote URL --archive id [--allow-incomplete] [--observation-order] [--cutover=false] [--json] + gitcrawl cloud publish --remote URL --archive id [--allow-incomplete] [--observation-order] [--cutover] [--json] `, "whoami": `gitcrawl whoami prints the configured remote archive identity. From 347fac03c920e84a27e10e95194e148d5b8664e5 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:21:23 +0200 Subject: [PATCH 11/52] fix(cloud): prove legacy PR file hydration --- internal/store/archive_coverage.go | 133 +++++++++++++++++++----- internal/store/archive_coverage_test.go | 91 ++++++++++++++++ 2 files changed, 199 insertions(+), 25 deletions(-) diff --git a/internal/store/archive_coverage.go b/internal/store/archive_coverage.go index f690938..dff0fd8 100644 --- a/internal/store/archive_coverage.go +++ b/internal/store/archive_coverage.go @@ -635,28 +635,86 @@ func (s *Store) archivePRFileCoverage( ctx context.Context, repoID int64, ) (EnrichmentCoverageMetric, error) { - if !s.archiveCoverageHasColumns( + hasReservations := s.archiveCoverageHasColumns( ctx, "thread_child_observation_reservations", "thread_id", "family", "source_updated_at", "observation_sequence", - ) { + ) + hasLegacyProof := s.archiveCoverageHasColumns( + ctx, + "pull_request_details", + "thread_id", + "changed_files", + "fetched_at", + ) && s.archiveCoverageHasColumns( + ctx, + "pull_request_files", + "thread_id", + "fetched_at", + ) + if !hasReservations && !hasLegacyProof { return EnrichmentCoverageMetric{}, nil } acceptedSourceUpdatedAt, acceptedObservationSequence := s.archiveAcceptedThreadObservationExpressions(ctx, "t") + reservationJoin := "" + reservationPresent := "0" + reservationSourceUpdatedAt := "''" + reservationObservationSequence := "0" + if hasReservations { + reservationJoin = ` + left join thread_child_observation_reservations reservation + on reservation.thread_id = t.id + and reservation.family = 'pull_request_files' + ` + reservationPresent = "case when reservation.thread_id is null then 0 else 1 end" + reservationSourceUpdatedAt = "coalesce(reservation.source_updated_at, '')" + reservationObservationSequence = "coalesce(reservation.observation_sequence, 0)" + } + legacyJoin := "" + detailPresent := "0" + detailChangedFiles := "0" + detailFetchedAt := "''" + fileCount := "0" + oldestFileFetchedAt := "''" + latestFileFetchedAt := "''" + if hasLegacyProof { + legacyJoin = ` + left join pull_request_details prd on prd.thread_id = t.id + left join ( + select thread_id, + count(*) as file_count, + min(fetched_at) as oldest_fetched_at, + max(fetched_at) as latest_fetched_at + from pull_request_files + group by thread_id + ) prf on prf.thread_id = t.id + ` + detailPresent = "case when prd.thread_id is null then 0 else 1 end" + detailChangedFiles = "coalesce(prd.changed_files, 0)" + detailFetchedAt = "coalesce(prd.fetched_at, '')" + fileCount = "coalesce(prf.file_count, 0)" + oldestFileFetchedAt = "coalesce(prf.oldest_fetched_at, '')" + latestFileFetchedAt = "coalesce(prf.latest_fetched_at, '')" + } rows, err := s.q().QueryContext(ctx, ` - select case when reservation.thread_id is null then 0 else 1 end, - coalesce(reservation.source_updated_at, ''), - coalesce(reservation.observation_sequence, 0), + select `+reservationPresent+`, + `+reservationSourceUpdatedAt+`, + `+reservationObservationSequence+`, + `+detailPresent+`, + `+detailChangedFiles+`, + `+detailFetchedAt+`, + `+fileCount+`, + `+oldestFileFetchedAt+`, + `+latestFileFetchedAt+`, `+acceptedSourceUpdatedAt+`, `+acceptedObservationSequence+` from threads t - left join thread_child_observation_reservations reservation - on reservation.thread_id = t.id - and reservation.family = 'pull_request_files' + `+reservationJoin+` + `+legacyJoin+` where t.repo_id = ? and t.kind = 'pull_request' `, repoID) if err != nil { @@ -667,34 +725,51 @@ func (s *Store) archivePRFileCoverage( metric := EnrichmentCoverageMetric{Supported: true} var latestObservedAt time.Time for rows.Next() { - var hasReservation int + var hasReservation, hasDetail, changedFiles, files int var reservationSequence, acceptedSequence int64 - var reservationSourceUpdatedAt, acceptedSource string + var reservationSourceUpdatedAt, detailFetchedAt, oldestFileFetchedAt string + var latestFileFetchedAt, acceptedSource string if err := rows.Scan( &hasReservation, &reservationSourceUpdatedAt, &reservationSequence, + &hasDetail, + &changedFiles, + &detailFetchedAt, + &files, + &oldestFileFetchedAt, + &latestFileFetchedAt, &acceptedSource, &acceptedSequence, ); err != nil { return EnrichmentCoverageMetric{}, fmt.Errorf("scan archive PR file coverage: %w", err) } metric.Eligible++ - if hasReservation == 0 { - continue - } - metric.Covered++ - if observedAt, ok := parseArchiveCoverageTimestamp(reservationSourceUpdatedAt); ok && - (latestObservedAt.IsZero() || observedAt.After(latestObservedAt)) { - latestObservedAt = observedAt - } - if archiveObservationAtOrAfter( - reservationSourceUpdatedAt, - reservationSequence, - acceptedSource, - observationSequenceOrderValue(acceptedSequence), - ) { - metric.Fresh++ + switch { + case hasReservation != 0: + metric.Covered++ + latestObservedAt = laterArchiveCoverageTimestamp( + latestObservedAt, + reservationSourceUpdatedAt, + ) + if archiveObservationAtOrAfter( + reservationSourceUpdatedAt, + reservationSequence, + acceptedSource, + observationSequenceOrderValue(acceptedSequence), + ) { + metric.Fresh++ + } + case hasDetail != 0 && changedFiles >= 0 && files == changedFiles: + metric.Covered++ + latestObservedAt = laterArchiveCoverageTimestamp(latestObservedAt, detailFetchedAt) + latestObservedAt = laterArchiveCoverageTimestamp(latestObservedAt, latestFileFetchedAt) + detailFresh := archiveCoverageTimestampAtOrAfter(detailFetchedAt, acceptedSource) + filesFresh := files == 0 || + archiveCoverageTimestampAtOrAfter(oldestFileFetchedAt, acceptedSource) + if detailFresh && filesFresh { + metric.Fresh++ + } } } if err := rows.Err(); err != nil { @@ -707,6 +782,14 @@ func (s *Store) archivePRFileCoverage( return metric, nil } +func laterArchiveCoverageTimestamp(latest time.Time, candidate string) time.Time { + parsed, ok := parseArchiveCoverageTimestamp(candidate) + if ok && (latest.IsZero() || parsed.After(latest)) { + return parsed + } + return latest +} + func (s *Store) archiveAcceptedThreadObservationExpressions( ctx context.Context, alias string, diff --git a/internal/store/archive_coverage_test.go b/internal/store/archive_coverage_test.go index a7d36d2..e14aa7f 100644 --- a/internal/store/archive_coverage_test.go +++ b/internal/store/archive_coverage_test.go @@ -286,6 +286,97 @@ func TestArchiveCoveragePRFilesRequireCurrentObservation(t *testing.T) { assertMetric(1, 0, 0, 1, false) } +func TestArchiveCoveragePRFilesUsesLegacySnapshotProof(t *testing.T) { + ctx := context.Background() + st, err := Open(ctx, filepath.Join(t.TempDir(), "gitcrawl.db")) + if err != nil { + t.Fatalf("open store: %v", err) + } + defer st.Close() + + repoID, err := st.UpsertRepository(ctx, Repository{ + Owner: "openclaw", Name: "gitcrawl", FullName: "openclaw/gitcrawl", + RawJSON: "{}", UpdatedAt: "2026-07-12T12:00:00Z", + }) + if err != nil { + t.Fatalf("repository: %v", err) + } + thread := archiveCoverageThread(repoID, 1, "pull_request") + thread.UpdatedAtGitHub = "2026-07-12T12:00:00Z" + thread.RawJSON = `{"version":1}` + thread.ContentHash = "version-1" + initial, err := st.UpsertThreadObservation(ctx, thread, UpsertThreadOptions{ + ObservationSequence: 1, + }) + if err != nil { + t.Fatalf("initial thread: %v", err) + } + thread.ID = initial.ID + if _, err := st.DB().ExecContext(ctx, `drop table thread_child_observation_reservations`); err != nil { + t.Fatalf("drop reservation table: %v", err) + } + + upsertFiles := func(changedFiles int, files []PullRequestFile) { + t.Helper() + if err := st.UpsertPullRequestCacheFamilies(ctx, PullRequestDetail{ + ThreadID: thread.ID, + RepoID: repoID, + Number: thread.Number, + ChangedFiles: changedFiles, + RawJSON: "{}", + FetchedAt: "2026-07-12T12:05:00Z", + UpdatedAt: "2026-07-12T12:05:00Z", + }, files, nil, nil, nil, PullRequestHydrationFamilies{ + Details: true, + Files: true, + }); err != nil { + t.Fatalf("PR cache: %v", err) + } + } + assertMetric := func(wantCovered, wantFresh int, wantComplete bool) { + t.Helper() + coverage, err := st.ArchiveCoverage(ctx, ArchiveCoverageOptions{}) + if err != nil { + t.Fatalf("archive coverage: %v", err) + } + metric := coverage.Rows[0].Enrichment.PRFiles + if !metric.Supported || metric.Eligible != 1 || + metric.Covered != wantCovered || metric.Fresh != wantFresh || + metric.Complete != wantComplete { + t.Fatalf("legacy PR file coverage = %+v", metric) + } + } + + upsertFiles(0, nil) + assertMetric(1, 1, true) + + upsertFiles(2, []PullRequestFile{{ + ThreadID: thread.ID, + Path: "README.md", + RawJSON: "{}", + FetchedAt: "2026-07-12T12:05:00Z", + }}) + assertMetric(0, 0, false) + + upsertFiles(1, []PullRequestFile{{ + ThreadID: thread.ID, + Path: "README.md", + RawJSON: "{}", + FetchedAt: "2026-07-12T12:05:00Z", + }}) + assertMetric(1, 1, true) + + thread.UpdatedAtGitHub = "2026-07-12T12:06:00Z" + thread.RawJSON = `{"version":2}` + thread.ContentHash = "version-2" + if accepted, err := st.UpsertThreadObservation(ctx, thread, UpsertThreadOptions{ + ObservationSequence: 2, + }); err != nil || !accepted.EvidenceApplied { + t.Fatalf("newer accepted evidence = %+v, %v", accepted, err) + } + assertMetric(1, 0, false) +} + func TestArchiveObservationFreshnessUsesSequenceForEqualSource(t *testing.T) { const source = "2026-07-12T12:00:00Z" if archiveObservationAtOrAfter(source, 10, source, 11) { From d1d6bc6ace8fb79163914d258f0e224b2d0e028b Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:21:27 +0200 Subject: [PATCH 12/52] fix(cloud): validate the full publish profile --- internal/cli/app_test.go | 260 +++++++++++++++++++++++++++++++++ internal/cli/cloud_commands.go | 144 ++++++++++++++++-- 2 files changed, 392 insertions(+), 12 deletions(-) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index cfc1eb7..1f86d80 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -13,6 +13,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "slices" "sort" "strconv" "strings" @@ -829,8 +830,44 @@ func testSnapshotPublishContract() crawlremote.Contract { contract := crawlremote.BaseContract() contract.Apps = []crawlremote.AppSpec{{ App: "gitcrawl", + IngestTables: []crawlremote.IngestTableSpec{ + {Name: "repositories", Columns: gitcrawlRepositoryColumns}, + {Name: "threads", Columns: gitcrawlThreadColumns}, + {Name: "thread_revisions", Columns: []string{ + "id", "thread_id", "source_updated_at", "content_hash", "title_hash", + "body_hash", "labels_hash", "created_at", + }}, + {Name: "thread_fingerprints", Columns: []string{ + "id", "thread_revision_id", "algorithm_version", "fingerprint_hash", + "fingerprint_slug", "body_token_hash", "file_set_hash", "simhash64", "created_at", + }}, + {Name: "thread_key_summaries", Columns: []string{ + "id", "thread_revision_id", "summary_kind", "prompt_version", "provider", + "model", "input_hash", "output_hash", "key_text", "created_at", + }}, + {Name: "cluster_groups", Columns: []string{ + "id", "repo_id", "stable_key", "stable_slug", "status", "cluster_type", + "representative_thread_id", "title", "member_count", "created_at", "updated_at", + "closed_at", + }}, + {Name: "cluster_memberships", Columns: []string{ + "cluster_id", "thread_id", "role", "state", "score_to_representative", + "created_at", "updated_at", "removed_at", + }}, + {Name: "pull_request_details", Columns: []string{ + "thread_id", "repo_id", "number", "base_sha", "head_sha", "head_ref", + "head_repo_full_name", "mergeable_state", "additions", "deletions", + "changed_files", "fetched_at", "updated_at", + }}, + {Name: "pull_request_files", Columns: []string{ + "thread_id", "position", "path", "status", "additions", "deletions", + "changes", "previous_path", "fetched_at", + }}, + {Name: "dataset_coverage", Columns: gitcrawlCloudCoverageColumns}, + }, Capabilities: []string{ gitcrawlSnapshotAtomicCapability, + gitcrawlSnapshotCutoverCapability, gitcrawlSnapshotProvenanceCapability, sqliteBundleGzipUploadCapability, }, @@ -838,6 +875,66 @@ func testSnapshotPublishContract() crawlremote.Contract { return contract } +func TestGitcrawlSnapshotStatusMatchesExactMetadata(t *testing.T) { + snapshotID := strings.Repeat("a", 64) + manifest := crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__openclaw", + SchemaName: gitcrawlCloudSchemaName, + SchemaVersion: gitcrawlCloudSchemaVersion, + SchemaHash: gitcrawlCloudSchemaHash, + SnapshotID: snapshotID, + SourceSHA256: snapshotID, + } + status := crawlremote.Status{ + App: "gitcrawl", + Archive: manifest.Archive, + ActiveSnapshotID: snapshotID, + CoverageComplete: true, + Snapshot: &crawlremote.ArchiveSnapshot{ + ID: snapshotID, + SourceSHA256: snapshotID, + SchemaName: gitcrawlCloudSchemaName, + SchemaVersion: gitcrawlCloudSchemaVersion, + SchemaHash: gitcrawlCloudSchemaHash, + CoverageComplete: true, + }, + } + if !gitcrawlSnapshotStatusMatches(status, manifest) { + t.Fatal("exact active snapshot did not match") + } + + sourceMismatch := status + sourceMismatch.Snapshot = &crawlremote.ArchiveSnapshot{} + *sourceMismatch.Snapshot = *status.Snapshot + sourceMismatch.Snapshot.SourceSHA256 = strings.Repeat("b", 64) + if gitcrawlSnapshotStatusMatches(sourceMismatch, manifest) { + t.Fatal("source digest mismatch was reused") + } + + schemaMismatch := status + schemaMismatch.Snapshot = &crawlremote.ArchiveSnapshot{} + *schemaMismatch.Snapshot = *status.Snapshot + schemaMismatch.Snapshot.SchemaVersion++ + if gitcrawlSnapshotStatusMatches(schemaMismatch, manifest) { + t.Fatal("schema mismatch was reused") + } + + manifest.Capabilities = []string{gitcrawlObservationOrderCapability} + if gitcrawlSnapshotStatusMatches(status, manifest) { + t.Fatal("staged snapshot reused without observable capability proof") + } + status.SnapshotCutoverAt = "2026-07-12T12:00:00Z" + status.Capabilities = []string{gitcrawlObservationOrderCapability} + if !gitcrawlSnapshotStatusMatches(status, manifest) { + t.Fatal("cut-over snapshot with requested capability did not match") + } + status.Capabilities = nil + if gitcrawlSnapshotStatusMatches(status, manifest) { + t.Fatal("snapshot reused without requested capability") + } +} + func TestCloudPublishSendsLocalRows(t *testing.T) { ctx := context.Background() dir := t.TempDir() @@ -1134,6 +1231,166 @@ func TestCloudPublishRejectsMissingSnapshotCapabilityBeforeUpload(t *testing.T) } } +func TestCloudPublishRejectsMissingRequestedCapabilityBeforeUpload(t *testing.T) { + tests := []struct { + name string + args []string + missingCapability string + }{ + { + name: "observation order", + args: []string{"--observation-order"}, + missingCapability: gitcrawlObservationOrderCapability, + }, + { + name: "cutover", + args: []string{"--cutover"}, + missingCapability: gitcrawlSnapshotCutoverCapability, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + cfg := config.Default() + cfg.DBPath = dbPath + if err := config.Save(configPath, cfg); err != nil { + t.Fatalf("save config: %v", err) + } + seedCommandFlowStore(t, dbPath) + + const tokenEnv = "GITCRAWL_TEST_OPTION_CONTRACT_TOKEN" + t.Setenv(tokenEnv, "publish-token") + mutations := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { + contract := testSnapshotPublishContract() + capabilities := make([]string, 0, len(contract.Apps[0].Capabilities)) + for _, capability := range contract.Apps[0].Capabilities { + if capability != test.missingCapability { + capabilities = append(capabilities, capability) + } + } + contract.Apps[0].Capabilities = capabilities + w.Header().Set("content-type", "application/json") + _ = json.NewEncoder(w).Encode(contract) + return + } + mutations++ + http.Error(w, "unexpected mutation", http.StatusInternalServerError) + })) + defer server.Close() + + args := []string{ + "--config", configPath, + "cloud", "publish", + "--remote", server.URL, + "--archive", "gitcrawl/openclaw__openclaw", + "--token-env", tokenEnv, + "--allow-incomplete", + "--json", + } + args = append(args, test.args...) + err := New().Run(ctx, args) + if err == nil || !strings.Contains(err.Error(), test.missingCapability) { + t.Fatalf("cloud publish error = %v", err) + } + if mutations != 0 { + t.Fatalf("remote mutations = %d, want 0", mutations) + } + }) + } +} + +func TestCloudPublishRejectsIncompleteRemoteSurfaceBeforeUpload(t *testing.T) { + tests := []struct { + name string + want string + mutate func(*crawlremote.Contract) + extraArgs []string + }{ + { + name: "missing ingest column", + want: "pull_request_files is missing required column position", + mutate: func(contract *crawlremote.Contract) { + for tableIndex := range contract.Apps[0].IngestTables { + table := &contract.Apps[0].IngestTables[tableIndex] + if table.Name == "pull_request_files" { + table.Columns = slices.DeleteFunc( + table.Columns, + func(column string) bool { return column == "position" }, + ) + } + } + }, + }, + { + name: "missing cutover route", + want: "POST /v1/apps/:app/archives/:archive/cutover", + extraArgs: []string{"--cutover"}, + mutate: func(contract *crawlremote.Contract) { + contract.Routes = slices.DeleteFunc( + contract.Routes, + func(route crawlremote.RouteSpec) bool { + return route.Method == http.MethodPost && + route.Path == "/v1/apps/:app/archives/:archive/cutover" + }, + ) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + cfg := config.Default() + cfg.DBPath = dbPath + if err := config.Save(configPath, cfg); err != nil { + t.Fatalf("save config: %v", err) + } + seedCommandFlowStore(t, dbPath) + + const tokenEnv = "GITCRAWL_TEST_SURFACE_CONTRACT_TOKEN" + t.Setenv(tokenEnv, "publish-token") + mutations := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { + contract := testSnapshotPublishContract() + test.mutate(&contract) + w.Header().Set("content-type", "application/json") + _ = json.NewEncoder(w).Encode(contract) + return + } + mutations++ + http.Error(w, "unexpected mutation", http.StatusInternalServerError) + })) + defer server.Close() + + args := []string{ + "--config", configPath, + "cloud", "publish", + "--remote", server.URL, + "--archive", "gitcrawl/openclaw__openclaw", + "--token-env", tokenEnv, + "--allow-incomplete", + "--json", + } + args = append(args, test.extraArgs...) + err := New().Run(ctx, args) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("cloud publish error = %v", err) + } + if mutations != 0 { + t.Fatalf("remote mutations = %d, want 0", mutations) + } + }) + } +} + func TestCloudPublishStagesByDefaultAndResumesCutover(t *testing.T) { ctx := context.Background() dir := t.TempDir() @@ -1219,6 +1476,9 @@ func TestCloudPublishStagesByDefaultAndResumesCutover(t *testing.T) { Snapshot: &crawlremote.ArchiveSnapshot{ ID: snapshotID, SourceSHA256: snapshotID, + SchemaName: gitcrawlCloudSchemaName, + SchemaVersion: gitcrawlCloudSchemaVersion, + SchemaHash: gitcrawlCloudSchemaHash, CoverageComplete: true, }, }) diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index e4f62b0..7aefc9a 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -24,10 +24,16 @@ const ( gitcrawlCloudSQLiteBundleChunkSize = int64(64 * 1024 * 1024) gitcrawlSnapshotAtomicCapability = "gitcrawl.snapshot.atomic" + gitcrawlSnapshotCutoverCapability = "gitcrawl.snapshot.cutover" gitcrawlSnapshotProvenanceCapability = "gitcrawl.snapshot.provenance.v1" sqliteBundleGzipUploadCapability = "sqlite.bundle.gzip.upload" ) +var gitcrawlCloudCoverageColumns = []string{ + "dataset", "row_count", "eligible_count", "covered_count", + "max_source_at", "dataset_generated_at", "complete", "mutation_token", +} + func (a *App) runCloud(ctx context.Context, args []string) error { if len(args) == 0 { return usageErr(fmt.Errorf("cloud requires a subcommand")) @@ -112,7 +118,12 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { } manifest := gitcrawlCloudManifest(archiveID, snapshot) counts := gitcrawlCloudDatasetCounts(snapshot) - if err := requireGitcrawlSnapshotPublishContract(ctx, client); err != nil { + if err := requireGitcrawlSnapshotPublishContract( + ctx, + client, + snapshot, + *cutover, + ); err != nil { return err } sqliteBundle, err := uploadSQLiteSnapshotArchive( @@ -130,7 +141,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { alreadyActive := false status, statusErr := client.Status(ctx, "gitcrawl", archiveID) if statusErr == nil { - alreadyActive = status.ActiveSnapshotID == snapshot.ID && status.CoverageComplete + alreadyActive = gitcrawlSnapshotStatusMatches(status, manifest) } else if !remoteNotFound(statusErr) { return statusErr } @@ -163,10 +174,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { archiveID, manifest, "dataset_coverage", - []string{ - "dataset", "row_count", "eligible_count", "covered_count", - "max_source_at", "dataset_generated_at", "complete", "mutation_token", - }, + gitcrawlCloudCoverageColumns, coverageRows, mutationToken, true, @@ -429,6 +437,8 @@ func remoteNotFound(err error) bool { func requireGitcrawlSnapshotPublishContract( ctx context.Context, client *crawlremote.Client, + snapshot gitcrawlCloudSnapshot, + cutover bool, ) error { contract, err := client.Contract(ctx) if err != nil { @@ -437,28 +447,138 @@ func requireGitcrawlSnapshotPublishContract( if err := contract.Validate(); err != nil { return fmt.Errorf("validate remote snapshot publish contract: %w", err) } - var capabilities []string - for _, app := range contract.Apps { + var appSpec *crawlremote.AppSpec + for index := range contract.Apps { + app := &contract.Apps[index] if app.App == "gitcrawl" { - capabilities = app.Capabilities + appSpec = app break } } - for _, capability := range []string{ + if appSpec == nil { + return fmt.Errorf("remote contract does not advertise the gitcrawl app") + } + requiredCapabilities := []string{ gitcrawlSnapshotAtomicCapability, gitcrawlSnapshotProvenanceCapability, sqliteBundleGzipUploadCapability, - } { - if !slices.Contains(capabilities, capability) { + } + requiredCapabilities = append(requiredCapabilities, snapshot.Capabilities...) + if cutover { + requiredCapabilities = append( + requiredCapabilities, + gitcrawlSnapshotCutoverCapability, + ) + } + for _, capability := range requiredCapabilities { + if !slices.Contains(appSpec.Capabilities, capability) { return fmt.Errorf( "remote does not advertise required snapshot publish capability %s", capability, ) } } + requiredRoutes := []crawlremote.RouteSpec{ + { + Method: http.MethodGet, + Path: "/v1/apps/:app/archives/:archive/status", + Auth: crawlremote.AuthReader, + }, + { + Method: http.MethodPost, + Path: "/v1/apps/:app/archives/:archive/ingest", + Auth: crawlremote.AuthPublisher, + }, + { + Method: http.MethodPut, + Path: "/v1/apps/:app/archives/:archive/sqlite", + Auth: crawlremote.AuthPublisher, + }, + } + if cutover { + requiredRoutes = append(requiredRoutes, crawlremote.RouteSpec{ + Method: http.MethodPost, + Path: "/v1/apps/:app/archives/:archive/cutover", + Auth: crawlremote.AuthPublisher, + }) + } + for _, required := range requiredRoutes { + if !slices.ContainsFunc(contract.Routes, func(route crawlremote.RouteSpec) bool { + return route == required + }) { + return fmt.Errorf( + "remote contract does not advertise required snapshot publish route %s %s with %s auth", + required.Method, + required.Path, + required.Auth, + ) + } + } + requiredTables := make([]crawlremote.IngestTableSpec, 0, len(snapshot.Datasets)+1) + for _, dataset := range snapshot.Datasets { + requiredTables = append(requiredTables, crawlremote.IngestTableSpec{ + Name: dataset.Name, + Columns: dataset.Columns, + }) + } + requiredTables = append(requiredTables, crawlremote.IngestTableSpec{ + Name: "dataset_coverage", + Columns: gitcrawlCloudCoverageColumns, + }) + for _, required := range requiredTables { + tableIndex := slices.IndexFunc(appSpec.IngestTables, func(table crawlremote.IngestTableSpec) bool { + return table.Name == required.Name + }) + if tableIndex < 0 { + return fmt.Errorf( + "remote contract does not advertise required snapshot ingest table %s", + required.Name, + ) + } + remoteColumns := appSpec.IngestTables[tableIndex].Columns + for _, column := range required.Columns { + if !slices.Contains(remoteColumns, column) { + return fmt.Errorf( + "remote contract snapshot ingest table %s is missing required column %s", + required.Name, + column, + ) + } + } + } return nil } +func gitcrawlSnapshotStatusMatches( + status crawlremote.Status, + manifest crawlremote.IngestManifest, +) bool { + snapshot := status.Snapshot + if snapshot == nil || + status.ActiveSnapshotID != manifest.SnapshotID || + !status.CoverageComplete || + snapshot.ID != manifest.SnapshotID || + snapshot.SourceSHA256 != manifest.SourceSHA256 || + snapshot.SchemaName != manifest.SchemaName || + snapshot.SchemaVersion != manifest.SchemaVersion || + snapshot.SchemaHash != manifest.SchemaHash || + !snapshot.CoverageComplete { + return false + } + if len(manifest.Capabilities) == 0 { + return true + } + if strings.TrimSpace(status.SnapshotCutoverAt) == "" { + return false + } + for _, capability := range manifest.Capabilities { + if !slices.Contains(status.Capabilities, capability) { + return false + } + } + return true +} + func gitcrawlCloudSQLiteBundlePrivacy() map[string]any { return map[string]any{ "includes_private_messages": true, From 918c0bc7a579d2d1277e6a42cee8476362841b27 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:21:29 +0200 Subject: [PATCH 13/52] chore(changelog): preserve release-owned heading --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff7a6e5..99964c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## 0.7.2 - Unreleased + ## 0.7.1 - 2026-07-09 - Add local, fail-closed packaging and independent verification for official macOS archives signed as `org.openclaw.gitcrawl` by `Developer ID Application: OpenClaw Foundation (FWJYW4S8P8)`, while keeping CI and cross-platform snapshots credential-free and non-publishing. From e67bc9c47ae1fb328ba77068161abdd93809050e Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:31:44 +0200 Subject: [PATCH 14/52] fix(cloud): preserve default reader cutover --- README.md | 13 +++++++------ internal/cli/app.go | 2 +- internal/cli/app_test.go | 17 +++++++---------- internal/cli/cloud_commands.go | 7 ++++--- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 5f338f6..17c4683 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ gitcrawl remote login --endpoint https://crawl.openclaw.ai --json gitcrawl remote status --json gitcrawl remote archives --json gitcrawl whoami --json -gitcrawl cloud publish --remote https://crawl.openclaw.ai --archive gitcrawl/openclaw__openclaw --cutover --json +gitcrawl cloud publish --remote https://crawl.openclaw.ai --archive gitcrawl/openclaw__openclaw --json gitcrawl sync owner/repo gitcrawl sync owner/repo --state open gitcrawl sync owner/repo --numbers 123,456 --include-comments @@ -72,11 +72,12 @@ SHA-256 as the snapshot identity, exports repositories, threads, revisions, fingerprints, summaries, durable clusters, and PR detail/file rows from that same image, negotiates the remote snapshot-provenance contract before touching R2, uploads its digest-scoped bundle, and activates complete D1 coverage. -Publishing stages the immutable snapshot by default; `--cutover` explicitly -moves unpinned reads to that snapshot and disables legacy ingest. Incomplete -local enrichment fails before any remote mutation; `--allow-incomplete` is an -explicit escape hatch, and `--observation-order` publishes durable fetch -ordering after the remote operator fence is enabled. +Publishing moves unpinned reads to the complete snapshot by default, preserving +the existing reader-refresh behavior. `--stage-only` keeps the immutable +snapshot staged for a later publish to resume and cut over. Incomplete local +enrichment fails before any remote mutation; `--allow-incomplete` is an explicit +escape hatch, and `--observation-order` publishes durable fetch ordering after +the remote operator fence is enabled. `gitcrawl clusters-report` writes a Markdown report for the top clusters using the same display view, with an at-a-glance table, per-cluster metadata, member tables, and key snippets. Use `--json` for the hydrated report payload. `gitcrawl cluster` and `gitcrawl refresh` build ghcrawl-shaped durable clusters by default (`--threshold 0.80`, `--min-size 1`, `--max-cluster-size 40`, `--k 16`, `--cross-kind-threshold 0.93`): every active vector-backed thread is represented, singleton rows use `singleton_orphan`, multi-member rows use `duplicate_candidate`, and stable IDs are derived from the representative thread. They also add deterministic GitHub reference evidence for direct issue/PR links such as `#123`, `issues/123`, and `pull/123`. Weak embedding edges need concrete title-token overlap unless their similarity is already high, which keeps generic low-confidence bridges from forming unrelated clusters. `gitcrawl tui` infers the most recently updated local repository when `owner/repo` is omitted. `serve` is intentionally not part of `gitcrawl`. diff --git a/internal/cli/app.go b/internal/cli/app.go index 69fa3e2..76ce7ec 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -4770,7 +4770,7 @@ Usage: "cloud": `gitcrawl cloud manages Worker-backed remote archives. Usage: - gitcrawl cloud publish --remote URL --archive id [--allow-incomplete] [--observation-order] [--cutover] [--json] + gitcrawl cloud publish --remote URL --archive id [--allow-incomplete] [--observation-order] [--stage-only] [--json] `, "whoami": `gitcrawl whoami prints the configured remote archive identity. diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 1f86d80..311e3a4 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -1136,7 +1136,6 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { "--archive", "gitcrawl/openclaw__openclaw", "--token-env", tokenEnv, "--allow-incomplete", - "--cutover", "--json", }); err != nil { t.Fatalf("cloud publish: %v", err) @@ -1239,12 +1238,11 @@ func TestCloudPublishRejectsMissingRequestedCapabilityBeforeUpload(t *testing.T) }{ { name: "observation order", - args: []string{"--observation-order"}, + args: []string{"--observation-order", "--stage-only"}, missingCapability: gitcrawlObservationOrderCapability, }, { name: "cutover", - args: []string{"--cutover"}, missingCapability: gitcrawlSnapshotCutoverCapability, }, } @@ -1327,9 +1325,8 @@ func TestCloudPublishRejectsIncompleteRemoteSurfaceBeforeUpload(t *testing.T) { }, }, { - name: "missing cutover route", - want: "POST /v1/apps/:app/archives/:archive/cutover", - extraArgs: []string{"--cutover"}, + name: "missing cutover route", + want: "POST /v1/apps/:app/archives/:archive/cutover", mutate: func(contract *crawlremote.Contract) { contract.Routes = slices.DeleteFunc( contract.Routes, @@ -1391,7 +1388,7 @@ func TestCloudPublishRejectsIncompleteRemoteSurfaceBeforeUpload(t *testing.T) { } } -func TestCloudPublishStagesByDefaultAndResumesCutover(t *testing.T) { +func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { ctx := context.Background() dir := t.TempDir() configPath := filepath.Join(dir, "config.toml") @@ -1520,7 +1517,7 @@ func TestCloudPublishStagesByDefaultAndResumesCutover(t *testing.T) { "--allow-incomplete", "--json", } - for index, extra := range [][]string{nil, {"--cutover"}} { + for index, extra := range [][]string{{"--stage-only"}, nil} { app := New() var output bytes.Buffer app.Stdout = &output @@ -1535,10 +1532,10 @@ func TestCloudPublishStagesByDefaultAndResumesCutover(t *testing.T) { t.Fatalf("decode run %d output: %v\n%s", index+1, err, output.String()) } if index == 0 && result.Cutover != nil { - t.Fatalf("default publish unexpectedly cut over: %#v", result.Cutover) + t.Fatalf("stage-only publish unexpectedly cut over: %#v", result.Cutover) } if index == 1 && result.Cutover == nil { - t.Fatalf("explicit cutover missing: %s", output.String()) + t.Fatalf("default cutover missing: %s", output.String()) } } if ingestRequests != 0 { diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 7aefc9a..ac083ff 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -54,7 +54,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { tokenEnv := fs.String("token-env", "", "remote token environment variable") allowIncomplete := fs.Bool("allow-incomplete", false, "publish even when local enrichment coverage is incomplete") observationOrder := fs.Bool("observation-order", false, "publish durable observation ordering when the remote fence is enabled") - cutover := fs.Bool("cutover", false, "cut over unpinned reads after activating the snapshot") + stageOnly := fs.Bool("stage-only", false, "stage the immutable snapshot without moving unpinned reads") jsonOut := fs.Bool("json", false, "write JSON output") if err := fs.Parse(normalizeCommandArgs(args, map[string]bool{"remote": true, "archive": true, "token-env": true})); err != nil { return usageErr(err) @@ -63,6 +63,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { if fs.NArg() != 0 { return usageErr(fmt.Errorf("cloud publish takes flags only")) } + cutover := !*stageOnly cfg, err := config.LoadRuntime(a.configPath) if err != nil { @@ -122,7 +123,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { ctx, client, snapshot, - *cutover, + cutover, ); err != nil { return err } @@ -185,7 +186,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { mutationToken = progress.MutationToken } var cutoverResult *crawlremote.CutoverResult - if *cutover { + if cutover { result, err := client.Cutover(ctx, "gitcrawl", archiveID, snapshot.ID) if err != nil { return fmt.Errorf("cut over cloud snapshot: %w", err) From a4b3fb41eca09873f72ce27c7ff7eae52e024d6c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:36:37 +0200 Subject: [PATCH 15/52] fix(cloud): resume staged capability snapshots --- go.mod | 2 +- go.sum | 4 ++-- internal/cli/app_test.go | 25 +++++++++++++++++++------ internal/cli/cloud_commands.go | 5 +---- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 0f72fbe..ee327a9 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 github.com/charmbracelet/x/ansi v0.11.7 github.com/mattn/go-isatty v0.0.22 - github.com/openclaw/crawlkit v0.13.5-0.20260712105859-a51e1868fb75 + github.com/openclaw/crawlkit v0.13.5-0.20260712113439-7aba6ba037e1 github.com/zalando/go-keyring v0.2.8 golang.org/x/sys v0.46.0 modernc.org/sqlite v1.53.0 diff --git a/go.sum b/go.sum index 56cf667..523d3c4 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/openclaw/crawlkit v0.13.5-0.20260712105859-a51e1868fb75 h1:Zrb8DB2rreGblEu3sVaBWfvSozrW2tD63XEN59DmOjQ= -github.com/openclaw/crawlkit v0.13.5-0.20260712105859-a51e1868fb75/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= +github.com/openclaw/crawlkit v0.13.5-0.20260712113439-7aba6ba037e1 h1:ir8m49bLLkMC0VIsJYIe4zjlDr5DBY3pYC+rcvFlyIM= +github.com/openclaw/crawlkit v0.13.5-0.20260712113439-7aba6ba037e1/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 311e3a4..3b76c5e 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -922,14 +922,14 @@ func TestGitcrawlSnapshotStatusMatchesExactMetadata(t *testing.T) { manifest.Capabilities = []string{gitcrawlObservationOrderCapability} if gitcrawlSnapshotStatusMatches(status, manifest) { - t.Fatal("staged snapshot reused without observable capability proof") + t.Fatal("staged snapshot reused without snapshot capability proof") } - status.SnapshotCutoverAt = "2026-07-12T12:00:00Z" - status.Capabilities = []string{gitcrawlObservationOrderCapability} + status.Capabilities = []string{"gitcrawl.threads.search"} + status.Snapshot.Capabilities = []string{gitcrawlObservationOrderCapability} if !gitcrawlSnapshotStatusMatches(status, manifest) { - t.Fatal("cut-over snapshot with requested capability did not match") + t.Fatal("staged snapshot with requested capability did not match") } - status.Capabilities = nil + status.Snapshot.Capabilities = nil if gitcrawlSnapshotStatusMatches(status, manifest) { t.Fatal("snapshot reused without requested capability") } @@ -1407,8 +1407,19 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { cutovers := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { + contract := testSnapshotPublishContract() + contract.Apps[0].Capabilities = append( + contract.Apps[0].Capabilities, + gitcrawlObservationOrderCapability, + ) + for index := range contract.Apps[0].IngestTables { + table := &contract.Apps[0].IngestTables[index] + if table.Name == "threads" || table.Name == "thread_revisions" { + table.Columns = append(table.Columns, "observation_sequence") + } + } w.Header().Set("content-type", "application/json") - _ = json.NewEncoder(w).Encode(testSnapshotPublishContract()) + _ = json.NewEncoder(w).Encode(contract) return } if got := r.Header.Get("authorization"); got != "Bearer publish-token" { @@ -1476,6 +1487,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { SchemaName: gitcrawlCloudSchemaName, SchemaVersion: gitcrawlCloudSchemaVersion, SchemaHash: gitcrawlCloudSchemaHash, + Capabilities: []string{gitcrawlObservationOrderCapability}, CoverageComplete: true, }, }) @@ -1515,6 +1527,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { "--archive", "gitcrawl/openclaw__openclaw", "--token-env", tokenEnv, "--allow-incomplete", + "--observation-order", "--json", } for index, extra := range [][]string{{"--stage-only"}, nil} { diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index ac083ff..21bfad9 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -569,11 +569,8 @@ func gitcrawlSnapshotStatusMatches( if len(manifest.Capabilities) == 0 { return true } - if strings.TrimSpace(status.SnapshotCutoverAt) == "" { - return false - } for _, capability := range manifest.Capabilities { - if !slices.Contains(status.Capabilities, capability) { + if !slices.Contains(snapshot.Capabilities, capability) { return false } } From a94c58c579ecd125edefac4300ef903ea0812e6f Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 13:39:18 +0200 Subject: [PATCH 16/52] build: pin crawlkit v0.14.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ee327a9..798ea25 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 github.com/charmbracelet/x/ansi v0.11.7 github.com/mattn/go-isatty v0.0.22 - github.com/openclaw/crawlkit v0.13.5-0.20260712113439-7aba6ba037e1 + github.com/openclaw/crawlkit v0.14.0 github.com/zalando/go-keyring v0.2.8 golang.org/x/sys v0.46.0 modernc.org/sqlite v1.53.0 diff --git a/go.sum b/go.sum index 523d3c4..0434ad5 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/openclaw/crawlkit v0.13.5-0.20260712113439-7aba6ba037e1 h1:ir8m49bLLkMC0VIsJYIe4zjlDr5DBY3pYC+rcvFlyIM= -github.com/openclaw/crawlkit v0.13.5-0.20260712113439-7aba6ba037e1/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= +github.com/openclaw/crawlkit v0.14.0 h1:SxhewcO9I706zQUHaSCt3Yn+F2qdrmHQ2NLI8QX0d3g= +github.com/openclaw/crawlkit v0.14.0/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= From efe13645007ac0f72ecf5cc15715b765b6d6e1cc Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:19:33 +0200 Subject: [PATCH 17/52] fix(cloud): require reader contract before publish --- internal/cli/app_test.go | 66 ++++++++++++++++++++++++++++++++-- internal/cli/cloud_commands.go | 46 ++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 3 deletions(-) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 3b76c5e..ce92a14 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -829,7 +829,8 @@ func TestRemoteCloudModeDoesNotCreateLocalDB(t *testing.T) { func testSnapshotPublishContract() crawlremote.Contract { contract := crawlremote.BaseContract() contract.Apps = []crawlremote.AppSpec{{ - App: "gitcrawl", + App: "gitcrawl", + Queries: gitcrawlCloudReaderQuerySpecs(), IngestTables: []crawlremote.IngestTableSpec{ {Name: "repositories", Columns: gitcrawlRepositoryColumns}, {Name: "threads", Columns: gitcrawlThreadColumns}, @@ -1303,12 +1304,13 @@ func TestCloudPublishRejectsMissingRequestedCapabilityBeforeUpload(t *testing.T) } func TestCloudPublishRejectsIncompleteRemoteSurfaceBeforeUpload(t *testing.T) { - tests := []struct { + type remoteSurfaceTest struct { name string want string mutate func(*crawlremote.Contract) extraArgs []string - }{ + } + tests := []remoteSurfaceTest{ { name: "missing ingest column", want: "pull_request_files is missing required column position", @@ -1337,6 +1339,64 @@ func TestCloudPublishRejectsIncompleteRemoteSurfaceBeforeUpload(t *testing.T) { ) }, }, + { + name: "missing reader query route", + want: "POST /v1/apps/:app/archives/:archive/query", + mutate: func(contract *crawlremote.Contract) { + contract.Routes = slices.DeleteFunc( + contract.Routes, + func(route crawlremote.RouteSpec) bool { + return route.Method == http.MethodPost && + route.Path == "/v1/apps/:app/archives/:archive/query" + }, + ) + }, + }, + { + name: "zero reader queries", + want: "required reader query gitcrawl.threads.search", + mutate: func(contract *crawlremote.Contract) { + contract.Apps[0].Queries = nil + }, + }, + { + name: "duplicate reader query", + want: "required reader query gitcrawl.threads.search more than once", + mutate: func(contract *crawlremote.Contract) { + contract.Apps[0].Queries = append( + contract.Apps[0].Queries, + contract.Apps[0].Queries[0], + ) + }, + }, + } + for _, required := range gitcrawlCloudReaderQuerySpecs() { + tests = append(tests, + remoteSurfaceTest{ + name: "missing reader query " + required.Name, + want: "required reader query " + required.Name, + mutate: func(contract *crawlremote.Contract) { + contract.Apps[0].Queries = slices.DeleteFunc( + contract.Apps[0].Queries, + func(query crawlremote.QuerySpec) bool { + return query.Name == required.Name + }, + ) + }, + }, + remoteSurfaceTest{ + name: "drifted reader query " + required.Name, + want: "reader query " + required.Name + " has arguments", + mutate: func(contract *crawlremote.Contract) { + for queryIndex := range contract.Apps[0].Queries { + query := &contract.Apps[0].Queries[queryIndex] + if query.Name == required.Name { + query.Args = append(slices.Clone(query.Args), "unexpected") + } + } + }, + }, + ) } for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 21bfad9..4242228 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -34,6 +34,17 @@ var gitcrawlCloudCoverageColumns = []string{ "max_source_at", "dataset_generated_at", "complete", "mutation_token", } +func gitcrawlCloudReaderQuerySpecs() []crawlremote.QuerySpec { + return []crawlremote.QuerySpec{ + {Name: "gitcrawl.threads.search", Args: []string{"owner", "repo", "query", "kind", "state"}}, + {Name: "gitcrawl.clusters.related", Args: []string{"owner", "repo", "number"}}, + {Name: "gitcrawl.clusters.list", Args: []string{"owner", "repo", "status", "min_size"}}, + {Name: "gitcrawl.clusters.members", Args: []string{"owner", "repo", "cluster_id"}}, + {Name: "gitcrawl.pull_requests.review_context", Args: []string{"owner", "repo", "number"}}, + {Name: "gitcrawl.coverage", Args: []string{"dataset"}}, + } +} + func (a *App) runCloud(ctx context.Context, args []string) error { if len(args) == 0 { return usageErr(fmt.Errorf("cloud requires a subcommand")) @@ -485,6 +496,11 @@ func requireGitcrawlSnapshotPublishContract( Path: "/v1/apps/:app/archives/:archive/status", Auth: crawlremote.AuthReader, }, + { + Method: http.MethodPost, + Path: "/v1/apps/:app/archives/:archive/query", + Auth: crawlremote.AuthReader, + }, { Method: http.MethodPost, Path: "/v1/apps/:app/archives/:archive/ingest", @@ -515,6 +531,36 @@ func requireGitcrawlSnapshotPublishContract( ) } } + for _, required := range gitcrawlCloudReaderQuerySpecs() { + queryIndex := -1 + for index, query := range appSpec.Queries { + if query.Name != required.Name { + continue + } + if queryIndex >= 0 { + return fmt.Errorf( + "remote contract advertises required reader query %s more than once", + required.Name, + ) + } + queryIndex = index + } + if queryIndex < 0 { + return fmt.Errorf( + "remote contract does not advertise required reader query %s", + required.Name, + ) + } + remoteArgs := appSpec.Queries[queryIndex].Args + if !slices.Equal(remoteArgs, required.Args) { + return fmt.Errorf( + "remote contract reader query %s has arguments %v, want %v", + required.Name, + remoteArgs, + required.Args, + ) + } + } requiredTables := make([]crawlremote.IngestTableSpec, 0, len(snapshot.Datasets)+1) for _, dataset := range snapshot.Datasets { requiredTables = append(requiredTables, crawlremote.IngestTableSpec{ From 1febffed719721a4b1f8ed2a38b87be01cea7c45 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:33:01 +0200 Subject: [PATCH 18/52] fix(cloud): align reader query contract --- internal/cli/app_test.go | 30 ++++++++++++++++++++++++------ internal/cli/cloud_commands.go | 2 +- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index ce92a14..01ebfad 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -694,7 +694,7 @@ func TestRemoteCloudModeDoesNotCreateLocalDB(t *testing.T) { t.Setenv("HOME", dir) t.Setenv("CRAWL_REMOTE_TOKEN", "test-token") - var sawQuery bool + var queryArgSets [][]string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if got := r.Header.Get("authorization"); got != "Bearer test-token" { http.Error(w, "missing bearer", http.StatusUnauthorized) @@ -723,7 +723,12 @@ func TestRemoteCloudModeDoesNotCreateLocalDB(t *testing.T) { http.Error(w, "unexpected query", http.StatusBadRequest) return } - sawQuery = true + argNames := make([]string, 0, len(req.Args)) + for name := range req.Args { + argNames = append(argNames, name) + } + sort.Strings(argNames) + queryArgSets = append(queryArgSets, argNames) _ = json.NewEncoder(w).Encode(crawlremote.QueryResult{ Values: []map[string]any{{ "thread_id": 10, @@ -790,8 +795,10 @@ func TestRemoteCloudModeDoesNotCreateLocalDB(t *testing.T) { if err := searchApp.Run(ctx, []string{"--config", configPath, "--json", "search", "openclaw/openclaw", "--query", "remote"}); err != nil { t.Fatalf("remote search: %v", err) } - if !sawQuery || !strings.Contains(searchOut.String(), `"remote search"`) { - t.Fatalf("remote search did not use worker query: saw=%v out=%s", sawQuery, searchOut.String()) + if len(queryArgSets) != 1 || + !slices.Equal(queryArgSets[0], []string{"limit", "mode", "owner", "query", "repo"}) || + !strings.Contains(searchOut.String(), `"remote search"`) { + t.Fatalf("remote search query args=%v out=%s", queryArgSets, searchOut.String()) } ghSearchApp := New() var ghSearchOut bytes.Buffer @@ -802,6 +809,10 @@ func TestRemoteCloudModeDoesNotCreateLocalDB(t *testing.T) { if !strings.Contains(ghSearchOut.String(), `"number": 42`) || !strings.Contains(ghSearchOut.String(), `"url": "https://github.com/openclaw/openclaw/issues/42"`) { t.Fatalf("remote gh search output = %s", ghSearchOut.String()) } + if len(queryArgSets) != 2 || + !slices.Equal(queryArgSets[1], []string{"kind", "limit", "owner", "query", "repo", "state"}) { + t.Fatalf("remote gh search query args=%v", queryArgSets) + } doctorApp := New() var doctorOut bytes.Buffer doctorApp.Stdout = &doctorOut @@ -829,8 +840,15 @@ func TestRemoteCloudModeDoesNotCreateLocalDB(t *testing.T) { func testSnapshotPublishContract() crawlremote.Contract { contract := crawlremote.BaseContract() contract.Apps = []crawlremote.AppSpec{{ - App: "gitcrawl", - Queries: gitcrawlCloudReaderQuerySpecs(), + App: "gitcrawl", + Queries: []crawlremote.QuerySpec{ + {Name: "gitcrawl.threads.search", Args: []string{"owner", "repo", "query", "kind", "state", "mode", "limit"}}, + {Name: "gitcrawl.clusters.related", Args: []string{"owner", "repo", "number"}}, + {Name: "gitcrawl.clusters.list", Args: []string{"owner", "repo", "status", "min_size"}}, + {Name: "gitcrawl.clusters.members", Args: []string{"owner", "repo", "cluster_id"}}, + {Name: "gitcrawl.pull_requests.review_context", Args: []string{"owner", "repo", "number"}}, + {Name: "gitcrawl.coverage", Args: []string{"dataset"}}, + }, IngestTables: []crawlremote.IngestTableSpec{ {Name: "repositories", Columns: gitcrawlRepositoryColumns}, {Name: "threads", Columns: gitcrawlThreadColumns}, diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 4242228..4c92e52 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -36,7 +36,7 @@ var gitcrawlCloudCoverageColumns = []string{ func gitcrawlCloudReaderQuerySpecs() []crawlremote.QuerySpec { return []crawlremote.QuerySpec{ - {Name: "gitcrawl.threads.search", Args: []string{"owner", "repo", "query", "kind", "state"}}, + {Name: "gitcrawl.threads.search", Args: []string{"owner", "repo", "query", "kind", "state", "mode", "limit"}}, {Name: "gitcrawl.clusters.related", Args: []string{"owner", "repo", "number"}}, {Name: "gitcrawl.clusters.list", Args: []string{"owner", "repo", "status", "min_size"}}, {Name: "gitcrawl.clusters.members", Args: []string{"owner", "repo", "cluster_id"}}, From 701fb8ffd4963a0dbd00a5210398506431f8b5e8 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 14:57:21 +0200 Subject: [PATCH 19/52] fix(cloud): resume staged publisher snapshots --- README.md | 10 +- go.mod | 2 +- go.sum | 4 +- internal/cli/app_test.go | 188 +++++++++++++++++++++++++++------ internal/cli/cloud_commands.go | 20 ++-- 5 files changed, 171 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 17c4683..5b3ba98 100644 --- a/README.md +++ b/README.md @@ -74,10 +74,12 @@ same image, negotiates the remote snapshot-provenance contract before touching R2, uploads its digest-scoped bundle, and activates complete D1 coverage. Publishing moves unpinned reads to the complete snapshot by default, preserving the existing reader-refresh behavior. `--stage-only` keeps the immutable -snapshot staged for a later publish to resume and cut over. Incomplete local -enrichment fails before any remote mutation; `--allow-incomplete` is an explicit -escape hatch, and `--observation-order` publishes durable fetch ordering after -the remote operator fence is enabled. +snapshot staged without changing serving state. A later publish verifies the +candidate through the publisher-only status projection, skips repeated ingest +when its digest, schema, capabilities, and coverage match, then cuts it over. +Incomplete local enrichment fails before any remote mutation; +`--allow-incomplete` is an explicit escape hatch, and `--observation-order` +publishes durable fetch ordering after the remote operator fence is enabled. `gitcrawl clusters-report` writes a Markdown report for the top clusters using the same display view, with an at-a-glance table, per-cluster metadata, member tables, and key snippets. Use `--json` for the hydrated report payload. `gitcrawl cluster` and `gitcrawl refresh` build ghcrawl-shaped durable clusters by default (`--threshold 0.80`, `--min-size 1`, `--max-cluster-size 40`, `--k 16`, `--cross-kind-threshold 0.93`): every active vector-backed thread is represented, singleton rows use `singleton_orphan`, multi-member rows use `duplicate_candidate`, and stable IDs are derived from the representative thread. They also add deterministic GitHub reference evidence for direct issue/PR links such as `#123`, `issues/123`, and `pull/123`. Weak embedding edges need concrete title-token overlap unless their similarity is already high, which keeps generic low-confidence bridges from forming unrelated clusters. `gitcrawl tui` infers the most recently updated local repository when `owner/repo` is omitted. `serve` is intentionally not part of `gitcrawl`. diff --git a/go.mod b/go.mod index 798ea25..74fcbcc 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 github.com/charmbracelet/x/ansi v0.11.7 github.com/mattn/go-isatty v0.0.22 - github.com/openclaw/crawlkit v0.14.0 + github.com/openclaw/crawlkit v0.14.1 github.com/zalando/go-keyring v0.2.8 golang.org/x/sys v0.46.0 modernc.org/sqlite v1.53.0 diff --git a/go.sum b/go.sum index 0434ad5..81dba97 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/openclaw/crawlkit v0.14.0 h1:SxhewcO9I706zQUHaSCt3Yn+F2qdrmHQ2NLI8QX0d3g= -github.com/openclaw/crawlkit v0.14.0/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= +github.com/openclaw/crawlkit v0.14.1 h1:B1exhgP/PZ//JCvr5asdhBZt6jZ5JwW8LHxGx0jfprc= +github.com/openclaw/crawlkit v0.14.1/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 01ebfad..a2f74f4 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -894,7 +894,7 @@ func testSnapshotPublishContract() crawlremote.Contract { return contract } -func TestGitcrawlSnapshotStatusMatchesExactMetadata(t *testing.T) { +func TestGitcrawlPublisherStatusMatchesExactMetadata(t *testing.T) { snapshotID := strings.Repeat("a", 64) manifest := crawlremote.IngestManifest{ App: "gitcrawl", @@ -905,11 +905,11 @@ func TestGitcrawlSnapshotStatusMatchesExactMetadata(t *testing.T) { SnapshotID: snapshotID, SourceSHA256: snapshotID, } - status := crawlremote.Status{ + status := crawlremote.PublisherStatus{ App: "gitcrawl", Archive: manifest.Archive, - ActiveSnapshotID: snapshotID, - CoverageComplete: true, + ActiveSnapshotID: strings.Repeat("f", 64), + CoverageComplete: false, Snapshot: &crawlremote.ArchiveSnapshot{ ID: snapshotID, SourceSHA256: snapshotID, @@ -919,15 +919,15 @@ func TestGitcrawlSnapshotStatusMatchesExactMetadata(t *testing.T) { CoverageComplete: true, }, } - if !gitcrawlSnapshotStatusMatches(status, manifest) { - t.Fatal("exact active snapshot did not match") + if !gitcrawlPublisherStatusMatches(status, manifest) { + t.Fatal("exact staged snapshot did not match") } sourceMismatch := status sourceMismatch.Snapshot = &crawlremote.ArchiveSnapshot{} *sourceMismatch.Snapshot = *status.Snapshot sourceMismatch.Snapshot.SourceSHA256 = strings.Repeat("b", 64) - if gitcrawlSnapshotStatusMatches(sourceMismatch, manifest) { + if gitcrawlPublisherStatusMatches(sourceMismatch, manifest) { t.Fatal("source digest mismatch was reused") } @@ -935,23 +935,26 @@ func TestGitcrawlSnapshotStatusMatchesExactMetadata(t *testing.T) { schemaMismatch.Snapshot = &crawlremote.ArchiveSnapshot{} *schemaMismatch.Snapshot = *status.Snapshot schemaMismatch.Snapshot.SchemaVersion++ - if gitcrawlSnapshotStatusMatches(schemaMismatch, manifest) { + if gitcrawlPublisherStatusMatches(schemaMismatch, manifest) { t.Fatal("schema mismatch was reused") } manifest.Capabilities = []string{gitcrawlObservationOrderCapability} - if gitcrawlSnapshotStatusMatches(status, manifest) { + if gitcrawlPublisherStatusMatches(status, manifest) { t.Fatal("staged snapshot reused without snapshot capability proof") } - status.Capabilities = []string{"gitcrawl.threads.search"} status.Snapshot.Capabilities = []string{gitcrawlObservationOrderCapability} - if !gitcrawlSnapshotStatusMatches(status, manifest) { + if !gitcrawlPublisherStatusMatches(status, manifest) { t.Fatal("staged snapshot with requested capability did not match") } status.Snapshot.Capabilities = nil - if gitcrawlSnapshotStatusMatches(status, manifest) { + if gitcrawlPublisherStatusMatches(status, manifest) { t.Fatal("snapshot reused without requested capability") } + status.Snapshot.CoverageComplete = false + if gitcrawlPublisherStatusMatches(status, manifest) { + t.Fatal("incomplete staged snapshot was reused") + } } func TestCloudPublishSendsLocalRows(t *testing.T) { @@ -1079,7 +1082,7 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { } return } - if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/status") { + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status") { http.NotFound(w, r) return } @@ -1357,6 +1360,32 @@ func TestCloudPublishRejectsIncompleteRemoteSurfaceBeforeUpload(t *testing.T) { ) }, }, + { + name: "missing publisher status route", + want: "GET /v1/apps/:app/archives/:archive/publish-status", + mutate: func(contract *crawlremote.Contract) { + contract.Routes = slices.DeleteFunc( + contract.Routes, + func(route crawlremote.RouteSpec) bool { + return route.Method == http.MethodGet && + route.Path == "/v1/apps/:app/archives/:archive/publish-status" + }, + ) + }, + }, + { + name: "publisher status route with reader auth", + want: "GET /v1/apps/:app/archives/:archive/publish-status", + mutate: func(contract *crawlremote.Contract) { + for index := range contract.Routes { + route := &contract.Routes[index] + if route.Method == http.MethodGet && + route.Path == "/v1/apps/:app/archives/:archive/publish-status" { + route.Auth = crawlremote.AuthReader + } + } + }, + }, { name: "missing reader query route", want: "POST /v1/apps/:app/archives/:archive/query", @@ -1377,6 +1406,15 @@ func TestCloudPublishRejectsIncompleteRemoteSurfaceBeforeUpload(t *testing.T) { contract.Apps[0].Queries = nil }, }, + { + name: "thread search missing mode and limit", + want: "reader query gitcrawl.threads.search has arguments", + mutate: func(contract *crawlremote.Contract) { + contract.Apps[0].Queries[0].Args = []string{ + "owner", "repo", "query", "kind", "state", + } + }, + }, { name: "duplicate reader query", want: "required reader query gitcrawl.threads.search more than once", @@ -1481,7 +1519,11 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { const tokenEnv = "GITCRAWL_TEST_RESUME_TOKEN" t.Setenv(tokenEnv, "publish-token") var snapshotID string + var stagedSnapshot *crawlremote.ArchiveSnapshot + servingSnapshotID := strings.Repeat("f", 64) ingestRequests := 0 + publisherStatusRequests := 0 + readerStatusRequests := 0 cutovers := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { @@ -1500,10 +1542,35 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { _ = json.NewEncoder(w).Encode(contract) return } + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/status") { + readerStatusRequests++ + if r.Header.Get("authorization") != "Bearer reader-token" { + http.Error(w, "publisher token has no reader role", http.StatusForbidden) + return + } + _ = json.NewEncoder(w).Encode(crawlremote.Status{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__openclaw", + ActiveSnapshotID: servingSnapshotID, + CoverageComplete: true, + }) + return + } if got := r.Header.Get("authorization"); got != "Bearer publish-token" { http.Error(w, "missing bearer", http.StatusUnauthorized) return } + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status") { + publisherStatusRequests++ + _ = json.NewEncoder(w).Encode(crawlremote.PublisherStatus{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__openclaw", + ActiveSnapshotID: servingSnapshotID, + CoverageComplete: servingSnapshotID == snapshotID, + Snapshot: stagedSnapshot, + }) + return + } if r.Method == http.MethodPut && strings.HasSuffix(r.URL.EscapedPath(), "/sqlite") { w.Header().Set("content-type", "application/json") switch r.Header.Get("x-crawl-sqlite-upload") { @@ -1553,27 +1620,42 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { } return } - if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/status") { - _ = json.NewEncoder(w).Encode(crawlremote.Status{ - App: "gitcrawl", - Archive: "gitcrawl/openclaw__openclaw", - ActiveSnapshotID: snapshotID, - CoverageComplete: true, - Snapshot: &crawlremote.ArchiveSnapshot{ - ID: snapshotID, - SourceSHA256: snapshotID, - SchemaName: gitcrawlCloudSchemaName, - SchemaVersion: gitcrawlCloudSchemaVersion, - SchemaHash: gitcrawlCloudSchemaHash, - Capabilities: []string{gitcrawlObservationOrderCapability}, - CoverageComplete: true, - }, - }) - return - } if r.Method == http.MethodPost && strings.HasSuffix(r.URL.EscapedPath(), "/ingest") { ingestRequests++ - http.Error(w, "activated snapshots must not be re-ingested", http.StatusConflict) + var body crawlremote.IngestRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if body.Manifest.SnapshotID != snapshotID || + body.Manifest.SourceSHA256 != snapshotID { + http.Error(w, "ingest provenance mismatch", http.StatusBadRequest) + return + } + mutationToken := fmt.Sprintf("resume-mutation-%d", ingestRequests) + if body.Final { + if body.Table != "dataset_coverage" || body.MutationToken == "" { + http.Error(w, "invalid snapshot activation", http.StatusBadRequest) + return + } + mutationToken = body.MutationToken + stagedSnapshot = &crawlremote.ArchiveSnapshot{ + ID: body.Manifest.SnapshotID, + SourceSHA256: body.Manifest.SourceSHA256, + SchemaName: body.Manifest.SchemaName, + SchemaVersion: body.Manifest.SchemaVersion, + SchemaHash: body.Manifest.SchemaHash, + Capabilities: slices.Clone(body.Manifest.Capabilities), + CoverageComplete: true, + } + } + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + Table: body.Table, + SnapshotID: body.Manifest.SnapshotID, + MutationToken: mutationToken, + RowsAccepted: int64(len(body.Rows)), + Complete: body.Final, + }) return } if r.Method == http.MethodPost && strings.HasSuffix(r.URL.EscapedPath(), "/cutover") { @@ -1585,7 +1667,12 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { http.Error(w, "cutover snapshot mismatch", http.StatusBadRequest) return } + if stagedSnapshot == nil || stagedSnapshot.ID != request.SnapshotID { + http.Error(w, "snapshot is not staged", http.StatusConflict) + return + } cutovers++ + servingSnapshotID = request.SnapshotID _ = json.NewEncoder(w).Encode(crawlremote.CutoverResult{ Archive: "gitcrawl/openclaw__openclaw", SnapshotID: snapshotID, @@ -1608,6 +1695,8 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { "--observation-order", "--json", } + stageIngestRequests := 0 + originalServingSnapshotID := servingSnapshotID for index, extra := range [][]string{{"--stage-only"}, nil} { app := New() var output bytes.Buffer @@ -1617,7 +1706,8 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { t.Fatalf("cloud publish run %d: %v", index+1, err) } var result struct { - Cutover *crawlremote.CutoverResult `json:"cutover"` + AlreadyStaged bool `json:"already_staged"` + Cutover *crawlremote.CutoverResult `json:"cutover"` } if err := json.Unmarshal(output.Bytes(), &result); err != nil { t.Fatalf("decode run %d output: %v\n%s", index+1, err, output.String()) @@ -1625,16 +1715,44 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { if index == 0 && result.Cutover != nil { t.Fatalf("stage-only publish unexpectedly cut over: %#v", result.Cutover) } + if index == 0 && result.AlreadyStaged { + t.Fatal("initial stage-only publish unexpectedly reused a candidate") + } + if index == 0 { + stageIngestRequests = ingestRequests + if stageIngestRequests == 0 || stagedSnapshot == nil { + t.Fatal("stage-only publish did not complete the candidate snapshot") + } + if servingSnapshotID != originalServingSnapshotID { + t.Fatalf("stage-only changed serving snapshot to %q", servingSnapshotID) + } + } if index == 1 && result.Cutover == nil { t.Fatalf("default cutover missing: %s", output.String()) } + if index == 1 && !result.AlreadyStaged { + t.Fatal("default publish did not report the staged candidate reuse") + } } - if ingestRequests != 0 { - t.Fatalf("ingest requests = %d, want 0 for active snapshot resume", ingestRequests) + if ingestRequests != stageIngestRequests { + t.Fatalf( + "ingest requests = %d after resume, want stage-only count %d", + ingestRequests, + stageIngestRequests, + ) } if cutovers != 1 { t.Fatalf("cutovers = %d, want 1", cutovers) } + if servingSnapshotID != snapshotID { + t.Fatalf("serving snapshot = %q, want staged snapshot %q", servingSnapshotID, snapshotID) + } + if publisherStatusRequests != 2 { + t.Fatalf("publisher status requests = %d, want 2", publisherStatusRequests) + } + if readerStatusRequests != 0 { + t.Fatalf("reader status requests = %d, publisher-only flow must not require reader role", readerStatusRequests) + } } func TestCloudPublishRejectsIncompleteHydrationBeforeRemoteMutation(t *testing.T) { diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 4c92e52..b95b856 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -150,15 +150,15 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { return err } - alreadyActive := false - status, statusErr := client.Status(ctx, "gitcrawl", archiveID) + alreadyStaged := false + status, statusErr := client.PublishStatus(ctx, "gitcrawl", archiveID) if statusErr == nil { - alreadyActive = gitcrawlSnapshotStatusMatches(status, manifest) + alreadyStaged = gitcrawlPublisherStatusMatches(status, manifest) } else if !remoteNotFound(statusErr) { return statusErr } var mutationToken string - if !alreadyActive { + if !alreadyStaged { for _, dataset := range snapshot.Datasets { progress, err := sendSnapshotIngestRows( ctx, @@ -215,7 +215,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { "capabilities": snapshot.Capabilities, "datasets": counts, "hydration": snapshot.Hydration, - "already_active": alreadyActive, + "already_staged": alreadyStaged, "mutation_token": mutationToken, "cutover": cutoverResult, "sqlite_bundle": sqliteBundle, @@ -493,8 +493,8 @@ func requireGitcrawlSnapshotPublishContract( requiredRoutes := []crawlremote.RouteSpec{ { Method: http.MethodGet, - Path: "/v1/apps/:app/archives/:archive/status", - Auth: crawlremote.AuthReader, + Path: "/v1/apps/:app/archives/:archive/publish-status", + Auth: crawlremote.AuthPublisher, }, { Method: http.MethodPost, @@ -596,14 +596,12 @@ func requireGitcrawlSnapshotPublishContract( return nil } -func gitcrawlSnapshotStatusMatches( - status crawlremote.Status, +func gitcrawlPublisherStatusMatches( + status crawlremote.PublisherStatus, manifest crawlremote.IngestManifest, ) bool { snapshot := status.Snapshot if snapshot == nil || - status.ActiveSnapshotID != manifest.SnapshotID || - !status.CoverageComplete || snapshot.ID != manifest.SnapshotID || snapshot.SourceSHA256 != manifest.SourceSHA256 || snapshot.SchemaName != manifest.SchemaName || From df857aaac43f0022e4ce66de68ae07b9f3563417 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:15:09 +0200 Subject: [PATCH 20/52] fix(cloud): harden staged snapshot resume --- internal/cli/app_test.go | 56 +++++++++++++----- internal/cli/cloud_commands.go | 87 ++++++++++++++++++++++------ internal/cli/cloud_snapshot.go | 29 +++++----- internal/cli/remote_commands_test.go | 36 ++++++++++-- 4 files changed, 156 insertions(+), 52 deletions(-) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index a2f74f4..1ca4f04 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -842,7 +842,7 @@ func testSnapshotPublishContract() crawlremote.Contract { contract.Apps = []crawlremote.AppSpec{{ App: "gitcrawl", Queries: []crawlremote.QuerySpec{ - {Name: "gitcrawl.threads.search", Args: []string{"owner", "repo", "query", "kind", "state", "mode", "limit"}}, + {Name: "gitcrawl.threads.search", Args: []string{"limit", "mode", "state", "kind", "query", "repo", "owner"}}, {Name: "gitcrawl.clusters.related", Args: []string{"owner", "repo", "number"}}, {Name: "gitcrawl.clusters.list", Args: []string{"owner", "repo", "status", "min_size"}}, {Name: "gitcrawl.clusters.members", Args: []string{"owner", "repo", "cluster_id"}}, @@ -909,17 +909,17 @@ func TestGitcrawlPublisherStatusMatchesExactMetadata(t *testing.T) { App: "gitcrawl", Archive: manifest.Archive, ActiveSnapshotID: strings.Repeat("f", 64), - CoverageComplete: false, + CoverageComplete: true, Snapshot: &crawlremote.ArchiveSnapshot{ ID: snapshotID, SourceSHA256: snapshotID, SchemaName: gitcrawlCloudSchemaName, SchemaVersion: gitcrawlCloudSchemaVersion, SchemaHash: gitcrawlCloudSchemaHash, - CoverageComplete: true, + CoverageComplete: false, }, } - if !gitcrawlPublisherStatusMatches(status, manifest) { + if !gitcrawlPublisherStatusMatches(status, manifest, false) { t.Fatal("exact staged snapshot did not match") } @@ -927,7 +927,7 @@ func TestGitcrawlPublisherStatusMatchesExactMetadata(t *testing.T) { sourceMismatch.Snapshot = &crawlremote.ArchiveSnapshot{} *sourceMismatch.Snapshot = *status.Snapshot sourceMismatch.Snapshot.SourceSHA256 = strings.Repeat("b", 64) - if gitcrawlPublisherStatusMatches(sourceMismatch, manifest) { + if gitcrawlPublisherStatusMatches(sourceMismatch, manifest, false) { t.Fatal("source digest mismatch was reused") } @@ -935,25 +935,40 @@ func TestGitcrawlPublisherStatusMatchesExactMetadata(t *testing.T) { schemaMismatch.Snapshot = &crawlremote.ArchiveSnapshot{} *schemaMismatch.Snapshot = *status.Snapshot schemaMismatch.Snapshot.SchemaVersion++ - if gitcrawlPublisherStatusMatches(schemaMismatch, manifest) { + if gitcrawlPublisherStatusMatches(schemaMismatch, manifest, false) { t.Fatal("schema mismatch was reused") } + status.CoverageComplete = false + if gitcrawlPublisherStatusMatches(status, manifest, false) { + t.Fatal("incomplete publisher status was reused without the escape hatch") + } + if !gitcrawlPublisherStatusMatches(status, manifest, true) { + t.Fatal("allow-incomplete did not resume an exact staged snapshot") + } + status.CoverageComplete = true + manifest.Capabilities = []string{gitcrawlObservationOrderCapability} - if gitcrawlPublisherStatusMatches(status, manifest) { + if gitcrawlPublisherStatusMatches(status, manifest, false) { t.Fatal("staged snapshot reused without snapshot capability proof") } status.Snapshot.Capabilities = []string{gitcrawlObservationOrderCapability} - if !gitcrawlPublisherStatusMatches(status, manifest) { + if !gitcrawlPublisherStatusMatches(status, manifest, false) { t.Fatal("staged snapshot with requested capability did not match") } - status.Snapshot.Capabilities = nil - if gitcrawlPublisherStatusMatches(status, manifest) { - t.Fatal("snapshot reused without requested capability") + status.Snapshot.Capabilities = []string{ + gitcrawlObservationOrderCapability, + "gitcrawl.unrequested-profile.v1", } - status.Snapshot.CoverageComplete = false - if gitcrawlPublisherStatusMatches(status, manifest) { - t.Fatal("incomplete staged snapshot was reused") + if gitcrawlPublisherStatusMatches(status, manifest, false) { + t.Fatal("snapshot with a broader publication profile was reused") + } + status.Snapshot.Capabilities = []string{ + gitcrawlObservationOrderCapability, + gitcrawlObservationOrderCapability, + } + if gitcrawlPublisherStatusMatches(status, manifest, false) { + t.Fatal("snapshot with duplicate publication capabilities was reused") } } @@ -1415,6 +1430,15 @@ func TestCloudPublishRejectsIncompleteRemoteSurfaceBeforeUpload(t *testing.T) { } }, }, + { + name: "thread search duplicate argument", + want: "reader query gitcrawl.threads.search has arguments", + mutate: func(contract *crawlremote.Contract) { + contract.Apps[0].Queries[0].Args = []string{ + "owner", "repo", "query", "kind", "state", "mode", "mode", + } + }, + }, { name: "duplicate reader query", want: "required reader query gitcrawl.threads.search more than once", @@ -1566,7 +1590,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { App: "gitcrawl", Archive: "gitcrawl/openclaw__openclaw", ActiveSnapshotID: servingSnapshotID, - CoverageComplete: servingSnapshotID == snapshotID, + CoverageComplete: stagedSnapshot != nil, Snapshot: stagedSnapshot, }) return @@ -1646,7 +1670,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { SchemaVersion: body.Manifest.SchemaVersion, SchemaHash: body.Manifest.SchemaHash, Capabilities: slices.Clone(body.Manifest.Capabilities), - CoverageComplete: true, + CoverageComplete: false, } } _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index b95b856..9761e66 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -153,7 +153,11 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { alreadyStaged := false status, statusErr := client.PublishStatus(ctx, "gitcrawl", archiveID) if statusErr == nil { - alreadyStaged = gitcrawlPublisherStatusMatches(status, manifest) + alreadyStaged = gitcrawlPublisherStatusMatches( + status, + manifest, + snapshot.AllowsIncompleteCoverage, + ) } else if !remoteNotFound(statusErr) { return statusErr } @@ -362,7 +366,16 @@ func sendIngestBatch( }) if err == nil { if result.ResetIncomplete { - if err := drainIngestReset(ctx, client, app, archive, manifest, table, columns); err != nil { + if err := drainIngestReset( + ctx, + client, + app, + archive, + manifest, + table, + columns, + mutationToken, + ); err != nil { return crawlremote.IngestResult{}, err } continue @@ -372,19 +385,37 @@ func sendIngestBatch( if !isResetIncomplete(err) { return crawlremote.IngestResult{}, err } - if err := drainIngestReset(ctx, client, app, archive, manifest, table, columns); err != nil { + if err := drainIngestReset( + ctx, + client, + app, + archive, + manifest, + table, + columns, + mutationToken, + ); err != nil { return crawlremote.IngestResult{}, err } } } -func drainIngestReset(ctx context.Context, client *crawlremote.Client, app, archive string, manifest crawlremote.IngestManifest, table string, columns []string) error { +func drainIngestReset( + ctx context.Context, + client *crawlremote.Client, + app, archive string, + manifest crawlremote.IngestManifest, + table string, + columns []string, + mutationToken string, +) error { for { result, err := client.Ingest(ctx, app, archive, crawlremote.IngestRequest{ - Manifest: manifest, - Table: table, - Columns: columns, - Rows: [][]any{}, + Manifest: manifest, + Table: table, + Columns: columns, + Rows: [][]any{}, + MutationToken: mutationToken, }) if err != nil { return err @@ -552,7 +583,7 @@ func requireGitcrawlSnapshotPublishContract( ) } remoteArgs := appSpec.Queries[queryIndex].Args - if !slices.Equal(remoteArgs, required.Args) { + if !equalUniqueStringSet(remoteArgs, required.Args) { return fmt.Errorf( "remote contract reader query %s has arguments %v, want %v", required.Name, @@ -596,27 +627,47 @@ func requireGitcrawlSnapshotPublishContract( return nil } +func equalUniqueStringSet(left, right []string) bool { + if len(left) != len(right) { + return false + } + seen := make(map[string]struct{}, len(left)) + for _, value := range left { + if _, duplicate := seen[value]; duplicate { + return false + } + seen[value] = struct{}{} + } + for _, value := range right { + if _, duplicate := seen[value]; !duplicate { + return false + } + delete(seen, value) + } + return len(seen) == 0 +} + func gitcrawlPublisherStatusMatches( status crawlremote.PublisherStatus, manifest crawlremote.IngestManifest, + allowIncomplete bool, ) bool { snapshot := status.Snapshot - if snapshot == nil || + if status.App != manifest.App || + status.Archive != manifest.Archive || + snapshot == nil || snapshot.ID != manifest.SnapshotID || snapshot.SourceSHA256 != manifest.SourceSHA256 || snapshot.SchemaName != manifest.SchemaName || snapshot.SchemaVersion != manifest.SchemaVersion || - snapshot.SchemaHash != manifest.SchemaHash || - !snapshot.CoverageComplete { + snapshot.SchemaHash != manifest.SchemaHash { return false } - if len(manifest.Capabilities) == 0 { - return true + if !status.CoverageComplete && !allowIncomplete { + return false } - for _, capability := range manifest.Capabilities { - if !slices.Contains(snapshot.Capabilities, capability) { - return false - } + if !equalUniqueStringSet(snapshot.Capabilities, manifest.Capabilities) { + return false } return true } diff --git a/internal/cli/cloud_snapshot.go b/internal/cli/cloud_snapshot.go index 01b01f7..af261ef 100644 --- a/internal/cli/cloud_snapshot.go +++ b/internal/cli/cloud_snapshot.go @@ -28,12 +28,13 @@ type gitcrawlCloudDataset struct { } type gitcrawlCloudSnapshot struct { - ID string - SourceSyncAt string - DatasetGeneratedAt string - Capabilities []string - Datasets []gitcrawlCloudDataset - Hydration crawlstore.EnrichmentCoverage + ID string + SourceSyncAt string + DatasetGeneratedAt string + Capabilities []string + Datasets []gitcrawlCloudDataset + Hydration crawlstore.EnrichmentCoverage + AllowsIncompleteCoverage bool } func buildGitcrawlCloudSnapshot( @@ -66,19 +67,21 @@ func buildGitcrawlCloudSnapshot( if err != nil { return gitcrawlCloudSnapshot{}, err } - if missing := incompleteGitcrawlCloudHydration(hydration); len(missing) > 0 && !allowIncomplete { + missing := incompleteGitcrawlCloudHydration(hydration) + if len(missing) > 0 && !allowIncomplete { return gitcrawlCloudSnapshot{}, fmt.Errorf( "cloud snapshot enrichment is incomplete (%s); hydrate the archive or pass --allow-incomplete", strings.Join(missing, ", "), ) } return gitcrawlCloudSnapshot{ - ID: snapshotID, - SourceSyncAt: sourceSyncAt, - DatasetGeneratedAt: time.Now().UTC().Format(time.RFC3339Nano), - Capabilities: capabilities, - Datasets: datasets, - Hydration: hydration, + ID: snapshotID, + SourceSyncAt: sourceSyncAt, + DatasetGeneratedAt: time.Now().UTC().Format(time.RFC3339Nano), + Capabilities: capabilities, + Datasets: datasets, + Hydration: hydration, + AllowsIncompleteCoverage: len(missing) > 0 && allowIncomplete, }, nil } diff --git a/internal/cli/remote_commands_test.go b/internal/cli/remote_commands_test.go index e8e866d..9d5bfbe 100644 --- a/internal/cli/remote_commands_test.go +++ b/internal/cli/remote_commands_test.go @@ -223,6 +223,7 @@ func TestSendSnapshotIngestRowsRotatesMutationTokens(t *testing.T) { func TestSendIngestRowsDrainsRemoteResetBeforeRetry(t *testing.T) { ctx := context.Background() var requests []crawlremote.IngestRequest + var drainRequests []crawlremote.IngestRequest resetCalls := 0 rejectedFirstBatch := false server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -237,8 +238,13 @@ func TestSendIngestRowsDrainsRemoteResetBeforeRetry(t *testing.T) { } w.Header().Set("content-type", "application/json") if len(body.Rows) == 0 { + drainRequests = append(drainRequests, body) resetCalls++ - _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ResetIncomplete: resetCalls == 1, ResetDeleted: 10000}) + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + MutationToken: body.MutationToken, + ResetIncomplete: resetCalls == 1, + ResetDeleted: 10000, + }) return } if body.Cursor == "" && !rejectedFirstBatch { @@ -251,7 +257,11 @@ func TestSendIngestRowsDrainsRemoteResetBeforeRetry(t *testing.T) { return } requests = append(requests, body) - _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{RowsAccepted: int64(len(body.Rows)), Complete: body.Final}) + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + MutationToken: body.MutationToken, + RowsAccepted: int64(len(body.Rows)), + Complete: body.Final, + }) })) defer server.Close() @@ -261,12 +271,23 @@ func TestSendIngestRowsDrainsRemoteResetBeforeRetry(t *testing.T) { if err != nil { t.Fatalf("client: %v", err) } - accepted, err := sendIngestRows(ctx, client, "gitcrawl", "gitcrawl/openclaw", crawlremote.IngestManifest{App: "gitcrawl"}, "threads", []string{"id"}, [][]any{{1}}, true) + progress, err := sendSnapshotIngestRows( + ctx, + client, + "gitcrawl", + "gitcrawl/openclaw", + crawlremote.IngestManifest{App: "gitcrawl"}, + "threads", + []string{"id"}, + [][]any{{1}}, + "generation-current", + true, + ) if err != nil { t.Fatalf("send ingest: %v", err) } - if accepted != 1 { - t.Fatalf("accepted = %d", accepted) + if progress.RowsAccepted != 1 || progress.MutationToken != "generation-current" { + t.Fatalf("progress = %#v", progress) } if resetCalls != 2 { t.Fatalf("resetCalls = %d", resetCalls) @@ -274,6 +295,11 @@ func TestSendIngestRowsDrainsRemoteResetBeforeRetry(t *testing.T) { if len(requests) != 1 || requests[0].Cursor != "" || !requests[0].Final { t.Fatalf("data requests = %#v", requests) } + for index, request := range drainRequests { + if request.MutationToken != "generation-current" { + t.Fatalf("drain request %d mutation token = %q", index, request.MutationToken) + } + } } func TestRemoteAndCloudCommandDispatchErrors(t *testing.T) { From 4b1dfab9535d02dd4c8fe9a59e46dfff37eb7f13 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:15:17 +0200 Subject: [PATCH 21/52] fix(store): require persisted PR file coverage --- internal/store/archive_coverage.go | 88 ++++++++++--------------- internal/store/archive_coverage_test.go | 21 ++++++ 2 files changed, 55 insertions(+), 54 deletions(-) diff --git a/internal/store/archive_coverage.go b/internal/store/archive_coverage.go index dff0fd8..b409ced 100644 --- a/internal/store/archive_coverage.go +++ b/internal/store/archive_coverage.go @@ -635,15 +635,7 @@ func (s *Store) archivePRFileCoverage( ctx context.Context, repoID int64, ) (EnrichmentCoverageMetric, error) { - hasReservations := s.archiveCoverageHasColumns( - ctx, - "thread_child_observation_reservations", - "thread_id", - "family", - "source_updated_at", - "observation_sequence", - ) - hasLegacyProof := s.archiveCoverageHasColumns( + hasPersistedProof := s.archiveCoverageHasColumns( ctx, "pull_request_details", "thread_id", @@ -655,9 +647,17 @@ func (s *Store) archivePRFileCoverage( "thread_id", "fetched_at", ) - if !hasReservations && !hasLegacyProof { + if !hasPersistedProof { return EnrichmentCoverageMetric{}, nil } + hasReservations := s.archiveCoverageHasColumns( + ctx, + "thread_child_observation_reservations", + "thread_id", + "family", + "source_updated_at", + "observation_sequence", + ) acceptedSourceUpdatedAt, acceptedObservationSequence := s.archiveAcceptedThreadObservationExpressions(ctx, "t") reservationJoin := "" @@ -674,15 +674,7 @@ func (s *Store) archivePRFileCoverage( reservationSourceUpdatedAt = "coalesce(reservation.source_updated_at, '')" reservationObservationSequence = "coalesce(reservation.observation_sequence, 0)" } - legacyJoin := "" - detailPresent := "0" - detailChangedFiles := "0" - detailFetchedAt := "''" - fileCount := "0" - oldestFileFetchedAt := "''" - latestFileFetchedAt := "''" - if hasLegacyProof { - legacyJoin = ` + persistedEvidenceJoin := ` left join pull_request_details prd on prd.thread_id = t.id left join ( select thread_id, @@ -693,28 +685,21 @@ func (s *Store) archivePRFileCoverage( group by thread_id ) prf on prf.thread_id = t.id ` - detailPresent = "case when prd.thread_id is null then 0 else 1 end" - detailChangedFiles = "coalesce(prd.changed_files, 0)" - detailFetchedAt = "coalesce(prd.fetched_at, '')" - fileCount = "coalesce(prf.file_count, 0)" - oldestFileFetchedAt = "coalesce(prf.oldest_fetched_at, '')" - latestFileFetchedAt = "coalesce(prf.latest_fetched_at, '')" - } rows, err := s.q().QueryContext(ctx, ` select `+reservationPresent+`, `+reservationSourceUpdatedAt+`, `+reservationObservationSequence+`, - `+detailPresent+`, - `+detailChangedFiles+`, - `+detailFetchedAt+`, - `+fileCount+`, - `+oldestFileFetchedAt+`, - `+latestFileFetchedAt+`, + case when prd.thread_id is null then 0 else 1 end, + coalesce(prd.changed_files, 0), + coalesce(prd.fetched_at, ''), + coalesce(prf.file_count, 0), + coalesce(prf.oldest_fetched_at, ''), + coalesce(prf.latest_fetched_at, ''), `+acceptedSourceUpdatedAt+`, `+acceptedObservationSequence+` from threads t `+reservationJoin+` - `+legacyJoin+` + `+persistedEvidenceJoin+` where t.repo_id = ? and t.kind = 'pull_request' `, repoID) if err != nil { @@ -745,31 +730,26 @@ func (s *Store) archivePRFileCoverage( return EnrichmentCoverageMetric{}, fmt.Errorf("scan archive PR file coverage: %w", err) } metric.Eligible++ - switch { - case hasReservation != 0: - metric.Covered++ - latestObservedAt = laterArchiveCoverageTimestamp( - latestObservedAt, - reservationSourceUpdatedAt, - ) - if archiveObservationAtOrAfter( + if hasDetail == 0 || changedFiles < 0 || files != changedFiles { + continue + } + metric.Covered++ + latestObservedAt = laterArchiveCoverageTimestamp(latestObservedAt, detailFetchedAt) + latestObservedAt = laterArchiveCoverageTimestamp(latestObservedAt, latestFileFetchedAt) + detailFresh := archiveCoverageTimestampAtOrAfter(detailFetchedAt, acceptedSource) + filesFresh := files == 0 || + archiveCoverageTimestampAtOrAfter(oldestFileFetchedAt, acceptedSource) + reservationFresh := true + if hasReservation != 0 { + reservationFresh = archiveObservationAtOrAfter( reservationSourceUpdatedAt, reservationSequence, acceptedSource, observationSequenceOrderValue(acceptedSequence), - ) { - metric.Fresh++ - } - case hasDetail != 0 && changedFiles >= 0 && files == changedFiles: - metric.Covered++ - latestObservedAt = laterArchiveCoverageTimestamp(latestObservedAt, detailFetchedAt) - latestObservedAt = laterArchiveCoverageTimestamp(latestObservedAt, latestFileFetchedAt) - detailFresh := archiveCoverageTimestampAtOrAfter(detailFetchedAt, acceptedSource) - filesFresh := files == 0 || - archiveCoverageTimestampAtOrAfter(oldestFileFetchedAt, acceptedSource) - if detailFresh && filesFresh { - metric.Fresh++ - } + ) + } + if detailFresh && filesFresh && reservationFresh { + metric.Fresh++ } } if err := rows.Err(); err != nil { diff --git a/internal/store/archive_coverage_test.go b/internal/store/archive_coverage_test.go index e14aa7f..27b9850 100644 --- a/internal/store/archive_coverage_test.go +++ b/internal/store/archive_coverage_test.go @@ -273,6 +273,27 @@ func TestArchiveCoveragePRFilesRequireCurrentObservation(t *testing.T) { ); err != nil || !applied { t.Fatalf("reserve PR file observation = %t, %v", applied, err) } + assertMetric(0, 0, 1, 0, false) + if err := st.UpsertPullRequestCacheFamilies(ctx, PullRequestDetail{ + ThreadID: thread.ID, + RepoID: repoID, + Number: thread.Number, + ChangedFiles: 1, + RawJSON: "{}", + FetchedAt: "2026-07-12T12:01:00Z", + UpdatedAt: "2026-07-12T12:01:00Z", + }, []PullRequestFile{{ + ThreadID: thread.ID, + Position: 0, + Path: "README.md", + RawJSON: "{}", + FetchedAt: "2026-07-12T12:01:00Z", + }}, nil, nil, nil, PullRequestHydrationFamilies{ + Details: true, + Files: true, + }); err != nil { + t.Fatalf("persist PR file evidence: %v", err) + } assertMetric(1, 1, 0, 0, true) thread.UpdatedAtGitHub = "2026-07-12T12:03:00Z" From 1a37504457f40dd7c880afb7e693ef5b589107fe Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:15:19 +0200 Subject: [PATCH 22/52] docs(cloud): define snapshot retention boundary --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 5b3ba98..3be148d 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,11 @@ when its digest, schema, capabilities, and coverage match, then cuts it over. Incomplete local enrichment fails before any remote mutation; `--allow-incomplete` is an explicit escape hatch, and `--observation-order` publishes durable fetch ordering after the remote operator fence is enabled. +Digest-scoped SQLite bundles can contain private issue and pull-request text. +Gitcrawl intentionally exposes no remote deletion command: operators must only +enable publication against a remote deployment with bounded lifecycle rules for +failed, superseded, and uncut staged bundles. `--stage-only` does not transfer +that retention responsibility back to the client. `gitcrawl clusters-report` writes a Markdown report for the top clusters using the same display view, with an at-a-glance table, per-cluster metadata, member tables, and key snippets. Use `--json` for the hydrated report payload. `gitcrawl cluster` and `gitcrawl refresh` build ghcrawl-shaped durable clusters by default (`--threshold 0.80`, `--min-size 1`, `--max-cluster-size 40`, `--k 16`, `--cross-kind-threshold 0.93`): every active vector-backed thread is represented, singleton rows use `singleton_orphan`, multi-member rows use `duplicate_candidate`, and stable IDs are derived from the representative thread. They also add deterministic GitHub reference evidence for direct issue/PR links such as `#123`, `issues/123`, and `pull/123`. Weak embedding edges need concrete title-token overlap unless their similarity is already high, which keeps generic low-confidence bridges from forming unrelated clusters. `gitcrawl tui` infers the most recently updated local repository when `owner/repo` is omitted. `serve` is intentionally not part of `gitcrawl`. From af5ae3272f58c5c59ea9189b1995ea3882b21931 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:19:59 +0200 Subject: [PATCH 23/52] fix(cloud): require complete staged coverage --- internal/cli/app_test.go | 59 ++++++++++++++++++++++------------ internal/cli/cloud_commands.go | 9 ++---- internal/cli/cloud_snapshot.go | 26 +++++++-------- 3 files changed, 52 insertions(+), 42 deletions(-) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 1ca4f04..8f2eb49 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -919,7 +919,7 @@ func TestGitcrawlPublisherStatusMatchesExactMetadata(t *testing.T) { CoverageComplete: false, }, } - if !gitcrawlPublisherStatusMatches(status, manifest, false) { + if !gitcrawlPublisherStatusMatches(status, manifest) { t.Fatal("exact staged snapshot did not match") } @@ -927,7 +927,7 @@ func TestGitcrawlPublisherStatusMatchesExactMetadata(t *testing.T) { sourceMismatch.Snapshot = &crawlremote.ArchiveSnapshot{} *sourceMismatch.Snapshot = *status.Snapshot sourceMismatch.Snapshot.SourceSHA256 = strings.Repeat("b", 64) - if gitcrawlPublisherStatusMatches(sourceMismatch, manifest, false) { + if gitcrawlPublisherStatusMatches(sourceMismatch, manifest) { t.Fatal("source digest mismatch was reused") } @@ -935,39 +935,36 @@ func TestGitcrawlPublisherStatusMatchesExactMetadata(t *testing.T) { schemaMismatch.Snapshot = &crawlremote.ArchiveSnapshot{} *schemaMismatch.Snapshot = *status.Snapshot schemaMismatch.Snapshot.SchemaVersion++ - if gitcrawlPublisherStatusMatches(schemaMismatch, manifest, false) { + if gitcrawlPublisherStatusMatches(schemaMismatch, manifest) { t.Fatal("schema mismatch was reused") } status.CoverageComplete = false - if gitcrawlPublisherStatusMatches(status, manifest, false) { - t.Fatal("incomplete publisher status was reused without the escape hatch") - } - if !gitcrawlPublisherStatusMatches(status, manifest, true) { - t.Fatal("allow-incomplete did not resume an exact staged snapshot") + if gitcrawlPublisherStatusMatches(status, manifest) { + t.Fatal("incomplete publisher status was reused") } status.CoverageComplete = true manifest.Capabilities = []string{gitcrawlObservationOrderCapability} - if gitcrawlPublisherStatusMatches(status, manifest, false) { + if gitcrawlPublisherStatusMatches(status, manifest) { t.Fatal("staged snapshot reused without snapshot capability proof") } status.Snapshot.Capabilities = []string{gitcrawlObservationOrderCapability} - if !gitcrawlPublisherStatusMatches(status, manifest, false) { + if !gitcrawlPublisherStatusMatches(status, manifest) { t.Fatal("staged snapshot with requested capability did not match") } status.Snapshot.Capabilities = []string{ gitcrawlObservationOrderCapability, "gitcrawl.unrequested-profile.v1", } - if gitcrawlPublisherStatusMatches(status, manifest, false) { + if gitcrawlPublisherStatusMatches(status, manifest) { t.Fatal("snapshot with a broader publication profile was reused") } status.Snapshot.Capabilities = []string{ gitcrawlObservationOrderCapability, gitcrawlObservationOrderCapability, } - if gitcrawlPublisherStatusMatches(status, manifest, false) { + if gitcrawlPublisherStatusMatches(status, manifest) { t.Fatal("snapshot with duplicate publication capabilities was reused") } } @@ -1549,6 +1546,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { publisherStatusRequests := 0 readerStatusRequests := 0 cutovers := 0 + publisherCoverageComplete := false server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { contract := testSnapshotPublishContract() @@ -1590,7 +1588,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { App: "gitcrawl", Archive: "gitcrawl/openclaw__openclaw", ActiveSnapshotID: servingSnapshotID, - CoverageComplete: stagedSnapshot != nil, + CoverageComplete: publisherCoverageComplete, Snapshot: stagedSnapshot, }) return @@ -1672,6 +1670,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { Capabilities: slices.Clone(body.Manifest.Capabilities), CoverageComplete: false, } + publisherCoverageComplete = true } _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ Table: body.Table, @@ -1720,8 +1719,9 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { "--json", } stageIngestRequests := 0 + recoveryIngestRequests := 0 originalServingSnapshotID := servingSnapshotID - for index, extra := range [][]string{{"--stage-only"}, nil} { + for index, extra := range [][]string{{"--stage-only"}, {"--stage-only"}, nil} { app := New() var output bytes.Buffer app.Stdout = &output @@ -1750,19 +1750,36 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { if servingSnapshotID != originalServingSnapshotID { t.Fatalf("stage-only changed serving snapshot to %q", servingSnapshotID) } + publisherCoverageComplete = false + } + if index == 1 { + if result.AlreadyStaged { + t.Fatal("incomplete publisher status was reused") + } + if result.Cutover != nil { + t.Fatalf("recovery stage-only publish unexpectedly cut over: %#v", result.Cutover) + } + recoveryIngestRequests = ingestRequests + if recoveryIngestRequests <= stageIngestRequests { + t.Fatalf( + "incomplete publisher status did not resume ingest: before=%d after=%d", + stageIngestRequests, + recoveryIngestRequests, + ) + } } - if index == 1 && result.Cutover == nil { + if index == 2 && result.Cutover == nil { t.Fatalf("default cutover missing: %s", output.String()) } - if index == 1 && !result.AlreadyStaged { + if index == 2 && !result.AlreadyStaged { t.Fatal("default publish did not report the staged candidate reuse") } } - if ingestRequests != stageIngestRequests { + if ingestRequests != recoveryIngestRequests { t.Fatalf( - "ingest requests = %d after resume, want stage-only count %d", + "ingest requests = %d after complete resume, want recovery count %d", ingestRequests, - stageIngestRequests, + recoveryIngestRequests, ) } if cutovers != 1 { @@ -1771,8 +1788,8 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { if servingSnapshotID != snapshotID { t.Fatalf("serving snapshot = %q, want staged snapshot %q", servingSnapshotID, snapshotID) } - if publisherStatusRequests != 2 { - t.Fatalf("publisher status requests = %d, want 2", publisherStatusRequests) + if publisherStatusRequests != 3 { + t.Fatalf("publisher status requests = %d, want 3", publisherStatusRequests) } if readerStatusRequests != 0 { t.Fatalf("reader status requests = %d, publisher-only flow must not require reader role", readerStatusRequests) diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 9761e66..8b1a2bf 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -153,11 +153,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { alreadyStaged := false status, statusErr := client.PublishStatus(ctx, "gitcrawl", archiveID) if statusErr == nil { - alreadyStaged = gitcrawlPublisherStatusMatches( - status, - manifest, - snapshot.AllowsIncompleteCoverage, - ) + alreadyStaged = gitcrawlPublisherStatusMatches(status, manifest) } else if !remoteNotFound(statusErr) { return statusErr } @@ -650,7 +646,6 @@ func equalUniqueStringSet(left, right []string) bool { func gitcrawlPublisherStatusMatches( status crawlremote.PublisherStatus, manifest crawlremote.IngestManifest, - allowIncomplete bool, ) bool { snapshot := status.Snapshot if status.App != manifest.App || @@ -663,7 +658,7 @@ func gitcrawlPublisherStatusMatches( snapshot.SchemaHash != manifest.SchemaHash { return false } - if !status.CoverageComplete && !allowIncomplete { + if !status.CoverageComplete { return false } if !equalUniqueStringSet(snapshot.Capabilities, manifest.Capabilities) { diff --git a/internal/cli/cloud_snapshot.go b/internal/cli/cloud_snapshot.go index af261ef..eaef905 100644 --- a/internal/cli/cloud_snapshot.go +++ b/internal/cli/cloud_snapshot.go @@ -28,13 +28,12 @@ type gitcrawlCloudDataset struct { } type gitcrawlCloudSnapshot struct { - ID string - SourceSyncAt string - DatasetGeneratedAt string - Capabilities []string - Datasets []gitcrawlCloudDataset - Hydration crawlstore.EnrichmentCoverage - AllowsIncompleteCoverage bool + ID string + SourceSyncAt string + DatasetGeneratedAt string + Capabilities []string + Datasets []gitcrawlCloudDataset + Hydration crawlstore.EnrichmentCoverage } func buildGitcrawlCloudSnapshot( @@ -75,13 +74,12 @@ func buildGitcrawlCloudSnapshot( ) } return gitcrawlCloudSnapshot{ - ID: snapshotID, - SourceSyncAt: sourceSyncAt, - DatasetGeneratedAt: time.Now().UTC().Format(time.RFC3339Nano), - Capabilities: capabilities, - Datasets: datasets, - Hydration: hydration, - AllowsIncompleteCoverage: len(missing) > 0 && allowIncomplete, + ID: snapshotID, + SourceSyncAt: sourceSyncAt, + DatasetGeneratedAt: time.Now().UTC().Format(time.RFC3339Nano), + Capabilities: capabilities, + Datasets: datasets, + Hydration: hydration, }, nil } From 4cd93f77b19f9602da3d7c6e4b6c7d64ac4346c0 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:41:13 +0200 Subject: [PATCH 24/52] fix(cloud): resume staged publish across processes --- README.md | 3 +- internal/cli/app_test.go | 159 +++++++++++++++++++++------------ internal/cli/cloud_commands.go | 26 +++++- 3 files changed, 125 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 3be148d..48f8628 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,8 @@ Publishing moves unpinned reads to the complete snapshot by default, preserving the existing reader-refresh behavior. `--stage-only` keeps the immutable snapshot staged without changing serving state. A later publish verifies the candidate through the publisher-only status projection, skips repeated ingest -when its digest, schema, capabilities, and coverage match, then cuts it over. +when its digest, source sync, schema, resolved publication profile, persisted +generation timestamp, and coverage match, then cuts it over. Incomplete local enrichment fails before any remote mutation; `--allow-incomplete` is an explicit escape hatch, and `--observation-order` publishes durable fetch ordering after the remote operator fence is enabled. diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 8f2eb49..fd5b7c5 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -902,69 +902,97 @@ func TestGitcrawlPublisherStatusMatchesExactMetadata(t *testing.T) { SchemaName: gitcrawlCloudSchemaName, SchemaVersion: gitcrawlCloudSchemaVersion, SchemaHash: gitcrawlCloudSchemaHash, + SourceSyncAt: "2026-07-12T12:00:00Z", SnapshotID: snapshotID, SourceSHA256: snapshotID, } + publicationCapabilities := gitcrawlCloudPublicationCapabilities(manifest.Capabilities) status := crawlremote.PublisherStatus{ App: "gitcrawl", Archive: manifest.Archive, - ActiveSnapshotID: strings.Repeat("f", 64), + ActiveSnapshotID: snapshotID, CoverageComplete: true, Snapshot: &crawlremote.ArchiveSnapshot{ - ID: snapshotID, - SourceSHA256: snapshotID, - SchemaName: gitcrawlCloudSchemaName, - SchemaVersion: gitcrawlCloudSchemaVersion, - SchemaHash: gitcrawlCloudSchemaHash, - CoverageComplete: false, + ID: snapshotID, + SourceSHA256: snapshotID, + SourceSyncAt: manifest.SourceSyncAt, + DatasetGeneratedAt: "2026-07-12T12:01:00Z", + SchemaName: gitcrawlCloudSchemaName, + SchemaVersion: gitcrawlCloudSchemaVersion, + SchemaHash: gitcrawlCloudSchemaHash, + Capabilities: publicationCapabilities, + CoverageComplete: false, }, } - if !gitcrawlPublisherStatusMatches(status, manifest) { + if !gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) { t.Fatal("exact staged snapshot did not match") } + activeMismatch := status + activeMismatch.ActiveSnapshotID = strings.Repeat("f", 64) + if gitcrawlPublisherStatusMatches(activeMismatch, manifest, publicationCapabilities) { + t.Fatal("non-active staged snapshot was reused") + } + sourceMismatch := status sourceMismatch.Snapshot = &crawlremote.ArchiveSnapshot{} *sourceMismatch.Snapshot = *status.Snapshot sourceMismatch.Snapshot.SourceSHA256 = strings.Repeat("b", 64) - if gitcrawlPublisherStatusMatches(sourceMismatch, manifest) { + if gitcrawlPublisherStatusMatches(sourceMismatch, manifest, publicationCapabilities) { t.Fatal("source digest mismatch was reused") } + sourceSyncMismatch := status + sourceSyncMismatch.Snapshot = &crawlremote.ArchiveSnapshot{} + *sourceSyncMismatch.Snapshot = *status.Snapshot + sourceSyncMismatch.Snapshot.SourceSyncAt = "2026-07-12T12:00:01Z" + if gitcrawlPublisherStatusMatches(sourceSyncMismatch, manifest, publicationCapabilities) { + t.Fatal("source sync mismatch was reused") + } + schemaMismatch := status schemaMismatch.Snapshot = &crawlremote.ArchiveSnapshot{} *schemaMismatch.Snapshot = *status.Snapshot schemaMismatch.Snapshot.SchemaVersion++ - if gitcrawlPublisherStatusMatches(schemaMismatch, manifest) { + if gitcrawlPublisherStatusMatches(schemaMismatch, manifest, publicationCapabilities) { t.Fatal("schema mismatch was reused") } + missingGeneratedAt := status + missingGeneratedAt.Snapshot = &crawlremote.ArchiveSnapshot{} + *missingGeneratedAt.Snapshot = *status.Snapshot + missingGeneratedAt.Snapshot.DatasetGeneratedAt = "" + if gitcrawlPublisherStatusMatches(missingGeneratedAt, manifest, publicationCapabilities) { + t.Fatal("snapshot without its staged generation timestamp was reused") + } + status.CoverageComplete = false - if gitcrawlPublisherStatusMatches(status, manifest) { + if gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) { t.Fatal("incomplete publisher status was reused") } status.CoverageComplete = true manifest.Capabilities = []string{gitcrawlObservationOrderCapability} - if gitcrawlPublisherStatusMatches(status, manifest) { + publicationCapabilities = gitcrawlCloudPublicationCapabilities(manifest.Capabilities) + if gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) { t.Fatal("staged snapshot reused without snapshot capability proof") } - status.Snapshot.Capabilities = []string{gitcrawlObservationOrderCapability} - if !gitcrawlPublisherStatusMatches(status, manifest) { + status.Snapshot.Capabilities = slices.Clone(publicationCapabilities) + if !gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) { t.Fatal("staged snapshot with requested capability did not match") } - status.Snapshot.Capabilities = []string{ - gitcrawlObservationOrderCapability, + status.Snapshot.Capabilities = append( + slices.Clone(publicationCapabilities), "gitcrawl.unrequested-profile.v1", - } - if gitcrawlPublisherStatusMatches(status, manifest) { + ) + if gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) { t.Fatal("snapshot with a broader publication profile was reused") } - status.Snapshot.Capabilities = []string{ + status.Snapshot.Capabilities = append( + slices.Clone(publicationCapabilities), gitcrawlObservationOrderCapability, - gitcrawlObservationOrderCapability, - } - if gitcrawlPublisherStatusMatches(status, manifest) { + ) + if gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) { t.Fatal("snapshot with duplicate publication capabilities was reused") } } @@ -1546,7 +1574,6 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { publisherStatusRequests := 0 readerStatusRequests := 0 cutovers := 0 - publisherCoverageComplete := false server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { contract := testSnapshotPublishContract() @@ -1584,11 +1611,15 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { } if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status") { publisherStatusRequests++ + activeSnapshotID := "" + if stagedSnapshot != nil { + activeSnapshotID = stagedSnapshot.ID + } _ = json.NewEncoder(w).Encode(crawlremote.PublisherStatus{ App: "gitcrawl", Archive: "gitcrawl/openclaw__openclaw", - ActiveSnapshotID: servingSnapshotID, - CoverageComplete: publisherCoverageComplete, + ActiveSnapshotID: activeSnapshotID, + CoverageComplete: stagedSnapshot != nil, Snapshot: stagedSnapshot, }) return @@ -1644,6 +1675,14 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { } if r.Method == http.MethodPost && strings.HasSuffix(r.URL.EscapedPath(), "/ingest") { ingestRequests++ + if stagedSnapshot != nil { + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(map[string]any{ + "error": "snapshot_active", + "message": "activated Gitcrawl snapshots are immutable", + }) + return + } var body crawlremote.IngestRequest if err := json.NewDecoder(r.Body).Decode(&body); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -1662,15 +1701,18 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { } mutationToken = body.MutationToken stagedSnapshot = &crawlremote.ArchiveSnapshot{ - ID: body.Manifest.SnapshotID, - SourceSHA256: body.Manifest.SourceSHA256, - SchemaName: body.Manifest.SchemaName, - SchemaVersion: body.Manifest.SchemaVersion, - SchemaHash: body.Manifest.SchemaHash, - Capabilities: slices.Clone(body.Manifest.Capabilities), - CoverageComplete: false, + ID: body.Manifest.SnapshotID, + SourceSHA256: body.Manifest.SourceSHA256, + SourceSyncAt: body.Manifest.SourceSyncAt, + DatasetGeneratedAt: fmt.Sprint(body.Rows[0][5]), + SchemaName: body.Manifest.SchemaName, + SchemaVersion: body.Manifest.SchemaVersion, + SchemaHash: body.Manifest.SchemaHash, + Capabilities: gitcrawlCloudPublicationCapabilities( + body.Manifest.Capabilities, + ), + CoverageComplete: true, } - publisherCoverageComplete = true } _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ Table: body.Table, @@ -1719,9 +1761,9 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { "--json", } stageIngestRequests := 0 - recoveryIngestRequests := 0 originalServingSnapshotID := servingSnapshotID - for index, extra := range [][]string{{"--stage-only"}, {"--stage-only"}, nil} { + var stagedDatasetGeneratedAt string + for index, extra := range [][]string{{"--stage-only"}, nil} { app := New() var output bytes.Buffer app.Stdout = &output @@ -1730,8 +1772,10 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { t.Fatalf("cloud publish run %d: %v", index+1, err) } var result struct { - AlreadyStaged bool `json:"already_staged"` - Cutover *crawlremote.CutoverResult `json:"cutover"` + DatasetGeneratedAt string `json:"dataset_generated_at"` + AlreadyStaged bool `json:"already_staged"` + MutationToken string `json:"mutation_token"` + Cutover *crawlremote.CutoverResult `json:"cutover"` } if err := json.Unmarshal(output.Bytes(), &result); err != nil { t.Fatalf("decode run %d output: %v\n%s", index+1, err, output.String()) @@ -1750,36 +1794,33 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { if servingSnapshotID != originalServingSnapshotID { t.Fatalf("stage-only changed serving snapshot to %q", servingSnapshotID) } - publisherCoverageComplete = false + stagedDatasetGeneratedAt = result.DatasetGeneratedAt + if stagedDatasetGeneratedAt == "" || result.MutationToken == "" { + t.Fatalf("stage-only did not bind its generation: %#v", result) + } + time.Sleep(2 * time.Millisecond) } if index == 1 { - if result.AlreadyStaged { - t.Fatal("incomplete publisher status was reused") - } - if result.Cutover != nil { - t.Fatalf("recovery stage-only publish unexpectedly cut over: %#v", result.Cutover) + if !result.AlreadyStaged || result.Cutover == nil { + t.Fatalf("default publish did not resume the staged candidate: %s", output.String()) } - recoveryIngestRequests = ingestRequests - if recoveryIngestRequests <= stageIngestRequests { + if result.DatasetGeneratedAt != stagedDatasetGeneratedAt { t.Fatalf( - "incomplete publisher status did not resume ingest: before=%d after=%d", - stageIngestRequests, - recoveryIngestRequests, + "resumed generation = %q, want staged generation %q", + result.DatasetGeneratedAt, + stagedDatasetGeneratedAt, ) } - } - if index == 2 && result.Cutover == nil { - t.Fatalf("default cutover missing: %s", output.String()) - } - if index == 2 && !result.AlreadyStaged { - t.Fatal("default publish did not report the staged candidate reuse") + if result.MutationToken != "" { + t.Fatalf("resumed publish minted mutation token %q", result.MutationToken) + } } } - if ingestRequests != recoveryIngestRequests { + if ingestRequests != stageIngestRequests { t.Fatalf( - "ingest requests = %d after complete resume, want recovery count %d", + "ingest requests = %d after complete resume, want stage-only count %d", ingestRequests, - recoveryIngestRequests, + stageIngestRequests, ) } if cutovers != 1 { @@ -1788,8 +1829,8 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { if servingSnapshotID != snapshotID { t.Fatalf("serving snapshot = %q, want staged snapshot %q", servingSnapshotID, snapshotID) } - if publisherStatusRequests != 3 { - t.Fatalf("publisher status requests = %d, want 3", publisherStatusRequests) + if publisherStatusRequests != 2 { + t.Fatalf("publisher status requests = %d, want 2", publisherStatusRequests) } if readerStatusRequests != 0 { t.Fatalf("reader status requests = %d, publisher-only flow must not require reader role", readerStatusRequests) diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 8b1a2bf..05715a7 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -45,6 +45,14 @@ func gitcrawlCloudReaderQuerySpecs() []crawlremote.QuerySpec { } } +func gitcrawlCloudPublicationCapabilities(requested []string) []string { + capabilities := make([]string, 0, len(gitcrawlCloudReaderQuerySpecs())+len(requested)) + for _, query := range gitcrawlCloudReaderQuerySpecs() { + capabilities = append(capabilities, query.Name) + } + return append(capabilities, requested...) +} + func (a *App) runCloud(ctx context.Context, args []string) error { if len(args) == 0 { return usageErr(fmt.Errorf("cloud requires a subcommand")) @@ -129,6 +137,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { return err } manifest := gitcrawlCloudManifest(archiveID, snapshot) + publicationCapabilities := gitcrawlCloudPublicationCapabilities(snapshot.Capabilities) counts := gitcrawlCloudDatasetCounts(snapshot) if err := requireGitcrawlSnapshotPublishContract( ctx, @@ -153,7 +162,14 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { alreadyStaged := false status, statusErr := client.PublishStatus(ctx, "gitcrawl", archiveID) if statusErr == nil { - alreadyStaged = gitcrawlPublisherStatusMatches(status, manifest) + alreadyStaged = gitcrawlPublisherStatusMatches( + status, + manifest, + publicationCapabilities, + ) + if alreadyStaged { + snapshot.DatasetGeneratedAt = status.Snapshot.DatasetGeneratedAt + } } else if !remoteNotFound(statusErr) { return statusErr } @@ -646,22 +662,26 @@ func equalUniqueStringSet(left, right []string) bool { func gitcrawlPublisherStatusMatches( status crawlremote.PublisherStatus, manifest crawlremote.IngestManifest, + publicationCapabilities []string, ) bool { snapshot := status.Snapshot if status.App != manifest.App || status.Archive != manifest.Archive || + status.ActiveSnapshotID != manifest.SnapshotID || snapshot == nil || snapshot.ID != manifest.SnapshotID || snapshot.SourceSHA256 != manifest.SourceSHA256 || + snapshot.SourceSyncAt != manifest.SourceSyncAt || snapshot.SchemaName != manifest.SchemaName || snapshot.SchemaVersion != manifest.SchemaVersion || - snapshot.SchemaHash != manifest.SchemaHash { + snapshot.SchemaHash != manifest.SchemaHash || + strings.TrimSpace(snapshot.DatasetGeneratedAt) == "" { return false } if !status.CoverageComplete { return false } - if !equalUniqueStringSet(snapshot.Capabilities, manifest.Capabilities) { + if !equalUniqueStringSet(snapshot.Capabilities, publicationCapabilities) { return false } return true From 577c34963493b9c4182ca418ad0b3c61a0eb41b0 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:56:38 +0200 Subject: [PATCH 25/52] fix(cloud): compare snapshot times chronologically --- internal/cli/cloud_snapshot.go | 89 ++++++++++++++++++++--------- internal/cli/cloud_snapshot_test.go | 84 +++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 28 deletions(-) create mode 100644 internal/cli/cloud_snapshot_test.go diff --git a/internal/cli/cloud_snapshot.go b/internal/cli/cloud_snapshot.go index eaef905..73367f1 100644 --- a/internal/cli/cloud_snapshot.go +++ b/internal/cli/cloud_snapshot.go @@ -132,28 +132,28 @@ from thread_revisions` } specs := []struct { - name string - columns []string - query string - maxSourceAt string + name string + columns []string + query string + sourceAtQuery string }{ { - name: "repositories", - columns: gitcrawlRepositoryColumns, - query: gitcrawlRepositoryExportSQL, - maxSourceAt: `select coalesce(max(updated_at), '') from repositories`, + name: "repositories", + columns: gitcrawlRepositoryColumns, + query: gitcrawlRepositoryExportSQL, + sourceAtQuery: `select coalesce(updated_at, '') from repositories`, }, { - name: "threads", - columns: threadColumns, - query: threadSelect + "\norder by repo_id, number", - maxSourceAt: `select coalesce(max(coalesce(nullif(updated_at_gh, ''), updated_at)), '') from threads`, + name: "threads", + columns: threadColumns, + query: threadSelect + "\norder by repo_id, number", + sourceAtQuery: `select coalesce(nullif(updated_at_gh, ''), updated_at, '') from threads`, }, { - name: "thread_revisions", - columns: revisionColumns, - query: revisionSelect + "\norder by id", - maxSourceAt: `select coalesce(max(coalesce(nullif(source_updated_at, ''), created_at)), '') from thread_revisions`, + name: "thread_revisions", + columns: revisionColumns, + query: revisionSelect + "\norder by id", + sourceAtQuery: `select coalesce(nullif(source_updated_at, ''), created_at, '') from thread_revisions`, }, { name: "thread_fingerprints", @@ -166,7 +166,7 @@ select id, thread_revision_id, algorithm_version, fingerprint_hash, fingerprint_slug, body_token_hash, file_set_hash, simhash64, created_at from thread_fingerprints order by id`, - maxSourceAt: `select coalesce(max(created_at), '') from thread_fingerprints`, + sourceAtQuery: `select coalesce(created_at, '') from thread_fingerprints`, }, { name: "thread_key_summaries", @@ -179,7 +179,7 @@ select id, thread_revision_id, summary_kind, prompt_version, provider, model, input_hash, output_hash, key_text, created_at from thread_key_summaries order by id`, - maxSourceAt: `select coalesce(max(created_at), '') from thread_key_summaries`, + sourceAtQuery: `select coalesce(created_at, '') from thread_key_summaries`, }, { name: "cluster_groups", @@ -196,7 +196,7 @@ select cluster.id, cluster.repo_id, cluster.stable_key, cluster.stable_slug, cluster.created_at, cluster.updated_at, coalesce(cluster.closed_at, '') from cluster_groups cluster order by cluster.id`, - maxSourceAt: `select coalesce(max(updated_at), '') from cluster_groups`, + sourceAtQuery: `select coalesce(updated_at, '') from cluster_groups`, }, { name: "cluster_memberships", @@ -209,7 +209,7 @@ select cluster_id, thread_id, role, state, score_to_representative, created_at, updated_at, coalesce(removed_at, '') from cluster_memberships order by cluster_id, thread_id`, - maxSourceAt: `select coalesce(max(updated_at), '') from cluster_memberships`, + sourceAtQuery: `select coalesce(updated_at, '') from cluster_memberships`, }, { name: "pull_request_details", @@ -225,7 +225,7 @@ select thread_id, repo_id, number, coalesce(base_sha, ''), coalesce(head_sha, '' fetched_at, updated_at from pull_request_details order by thread_id`, - maxSourceAt: `select coalesce(max(fetched_at), '') from pull_request_details`, + sourceAtQuery: `select coalesce(fetched_at, '') from pull_request_details`, }, { name: "pull_request_files", @@ -238,7 +238,7 @@ select thread_id, position, path, coalesce(status, ''), additions, deletions, changes, coalesce(previous_path, ''), fetched_at from pull_request_files order by thread_id, position`, - maxSourceAt: `select coalesce(max(fetched_at), '') from pull_request_files`, + sourceAtQuery: `select coalesce(fetched_at, '') from pull_request_files`, }, } @@ -248,8 +248,8 @@ order by thread_id, position`, if err != nil { return nil, fmt.Errorf("export cloud dataset %s: %w", spec.name, err) } - var maxSourceAt string - if err := db.QueryRowContext(ctx, spec.maxSourceAt).Scan(&maxSourceAt); err != nil { + maxSourceAt, err := latestRFC3339QueryValue(ctx, db, spec.sourceAtQuery) + if err != nil { return nil, fmt.Errorf("read cloud dataset %s freshness: %w", spec.name, err) } datasets = append(datasets, gitcrawlCloudDataset{ @@ -283,9 +283,8 @@ func gitcrawlCloudCapabilities(ctx context.Context, db *sql.DB, observationOrder } func gitcrawlCloudSourceSyncAt(ctx context.Context, db *sql.DB) (string, error) { - var sourceSyncAt string - err := db.QueryRowContext(ctx, ` -select coalesce(max(value), '') + sourceSyncAt, err := latestRFC3339QueryValue(ctx, db, ` +select value from ( select coalesce(finished_at, started_at, '') as value from sync_runs @@ -294,13 +293,47 @@ from ( select coalesce(nullif(updated_at_gh, ''), updated_at, '') from threads union all select coalesce(updated_at, '') from repositories -)`).Scan(&sourceSyncAt) +)`) if err != nil { return "", fmt.Errorf("read cloud snapshot source sync time: %w", err) } return sourceSyncAt, nil } +func latestRFC3339QueryValue(ctx context.Context, db *sql.DB, query string) (string, error) { + rows, err := db.QueryContext(ctx, query) + if err != nil { + return "", err + } + defer rows.Close() + + var latest time.Time + for rows.Next() { + var value string + if err := rows.Scan(&value); err != nil { + return "", err + } + value = strings.TrimSpace(value) + if value == "" { + continue + } + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return "", fmt.Errorf("parse RFC3339 timestamp %q: %w", value, err) + } + if latest.IsZero() || parsed.After(latest) { + latest = parsed + } + } + if err := rows.Err(); err != nil { + return "", err + } + if latest.IsZero() { + return "", nil + } + return latest.UTC().Format(time.RFC3339Nano), nil +} + func gitcrawlCloudHydration(ctx context.Context, snapshotPath string) (crawlstore.EnrichmentCoverage, error) { st, err := crawlstore.OpenReadOnly(ctx, snapshotPath) if err != nil { diff --git a/internal/cli/cloud_snapshot_test.go b/internal/cli/cloud_snapshot_test.go new file mode 100644 index 0000000..9242e98 --- /dev/null +++ b/internal/cli/cloud_snapshot_test.go @@ -0,0 +1,84 @@ +package cli + +import ( + "context" + "database/sql" + "testing" +) + +func TestLatestRFC3339QueryValueUsesParsedTimestampOrder(t *testing.T) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + defer db.Close() + if _, err := db.Exec(` + create table timestamps(value text not null); + insert into timestamps(value) values + ('2026-07-12T01:00:00+02:00'), + ('2026-07-12T00:30:00Z'), + (''); + `); err != nil { + t.Fatalf("seed timestamps: %v", err) + } + + got, err := latestRFC3339QueryValue( + context.Background(), + db, + `select value from timestamps`, + ) + if err != nil { + t.Fatalf("latest timestamp: %v", err) + } + if got != "2026-07-12T00:30:00Z" { + t.Fatalf("latest timestamp = %q, want parsed maximum", got) + } +} + +func TestGitcrawlCloudSourceSyncAtNormalizesRFC3339Maximum(t *testing.T) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + defer db.Close() + if _, err := db.Exec(` + create table sync_runs(status text, started_at text, finished_at text); + create table threads(updated_at_gh text, updated_at text); + create table repositories(updated_at text); + insert into sync_runs(status, started_at, finished_at) + values('success', '2026-07-12T01:00:00+02:00', ''); + insert into threads(updated_at_gh, updated_at) + values('2026-07-12T00:30:00Z', '2026-07-12T00:00:00Z'); + insert into repositories(updated_at) + values('2026-07-12T00:15:00Z'); + `); err != nil { + t.Fatalf("seed source clocks: %v", err) + } + + got, err := gitcrawlCloudSourceSyncAt(context.Background(), db) + if err != nil { + t.Fatalf("source sync at: %v", err) + } + if got != "2026-07-12T00:30:00Z" { + t.Fatalf("source sync at = %q, want parsed maximum", got) + } +} + +func TestLatestRFC3339QueryValueRejectsInvalidTimestamp(t *testing.T) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + defer db.Close() + if _, err := db.Exec(`create table timestamps(value text); insert into timestamps values('not-a-time')`); err != nil { + t.Fatalf("seed timestamp: %v", err) + } + + if _, err := latestRFC3339QueryValue( + context.Background(), + db, + `select value from timestamps`, + ); err == nil { + t.Fatal("invalid RFC3339 timestamp was accepted") + } +} From 2736106b83e2470338ba5b212e1b8581b2361752 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:56:46 +0200 Subject: [PATCH 26/52] fix(cloud): allow additive remote query args --- internal/cli/app_test.go | 36 +++++++++-------------------- internal/cli/cloud_commands.go | 20 ++++++++++++++-- internal/cli/cloud_commands_test.go | 23 ++++++++++++++++++ 3 files changed, 52 insertions(+), 27 deletions(-) create mode 100644 internal/cli/cloud_commands_test.go diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index fd5b7c5..7ffb084 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -1476,32 +1476,18 @@ func TestCloudPublishRejectsIncompleteRemoteSurfaceBeforeUpload(t *testing.T) { }, } for _, required := range gitcrawlCloudReaderQuerySpecs() { - tests = append(tests, - remoteSurfaceTest{ - name: "missing reader query " + required.Name, - want: "required reader query " + required.Name, - mutate: func(contract *crawlremote.Contract) { - contract.Apps[0].Queries = slices.DeleteFunc( - contract.Apps[0].Queries, - func(query crawlremote.QuerySpec) bool { - return query.Name == required.Name - }, - ) - }, - }, - remoteSurfaceTest{ - name: "drifted reader query " + required.Name, - want: "reader query " + required.Name + " has arguments", - mutate: func(contract *crawlremote.Contract) { - for queryIndex := range contract.Apps[0].Queries { - query := &contract.Apps[0].Queries[queryIndex] - if query.Name == required.Name { - query.Args = append(slices.Clone(query.Args), "unexpected") - } - } - }, + tests = append(tests, remoteSurfaceTest{ + name: "missing reader query " + required.Name, + want: "required reader query " + required.Name, + mutate: func(contract *crawlremote.Contract) { + contract.Apps[0].Queries = slices.DeleteFunc( + contract.Apps[0].Queries, + func(query crawlremote.QuerySpec) bool { + return query.Name == required.Name + }, + ) }, - ) + }) } for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 05715a7..4fce2bb 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -595,9 +595,9 @@ func requireGitcrawlSnapshotPublishContract( ) } remoteArgs := appSpec.Queries[queryIndex].Args - if !equalUniqueStringSet(remoteArgs, required.Args) { + if !uniqueStringSuperset(remoteArgs, required.Args) { return fmt.Errorf( - "remote contract reader query %s has arguments %v, want %v", + "remote contract reader query %s has arguments %v, missing required arguments from %v", required.Name, remoteArgs, required.Args, @@ -659,6 +659,22 @@ func equalUniqueStringSet(left, right []string) bool { return len(seen) == 0 } +func uniqueStringSuperset(values, required []string) bool { + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + if _, duplicate := seen[value]; duplicate { + return false + } + seen[value] = struct{}{} + } + for _, value := range required { + if _, ok := seen[value]; !ok { + return false + } + } + return true +} + func gitcrawlPublisherStatusMatches( status crawlremote.PublisherStatus, manifest crawlremote.IngestManifest, diff --git a/internal/cli/cloud_commands_test.go b/internal/cli/cloud_commands_test.go new file mode 100644 index 0000000..887cf4d --- /dev/null +++ b/internal/cli/cloud_commands_test.go @@ -0,0 +1,23 @@ +package cli + +import "testing" + +func TestUniqueStringSupersetAllowsAdditiveArguments(t *testing.T) { + if !uniqueStringSuperset( + []string{"owner", "repo", "query", "limit", "cursor"}, + []string{"owner", "repo", "query", "limit"}, + ) { + t.Fatal("additive optional remote argument was rejected") + } +} + +func TestUniqueStringSupersetRejectsMissingOrDuplicateArguments(t *testing.T) { + for _, values := range [][]string{ + {"owner", "repo"}, + {"owner", "repo", "query", "query"}, + } { + if uniqueStringSuperset(values, []string{"owner", "repo", "query"}) { + t.Fatalf("invalid remote arguments accepted: %v", values) + } + } +} From e063d6e489e8017d5adb565717ca0814cb1e40b9 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:58:44 +0200 Subject: [PATCH 27/52] fix(cloud): verify bound snapshot hydration --- internal/cli/app_test.go | 119 +++++++++++++++++++++++++++- internal/cli/cloud_commands.go | 114 ++++++++++++++++++++++++-- internal/cli/cloud_commands_test.go | 102 +++++++++++++++++++++++- 3 files changed, 323 insertions(+), 12 deletions(-) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 7ffb084..a4ca7e3 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -839,6 +839,11 @@ func TestRemoteCloudModeDoesNotCreateLocalDB(t *testing.T) { func testSnapshotPublishContract() crawlremote.Contract { contract := crawlremote.BaseContract() + contract.Routes = append(contract.Routes, crawlremote.RouteSpec{ + Method: http.MethodGet, + Path: "/v1/apps/:app/archives/:archive/sqlite", + Auth: crawlremote.AuthReader, + }) contract.Apps = []crawlremote.AppSpec{{ App: "gitcrawl", Queries: []crawlremote.QuerySpec{ @@ -887,6 +892,7 @@ func testSnapshotPublishContract() crawlremote.Contract { Capabilities: []string{ gitcrawlSnapshotAtomicCapability, gitcrawlSnapshotCutoverCapability, + gitcrawlSnapshotHydrationCapability, gitcrawlSnapshotProvenanceCapability, sqliteBundleGzipUploadCapability, }, @@ -1015,7 +1021,10 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { var sawSQLitePart bool var sawSQLiteManifest bool var sawCutover bool + var sawSQLiteRead bool var snapshotID string + var sqliteImage []byte + var publishedSnapshot *crawlremote.ArchiveSnapshot mutationCounter := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { @@ -1073,6 +1082,7 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { http.Error(w, "bundle did not contain sqlite", http.StatusBadRequest) return } + sqliteImage = decompressed _ = json.NewEncoder(w).Encode(crawlremote.SQLiteUploadResult{ App: "gitcrawl", Archive: "gitcrawl/openclaw__openclaw", @@ -1123,7 +1133,17 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { return } if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status") { - http.NotFound(w, r) + if publishedSnapshot == nil { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(crawlremote.PublisherStatus{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__openclaw", + ActiveSnapshotID: snapshotID, + CoverageComplete: true, + Snapshot: publishedSnapshot, + }) return } if r.Method == http.MethodPost && strings.HasSuffix(r.URL.EscapedPath(), "/cutover") { @@ -1143,6 +1163,17 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { }) return } + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/sqlite") { + if !sawCutover || len(sqliteImage) == 0 { + http.Error(w, "bound snapshot unavailable", http.StatusConflict) + return + } + sawSQLiteRead = true + w.Header().Set("content-type", "application/vnd.sqlite3") + w.Header().Set("x-crawl-content-sha256", snapshotID) + _, _ = w.Write(sqliteImage) + return + } if r.Method != http.MethodPost || r.URL.EscapedPath() != "/v1/apps/gitcrawl/archives/gitcrawl%2Fopenclaw__openclaw/ingest" { http.NotFound(w, r) return @@ -1170,6 +1201,19 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { return } } + publishedSnapshot = &crawlremote.ArchiveSnapshot{ + ID: body.Manifest.SnapshotID, + SourceSHA256: body.Manifest.SourceSHA256, + SchemaName: body.Manifest.SchemaName, + SchemaVersion: body.Manifest.SchemaVersion, + SchemaHash: body.Manifest.SchemaHash, + Capabilities: gitcrawlCloudPublicationCapabilities( + body.Manifest.Capabilities, + ), + SourceSyncAt: body.Manifest.SourceSyncAt, + DatasetGeneratedAt: fmt.Sprint(body.Rows[0][5]), + CoverageComplete: true, + } } seenTables[body.Table] = body mutationCounter++ @@ -1224,6 +1268,9 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { if !sawCutover { t.Fatal("cloud publish did not cut over the activated snapshot") } + if !sawSQLiteRead { + t.Fatal("cloud publish did not verify the readable bound SQLite snapshot") + } var payload map[string]any if err := json.Unmarshal(out.Bytes(), &payload); err != nil { t.Fatalf("decode output: %v\n%s", err, out.String()) @@ -1307,6 +1354,10 @@ func TestCloudPublishRejectsMissingRequestedCapabilityBeforeUpload(t *testing.T) name: "cutover", missingCapability: gitcrawlSnapshotCutoverCapability, }, + { + name: "bound snapshot hydration", + missingCapability: gitcrawlSnapshotHydrationCapability, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -1439,6 +1490,32 @@ func TestCloudPublishRejectsIncompleteRemoteSurfaceBeforeUpload(t *testing.T) { ) }, }, + { + name: "missing SQLite hydration route", + want: "GET /v1/apps/:app/archives/:archive/sqlite", + mutate: func(contract *crawlremote.Contract) { + contract.Routes = slices.DeleteFunc( + contract.Routes, + func(route crawlremote.RouteSpec) bool { + return route.Method == http.MethodGet && + route.Path == "/v1/apps/:app/archives/:archive/sqlite" + }, + ) + }, + }, + { + name: "SQLite hydration route with publisher auth", + want: "GET /v1/apps/:app/archives/:archive/sqlite", + mutate: func(contract *crawlremote.Contract) { + for index := range contract.Routes { + route := &contract.Routes[index] + if route.Method == http.MethodGet && + route.Path == "/v1/apps/:app/archives/:archive/sqlite" { + route.Auth = crawlremote.AuthPublisher + } + } + }, + }, { name: "zero reader queries", want: "required reader query gitcrawl.threads.search", @@ -1556,9 +1633,11 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { var snapshotID string var stagedSnapshot *crawlremote.ArchiveSnapshot servingSnapshotID := strings.Repeat("f", 64) + var sqliteImage []byte ingestRequests := 0 publisherStatusRequests := 0 readerStatusRequests := 0 + sqliteReadRequests := 0 cutovers := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { @@ -1595,6 +1674,17 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { http.Error(w, "missing bearer", http.StatusUnauthorized) return } + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/sqlite") { + sqliteReadRequests++ + if servingSnapshotID != snapshotID || len(sqliteImage) == 0 { + http.Error(w, "bound snapshot unavailable", http.StatusConflict) + return + } + w.Header().Set("content-type", "application/vnd.sqlite3") + w.Header().Set("x-crawl-content-sha256", snapshotID) + _, _ = w.Write(sqliteImage) + return + } if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status") { publisherStatusRequests++ activeSnapshotID := "" @@ -1621,6 +1711,24 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { http.Error(w, "invalid bundle part", http.StatusBadRequest) return } + compressed, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + reader, err := gzip.NewReader(bytes.NewReader(compressed)) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + sqliteImage, err = io.ReadAll(reader) + if closeErr := reader.Close(); err == nil { + err = closeErr + } + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } _ = json.NewEncoder(w).Encode(crawlremote.SQLiteUploadResult{ App: "gitcrawl", Archive: "gitcrawl/openclaw__openclaw", @@ -1815,11 +1923,14 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { if servingSnapshotID != snapshotID { t.Fatalf("serving snapshot = %q, want staged snapshot %q", servingSnapshotID, snapshotID) } - if publisherStatusRequests != 2 { - t.Fatalf("publisher status requests = %d, want 2", publisherStatusRequests) + if publisherStatusRequests != 3 { + t.Fatalf("publisher status requests = %d, want 3", publisherStatusRequests) } if readerStatusRequests != 0 { - t.Fatalf("reader status requests = %d, publisher-only flow must not require reader role", readerStatusRequests) + t.Fatalf("reader status requests = %d, post-cutover verification should use publisher status", readerStatusRequests) + } + if sqliteReadRequests != 1 { + t.Fatalf("SQLite read requests = %d, want 1 post-cutover hydration proof", sqliteReadRequests) } } diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 4fce2bb..bb14707 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "net/http" + "net/url" "os" "path/filepath" "slices" @@ -25,6 +26,7 @@ const ( gitcrawlSnapshotAtomicCapability = "gitcrawl.snapshot.atomic" gitcrawlSnapshotCutoverCapability = "gitcrawl.snapshot.cutover" + gitcrawlSnapshotHydrationCapability = "gitcrawl.snapshot.hydration.v1" gitcrawlSnapshotProvenanceCapability = "gitcrawl.snapshot.provenance.v1" sqliteBundleGzipUploadCapability = "sqlite.bundle.gzip.upload" ) @@ -109,9 +111,12 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { Archive: archiveID, TokenEnv: firstNonEmpty(*tokenEnv, cfg.Remote.TokenEnv, crawlremote.DefaultTokenEnv), } + httpClient := &http.Client{Timeout: 10 * time.Minute} + tokenProvider := crawlremote.EnvTokenProvider{Name: remoteCfg.TokenEnv} client, err := crawlremote.NewClientFromConfig(remoteCfg, crawlremote.Options{ - UserAgent: "gitcrawl/" + version, - HTTPClient: &http.Client{Timeout: 10 * time.Minute}, + UserAgent: "gitcrawl/" + version, + HTTPClient: httpClient, + TokenProvider: tokenProvider, }) if err != nil { return err @@ -219,6 +224,19 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { return fmt.Errorf("cut over cloud snapshot: %w", err) } cutoverResult = &result + if err := verifyGitcrawlSnapshotPublication( + ctx, + client, + httpClient, + tokenProvider, + endpoint, + archiveID, + snapshot, + manifest, + publicationCapabilities, + ); err != nil { + return fmt.Errorf("verify published cloud snapshot: %w", err) + } } sqliteBundlePrivacy := gitcrawlCloudSQLiteBundlePrivacy() return a.writeOutput("cloud publish", map[string]any{ @@ -523,6 +541,7 @@ func requireGitcrawlSnapshotPublishContract( requiredCapabilities = append( requiredCapabilities, gitcrawlSnapshotCutoverCapability, + gitcrawlSnapshotHydrationCapability, ) } for _, capability := range requiredCapabilities { @@ -556,11 +575,19 @@ func requireGitcrawlSnapshotPublishContract( }, } if cutover { - requiredRoutes = append(requiredRoutes, crawlremote.RouteSpec{ - Method: http.MethodPost, - Path: "/v1/apps/:app/archives/:archive/cutover", - Auth: crawlremote.AuthPublisher, - }) + requiredRoutes = append( + requiredRoutes, + crawlremote.RouteSpec{ + Method: http.MethodPost, + Path: "/v1/apps/:app/archives/:archive/cutover", + Auth: crawlremote.AuthPublisher, + }, + crawlremote.RouteSpec{ + Method: http.MethodGet, + Path: "/v1/apps/:app/archives/:archive/sqlite", + Auth: crawlremote.AuthReader, + }, + ) } for _, required := range requiredRoutes { if !slices.ContainsFunc(contract.Routes, func(route crawlremote.RouteSpec) bool { @@ -703,6 +730,79 @@ func gitcrawlPublisherStatusMatches( return true } +func verifyGitcrawlSnapshotPublication( + ctx context.Context, + client *crawlremote.Client, + httpClient *http.Client, + tokenProvider crawlremote.TokenProvider, + endpoint, archive string, + snapshot gitcrawlCloudSnapshot, + manifest crawlremote.IngestManifest, + publicationCapabilities []string, +) error { + status, err := client.PublishStatus(ctx, "gitcrawl", archive) + if err != nil { + return fmt.Errorf("read post-cutover publisher status: %w", err) + } + if !gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) || + status.Snapshot.DatasetGeneratedAt != snapshot.DatasetGeneratedAt { + return fmt.Errorf( + "post-cutover publisher status does not match snapshot %s digest, profile, generation, and coverage", + snapshot.ID, + ) + } + + token, err := tokenProvider.Token(ctx) + if err != nil { + return fmt.Errorf("read remote token for snapshot hydration: %w", err) + } + sqliteURL := strings.TrimRight(endpoint, "/") + + "/v1/apps/" + url.PathEscape("gitcrawl") + + "/archives/" + url.PathEscape(archive) + + "/sqlite" + request, err := http.NewRequestWithContext(ctx, http.MethodGet, sqliteURL, nil) + if err != nil { + return fmt.Errorf("build snapshot hydration request: %w", err) + } + request.Header.Set("accept", "application/vnd.sqlite3, application/octet-stream") + request.Header.Set("authorization", "Bearer "+token) + request.Header.Set("user-agent", "gitcrawl/"+version) + response, err := httpClient.Do(request) + if err != nil { + return fmt.Errorf("download bound SQLite snapshot: %w", err) + } + defer response.Body.Close() + if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { + body, _ := io.ReadAll(io.LimitReader(response.Body, 1<<20)) + return fmt.Errorf( + "download bound SQLite snapshot: status=%d body=%s", + response.StatusCode, + strings.TrimSpace(string(body)), + ) + } + if advertised := strings.TrimSpace(response.Header.Get("x-crawl-content-sha256")); advertised != "" && + !strings.EqualFold(advertised, snapshot.ID) { + return fmt.Errorf( + "downloaded SQLite snapshot advertises digest %s, want %s", + advertised, + snapshot.ID, + ) + } + hash := sha256.New() + if _, err := io.Copy(hash, response.Body); err != nil { + return fmt.Errorf("hash downloaded SQLite snapshot: %w", err) + } + actual := fmt.Sprintf("%x", hash.Sum(nil)) + if actual != snapshot.ID { + return fmt.Errorf( + "downloaded SQLite snapshot digest %s does not match source %s", + actual, + snapshot.ID, + ) + } + return nil +} + func gitcrawlCloudSQLiteBundlePrivacy() map[string]any { return map[string]any{ "includes_private_messages": true, diff --git a/internal/cli/cloud_commands_test.go b/internal/cli/cloud_commands_test.go index 887cf4d..5f99435 100644 --- a/internal/cli/cloud_commands_test.go +++ b/internal/cli/cloud_commands_test.go @@ -1,6 +1,18 @@ package cli -import "testing" +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + crawlremote "github.com/openclaw/crawlkit/remote" +) func TestUniqueStringSupersetAllowsAdditiveArguments(t *testing.T) { if !uniqueStringSuperset( @@ -21,3 +33,91 @@ func TestUniqueStringSupersetRejectsMissingOrDuplicateArguments(t *testing.T) { } } } + +func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t *testing.T) { + source := []byte("SQLite format 3\x00bound source") + snapshotID := fmt.Sprintf("%x", sha256.Sum256(source)) + snapshot := gitcrawlCloudSnapshot{ + ID: snapshotID, + SourceSyncAt: "2026-07-12T12:00:00Z", + DatasetGeneratedAt: "2026-07-12T12:01:00Z", + } + manifest := gitcrawlCloudManifest("gitcrawl/openclaw__openclaw", snapshot) + publicationCapabilities := gitcrawlCloudPublicationCapabilities(snapshot.Capabilities) + + for _, test := range []struct { + name string + statusCode int + body []byte + want string + }{ + { + name: "bound snapshot unavailable", + statusCode: http.StatusConflict, + body: []byte(`{"error":"snapshot_bundle_required"}`), + want: "status=409", + }, + { + name: "downloaded digest mismatch", + statusCode: http.StatusOK, + body: []byte("different sqlite image"), + want: "does not match source", + }, + } { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status"): + w.Header().Set("content-type", "application/json") + _ = json.NewEncoder(w).Encode(crawlremote.PublisherStatus{ + App: "gitcrawl", + Archive: manifest.Archive, + ActiveSnapshotID: snapshotID, + CoverageComplete: true, + Snapshot: &crawlremote.ArchiveSnapshot{ + ID: snapshotID, + SourceSHA256: snapshotID, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: publicationCapabilities, + }, + }) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/sqlite"): + w.WriteHeader(test.statusCode) + _, _ = w.Write(test.body) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + httpClient := &http.Client{Timeout: time.Second} + tokenProvider := crawlremote.StaticToken("reader-publisher-token") + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: httpClient, + TokenProvider: tokenProvider, + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + err = verifyGitcrawlSnapshotPublication( + context.Background(), + client, + httpClient, + tokenProvider, + server.URL, + manifest.Archive, + snapshot, + manifest, + publicationCapabilities, + ) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("verification error = %v, want %q", err, test.want) + } + }) + } +} From 2fd1ac8945002cd45152245ea44c56e0d69b1552 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 15:58:46 +0200 Subject: [PATCH 28/52] docs(cloud): require readable snapshot cutover --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 48f8628..0a5730e 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,11 @@ the existing reader-refresh behavior. `--stage-only` keeps the immutable snapshot staged without changing serving state. A later publish verifies the candidate through the publisher-only status projection, skips repeated ingest when its digest, source sync, schema, resolved publication profile, persisted -generation timestamp, and coverage match, then cuts it over. +generation timestamp, and coverage match, then cuts it over. Cutover requires +the remote contract to advertise reader-authenticated `GET /sqlite`; Gitcrawl +rechecks the exact publisher metadata and hashes the downloaded bound SQLite +image before reporting success. The publish credential therefore needs both +publisher and reader access for cutover. Incomplete local enrichment fails before any remote mutation; `--allow-incomplete` is an explicit escape hatch, and `--observation-order` publishes durable fetch ordering after the remote operator fence is enabled. From 00e87389e20d25ceb6b60956aa25891cc4ac72ab Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:00:17 +0200 Subject: [PATCH 29/52] fix(cloud): align bound hydration contract --- internal/cli/app_test.go | 5 ----- internal/cli/cloud_commands.go | 2 -- 2 files changed, 7 deletions(-) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index a4ca7e3..7613588 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -892,7 +892,6 @@ func testSnapshotPublishContract() crawlremote.Contract { Capabilities: []string{ gitcrawlSnapshotAtomicCapability, gitcrawlSnapshotCutoverCapability, - gitcrawlSnapshotHydrationCapability, gitcrawlSnapshotProvenanceCapability, sqliteBundleGzipUploadCapability, }, @@ -1354,10 +1353,6 @@ func TestCloudPublishRejectsMissingRequestedCapabilityBeforeUpload(t *testing.T) name: "cutover", missingCapability: gitcrawlSnapshotCutoverCapability, }, - { - name: "bound snapshot hydration", - missingCapability: gitcrawlSnapshotHydrationCapability, - }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index bb14707..a958254 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -26,7 +26,6 @@ const ( gitcrawlSnapshotAtomicCapability = "gitcrawl.snapshot.atomic" gitcrawlSnapshotCutoverCapability = "gitcrawl.snapshot.cutover" - gitcrawlSnapshotHydrationCapability = "gitcrawl.snapshot.hydration.v1" gitcrawlSnapshotProvenanceCapability = "gitcrawl.snapshot.provenance.v1" sqliteBundleGzipUploadCapability = "sqlite.bundle.gzip.upload" ) @@ -541,7 +540,6 @@ func requireGitcrawlSnapshotPublishContract( requiredCapabilities = append( requiredCapabilities, gitcrawlSnapshotCutoverCapability, - gitcrawlSnapshotHydrationCapability, ) } for _, capability := range requiredCapabilities { From 3f0f7967145d2c8b5f24a59c6924eeff02f80ed7 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:11:58 +0200 Subject: [PATCH 30/52] fix(cloud): harden snapshot publication integrity --- internal/cli/cloud_commands.go | 127 ++++++++++++++++++++++++++++----- 1 file changed, 110 insertions(+), 17 deletions(-) diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index a958254..387f5bf 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -13,6 +13,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "time" @@ -21,8 +22,10 @@ import ( ) const ( - gitcrawlCloudBatchSize = 250 - gitcrawlCloudSQLiteBundleChunkSize = int64(64 * 1024 * 1024) + gitcrawlCloudBatchSize = 250 + gitcrawlCloudSQLiteBundleChunkSize = int64(64 * 1024 * 1024) + gitcrawlCloudPublishPreflightTimeout = 30 * time.Second + gitcrawlCloudHydrationTimeout = 10 * time.Minute gitcrawlSnapshotAtomicCapability = "gitcrawl.snapshot.atomic" gitcrawlSnapshotCutoverCapability = "gitcrawl.snapshot.cutover" @@ -151,7 +154,10 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { ); err != nil { return err } - sqliteBundle, err := uploadSQLiteSnapshotArchive( + if err := requireGitcrawlCloudPublishRoles(ctx, client); err != nil { + return err + } + sqliteBundle, sqliteSourceSize, err := uploadSQLiteSnapshotArchive( ctx, client, "gitcrawl", @@ -233,6 +239,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { snapshot, manifest, publicationCapabilities, + sqliteSourceSize, ); err != nil { return fmt.Errorf("verify published cloud snapshot: %w", err) } @@ -473,7 +480,8 @@ func uploadSQLiteArchive(ctx context.Context, client *crawlremote.Client, app, a return nil, err } defer cleanup() - return uploadSQLiteSnapshotArchive(ctx, client, app, archive, snapshotPath, counts) + bundle, _, err := uploadSQLiteSnapshotArchive(ctx, client, app, archive, snapshotPath, counts) + return bundle, err } func uploadSQLiteSnapshotArchive( @@ -481,7 +489,7 @@ func uploadSQLiteSnapshotArchive( client *crawlremote.Client, app, archive, snapshotPath string, counts map[string]int64, -) (*crawlremote.SQLiteBundle, error) { +) (*crawlremote.SQLiteBundle, int64, error) { bundle, err := crawlremote.BuildSnapshotGzipSQLiteBundle(ctx, crawlremote.SQLiteBundleBuildOptions{ App: app, Archive: archive, @@ -491,14 +499,18 @@ func uploadSQLiteSnapshotArchive( Privacy: gitcrawlCloudSQLiteBundlePrivacy(), }) if err != nil { - return nil, err + return nil, 0, err } defer bundle.Cleanup() + sourceSize := bundle.Manifest.Object.Size + if sourceSize <= 0 { + return nil, 0, fmt.Errorf("SQLite bundle manifest has invalid source size %d", sourceSize) + } result, err := client.UploadSQLiteBundleFiles(ctx, app, archive, bundle.Manifest, bundle.Parts) if err != nil { - return nil, err + return nil, 0, err } - return result.Bundle, nil + return result.Bundle, sourceSize, nil } func remoteNotFound(err error) bool { @@ -551,6 +563,11 @@ func requireGitcrawlSnapshotPublishContract( } } requiredRoutes := []crawlremote.RouteSpec{ + { + Method: http.MethodGet, + Path: "/v1/whoami", + Auth: crawlremote.AuthReader, + }, { Method: http.MethodGet, Path: "/v1/apps/:app/archives/:archive/publish-status", @@ -664,6 +681,28 @@ func requireGitcrawlSnapshotPublishContract( return nil } +func requireGitcrawlCloudPublishRoles(ctx context.Context, client *crawlremote.Client) error { + preflightCtx, cancel := context.WithTimeout(ctx, gitcrawlCloudPublishPreflightTimeout) + defer cancel() + identity, err := client.Whoami(preflightCtx) + if err != nil { + return fmt.Errorf("read remote identity before snapshot publication: %w", err) + } + missing := make([]string, 0, 2) + for _, role := range []string{crawlremote.AuthPublisher, crawlremote.AuthReader} { + if !slices.Contains(identity.Roles, role) { + missing = append(missing, role) + } + } + if len(missing) > 0 { + return fmt.Errorf( + "remote token must have publisher and reader roles before snapshot publication; missing %s", + strings.Join(missing, ", "), + ) + } + return nil +} + func equalUniqueStringSet(left, right []string) bool { if len(left) != len(right) { return false @@ -719,7 +758,8 @@ func gitcrawlPublisherStatusMatches( strings.TrimSpace(snapshot.DatasetGeneratedAt) == "" { return false } - if !status.CoverageComplete { + if status.CoverageComplete != snapshot.CoverageComplete || + !status.CoverageComplete { return false } if !equalUniqueStringSet(snapshot.Capabilities, publicationCapabilities) { @@ -737,6 +777,7 @@ func verifyGitcrawlSnapshotPublication( snapshot gitcrawlCloudSnapshot, manifest crawlremote.IngestManifest, publicationCapabilities []string, + sourceSize int64, ) error { status, err := client.PublishStatus(ctx, "gitcrawl", archive) if err != nil { @@ -758,7 +799,9 @@ func verifyGitcrawlSnapshotPublication( "/v1/apps/" + url.PathEscape("gitcrawl") + "/archives/" + url.PathEscape(archive) + "/sqlite" - request, err := http.NewRequestWithContext(ctx, http.MethodGet, sqliteURL, nil) + hydrationCtx, cancel := context.WithTimeout(ctx, gitcrawlCloudHydrationTimeout) + defer cancel() + request, err := http.NewRequestWithContext(hydrationCtx, http.MethodGet, sqliteURL, nil) if err != nil { return fmt.Errorf("build snapshot hydration request: %w", err) } @@ -778,24 +821,74 @@ func verifyGitcrawlSnapshotPublication( strings.TrimSpace(string(body)), ) } - if advertised := strings.TrimSpace(response.Header.Get("x-crawl-content-sha256")); advertised != "" && - !strings.EqualFold(advertised, snapshot.ID) { + return verifyGitcrawlSQLiteHydration(response, snapshot.ID, sourceSize) +} + +func verifyGitcrawlSQLiteHydration( + response *http.Response, + expectedDigest string, + expectedSize int64, +) error { + advertised := strings.TrimSpace(response.Header.Get("x-crawl-content-sha256")) + if advertised == "" { + return fmt.Errorf("downloaded SQLite snapshot is missing x-crawl-content-sha256") + } + if !strings.EqualFold(advertised, expectedDigest) { return fmt.Errorf( "downloaded SQLite snapshot advertises digest %s, want %s", advertised, - snapshot.ID, + expectedDigest, + ) + } + if expectedSize <= 0 { + return fmt.Errorf("uploaded SQLite manifest source size must be positive, got %d", expectedSize) + } + contentLength := response.ContentLength + if contentLength <= 0 { + header := strings.TrimSpace(response.Header.Get("content-length")) + if header == "" { + return fmt.Errorf("downloaded SQLite snapshot is missing a positive Content-Length") + } + parsed, err := strconv.ParseInt(header, 10, 64) + if err != nil || parsed <= 0 { + return fmt.Errorf("downloaded SQLite snapshot has invalid Content-Length %q", header) + } + contentLength = parsed + } + if contentLength != expectedSize { + return fmt.Errorf( + "downloaded SQLite snapshot Content-Length %d does not match uploaded source size %d", + contentLength, + expectedSize, ) } hash := sha256.New() - if _, err := io.Copy(hash, response.Body); err != nil { - return fmt.Errorf("hash downloaded SQLite snapshot: %w", err) + written, err := io.CopyN(hash, response.Body, expectedSize) + if err != nil { + return fmt.Errorf( + "downloaded SQLite snapshot truncated after %d of %d bytes: %w", + written, + expectedSize, + err, + ) + } + var extra [1]byte + n, err := response.Body.Read(extra[:]) + if n > 0 { + return fmt.Errorf( + "downloaded SQLite snapshot exceeds Content-Length %d", + expectedSize, + ) + } + if err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("check downloaded SQLite snapshot boundary: %w", err) } actual := fmt.Sprintf("%x", hash.Sum(nil)) - if actual != snapshot.ID { + if actual != expectedDigest { return fmt.Errorf( "downloaded SQLite snapshot digest %s does not match source %s", actual, - snapshot.ID, + expectedDigest, ) } return nil From 5db7271f3a7c9349317581a3313a3bbe50c5a827 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:12:00 +0200 Subject: [PATCH 31/52] test(cloud): cover publication integrity guards --- internal/cli/app_test.go | 109 +++++++++++++++++++++++++++- internal/cli/cloud_commands_test.go | 101 +++++++++++++++++++++++++- 2 files changed, 208 insertions(+), 2 deletions(-) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 7613588..43e8cfe 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -899,6 +899,72 @@ func testSnapshotPublishContract() crawlremote.Contract { return contract } +func TestCloudPublishRejectsPublisherOnlyTokenBeforeMutation(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + cfg := config.Default() + cfg.DBPath = dbPath + if err := config.Save(configPath, cfg); err != nil { + t.Fatalf("save config: %v", err) + } + seedCommandFlowStore(t, dbPath) + + const tokenEnv = "GITCRAWL_TEST_PUBLISHER_ONLY_TOKEN" + t.Setenv(tokenEnv, "publisher-only-token") + whoamiRequests := 0 + mutations := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract": + w.Header().Set("content-type", "application/json") + _ = json.NewEncoder(w).Encode(testSnapshotPublishContract()) + case r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/whoami": + whoamiRequests++ + if got := r.Header.Get("authorization"); got != "Bearer publisher-only-token" { + http.Error(w, "missing bearer", http.StatusUnauthorized) + return + } + w.Header().Set("content-type", "application/json") + _ = json.NewEncoder(w).Encode(crawlremote.Identity{ + Owner: "openclaw", + Org: "openclaw", + Login: "publisher", + Roles: []string{crawlremote.AuthPublisher}, + }) + default: + if r.Method == http.MethodPost || + r.Method == http.MethodPut || + r.Method == http.MethodPatch || + r.Method == http.MethodDelete { + mutations++ + } + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + })) + defer server.Close() + + err := New().Run(ctx, []string{ + "--config", configPath, + "cloud", "publish", + "--remote", server.URL, + "--archive", "gitcrawl/openclaw__openclaw", + "--token-env", tokenEnv, + "--allow-incomplete", + "--json", + }) + if err == nil || !strings.Contains(err.Error(), "missing reader") { + t.Fatalf("cloud publish error = %v, want missing reader role", err) + } + if whoamiRequests != 1 { + t.Fatalf("whoami requests = %d, want 1", whoamiRequests) + } + if mutations != 0 { + t.Fatalf("remote mutations = %d, want 0", mutations) + } +} + func TestGitcrawlPublisherStatusMatchesExactMetadata(t *testing.T) { snapshotID := strings.Repeat("a", 64) manifest := crawlremote.IngestManifest{ @@ -926,7 +992,7 @@ func TestGitcrawlPublisherStatusMatchesExactMetadata(t *testing.T) { SchemaVersion: gitcrawlCloudSchemaVersion, SchemaHash: gitcrawlCloudSchemaHash, Capabilities: publicationCapabilities, - CoverageComplete: false, + CoverageComplete: true, }, } if !gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) { @@ -977,6 +1043,14 @@ func TestGitcrawlPublisherStatusMatchesExactMetadata(t *testing.T) { } status.CoverageComplete = true + nestedIncomplete := status + nestedIncomplete.Snapshot = &crawlremote.ArchiveSnapshot{} + *nestedIncomplete.Snapshot = *status.Snapshot + nestedIncomplete.Snapshot.CoverageComplete = false + if gitcrawlPublisherStatusMatches(nestedIncomplete, manifest, publicationCapabilities) { + t.Fatal("snapshot with nested incomplete coverage was reused") + } + manifest.Capabilities = []string{gitcrawlObservationOrderCapability} publicationCapabilities = gitcrawlCloudPublicationCapabilities(manifest.Capabilities) if gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) { @@ -1035,6 +1109,15 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { http.Error(w, "missing bearer", http.StatusUnauthorized) return } + if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/whoami" { + _ = json.NewEncoder(w).Encode(crawlremote.Identity{ + Owner: "openclaw", + Org: "openclaw", + Login: "publisher", + Roles: []string{crawlremote.AuthPublisher, crawlremote.AuthReader}, + }) + return + } if r.Method == http.MethodPut && r.URL.EscapedPath() == "/v1/apps/gitcrawl/archives/gitcrawl%2Fopenclaw__openclaw/sqlite" { uploadKind := r.Header.Get("x-crawl-sqlite-upload") payload, err := io.ReadAll(r.Body) @@ -1169,6 +1252,7 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { } sawSQLiteRead = true w.Header().Set("content-type", "application/vnd.sqlite3") + w.Header().Set("content-length", strconv.Itoa(len(sqliteImage))) w.Header().Set("x-crawl-content-sha256", snapshotID) _, _ = w.Write(sqliteImage) return @@ -1418,6 +1502,19 @@ func TestCloudPublishRejectsIncompleteRemoteSurfaceBeforeUpload(t *testing.T) { extraArgs []string } tests := []remoteSurfaceTest{ + { + name: "missing authenticated whoami route", + want: "GET /v1/whoami", + mutate: func(contract *crawlremote.Contract) { + contract.Routes = slices.DeleteFunc( + contract.Routes, + func(route crawlremote.RouteSpec) bool { + return route.Method == http.MethodGet && + route.Path == "/v1/whoami" + }, + ) + }, + }, { name: "missing ingest column", want: "pull_request_files is missing required column position", @@ -1669,6 +1766,15 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { http.Error(w, "missing bearer", http.StatusUnauthorized) return } + if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/whoami" { + _ = json.NewEncoder(w).Encode(crawlremote.Identity{ + Owner: "openclaw", + Org: "openclaw", + Login: "publisher", + Roles: []string{crawlremote.AuthPublisher, crawlremote.AuthReader}, + }) + return + } if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/sqlite") { sqliteReadRequests++ if servingSnapshotID != snapshotID || len(sqliteImage) == 0 { @@ -1676,6 +1782,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { return } w.Header().Set("content-type", "application/vnd.sqlite3") + w.Header().Set("content-length", strconv.Itoa(len(sqliteImage))) w.Header().Set("x-crawl-content-sha256", snapshotID) _, _ = w.Write(sqliteImage) return diff --git a/internal/cli/cloud_commands_test.go b/internal/cli/cloud_commands_test.go index 5f99435..e562f68 100644 --- a/internal/cli/cloud_commands_test.go +++ b/internal/cli/cloud_commands_test.go @@ -1,10 +1,12 @@ package cli import ( + "bytes" "context" "crypto/sha256" "encoding/json" "fmt" + "io" "net/http" "net/http/httptest" "strings" @@ -60,7 +62,7 @@ func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t { name: "downloaded digest mismatch", statusCode: http.StatusOK, - body: []byte("different sqlite image"), + body: bytes.Repeat([]byte("x"), len(source)), want: "does not match source", }, } { @@ -83,9 +85,14 @@ func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t SchemaVersion: manifest.SchemaVersion, SchemaHash: manifest.SchemaHash, Capabilities: publicationCapabilities, + CoverageComplete: true, }, }) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/sqlite"): + if test.statusCode == http.StatusOK { + w.Header().Set("content-length", fmt.Sprintf("%d", len(test.body))) + w.Header().Set("x-crawl-content-sha256", snapshotID) + } w.WriteHeader(test.statusCode) _, _ = w.Write(test.body) default: @@ -114,6 +121,7 @@ func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t snapshot, manifest, publicationCapabilities, + int64(len(source)), ) if err == nil || !strings.Contains(err.Error(), test.want) { t.Fatalf("verification error = %v, want %q", err, test.want) @@ -121,3 +129,94 @@ func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t }) } } + +func TestVerifyGitcrawlSQLiteHydrationRejectsInvalidFramingAndDigest(t *testing.T) { + source := []byte("SQLite format 3\x00bound source") + snapshotID := fmt.Sprintf("%x", sha256.Sum256(source)) + different := bytes.Repeat([]byte("x"), len(source)) + + tests := []struct { + name string + body []byte + digest string + contentLength int64 + want string + }{ + { + name: "valid", + body: source, + digest: snapshotID, + contentLength: int64(len(source)), + }, + { + name: "missing digest", + body: source, + contentLength: int64(len(source)), + want: "missing x-crawl-content-sha256", + }, + { + name: "wrong digest", + body: source, + digest: strings.Repeat("f", 64), + contentLength: int64(len(source)), + want: "advertises digest", + }, + { + name: "missing length", + body: source, + digest: snapshotID, + contentLength: -1, + want: "missing a positive Content-Length", + }, + { + name: "wrong length", + body: source, + digest: snapshotID, + contentLength: int64(len(source) + 1), + want: "does not match uploaded source size", + }, + { + name: "truncated body", + body: source[:len(source)-1], + digest: snapshotID, + contentLength: int64(len(source)), + want: "truncated", + }, + { + name: "oversized body", + body: append(append([]byte(nil), source...), 'x'), + digest: snapshotID, + contentLength: int64(len(source)), + want: "exceeds Content-Length", + }, + { + name: "body digest mismatch", + body: different, + digest: snapshotID, + contentLength: int64(len(source)), + want: "does not match source", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + response := &http.Response{ + Header: make(http.Header), + Body: io.NopCloser(bytes.NewReader(test.body)), + ContentLength: test.contentLength, + } + if test.digest != "" { + response.Header.Set("x-crawl-content-sha256", test.digest) + } + err := verifyGitcrawlSQLiteHydration(response, snapshotID, int64(len(source))) + if test.want == "" { + if err != nil { + t.Fatalf("verify hydration: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("verification error = %v, want %q", err, test.want) + } + }) + } +} From 90bb68359d01ec1312597d94801773440d7b49b1 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:12:01 +0200 Subject: [PATCH 32/52] docs(cloud): document dual-role publish preflight --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0a5730e..3f0960e 100644 --- a/README.md +++ b/README.md @@ -80,8 +80,9 @@ when its digest, source sync, schema, resolved publication profile, persisted generation timestamp, and coverage match, then cuts it over. Cutover requires the remote contract to advertise reader-authenticated `GET /sqlite`; Gitcrawl rechecks the exact publisher metadata and hashes the downloaded bound SQLite -image before reporting success. The publish credential therefore needs both -publisher and reader access for cutover. +image before reporting success. Before any upload or ingest, Gitcrawl verifies +the configured credential through the advertised `/v1/whoami` route and +requires both publisher and reader roles, including for stage-only publication. Incomplete local enrichment fails before any remote mutation; `--allow-incomplete` is an explicit escape hatch, and `--observation-order` publishes durable fetch ordering after the remote operator fence is enabled. From e927c0c6acd39080286a74257891c4c6e04816d6 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:26:50 +0200 Subject: [PATCH 33/52] fix(cloud): support portable snapshot preflight --- internal/cli/cloud_snapshot.go | 165 +++++++++++++++++++++------- internal/cli/cloud_snapshot_test.go | 150 +++++++++++++++++++++++++ 2 files changed, 277 insertions(+), 38 deletions(-) diff --git a/internal/cli/cloud_snapshot.go b/internal/cli/cloud_snapshot.go index 73367f1..aab7db0 100644 --- a/internal/cli/cloud_snapshot.go +++ b/internal/cli/cloud_snapshot.go @@ -21,10 +21,14 @@ const ( ) type gitcrawlCloudDataset struct { - Name string - Columns []string - Rows [][]any - MaxSourceAt string + Name string + Columns []string + Query string + RowCount int64 + EligibleCount int64 + CoveredCount int64 + MaxSourceAt string + Complete bool } type gitcrawlCloudSnapshot struct { @@ -55,17 +59,22 @@ func buildGitcrawlCloudSnapshot( if err != nil { return gitcrawlCloudSnapshot{}, err } - datasets, err := loadGitcrawlCloudDatasets(ctx, db, slices.Contains(capabilities, gitcrawlObservationOrderCapability)) + hydration, err := gitcrawlCloudHydration(ctx, snapshotPath) if err != nil { return gitcrawlCloudSnapshot{}, err } - if len(datasets) == 0 || len(datasets[0].Rows) == 0 { - return gitcrawlCloudSnapshot{}, fmt.Errorf("cloud snapshot has no repositories") - } - hydration, err := gitcrawlCloudHydration(ctx, snapshotPath) + datasets, err := loadGitcrawlCloudDatasets( + ctx, + db, + slices.Contains(capabilities, gitcrawlObservationOrderCapability), + hydration, + ) if err != nil { return gitcrawlCloudSnapshot{}, err } + if len(datasets) == 0 || datasets[0].RowCount == 0 { + return gitcrawlCloudSnapshot{}, fmt.Errorf("cloud snapshot has no repositories") + } missing := incompleteGitcrawlCloudHydration(hydration) if len(missing) > 0 && !allowIncomplete { return gitcrawlCloudSnapshot{}, fmt.Errorf( @@ -99,7 +108,12 @@ func gitcrawlCloudManifest(archive string, snapshot gitcrawlCloudSnapshot) crawl } } -func loadGitcrawlCloudDatasets(ctx context.Context, db *sql.DB, observationOrder bool) ([]gitcrawlCloudDataset, error) { +func loadGitcrawlCloudDatasets( + ctx context.Context, + db *sql.DB, + observationOrder bool, + hydration crawlstore.EnrichmentCoverage, +) ([]gitcrawlCloudDataset, error) { threadColumns := append([]string(nil), gitcrawlThreadColumns...) threadQuery, err := gitcrawlThreadExportSQL(ctx, db) if err != nil { @@ -244,20 +258,33 @@ order by thread_id, position`, datasets := make([]gitcrawlCloudDataset, 0, len(specs)) for _, spec := range specs { - rows, err := publishRows(ctx, db, spec.query, func(values []any) []any { return values }) - if err != nil { - return nil, fmt.Errorf("export cloud dataset %s: %w", spec.name, err) + var rowCount int64 + if err := db.QueryRowContext( + ctx, + "select count(*) from ("+spec.query+")", + ).Scan(&rowCount); err != nil { + return nil, fmt.Errorf("count cloud dataset %s: %w", spec.name, err) } maxSourceAt, err := latestRFC3339QueryValue(ctx, db, spec.sourceAtQuery) if err != nil { return nil, fmt.Errorf("read cloud dataset %s freshness: %w", spec.name, err) } - datasets = append(datasets, gitcrawlCloudDataset{ - Name: spec.name, - Columns: spec.columns, - Rows: rows, - MaxSourceAt: maxSourceAt, - }) + dataset := gitcrawlCloudDataset{ + Name: spec.name, + Columns: spec.columns, + Query: spec.query, + RowCount: rowCount, + EligibleCount: rowCount, + CoveredCount: rowCount, + MaxSourceAt: maxSourceAt, + Complete: true, + } + if observationOrder && spec.name == "thread_revisions" { + dataset.EligibleCount = int64(hydration.Revisions.Eligible) + dataset.CoveredCount = int64(hydration.Revisions.Fresh) + dataset.Complete = hydration.Revisions.Supported && hydration.Revisions.Complete + } + datasets = append(datasets, dataset) } return datasets, nil } @@ -283,21 +310,84 @@ func gitcrawlCloudCapabilities(ctx context.Context, db *sql.DB, observationOrder } func gitcrawlCloudSourceSyncAt(ctx context.Context, db *sql.DB) (string, error) { - sourceSyncAt, err := latestRFC3339QueryValue(ctx, db, ` + queries := make([]string, 0, 4) + if ok, err := sqliteTableHasColumns( + ctx, + db, + "sync_runs", + "status", + "started_at", + "finished_at", + ); err != nil { + return "", err + } else if ok { + queries = append(queries, ` +select coalesce(finished_at, started_at, '') +from sync_runs +where status in ('success', 'completed')`) + } + if ok, err := sqliteTableHasColumns(ctx, db, "portable_metadata", "key", "value"); err != nil { + return "", err + } else if ok { + queries = append(queries, ` select value -from ( - select coalesce(finished_at, started_at, '') as value - from sync_runs - where status in ('success', 'completed') - union all - select coalesce(nullif(updated_at_gh, ''), updated_at, '') from threads - union all - select coalesce(updated_at, '') from repositories -)`) - if err != nil { - return "", fmt.Errorf("read cloud snapshot source sync time: %w", err) +from portable_metadata +where key = 'exported_at'`) + } + if ok, err := sqliteTableHasColumns(ctx, db, "threads", "updated_at"); err != nil { + return "", err + } else if ok { + hasGitHubUpdatedAt, err := sqliteColumnExists(ctx, db, "threads", "updated_at_gh") + if err != nil { + return "", err + } + if hasGitHubUpdatedAt { + queries = append(queries, `select coalesce(nullif(updated_at_gh, ''), updated_at, '') from threads`) + } else { + queries = append(queries, `select coalesce(updated_at, '') from threads`) + } + } + if ok, err := sqliteTableHasColumns(ctx, db, "repositories", "updated_at"); err != nil { + return "", err + } else if ok { + queries = append(queries, `select coalesce(updated_at, '') from repositories`) + } + + var latest time.Time + for _, query := range queries { + value, err := latestRFC3339QueryValue(ctx, db, query) + if err != nil { + return "", fmt.Errorf("read cloud snapshot source sync time: %w", err) + } + if value == "" { + continue + } + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return "", fmt.Errorf("parse normalized cloud snapshot source sync time %q: %w", value, err) + } + if latest.IsZero() || parsed.After(latest) { + latest = parsed + } + } + if latest.IsZero() { + return "", nil + } + return latest.UTC().Format(time.RFC3339Nano), nil +} + +func sqliteTableHasColumns(ctx context.Context, db *sql.DB, table string, columns ...string) (bool, error) { + exists, err := sqliteTableExists(ctx, db, table) + if err != nil || !exists { + return false, err + } + for _, column := range columns { + exists, err := sqliteColumnExists(ctx, db, table, column) + if err != nil || !exists { + return false, err + } } - return sourceSyncAt, nil + return true, nil } func latestRFC3339QueryValue(ctx context.Context, db *sql.DB, query string) (string, error) { @@ -377,15 +467,14 @@ func incompleteGitcrawlCloudHydration(coverage crawlstore.EnrichmentCoverage) [] func gitcrawlCloudCoverageRows(snapshot gitcrawlCloudSnapshot, mutationToken string) [][]any { rows := make([][]any, 0, len(snapshot.Datasets)) for _, dataset := range snapshot.Datasets { - count := int64(len(dataset.Rows)) rows = append(rows, []any{ dataset.Name, - count, - count, - count, + dataset.RowCount, + dataset.EligibleCount, + dataset.CoveredCount, dataset.MaxSourceAt, snapshot.DatasetGeneratedAt, - true, + dataset.Complete, mutationToken, }) } @@ -395,7 +484,7 @@ func gitcrawlCloudCoverageRows(snapshot gitcrawlCloudSnapshot, mutationToken str func gitcrawlCloudDatasetCounts(snapshot gitcrawlCloudSnapshot) map[string]int64 { counts := make(map[string]int64, len(snapshot.Datasets)) for _, dataset := range snapshot.Datasets { - counts[dataset.Name] = int64(len(dataset.Rows)) + counts[dataset.Name] = dataset.RowCount } return counts } diff --git a/internal/cli/cloud_snapshot_test.go b/internal/cli/cloud_snapshot_test.go index 9242e98..b388f78 100644 --- a/internal/cli/cloud_snapshot_test.go +++ b/internal/cli/cloud_snapshot_test.go @@ -3,7 +3,11 @@ package cli import ( "context" "database/sql" + "fmt" + "path/filepath" "testing" + + crawlstore "github.com/openclaw/gitcrawl/internal/store" ) func TestLatestRFC3339QueryValueUsesParsedTimestampOrder(t *testing.T) { @@ -64,6 +68,152 @@ func TestGitcrawlCloudSourceSyncAtNormalizesRFC3339Maximum(t *testing.T) { } } +func TestGitcrawlCloudSourceSyncAtUsesPortableMetadataWithoutSyncRuns(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "portable.db") + st, err := crawlstore.Open(ctx, path) + if err != nil { + t.Fatalf("open store: %v", err) + } + defer st.Close() + repoID, err := st.UpsertRepository(ctx, crawlstore.Repository{ + Owner: "openclaw", + Name: "gitcrawl", + FullName: "openclaw/gitcrawl", + UpdatedAt: "2026-07-11T23:00:00Z", + }) + if err != nil { + t.Fatalf("seed repository: %v", err) + } + if _, err := st.UpsertThread(ctx, crawlstore.Thread{ + RepoID: repoID, + GitHubID: "portable-1", + Number: 1, + Kind: "issue", + State: "open", + Title: "Portable source clock", + HTMLURL: "https://github.com/openclaw/gitcrawl/issues/1", + LabelsJSON: "[]", + AssigneesJSON: "[]", + ContentHash: "portable-1", + UpdatedAtGitHub: "2026-07-11T23:30:00Z", + UpdatedAt: "2026-07-11T23:30:00Z", + }); err != nil { + t.Fatalf("seed thread: %v", err) + } + if _, err := st.PrunePortablePayloads(ctx, crawlstore.PortablePruneOptions{BodyChars: 64}); err != nil { + t.Fatalf("prune portable store: %v", err) + } + hasSyncRuns, err := sqliteTableExists(ctx, st.DB(), "sync_runs") + if err != nil { + t.Fatalf("inspect sync_runs: %v", err) + } + if hasSyncRuns { + t.Fatal("portable store retained sync_runs") + } + var exportedAt string + if err := st.DB().QueryRowContext( + ctx, + `select value from portable_metadata where key = 'exported_at'`, + ).Scan(&exportedAt); err != nil { + t.Fatalf("read portable exported_at: %v", err) + } + + got, err := gitcrawlCloudSourceSyncAt(ctx, st.DB()) + if err != nil { + t.Fatalf("source sync at: %v", err) + } + if got != exportedAt { + t.Fatalf("source sync at = %q, want portable exported_at %q", got, exportedAt) + } +} + +func TestObservationOrderRevisionCoverageCountsFreshSelectedThreads(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "coverage.db") + st, err := crawlstore.Open(ctx, path) + if err != nil { + t.Fatalf("open store: %v", err) + } + defer st.Close() + repoID, err := st.UpsertRepository(ctx, crawlstore.Repository{ + Owner: "openclaw", + Name: "gitcrawl", + FullName: "openclaw/gitcrawl", + UpdatedAt: "2026-07-12T10:00:00Z", + }) + if err != nil { + t.Fatalf("seed repository: %v", err) + } + threadIDs := make([]int64, 0, 3) + threads := make([]crawlstore.Thread, 0, 3) + for number := 1; number <= 3; number++ { + thread := crawlstore.Thread{ + RepoID: repoID, + GitHubID: fmt.Sprintf("thread-%d", number), + Number: number, + Kind: "issue", + State: "open", + Title: "Revision coverage", + HTMLURL: fmt.Sprintf("https://github.com/openclaw/gitcrawl/issues/%d", number), + LabelsJSON: "[]", + AssigneesJSON: "[]", + ContentHash: fmt.Sprintf("thread-%d", number), + UpdatedAtGitHub: "2026-07-12T10:00:00Z", + UpdatedAt: "2026-07-12T10:00:00Z", + } + threadID, err := st.UpsertThread(ctx, thread) + if err != nil { + t.Fatalf("seed thread %d: %v", number, err) + } + thread.ID = threadID + threadIDs = append(threadIDs, threadID) + threads = append(threads, thread) + } + if _, err := st.UpsertThreadRevisionAndFingerprint( + ctx, + crawlstore.ThreadEvidence{Thread: threads[1]}, + "2026-07-12T10:01:00Z", + ); err != nil { + t.Fatalf("seed fresh selected revision: %v", err) + } + if _, err := st.DB().ExecContext(ctx, ` + insert into thread_revisions( + thread_id, source_updated_at, content_hash, title_hash, body_hash, + labels_hash, observation_sequence, created_at + ) values + (?, '2026-07-12T09:00:00Z', 'later-stale', 'later-stale', 'later-stale', 'later-stale', 99, '2026-07-12T10:02:00Z'), + (?, '2026-07-12T09:00:00Z', 'stale', 'stale', 'stale', 'stale', 1, '2026-07-12T10:03:00Z') + `, threadIDs[1], threadIDs[2]); err != nil { + t.Fatalf("seed revisions: %v", err) + } + coverage, err := st.ArchiveCoverage(ctx, crawlstore.ArchiveCoverageOptions{}) + if err != nil { + t.Fatalf("archive coverage: %v", err) + } + revisions := coverage.Totals.Enrichment.Revisions + if revisions.Eligible != 3 || revisions.Covered != 2 || revisions.Fresh != 1 { + t.Fatalf("revision hydration = %#v, want eligible=3 covered=2 fresh=1", revisions) + } + datasets, err := loadGitcrawlCloudDatasets(ctx, st.DB(), true, coverage.Totals.Enrichment) + if err != nil { + t.Fatalf("load datasets: %v", err) + } + var revisionDataset gitcrawlCloudDataset + for _, dataset := range datasets { + if dataset.Name == "thread_revisions" { + revisionDataset = dataset + break + } + } + if revisionDataset.RowCount != 3 || + revisionDataset.EligibleCount != 3 || + revisionDataset.CoveredCount != 1 || + revisionDataset.Complete { + t.Fatalf("revision dataset coverage = %#v", revisionDataset) + } +} + func TestLatestRFC3339QueryValueRejectsInvalidTimestamp(t *testing.T) { db, err := sql.Open("sqlite", ":memory:") if err != nil { From 4904b208cdb7b919b519c801ae71e6274ce45c1a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:27:04 +0200 Subject: [PATCH 34/52] fix(cloud): stream staged snapshot publication --- internal/cli/app_test.go | 67 ++++++++-- internal/cli/cloud_commands.go | 189 +++++++++++++++++++++------- internal/cli/cloud_commands_test.go | 85 +++++++++++++ 3 files changed, 286 insertions(+), 55 deletions(-) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 43e8cfe..5f6bcb9 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -1228,6 +1228,19 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { }) return } + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/status") { + activeSnapshotID := "" + if sawCutover { + activeSnapshotID = snapshotID + } + _ = json.NewEncoder(w).Encode(crawlremote.Status{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__openclaw", + ActiveSnapshotID: activeSnapshotID, + CoverageComplete: sawCutover, + }) + return + } if r.Method == http.MethodPost && strings.HasSuffix(r.URL.EscapedPath(), "/cutover") { var request struct { SnapshotID string `json:"snapshot_id"` @@ -1726,11 +1739,13 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { var stagedSnapshot *crawlremote.ArchiveSnapshot servingSnapshotID := strings.Repeat("f", 64) var sqliteImage []byte + uploadRequests := 0 ingestRequests := 0 publisherStatusRequests := 0 readerStatusRequests := 0 sqliteReadRequests := 0 cutovers := 0 + hydrationFailuresRemaining := 1 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { contract := testSnapshotPublishContract() @@ -1750,8 +1765,8 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { } if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/status") { readerStatusRequests++ - if r.Header.Get("authorization") != "Bearer reader-token" { - http.Error(w, "publisher token has no reader role", http.StatusForbidden) + if r.Header.Get("authorization") != "Bearer publish-token" { + http.Error(w, "missing dual-role bearer", http.StatusForbidden) return } _ = json.NewEncoder(w).Encode(crawlremote.Status{ @@ -1781,6 +1796,11 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { http.Error(w, "bound snapshot unavailable", http.StatusConflict) return } + if hydrationFailuresRemaining > 0 { + hydrationFailuresRemaining-- + http.Error(w, "temporary hydration failure", http.StatusConflict) + return + } w.Header().Set("content-type", "application/vnd.sqlite3") w.Header().Set("content-length", strconv.Itoa(len(sqliteImage))) w.Header().Set("x-crawl-content-sha256", snapshotID) @@ -1803,6 +1823,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { return } if r.Method == http.MethodPut && strings.HasSuffix(r.URL.EscapedPath(), "/sqlite") { + uploadRequests++ w.Header().Set("content-type", "application/json") switch r.Header.Get("x-crawl-sqlite-upload") { case "bundle-part": @@ -1957,19 +1978,31 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { "--json", } stageIngestRequests := 0 + stageUploadRequests := 0 originalServingSnapshotID := servingSnapshotID var stagedDatasetGeneratedAt string - for index, extra := range [][]string{{"--stage-only"}, nil} { + for index, extra := range [][]string{{"--stage-only"}, nil, nil} { app := New() var output bytes.Buffer app.Stdout = &output args := append(append([]string(nil), baseArgs...), extra...) - if err := app.Run(ctx, args); err != nil { + err := app.Run(ctx, args) + if index == 1 { + if err == nil || !strings.Contains(err.Error(), "temporary hydration failure") { + t.Fatalf("first cutover hydration error = %v", err) + } + if servingSnapshotID != snapshotID { + t.Fatalf("failed hydration did not leave snapshot serving: %q", servingSnapshotID) + } + continue + } + if err != nil { t.Fatalf("cloud publish run %d: %v", index+1, err) } var result struct { DatasetGeneratedAt string `json:"dataset_generated_at"` AlreadyStaged bool `json:"already_staged"` + AlreadyCutOver bool `json:"already_cut_over"` MutationToken string `json:"mutation_token"` Cutover *crawlremote.CutoverResult `json:"cutover"` } @@ -1984,6 +2017,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { } if index == 0 { stageIngestRequests = ingestRequests + stageUploadRequests = uploadRequests if stageIngestRequests == 0 || stagedSnapshot == nil { t.Fatal("stage-only publish did not complete the candidate snapshot") } @@ -1996,9 +2030,9 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { } time.Sleep(2 * time.Millisecond) } - if index == 1 { - if !result.AlreadyStaged || result.Cutover == nil { - t.Fatalf("default publish did not resume the staged candidate: %s", output.String()) + if index == 2 { + if !result.AlreadyStaged || !result.AlreadyCutOver || result.Cutover != nil { + t.Fatalf("retry did not reuse the staged serving snapshot: %s", output.String()) } if result.DatasetGeneratedAt != stagedDatasetGeneratedAt { t.Fatalf( @@ -2019,20 +2053,27 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { stageIngestRequests, ) } + if uploadRequests != stageUploadRequests { + t.Fatalf( + "upload requests = %d after staged resume, want stage-only count %d", + uploadRequests, + stageUploadRequests, + ) + } if cutovers != 1 { t.Fatalf("cutovers = %d, want 1", cutovers) } if servingSnapshotID != snapshotID { t.Fatalf("serving snapshot = %q, want staged snapshot %q", servingSnapshotID, snapshotID) } - if publisherStatusRequests != 3 { - t.Fatalf("publisher status requests = %d, want 3", publisherStatusRequests) + if publisherStatusRequests != 5 { + t.Fatalf("publisher status requests = %d, want 5", publisherStatusRequests) } - if readerStatusRequests != 0 { - t.Fatalf("reader status requests = %d, post-cutover verification should use publisher status", readerStatusRequests) + if readerStatusRequests != 2 { + t.Fatalf("reader status requests = %d, want 2 serving-state checks", readerStatusRequests) } - if sqliteReadRequests != 1 { - t.Fatalf("SQLite read requests = %d, want 1 post-cutover hydration proof", sqliteReadRequests) + if sqliteReadRequests != 2 { + t.Fatalf("SQLite read requests = %d, want failed hydration plus retry", sqliteReadRequests) } } diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 387f5bf..b097480 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -157,16 +157,13 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { if err := requireGitcrawlCloudPublishRoles(ctx, client); err != nil { return err } - sqliteBundle, sqliteSourceSize, err := uploadSQLiteSnapshotArchive( - ctx, - client, - "gitcrawl", - archiveID, - snapshotPath, - counts, - ) + snapshotInfo, err := os.Stat(snapshotPath) if err != nil { - return err + return fmt.Errorf("stat frozen cloud snapshot: %w", err) + } + sqliteSourceSize := snapshotInfo.Size() + if sqliteSourceSize <= 0 { + return fmt.Errorf("frozen cloud snapshot has invalid size %d", sqliteSourceSize) } alreadyStaged := false @@ -183,20 +180,38 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { } else if !remoteNotFound(statusErr) { return statusErr } + var sqliteBundle *crawlremote.SQLiteBundle var mutationToken string if !alreadyStaged { + uploadedBundle, uploadedSourceSize, err := uploadSQLiteSnapshotArchive( + ctx, + client, + "gitcrawl", + archiveID, + snapshotPath, + counts, + ) + if err != nil { + return err + } + if uploadedSourceSize != sqliteSourceSize { + return fmt.Errorf( + "SQLite bundle source size changed from %d to %d", + sqliteSourceSize, + uploadedSourceSize, + ) + } + sqliteBundle = uploadedBundle for _, dataset := range snapshot.Datasets { - progress, err := sendSnapshotIngestRows( + progress, err := sendSnapshotIngestDataset( ctx, + snapshotDB, client, "gitcrawl", archiveID, manifest, - dataset.Name, - dataset.Columns, - dataset.Rows, + dataset, mutationToken, - false, ) if err != nil { return fmt.Errorf("publish cloud dataset %s: %w", dataset.Name, err) @@ -218,17 +233,32 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { true, ) if err != nil { - return fmt.Errorf("activate cloud snapshot: %w", err) + return fmt.Errorf("complete cloud snapshot staging: %w", err) } mutationToken = progress.MutationToken } var cutoverResult *crawlremote.CutoverResult + alreadyCutOver := false if cutover { - result, err := client.Cutover(ctx, "gitcrawl", archiveID, snapshot.ID) + readerStatus, err := client.Status(ctx, "gitcrawl", archiveID) if err != nil { - return fmt.Errorf("cut over cloud snapshot: %w", err) + return fmt.Errorf("read serving cloud snapshot status: %w", err) + } + if readerStatus.App != "gitcrawl" || readerStatus.Archive != archiveID { + return fmt.Errorf( + "serving cloud snapshot status returned app=%q archive=%q", + readerStatus.App, + readerStatus.Archive, + ) + } + alreadyCutOver = readerStatus.ActiveSnapshotID == snapshot.ID + if !alreadyCutOver { + result, err := client.Cutover(ctx, "gitcrawl", archiveID, snapshot.ID) + if err != nil { + return fmt.Errorf("cut over cloud snapshot: %w", err) + } + cutoverResult = &result } - cutoverResult = &result if err := verifyGitcrawlSnapshotPublication( ctx, client, @@ -256,6 +286,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { "datasets": counts, "hydration": snapshot.Hydration, "already_staged": alreadyStaged, + "already_cut_over": alreadyCutOver, "mutation_token": mutationToken, "cutover": cutoverResult, "sqlite_bundle": sqliteBundle, @@ -263,39 +294,108 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { }, true) } -func publishRows(ctx context.Context, db *sql.DB, query string, mapRow func([]any) []any) ([][]any, error) { - rows, err := db.QueryContext(ctx, query) +type ingestProgress struct { + RowsAccepted int64 + MutationToken string +} + +func sendSnapshotIngestDataset( + ctx context.Context, + db *sql.DB, + client *crawlremote.Client, + app, archive string, + manifest crawlremote.IngestManifest, + dataset gitcrawlCloudDataset, + mutationToken string, +) (ingestProgress, error) { + rows, err := db.QueryContext(ctx, dataset.Query) if err != nil { - return nil, err + return ingestProgress{}, err } defer rows.Close() - cols, err := rows.Columns() + columns, err := rows.Columns() if err != nil { - return nil, err + return ingestProgress{}, err } - out := make([][]any, 0) + if len(columns) != len(dataset.Columns) { + return ingestProgress{}, fmt.Errorf( + "dataset query returned %d columns, want %d", + len(columns), + len(dataset.Columns), + ) + } + + batch := make([][]any, 0, gitcrawlCloudBatchSize) + var scanned int64 + var accepted int64 + flush := func() error { + result, err := sendIngestBatch( + ctx, + client, + app, + archive, + manifest, + dataset.Name, + dataset.Columns, + batch, + scanned-int64(len(batch)), + mutationToken, + false, + ) + if err != nil { + return err + } + if result.RowsAccepted != int64(len(batch)) { + return fmt.Errorf( + "remote accepted %d rows from a %d-row batch", + result.RowsAccepted, + len(batch), + ) + } + accepted += result.RowsAccepted + mutationToken = result.MutationToken + batch = batch[:0] + return nil + } + for rows.Next() { - values := make([]any, len(cols)) - ptrs := make([]any, len(cols)) - for i := range values { - ptrs[i] = &values[i] + values := make([]any, len(columns)) + pointers := make([]any, len(columns)) + for index := range values { + pointers[index] = &values[index] } - if err := rows.Scan(ptrs...); err != nil { - return nil, err + if err := rows.Scan(pointers...); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err } - for i, value := range values { + for index, value := range values { if bytes, ok := value.([]byte); ok { - values[i] = string(bytes) + values[index] = string(bytes) + } + } + batch = append(batch, values) + scanned++ + if len(batch) == gitcrawlCloudBatchSize { + if err := flush(); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err } } - out = append(out, mapRow(values)) } - return out, rows.Err() -} - -type ingestProgress struct { - RowsAccepted int64 - MutationToken string + if err := rows.Err(); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + if len(batch) > 0 || scanned == 0 { + if err := flush(); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + } + if scanned != dataset.RowCount { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, fmt.Errorf( + "dataset row count changed from preflight %d to stream %d", + dataset.RowCount, + scanned, + ) + } + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, nil } func sendIngestRows( @@ -365,7 +465,7 @@ func sendSnapshotIngestRows( table, columns, rows[start:end], - start, + int64(start), mutationToken, final && end == len(rows), ) @@ -386,7 +486,7 @@ func sendIngestBatch( table string, columns []string, rows [][]any, - cursor int, + cursor int64, mutationToken string, final bool, ) (crawlremote.IngestResult, error) { @@ -467,7 +567,7 @@ func isResetIncomplete(err error) bool { return errors.As(err, &remoteErr) && remoteErr.Code == "reset_incomplete" } -func cursorFor(start int) string { +func cursorFor(start int64) string { if start == 0 { return "" } @@ -592,6 +692,11 @@ func requireGitcrawlSnapshotPublishContract( if cutover { requiredRoutes = append( requiredRoutes, + crawlremote.RouteSpec{ + Method: http.MethodGet, + Path: "/v1/apps/:app/archives/:archive/status", + Auth: crawlremote.AuthReader, + }, crawlremote.RouteSpec{ Method: http.MethodPost, Path: "/v1/apps/:app/archives/:archive/cutover", diff --git a/internal/cli/cloud_commands_test.go b/internal/cli/cloud_commands_test.go index e562f68..8aa2f8f 100644 --- a/internal/cli/cloud_commands_test.go +++ b/internal/cli/cloud_commands_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/sha256" + "database/sql" "encoding/json" "fmt" "io" @@ -36,6 +37,90 @@ func TestUniqueStringSupersetRejectsMissingOrDuplicateArguments(t *testing.T) { } } +func TestSendSnapshotIngestDatasetStreamsBoundedBatches(t *testing.T) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + defer db.Close() + if _, err := db.Exec(` + create table rows(id integer primary key, value text not null); + with recursive sequence(id) as ( + select 1 + union all + select id + 1 from sequence where id < 501 + ) + insert into rows(id, value) + select id, printf('row-%03d', id) from sequence; + `); err != nil { + t.Fatalf("seed rows: %v", err) + } + + var requests []crawlremote.IngestRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var request crawlremote.IngestRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + requests = append(requests, request) + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + Table: request.Table, + SnapshotID: request.Manifest.SnapshotID, + MutationToken: fmt.Sprintf("mutation-%d", len(requests)), + RowsAccepted: int64(len(request.Rows)), + }) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("publisher-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + dataset := gitcrawlCloudDataset{ + Name: "bounded", + Columns: []string{"id", "value"}, + Query: `select id, value from rows order by id`, + RowCount: 501, + } + progress, err := sendSnapshotIngestDataset( + context.Background(), + db, + client, + "gitcrawl", + "gitcrawl/openclaw__gitcrawl", + crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__gitcrawl", + SnapshotID: strings.Repeat("a", 64), + }, + dataset, + "", + ) + if err != nil { + t.Fatalf("stream dataset: %v", err) + } + if progress.RowsAccepted != 501 || progress.MutationToken != "mutation-3" { + t.Fatalf("progress = %#v", progress) + } + if len(requests) != 3 { + t.Fatalf("requests = %d, want 3", len(requests)) + } + for index, wantRows := range []int{250, 250, 1} { + if got := len(requests[index].Rows); got != wantRows { + t.Fatalf("request %d rows = %d, want %d", index, got, wantRows) + } + } + for index, wantCursor := range []string{"", "250", "500"} { + if requests[index].Cursor != wantCursor { + t.Fatalf("request %d cursor = %q, want %q", index, requests[index].Cursor, wantCursor) + } + } +} + func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t *testing.T) { source := []byte("SQLite format 3\x00bound source") snapshotID := fmt.Sprintf("%x", sha256.Sum256(source)) From 58caeb83002163c4a2179b29a79b15404a166cda Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:28:15 +0200 Subject: [PATCH 35/52] fix(cloud): require isolated snapshot staging --- README.md | 7 ++++--- internal/cli/app_test.go | 8 +++++++- internal/cli/cloud_commands.go | 2 ++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3f0960e..b5de94f 100644 --- a/README.md +++ b/README.md @@ -71,10 +71,11 @@ Use `gitcrawl remote login --github-token-env GITHUB_TOKEN` for non-browser boot SHA-256 as the snapshot identity, exports repositories, threads, revisions, fingerprints, summaries, durable clusters, and PR detail/file rows from that same image, negotiates the remote snapshot-provenance contract before touching -R2, uploads its digest-scoped bundle, and activates complete D1 coverage. +R2, uploads its digest-scoped bundle, and completes staged D1 coverage. Publishing moves unpinned reads to the complete snapshot by default, preserving -the existing reader-refresh behavior. `--stage-only` keeps the immutable -snapshot staged without changing serving state. A later publish verifies the +the existing reader-refresh behavior. The remote must advertise +`gitcrawl.snapshot.staging.v1`; `--stage-only` keeps the immutable snapshot +staged without changing serving state. A later publish verifies the candidate through the publisher-only status projection, skips repeated ingest when its digest, source sync, schema, resolved publication profile, persisted generation timestamp, and coverage match, then cuts it over. Cutover requires diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 5f6bcb9..ec68365 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -893,6 +893,7 @@ func testSnapshotPublishContract() crawlremote.Contract { gitcrawlSnapshotAtomicCapability, gitcrawlSnapshotCutoverCapability, gitcrawlSnapshotProvenanceCapability, + gitcrawlSnapshotStagingCapability, sqliteBundleGzipUploadCapability, }, }} @@ -1002,7 +1003,7 @@ func TestGitcrawlPublisherStatusMatchesExactMetadata(t *testing.T) { activeMismatch := status activeMismatch.ActiveSnapshotID = strings.Repeat("f", 64) if gitcrawlPublisherStatusMatches(activeMismatch, manifest, publicationCapabilities) { - t.Fatal("non-active staged snapshot was reused") + t.Fatal("publisher status candidate mismatch was reused") } sourceMismatch := status @@ -1446,6 +1447,11 @@ func TestCloudPublishRejectsMissingRequestedCapabilityBeforeUpload(t *testing.T) args: []string{"--observation-order", "--stage-only"}, missingCapability: gitcrawlObservationOrderCapability, }, + { + name: "stage isolation", + args: []string{"--stage-only"}, + missingCapability: gitcrawlSnapshotStagingCapability, + }, { name: "cutover", missingCapability: gitcrawlSnapshotCutoverCapability, diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index b097480..1189264 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -30,6 +30,7 @@ const ( gitcrawlSnapshotAtomicCapability = "gitcrawl.snapshot.atomic" gitcrawlSnapshotCutoverCapability = "gitcrawl.snapshot.cutover" gitcrawlSnapshotProvenanceCapability = "gitcrawl.snapshot.provenance.v1" + gitcrawlSnapshotStagingCapability = "gitcrawl.snapshot.staging.v1" sqliteBundleGzipUploadCapability = "sqlite.bundle.gzip.upload" ) @@ -645,6 +646,7 @@ func requireGitcrawlSnapshotPublishContract( requiredCapabilities := []string{ gitcrawlSnapshotAtomicCapability, gitcrawlSnapshotProvenanceCapability, + gitcrawlSnapshotStagingCapability, sqliteBundleGzipUploadCapability, } requiredCapabilities = append(requiredCapabilities, snapshot.Capabilities...) From b1828a8fb4d4afcff695d97ffdd4be1ff4f1e05d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:30:30 +0200 Subject: [PATCH 36/52] test(cloud): cover sparse revision observations --- internal/cli/cloud_snapshot_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/cli/cloud_snapshot_test.go b/internal/cli/cloud_snapshot_test.go index b388f78..6edc4e2 100644 --- a/internal/cli/cloud_snapshot_test.go +++ b/internal/cli/cloud_snapshot_test.go @@ -145,9 +145,9 @@ func TestObservationOrderRevisionCoverageCountsFreshSelectedThreads(t *testing.T if err != nil { t.Fatalf("seed repository: %v", err) } - threadIDs := make([]int64, 0, 3) - threads := make([]crawlstore.Thread, 0, 3) - for number := 1; number <= 3; number++ { + threadIDs := make([]int64, 0, 4) + threads := make([]crawlstore.Thread, 0, 4) + for number := 1; number <= 4; number++ { thread := crawlstore.Thread{ RepoID: repoID, GitHubID: fmt.Sprintf("thread-%d", number), @@ -192,8 +192,8 @@ func TestObservationOrderRevisionCoverageCountsFreshSelectedThreads(t *testing.T t.Fatalf("archive coverage: %v", err) } revisions := coverage.Totals.Enrichment.Revisions - if revisions.Eligible != 3 || revisions.Covered != 2 || revisions.Fresh != 1 { - t.Fatalf("revision hydration = %#v, want eligible=3 covered=2 fresh=1", revisions) + if revisions.Eligible != 4 || revisions.Covered != 2 || revisions.Fresh != 1 { + t.Fatalf("revision hydration = %#v, want eligible=4 covered=2 fresh=1", revisions) } datasets, err := loadGitcrawlCloudDatasets(ctx, st.DB(), true, coverage.Totals.Enrichment) if err != nil { @@ -207,7 +207,7 @@ func TestObservationOrderRevisionCoverageCountsFreshSelectedThreads(t *testing.T } } if revisionDataset.RowCount != 3 || - revisionDataset.EligibleCount != 3 || + revisionDataset.EligibleCount != 4 || revisionDataset.CoveredCount != 1 || revisionDataset.Complete { t.Fatalf("revision dataset coverage = %#v", revisionDataset) From e0621cb3eb01e195a231bd95d8f0e193753ebb32 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:31:06 +0200 Subject: [PATCH 37/52] fix(cloud): count snapshot datasets directly --- internal/cli/cloud_snapshot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cli/cloud_snapshot.go b/internal/cli/cloud_snapshot.go index aab7db0..5a3088f 100644 --- a/internal/cli/cloud_snapshot.go +++ b/internal/cli/cloud_snapshot.go @@ -261,7 +261,7 @@ order by thread_id, position`, var rowCount int64 if err := db.QueryRowContext( ctx, - "select count(*) from ("+spec.query+")", + "select count(*) from "+spec.name, ).Scan(&rowCount); err != nil { return nil, fmt.Errorf("count cloud dataset %s: %w", spec.name, err) } From 6a831eba8539e822eea45dbc05ac458da9afe8b3 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:31:58 +0200 Subject: [PATCH 38/52] test(cloud): preflight portable snapshots end to end --- internal/cli/cloud_snapshot_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/internal/cli/cloud_snapshot_test.go b/internal/cli/cloud_snapshot_test.go index 6edc4e2..1872ea5 100644 --- a/internal/cli/cloud_snapshot_test.go +++ b/internal/cli/cloud_snapshot_test.go @@ -126,6 +126,24 @@ func TestGitcrawlCloudSourceSyncAtUsesPortableMetadataWithoutSyncRuns(t *testing if got != exportedAt { t.Fatalf("source sync at = %q, want portable exported_at %q", got, exportedAt) } + if err := st.Close(); err != nil { + t.Fatalf("close portable store: %v", err) + } + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatalf("reopen portable sqlite: %v", err) + } + defer db.Close() + snapshot, err := buildGitcrawlCloudSnapshot(ctx, db, path, true, false) + if err != nil { + t.Fatalf("build portable cloud snapshot: %v", err) + } + if snapshot.SourceSyncAt != exportedAt { + t.Fatalf("snapshot source sync at = %q, want %q", snapshot.SourceSyncAt, exportedAt) + } + if counts := gitcrawlCloudDatasetCounts(snapshot); counts["repositories"] != 1 || counts["threads"] != 1 { + t.Fatalf("portable snapshot counts = %#v", counts) + } } func TestObservationOrderRevisionCoverageCountsFreshSelectedThreads(t *testing.T) { From 450b210da19363c165a318ed8ce82090fc93157d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:43:56 +0200 Subject: [PATCH 39/52] fix(cloud): accept admin publish credentials --- internal/cli/cloud_commands.go | 3 +++ internal/cli/cloud_commands_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 1189264..d28f78f 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -795,6 +795,9 @@ func requireGitcrawlCloudPublishRoles(ctx context.Context, client *crawlremote.C if err != nil { return fmt.Errorf("read remote identity before snapshot publication: %w", err) } + if slices.Contains(identity.Roles, "admin") { + return nil + } missing := make([]string, 0, 2) for _, role := range []string{crawlremote.AuthPublisher, crawlremote.AuthReader} { if !slices.Contains(identity.Roles, role) { diff --git a/internal/cli/cloud_commands_test.go b/internal/cli/cloud_commands_test.go index 8aa2f8f..1c0c830 100644 --- a/internal/cli/cloud_commands_test.go +++ b/internal/cli/cloud_commands_test.go @@ -37,6 +37,32 @@ func TestUniqueStringSupersetRejectsMissingOrDuplicateArguments(t *testing.T) { } } +func TestRequireGitcrawlCloudPublishRolesAcceptsAdmin(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.EscapedPath() != "/v1/whoami" { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(crawlremote.Identity{ + Login: "admin", + Roles: []string{"admin"}, + }) + })) + defer server.Close() + + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("admin-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + if err := requireGitcrawlCloudPublishRoles(context.Background(), client); err != nil { + t.Fatalf("admin role preflight: %v", err) + } +} + func TestSendSnapshotIngestDatasetStreamsBoundedBatches(t *testing.T) { db, err := sql.Open("sqlite", ":memory:") if err != nil { From 0505bb19fdf4b92b0781b14bea46ecb79f27d797 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:45:34 +0200 Subject: [PATCH 40/52] fix(cloud): verify publication acknowledgements --- internal/cli/app_test.go | 85 +++++++-- internal/cli/cloud_commands.go | 213 ++++++++++++++++++++-- internal/cli/cloud_commands_test.go | 265 ++++++++++++++++++++++++++++ 3 files changed, 540 insertions(+), 23 deletions(-) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index ec68365..f1b4d21 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -1743,6 +1743,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { t.Setenv(tokenEnv, "publish-token") var snapshotID string var stagedSnapshot *crawlremote.ArchiveSnapshot + var stagedDatasets []crawlremote.DatasetCoverage servingSnapshotID := strings.Repeat("f", 64) var sqliteImage []byte uploadRequests := 0 @@ -1775,12 +1776,25 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { http.Error(w, "missing dual-role bearer", http.StatusForbidden) return } - _ = json.NewEncoder(w).Encode(crawlremote.Status{ + status := crawlremote.Status{ App: "gitcrawl", Archive: "gitcrawl/openclaw__openclaw", ActiveSnapshotID: servingSnapshotID, - CoverageComplete: true, - }) + } + if servingSnapshotID == snapshotID && + readerStatusRequests >= 3 && + stagedSnapshot != nil { + status.SchemaName = stagedSnapshot.SchemaName + status.SchemaVersion = stagedSnapshot.SchemaVersion + status.SchemaHash = stagedSnapshot.SchemaHash + status.Capabilities = slices.Clone(stagedSnapshot.Capabilities) + status.SourceSyncAt = stagedSnapshot.SourceSyncAt + status.DatasetGeneratedAt = stagedSnapshot.DatasetGeneratedAt + status.CoverageComplete = true + status.Datasets = slices.Clone(stagedDatasets) + status.Snapshot = stagedSnapshot + } + _ = json.NewEncoder(w).Encode(status) return } if got := r.Header.Get("authorization"); got != "Bearer publish-token" { @@ -1923,6 +1937,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { return } mutationToken = body.MutationToken + stagedDatasets = testGitcrawlDatasetCoverageRows(t, body.Rows) stagedSnapshot = &crawlremote.ArchiveSnapshot{ ID: body.Manifest.SnapshotID, SourceSHA256: body.Manifest.SourceSHA256, @@ -1987,7 +2002,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { stageUploadRequests := 0 originalServingSnapshotID := servingSnapshotID var stagedDatasetGeneratedAt string - for index, extra := range [][]string{{"--stage-only"}, nil, nil} { + for index, extra := range [][]string{{"--stage-only"}, nil, nil, nil} { app := New() var output bytes.Buffer app.Stdout = &output @@ -2037,6 +2052,11 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { time.Sleep(2 * time.Millisecond) } if index == 2 { + if result.AlreadyCutOver || result.Cutover == nil { + t.Fatalf("incomplete reader status incorrectly skipped cutover: %s", output.String()) + } + } + if index == 3 { if !result.AlreadyStaged || !result.AlreadyCutOver || result.Cutover != nil { t.Fatalf("retry did not reuse the staged serving snapshot: %s", output.String()) } @@ -2066,21 +2086,62 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { stageUploadRequests, ) } - if cutovers != 1 { - t.Fatalf("cutovers = %d, want 1", cutovers) + if cutovers != 2 { + t.Fatalf("cutovers = %d, want 2 after incomplete reader status", cutovers) } if servingSnapshotID != snapshotID { t.Fatalf("serving snapshot = %q, want staged snapshot %q", servingSnapshotID, snapshotID) } - if publisherStatusRequests != 5 { - t.Fatalf("publisher status requests = %d, want 5", publisherStatusRequests) + if publisherStatusRequests != 7 { + t.Fatalf("publisher status requests = %d, want 7", publisherStatusRequests) } - if readerStatusRequests != 2 { - t.Fatalf("reader status requests = %d, want 2 serving-state checks", readerStatusRequests) + if readerStatusRequests != 3 { + t.Fatalf("reader status requests = %d, want 3 serving-state checks", readerStatusRequests) } - if sqliteReadRequests != 2 { - t.Fatalf("SQLite read requests = %d, want failed hydration plus retry", sqliteReadRequests) + if sqliteReadRequests != 3 { + t.Fatalf("SQLite read requests = %d, want failed hydration plus two retries", sqliteReadRequests) + } +} + +func testGitcrawlDatasetCoverageRows( + t *testing.T, + rows [][]any, +) []crawlremote.DatasetCoverage { + t.Helper() + datasets := make([]crawlremote.DatasetCoverage, 0, len(rows)) + for _, row := range rows { + if len(row) != len(gitcrawlCloudCoverageColumns) { + t.Fatalf("coverage row columns = %d, want %d", len(row), len(gitcrawlCloudCoverageColumns)) + } + number := func(value any) int64 { + t.Helper() + switch typed := value.(type) { + case float64: + return int64(typed) + case int64: + return typed + default: + t.Fatalf("coverage count type = %T", value) + return 0 + } + } + complete, ok := row[6].(bool) + if !ok { + t.Fatalf("coverage complete type = %T", row[6]) + } + covered := number(row[3]) + datasets = append(datasets, crawlremote.DatasetCoverage{ + Dataset: fmt.Sprint(row[0]), + RowCount: number(row[1]), + EligibleCount: number(row[2]), + CoveredCount: covered, + FreshCount: covered, + MaxSourceAt: fmt.Sprint(row[4]), + DatasetGeneratedAt: fmt.Sprint(row[5]), + Complete: complete, + }) } + return datasets } func TestCloudPublishRejectsIncompleteHydrationBeforeRemoteMutation(t *testing.T) { diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index d28f78f..58b773b 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -220,18 +220,14 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { counts[dataset.Name] = progress.RowsAccepted mutationToken = progress.MutationToken } - coverageRows := gitcrawlCloudCoverageRows(snapshot, mutationToken) - progress, err := sendSnapshotIngestRows( + progress, err := completeGitcrawlSnapshotStaging( ctx, client, "gitcrawl", archiveID, manifest, - "dataset_coverage", - gitcrawlCloudCoverageColumns, - coverageRows, + snapshot, mutationToken, - true, ) if err != nil { return fmt.Errorf("complete cloud snapshot staging: %w", err) @@ -252,7 +248,12 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { readerStatus.Archive, ) } - alreadyCutOver = readerStatus.ActiveSnapshotID == snapshot.ID + alreadyCutOver = gitcrawlReaderStatusMatches( + readerStatus, + snapshot, + manifest, + publicationCapabilities, + ) if !alreadyCutOver { result, err := client.Cutover(ctx, "gitcrawl", archiveID, snapshot.ID) if err != nil { @@ -298,6 +299,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { type ingestProgress struct { RowsAccepted int64 MutationToken string + Result crawlremote.IngestResult } func sendSnapshotIngestDataset( @@ -450,8 +452,13 @@ func sendSnapshotIngestRows( mutationToken, final, ) - return ingestProgress{RowsAccepted: result.RowsAccepted, MutationToken: result.MutationToken}, err + return ingestProgress{ + RowsAccepted: result.RowsAccepted, + MutationToken: result.MutationToken, + Result: result, + }, err } + var lastResult crawlremote.IngestResult for start := 0; start < len(rows); start += gitcrawlCloudBatchSize { end := start + gitcrawlCloudBatchSize if end > len(rows) { @@ -471,12 +478,80 @@ func sendSnapshotIngestRows( final && end == len(rows), ) if err != nil { - return ingestProgress{RowsAccepted: total, MutationToken: mutationToken}, err + return ingestProgress{ + RowsAccepted: total, + MutationToken: mutationToken, + Result: lastResult, + }, err } total += result.RowsAccepted mutationToken = result.MutationToken + lastResult = result + } + return ingestProgress{ + RowsAccepted: total, + MutationToken: mutationToken, + Result: lastResult, + }, nil +} + +func completeGitcrawlSnapshotStaging( + ctx context.Context, + client *crawlremote.Client, + app, archive string, + manifest crawlremote.IngestManifest, + snapshot gitcrawlCloudSnapshot, + mutationToken string, +) (ingestProgress, error) { + progress, err := sendSnapshotIngestRows( + ctx, + client, + app, + archive, + manifest, + "dataset_coverage", + gitcrawlCloudCoverageColumns, + gitcrawlCloudCoverageRows(snapshot, mutationToken), + mutationToken, + true, + ) + if err != nil { + return progress, err + } + result := progress.Result + expectedRows := int64(len(snapshot.Datasets)) + if result.Table != "dataset_coverage" { + return progress, fmt.Errorf( + "remote completed table %q, want dataset_coverage", + result.Table, + ) + } + if result.SnapshotID != snapshot.ID { + return progress, fmt.Errorf( + "remote completed snapshot %q, want %q", + result.SnapshotID, + snapshot.ID, + ) + } + if progress.RowsAccepted != expectedRows || result.RowsAccepted != expectedRows { + return progress, fmt.Errorf( + "remote accepted %d coverage rows with final batch count %d, want %d datasets", + progress.RowsAccepted, + result.RowsAccepted, + expectedRows, + ) + } + if result.MutationToken != mutationToken { + return progress, fmt.Errorf( + "remote completed mutation token %q, want %q", + result.MutationToken, + mutationToken, + ) + } + if !result.Complete { + return progress, fmt.Errorf("remote did not complete snapshot %s", snapshot.ID) } - return ingestProgress{RowsAccepted: total, MutationToken: mutationToken}, nil + return progress, nil } func sendIngestBatch( @@ -611,7 +686,61 @@ func uploadSQLiteSnapshotArchive( if err != nil { return nil, 0, err } - return result.Bundle, sourceSize, nil + uploadedBundle, err := validateGitcrawlSQLiteBundleUpload( + result, + app, + archive, + bundle.Manifest, + ) + if err != nil { + return nil, 0, err + } + return uploadedBundle, sourceSize, nil +} + +func validateGitcrawlSQLiteBundleUpload( + result crawlremote.SQLiteBundleUploadResult, + app, archive string, + expected crawlremote.SQLiteBundleManifest, +) (*crawlremote.SQLiteBundle, error) { + if result.App != app || result.Archive != archive { + return nil, fmt.Errorf( + "SQLite bundle upload returned app=%q archive=%q, want app=%q archive=%q", + result.App, + result.Archive, + app, + archive, + ) + } + if !result.Complete { + return nil, fmt.Errorf("SQLite bundle upload was not finalized") + } + if result.Bundle == nil || result.Bundle.Manifest == nil { + return nil, fmt.Errorf("SQLite bundle upload omitted the finalized manifest") + } + actual := result.Bundle.Manifest + if actual.SnapshotID != expected.SnapshotID { + return nil, fmt.Errorf( + "SQLite bundle upload acknowledged snapshot %q, want %q", + actual.SnapshotID, + expected.SnapshotID, + ) + } + if actual.Object.SHA256 != expected.Object.SHA256 { + return nil, fmt.Errorf( + "SQLite bundle upload acknowledged digest %q, want %q", + actual.Object.SHA256, + expected.Object.SHA256, + ) + } + if actual.Object.Size != expected.Object.Size { + return nil, fmt.Errorf( + "SQLite bundle upload acknowledged source size %d, want %d", + actual.Object.Size, + expected.Object.Size, + ) + } + return result.Bundle, nil } func remoteNotFound(err error) bool { @@ -878,6 +1007,68 @@ func gitcrawlPublisherStatusMatches( return true } +func gitcrawlReaderStatusMatches( + status crawlremote.Status, + snapshot gitcrawlCloudSnapshot, + manifest crawlremote.IngestManifest, + publicationCapabilities []string, +) bool { + if status.App != manifest.App || + status.Archive != manifest.Archive || + status.ActiveSnapshotID != snapshot.ID || + status.SchemaName != manifest.SchemaName || + status.SchemaVersion != manifest.SchemaVersion || + status.SchemaHash != manifest.SchemaHash || + status.SourceSyncAt != snapshot.SourceSyncAt || + status.DatasetGeneratedAt != snapshot.DatasetGeneratedAt || + !status.CoverageComplete || + !equalUniqueStringSet(status.Capabilities, publicationCapabilities) { + return false + } + if !gitcrawlPublisherStatusMatches(crawlremote.PublisherStatus{ + App: status.App, + Archive: status.Archive, + ActiveSnapshotID: status.ActiveSnapshotID, + CoverageComplete: status.CoverageComplete, + Snapshot: status.Snapshot, + }, manifest, publicationCapabilities) || + status.Snapshot.DatasetGeneratedAt != snapshot.DatasetGeneratedAt { + return false + } + return gitcrawlDatasetCoverageMatches(status.Datasets, snapshot) +} + +func gitcrawlDatasetCoverageMatches( + actual []crawlremote.DatasetCoverage, + snapshot gitcrawlCloudSnapshot, +) bool { + if len(actual) != len(snapshot.Datasets) { + return false + } + expected := make(map[string]gitcrawlCloudDataset, len(snapshot.Datasets)) + for _, dataset := range snapshot.Datasets { + if _, duplicate := expected[dataset.Name]; duplicate { + return false + } + expected[dataset.Name] = dataset + } + for _, dataset := range actual { + want, ok := expected[dataset.Dataset] + if !ok || + dataset.RowCount != want.RowCount || + dataset.EligibleCount != want.EligibleCount || + dataset.CoveredCount != want.CoveredCount || + dataset.FreshCount != want.CoveredCount || + dataset.MaxSourceAt != want.MaxSourceAt || + dataset.DatasetGeneratedAt != snapshot.DatasetGeneratedAt || + dataset.Complete != want.Complete { + return false + } + delete(expected, dataset.Dataset) + } + return len(expected) == 0 +} + func verifyGitcrawlSnapshotPublication( ctx context.Context, client *crawlremote.Client, diff --git a/internal/cli/cloud_commands_test.go b/internal/cli/cloud_commands_test.go index 1c0c830..7434a64 100644 --- a/internal/cli/cloud_commands_test.go +++ b/internal/cli/cloud_commands_test.go @@ -147,6 +147,271 @@ func TestSendSnapshotIngestDatasetStreamsBoundedBatches(t *testing.T) { } } +func TestCompleteGitcrawlSnapshotStagingRequiresExactAcknowledgement(t *testing.T) { + snapshotID := strings.Repeat("a", 64) + snapshot := gitcrawlCloudSnapshot{ + ID: snapshotID, + DatasetGeneratedAt: "2026-07-12T12:01:00Z", + Datasets: []gitcrawlCloudDataset{ + {Name: "repositories", RowCount: 1, EligibleCount: 1, CoveredCount: 1, Complete: true}, + {Name: "threads", RowCount: 3, EligibleCount: 3, CoveredCount: 3, Complete: true}, + }, + } + manifest := crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__gitcrawl", + SnapshotID: snapshotID, + } + const mutationToken = "mutation-final" + + for _, test := range []struct { + name string + mutate func(*crawlremote.IngestResult) + want string + }{ + {name: "exact acknowledgement"}, + { + name: "wrong table", + mutate: func(result *crawlremote.IngestResult) { + result.Table = "threads" + }, + want: "want dataset_coverage", + }, + { + name: "wrong snapshot", + mutate: func(result *crawlremote.IngestResult) { + result.SnapshotID = strings.Repeat("b", 64) + }, + want: "want \"" + snapshotID + "\"", + }, + { + name: "wrong dataset count", + mutate: func(result *crawlremote.IngestResult) { + result.RowsAccepted-- + }, + want: "want 2 datasets", + }, + { + name: "wrong mutation token", + mutate: func(result *crawlremote.IngestResult) { + result.MutationToken = "other-mutation" + }, + want: "want \"mutation-final\"", + }, + { + name: "partial candidate", + mutate: func(result *crawlremote.IngestResult) { + result.Complete = false + }, + want: "did not complete snapshot", + }, + } { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var request crawlremote.IngestRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if request.Table != "dataset_coverage" || + !request.Final || + request.MutationToken != mutationToken || + len(request.Rows) != len(snapshot.Datasets) { + http.Error(w, "invalid final coverage request", http.StatusBadRequest) + return + } + result := crawlremote.IngestResult{ + Table: request.Table, + SnapshotID: request.Manifest.SnapshotID, + MutationToken: request.MutationToken, + RowsAccepted: int64(len(request.Rows)), + Complete: true, + } + if test.mutate != nil { + test.mutate(&result) + } + _ = json.NewEncoder(w).Encode(result) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("publisher-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + + _, err = completeGitcrawlSnapshotStaging( + context.Background(), + client, + manifest.App, + manifest.Archive, + manifest, + snapshot, + mutationToken, + ) + if test.want == "" { + if err != nil { + t.Fatalf("complete staging: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("complete staging error = %v, want %q", err, test.want) + } + }) + } +} + +func TestValidateGitcrawlSQLiteBundleUploadRequiresFinalSnapshotDigest(t *testing.T) { + expected := crawlremote.SQLiteBundleManifest{ + SnapshotID: strings.Repeat("a", 64), + Object: crawlremote.SQLiteBundleObject{ + Size: 123, + SHA256: strings.Repeat("a", 64), + }, + } + validResult := func() crawlremote.SQLiteBundleUploadResult { + manifest := expected + return crawlremote.SQLiteBundleUploadResult{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__gitcrawl", + Complete: true, + Bundle: &crawlremote.SQLiteBundle{ + Manifest: &manifest, + }, + } + } + for _, test := range []struct { + name string + mutate func(*crawlremote.SQLiteBundleUploadResult) + want string + }{ + {name: "exact acknowledgement"}, + { + name: "not finalized", + mutate: func(result *crawlremote.SQLiteBundleUploadResult) { + result.Complete = false + }, + want: "not finalized", + }, + { + name: "wrong snapshot", + mutate: func(result *crawlremote.SQLiteBundleUploadResult) { + result.Bundle.Manifest.SnapshotID = strings.Repeat("b", 64) + }, + want: "acknowledged snapshot", + }, + { + name: "wrong digest", + mutate: func(result *crawlremote.SQLiteBundleUploadResult) { + result.Bundle.Manifest.Object.SHA256 = strings.Repeat("b", 64) + }, + want: "acknowledged digest", + }, + { + name: "wrong source size", + mutate: func(result *crawlremote.SQLiteBundleUploadResult) { + result.Bundle.Manifest.Object.Size++ + }, + want: "acknowledged source size", + }, + } { + t.Run(test.name, func(t *testing.T) { + result := validResult() + if test.mutate != nil { + test.mutate(&result) + } + _, err := validateGitcrawlSQLiteBundleUpload( + result, + "gitcrawl", + "gitcrawl/openclaw__gitcrawl", + expected, + ) + if test.want == "" { + if err != nil { + t.Fatalf("validate bundle upload: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("bundle validation error = %v, want %q", err, test.want) + } + }) + } +} + +func TestGitcrawlReaderStatusMatchesCompleteServingSnapshot(t *testing.T) { + snapshotID := strings.Repeat("a", 64) + snapshot := gitcrawlCloudSnapshot{ + ID: snapshotID, + SourceSyncAt: "2026-07-12T12:00:00Z", + DatasetGeneratedAt: "2026-07-12T12:01:00Z", + Datasets: []gitcrawlCloudDataset{{ + Name: "repositories", + RowCount: 1, + EligibleCount: 1, + CoveredCount: 1, + MaxSourceAt: "2026-07-12T12:00:00Z", + Complete: true, + }}, + } + manifest := gitcrawlCloudManifest("gitcrawl/openclaw__gitcrawl", snapshot) + capabilities := gitcrawlCloudPublicationCapabilities(snapshot.Capabilities) + status := crawlremote.Status{ + App: manifest.App, + Archive: manifest.Archive, + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: capabilities, + ActiveSnapshotID: snapshotID, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + CoverageComplete: true, + Datasets: []crawlremote.DatasetCoverage{{ + Dataset: "repositories", + RowCount: 1, + EligibleCount: 1, + CoveredCount: 1, + FreshCount: 1, + MaxSourceAt: "2026-07-12T12:00:00Z", + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + Complete: true, + }}, + Snapshot: &crawlremote.ArchiveSnapshot{ + ID: snapshotID, + SourceSHA256: snapshotID, + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: capabilities, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + CoverageComplete: true, + }, + } + if !gitcrawlReaderStatusMatches(status, snapshot, manifest, capabilities) { + t.Fatal("complete serving snapshot did not match") + } + + status.CoverageComplete = false + if gitcrawlReaderStatusMatches(status, snapshot, manifest, capabilities) { + t.Fatal("top-level incomplete reader status matched") + } + status.CoverageComplete = true + status.Snapshot.CoverageComplete = false + if gitcrawlReaderStatusMatches(status, snapshot, manifest, capabilities) { + t.Fatal("nested incomplete reader status matched") + } + status.Snapshot.CoverageComplete = true + status.Datasets[0].CoveredCount = 0 + if gitcrawlReaderStatusMatches(status, snapshot, manifest, capabilities) { + t.Fatal("reader dataset count drift matched") + } +} + func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t *testing.T) { source := []byte("SQLite format 3\x00bound source") snapshotID := fmt.Sprintf("%x", sha256.Sum256(source)) From 50bc3a4ac37f0eaa24654da33af7f034cbf7e254 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:45:58 +0200 Subject: [PATCH 41/52] fix(cloud): preflight export query schemas --- internal/cli/app_test.go | 41 ++++++++++++++++++++++++++++++++++ internal/cli/cloud_snapshot.go | 7 ++++++ 2 files changed, 48 insertions(+) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index f1b4d21..6a18d73 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -2181,6 +2181,47 @@ func TestCloudPublishRejectsIncompleteHydrationBeforeRemoteMutation(t *testing.T } } +func TestCloudPublishRejectsLegacyExportSchemaBeforeRemoteMutation(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + dbPath := filepath.Join(dir, "gitcrawl.db") + cfg := config.Default() + cfg.DBPath = dbPath + if err := config.Save(configPath, cfg); err != nil { + t.Fatalf("save config: %v", err) + } + seedCommandFlowStore(t, dbPath) + seedDoctorLegacyPullRequestFilesSchema(t, ctx, dbPath) + + const tokenEnv = "GITCRAWL_TEST_LEGACY_EXPORT_TOKEN" + t.Setenv(tokenEnv, "publish-token") + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + http.Error(w, "unexpected remote request", http.StatusInternalServerError) + })) + defer server.Close() + + err := New().Run(ctx, []string{ + "--config", configPath, + "cloud", "publish", + "--remote", server.URL, + "--archive", "gitcrawl/openclaw__openclaw", + "--token-env", tokenEnv, + "--allow-incomplete", + "--json", + }) + if err == nil || + !strings.Contains(err.Error(), "prepare cloud dataset pull_request_files export") || + !strings.Contains(err.Error(), "no such column: position") { + t.Fatalf("cloud publish error = %v", err) + } + if requests != 0 { + t.Fatalf("remote requests = %d, want 0", requests) + } +} + func TestRemoteLoginStoresKeyringToken(t *testing.T) { ctx := context.Background() dir := t.TempDir() diff --git a/internal/cli/cloud_snapshot.go b/internal/cli/cloud_snapshot.go index 5a3088f..4ebdc99 100644 --- a/internal/cli/cloud_snapshot.go +++ b/internal/cli/cloud_snapshot.go @@ -258,6 +258,13 @@ order by thread_id, position`, datasets := make([]gitcrawlCloudDataset, 0, len(specs)) for _, spec := range specs { + statement, err := db.PrepareContext(ctx, spec.query) + if err != nil { + return nil, fmt.Errorf("prepare cloud dataset %s export: %w", spec.name, err) + } + if err := statement.Close(); err != nil { + return nil, fmt.Errorf("close cloud dataset %s export: %w", spec.name, err) + } var rowCount int64 if err := db.QueryRowContext( ctx, From 15ab8ada81daebc6b24f557ec7f514133794bdcc Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:46:10 +0200 Subject: [PATCH 42/52] fix(store): trust current PR file reservations --- internal/store/archive_coverage.go | 6 ++--- internal/store/archive_coverage_test.go | 31 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/internal/store/archive_coverage.go b/internal/store/archive_coverage.go index b409ced..85a1c2e 100644 --- a/internal/store/archive_coverage.go +++ b/internal/store/archive_coverage.go @@ -739,16 +739,16 @@ func (s *Store) archivePRFileCoverage( detailFresh := archiveCoverageTimestampAtOrAfter(detailFetchedAt, acceptedSource) filesFresh := files == 0 || archiveCoverageTimestampAtOrAfter(oldestFileFetchedAt, acceptedSource) - reservationFresh := true + fresh := detailFresh && filesFresh if hasReservation != 0 { - reservationFresh = archiveObservationAtOrAfter( + fresh = archiveObservationAtOrAfter( reservationSourceUpdatedAt, reservationSequence, acceptedSource, observationSequenceOrderValue(acceptedSequence), ) } - if detailFresh && filesFresh && reservationFresh { + if fresh { metric.Fresh++ } } diff --git a/internal/store/archive_coverage_test.go b/internal/store/archive_coverage_test.go index 27b9850..b4194f9 100644 --- a/internal/store/archive_coverage_test.go +++ b/internal/store/archive_coverage_test.go @@ -305,6 +305,37 @@ func TestArchiveCoveragePRFilesRequireCurrentObservation(t *testing.T) { t.Fatalf("newer accepted evidence = %+v, %v", accepted, err) } assertMetric(1, 0, 0, 1, false) + + if applied, err := st.ReserveThreadChildObservation( + ctx, + thread.ID, + ThreadChildPullRequestFiles, + thread.UpdatedAtGitHub, + 2, + ); err != nil || !applied { + t.Fatalf("reserve current PR file observation = %t, %v", applied, err) + } + if err := st.UpsertPullRequestCacheFamilies(ctx, PullRequestDetail{ + ThreadID: thread.ID, + RepoID: repoID, + Number: thread.Number, + ChangedFiles: 1, + RawJSON: "{}", + FetchedAt: "2026-07-12T12:02:00Z", + UpdatedAt: "2026-07-12T12:02:00Z", + }, []PullRequestFile{{ + ThreadID: thread.ID, + Position: 0, + Path: "README.md", + RawJSON: "{}", + FetchedAt: "2026-07-12T12:02:00Z", + }}, nil, nil, nil, PullRequestHydrationFamilies{ + Details: true, + Files: true, + }); err != nil { + t.Fatalf("persist ordered PR file evidence: %v", err) + } + assertMetric(1, 1, 0, 0, true) } func TestArchiveCoveragePRFilesUsesLegacySnapshotProof(t *testing.T) { From b6b1e758521e21b47cd3d0d80b9b5dcf9b699993 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:46:49 +0200 Subject: [PATCH 43/52] fix(cloud): verify chunked SQLite hydration --- internal/cli/cloud_commands.go | 15 ++++---- internal/cli/cloud_commands_test.go | 60 +++++++++++++++++++++++++---- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 58b773b..f28b526 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -1144,19 +1144,18 @@ func verifyGitcrawlSQLiteHydration( if expectedSize <= 0 { return fmt.Errorf("uploaded SQLite manifest source size must be positive, got %d", expectedSize) } - contentLength := response.ContentLength - if contentLength <= 0 { - header := strings.TrimSpace(response.Header.Get("content-length")) - if header == "" { - return fmt.Errorf("downloaded SQLite snapshot is missing a positive Content-Length") - } + contentLength := int64(-1) + header := strings.TrimSpace(response.Header.Get("content-length")) + if header != "" { parsed, err := strconv.ParseInt(header, 10, 64) if err != nil || parsed <= 0 { return fmt.Errorf("downloaded SQLite snapshot has invalid Content-Length %q", header) } contentLength = parsed + } else if response.ContentLength >= 0 { + contentLength = response.ContentLength } - if contentLength != expectedSize { + if contentLength >= 0 && contentLength != expectedSize { return fmt.Errorf( "downloaded SQLite snapshot Content-Length %d does not match uploaded source size %d", contentLength, @@ -1177,7 +1176,7 @@ func verifyGitcrawlSQLiteHydration( n, err := response.Body.Read(extra[:]) if n > 0 { return fmt.Errorf( - "downloaded SQLite snapshot exceeds Content-Length %d", + "downloaded SQLite snapshot exceeds uploaded source size %d", expectedSize, ) } diff --git a/internal/cli/cloud_commands_test.go b/internal/cli/cloud_commands_test.go index 7434a64..4682088 100644 --- a/internal/cli/cloud_commands_test.go +++ b/internal/cli/cloud_commands_test.go @@ -10,6 +10,7 @@ import ( "io" "net/http" "net/http/httptest" + "slices" "strings" "testing" "time" @@ -512,11 +513,12 @@ func TestVerifyGitcrawlSQLiteHydrationRejectsInvalidFramingAndDigest(t *testing. different := bytes.Repeat([]byte("x"), len(source)) tests := []struct { - name string - body []byte - digest string - contentLength int64 - want string + name string + body []byte + digest string + contentLength int64 + contentLengthHeader string + want string }{ { name: "valid", @@ -538,11 +540,18 @@ func TestVerifyGitcrawlSQLiteHydrationRejectsInvalidFramingAndDigest(t *testing. want: "advertises digest", }, { - name: "missing length", + name: "chunked response", body: source, digest: snapshotID, contentLength: -1, - want: "missing a positive Content-Length", + }, + { + name: "malformed length", + body: source, + digest: snapshotID, + contentLength: -1, + contentLengthHeader: "not-a-number", + want: "invalid Content-Length", }, { name: "wrong length", @@ -563,7 +572,7 @@ func TestVerifyGitcrawlSQLiteHydrationRejectsInvalidFramingAndDigest(t *testing. body: append(append([]byte(nil), source...), 'x'), digest: snapshotID, contentLength: int64(len(source)), - want: "exceeds Content-Length", + want: "exceeds uploaded source size", }, { name: "body digest mismatch", @@ -583,6 +592,9 @@ func TestVerifyGitcrawlSQLiteHydrationRejectsInvalidFramingAndDigest(t *testing. if test.digest != "" { response.Header.Set("x-crawl-content-sha256", test.digest) } + if test.contentLengthHeader != "" { + response.Header.Set("content-length", test.contentLengthHeader) + } err := verifyGitcrawlSQLiteHydration(response, snapshotID, int64(len(source))) if test.want == "" { if err != nil { @@ -596,3 +608,35 @@ func TestVerifyGitcrawlSQLiteHydrationRejectsInvalidFramingAndDigest(t *testing. }) } } + +func TestVerifyGitcrawlSQLiteHydrationAcceptsChunkedResponse(t *testing.T) { + source := []byte("SQLite format 3\x00chunked source") + snapshotID := fmt.Sprintf("%x", sha256.Sum256(source)) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("x-crawl-content-sha256", snapshotID) + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + _, _ = w.Write(source) + })) + defer server.Close() + + response, err := server.Client().Get(server.URL) + if err != nil { + t.Fatalf("download chunked hydration: %v", err) + } + defer response.Body.Close() + if response.ContentLength != -1 || !slices.Contains(response.TransferEncoding, "chunked") { + t.Fatalf( + "response framing = length %d transfer %v, want chunked", + response.ContentLength, + response.TransferEncoding, + ) + } + if err := verifyGitcrawlSQLiteHydration( + response, + snapshotID, + int64(len(source)), + ); err != nil { + t.Fatalf("verify chunked hydration: %v", err) + } +} From 737a5b759b7daf915b8ef4ebf8c2d72e9674f5f3 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 16:54:54 +0200 Subject: [PATCH 44/52] fix(store): require persisted PR file freshness --- internal/store/archive_coverage.go | 6 +++--- internal/store/archive_coverage_test.go | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/internal/store/archive_coverage.go b/internal/store/archive_coverage.go index 85a1c2e..b409ced 100644 --- a/internal/store/archive_coverage.go +++ b/internal/store/archive_coverage.go @@ -739,16 +739,16 @@ func (s *Store) archivePRFileCoverage( detailFresh := archiveCoverageTimestampAtOrAfter(detailFetchedAt, acceptedSource) filesFresh := files == 0 || archiveCoverageTimestampAtOrAfter(oldestFileFetchedAt, acceptedSource) - fresh := detailFresh && filesFresh + reservationFresh := true if hasReservation != 0 { - fresh = archiveObservationAtOrAfter( + reservationFresh = archiveObservationAtOrAfter( reservationSourceUpdatedAt, reservationSequence, acceptedSource, observationSequenceOrderValue(acceptedSequence), ) } - if fresh { + if detailFresh && filesFresh && reservationFresh { metric.Fresh++ } } diff --git a/internal/store/archive_coverage_test.go b/internal/store/archive_coverage_test.go index b4194f9..9f3864c 100644 --- a/internal/store/archive_coverage_test.go +++ b/internal/store/archive_coverage_test.go @@ -315,20 +315,21 @@ func TestArchiveCoveragePRFilesRequireCurrentObservation(t *testing.T) { ); err != nil || !applied { t.Fatalf("reserve current PR file observation = %t, %v", applied, err) } + assertMetric(1, 0, 0, 1, false) if err := st.UpsertPullRequestCacheFamilies(ctx, PullRequestDetail{ ThreadID: thread.ID, RepoID: repoID, Number: thread.Number, ChangedFiles: 1, RawJSON: "{}", - FetchedAt: "2026-07-12T12:02:00Z", - UpdatedAt: "2026-07-12T12:02:00Z", + FetchedAt: "2026-07-12T12:04:00Z", + UpdatedAt: "2026-07-12T12:04:00Z", }, []PullRequestFile{{ ThreadID: thread.ID, Position: 0, Path: "README.md", RawJSON: "{}", - FetchedAt: "2026-07-12T12:02:00Z", + FetchedAt: "2026-07-12T12:04:00Z", }}, nil, nil, nil, PullRequestHydrationFamilies{ Details: true, Files: true, From 38d05453ab582e702ecf07fb437a01c547b238d4 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 17:55:37 +0200 Subject: [PATCH 45/52] fix(cloud): require staged snapshot manifests --- CHANGELOG.md | 2 ++ internal/cli/cloud_commands.go | 4 ++-- internal/cli/cloud_commands_test.go | 33 +++++++++++++++++++++++++++-- internal/cli/cloud_snapshot.go | 6 +++++- 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99964c6..f1b9de3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## 0.7.2 - Unreleased +- Require every Gitcrawl cloud publisher manifest to opt into snapshot staging so `--stage-only` and normal publish-then-cutover runs cannot activate serving state through the legacy compatibility path. + ## 0.7.1 - 2026-07-09 - Add local, fail-closed packaging and independent verification for official macOS archives signed as `org.openclaw.gitcrawl` by `Developer ID Application: OpenClaw Foundation (FWJYW4S8P8)`, while keeping CI and cross-platform snapshots credential-free and non-publishing. diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index f28b526..4aef39d 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -145,7 +145,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { return err } manifest := gitcrawlCloudManifest(archiveID, snapshot) - publicationCapabilities := gitcrawlCloudPublicationCapabilities(snapshot.Capabilities) + publicationCapabilities := gitcrawlCloudPublicationCapabilities(manifest.Capabilities) counts := gitcrawlCloudDatasetCounts(snapshot) if err := requireGitcrawlSnapshotPublishContract( ctx, @@ -284,7 +284,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { "source_sha256": snapshot.ID, "source_sync_at": snapshot.SourceSyncAt, "dataset_generated_at": snapshot.DatasetGeneratedAt, - "capabilities": snapshot.Capabilities, + "capabilities": manifest.Capabilities, "datasets": counts, "hydration": snapshot.Hydration, "already_staged": alreadyStaged, diff --git a/internal/cli/cloud_commands_test.go b/internal/cli/cloud_commands_test.go index 4682088..0e9bc68 100644 --- a/internal/cli/cloud_commands_test.go +++ b/internal/cli/cloud_commands_test.go @@ -38,6 +38,35 @@ func TestUniqueStringSupersetRejectsMissingOrDuplicateArguments(t *testing.T) { } } +func TestGitcrawlCloudManifestAlwaysOptsIntoStaging(t *testing.T) { + snapshot := gitcrawlCloudSnapshot{ + ID: strings.Repeat("a", 64), + Capabilities: []string{gitcrawlObservationOrderCapability}, + } + + manifest := gitcrawlCloudManifest("gitcrawl/openclaw__gitcrawl", snapshot) + + if !slices.Equal(manifest.Capabilities, []string{ + gitcrawlObservationOrderCapability, + gitcrawlSnapshotStagingCapability, + }) { + t.Fatalf("manifest capabilities = %v", manifest.Capabilities) + } + if !slices.Equal(snapshot.Capabilities, []string{gitcrawlObservationOrderCapability}) { + t.Fatalf("snapshot capabilities mutated = %v", snapshot.Capabilities) + } + + manifest = gitcrawlCloudManifest("gitcrawl/openclaw__gitcrawl", gitcrawlCloudSnapshot{ + ID: strings.Repeat("b", 64), + Capabilities: []string{ + gitcrawlSnapshotStagingCapability, + }, + }) + if !slices.Equal(manifest.Capabilities, []string{gitcrawlSnapshotStagingCapability}) { + t.Fatalf("staging capability duplicated = %v", manifest.Capabilities) + } +} + func TestRequireGitcrawlCloudPublishRolesAcceptsAdmin(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet || r.URL.EscapedPath() != "/v1/whoami" { @@ -359,7 +388,7 @@ func TestGitcrawlReaderStatusMatchesCompleteServingSnapshot(t *testing.T) { }}, } manifest := gitcrawlCloudManifest("gitcrawl/openclaw__gitcrawl", snapshot) - capabilities := gitcrawlCloudPublicationCapabilities(snapshot.Capabilities) + capabilities := gitcrawlCloudPublicationCapabilities(manifest.Capabilities) status := crawlremote.Status{ App: manifest.App, Archive: manifest.Archive, @@ -422,7 +451,7 @@ func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t DatasetGeneratedAt: "2026-07-12T12:01:00Z", } manifest := gitcrawlCloudManifest("gitcrawl/openclaw__openclaw", snapshot) - publicationCapabilities := gitcrawlCloudPublicationCapabilities(snapshot.Capabilities) + publicationCapabilities := gitcrawlCloudPublicationCapabilities(manifest.Capabilities) for _, test := range []struct { name string diff --git a/internal/cli/cloud_snapshot.go b/internal/cli/cloud_snapshot.go index 4ebdc99..4ece1b7 100644 --- a/internal/cli/cloud_snapshot.go +++ b/internal/cli/cloud_snapshot.go @@ -93,6 +93,10 @@ func buildGitcrawlCloudSnapshot( } func gitcrawlCloudManifest(archive string, snapshot gitcrawlCloudSnapshot) crawlremote.IngestManifest { + capabilities := slices.Clone(snapshot.Capabilities) + if !slices.Contains(capabilities, gitcrawlSnapshotStagingCapability) { + capabilities = append(capabilities, gitcrawlSnapshotStagingCapability) + } return crawlremote.IngestManifest{ App: "gitcrawl", Archive: archive, @@ -104,7 +108,7 @@ func gitcrawlCloudManifest(archive string, snapshot gitcrawlCloudSnapshot) crawl SourceSyncAt: snapshot.SourceSyncAt, SnapshotID: snapshot.ID, SourceSHA256: snapshot.ID, - Capabilities: snapshot.Capabilities, + Capabilities: capabilities, } } From 86fb98b377ba5daa51db1e064d16e0766f69bded Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:19:32 +0200 Subject: [PATCH 46/52] fix(cloud): scope publisher status to snapshots --- CHANGELOG.md | 1 + go.mod | 2 +- go.sum | 4 +-- internal/cli/app_test.go | 48 ++++++++++++++++++++++++++--- internal/cli/cloud_commands.go | 14 +++++++-- internal/cli/cloud_commands_test.go | 8 +++++ 6 files changed, 68 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1b9de3..70bee19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 0.7.2 - Unreleased - Require every Gitcrawl cloud publisher manifest to opt into snapshot staging so `--stage-only` and normal publish-then-cutover runs cannot activate serving state through the legacy compatibility path. +- Update CrawlKit to v0.14.2 and bind publisher resume plus post-cutover verification to the exact snapshot ID, so a newer concurrently staged candidate cannot be mistaken for the snapshot this publisher owns. ## 0.7.1 - 2026-07-09 diff --git a/go.mod b/go.mod index 74fcbcc..4c403a0 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 github.com/charmbracelet/x/ansi v0.11.7 github.com/mattn/go-isatty v0.0.22 - github.com/openclaw/crawlkit v0.14.1 + github.com/openclaw/crawlkit v0.14.2 github.com/zalando/go-keyring v0.2.8 golang.org/x/sys v0.46.0 modernc.org/sqlite v1.53.0 diff --git a/go.sum b/go.sum index 81dba97..2169d41 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/openclaw/crawlkit v0.14.1 h1:B1exhgP/PZ//JCvr5asdhBZt6jZ5JwW8LHxGx0jfprc= -github.com/openclaw/crawlkit v0.14.1/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= +github.com/openclaw/crawlkit v0.14.2 h1:NA0+fDlyYnQjrI6krkhajHDKMTOt9q8JTlMFsGH37pg= +github.com/openclaw/crawlkit v0.14.2/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 6a18d73..c905d23 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -1099,6 +1099,7 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { var snapshotID string var sqliteImage []byte var publishedSnapshot *crawlremote.ArchiveSnapshot + var publisherStatusSnapshotIDs []string mutationCounter := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { @@ -1216,6 +1217,10 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { return } if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status") { + publisherStatusSnapshotIDs = append( + publisherStatusSnapshotIDs, + r.URL.Query().Get("snapshot_id"), + ) if publishedSnapshot == nil { http.NotFound(w, r) return @@ -1382,6 +1387,15 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { if payload["sqlite_bundle"] == nil { t.Fatalf("missing sqlite bundle output: %#v", payload) } + for _, requestedSnapshotID := range publisherStatusSnapshotIDs { + if requestedSnapshotID != snapshotID { + t.Fatalf( + "publisher status requested snapshot %q, want %q", + requestedSnapshotID, + snapshotID, + ) + } + } privacy, ok := payload["sqlite_bundle_privacy"].(map[string]any) if !ok || privacy["includes_private_messages"] != true { t.Fatalf("missing sqlite bundle privacy output: %#v", payload) @@ -1749,6 +1763,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { uploadRequests := 0 ingestRequests := 0 publisherStatusRequests := 0 + var publisherStatusSnapshotIDs []string readerStatusRequests := 0 sqliteReadRequests := 0 cutovers := 0 @@ -1829,16 +1844,32 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { } if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status") { publisherStatusRequests++ + requestedSnapshotID := r.URL.Query().Get("snapshot_id") + publisherStatusSnapshotIDs = append( + publisherStatusSnapshotIDs, + requestedSnapshotID, + ) + if stagedSnapshot == nil { + http.NotFound(w, r) + return + } + statusSnapshot := stagedSnapshot + if requestedSnapshotID == "" { + concurrent := *stagedSnapshot + concurrent.ID = strings.Repeat("c", 64) + concurrent.SourceSHA256 = concurrent.ID + statusSnapshot = &concurrent + } activeSnapshotID := "" - if stagedSnapshot != nil { - activeSnapshotID = stagedSnapshot.ID + if statusSnapshot != nil { + activeSnapshotID = statusSnapshot.ID } _ = json.NewEncoder(w).Encode(crawlremote.PublisherStatus{ App: "gitcrawl", Archive: "gitcrawl/openclaw__openclaw", ActiveSnapshotID: activeSnapshotID, - CoverageComplete: stagedSnapshot != nil, - Snapshot: stagedSnapshot, + CoverageComplete: statusSnapshot != nil, + Snapshot: statusSnapshot, }) return } @@ -2095,6 +2126,15 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { if publisherStatusRequests != 7 { t.Fatalf("publisher status requests = %d, want 7", publisherStatusRequests) } + for _, requestedSnapshotID := range publisherStatusSnapshotIDs { + if requestedSnapshotID != snapshotID { + t.Fatalf( + "publisher status requested snapshot %q, want %q", + requestedSnapshotID, + snapshotID, + ) + } + } if readerStatusRequests != 3 { t.Fatalf("reader status requests = %d, want 3 serving-state checks", readerStatusRequests) } diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 4aef39d..8fcaa26 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -168,7 +168,12 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { } alreadyStaged := false - status, statusErr := client.PublishStatus(ctx, "gitcrawl", archiveID) + status, statusErr := client.PublishStatusForSnapshot( + ctx, + "gitcrawl", + archiveID, + snapshot.ID, + ) if statusErr == nil { alreadyStaged = gitcrawlPublisherStatusMatches( status, @@ -1080,7 +1085,12 @@ func verifyGitcrawlSnapshotPublication( publicationCapabilities []string, sourceSize int64, ) error { - status, err := client.PublishStatus(ctx, "gitcrawl", archive) + status, err := client.PublishStatusForSnapshot( + ctx, + "gitcrawl", + archive, + snapshot.ID, + ) if err != nil { return fmt.Errorf("read post-cutover publisher status: %w", err) } diff --git a/internal/cli/cloud_commands_test.go b/internal/cli/cloud_commands_test.go index 0e9bc68..554542c 100644 --- a/internal/cli/cloud_commands_test.go +++ b/internal/cli/cloud_commands_test.go @@ -476,6 +476,14 @@ func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status"): + if got := r.URL.Query().Get("snapshot_id"); got != snapshotID { + http.Error( + w, + fmt.Sprintf("snapshot_id = %q, want %q", got, snapshotID), + http.StatusBadRequest, + ) + return + } w.Header().Set("content-type", "application/json") _ = json.NewEncoder(w).Encode(crawlremote.PublisherStatus{ App: "gitcrawl", From 22a1401ff07859334af84345a22a4c6a96e1837c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:27:33 +0200 Subject: [PATCH 47/52] fix(cloud): resume incomplete snapshot ingest --- internal/cli/app_test.go | 10 ++++++++++ internal/cli/cloud_commands.go | 9 ++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index c905d23..fb3ccb7 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -1764,6 +1764,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { ingestRequests := 0 publisherStatusRequests := 0 var publisherStatusSnapshotIDs []string + snapshotMismatchResponsesRemaining := 1 readerStatusRequests := 0 sqliteReadRequests := 0 cutovers := 0 @@ -1849,6 +1850,15 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { publisherStatusSnapshotIDs, requestedSnapshotID, ) + if requestedSnapshotID != "" && snapshotMismatchResponsesRemaining > 0 { + snapshotMismatchResponsesRemaining-- + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(map[string]any{ + "error": "snapshot_mismatch", + "message": "snapshot ingest is incomplete", + }) + return + } if stagedSnapshot == nil { http.NotFound(w, r) return diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 8fcaa26..9df52f4 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -183,7 +183,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { if alreadyStaged { snapshot.DatasetGeneratedAt = status.Snapshot.DatasetGeneratedAt } - } else if !remoteNotFound(statusErr) { + } else if !remoteNotFound(statusErr) && !remoteSnapshotIncomplete(statusErr) { return statusErr } var sqliteBundle *crawlremote.SQLiteBundle @@ -753,6 +753,13 @@ func remoteNotFound(err error) bool { return errors.As(err, &remoteErr) && remoteErr.Status == http.StatusNotFound } +func remoteSnapshotIncomplete(err error) bool { + var remoteErr *crawlremote.Error + return errors.As(err, &remoteErr) && + remoteErr.Status == http.StatusConflict && + remoteErr.Code == "snapshot_mismatch" +} + func requireGitcrawlSnapshotPublishContract( ctx context.Context, client *crawlremote.Client, From 7003e67b6cf0ea8c0d4060f7ef588c9a9b280879 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:34:46 +0200 Subject: [PATCH 48/52] fix(cloud): accept concurrent snapshot completion --- internal/cli/app_test.go | 21 ++++++-- internal/cli/cloud_commands.go | 98 +++++++++++++++++++++++++++++----- 2 files changed, 103 insertions(+), 16 deletions(-) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index fb3ccb7..32549c6 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -1769,6 +1769,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { sqliteReadRequests := 0 cutovers := 0 hydrationFailuresRemaining := 1 + concurrentFinalCompletionsRemaining := 1 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" { contract := testSnapshotPublishContract() @@ -1992,6 +1993,15 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { ), CoverageComplete: true, } + if concurrentFinalCompletionsRemaining > 0 { + concurrentFinalCompletionsRemaining-- + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(map[string]any{ + "error": "snapshot_active", + "message": "a concurrent publisher completed the snapshot", + }) + return + } } _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ Table: body.Table, @@ -2074,8 +2084,8 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { if index == 0 && result.Cutover != nil { t.Fatalf("stage-only publish unexpectedly cut over: %#v", result.Cutover) } - if index == 0 && result.AlreadyStaged { - t.Fatal("initial stage-only publish unexpectedly reused a candidate") + if index == 0 && !result.AlreadyStaged { + t.Fatal("initial stage-only publish did not recover concurrent completion") } if index == 0 { stageIngestRequests = ingestRequests @@ -2133,8 +2143,11 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { if servingSnapshotID != snapshotID { t.Fatalf("serving snapshot = %q, want staged snapshot %q", servingSnapshotID, snapshotID) } - if publisherStatusRequests != 7 { - t.Fatalf("publisher status requests = %d, want 7", publisherStatusRequests) + if publisherStatusRequests != 8 { + t.Fatalf( + "publisher status requests = %d, want 8 including concurrent completion recovery", + publisherStatusRequests, + ) } for _, requestedSnapshotID := range publisherStatusSnapshotIDs { if requestedSnapshotID != snapshotID { diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 9df52f4..b8d4893 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -208,6 +208,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { ) } sqliteBundle = uploadedBundle + completedConcurrently := false for _, dataset := range snapshot.Datasets { progress, err := sendSnapshotIngestDataset( ctx, @@ -220,24 +221,63 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { mutationToken, ) if err != nil { + recovered, recoveryErr := recoverConcurrentGitcrawlSnapshot( + ctx, + client, + archiveID, + snapshot, + manifest, + publicationCapabilities, + err, + ) + if recoveryErr != nil { + return fmt.Errorf( + "publish cloud dataset %s: %w", + dataset.Name, + recoveryErr, + ) + } + if recovered { + alreadyStaged = true + completedConcurrently = true + break + } return fmt.Errorf("publish cloud dataset %s: %w", dataset.Name, err) } counts[dataset.Name] = progress.RowsAccepted mutationToken = progress.MutationToken } - progress, err := completeGitcrawlSnapshotStaging( - ctx, - client, - "gitcrawl", - archiveID, - manifest, - snapshot, - mutationToken, - ) - if err != nil { - return fmt.Errorf("complete cloud snapshot staging: %w", err) + if !completedConcurrently { + progress, err := completeGitcrawlSnapshotStaging( + ctx, + client, + "gitcrawl", + archiveID, + manifest, + snapshot, + mutationToken, + ) + if err != nil { + recovered, recoveryErr := recoverConcurrentGitcrawlSnapshot( + ctx, + client, + archiveID, + snapshot, + manifest, + publicationCapabilities, + err, + ) + if recoveryErr != nil { + return fmt.Errorf("complete cloud snapshot staging: %w", recoveryErr) + } + if !recovered { + return fmt.Errorf("complete cloud snapshot staging: %w", err) + } + alreadyStaged = true + } else { + mutationToken = progress.MutationToken + } } - mutationToken = progress.MutationToken } var cutoverResult *crawlremote.CutoverResult alreadyCutOver := false @@ -760,6 +800,40 @@ func remoteSnapshotIncomplete(err error) bool { remoteErr.Code == "snapshot_mismatch" } +func recoverConcurrentGitcrawlSnapshot( + ctx context.Context, + client *crawlremote.Client, + archive string, + snapshot gitcrawlCloudSnapshot, + manifest crawlremote.IngestManifest, + publicationCapabilities []string, + cause error, +) (bool, error) { + var remoteErr *crawlremote.Error + if !errors.As(cause, &remoteErr) || + remoteErr.Status != http.StatusConflict || + remoteErr.Code != "snapshot_active" { + return false, nil + } + status, err := client.PublishStatusForSnapshot( + ctx, + "gitcrawl", + archive, + snapshot.ID, + ) + if err != nil { + return false, fmt.Errorf("re-probe concurrent snapshot completion: %w", err) + } + if !gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) || + status.Snapshot.DatasetGeneratedAt != snapshot.DatasetGeneratedAt { + return false, fmt.Errorf( + "concurrent active snapshot %s does not match the requested digest, profile, generation, and coverage", + snapshot.ID, + ) + } + return true, nil +} + func requireGitcrawlSnapshotPublishContract( ctx context.Context, client *crawlremote.Client, From 1d2fcfbff9623b79e547ab11f4a9886ce6334f52 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:41:58 +0200 Subject: [PATCH 49/52] fix(cloud): adopt concurrent snapshot generation --- internal/cli/app_test.go | 12 ++- internal/cli/cloud_commands.go | 25 +++--- internal/cli/cloud_commands_test.go | 124 ++++++++++++++++++++++++++-- 3 files changed, 142 insertions(+), 19 deletions(-) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 32549c6..327887e 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -1754,6 +1754,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { seedCommandFlowStore(t, dbPath) const tokenEnv = "GITCRAWL_TEST_RESUME_TOKEN" + const concurrentDatasetGeneratedAt = "2000-01-02T03:04:05.678901234Z" t.Setenv(tokenEnv, "publish-token") var snapshotID string var stagedSnapshot *crawlremote.ArchiveSnapshot @@ -1980,11 +1981,17 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { } mutationToken = body.MutationToken stagedDatasets = testGitcrawlDatasetCoverageRows(t, body.Rows) + if got := fmt.Sprint(body.Rows[0][5]); got == concurrentDatasetGeneratedAt { + t.Fatalf("loser generation unexpectedly equals independent winner generation %q", got) + } + for index := range stagedDatasets { + stagedDatasets[index].DatasetGeneratedAt = concurrentDatasetGeneratedAt + } stagedSnapshot = &crawlremote.ArchiveSnapshot{ ID: body.Manifest.SnapshotID, SourceSHA256: body.Manifest.SourceSHA256, SourceSyncAt: body.Manifest.SourceSyncAt, - DatasetGeneratedAt: fmt.Sprint(body.Rows[0][5]), + DatasetGeneratedAt: concurrentDatasetGeneratedAt, SchemaName: body.Manifest.SchemaName, SchemaVersion: body.Manifest.SchemaVersion, SchemaHash: body.Manifest.SchemaHash, @@ -2097,7 +2104,8 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { t.Fatalf("stage-only changed serving snapshot to %q", servingSnapshotID) } stagedDatasetGeneratedAt = result.DatasetGeneratedAt - if stagedDatasetGeneratedAt == "" || result.MutationToken == "" { + if stagedDatasetGeneratedAt != concurrentDatasetGeneratedAt || + result.MutationToken == "" { t.Fatalf("stage-only did not bind its generation: %#v", result) } time.Sleep(2 * time.Millisecond) diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index b8d4893..5b86eec 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -221,7 +221,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { mutationToken, ) if err != nil { - recovered, recoveryErr := recoverConcurrentGitcrawlSnapshot( + recoveredGeneration, recoveryErr := recoverConcurrentGitcrawlSnapshot( ctx, client, archiveID, @@ -237,7 +237,8 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { recoveryErr, ) } - if recovered { + if recoveredGeneration != "" { + snapshot.DatasetGeneratedAt = recoveredGeneration alreadyStaged = true completedConcurrently = true break @@ -258,7 +259,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { mutationToken, ) if err != nil { - recovered, recoveryErr := recoverConcurrentGitcrawlSnapshot( + recoveredGeneration, recoveryErr := recoverConcurrentGitcrawlSnapshot( ctx, client, archiveID, @@ -270,9 +271,10 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { if recoveryErr != nil { return fmt.Errorf("complete cloud snapshot staging: %w", recoveryErr) } - if !recovered { + if recoveredGeneration == "" { return fmt.Errorf("complete cloud snapshot staging: %w", err) } + snapshot.DatasetGeneratedAt = recoveredGeneration alreadyStaged = true } else { mutationToken = progress.MutationToken @@ -808,12 +810,12 @@ func recoverConcurrentGitcrawlSnapshot( manifest crawlremote.IngestManifest, publicationCapabilities []string, cause error, -) (bool, error) { +) (string, error) { var remoteErr *crawlremote.Error if !errors.As(cause, &remoteErr) || remoteErr.Status != http.StatusConflict || remoteErr.Code != "snapshot_active" { - return false, nil + return "", nil } status, err := client.PublishStatusForSnapshot( ctx, @@ -822,16 +824,15 @@ func recoverConcurrentGitcrawlSnapshot( snapshot.ID, ) if err != nil { - return false, fmt.Errorf("re-probe concurrent snapshot completion: %w", err) + return "", fmt.Errorf("re-probe concurrent snapshot completion: %w", err) } - if !gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) || - status.Snapshot.DatasetGeneratedAt != snapshot.DatasetGeneratedAt { - return false, fmt.Errorf( - "concurrent active snapshot %s does not match the requested digest, profile, generation, and coverage", + if !gitcrawlPublisherStatusMatches(status, manifest, publicationCapabilities) { + return "", fmt.Errorf( + "concurrent active snapshot %s does not match the requested digest, profile, and coverage", snapshot.ID, ) } - return true, nil + return status.Snapshot.DatasetGeneratedAt, nil } func requireGitcrawlSnapshotPublishContract( diff --git a/internal/cli/cloud_commands_test.go b/internal/cli/cloud_commands_test.go index 554542c..0700965 100644 --- a/internal/cli/cloud_commands_test.go +++ b/internal/cli/cloud_commands_test.go @@ -442,6 +442,108 @@ func TestGitcrawlReaderStatusMatchesCompleteServingSnapshot(t *testing.T) { } } +func TestRecoverConcurrentGitcrawlSnapshotAdoptsOnlyMatchingCompletedSnapshot(t *testing.T) { + snapshotID := strings.Repeat("a", 64) + snapshot := gitcrawlCloudSnapshot{ + ID: snapshotID, + SourceSyncAt: "2026-07-12T12:00:00Z", + DatasetGeneratedAt: "2026-07-12T12:01:00Z", + } + manifest := gitcrawlCloudManifest("gitcrawl/openclaw__openclaw", snapshot) + publicationCapabilities := gitcrawlCloudPublicationCapabilities(manifest.Capabilities) + const winnerGeneration = "2026-07-12T12:02:00Z" + + for _, test := range []struct { + name string + activeSnapshotID string + coverageComplete bool + wantGeneration string + want string + }{ + { + name: "independent winner generation", + activeSnapshotID: snapshotID, + coverageComplete: true, + wantGeneration: winnerGeneration, + }, + { + name: "unrelated active candidate", + activeSnapshotID: strings.Repeat("b", 64), + coverageComplete: true, + want: "does not match the requested digest, profile, and coverage", + }, + { + name: "incomplete candidate", + activeSnapshotID: snapshotID, + want: "does not match the requested digest, profile, and coverage", + }, + } { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.URL.Query().Get("snapshot_id"); got != snapshotID { + http.Error(w, fmt.Sprintf("snapshot_id = %q, want %q", got, snapshotID), http.StatusBadRequest) + return + } + _ = json.NewEncoder(w).Encode(crawlremote.PublisherStatus{ + App: manifest.App, + Archive: manifest.Archive, + ActiveSnapshotID: test.activeSnapshotID, + CoverageComplete: test.coverageComplete, + Snapshot: &crawlremote.ArchiveSnapshot{ + ID: snapshotID, + SourceSHA256: snapshotID, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: winnerGeneration, + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: publicationCapabilities, + CoverageComplete: test.coverageComplete, + }, + }) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("publisher-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + + generation, err := recoverConcurrentGitcrawlSnapshot( + context.Background(), + client, + manifest.Archive, + snapshot, + manifest, + publicationCapabilities, + &crawlremote.Error{ + Status: http.StatusConflict, + Code: "snapshot_active", + Message: "a concurrent publisher completed the snapshot", + }, + ) + if test.want != "" { + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("recovery error = %v, want %q", err, test.want) + } + if generation != "" { + t.Fatalf("recovered generation = %q, want none", generation) + } + return + } + if err != nil { + t.Fatalf("recover concurrent snapshot: %v", err) + } + if generation != test.wantGeneration { + t.Fatalf("recovered generation = %q, want %q", generation, test.wantGeneration) + } + }) + } +} + func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t *testing.T) { source := []byte("SQLite format 3\x00bound source") snapshotID := fmt.Sprintf("%x", sha256.Sum256(source)) @@ -454,11 +556,19 @@ func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t publicationCapabilities := gitcrawlCloudPublicationCapabilities(manifest.Capabilities) for _, test := range []struct { - name string - statusCode int - body []byte - want string + name string + statusGeneration string + statusCode int + body []byte + want string }{ + { + name: "publisher generation mismatch", + statusGeneration: "2026-07-12T12:02:00Z", + statusCode: http.StatusOK, + body: source, + want: "post-cutover publisher status does not match", + }, { name: "bound snapshot unavailable", statusCode: http.StatusConflict, @@ -485,6 +595,10 @@ func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t return } w.Header().Set("content-type", "application/json") + statusGeneration := snapshot.DatasetGeneratedAt + if test.statusGeneration != "" { + statusGeneration = test.statusGeneration + } _ = json.NewEncoder(w).Encode(crawlremote.PublisherStatus{ App: "gitcrawl", Archive: manifest.Archive, @@ -494,7 +608,7 @@ func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t ID: snapshotID, SourceSHA256: snapshotID, SourceSyncAt: snapshot.SourceSyncAt, - DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + DatasetGeneratedAt: statusGeneration, SchemaName: manifest.SchemaName, SchemaVersion: manifest.SchemaVersion, SchemaHash: manifest.SchemaHash, From 65210f53686293d8c40fb896d8908b637778dbf5 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 18:54:17 +0200 Subject: [PATCH 50/52] fix(cloud): harden snapshot publication exactness --- CHANGELOG.md | 1 + README.md | 9 +- internal/cli/app_test.go | 43 ++- internal/cli/cloud_commands.go | 334 +++++++++++++++++++- internal/cli/cloud_commands_test.go | 462 ++++++++++++++++++++++++++++ 5 files changed, 823 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70bee19..e48d98b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Require every Gitcrawl cloud publisher manifest to opt into snapshot staging so `--stage-only` and normal publish-then-cutover runs cannot activate serving state through the legacy compatibility path. - Update CrawlKit to v0.14.2 and bind publisher resume plus post-cutover verification to the exact snapshot ID, so a newer concurrently staged candidate cannot be mistaken for the snapshot this publisher owns. +- Fail cloud publication before any remote mutation when local enrichment coverage is incomplete unless operators explicitly pass `--allow-incomplete`. ## 0.7.1 - 2026-07-09 diff --git a/README.md b/README.md index b5de94f..d101708 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,8 @@ Use `gitcrawl remote login --github-token-env GITHUB_TOKEN` for non-browser boot SHA-256 as the snapshot identity, exports repositories, threads, revisions, fingerprints, summaries, durable clusters, and PR detail/file rows from that same image, negotiates the remote snapshot-provenance contract before touching -R2, uploads its digest-scoped bundle, and completes staged D1 coverage. +R2, uploads its digest-scoped bundle, and completes staged D1 coverage through +row- and encoded-byte-bounded ingest requests. Publishing moves unpinned reads to the complete snapshot by default, preserving the existing reader-refresh behavior. The remote must advertise `gitcrawl.snapshot.staging.v1`; `--stage-only` keeps the immutable snapshot @@ -80,8 +81,10 @@ candidate through the publisher-only status projection, skips repeated ingest when its digest, source sync, schema, resolved publication profile, persisted generation timestamp, and coverage match, then cuts it over. Cutover requires the remote contract to advertise reader-authenticated `GET /sqlite`; Gitcrawl -rechecks the exact publisher metadata and hashes the downloaded bound SQLite -image before reporting success. Before any upload or ingest, Gitcrawl verifies +validates the cutover acknowledgement, retries the scoped reader projection +until its digest, profile, generation, and dataset coverage are exact, rechecks +the exact publisher metadata, and hashes the downloaded bound SQLite image +before reporting success. Before any upload or ingest, Gitcrawl verifies the configured credential through the advertised `/v1/whoami` route and requires both publisher and reader roles, including for stage-only publication. Incomplete local enrichment fails before any remote mutation; diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 327887e..bb4a9ac 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -1099,6 +1099,7 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { var snapshotID string var sqliteImage []byte var publishedSnapshot *crawlremote.ArchiveSnapshot + var publishedDatasets []crawlremote.DatasetCoverage var publisherStatusSnapshotIDs []string mutationCounter := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -1235,16 +1236,23 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { return } if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/status") { - activeSnapshotID := "" - if sawCutover { - activeSnapshotID = snapshotID + status := crawlremote.Status{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__openclaw", } - _ = json.NewEncoder(w).Encode(crawlremote.Status{ - App: "gitcrawl", - Archive: "gitcrawl/openclaw__openclaw", - ActiveSnapshotID: activeSnapshotID, - CoverageComplete: sawCutover, - }) + if sawCutover && publishedSnapshot != nil { + status.SchemaName = publishedSnapshot.SchemaName + status.SchemaVersion = publishedSnapshot.SchemaVersion + status.SchemaHash = publishedSnapshot.SchemaHash + status.Capabilities = slices.Clone(publishedSnapshot.Capabilities) + status.ActiveSnapshotID = publishedSnapshot.ID + status.SourceSyncAt = publishedSnapshot.SourceSyncAt + status.DatasetGeneratedAt = publishedSnapshot.DatasetGeneratedAt + status.CoverageComplete = true + status.Datasets = slices.Clone(publishedDatasets) + status.Snapshot = publishedSnapshot + } + _ = json.NewEncoder(w).Encode(status) return } if r.Method == http.MethodPost && strings.HasSuffix(r.URL.EscapedPath(), "/cutover") { @@ -1297,6 +1305,7 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { http.Error(w, "coverage activation mismatch", http.StatusBadRequest) return } + publishedDatasets = testGitcrawlDatasetCoverageRows(t, body.Rows) for _, row := range body.Rows { if row[len(row)-1] != body.MutationToken { http.Error(w, "coverage token mismatch", http.StatusBadRequest) @@ -1799,8 +1808,13 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { Archive: "gitcrawl/openclaw__openclaw", ActiveSnapshotID: servingSnapshotID, } + readerProjectionExact := false + switch readerStatusRequests { + case 3, 5, 6, 7: + readerProjectionExact = true + } if servingSnapshotID == snapshotID && - readerStatusRequests >= 3 && + readerProjectionExact && stagedSnapshot != nil { status.SchemaName = stagedSnapshot.SchemaName status.SchemaVersion = stagedSnapshot.SchemaVersion @@ -2105,7 +2119,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { } stagedDatasetGeneratedAt = result.DatasetGeneratedAt if stagedDatasetGeneratedAt != concurrentDatasetGeneratedAt || - result.MutationToken == "" { + result.MutationToken != "" { t.Fatalf("stage-only did not bind its generation: %#v", result) } time.Sleep(2 * time.Millisecond) @@ -2166,8 +2180,11 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { ) } } - if readerStatusRequests != 3 { - t.Fatalf("reader status requests = %d, want 3 serving-state checks", readerStatusRequests) + if readerStatusRequests != 7 { + t.Fatalf( + "reader status requests = %d, want 7 pre-cutover and exact post-cutover checks", + readerStatusRequests, + ) } if sqliteReadRequests != 3 { t.Fatalf("SQLite read requests = %d, want failed hydration plus two retries", sqliteReadRequests) diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 5b86eec..90e1347 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "database/sql" + "encoding/json" "errors" "flag" "fmt" @@ -22,10 +23,13 @@ import ( ) const ( - gitcrawlCloudBatchSize = 250 - gitcrawlCloudSQLiteBundleChunkSize = int64(64 * 1024 * 1024) - gitcrawlCloudPublishPreflightTimeout = 30 * time.Second - gitcrawlCloudHydrationTimeout = 10 * time.Minute + gitcrawlCloudBatchSize = 250 + gitcrawlCloudIngestRequestMaxBytes = int64(4 * 1024 * 1024) + gitcrawlCloudSQLiteBundleChunkSize = int64(64 * 1024 * 1024) + gitcrawlCloudPublishPreflightTimeout = 30 * time.Second + gitcrawlCloudPostCutoverStatusAttempts = 5 + gitcrawlCloudPostCutoverStatusRetryDelay = 100 * time.Millisecond + gitcrawlCloudHydrationTimeout = 10 * time.Minute gitcrawlSnapshotAtomicCapability = "gitcrawl.snapshot.atomic" gitcrawlSnapshotCutoverCapability = "gitcrawl.snapshot.cutover" @@ -239,6 +243,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { } if recoveredGeneration != "" { snapshot.DatasetGeneratedAt = recoveredGeneration + mutationToken = "" alreadyStaged = true completedConcurrently = true break @@ -275,6 +280,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { return fmt.Errorf("complete cloud snapshot staging: %w", err) } snapshot.DatasetGeneratedAt = recoveredGeneration + mutationToken = "" alreadyStaged = true } else { mutationToken = progress.MutationToken @@ -306,6 +312,9 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { if err != nil { return fmt.Errorf("cut over cloud snapshot: %w", err) } + if err := validateGitcrawlCutoverResult(result, archiveID, snapshot.ID); err != nil { + return fmt.Errorf("validate cloud snapshot cutover: %w", err) + } cutoverResult = &result } if err := verifyGitcrawlSnapshotPublication( @@ -349,6 +358,84 @@ type ingestProgress struct { Result crawlremote.IngestResult } +type gitcrawlIngestBatchSizer struct { + baseBytes int64 + finalBaseBytes int64 + rowBytes int64 + rowCount int +} + +func newGitcrawlIngestBatchSizer( + app, archive string, + manifest crawlremote.IngestManifest, + table string, + columns []string, + cursor, mutationToken string, +) (gitcrawlIngestBatchSizer, error) { + manifest.App = strings.TrimSpace(app) + manifest.Archive = strings.TrimSpace(archive) + request := crawlremote.IngestRequest{ + Manifest: manifest, + Table: table, + Columns: columns, + Rows: [][]any{}, + Cursor: cursor, + MutationToken: mutationToken, + } + baseBytes, err := encodedGitcrawlIngestRequestBytes(request) + if err != nil { + return gitcrawlIngestBatchSizer{}, err + } + request.Final = true + finalBaseBytes, err := encodedGitcrawlIngestRequestBytes(request) + if err != nil { + return gitcrawlIngestBatchSizer{}, err + } + return gitcrawlIngestBatchSizer{ + baseBytes: baseBytes, + finalBaseBytes: finalBaseBytes, + }, nil +} + +func encodedGitcrawlIngestRequestBytes(request crawlremote.IngestRequest) (int64, error) { + encoded, err := json.Marshal(request) + if err != nil { + return 0, fmt.Errorf("encode ingest request envelope: %w", err) + } + // crawlremote uses json.Encoder, which appends one newline byte. + return int64(len(encoded)) + 1, nil +} + +func (s *gitcrawlIngestBatchSizer) add(row []any, final bool) (bool, int64, error) { + encoded, err := json.Marshal(row) + if err != nil { + return false, 0, fmt.Errorf("encode ingest row: %w", err) + } + separatorBytes := int64(0) + if s.rowCount > 0 { + separatorBytes = 1 + } + baseBytes := s.baseBytes + if final { + baseBytes = s.finalBaseBytes + } + encodedBytes := baseBytes + s.rowBytes + separatorBytes + int64(len(encoded)) + if encodedBytes > gitcrawlCloudIngestRequestMaxBytes { + return false, encodedBytes, nil + } + s.rowBytes += separatorBytes + int64(len(encoded)) + s.rowCount++ + return true, encodedBytes, nil +} + +func (s gitcrawlIngestBatchSizer) encodedBytes(final bool) int64 { + baseBytes := s.baseBytes + if final { + baseBytes = s.finalBaseBytes + } + return baseBytes + s.rowBytes +} + func sendSnapshotIngestDataset( ctx context.Context, db *sql.DB, @@ -378,6 +465,27 @@ func sendSnapshotIngestDataset( batch := make([][]any, 0, gitcrawlCloudBatchSize) var scanned int64 var accepted int64 + var batchStart int64 + var batchSizer *gitcrawlIngestBatchSizer + ensureBatchSizer := func() error { + if batchSizer != nil { + return nil + } + sizer, err := newGitcrawlIngestBatchSizer( + app, + archive, + manifest, + dataset.Name, + dataset.Columns, + cursorFor(batchStart), + mutationToken, + ) + if err != nil { + return err + } + batchSizer = &sizer + return nil + } flush := func() error { result, err := sendIngestBatch( ctx, @@ -388,7 +496,7 @@ func sendSnapshotIngestDataset( dataset.Name, dataset.Columns, batch, - scanned-int64(len(batch)), + batchStart, mutationToken, false, ) @@ -405,6 +513,8 @@ func sendSnapshotIngestDataset( accepted += result.RowsAccepted mutationToken = result.MutationToken batch = batch[:0] + batchStart = scanned + batchSizer = nil return nil } @@ -422,6 +532,34 @@ func sendSnapshotIngestDataset( values[index] = string(bytes) } } + if err := ensureBatchSizer(); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + fits, encodedBytes, err := batchSizer.add(values, false) + if err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + if !fits && len(batch) > 0 { + if err := flush(); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + if err := ensureBatchSizer(); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + fits, encodedBytes, err = batchSizer.add(values, false) + if err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + } + if !fits { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, fmt.Errorf( + "dataset %s row %d encoded ingest request is %d bytes, limit %d", + dataset.Name, + scanned, + encodedBytes, + gitcrawlCloudIngestRequestMaxBytes, + ) + } batch = append(batch, values) scanned++ if len(batch) == gitcrawlCloudBatchSize { @@ -433,7 +571,23 @@ func sendSnapshotIngestDataset( if err := rows.Err(); err != nil { return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err } - if len(batch) > 0 || scanned == 0 { + if len(batch) > 0 { + if err := flush(); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + } + if scanned == 0 { + if err := ensureBatchSizer(); err != nil { + return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err + } + if encodedBytes := batchSizer.encodedBytes(false); encodedBytes > gitcrawlCloudIngestRequestMaxBytes { + return ingestProgress{}, fmt.Errorf( + "dataset %s empty encoded ingest request is %d bytes, limit %d", + dataset.Name, + encodedBytes, + gitcrawlCloudIngestRequestMaxBytes, + ) + } if err := flush(); err != nil { return ingestProgress{RowsAccepted: accepted, MutationToken: mutationToken}, err } @@ -486,6 +640,26 @@ func sendSnapshotIngestRows( ) (ingestProgress, error) { var total int64 if len(rows) == 0 { + sizer, sizeErr := newGitcrawlIngestBatchSizer( + app, + archive, + manifest, + table, + columns, + "", + mutationToken, + ) + if sizeErr != nil { + return ingestProgress{}, sizeErr + } + if encodedBytes := sizer.encodedBytes(final); encodedBytes > gitcrawlCloudIngestRequestMaxBytes { + return ingestProgress{}, fmt.Errorf( + "ingest table %s empty encoded request is %d bytes, limit %d", + table, + encodedBytes, + gitcrawlCloudIngestRequestMaxBytes, + ) + } result, err := sendIngestBatch( ctx, client, @@ -506,10 +680,50 @@ func sendSnapshotIngestRows( }, err } var lastResult crawlremote.IngestResult - for start := 0; start < len(rows); start += gitcrawlCloudBatchSize { - end := start + gitcrawlCloudBatchSize - if end > len(rows) { - end = len(rows) + for start := 0; start < len(rows); { + sizer, err := newGitcrawlIngestBatchSizer( + app, + archive, + manifest, + table, + columns, + cursorFor(int64(start)), + mutationToken, + ) + if err != nil { + return ingestProgress{ + RowsAccepted: total, + MutationToken: mutationToken, + Result: lastResult, + }, err + } + end := start + for end < len(rows) && end-start < gitcrawlCloudBatchSize { + fits, encodedBytes, err := sizer.add(rows[end], final && end == len(rows)-1) + if err != nil { + return ingestProgress{ + RowsAccepted: total, + MutationToken: mutationToken, + Result: lastResult, + }, err + } + if !fits { + if end == start { + return ingestProgress{ + RowsAccepted: total, + MutationToken: mutationToken, + Result: lastResult, + }, fmt.Errorf( + "ingest table %s row %d encoded request is %d bytes, limit %d", + table, + start, + encodedBytes, + gitcrawlCloudIngestRequestMaxBytes, + ) + } + break + } + end++ } result, err := sendIngestBatch( ctx, @@ -534,6 +748,7 @@ func sendSnapshotIngestRows( total += result.RowsAccepted mutationToken = result.MutationToken lastResult = result + start = end } return ingestProgress{ RowsAccepted: total, @@ -1125,6 +1340,95 @@ func gitcrawlReaderStatusMatches( return gitcrawlDatasetCoverageMatches(status.Datasets, snapshot) } +func validateGitcrawlCutoverResult( + result crawlremote.CutoverResult, + archive, snapshotID string, +) error { + if result.Archive != archive { + return fmt.Errorf("cutover returned archive %q, want %q", result.Archive, archive) + } + if result.SnapshotID != snapshotID { + return fmt.Errorf("cutover returned snapshot %q, want %q", result.SnapshotID, snapshotID) + } + if result.SnapshotMode != "snapshot" { + return fmt.Errorf("cutover returned snapshot mode %q, want snapshot", result.SnapshotMode) + } + if _, err := time.Parse(time.RFC3339Nano, result.CutoverAt); err != nil { + return fmt.Errorf("cutover returned invalid timestamp %q: %w", result.CutoverAt, err) + } + return nil +} + +func verifyGitcrawlReaderProjection( + ctx context.Context, + client *crawlremote.Client, + archive string, + snapshot gitcrawlCloudSnapshot, + manifest crawlremote.IngestManifest, + publicationCapabilities []string, +) error { + return verifyGitcrawlReaderProjectionWithRetry( + ctx, + client, + archive, + snapshot, + manifest, + publicationCapabilities, + gitcrawlCloudPostCutoverStatusAttempts, + gitcrawlCloudPostCutoverStatusRetryDelay, + ) +} + +func verifyGitcrawlReaderProjectionWithRetry( + ctx context.Context, + client *crawlremote.Client, + archive string, + snapshot gitcrawlCloudSnapshot, + manifest crawlremote.IngestManifest, + publicationCapabilities []string, + attempts int, + retryDelay time.Duration, +) error { + if attempts < 1 { + attempts = 1 + } + var lastErr error + for attempt := 1; attempt <= attempts; attempt++ { + status, err := client.Status(ctx, "gitcrawl", archive) + if err == nil && gitcrawlReaderStatusMatches( + status, + snapshot, + manifest, + publicationCapabilities, + ) { + return nil + } + lastErr = err + if attempt == attempts { + break + } + timer := time.NewTimer(retryDelay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } + if lastErr != nil { + return fmt.Errorf( + "read post-cutover reader status after %d attempts: %w", + attempts, + lastErr, + ) + } + return fmt.Errorf( + "post-cutover reader status does not match snapshot %s digest, profile, generation, and coverage after %d attempts", + snapshot.ID, + attempts, + ) +} + func gitcrawlDatasetCoverageMatches( actual []crawlremote.DatasetCoverage, snapshot gitcrawlCloudSnapshot, @@ -1167,6 +1471,16 @@ func verifyGitcrawlSnapshotPublication( publicationCapabilities []string, sourceSize int64, ) error { + if err := verifyGitcrawlReaderProjection( + ctx, + client, + archive, + snapshot, + manifest, + publicationCapabilities, + ); err != nil { + return err + } status, err := client.PublishStatusForSnapshot( ctx, "gitcrawl", diff --git a/internal/cli/cloud_commands_test.go b/internal/cli/cloud_commands_test.go index 0700965..9ddbe33 100644 --- a/internal/cli/cloud_commands_test.go +++ b/internal/cli/cloud_commands_test.go @@ -177,6 +177,270 @@ func TestSendSnapshotIngestDatasetStreamsBoundedBatches(t *testing.T) { } } +func TestSendSnapshotIngestDatasetFlushesBeforeEncodedByteLimit(t *testing.T) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + defer db.Close() + if _, err := db.Exec(`create table rows(id integer primary key, value text not null)`); err != nil { + t.Fatalf("create rows: %v", err) + } + value := strings.Repeat("x", int(gitcrawlCloudIngestRequestMaxBytes/2)) + for id := 1; id <= 3; id++ { + if _, err := db.Exec(`insert into rows(id, value) values(?, ?)`, id, value); err != nil { + t.Fatalf("seed row %d: %v", id, err) + } + } + + var requests []crawlremote.IngestRequest + var encodedSizes []int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if int64(len(body)) > gitcrawlCloudIngestRequestMaxBytes { + http.Error(w, "encoded request exceeded byte budget", http.StatusRequestEntityTooLarge) + return + } + var request crawlremote.IngestRequest + if err := json.Unmarshal(body, &request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + requests = append(requests, request) + encodedSizes = append(encodedSizes, len(body)) + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + Table: request.Table, + SnapshotID: request.Manifest.SnapshotID, + MutationToken: fmt.Sprintf("mutation-%d", len(requests)), + RowsAccepted: int64(len(request.Rows)), + }) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("publisher-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + progress, err := sendSnapshotIngestDataset( + context.Background(), + db, + client, + "gitcrawl", + "gitcrawl/openclaw__gitcrawl", + crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__gitcrawl", + SnapshotID: strings.Repeat("a", 64), + }, + gitcrawlCloudDataset{ + Name: "bounded_bytes", + Columns: []string{"id", "value"}, + Query: `select id, value from rows order by id`, + RowCount: 3, + }, + "", + ) + if err != nil { + t.Fatalf("stream byte-bounded dataset: %v", err) + } + if progress.RowsAccepted != 3 || progress.MutationToken != "mutation-3" { + t.Fatalf("progress = %#v", progress) + } + if len(requests) != 3 { + t.Fatalf("requests = %d, want one per large row", len(requests)) + } + for index := range requests { + if len(requests[index].Rows) != 1 { + t.Fatalf("request %d rows = %d, want 1", index, len(requests[index].Rows)) + } + if int64(encodedSizes[index]) > gitcrawlCloudIngestRequestMaxBytes { + t.Fatalf( + "request %d encoded bytes = %d, limit %d", + index, + encodedSizes[index], + gitcrawlCloudIngestRequestMaxBytes, + ) + } + } +} + +func TestSendSnapshotIngestRowsFlushesBeforeEncodedByteLimit(t *testing.T) { + var requests []crawlremote.IngestRequest + var encodedSizes []int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if int64(len(body)) > gitcrawlCloudIngestRequestMaxBytes { + http.Error(w, "encoded request exceeded byte budget", http.StatusRequestEntityTooLarge) + return + } + var request crawlremote.IngestRequest + if err := json.Unmarshal(body, &request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + requests = append(requests, request) + encodedSizes = append(encodedSizes, len(body)) + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + Table: request.Table, + SnapshotID: request.Manifest.SnapshotID, + MutationToken: fmt.Sprintf("mutation-%d", len(requests)), + RowsAccepted: int64(len(request.Rows)), + Complete: request.Final, + }) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("publisher-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + value := strings.Repeat("x", int(gitcrawlCloudIngestRequestMaxBytes/2)) + progress, err := sendSnapshotIngestRows( + context.Background(), + client, + "gitcrawl", + "gitcrawl/openclaw__gitcrawl", + crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__gitcrawl", + SnapshotID: strings.Repeat("a", 64), + }, + "threads", + []string{"body"}, + [][]any{{value}, {value}, {value}}, + "", + true, + ) + if err != nil { + t.Fatalf("send byte-bounded rows: %v", err) + } + if progress.RowsAccepted != 3 || progress.MutationToken != "mutation-3" { + t.Fatalf("progress = %#v", progress) + } + if len(requests) != 3 { + t.Fatalf("requests = %d, want one per large row", len(requests)) + } + for index := range requests { + if len(requests[index].Rows) != 1 { + t.Fatalf("request %d rows = %d, want 1", index, len(requests[index].Rows)) + } + if int64(encodedSizes[index]) > gitcrawlCloudIngestRequestMaxBytes { + t.Fatalf( + "request %d encoded bytes = %d, limit %d", + index, + encodedSizes[index], + gitcrawlCloudIngestRequestMaxBytes, + ) + } + wantCursor := "" + if index > 0 { + wantCursor = fmt.Sprint(index) + } + if requests[index].Cursor != wantCursor { + t.Fatalf("request %d cursor = %q, want %q", index, requests[index].Cursor, wantCursor) + } + if requests[index].Final != (index == len(requests)-1) { + t.Fatalf("request %d final = %v", index, requests[index].Final) + } + } +} + +func TestIngestBatchingRejectsSingleOversizedRowBeforeRequest(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + http.Error(w, "unexpected request", http.StatusInternalServerError) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("publisher-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + _, err = sendSnapshotIngestRows( + context.Background(), + client, + "gitcrawl", + "gitcrawl/openclaw__gitcrawl", + crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__gitcrawl", + SnapshotID: strings.Repeat("a", 64), + }, + "threads", + []string{"body"}, + [][]any{{strings.Repeat("x", int(gitcrawlCloudIngestRequestMaxBytes))}}, + "", + false, + ) + if err == nil || + !strings.Contains(err.Error(), "ingest table threads row 0 encoded request") || + !strings.Contains(err.Error(), fmt.Sprintf("limit %d", gitcrawlCloudIngestRequestMaxBytes)) { + t.Fatalf("oversized row error = %v", err) + } + if requests != 0 { + t.Fatalf("remote requests = %d, want 0", requests) + } + + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + defer db.Close() + if _, err := db.Exec(`create table rows(value text not null)`); err != nil { + t.Fatalf("create rows: %v", err) + } + if _, err := db.Exec( + `insert into rows(value) values(?)`, + strings.Repeat("x", int(gitcrawlCloudIngestRequestMaxBytes)), + ); err != nil { + t.Fatalf("seed oversized row: %v", err) + } + _, err = sendSnapshotIngestDataset( + context.Background(), + db, + client, + "gitcrawl", + "gitcrawl/openclaw__gitcrawl", + crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__gitcrawl", + SnapshotID: strings.Repeat("a", 64), + }, + gitcrawlCloudDataset{ + Name: "oversized", + Columns: []string{"value"}, + Query: `select value from rows`, + RowCount: 1, + }, + "", + ) + if err == nil || + !strings.Contains(err.Error(), "dataset oversized row 0 encoded ingest request") { + t.Fatalf("oversized dataset row error = %v", err) + } + if requests != 0 { + t.Fatalf("remote requests after streamed row = %d, want 0", requests) + } +} + func TestCompleteGitcrawlSnapshotStagingRequiresExactAcknowledgement(t *testing.T) { snapshotID := strings.Repeat("a", 64) snapshot := gitcrawlCloudSnapshot{ @@ -442,6 +706,180 @@ func TestGitcrawlReaderStatusMatchesCompleteServingSnapshot(t *testing.T) { } } +func TestValidateGitcrawlCutoverResultRequiresExactAcknowledgement(t *testing.T) { + const archive = "gitcrawl/openclaw__gitcrawl" + snapshotID := strings.Repeat("a", 64) + valid := crawlremote.CutoverResult{ + Archive: archive, + SnapshotID: snapshotID, + SnapshotMode: "snapshot", + CutoverAt: "2026-07-12T12:00:00.123456789Z", + } + for _, test := range []struct { + name string + mutate func(*crawlremote.CutoverResult) + want string + }{ + {name: "exact acknowledgement"}, + { + name: "wrong archive", + mutate: func(result *crawlremote.CutoverResult) { + result.Archive = "gitcrawl/other" + }, + want: "want \"" + archive + "\"", + }, + { + name: "wrong snapshot", + mutate: func(result *crawlremote.CutoverResult) { + result.SnapshotID = strings.Repeat("b", 64) + }, + want: "want \"" + snapshotID + "\"", + }, + { + name: "wrong mode", + mutate: func(result *crawlremote.CutoverResult) { + result.SnapshotMode = "mutable" + }, + want: "want snapshot", + }, + { + name: "invalid timestamp", + mutate: func(result *crawlremote.CutoverResult) { + result.CutoverAt = "later" + }, + want: "invalid timestamp", + }, + } { + t.Run(test.name, func(t *testing.T) { + result := valid + if test.mutate != nil { + test.mutate(&result) + } + err := validateGitcrawlCutoverResult(result, archive, snapshotID) + if test.want == "" { + if err != nil { + t.Fatalf("validate cutover: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("validation error = %v, want %q", err, test.want) + } + }) + } +} + +func TestVerifyGitcrawlReaderProjectionUsesBoundedExactRetries(t *testing.T) { + snapshotID := strings.Repeat("a", 64) + snapshot := gitcrawlCloudSnapshot{ + ID: snapshotID, + SourceSyncAt: "2026-07-12T12:00:00Z", + DatasetGeneratedAt: "2026-07-12T12:01:00Z", + Datasets: []gitcrawlCloudDataset{{ + Name: "repositories", + RowCount: 1, + EligibleCount: 1, + CoveredCount: 1, + MaxSourceAt: "2026-07-12T12:00:00Z", + Complete: true, + }}, + } + manifest := gitcrawlCloudManifest("gitcrawl/openclaw__gitcrawl", snapshot) + capabilities := gitcrawlCloudPublicationCapabilities(manifest.Capabilities) + exactStatus := crawlremote.Status{ + App: manifest.App, + Archive: manifest.Archive, + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: capabilities, + ActiveSnapshotID: snapshot.ID, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + CoverageComplete: true, + Datasets: []crawlremote.DatasetCoverage{{ + Dataset: "repositories", + RowCount: 1, + EligibleCount: 1, + CoveredCount: 1, + FreshCount: 1, + MaxSourceAt: "2026-07-12T12:00:00Z", + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + Complete: true, + }}, + Snapshot: &crawlremote.ArchiveSnapshot{ + ID: snapshot.ID, + SourceSHA256: snapshot.ID, + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: capabilities, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + CoverageComplete: true, + }, + } + + for _, test := range []struct { + name string + exactAt int + want string + wantCalls int + }{ + { + name: "eventually exact", + exactAt: 3, + wantCalls: 3, + }, + { + name: "retry bound exhausted", + want: "after 3 attempts", + wantCalls: 3, + }, + } { + t.Run(test.name, func(t *testing.T) { + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + status := exactStatus + if test.exactAt == 0 || calls < test.exactAt { + status.ActiveSnapshotID = strings.Repeat("b", 64) + } + _ = json.NewEncoder(w).Encode(status) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("reader-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + err = verifyGitcrawlReaderProjectionWithRetry( + context.Background(), + client, + manifest.Archive, + snapshot, + manifest, + capabilities, + 3, + 0, + ) + if test.want == "" { + if err != nil { + t.Fatalf("verify reader projection: %v", err) + } + } else if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("verification error = %v, want %q", err, test.want) + } + if calls != test.wantCalls { + t.Fatalf("status calls = %d, want %d", calls, test.wantCalls) + } + }) + } +} + func TestRecoverConcurrentGitcrawlSnapshotAdoptsOnlyMatchingCompletedSnapshot(t *testing.T) { snapshotID := strings.Repeat("a", 64) snapshot := gitcrawlCloudSnapshot{ @@ -585,6 +1023,30 @@ func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t t.Run(test.name, func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/status"): + _ = json.NewEncoder(w).Encode(crawlremote.Status{ + App: manifest.App, + Archive: manifest.Archive, + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: publicationCapabilities, + ActiveSnapshotID: snapshot.ID, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + CoverageComplete: true, + Snapshot: &crawlremote.ArchiveSnapshot{ + ID: snapshot.ID, + SourceSHA256: snapshot.ID, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: publicationCapabilities, + CoverageComplete: true, + }, + }) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status"): if got := r.URL.Query().Get("snapshot_id"); got != snapshotID { http.Error( From 57c5842fd489417baa1eb47b845d65d224eb9b5f Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 19:10:18 +0200 Subject: [PATCH 51/52] fix(cloud): attest snapshot cutover status --- internal/cli/app_test.go | 14 +- internal/cli/cloud_commands.go | 36 ++- internal/cli/cloud_commands_test.go | 398 +++++++++++++++++++++++++++- 3 files changed, 441 insertions(+), 7 deletions(-) diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index bb4a9ac..0806e2c 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -1241,6 +1241,9 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { Archive: "gitcrawl/openclaw__openclaw", } if sawCutover && publishedSnapshot != nil { + status.Mode = "cloud" + status.SnapshotMode = "snapshot" + status.SnapshotCutoverAt = "2026-07-12T09:00:00Z" status.SchemaName = publishedSnapshot.SchemaName status.SchemaVersion = publishedSnapshot.SchemaVersion status.SchemaHash = publishedSnapshot.SchemaHash @@ -1250,7 +1253,9 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { status.DatasetGeneratedAt = publishedSnapshot.DatasetGeneratedAt status.CoverageComplete = true status.Datasets = slices.Clone(publishedDatasets) - status.Snapshot = publishedSnapshot + snapshot := *publishedSnapshot + snapshot.CutoverAt = status.SnapshotCutoverAt + status.Snapshot = &snapshot } _ = json.NewEncoder(w).Encode(status) return @@ -1816,6 +1821,9 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { if servingSnapshotID == snapshotID && readerProjectionExact && stagedSnapshot != nil { + status.Mode = "cloud" + status.SnapshotMode = "snapshot" + status.SnapshotCutoverAt = "2026-07-12T11:00:00Z" status.SchemaName = stagedSnapshot.SchemaName status.SchemaVersion = stagedSnapshot.SchemaVersion status.SchemaHash = stagedSnapshot.SchemaHash @@ -1824,7 +1832,9 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { status.DatasetGeneratedAt = stagedSnapshot.DatasetGeneratedAt status.CoverageComplete = true status.Datasets = slices.Clone(stagedDatasets) - status.Snapshot = stagedSnapshot + snapshot := *stagedSnapshot + snapshot.CutoverAt = status.SnapshotCutoverAt + status.Snapshot = &snapshot } _ = json.NewEncoder(w).Encode(status) return diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 90e1347..6367a9f 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -306,7 +306,9 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { snapshot, manifest, publicationCapabilities, + "", ) + expectedCutoverAt := "" if !alreadyCutOver { result, err := client.Cutover(ctx, "gitcrawl", archiveID, snapshot.ID) if err != nil { @@ -316,6 +318,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { return fmt.Errorf("validate cloud snapshot cutover: %w", err) } cutoverResult = &result + expectedCutoverAt = result.CutoverAt } if err := verifyGitcrawlSnapshotPublication( ctx, @@ -327,6 +330,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { snapshot, manifest, publicationCapabilities, + expectedCutoverAt, sqliteSourceSize, ); err != nil { return fmt.Errorf("verify published cloud snapshot: %w", err) @@ -1314,9 +1318,12 @@ func gitcrawlReaderStatusMatches( snapshot gitcrawlCloudSnapshot, manifest crawlremote.IngestManifest, publicationCapabilities []string, + expectedCutoverAt string, ) bool { if status.App != manifest.App || status.Archive != manifest.Archive || + status.Mode != "cloud" || + status.SnapshotMode != "snapshot" || status.ActiveSnapshotID != snapshot.ID || status.SchemaName != manifest.SchemaName || status.SchemaVersion != manifest.SchemaVersion || @@ -1337,7 +1344,28 @@ func gitcrawlReaderStatusMatches( status.Snapshot.DatasetGeneratedAt != snapshot.DatasetGeneratedAt { return false } - return gitcrawlDatasetCoverageMatches(status.Datasets, snapshot) + return gitcrawlReaderCutoverMatches(status, expectedCutoverAt) && + gitcrawlDatasetCoverageMatches(status.Datasets, snapshot) +} + +func gitcrawlReaderCutoverMatches(status crawlremote.Status, expectedCutoverAt string) bool { + if status.Snapshot == nil { + return false + } + if _, err := time.Parse(time.RFC3339Nano, status.SnapshotCutoverAt); err != nil { + return false + } + if _, err := time.Parse(time.RFC3339Nano, status.Snapshot.CutoverAt); err != nil || + status.SnapshotCutoverAt != status.Snapshot.CutoverAt { + return false + } + if expectedCutoverAt == "" { + return true + } + if _, err := time.Parse(time.RFC3339Nano, expectedCutoverAt); err != nil { + return false + } + return status.SnapshotCutoverAt == expectedCutoverAt } func validateGitcrawlCutoverResult( @@ -1366,6 +1394,7 @@ func verifyGitcrawlReaderProjection( snapshot gitcrawlCloudSnapshot, manifest crawlremote.IngestManifest, publicationCapabilities []string, + expectedCutoverAt string, ) error { return verifyGitcrawlReaderProjectionWithRetry( ctx, @@ -1374,6 +1403,7 @@ func verifyGitcrawlReaderProjection( snapshot, manifest, publicationCapabilities, + expectedCutoverAt, gitcrawlCloudPostCutoverStatusAttempts, gitcrawlCloudPostCutoverStatusRetryDelay, ) @@ -1386,6 +1416,7 @@ func verifyGitcrawlReaderProjectionWithRetry( snapshot gitcrawlCloudSnapshot, manifest crawlremote.IngestManifest, publicationCapabilities []string, + expectedCutoverAt string, attempts int, retryDelay time.Duration, ) error { @@ -1400,6 +1431,7 @@ func verifyGitcrawlReaderProjectionWithRetry( snapshot, manifest, publicationCapabilities, + expectedCutoverAt, ) { return nil } @@ -1469,6 +1501,7 @@ func verifyGitcrawlSnapshotPublication( snapshot gitcrawlCloudSnapshot, manifest crawlremote.IngestManifest, publicationCapabilities []string, + expectedCutoverAt string, sourceSize int64, ) error { if err := verifyGitcrawlReaderProjection( @@ -1478,6 +1511,7 @@ func verifyGitcrawlSnapshotPublication( snapshot, manifest, publicationCapabilities, + expectedCutoverAt, ); err != nil { return err } diff --git a/internal/cli/cloud_commands_test.go b/internal/cli/cloud_commands_test.go index 9ddbe33..164d1cc 100644 --- a/internal/cli/cloud_commands_test.go +++ b/internal/cli/cloud_commands_test.go @@ -441,6 +441,254 @@ func TestIngestBatchingRejectsSingleOversizedRowBeforeRequest(t *testing.T) { } } +func TestIngestBatchingMatchesCrawlkitEscapingAtByteBoundary(t *testing.T) { + manifest := crawlremote.IngestManifest{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__gitcrawl", + SchemaName: gitcrawlCloudSchemaName, + SchemaVersion: gitcrawlCloudSchemaVersion, + SchemaHash: gitcrawlCloudSchemaHash, + SnapshotID: strings.Repeat("a", 64), + SourceSHA256: strings.Repeat("a", 64), + } + columns := []string{"value"} + var bodies [][]byte + var requests []crawlremote.IngestRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if int64(len(body)) > gitcrawlCloudIngestRequestMaxBytes { + http.Error(w, "encoded request exceeded byte budget", http.StatusRequestEntityTooLarge) + return + } + var request crawlremote.IngestRequest + if err := json.Unmarshal(body, &request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + bodies = append(bodies, body) + requests = append(requests, request) + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{ + Table: request.Table, + SnapshotID: request.Manifest.SnapshotID, + MutationToken: fmt.Sprintf("mutation-%d", len(requests)), + RowsAccepted: int64(len(request.Rows)), + Complete: request.Final, + }) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("publisher-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + + fitRows, oversizedRows := escapedIngestBoundaryValues( + t, + manifest, + "escaped_rows", + columns, + true, + ) + if _, err := sendSnapshotIngestRows( + context.Background(), + client, + manifest.App, + manifest.Archive, + manifest, + "escaped_rows", + columns, + [][]any{{fitRows}}, + "", + true, + ); err != nil { + t.Fatalf("send escaped in-memory boundary row: %v", err) + } + assertCrawlkitWireSizeParity(t, requests[0], bodies[0]) + assertEscapedIngestWireFragments(t, bodies[0]) + requestCount := len(requests) + if _, err := sendSnapshotIngestRows( + context.Background(), + client, + manifest.App, + manifest.Archive, + manifest, + "escaped_rows", + columns, + [][]any{{oversizedRows}}, + "", + true, + ); err == nil || !strings.Contains(err.Error(), "encoded request") { + t.Fatalf("escaped in-memory oversized row error = %v", err) + } + if len(requests) != requestCount { + t.Fatalf("escaped in-memory oversized row sent %d requests", len(requests)-requestCount) + } + + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("open escaped stream database: %v", err) + } + defer db.Close() + if _, err := db.Exec(`create table rows(value text not null)`); err != nil { + t.Fatalf("create escaped stream rows: %v", err) + } + fitStream, oversizedStream := escapedIngestBoundaryValues( + t, + manifest, + "escaped_stream", + columns, + false, + ) + if _, err := db.Exec(`insert into rows(value) values(?)`, fitStream); err != nil { + t.Fatalf("seed escaped stream boundary row: %v", err) + } + dataset := gitcrawlCloudDataset{ + Name: "escaped_stream", + Columns: columns, + Query: `select value from rows`, + RowCount: 1, + } + if _, err := sendSnapshotIngestDataset( + context.Background(), + db, + client, + manifest.App, + manifest.Archive, + manifest, + dataset, + "", + ); err != nil { + t.Fatalf("send escaped stream boundary row: %v", err) + } + assertCrawlkitWireSizeParity(t, requests[requestCount], bodies[requestCount]) + assertEscapedIngestWireFragments(t, bodies[requestCount]) + requestCount = len(requests) + if _, err := db.Exec(`update rows set value = ?`, oversizedStream); err != nil { + t.Fatalf("seed escaped stream oversized row: %v", err) + } + if _, err := sendSnapshotIngestDataset( + context.Background(), + db, + client, + manifest.App, + manifest.Archive, + manifest, + dataset, + "", + ); err == nil || !strings.Contains(err.Error(), "encoded ingest request") { + t.Fatalf("escaped stream oversized row error = %v", err) + } + if len(requests) != requestCount { + t.Fatalf("escaped stream oversized row sent %d requests", len(requests)-requestCount) + } +} + +func escapedIngestBoundaryValues( + t *testing.T, + manifest crawlremote.IngestManifest, + table string, + columns []string, + final bool, +) (fit string, oversized string) { + t.Helper() + const pattern = "\"\\\x00\b\f\n\r\t<>&\u2028\u2029\u00e9\u65e5\u672c\u8a9e\U0001F600" + fits := func(repeats int) bool { + t.Helper() + sizer, err := newGitcrawlIngestBatchSizer( + manifest.App, + manifest.Archive, + manifest, + table, + columns, + "", + "", + ) + if err != nil { + t.Fatalf("create escaped ingest batch sizer: %v", err) + } + ok, _, err := sizer.add([]any{strings.Repeat(pattern, repeats)}, final) + if err != nil { + t.Fatalf("size escaped ingest row: %v", err) + } + return ok + } + low, high := 0, 1 + for fits(high) { + low = high + high *= 2 + } + for low+1 < high { + middle := low + (high-low)/2 + if fits(middle) { + low = middle + } else { + high = middle + } + } + return strings.Repeat(pattern, low), strings.Repeat(pattern, high) +} + +func assertCrawlkitWireSizeParity( + t *testing.T, + request crawlremote.IngestRequest, + body []byte, +) { + t.Helper() + expected, err := encodedGitcrawlIngestRequestBytes(request) + if err != nil { + t.Fatalf("size CrawlKit ingest request: %v", err) + } + if int64(len(body)) != expected { + t.Fatalf("CrawlKit wire bytes = %d, batch sizer = %d", len(body), expected) + } + if int64(len(body)) > gitcrawlCloudIngestRequestMaxBytes { + t.Fatalf( + "CrawlKit wire bytes = %d, limit %d", + len(body), + gitcrawlCloudIngestRequestMaxBytes, + ) + } +} + +func assertEscapedIngestWireFragments(t *testing.T, body []byte) { + t.Helper() + for _, fragment := range []string{ + `\"`, + `\\`, + `\u0000`, + `\b`, + `\f`, + `\n`, + `\r`, + `\t`, + `\u003c`, + `\u003e`, + `\u0026`, + `\u2028`, + `\u2029`, + } { + if !bytes.Contains(body, []byte(fragment)) { + t.Fatalf("CrawlKit wire body is missing escaped fragment %q", fragment) + } + } + for _, value := range []string{ + "\u00e9", + "\u65e5\u672c\u8a9e", + "\U0001F600", + } { + if !bytes.Contains(body, []byte(value)) { + t.Fatalf("CrawlKit wire body is missing UTF-8 value %q", value) + } + } +} + func TestCompleteGitcrawlSnapshotStagingRequiresExactAcknowledgement(t *testing.T) { snapshotID := strings.Repeat("a", 64) snapshot := gitcrawlCloudSnapshot{ @@ -638,6 +886,7 @@ func TestValidateGitcrawlSQLiteBundleUploadRequiresFinalSnapshotDigest(t *testin func TestGitcrawlReaderStatusMatchesCompleteServingSnapshot(t *testing.T) { snapshotID := strings.Repeat("a", 64) + const cutoverAt = "2026-07-12T12:02:00.123456789Z" snapshot := gitcrawlCloudSnapshot{ ID: snapshotID, SourceSyncAt: "2026-07-12T12:00:00Z", @@ -656,10 +905,13 @@ func TestGitcrawlReaderStatusMatchesCompleteServingSnapshot(t *testing.T) { status := crawlremote.Status{ App: manifest.App, Archive: manifest.Archive, + Mode: "cloud", SchemaName: manifest.SchemaName, SchemaVersion: manifest.SchemaVersion, SchemaHash: manifest.SchemaHash, Capabilities: capabilities, + SnapshotMode: "snapshot", + SnapshotCutoverAt: cutoverAt, ActiveSnapshotID: snapshotID, SourceSyncAt: snapshot.SourceSyncAt, DatasetGeneratedAt: snapshot.DatasetGeneratedAt, @@ -684,28 +936,154 @@ func TestGitcrawlReaderStatusMatchesCompleteServingSnapshot(t *testing.T) { SourceSyncAt: snapshot.SourceSyncAt, DatasetGeneratedAt: snapshot.DatasetGeneratedAt, CoverageComplete: true, + CutoverAt: cutoverAt, }, } - if !gitcrawlReaderStatusMatches(status, snapshot, manifest, capabilities) { + if !gitcrawlReaderStatusMatches(status, snapshot, manifest, capabilities, cutoverAt) { t.Fatal("complete serving snapshot did not match") } status.CoverageComplete = false - if gitcrawlReaderStatusMatches(status, snapshot, manifest, capabilities) { + if gitcrawlReaderStatusMatches(status, snapshot, manifest, capabilities, cutoverAt) { t.Fatal("top-level incomplete reader status matched") } status.CoverageComplete = true status.Snapshot.CoverageComplete = false - if gitcrawlReaderStatusMatches(status, snapshot, manifest, capabilities) { + if gitcrawlReaderStatusMatches(status, snapshot, manifest, capabilities, cutoverAt) { t.Fatal("nested incomplete reader status matched") } status.Snapshot.CoverageComplete = true status.Datasets[0].CoveredCount = 0 - if gitcrawlReaderStatusMatches(status, snapshot, manifest, capabilities) { + if gitcrawlReaderStatusMatches(status, snapshot, manifest, capabilities, cutoverAt) { t.Fatal("reader dataset count drift matched") } } +func TestGitcrawlReaderStatusMatchesRequiresCutoverAttestation(t *testing.T) { + snapshotID := strings.Repeat("a", 64) + const cutoverAt = "2026-07-12T12:02:00.123456789Z" + snapshot := gitcrawlCloudSnapshot{ + ID: snapshotID, + SourceSyncAt: "2026-07-12T12:00:00Z", + DatasetGeneratedAt: "2026-07-12T12:01:00Z", + } + manifest := gitcrawlCloudManifest("gitcrawl/openclaw__gitcrawl", snapshot) + capabilities := gitcrawlCloudPublicationCapabilities(manifest.Capabilities) + valid := crawlremote.Status{ + App: manifest.App, + Archive: manifest.Archive, + Mode: "cloud", + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: capabilities, + SnapshotMode: "snapshot", + SnapshotCutoverAt: cutoverAt, + ActiveSnapshotID: snapshotID, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + CoverageComplete: true, + Snapshot: &crawlremote.ArchiveSnapshot{ + ID: snapshotID, + SourceSHA256: snapshotID, + SchemaName: manifest.SchemaName, + SchemaVersion: manifest.SchemaVersion, + SchemaHash: manifest.SchemaHash, + Capabilities: capabilities, + SourceSyncAt: snapshot.SourceSyncAt, + DatasetGeneratedAt: snapshot.DatasetGeneratedAt, + CoverageComplete: true, + CutoverAt: cutoverAt, + }, + } + + for _, test := range []struct { + name string + expectedCutoverAt string + mutate func(*crawlremote.Status) + }{ + {name: "resumed cutover", expectedCutoverAt: ""}, + {name: "fresh cutover", expectedCutoverAt: cutoverAt}, + { + name: "missing cloud mode", + mutate: func(status *crawlremote.Status) { + status.Mode = "" + }, + }, + { + name: "missing snapshot mode", + mutate: func(status *crawlremote.Status) { + status.SnapshotMode = "" + }, + }, + { + name: "legacy snapshot mode", + mutate: func(status *crawlremote.Status) { + status.SnapshotMode = "legacy" + }, + }, + { + name: "missing top-level cutover", + mutate: func(status *crawlremote.Status) { + status.SnapshotCutoverAt = "" + }, + }, + { + name: "missing nested cutover", + mutate: func(status *crawlremote.Status) { + status.Snapshot.CutoverAt = "" + }, + }, + { + name: "invalid top-level cutover", + mutate: func(status *crawlremote.Status) { + status.SnapshotCutoverAt = "later" + }, + }, + { + name: "inconsistent nested cutover", + mutate: func(status *crawlremote.Status) { + status.Snapshot.CutoverAt = "2026-07-12T12:02:01Z" + }, + }, + { + name: "equivalent nested cutover encoding", + mutate: func(status *crawlremote.Status) { + status.Snapshot.CutoverAt = "2026-07-12T12:02:00.123456789+00:00" + }, + }, + { + name: "fresh acknowledgement mismatch", + expectedCutoverAt: "2026-07-12T12:02:01Z", + }, + { + name: "equivalent acknowledgement encoding", + expectedCutoverAt: "2026-07-12T12:02:00.123456789+00:00", + }, + } { + t.Run(test.name, func(t *testing.T) { + status := valid + snapshotEnvelope := *valid.Snapshot + status.Snapshot = &snapshotEnvelope + if test.mutate != nil { + test.mutate(&status) + } + matched := gitcrawlReaderStatusMatches( + status, + snapshot, + manifest, + capabilities, + test.expectedCutoverAt, + ) + want := test.mutate == nil && + (test.expectedCutoverAt == "" || test.expectedCutoverAt == cutoverAt) + if matched != want { + t.Fatalf("cutover status match = %v, want %v", matched, want) + } + }) + } +} + func TestValidateGitcrawlCutoverResultRequiresExactAcknowledgement(t *testing.T) { const archive = "gitcrawl/openclaw__gitcrawl" snapshotID := strings.Repeat("a", 64) @@ -771,6 +1149,7 @@ func TestValidateGitcrawlCutoverResultRequiresExactAcknowledgement(t *testing.T) func TestVerifyGitcrawlReaderProjectionUsesBoundedExactRetries(t *testing.T) { snapshotID := strings.Repeat("a", 64) + const cutoverAt = "2026-07-12T12:02:00Z" snapshot := gitcrawlCloudSnapshot{ ID: snapshotID, SourceSyncAt: "2026-07-12T12:00:00Z", @@ -789,10 +1168,13 @@ func TestVerifyGitcrawlReaderProjectionUsesBoundedExactRetries(t *testing.T) { exactStatus := crawlremote.Status{ App: manifest.App, Archive: manifest.Archive, + Mode: "cloud", SchemaName: manifest.SchemaName, SchemaVersion: manifest.SchemaVersion, SchemaHash: manifest.SchemaHash, Capabilities: capabilities, + SnapshotMode: "snapshot", + SnapshotCutoverAt: cutoverAt, ActiveSnapshotID: snapshot.ID, SourceSyncAt: snapshot.SourceSyncAt, DatasetGeneratedAt: snapshot.DatasetGeneratedAt, @@ -817,6 +1199,7 @@ func TestVerifyGitcrawlReaderProjectionUsesBoundedExactRetries(t *testing.T) { SourceSyncAt: snapshot.SourceSyncAt, DatasetGeneratedAt: snapshot.DatasetGeneratedAt, CoverageComplete: true, + CutoverAt: cutoverAt, }, } @@ -863,6 +1246,7 @@ func TestVerifyGitcrawlReaderProjectionUsesBoundedExactRetries(t *testing.T) { snapshot, manifest, capabilities, + cutoverAt, 3, 0, ) @@ -985,6 +1369,7 @@ func TestRecoverConcurrentGitcrawlSnapshotAdoptsOnlyMatchingCompletedSnapshot(t func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t *testing.T) { source := []byte("SQLite format 3\x00bound source") snapshotID := fmt.Sprintf("%x", sha256.Sum256(source)) + const cutoverAt = "2026-07-12T12:02:00Z" snapshot := gitcrawlCloudSnapshot{ ID: snapshotID, SourceSyncAt: "2026-07-12T12:00:00Z", @@ -1027,10 +1412,13 @@ func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t _ = json.NewEncoder(w).Encode(crawlremote.Status{ App: manifest.App, Archive: manifest.Archive, + Mode: "cloud", SchemaName: manifest.SchemaName, SchemaVersion: manifest.SchemaVersion, SchemaHash: manifest.SchemaHash, Capabilities: publicationCapabilities, + SnapshotMode: "snapshot", + SnapshotCutoverAt: cutoverAt, ActiveSnapshotID: snapshot.ID, SourceSyncAt: snapshot.SourceSyncAt, DatasetGeneratedAt: snapshot.DatasetGeneratedAt, @@ -1045,6 +1433,7 @@ func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t SchemaHash: manifest.SchemaHash, Capabilities: publicationCapabilities, CoverageComplete: true, + CutoverAt: cutoverAt, }, }) case r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status"): @@ -1111,6 +1500,7 @@ func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t snapshot, manifest, publicationCapabilities, + cutoverAt, int64(len(source)), ) if err == nil || !strings.Contains(err.Error(), test.want) { From 0ac0a1749328d35ec93f02f2afd977c1f8860ffb Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 19:16:47 +0200 Subject: [PATCH 52/52] fix(cloud): bind bundle digest to snapshot --- internal/cli/cloud_commands.go | 30 +++++++++++++++-- internal/cli/cloud_commands_test.go | 52 +++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/internal/cli/cloud_commands.go b/internal/cli/cloud_commands.go index 6367a9f..3c64475 100644 --- a/internal/cli/cloud_commands.go +++ b/internal/cli/cloud_commands.go @@ -199,6 +199,7 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { "gitcrawl", archiveID, snapshotPath, + snapshot.ID, counts, ) if err != nil { @@ -922,14 +923,26 @@ func uploadSQLiteArchive(ctx context.Context, client *crawlremote.Client, app, a return nil, err } defer cleanup() - bundle, _, err := uploadSQLiteSnapshotArchive(ctx, client, app, archive, snapshotPath, counts) + snapshotID, err := cloudFileSHA256(snapshotPath) + if err != nil { + return nil, err + } + bundle, _, err := uploadSQLiteSnapshotArchive( + ctx, + client, + app, + archive, + snapshotPath, + snapshotID, + counts, + ) return bundle, err } func uploadSQLiteSnapshotArchive( ctx context.Context, client *crawlremote.Client, - app, archive, snapshotPath string, + app, archive, snapshotPath, expectedSnapshotID string, counts map[string]int64, ) (*crawlremote.SQLiteBundle, int64, error) { bundle, err := crawlremote.BuildSnapshotGzipSQLiteBundle(ctx, crawlremote.SQLiteBundleBuildOptions{ @@ -944,6 +957,19 @@ func uploadSQLiteSnapshotArchive( return nil, 0, err } defer bundle.Cleanup() + expectedSnapshotID = strings.TrimSpace(expectedSnapshotID) + if expectedSnapshotID == "" { + return nil, 0, fmt.Errorf("selected SQLite snapshot digest is required") + } + if bundle.Manifest.SnapshotID != expectedSnapshotID || + bundle.Manifest.Object.SHA256 != expectedSnapshotID { + return nil, 0, fmt.Errorf( + "SQLite bundle source digest changed from selected snapshot %s to snapshot %s with object digest %s", + expectedSnapshotID, + bundle.Manifest.SnapshotID, + bundle.Manifest.Object.SHA256, + ) + } sourceSize := bundle.Manifest.Object.Size if sourceSize <= 0 { return nil, 0, fmt.Errorf("SQLite bundle manifest has invalid source size %d", sourceSize) diff --git a/internal/cli/cloud_commands_test.go b/internal/cli/cloud_commands_test.go index 164d1cc..20a6624 100644 --- a/internal/cli/cloud_commands_test.go +++ b/internal/cli/cloud_commands_test.go @@ -10,6 +10,8 @@ import ( "io" "net/http" "net/http/httptest" + "os" + "path/filepath" "slices" "strings" "testing" @@ -884,6 +886,56 @@ func TestValidateGitcrawlSQLiteBundleUploadRequiresFinalSnapshotDigest(t *testin } } +func TestUploadSQLiteSnapshotArchiveRejectsSameSizeSourceDigestDrift(t *testing.T) { + snapshotPath := filepath.Join(t.TempDir(), "snapshot.db") + selectedSource := []byte("SQLite format 3\x00selected-source") + driftedSource := []byte("SQLite format 3\x00drifted--source") + if len(selectedSource) != len(driftedSource) { + t.Fatalf("test sources differ in size: %d != %d", len(selectedSource), len(driftedSource)) + } + if err := os.WriteFile(snapshotPath, selectedSource, 0o600); err != nil { + t.Fatalf("write selected source: %v", err) + } + selectedSnapshotID, err := cloudFileSHA256(snapshotPath) + if err != nil { + t.Fatalf("hash selected source: %v", err) + } + if err := os.WriteFile(snapshotPath, driftedSource, 0o600); err != nil { + t.Fatalf("rewrite drifted source: %v", err) + } + + uploadRequests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + uploadRequests++ + http.Error(w, "unexpected upload", http.StatusInternalServerError) + })) + defer server.Close() + client, err := crawlremote.NewClient(crawlremote.Options{ + Endpoint: server.URL, + HTTPClient: server.Client(), + TokenProvider: crawlremote.StaticToken("publisher-token"), + }) + if err != nil { + t.Fatalf("remote client: %v", err) + } + + _, _, err = uploadSQLiteSnapshotArchive( + context.Background(), + client, + "gitcrawl", + "gitcrawl/openclaw__gitcrawl", + snapshotPath, + selectedSnapshotID, + nil, + ) + if err == nil || !strings.Contains(err.Error(), "SQLite bundle source digest changed") { + t.Fatalf("same-size source drift error = %v", err) + } + if uploadRequests != 0 { + t.Fatalf("same-size source drift sent %d upload requests", uploadRequests) + } +} + func TestGitcrawlReaderStatusMatchesCompleteServingSnapshot(t *testing.T) { snapshotID := strings.Repeat("a", 64) const cutoverAt = "2026-07-12T12:02:00.123456789Z"