From ecb4dbdf82a0d1958d00a79156a25a3b84ea1d46 Mon Sep 17 00:00:00 2001 From: Ajit Pratap Singh Date: Fri, 16 Oct 2026 06:29:06 +0530 Subject: [PATCH] chore(ci): add weekly claims-drift check and GitHub signals log Add a --claims mode to gen-readme-metrics that verifies llms.txt, the GitHub About text and benchmark dates against metrics.json, and a weekly workflow that runs it plus logs stars, traffic and referrers to a tracking issue. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RkzVu3JY9Y6BRWfjesWts9 --- .github/workflows/claims-drift.yml | 112 +++++++++++++++++++ scripts/gen-readme-metrics/claims.go | 128 ++++++++++++++++++++++ scripts/gen-readme-metrics/claims_test.go | 61 +++++++++++ scripts/gen-readme-metrics/main.go | 20 ++++ scripts/gen-readme-metrics/metrics.json | 3 +- 5 files changed, 323 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/claims-drift.yml create mode 100644 scripts/gen-readme-metrics/claims.go create mode 100644 scripts/gen-readme-metrics/claims_test.go diff --git a/.github/workflows/claims-drift.yml b/.github/workflows/claims-drift.yml new file mode 100644 index 00000000..b5ae5714 --- /dev/null +++ b/.github/workflows/claims-drift.yml @@ -0,0 +1,112 @@ +name: Claims Drift and Signals + +# Weekly check that public claims (GitHub About text, llms.txt, README block) +# still match scripts/gen-readme-metrics/metrics.json, and that the benchmark +# data is not older than the latest release tag. Also logs weekly GitHub +# signals (stars, traffic, referrers) to a tracking issue. +# +# About text decision (Q10): the default GITHUB_TOKEN cannot edit repository +# settings, so this workflow only DETECTS About drift and fails; a maintainer +# fixes it by hand (Settings > About, or `gh repo edit --description ...`). +# A PAT-based auto-update was rejected to avoid storing an admin-scoped secret. +# +# Traffic endpoints (/traffic/*) need push access. Set the optional secret +# SIGNALS_TOKEN (fine-grained PAT, Administration: read) to record them; without +# it the traffic section is skipped with a note. pkg.go.dev "Imported by" has no +# API and stays a manual note in the tracking issue. + +on: + schedule: + - cron: '17 6 * * 1' + workflow_dispatch: + +permissions: + contents: read + +jobs: + claims-drift: + name: Claims match metrics.json + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Set up Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version: '1.26' + check-latest: true + + - name: Self-test (a seeded stale About claim must fail) + run: | + if go run ./scripts/gen-readme-metrics --claims --about "1.5M+ ops/sec" 2>/dev/null; then + echo "::error::seeded About mismatch was not detected" + exit 1 + fi + + - name: README block and version banners + run: go run ./scripts/gen-readme-metrics --check + + - name: llms.txt, About text and benchmark freshness + env: + GH_TOKEN: ${{ github.token }} + run: | + about="$(gh api "repos/${GITHUB_REPOSITORY}" --jq '.description // ""')" + go run ./scripts/gen-readme-metrics --claims --about "$about" + + signals: + name: Weekly GitHub signals + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + env: + TRACKING_TITLE: 'Weekly GitHub signals' + steps: + - name: Collect and log signals + env: + GH_TOKEN: ${{ github.token }} + SIGNALS_TOKEN: ${{ secrets.SIGNALS_TOKEN }} + run: | + set -euo pipefail + repo="${GITHUB_REPOSITORY}" + today="$(date -u +%F)" + body="$(mktemp)" + + stars="$(gh api "repos/${repo}" --jq '.stargazers_count')" + forks="$(gh api "repos/${repo}" --jq '.forks_count')" + { + echo "### Signals for ${today}" + echo + echo "- Stars: ${stars}" + echo "- Forks: ${forks}" + } > "$body" + + if [ -n "${SIGNALS_TOKEN:-}" ]; then + export GH_TOKEN="$SIGNALS_TOKEN" + { + echo "- Views (14d): $(gh api "repos/${repo}/traffic/views" --jq '"\(.count) total, \(.uniques) unique"')" + echo "- Clones (14d): $(gh api "repos/${repo}/traffic/clones" --jq '"\(.count) total, \(.uniques) unique"')" + echo "- Top referrers (14d):" + gh api "repos/${repo}/traffic/popular/referrers" \ + --jq '.[] | " - \(.referrer): \(.count) views, \(.uniques) unique"' + } >> "$body" + else + echo "- Traffic: skipped (secret SIGNALS_TOKEN not set; /traffic/* needs push access)" >> "$body" + fi + echo "- pkg.go.dev Imported by: record manually from https://pkg.go.dev/github.com/ajitpratap0/GoSQLX?tab=importedby" >> "$body" + + export GH_TOKEN="${{ github.token }}" + number="$(gh issue list --repo "$repo" --state all --search "\"${TRACKING_TITLE}\" in:title" \ + --json number,title --jq "[.[] | select(.title == \"${TRACKING_TITLE}\")][0].number // empty")" + if [ -z "$number" ]; then + number="$(gh issue create --repo "$repo" --title "$TRACKING_TITLE" \ + --body "Automated weekly log of stars, traffic and referrers (claims-drift workflow). One comment per week." \ + | grep -oE '[0-9]+$')" + fi + gh issue comment "$number" --repo "$repo" --body-file "$body" + cat "$body" >> "$GITHUB_STEP_SUMMARY" diff --git a/scripts/gen-readme-metrics/claims.go b/scripts/gen-readme-metrics/claims.go new file mode 100644 index 00000000..446ce03e --- /dev/null +++ b/scripts/gen-readme-metrics/claims.go @@ -0,0 +1,128 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "regexp" + "strings" + "time" +) + +// This file implements the --claims mode used by the weekly claims-drift +// workflow (.github/workflows/claims-drift.yml). It checks that public claims +// outside the README blocks (llms.txt and the GitHub About text) agree with +// scripts/gen-readme-metrics/metrics.json, and that the measurements are not +// older than the latest release tag. + +const dateLayout = "2006-01-02" + +var ( + opsClaim = regexp.MustCompile(`(?i)(\d+(?:\.\d+)?)\s?M\+?\s*(?:ops|parses)/sec`) + nsClaim = regexp.MustCompile(`(?i)(\d[\d,]*)\s?ns\b`) + generatedAt = regexp.MustCompile(`"generatedAt":\s*"([^"]+)"`) +) + +// ops returns the throughput figure with the given number of decimals. +func (m metrics) ops(decimals int) string { + return fmt.Sprintf("%.*f", decimals, 1e9/float64(m.ParseNsPerOp)/1e6) +} + +// llmsPerformanceLine is the exact claim llms.txt must carry. +func (m metrics) llmsPerformanceLine() string { + return fmt.Sprintf("~%d ns/op (~%sM ops/sec) for a simple SELECT (parse stage) on %s, Go %s.", + m.ParseNsPerOp, m.ops(2), m.Hardware, m.GoVersion) +} + +// checkLLMS verifies the llms.txt key facts against metrics.json. +func checkLLMS(text string, m metrics) []string { + var problems []string + if !strings.Contains(text, m.llmsPerformanceLine()) { + problems = append(problems, fmt.Sprintf("llms.txt: performance line does not contain %q", m.llmsPerformanceLine())) + } + if want := fmt.Sprintf("- SQL dialects: %d\n", m.Dialects); !strings.Contains(text, want) { + problems = append(problems, fmt.Sprintf("llms.txt: missing %q", strings.TrimSpace(want))) + } + if want := "SQL-99 compliance: " + m.SQL99Estimate; !strings.Contains(text, want) { + problems = append(problems, fmt.Sprintf("llms.txt: missing %q", want)) + } + return problems +} + +// checkAbout verifies the GitHub About (repository description) text. It may +// omit performance numbers, but any it states must match metrics.json. +func checkAbout(text string, m metrics) []string { + var problems []string + for _, mm := range opsClaim.FindAllStringSubmatch(text, -1) { + if mm[1] != m.ops(1) && mm[1] != m.ops(2) { + problems = append(problems, fmt.Sprintf("About text claims %sM ops/sec; metrics.json measures %sM (%s stage)", mm[1], m.ops(2), m.Stage)) + } + } + for _, mm := range nsClaim.FindAllStringSubmatch(text, -1) { + if strings.ReplaceAll(mm[1], ",", "") != fmt.Sprint(m.ParseNsPerOp) { + problems = append(problems, fmt.Sprintf("About text claims %s ns; metrics.json measures %d ns", mm[1], m.ParseNsPerOp)) + } + } + return problems +} + +// checkFresh fails when a measurement date is older than the latest tag date. +func checkFresh(label, measured, tagDate string) []string { + got, err := parseDate(measured) + if err != nil { + return []string{fmt.Sprintf("%s: cannot parse measurement date %q: %v", label, measured, err)} + } + tag, err := parseDate(tagDate) + if err != nil { + return []string{fmt.Sprintf("latest tag date %q: %v", tagDate, err)} + } + if got.Before(tag) { + return []string{fmt.Sprintf("%s: measured %s is older than the latest tag (%s); re-run the benchmark", label, got.Format(dateLayout), tag.Format(dateLayout))} + } + return nil +} + +func parseDate(s string) (time.Time, error) { + if len(s) >= len(dateLayout) { + s = s[:len(dateLayout)] + } + return time.Parse(dateLayout, s) +} + +func latestTagDate(version string) (string, error) { + out, err := exec.Command("git", "log", "-1", "--format=%cs", "v"+version).Output() + if err != nil { + return "", fmt.Errorf("reading date of tag v%s: %w", version, err) + } + return strings.TrimSpace(string(out)), nil +} + +// runClaims executes every claims check and returns the problems found. +func runClaims(m metrics, version, aboutText string, aboutSet bool) ([]string, error) { + var problems []string + + llms, err := os.ReadFile("llms.txt") + if err != nil { + return nil, fmt.Errorf("reading llms.txt (run from the repository root): %w", err) + } + problems = append(problems, checkLLMS(string(llms), m)...) + + if aboutSet { + problems = append(problems, checkAbout(aboutText, m)...) + } + + tagDate, err := latestTagDate(version) + if err != nil { + return nil, err + } + problems = append(problems, checkFresh("scripts/gen-readme-metrics/metrics.json", m.MeasuredAt, tagDate)...) + + if cmp, err := os.ReadFile("benchmarks/compare/results/latest.json"); err == nil { + if mm := generatedAt.FindSubmatch(cmp); mm != nil { + problems = append(problems, checkFresh("benchmarks/compare/results/latest.json", string(mm[1]), tagDate)...) + } else { + problems = append(problems, "benchmarks/compare/results/latest.json: generatedAt not found") + } + } + return problems, nil +} diff --git a/scripts/gen-readme-metrics/claims_test.go b/scripts/gen-readme-metrics/claims_test.go new file mode 100644 index 00000000..950592cc --- /dev/null +++ b/scripts/gen-readme-metrics/claims_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "strings" + "testing" +) + +var testMetrics = metrics{ + ParseNsPerOp: 318, Hardware: "Apple M4", GoVersion: "1.27", + Dialects: 8, SQL99Estimate: "~80–85%", Stage: "parse", +} + +func TestCheckLLMS(t *testing.T) { + ok := "- SQL dialects: 8\n- SQL-99 compliance: ~80–85% (estimated)\n- Performance: ~318 ns/op (~3.14M ops/sec) for a simple SELECT (parse stage) on Apple M4, Go 1.27. x" + if p := checkLLMS(ok, testMetrics); len(p) != 0 { + t.Fatalf("unexpected problems: %v", p) + } + seeded := strings.Replace(ok, "318 ns/op", "500 ns/op", 1) + if p := checkLLMS(seeded, testMetrics); len(p) == 0 { + t.Fatal("seeded llms.txt mismatch was not detected") + } +} + +func TestCheckAbout(t *testing.T) { + cases := []struct { + name, text string + wantBad bool + }{ + {"no numbers", "High-performance SQL parser for Go", false}, + {"matching ops", "SQL parser, ~3.1M ops/sec", false}, + {"matching ns", "parses in 318 ns", false}, + {"stale ops", "SQL parser - 1.5M+ ops/sec, multi-dialect", true}, + {"stale ns", "parses in 500 ns", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := len(checkAbout(tc.text, testMetrics)) > 0; got != tc.wantBad { + t.Fatalf("bad=%v, want %v", got, tc.wantBad) + } + }) + } +} + +func TestCheckFresh(t *testing.T) { + cases := []struct { + name, measured, tag string + wantBad bool + }{ + {"newer than tag", "2026-10-13", "2026-07-30", false}, + {"same day with timestamp", "2026-10-15T22:14:33Z", "2026-10-15", false}, + {"older than tag", "2026-07-01", "2026-07-30", true}, + {"missing date", "", "2026-07-30", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := len(checkFresh("x", tc.measured, tc.tag)) > 0; got != tc.wantBad { + t.Fatalf("bad=%v, want %v", got, tc.wantBad) + } + }) + } +} diff --git a/scripts/gen-readme-metrics/main.go b/scripts/gen-readme-metrics/main.go index 99e8b11c..7dc05e54 100644 --- a/scripts/gen-readme-metrics/main.go +++ b/scripts/gen-readme-metrics/main.go @@ -6,6 +6,9 @@ // // go run ./scripts/gen-readme-metrics # rewrite README.md and docs banners // go run ./scripts/gen-readme-metrics --check # exit 1 if anything has drifted +// go run ./scripts/gen-readme-metrics --claims [--about "text"] +// # exit 1 if llms.txt, the About text +// # or benchmark dates disagree with metrics.json // // Metric blocks live between / // (table + footnote) and / @@ -36,6 +39,7 @@ type metrics struct { SQL99Estimate string `json:"sql99_estimate"` Dialects int `json:"dialects"` RaceConditions int `json:"race_conditions"` + MeasuredAt string `json:"measured_at"` } func (m metrics) parsesPerSec() string { @@ -110,6 +114,8 @@ func latestTag() (string, error) { func run() error { check := flag.Bool("check", false, "fail if generated content differs from the files on disk") versionFlag := flag.String("version", "", "release version without leading v (default: latest git tag)") + claims := flag.Bool("claims", false, "check llms.txt, the About text and benchmark freshness against metrics.json instead of the README blocks") + about := flag.String("about", "", "with --claims: the GitHub About (repository description) text to verify") flag.Parse() version := strings.TrimPrefix(*versionFlag, "v") @@ -133,6 +139,20 @@ func run() error { return fmt.Errorf("metrics.json: parse_ns_per_op must be positive") } + if *claims { + aboutSet := false + flag.Visit(func(f *flag.Flag) { aboutSet = aboutSet || f.Name == "about" }) + problems, err := runClaims(m, version, *about, aboutSet) + if err != nil { + return err + } + if len(problems) > 0 { + return fmt.Errorf("claims drifted from metrics.json:\n - %s", strings.Join(problems, "\n - ")) + } + fmt.Printf("ok: claims match metrics.json at v%s (About text checked: %t)\n", version, aboutSet) + return nil + } + var drift []string update := func(path string, fn func(string) (string, error)) error { cur, err := os.ReadFile(path) diff --git a/scripts/gen-readme-metrics/metrics.json b/scripts/gen-readme-metrics/metrics.json index 20ddb691..e3d14c41 100644 --- a/scripts/gen-readme-metrics/metrics.json +++ b/scripts/gen-readme-metrics/metrics.json @@ -9,5 +9,6 @@ "benchmark_package": "./pkg/sql/parser/", "sql99_estimate": "~80–85%", "dialects": 8, - "race_conditions": 0 + "race_conditions": 0, + "measured_at": "2026-10-13" }