From 9e4c27a907aa16a75ef3407c331a77f2272c5b21 Mon Sep 17 00:00:00 2001 From: Jonathan Wrede Date: Sun, 3 May 2026 12:29:37 +0000 Subject: [PATCH 1/8] Add tokentoll GitHub Action for LLM cost analysis Adds a CI workflow that analyzes LLM API cost changes on pull requests. Posts a comment when model swaps, token limit changes, or new call sites are detected. Stays silent on PRs that don't touch LLM code. --- .github/workflows/llm-costs.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/llm-costs.yml diff --git a/.github/workflows/llm-costs.yml b/.github/workflows/llm-costs.yml new file mode 100644 index 000000000..cd1e460cc --- /dev/null +++ b/.github/workflows/llm-costs.yml @@ -0,0 +1,17 @@ +name: LLM Cost Analysis + +on: + pull_request: + branches: [main, master] + +permissions: + pull-requests: write + +jobs: + cost-analysis: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: Jwrede/tokentoll@v0.3.0 From 2d45140c6dd0cb475cf7bea2e7c4febdde2ac090 Mon Sep 17 00:00:00 2001 From: Jonathan Wrede Date: Sun, 3 May 2026 13:02:06 +0000 Subject: [PATCH 2/8] Add contents: read permission for checkout --- .github/workflows/llm-costs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/llm-costs.yml b/.github/workflows/llm-costs.yml index cd1e460cc..c1001c67c 100644 --- a/.github/workflows/llm-costs.yml +++ b/.github/workflows/llm-costs.yml @@ -5,6 +5,7 @@ on: branches: [main, master] permissions: + contents: read pull-requests: write jobs: From fb8ecdbba8ffa2b68f6cea3327bee03ced9c67ed Mon Sep 17 00:00:00 2001 From: Jonathan Wrede Date: Sun, 3 May 2026 14:02:52 +0000 Subject: [PATCH 3/8] Add .tokentoll.yml config and bump to v0.4.0 Set default model for dynamic call sites so cost estimates are shown instead of N/A. Bump action to v0.4.0 which supports config files. --- .github/workflows/llm-costs.yml | 2 +- .tokentoll.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 .tokentoll.yml diff --git a/.github/workflows/llm-costs.yml b/.github/workflows/llm-costs.yml index c1001c67c..19762053d 100644 --- a/.github/workflows/llm-costs.yml +++ b/.github/workflows/llm-costs.yml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: Jwrede/tokentoll@v0.3.0 + - uses: Jwrede/tokentoll@v0.4.0 diff --git a/.tokentoll.yml b/.tokentoll.yml new file mode 100644 index 000000000..ad09140e5 --- /dev/null +++ b/.tokentoll.yml @@ -0,0 +1 @@ +default_model: gpt-4o From 6de20149eb25ed5c5d7bf851065582a563d28452 Mon Sep 17 00:00:00 2001 From: Jonathan Wrede Date: Sun, 3 May 2026 15:23:15 +0000 Subject: [PATCH 4/8] Pin tokentoll action to commit SHA for supply-chain safety --- .github/workflows/llm-costs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/llm-costs.yml b/.github/workflows/llm-costs.yml index 19762053d..015e27a33 100644 --- a/.github/workflows/llm-costs.yml +++ b/.github/workflows/llm-costs.yml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: Jwrede/tokentoll@v0.4.0 + - uses: Jwrede/tokentoll@a4c8ce5f97e751f34498f6503087e5583c02cbbc From 76b268633528d837d55cbcbf719b891ebef1cbc1 Mon Sep 17 00:00:00 2001 From: Jonathan Wrede Date: Sun, 3 May 2026 17:36:36 +0000 Subject: [PATCH 5/8] Bump tokentoll action to v0.5.0 v0.5.0 includes built-in per-SDK default models (Anthropic calls use claude-sonnet, Google calls use gemini-flash, etc.) so a .tokentoll.yml config is no longer needed for basic usage. --- .tokentoll.yml | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .tokentoll.yml diff --git a/.tokentoll.yml b/.tokentoll.yml deleted file mode 100644 index ad09140e5..000000000 --- a/.tokentoll.yml +++ /dev/null @@ -1 +0,0 @@ -default_model: gpt-4o From 93b387f7b675397128bc3a5750ac8665462fff41 Mon Sep 17 00:00:00 2001 From: Jonathan Wrede Date: Mon, 4 May 2026 06:26:54 +0000 Subject: [PATCH 6/8] Bump tokentoll action SHA pin to v0.5.2 v0.5.2 fixes false positives where OpenAI-compatible SDKs were misidentified as openai, and skips AzureChatOpenAI calls without an explicit model name. Also pins the pip install version inside the action so the SHA pin is meaningful. --- .github/workflows/llm-costs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/llm-costs.yml b/.github/workflows/llm-costs.yml index 015e27a33..85685df47 100644 --- a/.github/workflows/llm-costs.yml +++ b/.github/workflows/llm-costs.yml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: Jwrede/tokentoll@a4c8ce5f97e751f34498f6503087e5583c02cbbc + - uses: Jwrede/tokentoll@cfc9307c2e820de1223fc707ee7e945f6f972379 # v0.5.2 From 36f0f0906c2a6635c406f9ce42611da2e42e7d27 Mon Sep 17 00:00:00 2001 From: Jonathan Wrede Date: Mon, 4 May 2026 06:43:46 +0000 Subject: [PATCH 7/8] Bump tokentoll action SHA pin to v0.6.0 v0.6.0 adds a ZhipuDetector (so Zhipu/GLM calls are attributed correctly) and a constructor-call fallback for OpenAI/Anthropic detectors, recovering DI-style detections that the v0.5.2 strict import check had dropped. --- .github/workflows/llm-costs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/llm-costs.yml b/.github/workflows/llm-costs.yml index 85685df47..f333db724 100644 --- a/.github/workflows/llm-costs.yml +++ b/.github/workflows/llm-costs.yml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: Jwrede/tokentoll@cfc9307c2e820de1223fc707ee7e945f6f972379 # v0.5.2 + - uses: Jwrede/tokentoll@ff50a35604b7858d9064cca6354b6229a94f7235 # v0.6.0 From 40397748688c02e75e5335ceb52e5d72feaf20c0 Mon Sep 17 00:00:00 2001 From: Jonathan Wrede Date: Mon, 4 May 2026 08:49:31 +0000 Subject: [PATCH 8/8] Bump tokentoll action SHA pin to v0.6.1 v0.6.1 reverts the v0.6.0 special-case that silently skipped AzureChatOpenAI(deployment_name=...) calls. Those calls now flow through the standard dynamic-default path like every other unresolved model, restoring consistency with the rest of the SDK detectors. The new skip_dynamic_models config option lets projects opt out of cost estimation for dynamic models project-wide or per path. --- .github/workflows/llm-costs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/llm-costs.yml b/.github/workflows/llm-costs.yml index f333db724..c58403d35 100644 --- a/.github/workflows/llm-costs.yml +++ b/.github/workflows/llm-costs.yml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: Jwrede/tokentoll@ff50a35604b7858d9064cca6354b6229a94f7235 # v0.6.0 + - uses: Jwrede/tokentoll@753ca4d1150c74169b52a843439049b65e256d2b # v0.6.1